Commit e4f49f7d authored by Christophe Siraut's avatar Christophe Siraut
Browse files

Update django-request module.

parent ea213b27
find . -name "*.pyc" -exec rm {} \;
find . -name "*~" -exec rm {} \;
find . -name ".svn" -exec rm -r {} \;
translation: makemessages -l fr
python makemessages -a
python compilemessages
__version__ = 0.30
__copyright__ = 'Copyright (c) 2009, Kyle Fuller'
__licence__ = 'BSD'
__author__ = 'Kyle Fuller <>, Jannis Leidel (jezdez), krisje8 <>'
__author__ = [
'Kyle Fuller <>',
'Jannis Leidel (jezdez)',
'krisje8 <>',
__URL__ = ''
......@@ -14,6 +14,7 @@ from request.models import Request
from request.traffic import modules
from request.plugins import *
class RequestAdmin(admin.ModelAdmin):
list_display = ('time', 'path', 'response', 'method', 'request_from')
fieldsets = (
......@@ -27,56 +28,58 @@ class RequestAdmin(admin.ModelAdmin):
'fields': ('referer', 'user_agent', 'ip', 'user', 'language')
raw_id_fields = ('user',)
readonly_fields = ('time',)
def lookup_allowed(self, key, value):
return key == 'user__username' or super(RequestAdmin, self).lookup_allowed(key, value)
def request_from(self, obj):
if obj.user:
return '<a href="?user__username=%s" title="%s">%s</a>' % (obj.user.username, _('Show only requests from this user.'), obj.user)
if obj.user_id:
user = obj.get_user()
return '<a href="?user__username=%s" title="%s">%s</a>' % (user.username, _('Show only requests from this user.'), user)
return '<a href="?ip=%s" title="%s">%s</a>' % (obj.ip, _('Show only requests from this IP address.'), obj.ip)
request_from.short_description = 'From'
request_from.allow_tags = True
def get_urls(self):
from django.conf.urls.defaults import patterns, url
def wrap(view):
def wrapper(*args, **kwargs):
return self.admin_site.admin_view(view)(*args, **kwargs)
return update_wrapper(wrapper, view)
info = self.model._meta.app_label, self.model._meta.module_name
return patterns('',
url(r'^overview/$', wrap(self.overview), name='%s_%s_overview' % info),
url(r'^overview/traffic.json$', wrap(self.traffic), name='%s_%s_traffic' % info),
) + super(RequestAdmin, self).get_urls()
def overview(self, request):
qs = Request.objects.this_month()
for plugin in plugins.plugins:
plugin.qs = qs
return render_to_response('admin/request/request/overview.html', {
'title': _('Request overview'),
'plugins': plugins.plugins,
'use_hosted_media': settings.REQUEST_USE_HOSTED_MEDIA,
'request_media_prefix': settings.REQUEST_MEDIA_PREFIX,
}, context_instance=RequestContext(request))
def traffic(self, request):
days_count = int(request.GET.get('days', 30))
except ValueError:
days_count = 30
if days_count < 10:
days_step = 1
elif days_count < 60:
days_step = 2
days_step = 30
days = [ - timedelta(day) for day in xrange(0, days_count, days_step)]
days_qs = [(day, for day in days]
return HttpResponse(simplejson.dumps(modules.graph(days_qs)), mimetype='text/javascript')
import datetime, time
import datetime
import time
from django.db import models
from django.contrib.auth.models import User
try: # For python <= 2.3
try: # For python <= 2.3
except NameError:
from sets import Set as set
QUERYSET_PROXY_METHODS = ('year', 'month', 'week', 'day', 'today', 'this_week', 'this_month', 'this_year', 'unique_visits', 'attr_list', 'search')
class RequestQuerySet(models.query.QuerySet):
def year(self, year):
return self.filter(time__year=year)
def month(self, year=None, month=None, month_format='%b', date=None):
if not date:
if year and month:
date =*time.strptime(year+month, '%Y'+month_format)[:3])
date =*time.strptime(year + month, '%Y' + month_format)[:3])
raise TypeError, 'Request.objects.month() takes exactly 2 arguments'
raise TypeError('Request.objects.month() takes exactly 2 arguments')
except ValueError:
# Calculate first and last day of month, for use in a date-range lookup.
first_day = date.replace(day=1)
if first_day.month == 12:
last_day = first_day.replace(year=first_day.year + 1, month=1)
last_day = first_day.replace(month=first_day.month + 1)
lookup_kwargs = {
'time__gte': first_day,
'time__lt': last_day,
return self.filter(**lookup_kwargs)
def week(self, year, week):
date =*time.strptime(year+'-0-'+week, '%Y-%w-%U')[:3])
date =*time.strptime(year + '-0-' + week, '%Y-%w-%U')[:3])
except ValueError:
# Calculate first and last day of week, for use in a date-range lookup.
first_day = date
last_day = date + datetime.timedelta(days=7)
......@@ -51,71 +53,72 @@ class RequestQuerySet(models.query.QuerySet):
'time__gte': first_day,
'time__lt': last_day,
return self.filter(**lookup_kwargs)
def day(self, year=None, month=None, day=None, month_format='%b', day_format='%d', date=None):
if not date:
if year and month and day:
date =*time.strptime(year+month+day, '%Y'+month_format+day_format)[:3])
date =*time.strptime(year + month + day, '%Y' + month_format + day_format)[:3])
raise TypeError, ' takes exactly 3 arguments'
raise TypeError(' takes exactly 3 arguments')
except ValueError:
return self.filter(time__range=(datetime.datetime.combine(date, datetime.time.min), datetime.datetime.combine(date, datetime.time.max)))
def today(self):
def this_year(self):
return self.year(
def this_month(self):
return self.month(
def this_week(self):
today =
return self.week(str(today.year), str(today.isocalendar()[1]-1))
return self.week(str(today.year), str(today.isocalendar()[1] - 1))
def unique_visits(self):
from request import settings
return self.exclude(referer__startswith=settings.REQUEST_BASE_URL)
def attr_list(self, name):
return [getattr(item, name, None) for item in self if hasattr(item, name)]
def search(self):
return self.filter(referer__contains='google')|self.filter(referer__contains='yahoo')|self.filter(referer__contains='bing')
return self.filter(referer__contains='google') | self.filter(referer__contains='yahoo') | self.filter(referer__contains='bing')
class RequestManager(models.Manager):
def __getattr__(self, attr):
return getattr(self.get_query_set(), attr, None)
super(RequestManager, self).__getattr__(*args, **kwargs)
def get_query_set(self):
return RequestQuerySet(self.model)
def active_users(self, **options):
Returns a list of active users.
Any arguments passed to this method will be
given to timedelta for time filtering.
>>> Request.object.active_users(minutes=15)
[<User: kylef>, <User: krisje8>]
qs = self.filter(user__isnull=False)
if options:
time = - datetime.timedelta(**options)
qs = qs.filter(time__gte=time)
requests = qs.select_related('user').only('user')
return set([request.user for request in requests])
......@@ -4,26 +4,30 @@ from request.models import Request
from request import settings
from request.router import patterns
class RequestMiddleware(object):
def process_response(self, request, response):
if request.method.lower() not in settings.REQUEST_VALID_METHOD_NAMES:
return response
if response.status_code < 400 and settings.REQUEST_ONLY_ERRORS:
return response
ignore = patterns(False, *settings.REQUEST_IGNORE_PATHS)
if ignore.resolve(request.path[1:]):
return response
if request.is_ajax() and settings.REQUEST_IGNORE_AJAX:
return response
if request.META.get('REMOTE_ADDR') in settings.REQUEST_IGNORE_IP:
return response
if getattr(request, 'user', False):
if request.user.username in settings.REQUEST_IGNORE_USERNAME:
return response
r = Request()
r.from_http_request(request, response)
return response
from datetime import datetime
from socket import gethostbyaddr
from django.db import models
......@@ -7,88 +6,104 @@ from django.utils.translation import ugettext_lazy as _
from request.managers import RequestManager
from request.utils import HTTP_STATUS_CODES, browsers, engines
from request import settings
class Request(models.Model):
# Response infomation
response = models.SmallIntegerField(_('response'), choices=HTTP_STATUS_CODES, default=200)
# Request infomation
method = models.CharField(_('method'), default='GET', max_length=7)
path = models.CharField(_('path'), max_length=255)
time = models.DateTimeField(_('time'),
time = models.DateTimeField(_('time'), auto_now_add=True)
is_secure = models.BooleanField(_('is secure'), default=False)
is_ajax = models.BooleanField(_('is ajax'), default=False, help_text=_('Wheather this request was used via javascript.'))
# User infomation
ip = models.IPAddressField(_('ip address'))
user = models.ForeignKey(User, blank=True, null=True, verbose_name=_('user'))
referer = models.URLField(_('referer'), verify_exists=False, max_length=255, blank=True, null=True)
referer = models.URLField(_('referer'), max_length=255, blank=True, null=True)
user_agent = models.CharField(_('user agent'), max_length=255, blank=True, null=True)
language = models.CharField(_('language'), max_length=255, blank=True, null=True)
objects = RequestManager()
class Meta:
verbose_name = _('request')
verbose_name_plural = _('requests')
ordering = ('-time',)
def __unicode__(self):
return u'[%s] %s %s %s' % (self.time, self.method, self.path, self.response)
def get_user(self):
return User.objects.get(pk=self.user_id)
def from_http_request(self, request, response=None, commit=True):
# Request infomation
self.method = request.method
self.path = request.path
self.is_secure = request.is_secure()
self.is_ajax = request.is_ajax()
# User infomation
self.ip = request.META.get('REMOTE_ADDR', '')
self.referer = request.META.get('HTTP_REFERER', '')[:255]
self.user_agent = request.META.get('HTTP_USER_AGENT', '')[:255]
self.language = request.META.get('HTTP_ACCEPT_LANGUAGE', '')[:255]
if getattr(request, 'user', False):
if request.user.is_authenticated():
self.user = request.user
if response:
self.response = response.status_code
if (response.status_code == 301) or (response.status_code == 302):
self.redirect = response['Location']
if commit:
def browser(self):
if not self.user_agent:
if not hasattr(self, '_browser'):
self._browser = browsers.resolve(self.user_agent)
return self._browser[0]
browser = property(browser)
def keywords(self):
if not self.referer:
if not hasattr(self, '_keywords'):
self._keywords = engines.resolve(self.referer)
if self._keywords:
return ' '.join(self._keywords[1]['keywords'].split('+'))
keywords = property(keywords)
def hostname(self):
return gethostbyaddr(self.ip)[0]
except Exception: # socket.gaierror, socket.herror, etc
except Exception: # socket.gaierror, socket.herror, etc
return self.ip
hostname = property(hostname)
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
if not settings.REQUEST_LOG_IP:
self.ip = settings.REQUEST_IP_DUMMY
parts = self.ip.split('.')[0:-1]
if not settings.REQUEST_LOG_USER:
self.user = None
return, force_insert, force_update, using, update_fields)
import re
from django.db.models import Count
from django.utils.translation import string_concat, ugettext, ugettext_lazy as _
from django.template.loader import render_to_string
......@@ -10,11 +11,12 @@ from request.traffic import modules
# Calculate the verbose_name by converting from InitialCaps to "lowercase with spaces".
get_verbose_name = lambda class_name: re.sub('(((?<=[a-z])[A-Z])|([A-Z](?![A-Z]|$)))', ' \\1', class_name).strip()
def set_count(items):
This is similar to "set", but this just creates a list with values.
The list will be ordered from most frequent down.
>>> inventory = ['apple', 'lemon', 'apple', 'orange', 'lemon', 'lemon']
>>> set_count(inventory)
......@@ -22,41 +24,44 @@ def set_count(items):
item_count = {}
for item in items:
if not item: continue
if not item_count.has_key(item): item_count[item] = 0
if not item:
if item not in item_count:
item_count[item] = 0
item_count[item] += 1
items = [(v, k) for k, v in item_count.iteritems()]
return [(k, v) for v, k in items]
class Plugins(object):
def load(self):
from django.utils.importlib import import_module
from django.core import exceptions
self._plugins = []
for module_path in settings.REQUEST_PLUGINS:
dot = module_path.rindex('.')
except ValueError:
raise exceptions.ImproperlyConfigured, '%s isn\'t a plugin' % module_path
plugin, plugin_classname = module_path[:dot], module_path[dot+1:]
plugin, plugin_classname = module_path[:dot], module_path[dot + 1:]
mod = import_module(plugin)
except ImportError, e:
raise exceptions.ImproperlyConfigured, 'Error importing plugin %s: "%s"' % (plugin, e)
plugin_class = getattr(mod, plugin_classname)
except AttributeError:
raise exceptions.ImproperlyConfigured, 'Plugin "%s" does not define a "%s" class' % (plugin, plugin_classname)
def plugins(self):
if not hasattr(self, '_plugins'):
......@@ -65,76 +70,88 @@ class Plugins(object):
plugins = Plugins()
class Plugin(object):
def __init__(self):
self.module_name = self.__class__.__name__
if not hasattr(self, 'verbose_name'):
self.verbose_name = _(get_verbose_name(self.module_name))
def template_context(self):
def render(self):
templates = [
"request/plugins/%s.html" % (self.__class__.__name__.lower()),
if hasattr(self, 'template'):
templates.insert(0, self.template)
kwargs = self.template_context()
kwargs['verbose_name'] = self.verbose_name
kwargs['plugin'] = self
return render_to_string(templates, kwargs)
class LatestRequests(Plugin):
def template_context(self):
return {'requests': Request.objects.all()[:5]}
class TrafficInformation(Plugin):
def template_context(self):
INFO_TABLE = ('today', 'this_week', 'this_month', 'this_year', 'all')
INFO_TABLE_QUERIES = [getattr(Request.objects, query, None)() for query in INFO_TABLE]
return {
'traffic': modules.table(INFO_TABLE_QUERIES)
class TopPaths(Plugin):
def queryset(self):
return self.qs.filter(response__lt=400).values_list('path', flat=True)
return self.qs.filter(response__lt=400)
def template_context(self):
return {
'paths': set_count(self.queryset())[:10]
'paths': self.queryset().values('path').annotate(Count('path')).order_by('-path__count')[:10]
class TopErrorPaths(TopPaths):
template = 'request/plugins/toppaths.html'
def queryset(self):
return self.qs.filter(response__gte=400).values_list('path', flat=True)
return self.qs.filter(response__gte=400)
class TopReferrers(Plugin):
def queryset(self):
return self.qs.unique_visits().exclude(referer='')
def template_context(self):
return {
'referrers': set_count(self.qs.unique_visits().exclude(referer='').values_list('referer', flat=True))[:10]
'referrers': self.queryset().values('referer').annotate(Count('referer')).order_by('-referer__count')[:10]
class TopSearchPhrases(Plugin):
def template_context(self):
return {
'phrases': set_count('referer').attr_list('keywords'))[:10]
class TopBrowsers(Plugin):
def template_context(self):
return {
'browsers': set_count(self.qs.only('user_agent').attr_list('browser'))[:5]
class ActiveUsers(Plugin):
def template_context(self):
return {}
import re
class RegexPattern(object):
def __init__(self, regex, name=''):
self.regex = re.compile(regex, re.UNICODE)