From 5892538fb7b6363b585eb7c5ffc52dfaae73ccef Mon Sep 17 00:00:00 2001 From: Andrew Williams Date: Mon, 23 Apr 2012 23:56:55 +0100 Subject: [PATCH 01/23] Basic upgrade to all base packages, Django, Celery and others. --- app/api/admin.py | 3 --- app/conf/common.py | 1 - app/eve_api/tasks/account.py | 2 +- app/eve_api/tasks/alliance.py | 2 +- app/eve_api/tasks/character.py | 2 +- app/eve_api/tasks/corporation.py | 14 ++++++------- app/eve_api/tasks/static.py | 3 ++- app/eve_proxy/models.py | 1 + app/eve_proxy/tasks.py | 4 +++- app/groups/urls.py | 4 ++-- app/hr/tasks.py | 4 +++- app/reddit/tasks.py | 3 +-- app/sso/tasks.py | 2 +- app/sso/urls.py | 4 ++-- requirements.txt | 35 +++++++++++++++++--------------- 15 files changed, 44 insertions(+), 40 deletions(-) diff --git a/app/api/admin.py b/app/api/admin.py index cde57ba..5ec9e8c 100644 --- a/app/api/admin.py +++ b/app/api/admin.py @@ -1,6 +1,5 @@ from django.contrib import admin from api.models import AuthAPIKey, AuthAPILog -from piston.models import Consumer, Token class AuthAPIKeyAdmin(admin.ModelAdmin): list_display = ('key', 'name', 'url', 'active') @@ -18,5 +17,3 @@ class AuthAPILogAdmin(admin.ModelAdmin): admin.site.register(AuthAPIKey, AuthAPIKeyAdmin) admin.site.register(AuthAPILog, AuthAPILogAdmin) -admin.site.register(Consumer, admin.ModelAdmin) -admin.site.register(Token, admin.ModelAdmin) diff --git a/app/conf/common.py b/app/conf/common.py index 47f20f4..4614235 100644 --- a/app/conf/common.py +++ b/app/conf/common.py @@ -14,7 +14,6 @@ USE_I18N = True # Defines the Static Media storage as per staticfiles contrib STATIC_ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'static') STATIC_URL = '/static/' -ADMIN_MEDIA_PREFIX = STATIC_URL + 'admin/' # Make this unique, and don't share it with anybody. SECRET_KEY = '' diff --git a/app/eve_api/tasks/account.py b/app/eve_api/tasks/account.py index 526748b..473c840 100644 --- a/app/eve_api/tasks/account.py +++ b/app/eve_api/tasks/account.py @@ -3,7 +3,7 @@ from datetime import datetime, timedelta from xml.dom import minidom import logging -from celery.decorators import task +from celery.task import task from celery.task.sets import TaskSet from gargoyle import gargoyle diff --git a/app/eve_api/tasks/alliance.py b/app/eve_api/tasks/alliance.py index 58835e5..f1dfc85 100644 --- a/app/eve_api/tasks/alliance.py +++ b/app/eve_api/tasks/alliance.py @@ -1,7 +1,7 @@ from datetime import datetime from xml.dom import minidom -from celery.decorators import task +from celery.task import task from eve_proxy.models import CachedDocument from eve_proxy.exceptions import DocumentRetrievalError diff --git a/app/eve_api/tasks/character.py b/app/eve_api/tasks/character.py index 50c2ede..1117146 100644 --- a/app/eve_api/tasks/character.py +++ b/app/eve_api/tasks/character.py @@ -2,7 +2,7 @@ from datetime import datetime, timedelta from xml.dom import minidom import logging -from celery.decorators import task +from celery.task import task from celery.task.sets import subtask from gargoyle import gargoyle diff --git a/app/eve_api/tasks/corporation.py b/app/eve_api/tasks/corporation.py index 934d73d..e6bc045 100644 --- a/app/eve_api/tasks/corporation.py +++ b/app/eve_api/tasks/corporation.py @@ -3,7 +3,7 @@ import logging from datetime import datetime, timedelta from xml.dom import minidom -from celery.decorators import task +from celery.task import task from gargoyle import gargoyle from eve_proxy.models import CachedDocument @@ -98,14 +98,12 @@ def import_corp_details_func(corp_id, log=logging.getLogger(__name__)): if int(d['allianceID']): corpobj.alliance, created = EVEPlayerAlliance.objects.get_or_create(id=d['allianceID']) + corpobj.api_last_updated = datetime.utcnow() + corpobj.save() + # Skip looking up the CEOs for NPC corps and ones with no CEO defined (dead corps) if corp_id > 1000182 and int(d['ceoID']) > 1: import_eve_character.delay(d['ceoID'], callback=link_ceo.subtask(corporation=corpobj.id)) - else: - corpobj.ceo_character = None - - corpobj.api_last_updated = datetime.utcnow() - corpobj.save() return EVEPlayerCorporation.objects.get(pk=corpobj.pk) @@ -113,7 +111,9 @@ def import_corp_details_func(corp_id, log=logging.getLogger(__name__)): @task(ignore_result=True) def link_ceo(corporation, character): """ Links a character to the CEO position of a corporation """ - corpobj = EVEPlayerCorporation.objects.filter(id=corporation).update(ceo_character=EVEPlayerCharacter.objects.get(id=character)) + corp = EVEPlayerCorporation.objects.filter(id=corporation) + char = EVEPlayerCharacter.objects.get(id=character) + corp.update(ceo_character=char) @task(ignore_result=True) diff --git a/app/eve_api/tasks/static.py b/app/eve_api/tasks/static.py index ffc9ef6..f778f93 100644 --- a/app/eve_api/tasks/static.py +++ b/app/eve_api/tasks/static.py @@ -1,4 +1,5 @@ -from celery.decorators import task +from celery.task import task + from eve_proxy.models import CachedDocument from eve_api.utils import basic_xml_parse_doc from eve_api.models import EVESkill, EVESkillGroup diff --git a/app/eve_proxy/models.py b/app/eve_proxy/models.py index 482c25c..394e734 100644 --- a/app/eve_proxy/models.py +++ b/app/eve_proxy/models.py @@ -70,6 +70,7 @@ class CachedDocumentManager(models.Manager): logger.debug('Requesting URL: %s' % url) try: + print doc_key doc = super(CachedDocumentManager, self).get_query_set().get(pk=doc_key) created = False except self.model.DoesNotExist: diff --git a/app/eve_proxy/tasks.py b/app/eve_proxy/tasks.py index 5792da7..65a46ca 100644 --- a/app/eve_proxy/tasks.py +++ b/app/eve_proxy/tasks.py @@ -1,7 +1,9 @@ from django.conf import settings import logging from datetime import datetime, timedelta -from celery.decorators import task + +from celery.task import task + from eve_proxy.models import CachedDocument, ApiAccessLog @task(ignore_result=True) diff --git a/app/groups/urls.py b/app/groups/urls.py index 02bfb3f..c88c49e 100644 --- a/app/groups/urls.py +++ b/app/groups/urls.py @@ -1,5 +1,5 @@ from django.conf.urls.defaults import * -from django.core.urlresolvers import reverse +from django.core.urlresolvers import reverse_lazy from groups import views urlpatterns = patterns('', @@ -14,5 +14,5 @@ urlpatterns = patterns('', ) urlpatterns += patterns('django.views.generic.simple', - ('^$', 'redirect_to', {'url': reverse('groups.views.group_list')}), + ('^$', 'redirect_to', {'url': reverse_lazy('groups.views.group_list')}), ) diff --git a/app/hr/tasks.py b/app/hr/tasks.py index 8109c96..a0a8467 100644 --- a/app/hr/tasks.py +++ b/app/hr/tasks.py @@ -1,7 +1,9 @@ from django.conf import settings import logging from datetime import datetime, timedelta -from celery.decorators import task + +from celery.task import task + from hr.utils import blacklist_values from django.contrib.auth.models import User from django.core.mail import send_mail diff --git a/app/reddit/tasks.py b/app/reddit/tasks.py index 7910ea5..eb73ba1 100644 --- a/app/reddit/tasks.py +++ b/app/reddit/tasks.py @@ -1,7 +1,6 @@ from datetime import datetime, timedelta from urllib2 import HTTPError, URLError -from celery.task import Task -from celery.decorators import task +from celery.task import Task, task from reddit.models import RedditAccount from reddit.api import Inbox, LoginError, Flair from django.conf import settings diff --git a/app/sso/tasks.py b/app/sso/tasks.py index 7af87ab..5945f69 100644 --- a/app/sso/tasks.py +++ b/app/sso/tasks.py @@ -10,7 +10,7 @@ from django.utils import simplejson as json from django.contrib.auth.models import User from celery.signals import task_failure -from celery.decorators import task +from celery.task import task from api.models import AuthAPIKey from eve_api.models import EVEAccount, EVEPlayerCorporation, EVEPlayerAlliance diff --git a/app/sso/urls.py b/app/sso/urls.py index bdacf9d..dbb668a 100644 --- a/app/sso/urls.py +++ b/app/sso/urls.py @@ -1,5 +1,5 @@ from django.conf.urls.defaults import * -from django.core.urlresolvers import reverse +from django.core.urlresolvers import reverse, reverse_lazy from django.contrib.auth.views import password_change, password_change_done from django.contrib.auth.decorators import login_required @@ -28,5 +28,5 @@ urlpatterns = patterns('', ) urlpatterns += patterns('django.views.generic.simple', - ('^$', 'redirect_to', {'url': reverse('sso.views.profile')}), + ('^$', 'redirect_to', {'url': reverse_lazy('sso.views.profile')}), ) diff --git a/requirements.txt b/requirements.txt index acea387..d253df1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,21 +1,24 @@ -MySQL-python -Django==1.3 --e hg+https://bitbucket.org/jespern/django-piston@c4b2d21db51a#egg=django_piston --e hg+https://bitbucket.org/ubernostrum/django-registration@d36a38202ee3#egg=django-registration -yolk==0.4.1 --e hg+http://bitbucket.org/schinckel/django-jsonfield#egg=django-jsonfield xmlrpclib==1.0.1 -South==0.7.3 -fabric -flup -celery==2.2.6 -django-celery==2.2.4 xmpppy -django-sentry==1.13.5 -raven==0.7.0 -nexus --e git+https://github.com/nikdoof/gargoyle.git@dca57fc4b437b85f8cbc#egg=gargoyle +dnspython beautifulsoup +fabric + +Django==1.4 +MySQL-python +Celery==2.5.3 +django-celery==2.5.5 + +-e hg+http://bitbucket.org/schinckel/django-jsonfield#egg=django-jsonfield +South==0.7.3 django-redis-cache IPy==0.75 -dnspython + +sentry==4.0.15 +raven==1.7.3 + +nexus +gargoyle==0.8.0 +-e hg+https://bitbucket.org/jespern/django-piston@7c90898072ce#egg=django_piston +-e hg+https://bitbucket.org/ubernostrum/django-registration@27bccd108cde#egg=django-registration + From 8350ccb9fdb202c32ec5acd14d0f17d37f4734a6 Mon Sep 17 00:00:00 2001 From: Andrew Williams Date: Mon, 23 Apr 2012 23:59:34 +0100 Subject: [PATCH 02/23] Remove direct dependency for BeautifulSoup, update South --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index d253df1..43a8b54 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ xmlrpclib==1.0.1 xmpppy dnspython -beautifulsoup fabric Django==1.4 @@ -10,7 +9,7 @@ Celery==2.5.3 django-celery==2.5.5 -e hg+http://bitbucket.org/schinckel/django-jsonfield#egg=django-jsonfield -South==0.7.3 +South==0.7.4 django-redis-cache IPy==0.75 From 474f067f069999c30ce05075a6c478f627636591 Mon Sep 17 00:00:00 2001 From: Andrew Williams Date: Tue, 24 Apr 2012 00:05:46 +0100 Subject: [PATCH 03/23] Updated ignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index d0144b1..d99e213 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ env *.pid app/conf/brokersettings.py ./static/ +.env From 9ebacb1e1333caaeec44dffe8c1485e59d1cb817 Mon Sep 17 00:00:00 2001 From: Andrew Williams Date: Sun, 20 May 2012 10:52:54 +0100 Subject: [PATCH 04/23] Update documentation and requirements --- LICENSE | 7 +------ README | 40 ++++++---------------------------------- requirements.txt | 4 ++-- 3 files changed, 9 insertions(+), 42 deletions(-) diff --git a/LICENSE b/LICENSE index dc27802..d808fa0 100644 --- a/LICENSE +++ b/LICENSE @@ -1,9 +1,4 @@ -Auth is officially ISKware/Beerware. - -You are free to copy, modify and use this software as you see fit, as long as -you give attribution to TEST Alliance for the software. If you find this -useful then please consider tipping the following people some beer money or -ISKies for exotic dancers. +This software should be classified as all rights reserved, you may not redistribute the source or applications to any other parties, or use its code as the basis for any further development outside of Test Alliance Please Ignore. Main Developers: diff --git a/README b/README index a45360d..75b7ba9 100644 --- a/README +++ b/README @@ -1,44 +1,16 @@ REQUIREMENTS ------------ -Use ./setup-env.sh to setup the virtualenv for Auth, you'll also need the following packages (as named on Debian): +The requirements.txt covers all dependencies for Auth, setup a virtual env and install the requirements: -python-zeroc-ice (ZeroC ICE Bindings for Python, only needed if you are using Mumble connecitivty) -python-imaging (PIL) -python-mysqldb (MySQL connectors for Python) -python-crypto (SSL and related stuff for Django) -python-virtualenv (distribute enabled version) -python-distribute +virtualenv env +. env/bin/activate +pip install -r requirements.txt -Also you'll need a working RabbitMQ install, and your database software of choice. - -RABBITMQ SETUP --------------- - -The fabric config has all the options to auto configure this for you, just give sudo access to /usr/sbin/rabbitmqctl to -your user. - -VIRTUALENV SETUP ----------------- - -Most of Auth's dependancies are pulled through Virtualenv, to setup the enviroment use the ./setup-env.sh script. +As we're using system wide packages, its advisable to install python-mysql packages system wide, otherwise you'll need a basic build env on your machine (build-essentials, python-dev on Debian). RUNNING ------- -For live envs, use ./start.sh, which runs a FCGI instance on port 9981. +For dev, use ./manage.py runserver :, after loading the virtualenv. In development Celery will operate in-process and doesn't require a seperate celeryd process to execute. -For dev, use ./manage.py runserver :, after loading the virtualenv. Load a seperate shell and start the Celery -work processor with ./manage.py celeryd - -DB SETUP --------- - -Copy over dbsettings.py.example and modify to your needs. - - -MUMBLE SETUP ------------- - -If you are using the Mumble SSO connector, then please copy over your Murmur.ice file to the root of the directory, if this -doesnt match the current running Mumble server it'll cause a world of pain. diff --git a/requirements.txt b/requirements.txt index 43a8b54..54039b4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ MySQL-python Celery==2.5.3 django-celery==2.5.5 --e hg+http://bitbucket.org/schinckel/django-jsonfield#egg=django-jsonfield +django-jsonfield==0.8.7 South==0.7.4 django-redis-cache IPy==0.75 @@ -17,7 +17,7 @@ sentry==4.0.15 raven==1.7.3 nexus -gargoyle==0.8.0 +gargoyle==0.9.0 -e hg+https://bitbucket.org/jespern/django-piston@7c90898072ce#egg=django_piston -e hg+https://bitbucket.org/ubernostrum/django-registration@27bccd108cde#egg=django-registration From c03989bd00d82a7d4b417ad44a2e4c83c113db62 Mon Sep 17 00:00:00 2001 From: Andrew Williams Date: Sun, 20 May 2012 10:53:14 +0100 Subject: [PATCH 05/23] Add redirect on registration completed. --- app/templates/registration/registration_complete.html | 1 + 1 file changed, 1 insertion(+) diff --git a/app/templates/registration/registration_complete.html b/app/templates/registration/registration_complete.html index f026a47..5c3edc1 100644 --- a/app/templates/registration/registration_complete.html +++ b/app/templates/registration/registration_complete.html @@ -1,6 +1,7 @@ {% extends "registration/registration_base.html" %} {% block title %}Activation email sent{% endblock %} {% block content %} +

An activation email has been sent. Please check your email and click on the link to activate your account.

{% endblock %} From e1dc20ff2ef52a57607fba04926fbdd320d448c8 Mon Sep 17 00:00:00 2001 From: Andrew Williams Date: Sun, 20 May 2012 11:31:56 +0100 Subject: [PATCH 06/23] Work on updating eve_proxy for Django 1.4 and TZ support. --- app/eve_proxy/__init__.py | 2 +- app/eve_proxy/models.py | 21 ++++++++++------ app/eve_proxy/tests.py | 7 +++--- app/eve_proxy/urls.py | 4 ++- app/eve_proxy/views.py | 52 +++++++++++++++++---------------------- 5 files changed, 45 insertions(+), 41 deletions(-) diff --git a/app/eve_proxy/__init__.py b/app/eve_proxy/__init__.py index dab0451..0fe5b46 100644 --- a/app/eve_proxy/__init__.py +++ b/app/eve_proxy/__init__.py @@ -1,4 +1,4 @@ -VERSION = (0, 4) +VERSION = (0, 5) # Dynamically calculate the version based on VERSION tuple if len(VERSION)>2 and VERSION[2] is not None: diff --git a/app/eve_proxy/models.py b/app/eve_proxy/models.py index 394e734..f548a42 100644 --- a/app/eve_proxy/models.py +++ b/app/eve_proxy/models.py @@ -8,6 +8,7 @@ from xml.dom import minidom from django.db import models, IntegrityError from django.conf import settings from django.core.cache import cache +from django.utils.timezone import utc, now from eve_proxy.exceptions import * @@ -77,7 +78,7 @@ class CachedDocumentManager(models.Manager): doc = CachedDocument(pk=doc_key, url_path=url) created = True - if created or not doc.cached_until or datetime.utcnow() > doc.cached_until or no_cache: + if created or not doc.cached_until or now() > doc.cached_until or no_cache: stat_update_count('eve_proxy_api_requests') req = urllib2.Request(url) @@ -104,23 +105,29 @@ class CachedDocumentManager(models.Manager): raise DocumentRetrievalError(e.reason) else: doc.body = unicode(conn.read(), 'utf-8') - doc.time_retrieved = datetime.utcnow() + doc.time_retrieved = now() error = 0 try: # Parse the response via minidom dom = minidom.parseString(doc.body.encode('utf-8')) except: - doc.cached_until = datetime.utcnow() + doc.cached_until = now() else: - date = datetime.strptime(dom.getElementsByTagName('cachedUntil')[0].childNodes[0].nodeValue, '%Y-%m-%d %H:%M:%S') - # Add 30 seconds to the cache timers, avoid hitting too early and account for some minor clock skew. - doc.cached_until = date + timedelta(seconds=getattr(settings, 'EVE_PROXY_GLOBAL_CACHE_ADJUSTMENT', 30)) + + # Calculate the cache timer on the basis of the currentTime and cachedUntil fields provided by the API + # This allows for clock skew not to fuck with the timers. + currenttime = datetime.strptime(dom.getElementsByTagName('currentTime')[0].childNodes[0].nodeValue, '%Y-%m-%d %H:%M:%S') + cacheuntil = datetime.strptime(dom.getElementsByTagName('cachedUntil')[0].childNodes[0].nodeValue, '%Y-%m-%d %H:%M:%S') + date = now() + (cacheuntil - currenttime) + + # Add the global adjustment, to avoid CCP's hardline cache timers + doc.cached_until += timedelta(seconds=getattr(settings, 'EVE_PROXY_GLOBAL_CACHE_ADJUSTMENT', 30)) # Allow for tuning of individual pages with bad cache returns adjustconfig = getattr(settings, 'EVE_PROXY_CACHE_ADJUSTMENTS', {}) if url_path.lower() in adjustconfig: - doc.cached_until = date + timedelta(seconds=adjustconfig[url_path.lower()]) + doc.cached_until += timedelta(seconds=adjustconfig[url_path.lower()]) enode = dom.getElementsByTagName('error') if enode: diff --git a/app/eve_proxy/tests.py b/app/eve_proxy/tests.py index fb47f47..6965548 100644 --- a/app/eve_proxy/tests.py +++ b/app/eve_proxy/tests.py @@ -4,6 +4,7 @@ from datetime import datetime import time from django.utils import unittest +from django.utils.timezone import now from eve_proxy.models import CachedDocument from eve_proxy.exceptions import * @@ -40,8 +41,8 @@ class CachedDocumentTestCase(unittest.TestCase): """ Tests if objects are being cached correctly """ url = '/server/ServerStatus.xml.aspx' - obj = CachedDocument.objects.api_query(url, no_cache=True) - obj2 = CachedDocument.objects.api_query(url) + obj = CachedDocument.objects.api_query(url, no_cache=True) + obj2 = CachedDocument.objects.api_query(url) self.assertEqual(obj.pk, obj2.pk, "Objects are not caching correctly") @@ -51,7 +52,7 @@ class CachedDocumentTestCase(unittest.TestCase): url = '/server/ServerStatus.xml.aspx' obj = CachedDocument.objects.api_query(url, no_cache=True) ret_time = obj.time_retrieved - obj.cached_until = datetime.utcnow() + obj.cached_until = now() obj.save() time.sleep(1) diff --git a/app/eve_proxy/urls.py b/app/eve_proxy/urls.py index 6c2e37f..02d091e 100644 --- a/app/eve_proxy/urls.py +++ b/app/eve_proxy/urls.py @@ -1,8 +1,10 @@ from django.conf.urls.defaults import * +from eve_proxy.views import EVEAPIProxyView + urlpatterns = patterns('eve_proxy.views', # This view can be used just like EVE API's http://api.eve-online.com. # Any parameters or URL paths are sent through the cache system and # forwarded to the EVE API site as needed. - url(r'^', 'retrieve_xml', name='eve_proxy-retrieve_xml'), + url(r'^', EVEAPIProxyView.as_view(), name='eveproxy-apiproxy'), ) diff --git a/app/eve_proxy/views.py b/app/eve_proxy/views.py index 83f687b..3108cdc 100644 --- a/app/eve_proxy/views.py +++ b/app/eve_proxy/views.py @@ -2,38 +2,32 @@ from django.core.urlresolvers import reverse from django.http import HttpResponse, HttpResponseNotFound, HttpResponseServerError from eve_proxy.models import CachedDocument -def retrieve_xml(request): - """ - A view that forwards EVE API requests through the cache system, either - retrieving a cached document or querying and caching as needed. - """ - # This is the URL path (minus the parameters). - url_path = request.META['PATH_INFO'].replace(reverse('eve_proxy.views.retrieve_xml'),"/") - # The parameters attached to the end of the URL path. - if request.method == 'POST': - p = request.POST - else: - p = request.GET +class EVEAPIProxyView(View): + """Allows for standard EVE API calls to be proxied through your application""" - # Convert the QuerySet object into a dict - params = {} - for key,value in p.items(): - params[key] = value - - if url_path == '/' or url_path == '': - # If they don't provide any kind of query, shoot a quick error message. - return HttpResponseNotFound('No API query specified.') + def get(self, request, *args, **kwargs): + return self.get_document(request, request.GET) - if 'userID' in params and not 'service' in params: - return HttpResponse('No Service ID provided.') + def post(self, request, *args, **kwargs): + return self.get_document(request, request.POST) + + def get_document(self, request, params): + url_path = request.META['PATH_INFO'].replace(reverse('eveproxy-apiproxy'),"/") - try: - cached_doc = CachedDocument.objects.api_query(url_path, params, exceptions=False) - except: - return HttpResponseServerError('Error occured') + if url_path == '/' or url_path == '': + # If they don't provide any kind of query, shoot a quick error message. + return HttpResponseNotFound('No API query specified.') + + if 'userID' in params and not 'service' in params: + return HttpResponse('No Service ID provided.') - if cached_doc: - return HttpResponse(cached_doc.body, mimetype='text/xml') + try: + cached_doc = CachedDocument.objects.api_query(url_path, params, exceptions=False) + except: + return HttpResponseServerError('Error occured') - return HttpResponseNotFound('Error retrieving the document') + if cached_doc: + return HttpResponse(cached_doc.body, mimetype='text/xml') + + return HttpResponseNotFound('Error retrieving the document') From 4a7cde91763aab4478b5292b6adb50836b70312e Mon Sep 17 00:00:00 2001 From: Andrew Williams Date: Sun, 20 May 2012 11:39:57 +0100 Subject: [PATCH 07/23] Fix some minor issues in eve_proxy stopping API proxy from working correctly --- app/eve_proxy/models.py | 2 +- app/eve_proxy/views.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/app/eve_proxy/models.py b/app/eve_proxy/models.py index f548a42..0709e0e 100644 --- a/app/eve_proxy/models.py +++ b/app/eve_proxy/models.py @@ -119,7 +119,7 @@ class CachedDocumentManager(models.Manager): # This allows for clock skew not to fuck with the timers. currenttime = datetime.strptime(dom.getElementsByTagName('currentTime')[0].childNodes[0].nodeValue, '%Y-%m-%d %H:%M:%S') cacheuntil = datetime.strptime(dom.getElementsByTagName('cachedUntil')[0].childNodes[0].nodeValue, '%Y-%m-%d %H:%M:%S') - date = now() + (cacheuntil - currenttime) + doc.cached_until = now() + (cacheuntil - currenttime) # Add the global adjustment, to avoid CCP's hardline cache timers doc.cached_until += timedelta(seconds=getattr(settings, 'EVE_PROXY_GLOBAL_CACHE_ADJUSTMENT', 30)) diff --git a/app/eve_proxy/views.py b/app/eve_proxy/views.py index 3108cdc..8d52a08 100644 --- a/app/eve_proxy/views.py +++ b/app/eve_proxy/views.py @@ -1,5 +1,7 @@ from django.core.urlresolvers import reverse from django.http import HttpResponse, HttpResponseNotFound, HttpResponseServerError +from django.views.generic import View + from eve_proxy.models import CachedDocument @@ -22,10 +24,10 @@ class EVEAPIProxyView(View): if 'userID' in params and not 'service' in params: return HttpResponse('No Service ID provided.') - try: - cached_doc = CachedDocument.objects.api_query(url_path, params, exceptions=False) - except: - return HttpResponseServerError('Error occured') + #try: + cached_doc = CachedDocument.objects.api_query(url_path, params, exceptions=False) + #except: + # return HttpResponseServerError('Error occured') if cached_doc: return HttpResponse(cached_doc.body, mimetype='text/xml') From 605a063e80a11fe65bbd1a3308f16931fa6ba7f0 Mon Sep 17 00:00:00 2001 From: Andrew Williams Date: Sun, 20 May 2012 11:40:25 +0100 Subject: [PATCH 08/23] Fix naturaltimediff to work in Django 1.4 timezones --- app/eve_api/templatetags/naturaltimediff.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/eve_api/templatetags/naturaltimediff.py b/app/eve_api/templatetags/naturaltimediff.py index 657155a..c79d871 100644 --- a/app/eve_api/templatetags/naturaltimediff.py +++ b/app/eve_api/templatetags/naturaltimediff.py @@ -1,4 +1,5 @@ from django import template +from django.utils.timezone import now register = template.Library() @@ -15,7 +16,7 @@ def naturaltimediff(value): from datetime import datetime if isinstance(value, datetime): - delta = datetime.now() - value + delta = now() - value if delta.days > 6: return value.strftime("%b %d") # May 15 if delta.days > 1: From 27abcf86612e186f00cb9b91e604a222c9666438 Mon Sep 17 00:00:00 2001 From: Andrew Williams Date: Sun, 20 May 2012 12:21:10 +0100 Subject: [PATCH 09/23] Update eve_proxy taks for Django 1.4 --- app/eve_proxy/tasks.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/app/eve_proxy/tasks.py b/app/eve_proxy/tasks.py index 65a46ca..ada25b3 100644 --- a/app/eve_proxy/tasks.py +++ b/app/eve_proxy/tasks.py @@ -1,16 +1,19 @@ -from django.conf import settings import logging from datetime import datetime, timedelta +from django.conf import settings +from django.utils.timezone import now + from celery.task import task from eve_proxy.models import CachedDocument, ApiAccessLog + @task(ignore_result=True) def clear_stale_cache(cache_extension=0): log = clear_stale_cache.get_logger() - time = datetime.utcnow() - timedelta(seconds=cache_extension) + time = now() - timedelta(seconds=cache_extension) objs = CachedDocument.objects.filter(cached_until__lt=time) log.info('Removing %s stale cache documents' % objs.count()) objs.delete() @@ -20,7 +23,7 @@ def clear_stale_cache(cache_extension=0): def clear_old_logs(): log = clear_old_logs.get_logger() - time = datetime.utcnow() - timedelta(days=settings.EVE_PROXY_KEEP_LOGS) + time = now() - timedelta(days=getattr(settings, 'EVE_PROXY_KEEP_LOGS', 30)) objs = ApiAccessLog.objects.filter(time_access__lt=time) log.info('Removing %s old access logs' % objs.count()) objs.delete() From 0a4b30244fad97d1129fe0dd2ffe32712cfe41b9 Mon Sep 17 00:00:00 2001 From: Andrew Williams Date: Sun, 20 May 2012 12:21:24 +0100 Subject: [PATCH 10/23] Initial work on eve_api for Django 1.4 --- app/eve_api/models/alliance.py | 2 ++ app/eve_api/models/character.py | 1 + app/eve_api/models/corporation.py | 3 +++ app/eve_api/tasks/account.py | 13 +++++++------ app/eve_api/tasks/alliance.py | 5 +++-- app/eve_api/tasks/character.py | 11 +++++++---- app/eve_api/tasks/corporation.py | 10 +++++----- app/eve_api/utils.py | 8 ++++++++ app/eve_api/views/base.py | 2 +- 9 files changed, 37 insertions(+), 18 deletions(-) diff --git a/app/eve_api/models/alliance.py b/app/eve_api/models/alliance.py index de0a8ce..6890fde 100644 --- a/app/eve_api/models/alliance.py +++ b/app/eve_api/models/alliance.py @@ -1,7 +1,9 @@ from django.db import models from django.contrib.auth.models import Group + from eve_api.models import EVEAPIModel + class EVEPlayerAlliance(EVEAPIModel): """ Represents a player-controlled alliance. Updated from the alliance diff --git a/app/eve_api/models/character.py b/app/eve_api/models/character.py index f4df5cf..114ad90 100644 --- a/app/eve_api/models/character.py +++ b/app/eve_api/models/character.py @@ -1,5 +1,6 @@ from django.db import models from django.core.exceptions import ObjectDoesNotExist + from eve_api.app_defines import * from eve_api.models import EVEAPIModel diff --git a/app/eve_api/models/corporation.py b/app/eve_api/models/corporation.py index 871e801..2db246f 100644 --- a/app/eve_api/models/corporation.py +++ b/app/eve_api/models/corporation.py @@ -1,9 +1,12 @@ from django.db import models from django.contrib.auth.models import Group + from gargoyle import gargoyle + from eve_api.models import EVEAPIModel from eve_api.app_defines import * + class EVEPlayerCorporation(EVEAPIModel): """ Represents a player-controlled corporation. Updated from a mixture of diff --git a/app/eve_api/tasks/account.py b/app/eve_api/tasks/account.py index 473c840..d7746f8 100644 --- a/app/eve_api/tasks/account.py +++ b/app/eve_api/tasks/account.py @@ -22,6 +22,7 @@ from eve_api.tasks.corporation import import_corp_members, import_corp_details from sso.tasks import update_user_access from django.contrib.auth.models import User +from django.utils.timezone import now, utc @task(ignore_result=True, expires=120) def queue_apikey_updates(update_delay=86400, batch_size=50): @@ -40,9 +41,9 @@ def queue_apikey_updates(update_delay=86400, batch_size=50): log.info("Updating APIs older than %s" % (datetime.now() - delta)) if gargoyle.is_active('eve-cak'): - accounts = EVEAccount.objects.filter(api_last_updated__lt=(datetime.now() - delta)).exclude(api_status__in=[API_STATUS_ACC_EXPIRED, API_STATUS_KEY_EXPIRED, API_STATUS_AUTH_ERROR]).order_by('api_last_updated')[:batch_size] + accounts = EVEAccount.objects.filter(api_last_updated__lt=(now() - delta)).exclude(api_status__in=[API_STATUS_ACC_EXPIRED, API_STATUS_KEY_EXPIRED, API_STATUS_AUTH_ERROR]).order_by('api_last_updated')[:batch_size] else: - accounts = EVEAccount.objects.filter(api_last_updated__lt=(datetime.now() - delta)).exclude(api_status__in=[API_STATUS_ACC_EXPIRED, API_STATUS_KEY_EXPIRED, API_STATUS_AUTH_ERROR]).exclude(api_keytype__gt=2).order_by('api_last_updated')[:batch_size] + accounts = EVEAccount.objects.filter(api_last_updated__lt=(now() - delta)).exclude(api_status__in=[API_STATUS_ACC_EXPIRED, API_STATUS_KEY_EXPIRED, API_STATUS_AUTH_ERROR]).exclude(api_keytype__gt=2).order_by('api_last_updated')[:batch_size] log.info("%s account(s) to update" % accounts.count()) for acc in accounts: log.debug("Queueing UserID %s for update" % acc.pk) @@ -115,7 +116,7 @@ def import_apikey_func(api_userid, api_key, user=None, force_cache=False, log=lo account.api_keytype = API_KEYTYPE_ACCOUNT account.api_accessmask = int(keydoc['accessMask']) if not keydoc['expires'] == '': - account.api_expiry = datetime.strptime(keydoc['expires'], '%Y-%m-%d %H:%M:%S') + account.api_expiry = datetime.strptime(keydoc['expires'], '%Y-%m-%d %H:%M:%S').replace(tzinfo=utc) # Checks account status to see if the account is still active if not account.api_keytype == API_KEYTYPE_CORPORATION: @@ -124,8 +125,8 @@ def import_apikey_func(api_userid, api_key, user=None, force_cache=False, log=lo status = CachedDocument.objects.api_query('/account/AccountStatus.xml.aspx', params=auth_params, no_cache=True) status = basic_xml_parse_doc(status)['eveapi'] if not status.get('error', None): - paiddate = datetime.strptime(status['result']['paidUntil'], '%Y-%m-%d %H:%M:%S') - if paiddate <= datetime.utcnow(): + paiddate = datetime.strptime(status['result']['paidUntil'], '%Y-%m-%d %H:%M:%S').replace(tzinfo=utc) + if paiddate <= now(): account.api_status = API_STATUS_ACC_EXPIRED else: account.api_status = API_STATUS_OK @@ -236,7 +237,7 @@ def import_apikey_func(api_userid, api_key, user=None, force_cache=False, log=lo if account.user: update_user_access.delay(account.user.id) - account.api_last_updated = datetime.utcnow() + account.api_last_updated = now() account.save() return account diff --git a/app/eve_api/tasks/alliance.py b/app/eve_api/tasks/alliance.py index f1dfc85..ce3c30e 100644 --- a/app/eve_api/tasks/alliance.py +++ b/app/eve_api/tasks/alliance.py @@ -11,6 +11,7 @@ from eve_api.utils import basic_xml_parse_doc from eve_api.tasks.corporation import import_corp_details, import_corp_details_result from django.core.exceptions import ValidationError +from django.utils.timezone import now, utc @task(ignore_result=True, default_retry_delay=10 * 60) def import_alliance_details(): @@ -32,10 +33,10 @@ def import_alliance_details(): allobj, created = EVEPlayerAlliance.objects.get_or_create(pk=alliance['allianceID']) allobj.name = alliance['name'] allobj.ticker = alliance['shortName'] - allobj.date_founded = datetime.strptime(alliance['startDate'], "%Y-%m-%d %H:%M:%S") + allobj.date_founded = datetime.strptime(alliance['startDate'], "%Y-%m-%d %H:%M:%S").replace(tzinfo=utc) allobj.executor, created = EVEPlayerCorporation.objects.get_or_create(id=alliance['executorCorpID']) allobj.member_count = alliance['memberCount'] - allobj.api_last_updated = datetime.utcnow() + allobj.api_last_updated = now() allobj.save() members = [int(corp['corporationID']) for corp in alliance['memberCorporations']] diff --git a/app/eve_api/tasks/character.py b/app/eve_api/tasks/character.py index 1117146..363f1f2 100644 --- a/app/eve_api/tasks/character.py +++ b/app/eve_api/tasks/character.py @@ -2,6 +2,8 @@ from datetime import datetime, timedelta from xml.dom import minidom import logging +from django.utils.timezone import now, utc + from celery.task import task from celery.task.sets import subtask from gargoyle import gargoyle @@ -85,11 +87,11 @@ def import_eve_character_func(character_id, key_id=None, logger=logging.getLogge # Set corporation and join date corp, created = EVEPlayerCorporation.objects.get_or_create(pk=values['corporationID']) from eve_api.tasks.corporation import import_corp_details - if created or not corp.name or corp.api_last_updated < (datetime.utcnow() - timedelta(hours=12)): + if created or not corp.name or corp.api_last_updated < (now() - timedelta(hours=12)): import_corp_details.delay(values['corporationID']) pchar.corporation = corp - pchar.corporation_date = values['corporationDate'] + pchar.corporation_date = datetime.strptime(values['corporationDate'], "%Y-%m-%d %H:%M:%S").replace(tzinfo=utc) # Derrive Race value from the choices for v in API_RACES_CHOICES: @@ -106,7 +108,8 @@ def import_eve_character_func(character_id, key_id=None, logger=logging.getLogge corp, created = EVEPlayerCorporation.objects.get_or_create(pk=emp['corporationID']) if created: import_corp_details.delay(emp['corporationID']) - eobj, created = EVEPlayerCharacterEmploymentHistory.objects.get_or_create(pk=emp['recordID'], corporation=corp, character=pchar, start_date=emp['startDate']) + startdate = datetime.strptime(emp['startDate'], "%Y-%m-%d %H:%M:%S").replace(tzinfo=utc) + eobj, created = EVEPlayerCharacterEmploymentHistory.objects.get_or_create(pk=emp['recordID'], corporation=corp, character=pchar, start_date=startdate) # We've been passed a Key ID, try and work with it if key_id: @@ -183,7 +186,7 @@ def import_eve_character_func(character_id, key_id=None, logger=logging.getLogge else: pchar.gender = API_GENDER_FEMALE - pchar.api_last_updated = datetime.utcnow() + pchar.api_last_updated = now() pchar.save() if acc: diff --git a/app/eve_api/tasks/corporation.py b/app/eve_api/tasks/corporation.py index e6bc045..61296b0 100644 --- a/app/eve_api/tasks/corporation.py +++ b/app/eve_api/tasks/corporation.py @@ -55,7 +55,7 @@ def import_corp_details_result(corp_id, callback=None): def import_corp_details_func(corp_id, log=logging.getLogger(__name__)): corpobj, created = EVEPlayerCorporation.objects.get_or_create(id=corp_id) - if created or not corpobj.api_last_updated or corpobj.api_last_updated < (datetime.utcnow() - timedelta(hours=12)): + if created or not corpobj.api_last_updated or corpobj.api_last_updated < (now() - timedelta(hours=12)): try: doc = CachedDocument.objects.api_query('/corp/CorporationSheet.xml.aspx', {'corporationID': corp_id}) @@ -98,7 +98,7 @@ def import_corp_details_func(corp_id, log=logging.getLogger(__name__)): if int(d['allianceID']): corpobj.alliance, created = EVEPlayerAlliance.objects.get_or_create(id=d['allianceID']) - corpobj.api_last_updated = datetime.utcnow() + corpobj.api_last_updated = now() corpobj.save() # Skip looking up the CEOs for NPC corps and ones with no CEO defined (dead corps) @@ -159,10 +159,10 @@ def import_corp_members(key_id, character_id): if created: charobj.name = character['name'] charobj.corporation = corp - charobj.corporation_date = character['startDateTime'] + charobj.corporation_date = datetime.strptime(character['startDateTime'], "%Y-%m-%d %H:%M:%S").replace(tzinfo=utc) if 'logonDateTime' in character: - charobj.last_login = character['logonDateTime'] - charobj.last_logoff = character['logoffDateTime'] + charobj.last_login = datetime.strptime(character['logonDateTime'], "%Y-%m-%d %H:%M:%S").replace(tzinfo=utc) + charobj.last_logoff = datetime.strptime(character['logoffDateTime'], "%Y-%m-%d %H:%M:%S").replace(tzinfo=utc) charobj.current_location_id = int(character['locationID']) else: charobj.last_login = None diff --git a/app/eve_api/utils.py b/app/eve_api/utils.py index 840bcb5..2cbed6e 100644 --- a/app/eve_api/utils.py +++ b/app/eve_api/utils.py @@ -1,4 +1,8 @@ +from datetime import datetime from xml.dom import minidom + +from django.utils.timezone import utc + from eve_proxy.models import CachedDocument def basic_xml_parse(nodes): @@ -51,3 +55,7 @@ def basic_xml_parse_doc(doc): return basic_xml_parse(dom.childNodes) return {} + + +def parse_eveapi_date(datestring): + return datetime.strptime(datestring, "%Y-%m-%d %H:%M:%S").replace(tzinfo=utc) \ No newline at end of file diff --git a/app/eve_api/views/base.py b/app/eve_api/views/base.py index d8f6d7a..1293953 100644 --- a/app/eve_api/views/base.py +++ b/app/eve_api/views/base.py @@ -6,7 +6,6 @@ from django.http import HttpResponse, Http404 from django.shortcuts import render_to_response, get_object_or_404, redirect from django.template import RequestContext from django.views.generic import DetailView, ListView - from django.contrib import messages from django.contrib.auth.decorators import login_required @@ -22,6 +21,7 @@ from eve_api.tasks import import_apikey_result from eve_api.utils import basic_xml_parse_doc from eve_api.views.mixins import DetailPaginationMixin + @login_required def eveapi_add(request, post_save_redirect='/', template='eve_api/add.html'): """ Add a EVE API key to a user's account """ From 1bbd15abceeda6b3adffb8fceeee70342dba5cd4 Mon Sep 17 00:00:00 2001 From: Andrew Williams Date: Sun, 20 May 2012 14:23:08 +0100 Subject: [PATCH 11/23] Fix date/time fuctions for timezone information --- app/api/auth.py | 5 ++++- app/api/handlers/oauth.py | 15 ++++++--------- app/api/handlers/v1.py | 14 ++++++-------- app/hr/views.py | 3 ++- app/sso/middleware.py | 5 ++--- 5 files changed, 20 insertions(+), 22 deletions(-) diff --git a/app/api/auth.py b/app/api/auth.py index ca5bd4e..cc3ca39 100644 --- a/app/api/auth.py +++ b/app/api/auth.py @@ -1,7 +1,10 @@ from urllib import urlencode from datetime import datetime + from django.http import HttpResponseForbidden from django.contrib.auth.models import AnonymousUser +from django.utils.timezone import now + from api.models import AuthAPIKey, AuthAPILog @@ -21,7 +24,7 @@ class APIKeyAuthentication(object): url = "%s?%s" % (request.path, urlencode(params)) else: url = request.path - AuthAPILog(key=keyobj, access_datetime=datetime.utcnow(), url=url).save() + AuthAPILog(key=keyobj, access_datetime=now(), url=url).save() request.user = AnonymousUser() request.api_key = keyobj return True diff --git a/app/api/handlers/oauth.py b/app/api/handlers/oauth.py index 6d53023..1446774 100644 --- a/app/api/handlers/oauth.py +++ b/app/api/handlers/oauth.py @@ -5,17 +5,16 @@ from operator import itemgetter from django.contrib.auth import login, logout, authenticate from django.contrib.auth.models import User - from django.http import HttpResponse from django.core.urlresolvers import reverse from django.shortcuts import get_object_or_404 from django.conf import settings +from django.utils.timezone import now, utc from piston.handler import BaseHandler from piston.utils import rc, throttle from api.models import AuthAPIKey, AuthAPILog - from eve_proxy.models import CachedDocument from eve_proxy.exceptions import * from eve_api.app_defines import * @@ -54,12 +53,10 @@ class OAuthOpTimerHandler(BaseHandler): for node in dom.getElementsByTagName('rowset')[0].childNodes: if node.nodeType == 1: - ownerID = node.getAttribute('ownerID') + ownerID = node.getAttribute('ownerID') if ownerID != '1': - date = node.getAttribute('eventDate') - dt = datetime.strptime(date, '%Y-%m-%d %H:%M:%S') - now = datetime.utcnow() - startsIn = int(dt.strftime('%s')) - int(now.strftime('%s')) + dt = datetime.strptime(node.getAttribute('eventDate'), '%Y-%m-%d %H:%M:%S').replace(tzinfo=utc) + startsIn = int(dt.strftime('%s')) - int(now().strftime('%s')) duration = int(node.getAttribute('duration')) fid = re.search('topic=[\d]+', node.getAttribute('eventText')) @@ -84,7 +81,7 @@ class OAuthOpTimerHandler(BaseHandler): 'isImportant': int(node.getAttribute('importance')), 'eventText': node.getAttribute('eventText'), 'endsIn':endsIn, - 'forumLink': forumlink} + 'forumLink': forumlink} events.append(event) if len(events) == 0: @@ -101,7 +98,7 @@ class OAuthOpTimerHandler(BaseHandler): 'forumLink': ''}]} else: events.sort(key=itemgetter('startsIn')) - return {'ops': events, 'doc_time': cached_doc.time_retrieved, 'cache_until': cached_doc.cached_until, 'current_time': datetime.utcnow() } + return {'ops': events, 'doc_time': cached_doc.time_retrieved, 'cache_until': cached_doc.cached_until, 'current_time': now() } class OAuthCharacterHandler(BaseHandler): diff --git a/app/api/handlers/v1.py b/app/api/handlers/v1.py index 2dd59c5..bfc96ec 100644 --- a/app/api/handlers/v1.py +++ b/app/api/handlers/v1.py @@ -4,11 +4,11 @@ from xml.dom import minidom from operator import itemgetter from django.contrib.auth.models import User - from django.http import HttpResponse from django.core.urlresolvers import reverse from django.shortcuts import get_object_or_404 from django.conf import settings +from django.utils.timezone import now from BeautifulSoup import BeautifulSoup from piston.handler import BaseHandler @@ -194,12 +194,10 @@ class OpTimerHandler(BaseHandler): events = [] for node in dom.getElementsByTagName('rowset')[0].childNodes: if node.nodeType == 1: - ownerID = node.getAttribute('ownerID') + ownerID = node.getAttribute('ownerID') if ownerID != '1': - date = node.getAttribute('eventDate') - dt = datetime.strptime(date, '%Y-%m-%d %H:%M:%S') - now = datetime.utcnow() - startsIn = int(dt.strftime('%s')) - int(now.strftime('%s')) + dt = datetime.strptime(node.getAttribute('eventDate'), '%Y-%m-%d %H:%M:%S').replace(tzinfo=utc) + startsIn = int(dt.strftime('%s')) - int(now().strftime('%s')) duration = int(node.getAttribute('duration')) fid = re.search('topic=[\d]+', node.getAttribute('eventText')) @@ -224,7 +222,7 @@ class OpTimerHandler(BaseHandler): 'isImportant': int(node.getAttribute('importance')), 'eventText': ' '.join(BeautifulSoup(node.getAttribute('eventText')).findAll(text=True)), 'endsIn':endsIn, - 'forumLink': forumlink} + 'forumLink': forumlink} events.append(event) if len(events) == 0: return {'ops':[{ @@ -240,7 +238,7 @@ class OpTimerHandler(BaseHandler): 'forumLink': ''}]} else: events.sort(key=itemgetter('startsIn')) - return {'ops': events, 'doc_time': cached_doc.time_retrieved, 'cache_until': cached_doc.cached_until, 'current_time': datetime.utcnow() } + return {'ops': events, 'doc_time': cached_doc.time_retrieved, 'cache_until': cached_doc.cached_until, 'current_time': now() } class BlacklistHandler(BaseHandler): diff --git a/app/hr/views.py b/app/hr/views.py index 66a192b..0994da4 100644 --- a/app/hr/views.py +++ b/app/hr/views.py @@ -13,6 +13,7 @@ from django.forms.extras.widgets import SelectDateWidget from django.views.generic import TemplateView, DetailView, FormView, CreateView, ListView from django.views.generic.detail import BaseDetailView from django.conf import settings +from django.utils.timezone import now from gargoyle import gargoyle @@ -354,7 +355,7 @@ class HrBlacklistUser(FormView): self.source = BlacklistSource.objects.get(id=1) self.expiry = form.cleaned_data.get('expiry_date', None) if not self.expiry: - self.expiry = datetime.utcnow() + timedelta(days=50*365) # 50 year default + self.expiry = now() + timedelta(days=50*365) # 50 year default self.level = form.cleaned_data.get('level', 0) self.reason = form.cleaned_data.get('reason', 'No reason provided') diff --git a/app/sso/middleware.py b/app/sso/middleware.py index 0112476..a04e86e 100644 --- a/app/sso/middleware.py +++ b/app/sso/middleware.py @@ -1,5 +1,6 @@ from django.db import IntegrityError from django.contrib.auth import logout +from django.utils.timezone import utc, now class InactiveLogoutMiddleware(object): """ @@ -59,7 +60,5 @@ class IPTrackingMiddleware(object): if request.user and not request.user.is_anonymous(): ip, created = SSOUserIPAddress.objects.get_or_create(user=request.user, ip_address=request.META['REMOTE_ADDR']) - if created: - ip.first_seen = datetime.utcnow() - ip.last_seen = datetime.utcnow() + ip.last_seen = now() ip.save() From 1d2bf5258b332aa4e754fa6f81cc842c33793432 Mon Sep 17 00:00:00 2001 From: Andrew Williams Date: Sun, 20 May 2012 15:19:11 +0100 Subject: [PATCH 12/23] Fix import errors --- app/eve_api/tasks/account.py | 10 ++++------ app/eve_api/tasks/corporation.py | 2 ++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/eve_api/tasks/account.py b/app/eve_api/tasks/account.py index d7746f8..b156718 100644 --- a/app/eve_api/tasks/account.py +++ b/app/eve_api/tasks/account.py @@ -3,26 +3,24 @@ from datetime import datetime, timedelta from xml.dom import minidom import logging +from django.conf import settings +from django.contrib.auth.models import User +from django.utils.timezone import now, utc + from celery.task import task from celery.task.sets import TaskSet from gargoyle import gargoyle -from django.conf import settings - from eve_proxy.exceptions import * from eve_proxy.models import CachedDocument - from eve_api.models import EVEAccount, EVEPlayerCharacter from eve_api.app_defines import * from eve_api.api_exceptions import * from eve_api.utils import basic_xml_parse_doc from eve_api.tasks.character import import_eve_characters from eve_api.tasks.corporation import import_corp_members, import_corp_details - from sso.tasks import update_user_access -from django.contrib.auth.models import User -from django.utils.timezone import now, utc @task(ignore_result=True, expires=120) def queue_apikey_updates(update_delay=86400, batch_size=50): diff --git a/app/eve_api/tasks/corporation.py b/app/eve_api/tasks/corporation.py index 61296b0..3bf7d16 100644 --- a/app/eve_api/tasks/corporation.py +++ b/app/eve_api/tasks/corporation.py @@ -3,6 +3,8 @@ import logging from datetime import datetime, timedelta from xml.dom import minidom +from django.utils.timezone import now, utc + from celery.task import task from gargoyle import gargoyle From 6f571539b99a7615f572494315be5b28e8495990 Mon Sep 17 00:00:00 2001 From: Andrew Williams Date: Sun, 20 May 2012 15:40:24 +0100 Subject: [PATCH 13/23] Removed Sentry/Raven, as it seems to be broke under Django 1.4 --- app/conf/common.py | 2 -- requirements.txt | 3 --- 2 files changed, 5 deletions(-) diff --git a/app/conf/common.py b/app/conf/common.py index 4614235..93f8cac 100644 --- a/app/conf/common.py +++ b/app/conf/common.py @@ -63,8 +63,6 @@ INSTALLED_APPS = ( 'django.contrib.staticfiles', 'nexus', 'gargoyle', - 'sentry', - 'raven.contrib.django', 'south', 'piston', 'djcelery', diff --git a/requirements.txt b/requirements.txt index 54039b4..2c4dec7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,9 +13,6 @@ South==0.7.4 django-redis-cache IPy==0.75 -sentry==4.0.15 -raven==1.7.3 - nexus gargoyle==0.9.0 -e hg+https://bitbucket.org/jespern/django-piston@7c90898072ce#egg=django_piston From af1ba8744ede33954391dc344d1027ab2f5937ee Mon Sep 17 00:00:00 2001 From: Andrew Williams Date: Sun, 20 May 2012 16:10:47 +0100 Subject: [PATCH 14/23] Fix Blacklist checker for new TZ info --- app/hr/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/hr/utils.py b/app/hr/utils.py index f3eedfd..13a524a 100644 --- a/app/hr/utils.py +++ b/app/hr/utils.py @@ -3,6 +3,7 @@ from datetime import datetime from django.db import models from django.template.loader import render_to_string +from django.utils.timezone import now from eve_api.models import EVEPlayerCharacter from hr.app_defines import * @@ -29,7 +30,7 @@ def blacklist_values(user, level=BLACKLIST_LEVEL_NOTE): """ blacklist = [] - bl_items = Blacklist.objects.filter(models.Q(expiry_date__gt=datetime.now()) | models.Q(expiry_date=None), level__lte=level) + bl_items = Blacklist.objects.filter(models.Q(expiry_date__gt=now()) | models.Q(expiry_date=None), level__lte=level) # Check Reddit blacklists if installed('reddit'): From e6d5cc32019790262250770bff8546d95e0846b0 Mon Sep 17 00:00:00 2001 From: Andrew Williams Date: Sun, 20 May 2012 16:11:14 +0100 Subject: [PATCH 15/23] Bring reddit app up to date with new TZ requirements --- app/reddit/__init__.py | 2 +- app/reddit/models.py | 5 +++-- app/reddit/tasks.py | 11 ++++++++--- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/app/reddit/__init__.py b/app/reddit/__init__.py index 805a0d1..ee1709b 100644 --- a/app/reddit/__init__.py +++ b/app/reddit/__init__.py @@ -1,4 +1,4 @@ -VERSION = (0, 1) +VERSION = (0, 2) # Dynamically calculate the version based on VERSION tuple if len(VERSION)>2 and VERSION[2] is not None: diff --git a/app/reddit/models.py b/app/reddit/models.py index ae05ead..55f8126 100644 --- a/app/reddit/models.py +++ b/app/reddit/models.py @@ -4,6 +4,7 @@ import urllib from django.utils import simplejson as json from django.db import models from django.contrib.auth.models import User +from django.utils.timezone import now, utc from reddit.api import Comment @@ -49,8 +50,8 @@ class RedditAccount(models.Model): self.comment_karma = int(data['comment_karma']) self.reddit_id = unicode(data['id'], 'utf-8') - self.date_created = datetime.fromtimestamp(data['created_utc']) - self.last_update = datetime.now() + self.date_created = datetime.fromtimestamp(data['created_utc']).replace(tzinfo=utc) + self.last_update = now() def recent_posts(self): """ Returns the first page of posts visible on the user's profile page """ diff --git a/app/reddit/tasks.py b/app/reddit/tasks.py index eb73ba1..1da04ef 100644 --- a/app/reddit/tasks.py +++ b/app/reddit/tasks.py @@ -1,9 +1,14 @@ from datetime import datetime, timedelta from urllib2 import HTTPError, URLError + +from django.conf import settings +from django.utils.timezone import now + from celery.task import Task, task + from reddit.models import RedditAccount from reddit.api import Inbox, LoginError, Flair -from django.conf import settings + class send_reddit_message(Task): @@ -71,8 +76,8 @@ def queue_account_updates(update_delay=604800, batch_size=50): log = queue_account_updates.get_logger() # Update all the eve accounts and related corps delta = timedelta(seconds=update_delay) - log.info("Updating Accounts older than %s" % (datetime.now() - delta)) - accounts = RedditAccount.objects.order_by('last_update').filter(last_update__lt=(datetime.now() - delta))[:batch_size] + log.info("Updating Accounts older than %s" % (now() - delta)) + accounts = RedditAccount.objects.order_by('last_update').filter(last_update__lt=(now() - delta))[:batch_size] log.info("%s account(s) to update" % accounts.count()) for acc in accounts: log.debug("Queueing Account %s for update" % acc.username) From d7b364011f3abb1b0e3aff19e983a53ef25e78b2 Mon Sep 17 00:00:00 2001 From: Andrew Williams Date: Sun, 20 May 2012 16:11:37 +0100 Subject: [PATCH 16/23] Fix naturaltimediff for the SSO app --- app/sso/templatetags/naturaltimediff.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/sso/templatetags/naturaltimediff.py b/app/sso/templatetags/naturaltimediff.py index 657155a..c79d871 100644 --- a/app/sso/templatetags/naturaltimediff.py +++ b/app/sso/templatetags/naturaltimediff.py @@ -1,4 +1,5 @@ from django import template +from django.utils.timezone import now register = template.Library() @@ -15,7 +16,7 @@ def naturaltimediff(value): from datetime import datetime if isinstance(value, datetime): - delta = datetime.now() - value + delta = now() - value if delta.days > 6: return value.strftime("%b %d") # May 15 if delta.days > 1: From 04328bb0ed84180aa9e5ce7f749eafb1ab96d4fc Mon Sep 17 00:00:00 2001 From: Andrew Williams Date: Sun, 20 May 2012 16:12:10 +0100 Subject: [PATCH 17/23] Use create instead of instance and save --- app/api/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/auth.py b/app/api/auth.py index cc3ca39..40b4cda 100644 --- a/app/api/auth.py +++ b/app/api/auth.py @@ -24,7 +24,7 @@ class APIKeyAuthentication(object): url = "%s?%s" % (request.path, urlencode(params)) else: url = request.path - AuthAPILog(key=keyobj, access_datetime=now(), url=url).save() + AuthAPILog.objects.create(key=keyobj, access_datetime=now(), url=url) request.user = AnonymousUser() request.api_key = keyobj return True From 58e6e76bc53f8d82615b9614fe31ec5a82487dd5 Mon Sep 17 00:00:00 2001 From: Andrew Williams Date: Sun, 20 May 2012 16:12:34 +0100 Subject: [PATCH 18/23] Use now() instead of datetime.now() --- app/eve_api/tasks/account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/eve_api/tasks/account.py b/app/eve_api/tasks/account.py index b156718..384fd60 100644 --- a/app/eve_api/tasks/account.py +++ b/app/eve_api/tasks/account.py @@ -36,7 +36,7 @@ def queue_apikey_updates(update_delay=86400, batch_size=50): # Update all the eve accounts and related corps delta = timedelta(seconds=update_delay) - log.info("Updating APIs older than %s" % (datetime.now() - delta)) + log.info("Updating APIs older than %s" % (now() - delta)) if gargoyle.is_active('eve-cak'): accounts = EVEAccount.objects.filter(api_last_updated__lt=(now() - delta)).exclude(api_status__in=[API_STATUS_ACC_EXPIRED, API_STATUS_KEY_EXPIRED, API_STATUS_AUTH_ERROR]).order_by('api_last_updated')[:batch_size] From 31979cbab8368d814a65751acd81c4331d9add80 Mon Sep 17 00:00:00 2001 From: Andrew Williams Date: Sun, 20 May 2012 16:38:18 +0100 Subject: [PATCH 19/23] Update logging setup --- app/conf/common.py | 83 +++++++++++++++++++++++++++++----------------- 1 file changed, 52 insertions(+), 31 deletions(-) diff --git a/app/conf/common.py b/app/conf/common.py index 93f8cac..a44d7dd 100644 --- a/app/conf/common.py +++ b/app/conf/common.py @@ -10,6 +10,7 @@ TIME_ZONE = 'UTC' LANGUAGE_CODE = 'en-us' SITE_ID = 1 USE_I18N = True +USE_TZ = False # Defines the Static Media storage as per staticfiles contrib STATIC_ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'static') @@ -61,6 +62,7 @@ INSTALLED_APPS = ( 'django.contrib.sites', 'django.contrib.humanize', 'django.contrib.staticfiles', + 'raven.contrib.django', 'nexus', 'gargoyle', 'south', @@ -128,7 +130,7 @@ GARGOYLE_SWITCH_DEFAULTS = { 'description': 'Enables/Disables the HR functionality.', }, 'eve-cak': { - 'is_active': False, + 'is_active': True, 'label': 'EVE Customizable API Keys', 'description': 'Enables/Disables EVE API CAK support.', }, @@ -152,45 +154,64 @@ GARGOYLE_SWITCH_DEFAULTS = { LOGGING = { 'version': 1, - 'disable_existing_loggers': True, + 'disable_existing_loggers': False, + 'root': { + 'level': 'WARNING', + 'handlers': ['sentry', 'console'], + }, + 'filters': { + 'require_debug_false': { + '()': 'django.utils.log.RequireDebugFalse' + } + }, + 'formatters': { + 'verbose': { + 'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s' + }, + }, 'handlers': { - 'null': { - 'level':'DEBUG', - 'class':'django.utils.log.NullHandler', + 'sentry': { + 'level': 'DEBUG', + 'class': 'raven.contrib.django.handlers.SentryHandler', + 'filters': ['require_debug_false'], + }, + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'formatter': 'verbose' }, 'mail_admins': { 'level': 'ERROR', 'class': 'django.utils.log.AdminEmailHandler', - }, - 'sentry': { - 'level': 'DEBUG', - 'class': 'raven.contrib.django.handlers.SentryHandler', - }, + 'include_html': True, + 'filters': ['require_debug_false'], + } }, 'loggers': { - 'root': { - 'level': 'WARNING', - 'handlers': ['sentry'], + 'django.request': { + 'handlers': ['console'], + 'level': 'ERROR', + 'propagate': True, + }, + 'django.db.backends': { + 'level': 'ERROR', + 'handlers': ['console'], + 'propagate': False, + }, + 'raven': { + 'level': 'DEBUG', + 'handlers': ['console'], + 'propagate': False, }, 'sentry.errors': { + 'level': 'DEBUG', + 'handlers': ['console'], + 'propagate': False, + }, + 'celery': { 'level': 'WARNING', - 'handlers': ['null'], - 'propagate': True, - }, - 'django': { - 'handlers':['null'], - 'propagate': True, - 'level':'INFO', - }, - 'django.request': { - 'handlers': ['null'], - 'level': 'ERROR', - 'propagate': True, - }, - 'celery.task.default': { - 'handlers': ['null'], - 'level': 'ERROR', - 'propagate': True, + 'handlers': ['console'], + 'propagate': False, }, } -} +} \ No newline at end of file From 1c675338c6b684207bc4efde13f4599de408a988 Mon Sep 17 00:00:00 2001 From: Andrew Williams Date: Sun, 20 May 2012 17:35:21 +0100 Subject: [PATCH 20/23] Updated README --- README => README.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) rename README => README.md (56%) diff --git a/README b/README.md similarity index 56% rename from README rename to README.md index 75b7ba9..189d899 100644 --- a/README +++ b/README.md @@ -1,4 +1,12 @@ -REQUIREMENTS +TEST Auth +========= + +TEST Auth is a central access management system for EVE Online corporations and alliances, it stores and manages EVE API keys and assigns access permissions based on defined permissions. + +Included is a HR management system, a group management system and a basic API key viewer showing Character and skill information. + + +Requirements ------------ The requirements.txt covers all dependencies for Auth, setup a virtual env and install the requirements: @@ -9,8 +17,10 @@ pip install -r requirements.txt As we're using system wide packages, its advisable to install python-mysql packages system wide, otherwise you'll need a basic build env on your machine (build-essentials, python-dev on Debian). -RUNNING +Running ------- For dev, use ./manage.py runserver :, after loading the virtualenv. In development Celery will operate in-process and doesn't require a seperate celeryd process to execute. +For Live instances its advisable to run within a WSGI container server such as uWSGI. + From 3bad0c6c4e0e8bb90702e65421bba73625cade9a Mon Sep 17 00:00:00 2001 From: Andrew Williams Date: Sun, 20 May 2012 18:13:35 +0100 Subject: [PATCH 21/23] Removed Nexus, switched to default backends --- app/conf/common.py | 5 ----- app/sso/backends.py | 26 -------------------------- app/templates/base.html | 2 +- app/urls.py | 8 ++++++-- 4 files changed, 7 insertions(+), 34 deletions(-) delete mode 100644 app/sso/backends.py diff --git a/app/conf/common.py b/app/conf/common.py index a44d7dd..3d11112 100644 --- a/app/conf/common.py +++ b/app/conf/common.py @@ -63,7 +63,6 @@ INSTALLED_APPS = ( 'django.contrib.humanize', 'django.contrib.staticfiles', 'raven.contrib.django', - 'nexus', 'gargoyle', 'south', 'piston', @@ -80,10 +79,6 @@ INSTALLED_APPS = ( 'tools', ) -AUTHENTICATION_BACKENDS = ( - 'sso.backends.SimpleHashModelBackend', -) - AUTH_PROFILE_MODULE = 'sso.SSOUser' LOGIN_REDIRECT_URL = "/profile" LOGIN_URL = "/login" diff --git a/app/sso/backends.py b/app/sso/backends.py deleted file mode 100644 index 712d483..0000000 --- a/app/sso/backends.py +++ /dev/null @@ -1,26 +0,0 @@ -from django.contrib.auth.backends import ModelBackend -from django.contrib.auth.models import User -from hashlib import sha1 - - -class SimpleHashModelBackend(ModelBackend): - - supports_anonymous_user = False - supports_object_permissions = False - supports_inactive_user = False - - def authenticate(self, username=None, password=None): - try: - user = User.objects.get(username=username) - except User.DoesNotExist: - return None - - if '$' in user.password: - if user.check_password(password): - return user - else: - if user.password == sha1(password).hexdigest(): - user.set_password(password) - return user - - return None diff --git a/app/templates/base.html b/app/templates/base.html index f27c8a5..d63bb26 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -62,7 +62,7 @@
  • Lookup User
    • {% endif %} {% if request.user.is_staff %} -
  • Admin
    • +
  • Admin
    • {% endif %} {% if "sentry"|installed %} {% if request.user.is_superuser %} diff --git a/app/urls.py b/app/urls.py index 342b278..5165dad 100644 --- a/app/urls.py +++ b/app/urls.py @@ -4,11 +4,11 @@ from django.contrib.auth.views import login from django.contrib.staticfiles.urls import staticfiles_urlpatterns from django.conf import settings - from utils import installed from registration.views import register from sso.forms import RegistrationFormUniqueEmailBlocked + admin.autodiscover() urlpatterns = patterns('', @@ -45,7 +45,11 @@ if installed('nexus'): nexus.autodiscover() urlpatterns += patterns('', - (r'^nexus/', include(nexus.site.urls)), + (r'^admin/', include('nexus.site.urls')), + ) +else: + urlpatterns += patterns('', + url(r'^admin/', include('admin.site.urls')), ) if settings.DEBUG: From eef6e593358a5a70766dbc1841d05c86d11abc67 Mon Sep 17 00:00:00 2001 From: Andrew Williams Date: Sun, 20 May 2012 18:22:07 +0100 Subject: [PATCH 22/23] Updated README regarding bootstrap --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 189d899..40e2317 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,13 @@ pip install -r requirements.txt As we're using system wide packages, its advisable to install python-mysql packages system wide, otherwise you'll need a basic build env on your machine (build-essentials, python-dev on Debian). +Bootstrap +--------- + +Auth uses Twitter Bootstrap v1 for its design layout, its yet to be updated to v2.0. You can grab the older version from the URL below, extract it into app/sso/static/bootstrap/ + +https://github.com/twitter/bootstrap/tarball/v1.4.0 + Running ------- From 959b4f1ba8a1a8cbcc338e724f49fb2633e1ba9c Mon Sep 17 00:00:00 2001 From: Andrew Williams Date: Mon, 21 May 2012 01:43:29 +0100 Subject: [PATCH 23/23] Fix admin urls --- app/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/urls.py b/app/urls.py index 5165dad..5c2ae29 100644 --- a/app/urls.py +++ b/app/urls.py @@ -49,7 +49,7 @@ if installed('nexus'): ) else: urlpatterns += patterns('', - url(r'^admin/', include('admin.site.urls')), + url(r'^admin/', include(admin.site.urls)), ) if settings.DEBUG: