diff --git a/.gitignore b/.gitignore index d0144b1..d99e213 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ env *.pid app/conf/brokersettings.py ./static/ +.env diff --git a/LICENSE b/LICENSE index dc27802..d808fa0 100644 --- a/LICENSE +++ b/LICENSE @@ -1,9 +1,4 @@ -Auth is officially ISKware/Beerware. - -You are free to copy, modify and use this software as you see fit, as long as -you give attribution to TEST Alliance for the software. If you find this -useful then please consider tipping the following people some beer money or -ISKies for exotic dancers. +This software should be classified as all rights reserved, you may not redistribute the source or applications to any other parties, or use its code as the basis for any further development outside of Test Alliance Please Ignore. Main Developers: diff --git a/README b/README deleted file mode 100644 index a45360d..0000000 --- a/README +++ /dev/null @@ -1,44 +0,0 @@ -REQUIREMENTS ------------- - -Use ./setup-env.sh to setup the virtualenv for Auth, you'll also need the following packages (as named on Debian): - -python-zeroc-ice (ZeroC ICE Bindings for Python, only needed if you are using Mumble connecitivty) -python-imaging (PIL) -python-mysqldb (MySQL connectors for Python) -python-crypto (SSL and related stuff for Django) -python-virtualenv (distribute enabled version) -python-distribute - -Also you'll need a working RabbitMQ install, and your database software of choice. - -RABBITMQ SETUP --------------- - -The fabric config has all the options to auto configure this for you, just give sudo access to /usr/sbin/rabbitmqctl to -your user. - -VIRTUALENV SETUP ----------------- - -Most of Auth's dependancies are pulled through Virtualenv, to setup the enviroment use the ./setup-env.sh script. - -RUNNING -------- - -For live envs, use ./start.sh, which runs a FCGI instance on port 9981. - -For dev, use ./manage.py runserver :, after loading the virtualenv. Load a seperate shell and start the Celery -work processor with ./manage.py celeryd - -DB SETUP --------- - -Copy over dbsettings.py.example and modify to your needs. - - -MUMBLE SETUP ------------- - -If you are using the Mumble SSO connector, then please copy over your Murmur.ice file to the root of the directory, if this -doesnt match the current running Mumble server it'll cause a world of pain. diff --git a/README.md b/README.md new file mode 100644 index 0000000..40e2317 --- /dev/null +++ b/README.md @@ -0,0 +1,33 @@ +TEST Auth +========= + +TEST Auth is a central access management system for EVE Online corporations and alliances, it stores and manages EVE API keys and assigns access permissions based on defined permissions. + +Included is a HR management system, a group management system and a basic API key viewer showing Character and skill information. + + +Requirements +------------ + +The requirements.txt covers all dependencies for Auth, setup a virtual env and install the requirements: + +virtualenv env +. env/bin/activate +pip install -r requirements.txt + +As we're using system wide packages, its advisable to install python-mysql packages system wide, otherwise you'll need a basic build env on your machine (build-essentials, python-dev on Debian). + +Bootstrap +--------- + +Auth uses Twitter Bootstrap v1 for its design layout, its yet to be updated to v2.0. You can grab the older version from the URL below, extract it into app/sso/static/bootstrap/ + +https://github.com/twitter/bootstrap/tarball/v1.4.0 + +Running +------- + +For dev, use ./manage.py runserver :, after loading the virtualenv. In development Celery will operate in-process and doesn't require a seperate celeryd process to execute. + +For Live instances its advisable to run within a WSGI container server such as uWSGI. + diff --git a/app/api/admin.py b/app/api/admin.py index cde57ba..5ec9e8c 100644 --- a/app/api/admin.py +++ b/app/api/admin.py @@ -1,6 +1,5 @@ from django.contrib import admin from api.models import AuthAPIKey, AuthAPILog -from piston.models import Consumer, Token class AuthAPIKeyAdmin(admin.ModelAdmin): list_display = ('key', 'name', 'url', 'active') @@ -18,5 +17,3 @@ class AuthAPILogAdmin(admin.ModelAdmin): admin.site.register(AuthAPIKey, AuthAPIKeyAdmin) admin.site.register(AuthAPILog, AuthAPILogAdmin) -admin.site.register(Consumer, admin.ModelAdmin) -admin.site.register(Token, admin.ModelAdmin) diff --git a/app/api/auth.py b/app/api/auth.py index ca5bd4e..40b4cda 100644 --- a/app/api/auth.py +++ b/app/api/auth.py @@ -1,7 +1,10 @@ from urllib import urlencode from datetime import datetime + from django.http import HttpResponseForbidden from django.contrib.auth.models import AnonymousUser +from django.utils.timezone import now + from api.models import AuthAPIKey, AuthAPILog @@ -21,7 +24,7 @@ class APIKeyAuthentication(object): url = "%s?%s" % (request.path, urlencode(params)) else: url = request.path - AuthAPILog(key=keyobj, access_datetime=datetime.utcnow(), url=url).save() + AuthAPILog.objects.create(key=keyobj, access_datetime=now(), url=url) request.user = AnonymousUser() request.api_key = keyobj return True diff --git a/app/api/handlers/oauth.py b/app/api/handlers/oauth.py index 6d53023..1446774 100644 --- a/app/api/handlers/oauth.py +++ b/app/api/handlers/oauth.py @@ -5,17 +5,16 @@ from operator import itemgetter from django.contrib.auth import login, logout, authenticate from django.contrib.auth.models import User - from django.http import HttpResponse from django.core.urlresolvers import reverse from django.shortcuts import get_object_or_404 from django.conf import settings +from django.utils.timezone import now, utc from piston.handler import BaseHandler from piston.utils import rc, throttle from api.models import AuthAPIKey, AuthAPILog - from eve_proxy.models import CachedDocument from eve_proxy.exceptions import * from eve_api.app_defines import * @@ -54,12 +53,10 @@ class OAuthOpTimerHandler(BaseHandler): for node in dom.getElementsByTagName('rowset')[0].childNodes: if node.nodeType == 1: - ownerID = node.getAttribute('ownerID') + ownerID = node.getAttribute('ownerID') if ownerID != '1': - date = node.getAttribute('eventDate') - dt = datetime.strptime(date, '%Y-%m-%d %H:%M:%S') - now = datetime.utcnow() - startsIn = int(dt.strftime('%s')) - int(now.strftime('%s')) + dt = datetime.strptime(node.getAttribute('eventDate'), '%Y-%m-%d %H:%M:%S').replace(tzinfo=utc) + startsIn = int(dt.strftime('%s')) - int(now().strftime('%s')) duration = int(node.getAttribute('duration')) fid = re.search('topic=[\d]+', node.getAttribute('eventText')) @@ -84,7 +81,7 @@ class OAuthOpTimerHandler(BaseHandler): 'isImportant': int(node.getAttribute('importance')), 'eventText': node.getAttribute('eventText'), 'endsIn':endsIn, - 'forumLink': forumlink} + 'forumLink': forumlink} events.append(event) if len(events) == 0: @@ -101,7 +98,7 @@ class OAuthOpTimerHandler(BaseHandler): 'forumLink': ''}]} else: events.sort(key=itemgetter('startsIn')) - return {'ops': events, 'doc_time': cached_doc.time_retrieved, 'cache_until': cached_doc.cached_until, 'current_time': datetime.utcnow() } + return {'ops': events, 'doc_time': cached_doc.time_retrieved, 'cache_until': cached_doc.cached_until, 'current_time': now() } class OAuthCharacterHandler(BaseHandler): diff --git a/app/api/handlers/v1.py b/app/api/handlers/v1.py index 2dd59c5..bfc96ec 100644 --- a/app/api/handlers/v1.py +++ b/app/api/handlers/v1.py @@ -4,11 +4,11 @@ from xml.dom import minidom from operator import itemgetter from django.contrib.auth.models import User - from django.http import HttpResponse from django.core.urlresolvers import reverse from django.shortcuts import get_object_or_404 from django.conf import settings +from django.utils.timezone import now from BeautifulSoup import BeautifulSoup from piston.handler import BaseHandler @@ -194,12 +194,10 @@ class OpTimerHandler(BaseHandler): events = [] for node in dom.getElementsByTagName('rowset')[0].childNodes: if node.nodeType == 1: - ownerID = node.getAttribute('ownerID') + ownerID = node.getAttribute('ownerID') if ownerID != '1': - date = node.getAttribute('eventDate') - dt = datetime.strptime(date, '%Y-%m-%d %H:%M:%S') - now = datetime.utcnow() - startsIn = int(dt.strftime('%s')) - int(now.strftime('%s')) + dt = datetime.strptime(node.getAttribute('eventDate'), '%Y-%m-%d %H:%M:%S').replace(tzinfo=utc) + startsIn = int(dt.strftime('%s')) - int(now().strftime('%s')) duration = int(node.getAttribute('duration')) fid = re.search('topic=[\d]+', node.getAttribute('eventText')) @@ -224,7 +222,7 @@ class OpTimerHandler(BaseHandler): 'isImportant': int(node.getAttribute('importance')), 'eventText': ' '.join(BeautifulSoup(node.getAttribute('eventText')).findAll(text=True)), 'endsIn':endsIn, - 'forumLink': forumlink} + 'forumLink': forumlink} events.append(event) if len(events) == 0: return {'ops':[{ @@ -240,7 +238,7 @@ class OpTimerHandler(BaseHandler): 'forumLink': ''}]} else: events.sort(key=itemgetter('startsIn')) - return {'ops': events, 'doc_time': cached_doc.time_retrieved, 'cache_until': cached_doc.cached_until, 'current_time': datetime.utcnow() } + return {'ops': events, 'doc_time': cached_doc.time_retrieved, 'cache_until': cached_doc.cached_until, 'current_time': now() } class BlacklistHandler(BaseHandler): diff --git a/app/conf/common.py b/app/conf/common.py index 47f20f4..3d11112 100644 --- a/app/conf/common.py +++ b/app/conf/common.py @@ -10,11 +10,11 @@ TIME_ZONE = 'UTC' LANGUAGE_CODE = 'en-us' SITE_ID = 1 USE_I18N = True +USE_TZ = False # Defines the Static Media storage as per staticfiles contrib STATIC_ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'static') STATIC_URL = '/static/' -ADMIN_MEDIA_PREFIX = STATIC_URL + 'admin/' # Make this unique, and don't share it with anybody. SECRET_KEY = '' @@ -62,10 +62,8 @@ INSTALLED_APPS = ( 'django.contrib.sites', 'django.contrib.humanize', 'django.contrib.staticfiles', - 'nexus', - 'gargoyle', - 'sentry', 'raven.contrib.django', + 'gargoyle', 'south', 'piston', 'djcelery', @@ -81,10 +79,6 @@ INSTALLED_APPS = ( 'tools', ) -AUTHENTICATION_BACKENDS = ( - 'sso.backends.SimpleHashModelBackend', -) - AUTH_PROFILE_MODULE = 'sso.SSOUser' LOGIN_REDIRECT_URL = "/profile" LOGIN_URL = "/login" @@ -131,7 +125,7 @@ GARGOYLE_SWITCH_DEFAULTS = { 'description': 'Enables/Disables the HR functionality.', }, 'eve-cak': { - 'is_active': False, + 'is_active': True, 'label': 'EVE Customizable API Keys', 'description': 'Enables/Disables EVE API CAK support.', }, @@ -155,45 +149,64 @@ GARGOYLE_SWITCH_DEFAULTS = { LOGGING = { 'version': 1, - 'disable_existing_loggers': True, + 'disable_existing_loggers': False, + 'root': { + 'level': 'WARNING', + 'handlers': ['sentry', 'console'], + }, + 'filters': { + 'require_debug_false': { + '()': 'django.utils.log.RequireDebugFalse' + } + }, + 'formatters': { + 'verbose': { + 'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s' + }, + }, 'handlers': { - 'null': { - 'level':'DEBUG', - 'class':'django.utils.log.NullHandler', + 'sentry': { + 'level': 'DEBUG', + 'class': 'raven.contrib.django.handlers.SentryHandler', + 'filters': ['require_debug_false'], + }, + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'formatter': 'verbose' }, 'mail_admins': { 'level': 'ERROR', 'class': 'django.utils.log.AdminEmailHandler', - }, - 'sentry': { - 'level': 'DEBUG', - 'class': 'raven.contrib.django.handlers.SentryHandler', - }, + 'include_html': True, + 'filters': ['require_debug_false'], + } }, 'loggers': { - 'root': { - 'level': 'WARNING', - 'handlers': ['sentry'], + 'django.request': { + 'handlers': ['console'], + 'level': 'ERROR', + 'propagate': True, + }, + 'django.db.backends': { + 'level': 'ERROR', + 'handlers': ['console'], + 'propagate': False, + }, + 'raven': { + 'level': 'DEBUG', + 'handlers': ['console'], + 'propagate': False, }, 'sentry.errors': { + 'level': 'DEBUG', + 'handlers': ['console'], + 'propagate': False, + }, + 'celery': { 'level': 'WARNING', - 'handlers': ['null'], - 'propagate': True, - }, - 'django': { - 'handlers':['null'], - 'propagate': True, - 'level':'INFO', - }, - 'django.request': { - 'handlers': ['null'], - 'level': 'ERROR', - 'propagate': True, - }, - 'celery.task.default': { - 'handlers': ['null'], - 'level': 'ERROR', - 'propagate': True, + 'handlers': ['console'], + 'propagate': False, }, } -} +} \ No newline at end of file diff --git a/app/eve_api/models/alliance.py b/app/eve_api/models/alliance.py index de0a8ce..6890fde 100644 --- a/app/eve_api/models/alliance.py +++ b/app/eve_api/models/alliance.py @@ -1,7 +1,9 @@ from django.db import models from django.contrib.auth.models import Group + from eve_api.models import EVEAPIModel + class EVEPlayerAlliance(EVEAPIModel): """ Represents a player-controlled alliance. Updated from the alliance diff --git a/app/eve_api/models/character.py b/app/eve_api/models/character.py index f4df5cf..114ad90 100644 --- a/app/eve_api/models/character.py +++ b/app/eve_api/models/character.py @@ -1,5 +1,6 @@ from django.db import models from django.core.exceptions import ObjectDoesNotExist + from eve_api.app_defines import * from eve_api.models import EVEAPIModel diff --git a/app/eve_api/models/corporation.py b/app/eve_api/models/corporation.py index 871e801..2db246f 100644 --- a/app/eve_api/models/corporation.py +++ b/app/eve_api/models/corporation.py @@ -1,9 +1,12 @@ from django.db import models from django.contrib.auth.models import Group + from gargoyle import gargoyle + from eve_api.models import EVEAPIModel from eve_api.app_defines import * + class EVEPlayerCorporation(EVEAPIModel): """ Represents a player-controlled corporation. Updated from a mixture of diff --git a/app/eve_api/tasks/account.py b/app/eve_api/tasks/account.py index 526748b..384fd60 100644 --- a/app/eve_api/tasks/account.py +++ b/app/eve_api/tasks/account.py @@ -3,25 +3,24 @@ from datetime import datetime, timedelta from xml.dom import minidom import logging -from celery.decorators import task +from django.conf import settings +from django.contrib.auth.models import User +from django.utils.timezone import now, utc + +from celery.task import task from celery.task.sets import TaskSet from gargoyle import gargoyle -from django.conf import settings - from eve_proxy.exceptions import * from eve_proxy.models import CachedDocument - from eve_api.models import EVEAccount, EVEPlayerCharacter from eve_api.app_defines import * from eve_api.api_exceptions import * from eve_api.utils import basic_xml_parse_doc from eve_api.tasks.character import import_eve_characters from eve_api.tasks.corporation import import_corp_members, import_corp_details - from sso.tasks import update_user_access -from django.contrib.auth.models import User @task(ignore_result=True, expires=120) def queue_apikey_updates(update_delay=86400, batch_size=50): @@ -37,12 +36,12 @@ def queue_apikey_updates(update_delay=86400, batch_size=50): # Update all the eve accounts and related corps delta = timedelta(seconds=update_delay) - log.info("Updating APIs older than %s" % (datetime.now() - delta)) + log.info("Updating APIs older than %s" % (now() - delta)) if gargoyle.is_active('eve-cak'): - accounts = EVEAccount.objects.filter(api_last_updated__lt=(datetime.now() - delta)).exclude(api_status__in=[API_STATUS_ACC_EXPIRED, API_STATUS_KEY_EXPIRED, API_STATUS_AUTH_ERROR]).order_by('api_last_updated')[:batch_size] + accounts = EVEAccount.objects.filter(api_last_updated__lt=(now() - delta)).exclude(api_status__in=[API_STATUS_ACC_EXPIRED, API_STATUS_KEY_EXPIRED, API_STATUS_AUTH_ERROR]).order_by('api_last_updated')[:batch_size] else: - accounts = EVEAccount.objects.filter(api_last_updated__lt=(datetime.now() - delta)).exclude(api_status__in=[API_STATUS_ACC_EXPIRED, API_STATUS_KEY_EXPIRED, API_STATUS_AUTH_ERROR]).exclude(api_keytype__gt=2).order_by('api_last_updated')[:batch_size] + accounts = EVEAccount.objects.filter(api_last_updated__lt=(now() - delta)).exclude(api_status__in=[API_STATUS_ACC_EXPIRED, API_STATUS_KEY_EXPIRED, API_STATUS_AUTH_ERROR]).exclude(api_keytype__gt=2).order_by('api_last_updated')[:batch_size] log.info("%s account(s) to update" % accounts.count()) for acc in accounts: log.debug("Queueing UserID %s for update" % acc.pk) @@ -115,7 +114,7 @@ def import_apikey_func(api_userid, api_key, user=None, force_cache=False, log=lo account.api_keytype = API_KEYTYPE_ACCOUNT account.api_accessmask = int(keydoc['accessMask']) if not keydoc['expires'] == '': - account.api_expiry = datetime.strptime(keydoc['expires'], '%Y-%m-%d %H:%M:%S') + account.api_expiry = datetime.strptime(keydoc['expires'], '%Y-%m-%d %H:%M:%S').replace(tzinfo=utc) # Checks account status to see if the account is still active if not account.api_keytype == API_KEYTYPE_CORPORATION: @@ -124,8 +123,8 @@ def import_apikey_func(api_userid, api_key, user=None, force_cache=False, log=lo status = CachedDocument.objects.api_query('/account/AccountStatus.xml.aspx', params=auth_params, no_cache=True) status = basic_xml_parse_doc(status)['eveapi'] if not status.get('error', None): - paiddate = datetime.strptime(status['result']['paidUntil'], '%Y-%m-%d %H:%M:%S') - if paiddate <= datetime.utcnow(): + paiddate = datetime.strptime(status['result']['paidUntil'], '%Y-%m-%d %H:%M:%S').replace(tzinfo=utc) + if paiddate <= now(): account.api_status = API_STATUS_ACC_EXPIRED else: account.api_status = API_STATUS_OK @@ -236,7 +235,7 @@ def import_apikey_func(api_userid, api_key, user=None, force_cache=False, log=lo if account.user: update_user_access.delay(account.user.id) - account.api_last_updated = datetime.utcnow() + account.api_last_updated = now() account.save() return account diff --git a/app/eve_api/tasks/alliance.py b/app/eve_api/tasks/alliance.py index 58835e5..ce3c30e 100644 --- a/app/eve_api/tasks/alliance.py +++ b/app/eve_api/tasks/alliance.py @@ -1,7 +1,7 @@ from datetime import datetime from xml.dom import minidom -from celery.decorators import task +from celery.task import task from eve_proxy.models import CachedDocument from eve_proxy.exceptions import DocumentRetrievalError @@ -11,6 +11,7 @@ from eve_api.utils import basic_xml_parse_doc from eve_api.tasks.corporation import import_corp_details, import_corp_details_result from django.core.exceptions import ValidationError +from django.utils.timezone import now, utc @task(ignore_result=True, default_retry_delay=10 * 60) def import_alliance_details(): @@ -32,10 +33,10 @@ def import_alliance_details(): allobj, created = EVEPlayerAlliance.objects.get_or_create(pk=alliance['allianceID']) allobj.name = alliance['name'] allobj.ticker = alliance['shortName'] - allobj.date_founded = datetime.strptime(alliance['startDate'], "%Y-%m-%d %H:%M:%S") + allobj.date_founded = datetime.strptime(alliance['startDate'], "%Y-%m-%d %H:%M:%S").replace(tzinfo=utc) allobj.executor, created = EVEPlayerCorporation.objects.get_or_create(id=alliance['executorCorpID']) allobj.member_count = alliance['memberCount'] - allobj.api_last_updated = datetime.utcnow() + allobj.api_last_updated = now() allobj.save() members = [int(corp['corporationID']) for corp in alliance['memberCorporations']] diff --git a/app/eve_api/tasks/character.py b/app/eve_api/tasks/character.py index 50c2ede..363f1f2 100644 --- a/app/eve_api/tasks/character.py +++ b/app/eve_api/tasks/character.py @@ -2,7 +2,9 @@ from datetime import datetime, timedelta from xml.dom import minidom import logging -from celery.decorators import task +from django.utils.timezone import now, utc + +from celery.task import task from celery.task.sets import subtask from gargoyle import gargoyle @@ -85,11 +87,11 @@ def import_eve_character_func(character_id, key_id=None, logger=logging.getLogge # Set corporation and join date corp, created = EVEPlayerCorporation.objects.get_or_create(pk=values['corporationID']) from eve_api.tasks.corporation import import_corp_details - if created or not corp.name or corp.api_last_updated < (datetime.utcnow() - timedelta(hours=12)): + if created or not corp.name or corp.api_last_updated < (now() - timedelta(hours=12)): import_corp_details.delay(values['corporationID']) pchar.corporation = corp - pchar.corporation_date = values['corporationDate'] + pchar.corporation_date = datetime.strptime(values['corporationDate'], "%Y-%m-%d %H:%M:%S").replace(tzinfo=utc) # Derrive Race value from the choices for v in API_RACES_CHOICES: @@ -106,7 +108,8 @@ def import_eve_character_func(character_id, key_id=None, logger=logging.getLogge corp, created = EVEPlayerCorporation.objects.get_or_create(pk=emp['corporationID']) if created: import_corp_details.delay(emp['corporationID']) - eobj, created = EVEPlayerCharacterEmploymentHistory.objects.get_or_create(pk=emp['recordID'], corporation=corp, character=pchar, start_date=emp['startDate']) + startdate = datetime.strptime(emp['startDate'], "%Y-%m-%d %H:%M:%S").replace(tzinfo=utc) + eobj, created = EVEPlayerCharacterEmploymentHistory.objects.get_or_create(pk=emp['recordID'], corporation=corp, character=pchar, start_date=startdate) # We've been passed a Key ID, try and work with it if key_id: @@ -183,7 +186,7 @@ def import_eve_character_func(character_id, key_id=None, logger=logging.getLogge else: pchar.gender = API_GENDER_FEMALE - pchar.api_last_updated = datetime.utcnow() + pchar.api_last_updated = now() pchar.save() if acc: diff --git a/app/eve_api/tasks/corporation.py b/app/eve_api/tasks/corporation.py index 934d73d..3bf7d16 100644 --- a/app/eve_api/tasks/corporation.py +++ b/app/eve_api/tasks/corporation.py @@ -3,7 +3,9 @@ import logging from datetime import datetime, timedelta from xml.dom import minidom -from celery.decorators import task +from django.utils.timezone import now, utc + +from celery.task import task from gargoyle import gargoyle from eve_proxy.models import CachedDocument @@ -55,7 +57,7 @@ def import_corp_details_result(corp_id, callback=None): def import_corp_details_func(corp_id, log=logging.getLogger(__name__)): corpobj, created = EVEPlayerCorporation.objects.get_or_create(id=corp_id) - if created or not corpobj.api_last_updated or corpobj.api_last_updated < (datetime.utcnow() - timedelta(hours=12)): + if created or not corpobj.api_last_updated or corpobj.api_last_updated < (now() - timedelta(hours=12)): try: doc = CachedDocument.objects.api_query('/corp/CorporationSheet.xml.aspx', {'corporationID': corp_id}) @@ -98,14 +100,12 @@ def import_corp_details_func(corp_id, log=logging.getLogger(__name__)): if int(d['allianceID']): corpobj.alliance, created = EVEPlayerAlliance.objects.get_or_create(id=d['allianceID']) + corpobj.api_last_updated = now() + corpobj.save() + # Skip looking up the CEOs for NPC corps and ones with no CEO defined (dead corps) if corp_id > 1000182 and int(d['ceoID']) > 1: import_eve_character.delay(d['ceoID'], callback=link_ceo.subtask(corporation=corpobj.id)) - else: - corpobj.ceo_character = None - - corpobj.api_last_updated = datetime.utcnow() - corpobj.save() return EVEPlayerCorporation.objects.get(pk=corpobj.pk) @@ -113,7 +113,9 @@ def import_corp_details_func(corp_id, log=logging.getLogger(__name__)): @task(ignore_result=True) def link_ceo(corporation, character): """ Links a character to the CEO position of a corporation """ - corpobj = EVEPlayerCorporation.objects.filter(id=corporation).update(ceo_character=EVEPlayerCharacter.objects.get(id=character)) + corp = EVEPlayerCorporation.objects.filter(id=corporation) + char = EVEPlayerCharacter.objects.get(id=character) + corp.update(ceo_character=char) @task(ignore_result=True) @@ -159,10 +161,10 @@ def import_corp_members(key_id, character_id): if created: charobj.name = character['name'] charobj.corporation = corp - charobj.corporation_date = character['startDateTime'] + charobj.corporation_date = datetime.strptime(character['startDateTime'], "%Y-%m-%d %H:%M:%S").replace(tzinfo=utc) if 'logonDateTime' in character: - charobj.last_login = character['logonDateTime'] - charobj.last_logoff = character['logoffDateTime'] + charobj.last_login = datetime.strptime(character['logonDateTime'], "%Y-%m-%d %H:%M:%S").replace(tzinfo=utc) + charobj.last_logoff = datetime.strptime(character['logoffDateTime'], "%Y-%m-%d %H:%M:%S").replace(tzinfo=utc) charobj.current_location_id = int(character['locationID']) else: charobj.last_login = None diff --git a/app/eve_api/tasks/static.py b/app/eve_api/tasks/static.py index ffc9ef6..f778f93 100644 --- a/app/eve_api/tasks/static.py +++ b/app/eve_api/tasks/static.py @@ -1,4 +1,5 @@ -from celery.decorators import task +from celery.task import task + from eve_proxy.models import CachedDocument from eve_api.utils import basic_xml_parse_doc from eve_api.models import EVESkill, EVESkillGroup diff --git a/app/eve_api/templatetags/naturaltimediff.py b/app/eve_api/templatetags/naturaltimediff.py index 657155a..c79d871 100644 --- a/app/eve_api/templatetags/naturaltimediff.py +++ b/app/eve_api/templatetags/naturaltimediff.py @@ -1,4 +1,5 @@ from django import template +from django.utils.timezone import now register = template.Library() @@ -15,7 +16,7 @@ def naturaltimediff(value): from datetime import datetime if isinstance(value, datetime): - delta = datetime.now() - value + delta = now() - value if delta.days > 6: return value.strftime("%b %d") # May 15 if delta.days > 1: diff --git a/app/eve_api/utils.py b/app/eve_api/utils.py index 840bcb5..2cbed6e 100644 --- a/app/eve_api/utils.py +++ b/app/eve_api/utils.py @@ -1,4 +1,8 @@ +from datetime import datetime from xml.dom import minidom + +from django.utils.timezone import utc + from eve_proxy.models import CachedDocument def basic_xml_parse(nodes): @@ -51,3 +55,7 @@ def basic_xml_parse_doc(doc): return basic_xml_parse(dom.childNodes) return {} + + +def parse_eveapi_date(datestring): + return datetime.strptime(datestring, "%Y-%m-%d %H:%M:%S").replace(tzinfo=utc) \ No newline at end of file diff --git a/app/eve_api/views/base.py b/app/eve_api/views/base.py index d8f6d7a..1293953 100644 --- a/app/eve_api/views/base.py +++ b/app/eve_api/views/base.py @@ -6,7 +6,6 @@ from django.http import HttpResponse, Http404 from django.shortcuts import render_to_response, get_object_or_404, redirect from django.template import RequestContext from django.views.generic import DetailView, ListView - from django.contrib import messages from django.contrib.auth.decorators import login_required @@ -22,6 +21,7 @@ from eve_api.tasks import import_apikey_result from eve_api.utils import basic_xml_parse_doc from eve_api.views.mixins import DetailPaginationMixin + @login_required def eveapi_add(request, post_save_redirect='/', template='eve_api/add.html'): """ Add a EVE API key to a user's account """ diff --git a/app/eve_proxy/__init__.py b/app/eve_proxy/__init__.py index dab0451..0fe5b46 100644 --- a/app/eve_proxy/__init__.py +++ b/app/eve_proxy/__init__.py @@ -1,4 +1,4 @@ -VERSION = (0, 4) +VERSION = (0, 5) # Dynamically calculate the version based on VERSION tuple if len(VERSION)>2 and VERSION[2] is not None: diff --git a/app/eve_proxy/models.py b/app/eve_proxy/models.py index 482c25c..0709e0e 100644 --- a/app/eve_proxy/models.py +++ b/app/eve_proxy/models.py @@ -8,6 +8,7 @@ from xml.dom import minidom from django.db import models, IntegrityError from django.conf import settings from django.core.cache import cache +from django.utils.timezone import utc, now from eve_proxy.exceptions import * @@ -70,13 +71,14 @@ class CachedDocumentManager(models.Manager): logger.debug('Requesting URL: %s' % url) try: + print doc_key doc = super(CachedDocumentManager, self).get_query_set().get(pk=doc_key) created = False except self.model.DoesNotExist: doc = CachedDocument(pk=doc_key, url_path=url) created = True - if created or not doc.cached_until or datetime.utcnow() > doc.cached_until or no_cache: + if created or not doc.cached_until or now() > doc.cached_until or no_cache: stat_update_count('eve_proxy_api_requests') req = urllib2.Request(url) @@ -103,23 +105,29 @@ class CachedDocumentManager(models.Manager): raise DocumentRetrievalError(e.reason) else: doc.body = unicode(conn.read(), 'utf-8') - doc.time_retrieved = datetime.utcnow() + doc.time_retrieved = now() error = 0 try: # Parse the response via minidom dom = minidom.parseString(doc.body.encode('utf-8')) except: - doc.cached_until = datetime.utcnow() + doc.cached_until = now() else: - date = datetime.strptime(dom.getElementsByTagName('cachedUntil')[0].childNodes[0].nodeValue, '%Y-%m-%d %H:%M:%S') - # Add 30 seconds to the cache timers, avoid hitting too early and account for some minor clock skew. - doc.cached_until = date + timedelta(seconds=getattr(settings, 'EVE_PROXY_GLOBAL_CACHE_ADJUSTMENT', 30)) + + # Calculate the cache timer on the basis of the currentTime and cachedUntil fields provided by the API + # This allows for clock skew not to fuck with the timers. + currenttime = datetime.strptime(dom.getElementsByTagName('currentTime')[0].childNodes[0].nodeValue, '%Y-%m-%d %H:%M:%S') + cacheuntil = datetime.strptime(dom.getElementsByTagName('cachedUntil')[0].childNodes[0].nodeValue, '%Y-%m-%d %H:%M:%S') + doc.cached_until = now() + (cacheuntil - currenttime) + + # Add the global adjustment, to avoid CCP's hardline cache timers + doc.cached_until += timedelta(seconds=getattr(settings, 'EVE_PROXY_GLOBAL_CACHE_ADJUSTMENT', 30)) # Allow for tuning of individual pages with bad cache returns adjustconfig = getattr(settings, 'EVE_PROXY_CACHE_ADJUSTMENTS', {}) if url_path.lower() in adjustconfig: - doc.cached_until = date + timedelta(seconds=adjustconfig[url_path.lower()]) + doc.cached_until += timedelta(seconds=adjustconfig[url_path.lower()]) enode = dom.getElementsByTagName('error') if enode: diff --git a/app/eve_proxy/tasks.py b/app/eve_proxy/tasks.py index 5792da7..ada25b3 100644 --- a/app/eve_proxy/tasks.py +++ b/app/eve_proxy/tasks.py @@ -1,14 +1,19 @@ -from django.conf import settings import logging from datetime import datetime, timedelta -from celery.decorators import task + +from django.conf import settings +from django.utils.timezone import now + +from celery.task import task + from eve_proxy.models import CachedDocument, ApiAccessLog + @task(ignore_result=True) def clear_stale_cache(cache_extension=0): log = clear_stale_cache.get_logger() - time = datetime.utcnow() - timedelta(seconds=cache_extension) + time = now() - timedelta(seconds=cache_extension) objs = CachedDocument.objects.filter(cached_until__lt=time) log.info('Removing %s stale cache documents' % objs.count()) objs.delete() @@ -18,7 +23,7 @@ def clear_stale_cache(cache_extension=0): def clear_old_logs(): log = clear_old_logs.get_logger() - time = datetime.utcnow() - timedelta(days=settings.EVE_PROXY_KEEP_LOGS) + time = now() - timedelta(days=getattr(settings, 'EVE_PROXY_KEEP_LOGS', 30)) objs = ApiAccessLog.objects.filter(time_access__lt=time) log.info('Removing %s old access logs' % objs.count()) objs.delete() diff --git a/app/eve_proxy/tests.py b/app/eve_proxy/tests.py index fb47f47..6965548 100644 --- a/app/eve_proxy/tests.py +++ b/app/eve_proxy/tests.py @@ -4,6 +4,7 @@ from datetime import datetime import time from django.utils import unittest +from django.utils.timezone import now from eve_proxy.models import CachedDocument from eve_proxy.exceptions import * @@ -40,8 +41,8 @@ class CachedDocumentTestCase(unittest.TestCase): """ Tests if objects are being cached correctly """ url = '/server/ServerStatus.xml.aspx' - obj = CachedDocument.objects.api_query(url, no_cache=True) - obj2 = CachedDocument.objects.api_query(url) + obj = CachedDocument.objects.api_query(url, no_cache=True) + obj2 = CachedDocument.objects.api_query(url) self.assertEqual(obj.pk, obj2.pk, "Objects are not caching correctly") @@ -51,7 +52,7 @@ class CachedDocumentTestCase(unittest.TestCase): url = '/server/ServerStatus.xml.aspx' obj = CachedDocument.objects.api_query(url, no_cache=True) ret_time = obj.time_retrieved - obj.cached_until = datetime.utcnow() + obj.cached_until = now() obj.save() time.sleep(1) diff --git a/app/eve_proxy/urls.py b/app/eve_proxy/urls.py index 6c2e37f..02d091e 100644 --- a/app/eve_proxy/urls.py +++ b/app/eve_proxy/urls.py @@ -1,8 +1,10 @@ from django.conf.urls.defaults import * +from eve_proxy.views import EVEAPIProxyView + urlpatterns = patterns('eve_proxy.views', # This view can be used just like EVE API's http://api.eve-online.com. # Any parameters or URL paths are sent through the cache system and # forwarded to the EVE API site as needed. - url(r'^', 'retrieve_xml', name='eve_proxy-retrieve_xml'), + url(r'^', EVEAPIProxyView.as_view(), name='eveproxy-apiproxy'), ) diff --git a/app/eve_proxy/views.py b/app/eve_proxy/views.py index 83f687b..8d52a08 100644 --- a/app/eve_proxy/views.py +++ b/app/eve_proxy/views.py @@ -1,39 +1,35 @@ from django.core.urlresolvers import reverse from django.http import HttpResponse, HttpResponseNotFound, HttpResponseServerError +from django.views.generic import View + from eve_proxy.models import CachedDocument -def retrieve_xml(request): - """ - A view that forwards EVE API requests through the cache system, either - retrieving a cached document or querying and caching as needed. - """ - # This is the URL path (minus the parameters). - url_path = request.META['PATH_INFO'].replace(reverse('eve_proxy.views.retrieve_xml'),"/") - # The parameters attached to the end of the URL path. - if request.method == 'POST': - p = request.POST - else: - p = request.GET +class EVEAPIProxyView(View): + """Allows for standard EVE API calls to be proxied through your application""" - # Convert the QuerySet object into a dict - params = {} - for key,value in p.items(): - params[key] = value - - if url_path == '/' or url_path == '': - # If they don't provide any kind of query, shoot a quick error message. - return HttpResponseNotFound('No API query specified.') + def get(self, request, *args, **kwargs): + return self.get_document(request, request.GET) - if 'userID' in params and not 'service' in params: - return HttpResponse('No Service ID provided.') + def post(self, request, *args, **kwargs): + return self.get_document(request, request.POST) + + def get_document(self, request, params): + url_path = request.META['PATH_INFO'].replace(reverse('eveproxy-apiproxy'),"/") - try: + if url_path == '/' or url_path == '': + # If they don't provide any kind of query, shoot a quick error message. + return HttpResponseNotFound('No API query specified.') + + if 'userID' in params and not 'service' in params: + return HttpResponse('No Service ID provided.') + + #try: cached_doc = CachedDocument.objects.api_query(url_path, params, exceptions=False) - except: - return HttpResponseServerError('Error occured') + #except: + # return HttpResponseServerError('Error occured') - if cached_doc: - return HttpResponse(cached_doc.body, mimetype='text/xml') + if cached_doc: + return HttpResponse(cached_doc.body, mimetype='text/xml') - return HttpResponseNotFound('Error retrieving the document') + return HttpResponseNotFound('Error retrieving the document') diff --git a/app/groups/urls.py b/app/groups/urls.py index 02bfb3f..c88c49e 100644 --- a/app/groups/urls.py +++ b/app/groups/urls.py @@ -1,5 +1,5 @@ from django.conf.urls.defaults import * -from django.core.urlresolvers import reverse +from django.core.urlresolvers import reverse_lazy from groups import views urlpatterns = patterns('', @@ -14,5 +14,5 @@ urlpatterns = patterns('', ) urlpatterns += patterns('django.views.generic.simple', - ('^$', 'redirect_to', {'url': reverse('groups.views.group_list')}), + ('^$', 'redirect_to', {'url': reverse_lazy('groups.views.group_list')}), ) diff --git a/app/hr/tasks.py b/app/hr/tasks.py index 8109c96..a0a8467 100644 --- a/app/hr/tasks.py +++ b/app/hr/tasks.py @@ -1,7 +1,9 @@ from django.conf import settings import logging from datetime import datetime, timedelta -from celery.decorators import task + +from celery.task import task + from hr.utils import blacklist_values from django.contrib.auth.models import User from django.core.mail import send_mail diff --git a/app/hr/utils.py b/app/hr/utils.py index f3eedfd..13a524a 100644 --- a/app/hr/utils.py +++ b/app/hr/utils.py @@ -3,6 +3,7 @@ from datetime import datetime from django.db import models from django.template.loader import render_to_string +from django.utils.timezone import now from eve_api.models import EVEPlayerCharacter from hr.app_defines import * @@ -29,7 +30,7 @@ def blacklist_values(user, level=BLACKLIST_LEVEL_NOTE): """ blacklist = [] - bl_items = Blacklist.objects.filter(models.Q(expiry_date__gt=datetime.now()) | models.Q(expiry_date=None), level__lte=level) + bl_items = Blacklist.objects.filter(models.Q(expiry_date__gt=now()) | models.Q(expiry_date=None), level__lte=level) # Check Reddit blacklists if installed('reddit'): diff --git a/app/hr/views.py b/app/hr/views.py index 66a192b..0994da4 100644 --- a/app/hr/views.py +++ b/app/hr/views.py @@ -13,6 +13,7 @@ from django.forms.extras.widgets import SelectDateWidget from django.views.generic import TemplateView, DetailView, FormView, CreateView, ListView from django.views.generic.detail import BaseDetailView from django.conf import settings +from django.utils.timezone import now from gargoyle import gargoyle @@ -354,7 +355,7 @@ class HrBlacklistUser(FormView): self.source = BlacklistSource.objects.get(id=1) self.expiry = form.cleaned_data.get('expiry_date', None) if not self.expiry: - self.expiry = datetime.utcnow() + timedelta(days=50*365) # 50 year default + self.expiry = now() + timedelta(days=50*365) # 50 year default self.level = form.cleaned_data.get('level', 0) self.reason = form.cleaned_data.get('reason', 'No reason provided') diff --git a/app/reddit/__init__.py b/app/reddit/__init__.py index 805a0d1..ee1709b 100644 --- a/app/reddit/__init__.py +++ b/app/reddit/__init__.py @@ -1,4 +1,4 @@ -VERSION = (0, 1) +VERSION = (0, 2) # Dynamically calculate the version based on VERSION tuple if len(VERSION)>2 and VERSION[2] is not None: diff --git a/app/reddit/models.py b/app/reddit/models.py index ae05ead..55f8126 100644 --- a/app/reddit/models.py +++ b/app/reddit/models.py @@ -4,6 +4,7 @@ import urllib from django.utils import simplejson as json from django.db import models from django.contrib.auth.models import User +from django.utils.timezone import now, utc from reddit.api import Comment @@ -49,8 +50,8 @@ class RedditAccount(models.Model): self.comment_karma = int(data['comment_karma']) self.reddit_id = unicode(data['id'], 'utf-8') - self.date_created = datetime.fromtimestamp(data['created_utc']) - self.last_update = datetime.now() + self.date_created = datetime.fromtimestamp(data['created_utc']).replace(tzinfo=utc) + self.last_update = now() def recent_posts(self): """ Returns the first page of posts visible on the user's profile page """ diff --git a/app/reddit/tasks.py b/app/reddit/tasks.py index 7910ea5..1da04ef 100644 --- a/app/reddit/tasks.py +++ b/app/reddit/tasks.py @@ -1,10 +1,14 @@ from datetime import datetime, timedelta from urllib2 import HTTPError, URLError -from celery.task import Task -from celery.decorators import task + +from django.conf import settings +from django.utils.timezone import now + +from celery.task import Task, task + from reddit.models import RedditAccount from reddit.api import Inbox, LoginError, Flair -from django.conf import settings + class send_reddit_message(Task): @@ -72,8 +76,8 @@ def queue_account_updates(update_delay=604800, batch_size=50): log = queue_account_updates.get_logger() # Update all the eve accounts and related corps delta = timedelta(seconds=update_delay) - log.info("Updating Accounts older than %s" % (datetime.now() - delta)) - accounts = RedditAccount.objects.order_by('last_update').filter(last_update__lt=(datetime.now() - delta))[:batch_size] + log.info("Updating Accounts older than %s" % (now() - delta)) + accounts = RedditAccount.objects.order_by('last_update').filter(last_update__lt=(now() - delta))[:batch_size] log.info("%s account(s) to update" % accounts.count()) for acc in accounts: log.debug("Queueing Account %s for update" % acc.username) diff --git a/app/sso/backends.py b/app/sso/backends.py deleted file mode 100644 index 712d483..0000000 --- a/app/sso/backends.py +++ /dev/null @@ -1,26 +0,0 @@ -from django.contrib.auth.backends import ModelBackend -from django.contrib.auth.models import User -from hashlib import sha1 - - -class SimpleHashModelBackend(ModelBackend): - - supports_anonymous_user = False - supports_object_permissions = False - supports_inactive_user = False - - def authenticate(self, username=None, password=None): - try: - user = User.objects.get(username=username) - except User.DoesNotExist: - return None - - if '$' in user.password: - if user.check_password(password): - return user - else: - if user.password == sha1(password).hexdigest(): - user.set_password(password) - return user - - return None diff --git a/app/sso/middleware.py b/app/sso/middleware.py index 0112476..a04e86e 100644 --- a/app/sso/middleware.py +++ b/app/sso/middleware.py @@ -1,5 +1,6 @@ from django.db import IntegrityError from django.contrib.auth import logout +from django.utils.timezone import utc, now class InactiveLogoutMiddleware(object): """ @@ -59,7 +60,5 @@ class IPTrackingMiddleware(object): if request.user and not request.user.is_anonymous(): ip, created = SSOUserIPAddress.objects.get_or_create(user=request.user, ip_address=request.META['REMOTE_ADDR']) - if created: - ip.first_seen = datetime.utcnow() - ip.last_seen = datetime.utcnow() + ip.last_seen = now() ip.save() diff --git a/app/sso/tasks.py b/app/sso/tasks.py index 7af87ab..5945f69 100644 --- a/app/sso/tasks.py +++ b/app/sso/tasks.py @@ -10,7 +10,7 @@ from django.utils import simplejson as json from django.contrib.auth.models import User from celery.signals import task_failure -from celery.decorators import task +from celery.task import task from api.models import AuthAPIKey from eve_api.models import EVEAccount, EVEPlayerCorporation, EVEPlayerAlliance diff --git a/app/sso/templatetags/naturaltimediff.py b/app/sso/templatetags/naturaltimediff.py index 657155a..c79d871 100644 --- a/app/sso/templatetags/naturaltimediff.py +++ b/app/sso/templatetags/naturaltimediff.py @@ -1,4 +1,5 @@ from django import template +from django.utils.timezone import now register = template.Library() @@ -15,7 +16,7 @@ def naturaltimediff(value): from datetime import datetime if isinstance(value, datetime): - delta = datetime.now() - value + delta = now() - value if delta.days > 6: return value.strftime("%b %d") # May 15 if delta.days > 1: diff --git a/app/sso/urls.py b/app/sso/urls.py index bdacf9d..dbb668a 100644 --- a/app/sso/urls.py +++ b/app/sso/urls.py @@ -1,5 +1,5 @@ from django.conf.urls.defaults import * -from django.core.urlresolvers import reverse +from django.core.urlresolvers import reverse, reverse_lazy from django.contrib.auth.views import password_change, password_change_done from django.contrib.auth.decorators import login_required @@ -28,5 +28,5 @@ urlpatterns = patterns('', ) urlpatterns += patterns('django.views.generic.simple', - ('^$', 'redirect_to', {'url': reverse('sso.views.profile')}), + ('^$', 'redirect_to', {'url': reverse_lazy('sso.views.profile')}), ) diff --git a/app/templates/base.html b/app/templates/base.html index f27c8a5..d63bb26 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -62,7 +62,7 @@
  • Lookup User
  • {% endif %} {% if request.user.is_staff %} -
  • Admin
  • +
  • Admin
  • {% endif %} {% if "sentry"|installed %} {% if request.user.is_superuser %} diff --git a/app/templates/registration/registration_complete.html b/app/templates/registration/registration_complete.html index f026a47..5c3edc1 100644 --- a/app/templates/registration/registration_complete.html +++ b/app/templates/registration/registration_complete.html @@ -1,6 +1,7 @@ {% extends "registration/registration_base.html" %} {% block title %}Activation email sent{% endblock %} {% block content %} +

    An activation email has been sent. Please check your email and click on the link to activate your account.

    {% endblock %} diff --git a/app/urls.py b/app/urls.py index 342b278..5c2ae29 100644 --- a/app/urls.py +++ b/app/urls.py @@ -4,11 +4,11 @@ from django.contrib.auth.views import login from django.contrib.staticfiles.urls import staticfiles_urlpatterns from django.conf import settings - from utils import installed from registration.views import register from sso.forms import RegistrationFormUniqueEmailBlocked + admin.autodiscover() urlpatterns = patterns('', @@ -45,7 +45,11 @@ if installed('nexus'): nexus.autodiscover() urlpatterns += patterns('', - (r'^nexus/', include(nexus.site.urls)), + (r'^admin/', include('nexus.site.urls')), + ) +else: + urlpatterns += patterns('', + url(r'^admin/', include(admin.site.urls)), ) if settings.DEBUG: diff --git a/requirements.txt b/requirements.txt index acea387..2c4dec7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,21 +1,20 @@ -MySQL-python -Django==1.3 --e hg+https://bitbucket.org/jespern/django-piston@c4b2d21db51a#egg=django_piston --e hg+https://bitbucket.org/ubernostrum/django-registration@d36a38202ee3#egg=django-registration -yolk==0.4.1 --e hg+http://bitbucket.org/schinckel/django-jsonfield#egg=django-jsonfield xmlrpclib==1.0.1 -South==0.7.3 -fabric -flup -celery==2.2.6 -django-celery==2.2.4 xmpppy -django-sentry==1.13.5 -raven==0.7.0 -nexus --e git+https://github.com/nikdoof/gargoyle.git@dca57fc4b437b85f8cbc#egg=gargoyle -beautifulsoup +dnspython +fabric + +Django==1.4 +MySQL-python +Celery==2.5.3 +django-celery==2.5.5 + +django-jsonfield==0.8.7 +South==0.7.4 django-redis-cache IPy==0.75 -dnspython + +nexus +gargoyle==0.9.0 +-e hg+https://bitbucket.org/jespern/django-piston@7c90898072ce#egg=django_piston +-e hg+https://bitbucket.org/ubernostrum/django-registration@27bccd108cde#egg=django-registration +