From 30d1a129753cadd9bd6ad8f0c36d8504b92d6865 Mon Sep 17 00:00:00 2001 From: Andrew Williams Date: Mon, 8 Nov 2010 11:26:55 +0000 Subject: [PATCH] Initial work in bringing celery into Auth --- eve_api/admin.py | 4 ++-- eve_api/cron.py | 23 ++++------------------- eve_api/tasks.py | 32 ++++++++++++++++++++++++++++++++ requirements.txt | 1 + run-cron.py | 2 +- settings.py | 12 ++++++++++++ sso/views.py | 40 +++++++++++++++++++--------------------- 7 files changed, 71 insertions(+), 43 deletions(-) create mode 100644 eve_api/tasks.py diff --git a/eve_api/admin.py b/eve_api/admin.py index 2fdd043..6ac3dfc 100644 --- a/eve_api/admin.py +++ b/eve_api/admin.py @@ -4,11 +4,11 @@ Admin interface models. Automatically detected by admin.autodiscover(). from django.contrib import admin from eve_api.models import * -from eve_api.api_puller.accounts import import_eve_account +from eve_api.tasks import import_apikey def account_api_update(modeladmin, request, queryset): for obj in queryset: - import_eve_account(obj.api_key, obj.api_user_id) + import_apikey.delay(api_key=obj.api_key, api_userid=obj.api_user_id) account_api_update.short_description = "Update account from the EVE API" diff --git a/eve_api/cron.py b/eve_api/cron.py index 8ace559..a342281 100644 --- a/eve_api/cron.py +++ b/eve_api/cron.py @@ -8,6 +8,8 @@ from eve_api.api_puller.corp_management import pull_corp_members from eve_api.api_exceptions import APIAuthException, APINoUserIDException from eve_api.app_defines import * +from eve_api.tasks import * + class UpdateAPIs(): """ Updates all Eve API elements in the database @@ -33,28 +35,11 @@ class UpdateAPIs(): accounts = EVEAccount.objects.filter(api_last_updated__lt=(datetime.datetime.now() - delta)).exclude(api_status=API_STATUS_ACC_EXPIRED).exclude(api_status=API_STATUS_AUTH_ERROR).order_by('api_last_updated')[:self.batches] self._logger.debug("%s account(s) to update" % len(accounts)) for acc in accounts: - self._logger.info("Updating UserID %s" % acc.api_user_id) + self._logger.info("Queueing UserID %s for update" % acc.api_user_id) if not acc.user: acc.delete() continue - eve_api.api_puller.accounts.import_eve_account(acc.api_key, acc.api_user_id) - if acc.api_status == API_STATUS_OK: - if acc.api_keytype == API_KEYTYPE_FULL and acc.characters.filter(director=1).count(): - donecorps = [] - for char in acc.characters.filter(director=1): - if not char.corporation.id in donecorps: - self._logger.info("Updating Corp %s" % char.corporation) - pull_corp_members(acc.api_key, acc.api_user_id, char.id) - char.corporation.query_and_update_corp() - donecorps.append(char.corporation.id) - - if self.settings['update_corp']: - for char in acc.characters.all(): - try: - char.corporation.query_and_update_corp() - except: - self._logger.error('Error updating %s' % corp) - continue + import_apikey.delay(api_key=acc.api_key, api_userid=acc.api_user_id) class AllianceUpdate(): """ diff --git a/eve_api/tasks.py b/eve_api/tasks.py new file mode 100644 index 0000000..c7ff257 --- /dev/null +++ b/eve_api/tasks.py @@ -0,0 +1,32 @@ +from celery.decorators import task +from eve_api.api_puller.accounts import import_eve_account +from eve_api.app_defines import * + +@task() +def import_apikey(api_userid, api_key, user=None, force_cache=False): + acc = import_eve_account(api_key, api_userid, force_cache=force_cache) + donecorps = [] + if acc and acc.api_status == API_STATUS_OK: + if user and not acc.user: + acc.user = user + if acc.api_keytype == API_KEYTYPE_FULL and acc.characters.filter(director=1).count(): + donecorps = [] + for char in acc.characters.filter(director=1): + if not char.corporation.id in donecorps: + #pull_corp_members(acc.api_key, acc.api_user_id, char.id) + char.corporation.query_and_update_corp() + donecorps.append(char.corporation.id) + + for char in acc.characters.all(): + try: + if char.corporation.id not in donecorps: + char.corporation.query_and_update_corp() + donecorps.append(char.corporation.id) + except: + continue + + acc.save() + if acc.user: + acc.user.get_profile().update_access() + + return acc diff --git a/requirements.txt b/requirements.txt index 0420f15..d272a8f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,4 @@ fabric==0.9.1 flup django-debug-toolbar==0.8.3 simplejson +django-celery diff --git a/run-cron.py b/run-cron.py index 90d5ba9..b9f9536 100755 --- a/run-cron.py +++ b/run-cron.py @@ -16,7 +16,7 @@ import settings setup_environ(settings) -logging.basicConfig(level=logging.DEBUG) +logging.basicConfig(level=logging.INFO) log = logging.getLogger('runcron') try: diff --git a/settings.py b/settings.py index 51336c4..ada56b6 100755 --- a/settings.py +++ b/settings.py @@ -1,4 +1,5 @@ import os +import djcelery # Django settings for login project. @@ -90,6 +91,7 @@ INSTALLED_APPS = ( 'django.contrib.humanize', 'south', 'piston', + 'djcelery', 'registration', 'debug_toolbar', 'eve_proxy', @@ -145,8 +147,18 @@ HR_STAFF_GROUP = 'HR Staff' FULL_API_USER_ID = 415631 FULL_API_CHARACTER_ID = 246102445 +### Celery Broker + +BROKER_HOST = "localhost" +BROKER_PORT = 5672 +BROKER_USER = "guest" +BROKER_PASSWORD = "guest" +BROKER_VHOST = "/" + # try and import local settings try: from settingslocal import * except: pass + +djcelery.setup_loader() diff --git a/sso/views.py b/sso/views.py index ff6a9ba..e9ef163 100644 --- a/sso/views.py +++ b/sso/views.py @@ -2,6 +2,7 @@ import hashlib import random import re import unicodedata +import celery from django.http import HttpResponse from django.shortcuts import render_to_response, get_object_or_404, redirect @@ -20,6 +21,8 @@ from sso.models import ServiceAccount, Service, SSOUser, ExistingUser, ServiceEr from sso.forms import EveAPIForm, UserServiceAccountForm, ServiceAccountResetForm, RedditAccountForm, UserLookupForm, APIPasswordForm from reddit.models import RedditAccount +from eve_api.tasks import import_apikey + import settings def index(request): @@ -54,23 +57,16 @@ def eveapi_add(request): if request.method == 'POST': form = EveAPIForm(request.POST) if form.is_valid(): - try: - acc = import_eve_account(form.cleaned_data['api_key'], form.cleaned_data['user_id']) - except APIAuthException: - return redirect('sso.views.profile') - if not acc: - messages.add_message(request, messages.ERROR, "A error was encountered while adding your API key, try again later. If the issue persists, contact a Admin.") - return redirect('sso.views.profile') - - acc.user = request.user - acc.description = form.cleaned_data['description'] - acc.save() - messages.add_message(request, messages.INFO, "EVE API successfully added.") - if len(ServiceAccount.objects.filter(user=request.user, active=0)) > 0: - messages.add_message(request, messages.INFO, "It can take up to 10 minutes for inactive accounts to be reenabled, please check back later.") - - request.user.get_profile().update_access() + task = import_apikey.delay(api_key=form.cleaned_data['api_key'], api_userid=form.cleaned_data['user_id'], user=request.user) + try: + acc = task.wait(5) + except celery.exceptions.TimeoutError: + messages.add_message(request, messages.INFO, "The addition of your API key is still processing, please check back in a minute or so") + pass + else: + messages.add_message(request, messages.INFO, "EVE API successfully added.") + request.user.get_profile().update_access() return redirect('sso.views.profile') else: @@ -105,14 +101,16 @@ def eveapi_refresh(request, userid=0): pass else: if acc.user == request.user or request.user.is_superuser: - import_eve_account(acc.api_key, acc.api_user_id, force_cache=True) - acc.user.get_profile().update_access() + task = import_apikey.delay(api_key=acc.api_key, api_userid=acc.api_user_id, force_cache=True, user=request.user) if request.is_ajax(): - acc = EVEAccount.objects.get(id=userid) - return HttpResponse(serializers.serialize('json', [acc]), mimetype='application/javascript') + try: + acc = task.wait(60) + return HttpResponse(serializers.serialize('json', [acc]), mimetype='application/javascript') + except celery.exceptions.TimeoutError: + return HttpResponse(serializers.serialize('json', [None]), mimetype='application/javascript') else: - messages.add_message(request, messages.INFO,"Key %s has been refreshed from the EVE API." % acc.api_user_id) + messages.add_message(request, messages.INFO,"Key %s has been queued to be refreshed from the API" % acc.api_user_id) return redirect('sso.views.profile')