Initial work in bringing celery into Auth

eve_api/admin.py

@@ -4,11 +4,11 @@ Admin interface models. Automatically detected by admin.autodiscover().
 from django.contrib import admin
 from eve_api.models import *
 
-from eve_api.api_puller.accounts import import_eve_account
+from eve_api.tasks import import_apikey
 
 def account_api_update(modeladmin, request, queryset):
     for obj in queryset:
-        import_eve_account(obj.api_key, obj.api_user_id)
+        import_apikey.delay(api_key=obj.api_key, api_userid=obj.api_user_id)
 
 account_api_update.short_description = "Update account from the EVE API"
 

eve_api/cron.py

@@ -8,6 +8,8 @@ from eve_api.api_puller.corp_management import pull_corp_members
 from eve_api.api_exceptions import APIAuthException, APINoUserIDException
 from eve_api.app_defines import *
 
+from eve_api.tasks import *
+
 class UpdateAPIs():
         """
         Updates all Eve API elements in the database
@@ -33,28 +35,11 @@ class UpdateAPIs():
             accounts = EVEAccount.objects.filter(api_last_updated__lt=(datetime.datetime.now() - delta)).exclude(api_status=API_STATUS_ACC_EXPIRED).exclude(api_status=API_STATUS_AUTH_ERROR).order_by('api_last_updated')[:self.batches]
             self._logger.debug("%s account(s) to update" % len(accounts))
             for acc in accounts:
-               self._logger.info("Updating UserID %s" % acc.api_user_id)
+               self._logger.info("Queueing UserID %s for update" % acc.api_user_id)
                if not acc.user:
                    acc.delete()
                    continue
-               eve_api.api_puller.accounts.import_eve_account(acc.api_key, acc.api_user_id)
-               if acc.api_status == API_STATUS_OK:
-                   if acc.api_keytype == API_KEYTYPE_FULL and acc.characters.filter(director=1).count():
-                       donecorps = []
-                       for char in acc.characters.filter(director=1):
-                           if not char.corporation.id in donecorps:
-                               self._logger.info("Updating Corp %s" % char.corporation)
-                               pull_corp_members(acc.api_key, acc.api_user_id, char.id)
-                               char.corporation.query_and_update_corp()
-                               donecorps.append(char.corporation.id)
-
-                   if self.settings['update_corp']:
-                       for char in acc.characters.all():
-                        try:
-                            char.corporation.query_and_update_corp()
-                        except:
-                            self._logger.error('Error updating %s' % corp)
-                        continue
+               import_apikey.delay(api_key=acc.api_key, api_userid=acc.api_user_id)
 
 class AllianceUpdate():
         """

eve_api/tasks.py

@@ -0,0 +1,32 @@
+from celery.decorators import task
+from eve_api.api_puller.accounts import import_eve_account
+from eve_api.app_defines import *
+
+@task()
+def import_apikey(api_userid, api_key, user=None, force_cache=False):
+    acc = import_eve_account(api_key, api_userid, force_cache=force_cache)
+    donecorps = []
+    if acc and acc.api_status == API_STATUS_OK:
+        if user and not acc.user:
+            acc.user = user
+        if acc.api_keytype == API_KEYTYPE_FULL and acc.characters.filter(director=1).count():
+            donecorps = []
+            for char in acc.characters.filter(director=1):
+                if not char.corporation.id in donecorps:
+                    #pull_corp_members(acc.api_key, acc.api_user_id, char.id)
+                    char.corporation.query_and_update_corp()
+                    donecorps.append(char.corporation.id)
+
+        for char in acc.characters.all():
+            try:
+                if char.corporation.id not in donecorps:
+                    char.corporation.query_and_update_corp()
+                    donecorps.append(char.corporation.id)
+            except:
+                continue
+
+        acc.save()
+        if acc.user:
+            acc.user.get_profile().update_access()
+
+    return acc

requirements.txt

@@ -9,3 +9,4 @@ fabric==0.9.1
 flup
 django-debug-toolbar==0.8.3
 simplejson
+django-celery

run-cron.py

@@ -16,7 +16,7 @@ import settings
 
 setup_environ(settings)
 
-logging.basicConfig(level=logging.DEBUG)
+logging.basicConfig(level=logging.INFO)
 log = logging.getLogger('runcron')
 
 try:

settings.py

@@ -1,4 +1,5 @@
 import os
+import djcelery
 
 # Django settings for login project.
 
@@ -90,6 +91,7 @@ INSTALLED_APPS = (
     'django.contrib.humanize',
     'south',
     'piston',
+    'djcelery',
     'registration',
     'debug_toolbar',
     'eve_proxy',
@@ -145,8 +147,18 @@ HR_STAFF_GROUP = 'HR Staff'
 FULL_API_USER_ID = 415631
 FULL_API_CHARACTER_ID = 246102445
 
+### Celery Broker
+
+BROKER_HOST = "localhost"
+BROKER_PORT = 5672
+BROKER_USER = "guest"
+BROKER_PASSWORD = "guest"
+BROKER_VHOST = "/"
+
 # try and import local settings
 try:
     from settingslocal import *
 except:
     pass
+
+djcelery.setup_loader()

sso/views.py

@@ -2,6 +2,7 @@ import hashlib
 import random
 import re
 import unicodedata
+import celery
 
 from django.http import HttpResponse
 from django.shortcuts import render_to_response, get_object_or_404, redirect
@@ -20,6 +21,8 @@ from sso.models import ServiceAccount, Service, SSOUser, ExistingUser, ServiceEr
 from sso.forms import EveAPIForm, UserServiceAccountForm, ServiceAccountResetForm, RedditAccountForm, UserLookupForm, APIPasswordForm
 from reddit.models import RedditAccount
 
+from eve_api.tasks import import_apikey
+
 import settings
 
 def index(request):
@@ -54,23 +57,16 @@ def eveapi_add(request):
     if request.method == 'POST': 
         form = EveAPIForm(request.POST) 
         if form.is_valid():
+
+            task = import_apikey.delay(api_key=form.cleaned_data['api_key'], api_userid=form.cleaned_data['user_id'], user=request.user)
             try:
-                acc = import_eve_account(form.cleaned_data['api_key'], form.cleaned_data['user_id'])
-            except APIAuthException:
-                return redirect('sso.views.profile')
-
-            if not acc:
-                messages.add_message(request, messages.ERROR, "A error was encountered while adding your API key, try again later. If the issue persists, contact a Admin.")
-                return redirect('sso.views.profile')
-
-            acc.user = request.user
-            acc.description = form.cleaned_data['description']
-            acc.save()
-            messages.add_message(request, messages.INFO, "EVE API successfully added.")
-            if len(ServiceAccount.objects.filter(user=request.user, active=0)) > 0:
-                messages.add_message(request, messages.INFO, "It can take up to 10 minutes for inactive accounts to be reenabled, please check back later.")
-
-            request.user.get_profile().update_access()
+                acc = task.wait(5)
+            except celery.exceptions.TimeoutError:
+                messages.add_message(request, messages.INFO, "The addition of your API key is still processing, please check back in a minute or so")
+                pass    
+            else:
+                messages.add_message(request, messages.INFO, "EVE API successfully added.")
+                request.user.get_profile().update_access()
 
             return redirect('sso.views.profile')
     else:
@@ -105,14 +101,16 @@ def eveapi_refresh(request, userid=0):
             pass
         else:
             if acc.user == request.user or request.user.is_superuser:
-                import_eve_account(acc.api_key, acc.api_user_id, force_cache=True)
-                acc.user.get_profile().update_access()
+                task = import_apikey.delay(api_key=acc.api_key, api_userid=acc.api_user_id, force_cache=True, user=request.user)
 
                 if request.is_ajax():
-                    acc = EVEAccount.objects.get(id=userid)
-                    return HttpResponse(serializers.serialize('json', [acc]), mimetype='application/javascript')
+                    try:
+                        acc = task.wait(60)
+                        return HttpResponse(serializers.serialize('json', [acc]), mimetype='application/javascript')
+                    except celery.exceptions.TimeoutError:
+                        return HttpResponse(serializers.serialize('json', [None]), mimetype='application/javascript')
                 else:
-                    messages.add_message(request, messages.INFO,"Key %s has been refreshed from the EVE API." % acc.api_user_id)
+                    messages.add_message(request, messages.INFO,"Key %s has been queued to be refreshed from the API" % acc.api_user_id)
 
     return redirect('sso.views.profile')