Various fixes and improvements to the API tasks

eve_api/tasks/account.py

@@ -2,12 +2,15 @@ from datetime import datetime, timedelta
 from xml.dom import minidom
 
 from celery.decorators import task
+from celery.task.sets import TaskSet
 
 from eve_proxy.models import CachedDocument
 
-from eve_api.models import EVEAccount
+from eve_api.models import EVEAccount, EVEPlayerCharacter
 from eve_api.app_defines import *
 from eve_api.utils import basic_xml_parse
+from eve_api.tasks.character import import_eve_character
+from eve_api.tasks.corporation import import_corp_members, import_corp_details
 
 from sso.tasks import update_user_access
 
@@ -55,21 +58,17 @@ def import_apikey_func(api_userid, api_key, user=None, force_cache=False):
     auth_params = {'userid': api_userid, 'apikey': api_key}
     account_doc = CachedDocument.objects.api_query('/account/Characters.xml.aspx', params=auth_params, no_cache=force_cache)
 
-    if account_doc and account_doc.body:
+    doc = basic_xml_parse_doc(account_doc)['eveapi']
-        dom = minidom.parseString(account_doc.body.encode('utf-8'))
-    else:
-        return
 
     # Checks for a document error
-    enode = dom.getElementsByTagName('error')
+    if 'error' in doc:
-    if enode:
         try:
             account = EVEAccount.objects.get(id=api_userid)
         except EVEAccount.DoesNotExist:
             # If no Account exists in the DB, just ignore it
             return
 
-        error = enode[0].getAttribute('code')
+        error = doc['error']['code']
         if int(error) >= 500:
             # API disabled, down or rejecting, return without changes
             return
@@ -88,52 +87,40 @@ def import_apikey_func(api_userid, api_key, user=None, force_cache=False):
     account, created = EVEAccount.objects.get_or_create(id=api_userid, api_user_id=api_userid, api_key=api_key)
     account.api_status = API_STATUS_OK
     if user and created:
-        account.user = user
+        account.user = User.objects.get(id=user)
-    account.save()
-
-    account.characters.clear()
-    for node in dom.getElementsByTagName('rowset')[0].childNodes:
-         try:
-             char = import_eve_character.delay(node.getAttribute('characterID'), api_key, api_userid).get()
-             account.characters.add(char)
-         except AttributeError:
-             continue
-
-    # Check API keytype if we have a character and a unknown key status
-    if account.api_keytype == API_KEYTYPE_UNKNOWN and len(account.characters.all()):
-        auth_params['characterID'] = account.characters.all()[0].id
-        keycheck = CachedDocument.objects.api_query('/char/AccountBalance.xml.aspx', params=auth_params, no_cache=True)
-
-        if keycheck:
-            dom = minidom.parseString(keycheck.body.encode('utf-8'))
-            enode = dom.getElementsByTagName('error')
-
-            if enode and int(enode[0].getAttribute('code')) == 200:
-                account.api_keytype = API_KEYTYPE_LIMITED
-            elif not enode:
-                account.api_keytype = API_KEYTYPE_FULL
-            else:
-                account.api_keytype = API_KEYTYPE_UNKNOWN
-        else:
-            account.api_keytype = API_KEYTYPE_UNKNOWN
 
     account.api_last_updated = datetime.utcnow()
     account.save()
-    log.debug('Completed')
 
-    donecorps = []
+    # Check API keytype if we have a character and a unknown key status
-    if account.api_keytype == API_KEYTYPE_FULL and account.characters.filter(director=1).count():
+    if account.api_keytype == API_KEYTYPE_UNKNOWN:
-        donecorps = []
+        keycheck = CachedDocument.objects.api_query('/account/AccountStatus.xml.aspx', params=auth_params, no_cache=True)
-        for char in account.characters.filter(director=1):
+        doc = basic_xml_parse_doc(keycheck)['eveapi']
-            if not char.corporation.id in donecorps:
-                import_corp_members.delay(api_key=account.api_key, api_userid=account.api_user_id, character_id=char.id)
-                donecorps.append(char.corporation.id)
 
-    for id in set(account.characters.all().values_list('corporation__id', flat=True)):
+        if 'error' in doc and doc['error']['code'] == '200':
-        import_corp_details.delay(corp_id=id)
+            account.api_keytype = API_KEYTYPE_LIMITED
+        elif not 'error' in doc:
+            account.api_keytype = API_KEYTYPE_FULL
+        else:
+            account.api_keytype = API_KEYTYPE_UNKNOWN
 
-    account.save()
+        account.api_last_updated = datetime.utcnow()
-    if account.user:
+        account.save()
-         update_user_access.delay(user=account.user.id)
+
+    # Process the account's character list
+    charlist = account.characters.all().values_list('id', flat=True)
+    for char in doc['result']['characters']:
+        import_eve_character.delay(char['characterID'], api_key, api_userid, callback=link_char_to_account.subtask(account=account.id)).wait()
 
     return account
+
+
+@task(ignore_result=True)
+def link_char_to_account(character, account):
+    acc = EVEAccount.objects.get(id=account)
+    char = EVEPlayerCharacter.objects.get(id=character)
+
+    acc.characters.add(char)
+
+    if acc.user:
+        update_user_access.delay(user=acc.user.id)

eve_api/tasks/alliance.py

@@ -9,6 +9,7 @@ from eve_api.models import EVEAccount, EVEPlayerCorporation, EVEPlayerAlliance
 from eve_api.utils import basic_xml_parse_doc
 from eve_api.tasks.corporation import import_corp_details, import_corp_details_result
 
+from django.core.exceptions import ValidationError
 
 @task(ignore_result=True)
 def import_alliance_details():
@@ -21,13 +22,10 @@ def import_alliance_details():
 
     for alliance in basic_xml_parse_doc(doc)['eveapi']['result']['alliances']:
         allobj, created = EVEPlayerAlliance.objects.get_or_create(pk=alliance['allianceID'])
-        if created:
+        allobj.name = alliance['name']
-            allobj.name = alliance['name']
+        allobj.ticker = alliance['shortName']
-            allobj.ticker = alliance['shortName']
+        allobj.date_founded = datetime.strptime(alliance['startDate'],"%Y-%m-%d %H:%M:%S")
-            allobj.date_founded = alliance['startDate']
         allobj.executor, created = EVEPlayerCorporation.objects.get_or_create(id=alliance['executorCorpID'])
-        if created:
-            import_corp_details.delay(alliance['executorCorpID'])
         allobj.member_count = alliance['memberCount']
         allobj.api_last_updated = datetime.utcnow()
         allobj.save()
@@ -37,12 +35,8 @@ def import_alliance_details():
         validcorps = []
         for corp in alliance['memberCorporations']:
             if int(corp['corporationID']) not in corplist:
-                corpobj, created = EVEPlayerCorporation.objects.get_or_create(pk=corp['corporationID'])
+                import_corp_details.delay(corp['corporationID'])
-                corpobj.alliance = allobj
+                pass
-                corpobj.save()
-
-                if created:
-                    import_corp_details.delay(corp['corporationID'])
             validcorps.append(int(corp['corporationID']))
 
         delcorps = set(corplist) - set(validcorps)

eve_api/tasks/character.py

@@ -32,9 +32,10 @@ def import_eve_character(character_id, api_key=None, user_id=None, callback=None
     pchar.security_status = values['securityStatus']
 
     corp, created = EVEPlayerCorporation.objects.get_or_create(id=values['corporationID'])
-    if created:
+
-        from eve_api.tasks.corporation import import_corp_details
+    from eve_api.tasks.corporation import import_corp_details
-        import_corp_details.delay(values['corporationID'])
+    import_corp_details.delay(values['corporationID'])
+
     pchar.corporation = corp
     pchar.corporation_date = values['corporationDate']
 
@@ -50,10 +51,10 @@ def import_eve_character(character_id, api_key=None, user_id=None, callback=None
                                                        params=auth_params,
                                                        no_cache=False)
 
-        dom = minidom.parseString(char_doc.body.encode('utf-8'))
+        doc = basic_xml_parse_doc(char_doc)['eveapi']
-        if not dom.getElementsByTagName('error'):
+        if not 'error' in doc:
 
-            values = basic_xml_parse(dom.getElementsByTagName('result')[0].childNodes)
+            values = doc['result']
             pchar.balance = values['balance']
             pchar.attrib_intelligence = values['attributes']['intelligence']
             pchar.attrib_charisma = values['attributes']['charisma']
@@ -85,6 +86,9 @@ def import_eve_character(character_id, api_key=None, user_id=None, callback=None
     pchar.api_last_updated = datetime.utcnow()
     pchar.save()
 
+    if pchar.director and api_key and user_id:
+        import_corp_members.delay(api_key=account.api_key, api_userid=account.api_user_id, character_id=char.id)
+
     if callback:
         callback.delay(character=pchar.id)
     else:

eve_api/tasks/corporation.py

@@ -3,7 +3,7 @@ from xml.dom import minidom
 
 from celery.decorators import task
 from eve_proxy.models import CachedDocument
-from eve_api.models import EVEPlayerCorporation, EVEPlayerCharacter
+from eve_api.models import EVEPlayerCorporation, EVEPlayerCharacter, EVEPlayerAlliance
 from eve_api.utils import basic_xml_parse_doc
 from eve_api.tasks.character import import_eve_character
         
@@ -17,8 +17,8 @@ def import_corp_details_result(corp_id):
 
 
 def import_corp_details_func(corp_id):
-    corpobj, created = EVEPlayerCorporation.objects.get_or_create(id=corp_id)
+
-    
+    corpobj, created = EVEPlayerCorporation.objects.get_or_create(id=corp_id)   
     if created or not corpobj.api_last_updated or corpobj.api_last_updated < (datetime.utcnow() - timedelta(hours=12)):
 
         doc = CachedDocument.objects.api_query('/corp/CorporationSheet.xml.aspx', {'corporationID': corp_id})
@@ -48,6 +48,8 @@ def import_corp_details_func(corp_id):
         for logo_map in logo_mappings:
             setattr(corpobj, logo_map[1], d['logo'][logo_map[0]])
 
+        corpobj.alliance, created = EVEPlayerAlliance.objects.get_or_create(id=d['allianceID'])
+
         import_eve_character.delay(d['ceoID'], callback=link_ceo.subtask(corporation=corpobj.id))
 
         corpobj.api_last_updated = datetime.utcnow()
@@ -66,5 +68,29 @@ def link_ceo(corporation, character):
 
 @task(ignore_result=True)
 def import_corp_members(api_userid, api_key, character_id):
-    pull_corp_members(api_key, api_userid, character_id)
+    """
+    This function pulls all corporation members from the EVE API using a director's
+    API key. It'll add as much information as it can about the character.
+    """
+
+    # grab and decode /corp/MemberTracking.xml.aspx
+    auth_params = {'userID': api_userid, 'apiKey': api_key, 'characterID': character_id }
+    char_doc = CachedDocument.objects.api_query('/corp/MemberTracking.xml.aspx',
+                                                   params=auth_params,
+                                                   no_cache=False)
+
+    set = basic_xml_parse_doc(char_doc)['eveapi']['result']['members']
+    corp = EVEPlayerCharacter.objects.get(id=character_id).corporation
+
+    for character in set:
+        import_eve_character.delay(character['characterID'], callback=update_character_tracking.subtask(tracking_details=character))
+
+@task()
+def update_character_tracking(character, tracking_details):
+    charobj = EVEPlayerCharacter.objects.get(id=character)
+
+    charobj.last_login = tracking_details['logonDateTime']
+    charobj.last_logoff = tracking_details['logoffDateTime']
+    charobj.current_location_id = int(tracking_details['locationID'])
+    charobj.save()
 

settings.py

@@ -181,6 +181,7 @@ CELERYBEAT_SCHEDULE = {
 CELERY_SEND_TASK_ERROR_EMAILS = True
 CELERY_RESULT_BACKEND = "amqp"
 CELERY_DISABLE_RATE_LIMITS = True
+CELERYD_PREFETCH_MULTIPLIER = 128
 
 # Load the Celery tasks
 djcelery.setup_loader()