Rework of the API import routines, now more task based

This is step one of cleaning up the eve_api application and replacing the working core with a more stable, celery based import queue. In addition we've added the ability to import a character just based on character ID, this should allow for populated corporate CEO positions on corps.

eve_api/api_puller/accounts.py

@@ -1,193 +0,0 @@
-#!/usr/bin/env python
-"""
-This module abstracts the pulling of account data from the EVE API.
-"""
-from xml.dom import minidom
-from datetime import datetime
-
-if __name__ == "__main__":
-    # Only mess with the environmental stuff if this is being ran directly.
-    from importer_path import fix_environment
-    fix_environment() 
-
-from datetime import datetime
-
-from django.conf import settings
-from eve_proxy.models import CachedDocument
-from eve_api.app_defines import *
-from eve_api.api_exceptions import APIAuthException, APINoUserIDException
-from eve_api.models import EVEAccount, EVEPlayerCharacter, EVEPlayerCharacterRole, EVEPlayerCorporation
-
-def import_eve_account(api_key, user_id, force_cache=False):
-    """
-    Imports an account from the API into the EVEAccount model.
-    """
-
-    auth_params = {'userid': user_id, 'apikey': api_key}
-    account_doc = CachedDocument.objects.api_query('/account/Characters.xml.aspx', params=auth_params, no_cache=force_cache)
-
-    if account_doc and account_doc.body:
-        dom = minidom.parseString(account_doc.body.encode('utf-8'))
-    else:
-        return
-
-    enode = dom.getElementsByTagName('error')
-    if enode:
-        try:
-            account = EVEAccount.objects.get(id=user_id)
-        except EVEAccount.DoesNotExist:
-            return
-
-        error = enode[0].getAttribute('code')
-
-        if int(error) >= 500:
-            # API disabled, down or rejecting, return without changes
-            return
-        elif error in ['202', '203', '204', '205', '212']:
-            account.api_status = API_STATUS_AUTH_ERROR
-        elif error == '211':
-            account.api_status = API_STATUS_ACC_EXPIRED
-        else:
-            account.api_status = API_STATUS_OTHER_ERROR
-        account.api_last_updated = datetime.utcnow()
-        account.save()
-        return
-
-    characters_node_children = dom.getElementsByTagName('rowset')[0].childNodes
-
-    # Create or retrieve the account last to make sure everything
-    # before here is good to go.
-    account, created = EVEAccount.objects.get_or_create(id=user_id, api_user_id=user_id, api_key=api_key)
-    account.api_status = API_STATUS_OK
-    account.save()
-
-    account.characters.clear()
-    for node in characters_node_children:
-        try:
-            char = import_eve_character(api_key, user_id, node.getAttribute('characterID'))
-            if char:
-                account.characters.add(char)
-        except AttributeError:
-            # This must be a Text node, ignore it.
-            continue
-
-    # Check API keytype if we have a character and a unknown key status
-    if account.api_keytype == API_KEYTYPE_UNKNOWN and len(account.characters.all()):
-        auth_params['characterID'] = account.characters.all()[0].id
-        keycheck = CachedDocument.objects.api_query('/char/AccountBalance.xml.aspx', params=auth_params, no_cache=True)
-
-        if keycheck:
-            dom = minidom.parseString(keycheck.body.encode('utf-8'))
-            enode = dom.getElementsByTagName('error')
-
-            if enode and int(enode[0].getAttribute('code')) == 200:
-                account.api_keytype = API_KEYTYPE_LIMITED
-            elif not enode:
-                account.api_keytype = API_KEYTYPE_FULL
-            else:
-                account.api_keytype = API_KEYTYPE_UNKNOWN
-        else:
-            account.api_keytype = API_KEYTYPE_UNKNOWN
-
-    account.api_last_updated = datetime.utcnow()
-    account.save()
-    return account
-    
-def import_eve_character(api_key, user_id, character_id):
-
-    auth_params = {'userID': user_id, 'apiKey': api_key, 'characterID': character_id }
-    char_doc = CachedDocument.objects.api_query('/char/CharacterSheet.xml.aspx',
-                                                   params=auth_params,
-                                                   no_cache=False)
-
-    dom = minidom.parseString(char_doc.body.encode('utf-8'))
-    if dom.getElementsByTagName('error'):
-        return
-
-    nodes = dom.getElementsByTagName('result')[0].childNodes
-    pchar, created = EVEPlayerCharacter.objects.get_or_create(id=character_id)
-
-    values = {}
-    for node in nodes:
-        if node.nodeType == 1:
-            node.normalize()
-            if len(node.childNodes) == 1:
-                values[node.tagName] = node.childNodes[0].nodeValue
-            else:
-                nv = {}
-                if node.tagName == "rowset":
-                    rset = []
-                    for nd in node.childNodes:
-                        if nd.nodeType == 1:
-                            d = {}
-                            for e in nd.attributes.keys():
-                                d[e] = nd.attributes[e].value
-                            rset.append(d)
-                    values[node.attributes['name'].value] = rset
-                else:
-                    for nd in node.childNodes:
-                        if nd.nodeType == 1:
-                            nv[nd.tagName] = nd.childNodes[0].nodeValue
-                    values[node.tagName] = nv
-
-    # Get this first, as it's safe.
-    corporation_id = values['corporationID']
-    corp, created = EVEPlayerCorporation.objects.get_or_create(id=corporation_id)
-    if not corp.name:
-        try:
-            corp.query_and_update_corp()
-        except:
-            pass
-
-    name = values['name']
-    # Save these for last to keep the save count low.
-    pchar.name = name
-    pchar.corporation = corp
-
-    pchar.balance = values['balance']
-    pchar.attrib_intelligence = values['attributes']['intelligence']
-    pchar.attrib_charisma = values['attributes']['charisma']
-    pchar.attrib_perception = values['attributes']['perception']
-    pchar.attrib_willpower = values['attributes']['willpower']
-    pchar.attrib_memory = values['attributes']['memory']
-
-    # Process the character's roles
-    pchar.director = False
-    pchar.roles.clear()
-    roles = values.get('corporationRoles', None)
-    if roles and len(roles):
-        for r in roles:
-            role, created = EVEPlayerCharacterRole.objects.get_or_create(roleid=r['roleID'], name=r['roleName'])
-            pchar.roles.add(role)
-            if r['roleName'] == 'roleDirector':
-                pchar.director = True
-
-    if values['gender'] == 'Male':
-        pchar.gender = 1
-    else:
-        pchar.gender = 2
-
-    for v in API_RACES_CHOICES:
-        val, race = v
-        if race == values['race']:
-            pchar.race = val
-            break
-
-    total = 0
-    for skill in values['skills']:
-        total = total + int(skill['skillpoints'])
-    pchar.total_sp = total
-
-    pchar.api_last_updated = datetime.utcnow()
-    pchar.save()
-
-    return pchar
-
-if __name__ == "__main__":
-    """
-    Test import.
-    """
-    api_key = settings.EVE_API_USER_KEY 
-    #api_key += "1"
-    user_id = settings.EVE_API_USER_ID
-    import_eve_account(api_key, user_id)

eve_api/api_puller/importer_path.py

@@ -1,16 +0,0 @@
-import os
-import sys
-# The path to the folder containing settings.py.
-BASE_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
-APPS_PATH = os.path.join(BASE_PATH, 'apps')
-
-def fix_environment():
-    """
-    Callable function to set up all of the Django environmental variables and
-    pathing for directly executable python modules.
-    """
-    from importer_path import BASE_PATH
-    # Prepare the environment
-    sys.path.insert(0, APPS_PATH)
-    sys.path.insert(0, BASE_PATH)
-    os.environ['DJANGO_SETTINGS_MODULE'] = 'settings' 

eve_api/app_defines.py

@@ -26,9 +26,12 @@ API_KEYTYPE_CHOICES = (
     (API_KEYTYPE_FULL, 'Full'),
 )
 
+API_GENDER_MALE = 1
+API_GENDER_FEMALE = 2
+
 API_GENDER_CHOICES = (
-    (1, 'Male'),
+    (API_GENDER_MALE, 'Male'),
-    (2, 'Female'),
+    (API_GENDER_FEMALE, 'Female'),
 )
 
 API_RACES_CHOICES = (

eve_api/migrations/0006_auto__add_field_eveplayercharacter_corporation_date__add_field_eveplay.py

@@ -0,0 +1,142 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Adding field 'EVEPlayerCharacter.corporation_date'
+        db.add_column('eve_api_eveplayercharacter', 'corporation_date', self.gf('django.db.models.fields.DateTimeField')(null=True, blank=True), keep_default=False)
+
+        # Adding field 'EVEPlayerCharacter.security_status'
+        db.add_column('eve_api_eveplayercharacter', 'security_status', self.gf('django.db.models.fields.FloatField')(null=True, blank=True), keep_default=False)
+
+
+    def backwards(self, orm):
+        
+        # Deleting field 'EVEPlayerCharacter.corporation_date'
+        db.delete_column('eve_api_eveplayercharacter', 'corporation_date')
+
+        # Deleting field 'EVEPlayerCharacter.security_status'
+        db.delete_column('eve_api_eveplayercharacter', 'security_status')
+
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'eve_api.eveaccount': {
+            'Meta': {'object_name': 'EVEAccount'},
+            'api_key': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'api_keytype': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'api_last_updated': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'api_status': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'api_user_id': ('django.db.models.fields.IntegerField', [], {}),
+            'characters': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': "orm['eve_api.EVEPlayerCharacter']", 'null': 'True', 'blank': 'True'}),
+            'description': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'blank': 'True'})
+        },
+        'eve_api.eveplayeralliance': {
+            'Meta': {'object_name': 'EVEPlayerAlliance'},
+            'api_last_updated': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'date_founded': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
+            'group': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.Group']", 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'member_count': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+            'ticker': ('django.db.models.fields.CharField', [], {'max_length': '15', 'blank': 'True'})
+        },
+        'eve_api.eveplayercharacter': {
+            'Meta': {'object_name': 'EVEPlayerCharacter'},
+            'api_last_updated': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'attrib_charisma': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'attrib_intelligence': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'attrib_memory': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'attrib_perception': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'attrib_willpower': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'balance': ('django.db.models.fields.FloatField', [], {'null': 'True', 'blank': 'True'}),
+            'corporation': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['eve_api.EVEPlayerCorporation']", 'null': 'True', 'blank': 'True'}),
+            'corporation_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'current_location_id': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'director': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}),
+            'gender': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'last_logoff': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+            'race': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'roles': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': "orm['eve_api.EVEPlayerCharacterRole']", 'null': 'True', 'blank': 'True'}),
+            'security_status': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '2', 'decimal_places': '2', 'blank': 'True'}),
+            'total_sp': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'})
+        },
+        'eve_api.eveplayercharacterrole': {
+            'Meta': {'object_name': 'EVEPlayerCharacterRole'},
+            'api_last_updated': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'roleid': ('django.db.models.fields.CharField', [], {'max_length': '64'})
+        },
+        'eve_api.eveplayercorporation': {
+            'Meta': {'object_name': 'EVEPlayerCorporation'},
+            'alliance': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['eve_api.EVEPlayerAlliance']", 'null': 'True', 'blank': 'True'}),
+            'alliance_join_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
+            'api_last_updated': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'applications': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}),
+            'ceo_character': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['eve_api.EVEPlayerCharacter']", 'null': 'True', 'blank': 'True'}),
+            'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'group': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.Group']", 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'logo_color1': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'logo_color2': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'logo_color3': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'logo_graphic_id': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'logo_shape1': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'logo_shape2': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'logo_shape3': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'member_count': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'shares': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'tax_rate': ('django.db.models.fields.FloatField', [], {'null': 'True', 'blank': 'True'}),
+            'ticker': ('django.db.models.fields.CharField', [], {'max_length': '15', 'null': 'True', 'blank': 'True'}),
+            'url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['eve_api']

eve_api/models/api_player.py

@@ -69,6 +69,7 @@ class EVEPlayerCharacter(EVEAPIModel):
     """
     name = models.CharField(max_length=255, blank=True, null=False)
     corporation = models.ForeignKey('EVEPlayerCorporation', blank=True, null=True)
+    corporation_date = models.DateTimeField(blank=True, null=True, verbose_name="Corporation Join Date")
     race = models.IntegerField(blank=True, null=True, choices=API_RACES_CHOICES)
     gender = models.IntegerField(blank=True, null=True, choices=API_GENDER_CHOICES)
     balance = models.FloatField("Account Balance", blank=True, null=True)
@@ -81,6 +82,7 @@ class EVEPlayerCharacter(EVEAPIModel):
     attrib_willpower = models.IntegerField("Willpower", blank=True, null=True)
     total_sp = models.IntegerField("Total SP", blank=True, null=True)
 
+    security_status = models.FloatField("Security Status", blank=True, null=True)
     current_location_id = models.IntegerField("Current Location ID", blank=True, null=True)
     last_login = models.DateTimeField(blank=True, null=True,
                                             verbose_name="Last Login Date/Time",
@@ -210,6 +212,9 @@ class EVEPlayerCorporation(EVEAPIModel):
         Takes an EVEPlayerCorporation object and updates it from the 
         EVE API service.
         """
+
+        from eve_api.api_puller.accounts import import_eve_character
+
         # Pull XML from the EVE API via eve_proxy.
         dom = EVEPlayerCorporation.objects.api_corp_sheet_xml(self.id)
         
@@ -245,7 +250,7 @@ class EVEPlayerCorporation(EVEAPIModel):
                 continue
 
         ceoid = dom.getElementsByTagName('ceoID')[0].firstChild.nodeValue
-        self.ceo_character, created = EVEPlayerCharacter.objects.get_or_create(id=ceoid)
+        self.ceo_character = import_eve_character(ceoid)
 
         self.api_last_updated = datetime.utcnow()
         self.save()

eve_api/tasks.py

@@ -1,9 +1,12 @@
-import datetime
+from datetime import datetime, timedelta
+from xml.dom import minidom
+
 from celery.decorators import task
-from eve_api.models import EVEAccount, EVEPlayerCorporation
+from eve_api.models import EVEAccount, EVEPlayerCorporation, EVEPlayerCharacter, EVEPlayerCharacterRole
-from eve_api.api_puller.accounts import import_eve_account
 from eve_api.api_puller.corp_management import pull_corp_members
 from eve_api.app_defines import *
+from eve_api.utils import basic_xml_parse
+from eve_proxy.models import CachedDocument
 from sso.tasks import update_user_access
 from django.contrib.auth.models import User
 
@@ -15,10 +18,10 @@ def queue_apikey_updates(update_delay=86400, batch_size=50):
 
     log = queue_apikey_updates.get_logger()
     # Update all the eve accounts and related corps
-    delta = datetime.timedelta(seconds=update_delay)
+    delta = timedelta(seconds=update_delay)
-    log.info("Updating APIs older than %s" % (datetime.datetime.now() - delta))
+    log.info("Updating APIs older than %s" % (datetime.now() - delta))
 
-    accounts = EVEAccount.objects.filter(api_last_updated__lt=(datetime.datetime.now() - delta)).exclude(api_status=API_STATUS_ACC_EXPIRED).exclude(api_status=API_STATUS_AUTH_ERROR).order_by('api_last_updated')[:batch_size]
+    accounts = EVEAccount.objects.filter(api_last_updated__lt=(datetime.now() - delta)).exclude(api_status=API_STATUS_ACC_EXPIRED).exclude(api_status=API_STATUS_AUTH_ERROR).order_by('api_last_updated')[:batch_size]
     log.info("%s account(s) to update" % accounts.count())
     for acc in accounts:
         log.debug("Queueing UserID %s for update" % acc.api_user_id)
@@ -39,33 +42,164 @@ def import_apikey_result(api_userid, api_key, user=None, force_cache=False):
 def import_apikey_func(api_userid, api_key, user=None, force_cache=False):
     log = import_apikey.get_logger('import_apikey_result')
     log.info('Importing %s/%s' % (api_userid, api_key))
-    acc = import_eve_account(api_key, api_userid, force_cache=force_cache)
+
+    auth_params = {'userid': api_userid, 'apikey': api_key}
+    account_doc = CachedDocument.objects.api_query('/account/Characters.xml.aspx', params=auth_params, no_cache=force_cache)
+
+    if account_doc and account_doc.body:
+        dom = minidom.parseString(account_doc.body.encode('utf-8'))
+    else:
+        return
+
+    # Checks for a document error
+    enode = dom.getElementsByTagName('error')
+    if enode:
+        try:
+            account = EVEAccount.objects.get(id=api_userid)
+        except EVEAccount.DoesNotExist:
+            # If no Account exists in the DB, just ignore it
+            return
+
+        error = enode[0].getAttribute('code')
+        if int(error) >= 500:
+            # API disabled, down or rejecting, return without changes
+            return
+        elif error in ['202', '203', '204', '205', '212']:
+            account.api_status = API_STATUS_AUTH_ERROR
+        elif error == '211':
+            account.api_status = API_STATUS_ACC_EXPIRED
+        else:
+            account.api_status = API_STATUS_OTHER_ERROR
+        account.api_last_updated = datetime.utcnow()
+        account.save()
+        return account
+
+    # Create or retrieve the account last to make sure everything
+    # before here is good to go.
+    account, created = EVEAccount.objects.get_or_create(id=api_userid, api_user_id=api_userid, api_key=api_key)
+    account.api_status = API_STATUS_OK
+    if user and created:
+        account.user = user
+    account.save()
+
+    account.characters.clear()
+    for node in dom.getElementsByTagName('rowset')[0].childNodes:
+         try:
+             char = import_eve_character.delay(node.getAttribute('characterID'), api_key, api_userid).get()
+             account.characters.add(char)
+         except AttributeError:
+             continue
+
+    # Check API keytype if we have a character and a unknown key status
+    if account.api_keytype == API_KEYTYPE_UNKNOWN and len(account.characters.all()):
+        auth_params['characterID'] = account.characters.all()[0].id
+        keycheck = CachedDocument.objects.api_query('/char/AccountBalance.xml.aspx', params=auth_params, no_cache=True)
+
+        if keycheck:
+            dom = minidom.parseString(keycheck.body.encode('utf-8'))
+            enode = dom.getElementsByTagName('error')
+
+            if enode and int(enode[0].getAttribute('code')) == 200:
+                account.api_keytype = API_KEYTYPE_LIMITED
+            elif not enode:
+                account.api_keytype = API_KEYTYPE_FULL
+            else:
+                account.api_keytype = API_KEYTYPE_UNKNOWN
+        else:
+            account.api_keytype = API_KEYTYPE_UNKNOWN
+
+    account.api_last_updated = datetime.utcnow()
+    account.save()
     log.debug('Completed')
+
     donecorps = []
-    if acc and acc.api_status == API_STATUS_OK:
+    if account.api_keytype == API_KEYTYPE_FULL and account.characters.filter(director=1).count():
-        if user and not acc.user:
+        donecorps = []
-            acc.user = User.objects.get(id=user)
+        for char in account.characters.filter(director=1):
-        if acc.api_keytype == API_KEYTYPE_FULL and acc.characters.filter(director=1).count():
+            if not char.corporation.id in donecorps:
-            donecorps = []
+                import_corp_members.delay(api_key=account.api_key, api_userid=account.api_user_id, character_id=char.id)
-            for char in acc.characters.filter(director=1):
-                if not char.corporation.id in donecorps:
-                    import_corp_members.delay(api_key=acc.api_key, api_userid=acc.api_user_id, character_id=char.id)
-
-                    if char.corporation.api_last_updated < (datetime.datetime.now() - datetime.timedelta(hours=12)):
-                        import_corp_details.delay(corp_id=char.corporation.id)
-                    donecorps.append(char.corporation.id)
-
-        for char in acc.characters.all():
-            if char.corporation.id not in donecorps:
-                if char.corporation.api_last_updated < (datetime.datetime.now() - datetime.timedelta(hours=12)):
-                    import_corp_details.delay(corp_id=char.corporation.id)
                 donecorps.append(char.corporation.id)
 
-        acc.save()
+    for id in set(account.characters.all().values_list('corporation__id', flat=True)):
-        if acc.user:
+        import_corp_details.delay(corp_id=id)
-             update_user_access.delay(user=acc.user.id)
 
-        return acc
+    account.save()
+    if account.user:
+         update_user_access.delay(user=account.user.id)
+
+    return account
+
+
+@task()
+def import_eve_character(character_id, api_key=None, user_id=None):
+
+    char_doc = CachedDocument.objects.api_query('/eve/CharacterInfo.xml.aspx',
+                                               params={'characterID': character_id},
+                                               no_cache=False)
+
+    dom = minidom.parseString(char_doc.body.encode('utf-8'))
+    if dom.getElementsByTagName('error'):
+        return
+    values = basic_xml_parse(dom.getElementsByTagName('result')[0].childNodes)
+    pchar, created = EVEPlayerCharacter.objects.get_or_create(id=character_id)
+
+    pchar.name = values['characterName']
+    pchar.security_status = values['securityStatus']
+
+    corp, created = EVEPlayerCorporation.objects.get_or_create(id=values['corporationID'])
+    if created:
+        import_corp_details.delay(values['corporationID'])
+    pchar.corporation = corp
+    pchar.corporation_date = values['corporationDate']
+
+    for v in API_RACES_CHOICES:
+        val, race = v
+        if race == values['race']:
+            pchar.race = val
+            break
+
+    if api_key and user_id:
+        auth_params = {'userID': user_id, 'apiKey': api_key, 'characterID': character_id }
+        char_doc = CachedDocument.objects.api_query('/char/CharacterSheet.xml.aspx',
+                                                       params=auth_params,
+                                                       no_cache=False)
+
+        dom = minidom.parseString(char_doc.body.encode('utf-8'))
+        if not dom.getElementsByTagName('error'):
+
+            values = basic_xml_parse(dom.getElementsByTagName('result')[0].childNodes)
+            pchar.balance = values['balance']
+            pchar.attrib_intelligence = values['attributes']['intelligence']
+            pchar.attrib_charisma = values['attributes']['charisma']
+            pchar.attrib_perception = values['attributes']['perception']
+            pchar.attrib_willpower = values['attributes']['willpower']
+            pchar.attrib_memory = values['attributes']['memory']
+
+            # Process the character's roles
+            pchar.director = False
+            pchar.roles.clear()
+            roles = values.get('corporationRoles', None)
+            if roles and len(roles):
+                for r in roles:
+                    role, created = EVEPlayerCharacterRole.objects.get_or_create(roleid=r['roleID'], name=r['roleName'])
+                    pchar.roles.add(role)
+                    if r['roleName'] == 'roleDirector':
+                        pchar.director = True
+
+            if values['gender'] == 'Male':
+                pchar.gender = API_GENDER_MALE
+            else:
+                pchar.gender = API_GENDER_FEMALE
+
+            total = 0
+            for skill in values['skills']:
+                total = total + int(skill['skillpoints'])
+            pchar.total_sp = total
+
+    pchar.api_last_updated = datetime.utcnow()
+    pchar.save()
+
+    return pchar
 
 
 @task(ignore_result=True)
@@ -82,5 +216,7 @@ def import_corp_members(api_userid, api_key, character_id):
 @task(ignore_result=True)
 def import_corp_details(corp_id):
     corp, created = EVEPlayerCorporation.objects.get_or_create(id=corp_id)
-    corp.query_and_update_corp()
+    
-    corp.save()
+    if created or not corp.api_last_updated or corp.api_last_updated < (datetime.utcnow() - timedelta(hours=12)):
+        corp.query_and_update_corp()
+        corp.save()

eve_api/utils.py

@@ -0,0 +1,26 @@
+def basic_xml_parse(nodes):
+    """ Parses a minidom set of nodes into a tree dict """
+    values = {}
+    for node in nodes:
+        if node.nodeType == 1:
+            node.normalize()
+            if len(node.childNodes) == 1:
+                values[node.tagName] = node.childNodes[0].nodeValue
+            else:
+                nv = {}
+                if node.tagName == "rowset":
+                    rset = []
+                    for nd in node.childNodes:
+                        if nd.nodeType == 1:
+                            d = {}
+                            for e in nd.attributes.keys():
+                                d[e] = nd.attributes[e].value
+                            rset.append(d)
+                    values[node.attributes['name'].value] = rset
+                else:
+                    for nd in node.childNodes:
+                        if nd.nodeType == 1:
+                            nv[nd.tagName] = nd.childNodes[0].nodeValue
+                    values[node.tagName] = nv
+
+    return values

templates/sso/character.html

@@ -15,8 +15,10 @@
 <li>Race: {{ character.get_race_display }}</li>
 <li>Gender: {{ character.get_gender_display }}</li>
 <li>Corporation: {{ character.corporation.name }}</li>
+<li>Joined Corporation: {{ character.corporation_date }}</li>
 <li>Alliance: {{ character.corporation.alliance.name }}</li>
 <li>Total SP: {{ character.total_sp|intcomma }}</li>
+<li>Security Status: {{ character.security_status }}</li>
 <li>Director: {{ character.director }}</li>
 </ul>