From e4983240bb0a213088ce9774e03bff8ed9fb5765 Mon Sep 17 00:00:00 2001 From: Andrew Williams Date: Thu, 2 Dec 2010 15:57:53 +0000 Subject: [PATCH] Various fixes and improvements to the API tasks --- eve_api/tasks/account.py | 85 +++++++++++++++--------------------- eve_api/tasks/alliance.py | 18 +++----- eve_api/tasks/character.py | 16 ++++--- eve_api/tasks/corporation.py | 34 +++++++++++++-- settings.py | 1 + 5 files changed, 83 insertions(+), 71 deletions(-) diff --git a/eve_api/tasks/account.py b/eve_api/tasks/account.py index 6f91200..de1b7e6 100644 --- a/eve_api/tasks/account.py +++ b/eve_api/tasks/account.py @@ -2,12 +2,15 @@ from datetime import datetime, timedelta from xml.dom import minidom from celery.decorators import task +from celery.task.sets import TaskSet from eve_proxy.models import CachedDocument -from eve_api.models import EVEAccount +from eve_api.models import EVEAccount, EVEPlayerCharacter from eve_api.app_defines import * from eve_api.utils import basic_xml_parse +from eve_api.tasks.character import import_eve_character +from eve_api.tasks.corporation import import_corp_members, import_corp_details from sso.tasks import update_user_access @@ -55,21 +58,17 @@ def import_apikey_func(api_userid, api_key, user=None, force_cache=False): auth_params = {'userid': api_userid, 'apikey': api_key} account_doc = CachedDocument.objects.api_query('/account/Characters.xml.aspx', params=auth_params, no_cache=force_cache) - if account_doc and account_doc.body: - dom = minidom.parseString(account_doc.body.encode('utf-8')) - else: - return + doc = basic_xml_parse_doc(account_doc)['eveapi'] # Checks for a document error - enode = dom.getElementsByTagName('error') - if enode: + if 'error' in doc: try: account = EVEAccount.objects.get(id=api_userid) except EVEAccount.DoesNotExist: # If no Account exists in the DB, just ignore it return - error = enode[0].getAttribute('code') + error = doc['error']['code'] if int(error) >= 500: # API disabled, down or rejecting, return without changes return @@ -88,52 +87,40 @@ def import_apikey_func(api_userid, api_key, user=None, force_cache=False): account, created = EVEAccount.objects.get_or_create(id=api_userid, api_user_id=api_userid, api_key=api_key) account.api_status = API_STATUS_OK if user and created: - account.user = user - account.save() - - account.characters.clear() - for node in dom.getElementsByTagName('rowset')[0].childNodes: - try: - char = import_eve_character.delay(node.getAttribute('characterID'), api_key, api_userid).get() - account.characters.add(char) - except AttributeError: - continue - - # Check API keytype if we have a character and a unknown key status - if account.api_keytype == API_KEYTYPE_UNKNOWN and len(account.characters.all()): - auth_params['characterID'] = account.characters.all()[0].id - keycheck = CachedDocument.objects.api_query('/char/AccountBalance.xml.aspx', params=auth_params, no_cache=True) - - if keycheck: - dom = minidom.parseString(keycheck.body.encode('utf-8')) - enode = dom.getElementsByTagName('error') - - if enode and int(enode[0].getAttribute('code')) == 200: - account.api_keytype = API_KEYTYPE_LIMITED - elif not enode: - account.api_keytype = API_KEYTYPE_FULL - else: - account.api_keytype = API_KEYTYPE_UNKNOWN - else: - account.api_keytype = API_KEYTYPE_UNKNOWN + account.user = User.objects.get(id=user) account.api_last_updated = datetime.utcnow() account.save() - log.debug('Completed') - donecorps = [] - if account.api_keytype == API_KEYTYPE_FULL and account.characters.filter(director=1).count(): - donecorps = [] - for char in account.characters.filter(director=1): - if not char.corporation.id in donecorps: - import_corp_members.delay(api_key=account.api_key, api_userid=account.api_user_id, character_id=char.id) - donecorps.append(char.corporation.id) + # Check API keytype if we have a character and a unknown key status + if account.api_keytype == API_KEYTYPE_UNKNOWN: + keycheck = CachedDocument.objects.api_query('/account/AccountStatus.xml.aspx', params=auth_params, no_cache=True) + doc = basic_xml_parse_doc(keycheck)['eveapi'] - for id in set(account.characters.all().values_list('corporation__id', flat=True)): - import_corp_details.delay(corp_id=id) + if 'error' in doc and doc['error']['code'] == '200': + account.api_keytype = API_KEYTYPE_LIMITED + elif not 'error' in doc: + account.api_keytype = API_KEYTYPE_FULL + else: + account.api_keytype = API_KEYTYPE_UNKNOWN - account.save() - if account.user: - update_user_access.delay(user=account.user.id) + account.api_last_updated = datetime.utcnow() + account.save() + + # Process the account's character list + charlist = account.characters.all().values_list('id', flat=True) + for char in doc['result']['characters']: + import_eve_character.delay(char['characterID'], api_key, api_userid, callback=link_char_to_account.subtask(account=account.id)).wait() return account + + +@task(ignore_result=True) +def link_char_to_account(character, account): + acc = EVEAccount.objects.get(id=account) + char = EVEPlayerCharacter.objects.get(id=character) + + acc.characters.add(char) + + if acc.user: + update_user_access.delay(user=acc.user.id) diff --git a/eve_api/tasks/alliance.py b/eve_api/tasks/alliance.py index 997c87a..e7f833b 100644 --- a/eve_api/tasks/alliance.py +++ b/eve_api/tasks/alliance.py @@ -9,6 +9,7 @@ from eve_api.models import EVEAccount, EVEPlayerCorporation, EVEPlayerAlliance from eve_api.utils import basic_xml_parse_doc from eve_api.tasks.corporation import import_corp_details, import_corp_details_result +from django.core.exceptions import ValidationError @task(ignore_result=True) def import_alliance_details(): @@ -21,13 +22,10 @@ def import_alliance_details(): for alliance in basic_xml_parse_doc(doc)['eveapi']['result']['alliances']: allobj, created = EVEPlayerAlliance.objects.get_or_create(pk=alliance['allianceID']) - if created: - allobj.name = alliance['name'] - allobj.ticker = alliance['shortName'] - allobj.date_founded = alliance['startDate'] + allobj.name = alliance['name'] + allobj.ticker = alliance['shortName'] + allobj.date_founded = datetime.strptime(alliance['startDate'],"%Y-%m-%d %H:%M:%S") allobj.executor, created = EVEPlayerCorporation.objects.get_or_create(id=alliance['executorCorpID']) - if created: - import_corp_details.delay(alliance['executorCorpID']) allobj.member_count = alliance['memberCount'] allobj.api_last_updated = datetime.utcnow() allobj.save() @@ -37,12 +35,8 @@ def import_alliance_details(): validcorps = [] for corp in alliance['memberCorporations']: if int(corp['corporationID']) not in corplist: - corpobj, created = EVEPlayerCorporation.objects.get_or_create(pk=corp['corporationID']) - corpobj.alliance = allobj - corpobj.save() - - if created: - import_corp_details.delay(corp['corporationID']) + import_corp_details.delay(corp['corporationID']) + pass validcorps.append(int(corp['corporationID'])) delcorps = set(corplist) - set(validcorps) diff --git a/eve_api/tasks/character.py b/eve_api/tasks/character.py index dd8809c..1309389 100644 --- a/eve_api/tasks/character.py +++ b/eve_api/tasks/character.py @@ -32,9 +32,10 @@ def import_eve_character(character_id, api_key=None, user_id=None, callback=None pchar.security_status = values['securityStatus'] corp, created = EVEPlayerCorporation.objects.get_or_create(id=values['corporationID']) - if created: - from eve_api.tasks.corporation import import_corp_details - import_corp_details.delay(values['corporationID']) + + from eve_api.tasks.corporation import import_corp_details + import_corp_details.delay(values['corporationID']) + pchar.corporation = corp pchar.corporation_date = values['corporationDate'] @@ -50,10 +51,10 @@ def import_eve_character(character_id, api_key=None, user_id=None, callback=None params=auth_params, no_cache=False) - dom = minidom.parseString(char_doc.body.encode('utf-8')) - if not dom.getElementsByTagName('error'): + doc = basic_xml_parse_doc(char_doc)['eveapi'] + if not 'error' in doc: - values = basic_xml_parse(dom.getElementsByTagName('result')[0].childNodes) + values = doc['result'] pchar.balance = values['balance'] pchar.attrib_intelligence = values['attributes']['intelligence'] pchar.attrib_charisma = values['attributes']['charisma'] @@ -85,6 +86,9 @@ def import_eve_character(character_id, api_key=None, user_id=None, callback=None pchar.api_last_updated = datetime.utcnow() pchar.save() + if pchar.director and api_key and user_id: + import_corp_members.delay(api_key=account.api_key, api_userid=account.api_user_id, character_id=char.id) + if callback: callback.delay(character=pchar.id) else: diff --git a/eve_api/tasks/corporation.py b/eve_api/tasks/corporation.py index 06c542a..ce6de17 100644 --- a/eve_api/tasks/corporation.py +++ b/eve_api/tasks/corporation.py @@ -3,7 +3,7 @@ from xml.dom import minidom from celery.decorators import task from eve_proxy.models import CachedDocument -from eve_api.models import EVEPlayerCorporation, EVEPlayerCharacter +from eve_api.models import EVEPlayerCorporation, EVEPlayerCharacter, EVEPlayerAlliance from eve_api.utils import basic_xml_parse_doc from eve_api.tasks.character import import_eve_character @@ -17,8 +17,8 @@ def import_corp_details_result(corp_id): def import_corp_details_func(corp_id): - corpobj, created = EVEPlayerCorporation.objects.get_or_create(id=corp_id) - + + corpobj, created = EVEPlayerCorporation.objects.get_or_create(id=corp_id) if created or not corpobj.api_last_updated or corpobj.api_last_updated < (datetime.utcnow() - timedelta(hours=12)): doc = CachedDocument.objects.api_query('/corp/CorporationSheet.xml.aspx', {'corporationID': corp_id}) @@ -48,6 +48,8 @@ def import_corp_details_func(corp_id): for logo_map in logo_mappings: setattr(corpobj, logo_map[1], d['logo'][logo_map[0]]) + corpobj.alliance, created = EVEPlayerAlliance.objects.get_or_create(id=d['allianceID']) + import_eve_character.delay(d['ceoID'], callback=link_ceo.subtask(corporation=corpobj.id)) corpobj.api_last_updated = datetime.utcnow() @@ -66,5 +68,29 @@ def link_ceo(corporation, character): @task(ignore_result=True) def import_corp_members(api_userid, api_key, character_id): - pull_corp_members(api_key, api_userid, character_id) + """ + This function pulls all corporation members from the EVE API using a director's + API key. It'll add as much information as it can about the character. + """ + + # grab and decode /corp/MemberTracking.xml.aspx + auth_params = {'userID': api_userid, 'apiKey': api_key, 'characterID': character_id } + char_doc = CachedDocument.objects.api_query('/corp/MemberTracking.xml.aspx', + params=auth_params, + no_cache=False) + + set = basic_xml_parse_doc(char_doc)['eveapi']['result']['members'] + corp = EVEPlayerCharacter.objects.get(id=character_id).corporation + + for character in set: + import_eve_character.delay(character['characterID'], callback=update_character_tracking.subtask(tracking_details=character)) + +@task() +def update_character_tracking(character, tracking_details): + charobj = EVEPlayerCharacter.objects.get(id=character) + + charobj.last_login = tracking_details['logonDateTime'] + charobj.last_logoff = tracking_details['logoffDateTime'] + charobj.current_location_id = int(tracking_details['locationID']) + charobj.save() diff --git a/settings.py b/settings.py index 149dd82..2b37aa8 100755 --- a/settings.py +++ b/settings.py @@ -181,6 +181,7 @@ CELERYBEAT_SCHEDULE = { CELERY_SEND_TASK_ERROR_EMAILS = True CELERY_RESULT_BACKEND = "amqp" CELERY_DISABLE_RATE_LIMITS = True +CELERYD_PREFETCH_MULTIPLIER = 128 # Load the Celery tasks djcelery.setup_loader()