Rework eve_proxy to remove dependancy on eve_api and allow it to handle downtime situations better

api/handlers.py

@@ -13,6 +13,7 @@ from django.shortcuts import get_object_or_404
 
 from api.models import AuthAPIKey, AuthAPILog
 from eve_proxy.models import CachedDocument
+from eve_proxy.exceptions import *
 from eve_api.models import EVEAccount
 from sso.models import ServiceAccount, Service
 from hr.models import Blacklist
@@ -119,9 +120,12 @@ class EveAPIProxyHandler(BaseHandler):
             obj = get_object_or_404(EVEAccount, pk=params['userid'])
             params['apikey'] = obj.api_key
 
-        cached_doc = CachedDocument.objects.api_query(url_path, params, exceptions=False)
-
-        return HttpResponse(cached_doc.body)
+        try:
+            cached_doc = CachedDocument.objects.api_query(url_path, params)
+        except DocumentRetrievalError:
+            return HttpResponse(status=500)
+        else:
+            return HttpResponse(cached_doc.body)
 
 
 class OpTimerHandler(BaseHandler):
@@ -132,28 +136,19 @@ class OpTimerHandler(BaseHandler):
 
         params = {'userID': obj.id, 'apiKey': obj.api_key, 'characterID': FULL_API_CHARACTER_ID}
 
-        cached_doc = CachedDocument.objects.api_query('/char/UpcomingCalendarEvents.xml.aspx', params, exceptions=False)
+        error_doc = {'ops': [{'startsIn': -1, 'eventID': 0, 'ownerName': '', 'eventDate': '', 'eventTitle': '<div style="text-align:center">The EVE API calendar is unavailable</div>', 'duration': 0, 'isImportant': 0, 'eventText': 'Fuck CCP tbqh imho srsly', 'endsIn':-1, 'forumLink': ''}]}
 
-        if cached_doc:
-            dom = minidom.parseString(cached_doc.body.encode('utf-8'))
-            enode = dom.getElementsByTagName('error')
-        if not cached_doc or enode:
-            return {'ops': [{
-                'startsIn': -1,
-                'eventID': 0,
-                'ownerName': '',
-                'eventDate': '',
-                'eventTitle': '<div style="text-align:center">The EVE API is currently down</div>',
-                'duration': 0,
-                'isImportant': 0,
-                'eventText': 'Fuck CCP tbqh imho srsly',
-                'endsIn':-1,
-                'forumLink': ''}]}
+        try:
+            cached_doc = CachedDocument.objects.api_query('/char/UpcomingCalendarEvents.xml.aspx', params)
+        except DocumentRetrievalError:
+            return error_doc
+        dom = minidom.parseString(cached_doc.body.encode('utf-8'))
+        if dom.getElementsByTagName('error'):
+            error_doc['raw_xml'] = cached_doc.body
+            return error_doc
 
         events = []
-        events_node_children = dom.getElementsByTagName('rowset')[0].childNodes
-       
-        for node in events_node_children:
+        for node in dom.getElementsByTagName('rowset')[0].childNodes:
             if node.nodeType == 1:
                 ownerID = node.getAttribute('ownerID')                
                 if ownerID != '1':
@@ -200,7 +195,7 @@ class OpTimerHandler(BaseHandler):
                 'endsIn':-1,
                 'forumLink': ''}]}
         else:
-            return {'ops':events}
+            return {'ops': events, 'doc_time': cached_doc.time_retrieved, 'cache_until': cached_doc.cached_until, 'current_time': datetime.utcnow() }
 
 
 class BlacklistHandler(BaseHandler):

eve_api/api_puller/accounts.py

@@ -22,28 +22,12 @@ def import_eve_account(api_key, user_id, force_cache=False):
     """
     Imports an account from the API into the EVEAccount model.
     """
-    auth_params = {'userID': user_id, 'apiKey': api_key}
 
+    auth_params = {'userid': user_id, 'apikey': api_key}
     try:
-        account_doc = CachedDocument.objects.api_query('/account/Characters.xml.aspx',
-                                                   params=auth_params,
-                                                   no_cache=force_cache)
-    except APIAuthException:
-        try:
-            account = EVEAccount.objects.get(id=user_id)
-        except EVEAccount.DoesNotExist:
-            return
-        if api_key == account.api_key:
-            account.api_status = API_STATUS_AUTH_ERROR
-            account.api_last_updated = datetime.utcnow()
-            account.save()
-            return
-    except APINoUserIDException:
-        try:
-            account = EVEAccount.objects.get(id=user_id)
-            account.delete()
-        except EVEAccount.DoesNotExist:
-            return
+        account_doc = CachedDocument.objects.api_query('/account/Characters.xml.aspx', params=auth_params, no_cache=force_cache)
+    except:
+        return
 
     if account_doc and account_doc.body:
         dom = minidom.parseString(account_doc.body.encode('utf-8'))
@@ -59,11 +43,12 @@ def import_eve_account(api_key, user_id, force_cache=False):
 
         error = enode[0].getAttribute('code')
 
-        if int(error) >= 900:
+        if int(error) >= 500:
             # API disabled, down or rejecting, return without changes
             return
-
-        if error == '211':
+        elif error in ['202', '203', '204', '205', '212']:
+            account.api_status = API_STATUS_AUTH_ERROR
+        elif error == '211':
             account.api_status = API_STATUS_ACC_EXPIRED
         else:
             account.api_status = API_STATUS_OTHER_ERROR

eve_api/managers.py

@@ -82,13 +82,11 @@ class EVEPlayerCorporationManager(models.Manager):
         
         # Convert incoming data to UTF-8.
         dom = minidom.parseString(corp_doc.body.encode('utf-8'))
-        
         error_node = dom.getElementsByTagName('error')
         
         # If there's an error, see if it's because the corp doesn't exist.
         if error_node:
-            error_code = error_node[0].getAttribute('code')
-            if error_code == '523':
+            if error_node[0].getAttribute('code') == '523':
                 raise InvalidCorpID(id)
         
         return dom

eve_proxy/exceptions.py

@@ -0,0 +1,17 @@
+class DocumentRetrievalError(Exception):
+    """
+    Unable to retrieve a document from the EVE API: %s
+    """
+    def __init__(self, value):
+        self.value = value
+
+    def __str__(self):
+        return self.__doc_ % value_
+
+class InvalidDocument(Exception):
+    """
+    The document retrieved from the EVE API is not a valid XML document
+    """
+    def __str__(self):
+        return self.__doc__
+

eve_proxy/models.py

@@ -1,95 +1,46 @@
-import httplib
-import urllib
+import urllib, urllib2
 import xml
 import hashlib
 import socket
 from datetime import datetime, timedelta
 from xml.dom import minidom
 from django.db import models
-from eve_api.api_exceptions import APIAuthException, APINoUserIDException
+from eve_proxy.exceptions import *
+import settings
 
 # You generally never want to change this unless you have a very good reason.
-API_URL = 'api.eve-online.com'
+
+try:
+    API_URL = getattr(settings, 'EVE_API_URL')
+except AttributeError:
+    API_URL = 'http://api.eve-online.com'
+
+# Errors to rollback if we have a cached version of the document
+# Errors 500-999 at the moment, this can be trimmed down as needed
+ROLLBACK_ERRORS = range(500, 999)
 
 class CachedDocumentManager(models.Manager):
     """
     This manager handles querying or retrieving CachedDocuments.
     """
 
-    def get_document_id(self, url_path, params):
-        if params:
-            p = params.copy()
-            if 'service' in p:
-                del p['service']
-            paramstr = urllib.urlencode(p)
+    def construct_url(self, url_path, params):
+
+         # Valid arguments for EVE API Calls
+        allowed_params = ['userid', 'apikey', 'characterid', 'version', 'names', 'ids', 'corporationid', 'beforerefid', 'accountkey']
+
+        if len(params):
+            for k, v in params.items():
+                del params[k]
+                if k.lower() in allowed_params:
+                   params[k.lower()] = v
+            url = "%s%s?%s" % (API_URL, url_path, urllib.urlencode(params))
         else:
-            paramstr = ''
+            url = "%s%s" % (API_URL, url_path)
 
-        return hashlib.sha1('%s?%s' % (url_path, paramstr)).hexdigest()
+        return url
 
-    def cache_from_eve_api(self, url_path, params):
-        """
-        Connect to the EVE API server, send the request, and cache it to
-        a CachedDocument. This is typically not something you want to call
-        directly. Use api_query().
-        """
-
-        method = 'GET'
-        paramstr = ''
-        service = 'auth'
-
-        if params:
-            if 'service' in params:
-                service = params['service']
-                del params['service']
-            paramstr = urllib.urlencode(params)
-
-        if len(paramstr.strip()) > 0:
-            method = 'POST'
-
-        headers = {"Content-type": "application/x-www-form-urlencoded"}
-        conn = httplib.HTTPConnection(API_URL)
-        try:
-            conn.request(method, url_path, paramstr, headers)
-        except socket.error:
-            return None
-        response = conn.getresponse()
-
-        if response.status == 200:
-            doc_id = self.get_document_id(url_path, params)
-            cached_doc, created = self.get_or_create(url_path=doc_id)
-            cached_doc.body = unicode(response.read(), 'utf-8')
-            cached_doc.time_retrieved = datetime.utcnow()
-
-            try:
-                # Parse the response via minidom
-                dom = minidom.parseString(cached_doc.body.encode('utf-8'))
-            except xml.parsers.expat.ExpatError:
-                cached_doc.cached_until = datetime.utcnow()
-            else:
-                cached_doc.cached_until = dom.getElementsByTagName('cachedUntil')[0].childNodes[0].nodeValue           
-
-            # If this is user related, write a log instance
-            if params and 'userID' in params:
-                try:
-                    v = int(params['userID'])
-                except ValueError:
-                    pass
-                else:
-                    log = ApiAccessLog()
-                    log.userid = v
-                    log.service = service
-                    log.time_access = cached_doc.time_retrieved
-                    log.document = url_path
-                    log.save()
-
-            # Finish up and return the resulting document just in case.
-            cached_doc.save()
-            cached_doc = self.get(id=cached_doc.pk)
-
-            return cached_doc
-    
-    def api_query(self, url_path, params=None, no_cache=False, exceptions=True):
+    def api_query(self, url_path, params={}, no_cache=False, exceptions=True):
         """
         Transparently handles querying EVE API or retrieving the document from
         the cache.
@@ -102,35 +53,59 @@ class CachedDocumentManager(models.Manager):
                                     the query: userID=1&characterID=xxxxxxxx
         """
 
-        doc_id = self.get_document_id(url_path, params)
+        url = self.construct_url(url_path, params)
      
-        doc = None
         if not no_cache:
             try:
-                doc = super(CachedDocumentManager, self).get_query_set().get(url_path=doc_id)
+                doc = super(CachedDocumentManager, self).get_query_set().get(url_path=url)
             except self.model.DoesNotExist:
-                pass
+                doc = None
+        else:
+            doc = None
 
-        # EVE uses UTC.
-        current_eve_time = datetime.utcnow()
+        if not doc or not doc.cached_until or datetime.utcnow() > doc.cached_until:
 
-        if not doc or not doc.cached_until or current_eve_time > doc.cached_until:
-            doc = self.cache_from_eve_api(url_path, params)
+            req = urllib2.Request(url)
+            req.add_header('CCP-Contact', 'matalok@pleaseignore.com')
+            try:
+                conn = urllib2.urlopen(req)
+            except urllib2.HTTPError, e:
+                raise DocumentRetrievalError(e.code)
+            except urllib2.URLError, e:
+                raise DocumentRetrievalError(e.reason)
 
-        if doc:
-            dom = minidom.parseString(doc.body.encode('utf-8'))
+            cached_doc, created = self.get_or_create(url_path=url)
+            cached_doc.body = unicode(conn.read(), 'utf-8')
+            cached_doc.time_retrieved = datetime.utcnow()
 
-            if dom:
-                error_node = dom.getElementsByTagName('error')
-                if error_node and exceptions:
-                    error_code = error_node[0].getAttribute('code')
-                    # User specified an invalid userid and/or auth key.
-                    if error_code == '203':
-                        raise APIAuthException()
-                    elif error_code == '106':
-                        raise APINoUserIDException()
+            try:
+                # Parse the response via minidom
+                dom = minidom.parseString(cached_doc.body.encode('utf-8'))
+            except xml.parsers.expat.ExpatError:
+                cached_doc.cached_until = datetime.utcnow()
+            else:
+                cached_doc.cached_until = dom.getElementsByTagName('cachedUntil')[0].childNodes[0].nodeValue
+                enode = dom.getElementsByTagName('error')
+                if enode:
+                   error = enode[0].getAttribute('code')
+                else:
+                   error = 0
 
-            return doc
+            # If we have a error in the ignored error list use the cached doc, otherwise return the new doc
+            if not doc or (not error or not error in ROLLBACK_ERRORS):
+                cached_doc.save()
+                doc = self.get(id=cached_doc.pk)
+
+            # If this is user related, write a log instance
+            if params and params.get('userid', None):
+                try:
+                    v = int(params.get('userid', None))
+                except:
+                    pass
+                else:
+                    ApiAccessLog(userid=v, service='Unknown', time_access=cached_doc.time_retrieved, document=url).save()
+
+        return doc
 
 class CachedDocument(models.Model):
     """