Massive reworking of eve_proxy to use sha1 ids for url_paths and various other small fixes

eve_proxy/models.py

@@ -1,6 +1,7 @@
 import httplib
 import urllib
 import xml
+import hashlib
 from datetime import datetime, timedelta
 from xml.dom import minidom
 from django.db import models
@@ -13,40 +14,74 @@ class CachedDocumentManager(models.Manager):
     """
     This manager handles querying or retrieving CachedDocuments.
     """
-    def cache_from_eve_api(self, cached_doc, url_path, params, no_cache=False):
+
+    def get_document_id(self, url_path, params):
+        if params:
+            if 'service' in params:
+                del params['service']
+            paramstr = urllib.urlencode(params)
+        else:
+            paramstr = ''
+
+        return hashlib.sha1('%s?%s' % (url_path, paramstr)).hexdigest()
+
+    def cache_from_eve_api(self, url_path, params, no_cache=False):
         """
         Connect to the EVE API server, send the request, and cache it to
         a CachedDocument. This is typically not something you want to call
         directly. Use api_query().
         """
+
+        method = 'GET'
+        paramstr = ''
+
+        if params:
+            if 'service' in params:
+                service = params['service']
+                del params['service']
+            else:
+                service = 'auth'
+            paramstr = urllib.urlencode(params)
+
+        if len(paramstr.strip()) > 0:
+            method = 'POST'
+
         headers = {"Content-type": "application/x-www-form-urlencoded"}
-        # This is the connection to the EVE API server.
         conn = httplib.HTTPConnection(API_URL)
-        # Combine everything into an HTTP request.
+        conn.request(method, url_path, paramstr, headers)
-        conn.request("POST", url_path, params, headers)
-        # Retrieve the response from the server.
         response = conn.getresponse()
-        # Save the response (an XML document) to the CachedDocument.
+        
+        print url_path, paramstr, response.status
+
+        if response.status == 200:
+            doc_id = self.get_document_id(url_path, params)
+            cached_doc, created = self.get_or_create(url_path=doc_id)
             cached_doc.body = unicode(response.read(), 'utf-8')
+            cached_doc.time_retrieved = datetime.utcnow()
 
             try:
                 # Parse the response via minidom
                 dom = minidom.parseString(cached_doc.body.encode('utf-8'))
             except xml.parsers.expat.ExpatError:
-            return
+                cached_doc.cached_until = datetime.utcnow()
-
+            else:
-        # Set the CachedDocument's time_retrieved and cached_until times based
-        # on the values in the XML response. This will be used in future
-        # requests to see if the CachedDocument can be retrieved directly or
-        # if it needs to be re-cached.
-        cached_doc.time_retrieved = datetime.utcnow()
                 cached_doc.cached_until = dom.getElementsByTagName('cachedUntil')[0].childNodes[0].nodeValue           
 
+            # If this is user related, write a log instance
+            if params and 'userID' in params:
+                log = ApiAccessLog()
+                log.userid = params['userID']
+                log.service = service
+                log.time_access = cached_doc.time_retrieved
+                log.document = url_path
+                log.save()
+
             # Finish up and return the resulting document just in case.
             if no_cache == False:
                 cached_doc.save()
+                cached_doc = self.get(id=cached_doc.pk)
 
-        return dom
+            return cached_doc
     
     def api_query(self, url_path, params=None, no_cache=False, exceptions=True):
         """
@@ -61,64 +96,24 @@ class CachedDocumentManager(models.Manager):
                                     the query: userID=1&characterID=xxxxxxxx
         """
 
-        # If we have a service ID, store and strip it from the query string.
+        doc_id = self.get_document_id(url_path, params)
-        if 'service' in params:
-            service = params['service']
-            del params['service']
-        else:
-            service = 'auth'
      
-        if 'userID' in params:
+        doc = None
-            userid = params['userID']
+        if not no_cache:
-
+            try:
-        paramstr = urllib.urlencode(params)
+                doc = super(CachedDocumentManager, self).get_query_set().get(url_path=doc_id)
-
+            except self.model.DoesNotExist:
-        if params == None or paramstr.strip() == '':
+                pass
-            # For whatever reason, EVE API freaks out if there are no parameters.
-            # Add a bogus parameter if none are specified. I'm sure there's a
-            # better fix for this.
-            paramstr = 'odd_parm=1'
-        
-        # Combine the URL path and the parameters to create the full query.
-        query_name = '%s?%s' % (url_path, paramstr)
-        
-        if no_cache:
-            # If no_cache is enabled, don't even attempt a lookup.
-            cached_doc = CachedDocument()
-            created = False
-        else:
-            # Retrieve or create a new CachedDocument based on the full URL
-            # and parameters.
-            cached_doc, created = self.get_or_create(url_path=query_name)
     
         # EVE uses UTC.
         current_eve_time = datetime.utcnow()
 
-        # Figure out if we need hit EVE API and re-cache, or just pull from
+        if not doc or current_eve_time > doc.cached_until:
-        # the local cache (based on cached_until).
+            doc = self.cache_from_eve_api(url_path, params, no_cache=no_cache)
-        if no_cache or created or \
-          cached_doc.cached_until == None or \
-          current_eve_time > cached_doc.cached_until:
-            # Cache from EVE API
-            dom = self.cache_from_eve_api(cached_doc, url_path, paramstr, 
-                                    no_cache=no_cache)
 
-            # If its a user related request, log the transaction.
+        if doc:
-            if 'userID' in params:
+            dom = minidom.parseString(doc.body.encode('utf-8'))
-                log = ApiAccessLog()
-                log.userid = userid
-                log.service = service
-                log.time_access = datetime.utcnow()
-                log.document = url_path
-                log.save()
-        else:
-            # Parse the document here since it was retrieved from the
-            # database cache instead of queried for.
-            dom = minidom.parseString(cached_doc.body.encode('utf-8'))
             
-        # Check for the presence errors. Only check the bare minimum,
-        # generic stuff that applies to most or all queries. User-level code
-        # should check for the more specific errors.
             if dom:
                 error_node = dom.getElementsByTagName('error')
                 if error_node and exceptions:
@@ -129,7 +124,7 @@ class CachedDocumentManager(models.Manager):
                     elif error_code == '106':
                         raise APINoUserIDException()
 
-        return cached_doc
+            return doc
 
 class CachedDocument(models.Model):
     """

eve_proxy/views.py

@@ -1,5 +1,5 @@
 from django.core.urlresolvers import reverse
-from django.http import HttpResponse, HttpResponseNotFound
+from django.http import HttpResponse, HttpResponseNotFound, HttpResponseServerError
 from eve_proxy.models import CachedDocument
 
 def retrieve_xml(request):
@@ -28,14 +28,7 @@ def retrieve_xml(request):
     if 'userID' in params and not 'service' in params:
         return HttpResponse('No Service ID provided.')
 
-    # The query system will retrieve a cached_doc that was either previously
-    # or newly cached depending on cache intervals.
-    try:
     cached_doc = CachedDocument.objects.api_query(url_path, params, exceptions=False)
-    except:
-        return HttpResponseServerError()
-
-    # Return the document's body as XML.
 
     if cached_doc:
         return HttpResponse(cached_doc.body, mimetype='text/xml')