Reorganise the file structure into a project tree

app/eve_proxy/__init__.py

@@ -0,0 +1,9 @@
+VERSION = (0, 4)
+
+# Dynamically calculate the version based on VERSION tuple
+if len(VERSION)>2 and VERSION[2] is not None:
+    str_version = "%d.%d_%s" % VERSION[:3]
+else:
+    str_version = "%d.%d" % VERSION[:2]
+
+__version__ = str_version

app/eve_proxy/admin.py

@@ -0,0 +1,16 @@
+from django.contrib import admin
+from eve_proxy.models import CachedDocument, ApiAccessLog
+
+class CachedDocumentAdmin(admin.ModelAdmin):
+    model = CachedDocument
+    list_display = ('url_path', 'time_retrieved', 'cached_until')
+    verbose_name = 'Cached Document'
+    verbose_name_plural = 'Cached Documents'
+admin.site.register(CachedDocument, CachedDocumentAdmin)
+
+class ApiAccessLogAdmin(admin.ModelAdmin):
+    model = ApiAccessLog
+    list_display = ('userid', 'service', 'document', 'time_access')
+    verbose_name = 'API Access Log'
+    verbose_name_plural = 'API Access Logs'
+admin.site.register(ApiAccessLog, ApiAccessLogAdmin)

app/eve_proxy/exceptions.py

@@ -0,0 +1,17 @@
+class DocumentRetrievalError(Exception):
+    """
+    Unable to retrieve a document from the EVE API: %s
+    """
+    def __init__(self, value):
+        self.value = value
+
+    def __str__(self):
+        return self.__doc__ % self.value
+
+class InvalidDocument(Exception):
+    """
+    The document retrieved from the EVE API is not a valid XML document
+    """
+    def __str__(self):
+        return self.__doc__
+

app/eve_proxy/migrations/0001_initial.py

@@ -0,0 +1,60 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Adding model 'CachedDocument'
+        db.create_table('eve_proxy_cacheddocument', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('url_path', self.gf('django.db.models.fields.CharField')(max_length=255)),
+            ('body', self.gf('django.db.models.fields.TextField')()),
+            ('time_retrieved', self.gf('django.db.models.fields.DateTimeField')(null=True, blank=True)),
+            ('cached_until', self.gf('django.db.models.fields.DateTimeField')(null=True, blank=True)),
+        ))
+        db.send_create_signal('eve_proxy', ['CachedDocument'])
+
+        # Adding model 'ApiAccessLog'
+        db.create_table('eve_proxy_apiaccesslog', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('userid', self.gf('django.db.models.fields.IntegerField')()),
+            ('service', self.gf('django.db.models.fields.CharField')(max_length=255)),
+            ('time_access', self.gf('django.db.models.fields.DateTimeField')()),
+            ('document', self.gf('django.db.models.fields.CharField')(max_length=255)),
+        ))
+        db.send_create_signal('eve_proxy', ['ApiAccessLog'])
+
+
+    def backwards(self, orm):
+        
+        # Deleting model 'CachedDocument'
+        db.delete_table('eve_proxy_cacheddocument')
+
+        # Deleting model 'ApiAccessLog'
+        db.delete_table('eve_proxy_apiaccesslog')
+
+
+    models = {
+        'eve_proxy.apiaccesslog': {
+            'Meta': {'object_name': 'ApiAccessLog'},
+            'document': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'service': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'time_access': ('django.db.models.fields.DateTimeField', [], {}),
+            'userid': ('django.db.models.fields.IntegerField', [], {})
+        },
+        'eve_proxy.cacheddocument': {
+            'Meta': {'object_name': 'CachedDocument'},
+            'body': ('django.db.models.fields.TextField', [], {}),
+            'cached_until': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'time_retrieved': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'url_path': ('django.db.models.fields.CharField', [], {'max_length': '255'})
+        }
+    }
+
+    complete_apps = ['eve_proxy']

app/eve_proxy/migrations/0002_auto__add_unique_cacheddocument_url_path.py

@@ -0,0 +1,40 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Adding unique constraint on 'CachedDocument', fields ['url_path']
+        db.create_unique('eve_proxy_cacheddocument', ['url_path'])
+
+
+    def backwards(self, orm):
+        
+        # Removing unique constraint on 'CachedDocument', fields ['url_path']
+        db.delete_unique('eve_proxy_cacheddocument', ['url_path'])
+
+
+    models = {
+        'eve_proxy.apiaccesslog': {
+            'Meta': {'object_name': 'ApiAccessLog'},
+            'document': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'service': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'time_access': ('django.db.models.fields.DateTimeField', [], {}),
+            'userid': ('django.db.models.fields.IntegerField', [], {})
+        },
+        'eve_proxy.cacheddocument': {
+            'Meta': {'object_name': 'CachedDocument'},
+            'body': ('django.db.models.fields.TextField', [], {}),
+            'cached_until': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'time_retrieved': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'url_path': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
+        }
+    }
+
+    complete_apps = ['eve_proxy']

app/eve_proxy/migrations/0003_auto__del_field_cacheddocument_id__add_field_cacheddocument_doc_key__d.py

@@ -0,0 +1,55 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+
+        # Delete all records
+        db.execute("DELETE FROM eve_proxy_cacheddocument")
+        
+        # Removing unique constraint on 'CachedDocument', fields ['url_path']
+        db.delete_unique('eve_proxy_cacheddocument', ['url_path'])
+
+        # Deleting field 'CachedDocument.id'
+        db.delete_column('eve_proxy_cacheddocument', 'id')
+
+        # Adding field 'CachedDocument.doc_key'
+        db.add_column('eve_proxy_cacheddocument', 'doc_key', self.gf('django.db.models.fields.CharField')(default='xxx', max_length=40, primary_key=True), keep_default=False)
+
+
+    def backwards(self, orm):
+        
+        # User chose to not deal with backwards NULL issues for 'CachedDocument.id'
+        raise RuntimeError("Cannot reverse this migration. 'CachedDocument.id' and its values cannot be restored.")
+
+        # Deleting field 'CachedDocument.doc_key'
+        db.delete_column('eve_proxy_cacheddocument', 'doc_key')
+
+        # Adding unique constraint on 'CachedDocument', fields ['url_path']
+        db.create_unique('eve_proxy_cacheddocument', ['url_path'])
+
+
+    models = {
+        'eve_proxy.apiaccesslog': {
+            'Meta': {'ordering': "['time_access']", 'object_name': 'ApiAccessLog'},
+            'document': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'service': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'time_access': ('django.db.models.fields.DateTimeField', [], {}),
+            'userid': ('django.db.models.fields.IntegerField', [], {})
+        },
+        'eve_proxy.cacheddocument': {
+            'Meta': {'ordering': "['time_retrieved']", 'object_name': 'CachedDocument'},
+            'body': ('django.db.models.fields.TextField', [], {}),
+            'cached_until': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'doc_key': ('django.db.models.fields.CharField', [], {'max_length': '40', 'primary_key': 'True'}),
+            'time_retrieved': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'url_path': ('django.db.models.fields.CharField', [], {'max_length': '255'})
+        }
+    }
+
+    complete_apps = ['eve_proxy']

app/eve_proxy/models.py

@@ -0,0 +1,139 @@
+import urllib, urllib2
+import xml
+import hashlib
+import socket
+from datetime import datetime, timedelta
+from xml.dom import minidom
+from django.db import models
+from eve_proxy.exceptions import *
+import settings
+
+# You generally never want to change this unless you have a very good reason.
+
+try:
+    API_URL = getattr(settings, 'EVE_API_URL')
+except AttributeError:
+    API_URL = 'https://api.eve-online.com'
+
+# Errors to rollback if we have a cached version of the document
+# Errors 500-999 at the moment, this can be trimmed down as needed
+ROLLBACK_ERRORS = range(500, 999)
+
+class CachedDocumentManager(models.Manager):
+    """
+    This manager handles querying or retrieving CachedDocuments.
+    """
+
+    def construct_url(self, url_path, params):
+
+         # Valid arguments for EVE API Calls
+        allowed_params = ['userid', 'apikey', 'characterid', 'version', 'names', 'ids', 'corporationid', 'beforerefid', 'accountkey']
+
+        if len(params):
+            for k, v in params.items():
+                del params[k]
+                if k.lower() in allowed_params:
+                   params[k.lower()] = v
+            url = "%s%s?%s" % (API_URL, url_path, urllib.urlencode(params))
+        else:
+            url = "%s%s" % (API_URL, url_path)
+
+        return url
+
+    def api_query(self, url_path, params={}, no_cache=False, exceptions=True):
+        """
+        Transparently handles querying EVE API or retrieving the document from
+        the cache.
+        
+        Arguments:
+        url_path: (string) Path to the EVE API page to query. For example:
+                           /eve/ErrorList.xml.aspx
+        params: (dictionary/string) A dictionary of extra parameters to include.
+                                    May also be a string representation of
+                                    the query: userID=1&characterID=xxxxxxxx
+        """
+
+        url = self.construct_url(url_path, params)
+        doc_key = hashlib.sha1(url).hexdigest()
+
+        
+        try:
+            doc = super(CachedDocumentManager, self).get_query_set().get(pk=doc_key)
+        except self.model.DoesNotExist:
+            doc = CachedDocument(pk=doc_key, url_path=url)
+
+        if not doc.cached_until or datetime.utcnow() > doc.cached_until or no_cache:
+
+            req = urllib2.Request(url)
+            req.add_header('CCP-Contact', 'matalok@pleaseignore.com')
+            try:
+                conn = urllib2.urlopen(req)
+            except urllib2.HTTPError, e:
+                print "HTTP Error Code: %s" % e.code
+                raise DocumentRetrievalError(e.code)
+            except urllib2.URLError, e:
+                print "URLError: %s" % e.reason
+                raise DocumentRetrievalError(e.reason)
+
+            doc.body = unicode(conn.read(), 'utf-8')
+            doc.time_retrieved = datetime.utcnow()
+
+            error = 0
+            try:
+                # Parse the response via minidom
+                dom = minidom.parseString(doc.body.encode('utf-8'))
+            except:
+                doc.cached_until = datetime.utcnow()
+            else:
+                doc.cached_until = dom.getElementsByTagName('cachedUntil')[0].childNodes[0].nodeValue
+                enode = dom.getElementsByTagName('error')
+                if enode:
+                   error = enode[0].getAttribute('code')
+
+            # If we have a error in the ignored error list use the cached doc, otherwise return the new doc
+            if not error or not error in ROLLBACK_ERRORS:
+                doc.save()
+                doc = self.get(pk=doc.pk)
+
+            # If this is user related, write a log instance
+            if params and params.get('userid', None):
+                try:
+                    v = int(params.get('userid', None))
+                except:
+                    pass
+                else:
+                    ApiAccessLog(userid=v, service='Unknown', time_access=doc.time_retrieved, document=url).save()
+
+        return doc
+
+class CachedDocument(models.Model):
+    """
+    This is a cached XML document from the EVE API.
+    """
+    doc_key = models.CharField(max_length=40, primary_key=True)
+    url_path = models.CharField(max_length=255)
+    body = models.TextField()
+    time_retrieved = models.DateTimeField(blank=True, null=True)
+    cached_until = models.DateTimeField(blank=True, null=True)
+
+    # The custom manager handles the querying.
+    objects = CachedDocumentManager()
+
+    class Meta:
+        verbose_name = 'Cached Document'
+        verbose_name_plural = 'Cached Documents'
+        ordering = ['time_retrieved']
+
+class ApiAccessLog(models.Model):
+    """
+    Provides a list of API accesses made by applications or Auth
+    """
+    userid = models.IntegerField()
+    service = models.CharField(max_length=255)
+    time_access = models.DateTimeField()
+    document = models.CharField(max_length=255)
+
+    class Meta:
+        verbose_name = 'API Access Log'
+        verbose_name_plural = 'API Access Logs'
+        ordering = ['time_access']

app/eve_proxy/tasks.py

@@ -0,0 +1,24 @@
+from django.conf import settings
+import logging
+from datetime import datetime, timedelta
+from celery.decorators import task
+from eve_proxy.models import CachedDocument, ApiAccessLog
+
+@task(ignore_result=True)
+def clear_stale_cache(cache_extension=0):
+    log = clear_stale_cache.get_logger()
+
+    time = datetime.utcnow() - timedelta(seconds=cache_extension)
+    objs = CachedDocument.objects.filter(cached_until__lt=time)
+    log.info('Removing %s stale cache documents' % objs.count())
+    objs.delete()
+
+
+@task(ignore_result=True)
+def clear_old_logs():
+    log = clear_old_logs.get_logger()
+
+    time = datetime.utcnow() - timedelta(days=settings.EVE_PROXY_KEEP_LOGS)
+    objs = ApiAccessLog.objects.filter(time_access__lt=time)
+    log.info('Removing %s old access logs' % objs.count())
+    objs.delete()

app/eve_proxy/tests.py

@@ -0,0 +1,70 @@
+from __future__ import with_statement
+import unittest
+from datetime import datetime
+import time
+from eve_proxy.models import CachedDocument
+from eve_proxy.exceptions import *
+
+class CachedDocumentTestCase(unittest.TestCase):
+
+    params = {'apikey': 'B149FF0C66D6488CA9C0DA8B7E49F4C9CC0064BA38B043819146992510DA8C83' , 'userid': 415631 }
+
+    def tearDown(self):
+        CachedDocument.objects.all().delete()
+
+    def testDumbApiQuery(self):
+        """ Queries the EVE API with a non authenticated request """
+        url = '/server/ServerStatus.xml.aspx'
+
+        obj = CachedDocument.objects.api_query(url, no_cache=True)
+
+        self.assertNotEqual(obj, None, "No CachedDocument returned")
+        self.assertNotEqual(obj.body, None, "CachedDocument has no body")
+        self.assertNotEqual(obj.cached_until, None, "CachedDocument has no cache expiry time")
+        self.assertNotEqual(obj.time_retrieved, None, "CachedDocument has no retrieval time")
+
+    def testUserApiQuery(self):
+        """ Queries the EVE API with a authenticated request """
+
+        url = '/account/Characters.xml.aspx'
+
+        obj = CachedDocument.objects.api_query(url, params=self.params, no_cache=True)
+
+        self.assertNotEqual(obj, None, "No CachedDocument returned")
+        self.assertNotEqual(obj.body, None, "CachedDocument has no body")
+        self.assertNotEqual(obj.cached_until, None, "CachedDocument has no cache expiry time")
+        self.assertNotEqual(obj.time_retrieved, None, "CachedDocument has no retrieval time")        
+
+    def testCaching(self):
+        """ Tests if objects are being cached correctly """
+
+        url = '/server/ServerStatus.xml.aspx'
+	obj = CachedDocument.objects.api_query(url, no_cache=True)
+	obj2 = CachedDocument.objects.api_query(url)
+
+        self.assertEqual(obj.pk, obj2.pk, "Objects are not caching correctly")
+
+    def testCacheExpiry(self):
+        """ Tests that cache expiry is working """
+
+        url = '/server/ServerStatus.xml.aspx'
+        obj = CachedDocument.objects.api_query(url, no_cache=True)
+        ret_time = obj.time_retrieved
+        obj.cached_until = datetime.utcnow()
+        obj.save()
+
+        time.sleep(1)
+
+        obj2 = CachedDocument.objects.api_query(url)
+        
+        self.assertEqual(obj.pk, obj2.pk, "Cache Expiry test returned different objects")
+        self.assertNotEqual(obj2.time_retrieved, ret_time, "Retrieval time not updated")
+
+    def testInvalidApiQuery(self):
+        """ Attempts to request a invalid EVE API endpoint """
+
+        url = '/server/ServerStatus.xml.aspx'
+        
+        with self.assertRaises(DocumentRetrievalError):
+            obj = CachedDocument.objects.api_query(url, no_cache=True)
+

app/eve_proxy/urls.py

@@ -0,0 +1,8 @@
+from django.conf.urls.defaults import *
+
+urlpatterns = patterns('eve_proxy.views',
+    # This view can be used just like EVE API's http://api.eve-online.com.
+    # Any parameters or URL paths are sent through the cache system and
+    # forwarded to the EVE API site as needed.
+    url(r'^', 'retrieve_xml', name='eve_proxy-retrieve_xml'),
+)

app/eve_proxy/views.py

@@ -0,0 +1,39 @@
+from django.core.urlresolvers import reverse
+from django.http import HttpResponse, HttpResponseNotFound, HttpResponseServerError
+from eve_proxy.models import CachedDocument
+
+def retrieve_xml(request):
+    """
+    A view that forwards EVE API requests through the cache system, either
+    retrieving a cached document or querying and caching as needed.
+    """
+    # This is the URL path (minus the parameters).
+    url_path = request.META['PATH_INFO'].replace(reverse('eve_proxy.views.retrieve_xml'),"/")
+    # The parameters attached to the end of the URL path.
+
+    if request.method == 'POST':
+        p = request.POST
+    else:
+        p = request.GET
+
+    # Convert the QuerySet object into a dict
+    params = {}
+    for key,value in p.items():
+        params[key] = value
+
+    if url_path == '/' or url_path == '':
+        # If they don't provide any kind of query, shoot a quick error message.
+        return HttpResponseNotFound('No API query specified.')
+    
+    if 'userID' in params and not 'service' in params:
+        return HttpResponse('No Service ID provided.')
+
+    try:
+        cached_doc = CachedDocument.objects.api_query(url_path, params, exceptions=False)
+    except:
+        return HttpResponseServerError('Error occured')
+
+    if cached_doc:
+        return HttpResponse(cached_doc.body, mimetype='text/xml')
+
+    return HttpResponseNotFound('Error retrieving the document')