Rewowrking on the eve_proxy application, more resilient

eve_proxy/exceptions.py

@@ -6,7 +6,7 @@ class DocumentRetrievalError(Exception):
         self.value = value
 
     def __str__(self):
-        return self.__doc_ % value_
+        return self.__doc__ % value
 
 class InvalidDocument(Exception):
     """

eve_proxy/migrations/0002_auto__add_unique_cacheddocument_url_path.py

@@ -0,0 +1,40 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Adding unique constraint on 'CachedDocument', fields ['url_path']
+        db.create_unique('eve_proxy_cacheddocument', ['url_path'])
+
+
+    def backwards(self, orm):
+        
+        # Removing unique constraint on 'CachedDocument', fields ['url_path']
+        db.delete_unique('eve_proxy_cacheddocument', ['url_path'])
+
+
+    models = {
+        'eve_proxy.apiaccesslog': {
+            'Meta': {'object_name': 'ApiAccessLog'},
+            'document': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'service': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'time_access': ('django.db.models.fields.DateTimeField', [], {}),
+            'userid': ('django.db.models.fields.IntegerField', [], {})
+        },
+        'eve_proxy.cacheddocument': {
+            'Meta': {'object_name': 'CachedDocument'},
+            'body': ('django.db.models.fields.TextField', [], {}),
+            'cached_until': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'time_retrieved': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'url_path': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
+        }
+    }
+
+    complete_apps = ['eve_proxy']

eve_proxy/models.py

@@ -54,47 +54,53 @@ class CachedDocumentManager(models.Manager):
         """
 
         url = self.construct_url(url_path, params)
-     
-        if not no_cache:
-            try:
-                doc = super(CachedDocumentManager, self).get_query_set().get(url_path=url)
-            except self.model.DoesNotExist:
-                doc = None
-        else:
-            doc = None
 
-        if not doc or not doc.cached_until or datetime.utcnow() > doc.cached_until:
+        try:
+            doc = super(CachedDocumentManager, self).get_query_set().get(url_path=url)
+            print "Doc found"
+        except self.model.MultipleObjectsReturned:
+            super(CachedDocumentManager, self).get_query_set().filter(url_path=url).delete()
+            doc = CachedDocument(url_path=url)
+            print "Multiple doc"
+        except self.model.DoesNotExist:
+            doc = CachedDocument(url_path=url)
+            print "No doc found"
+
+        if doc.pk and no_cache:
+            doc.delete()
+            doc = CachedDocument(url_path=url)
+
+        if not doc.cached_until or datetime.utcnow() > doc.cached_until:
+            print "Doc expired"
 
             req = urllib2.Request(url)
             req.add_header('CCP-Contact', 'matalok@pleaseignore.com')
             try:
-                conn = urllib2.urlopen(req)
+                conn = urllib2.urlopen(req, timeout=60)
             except urllib2.HTTPError, e:
                 raise DocumentRetrievalError(e.code)
             except urllib2.URLError, e:
                 raise DocumentRetrievalError(e.reason)
 
-            cached_doc, created = self.get_or_create(url_path=url)
+            doc.body = unicode(conn.read(), 'utf-8')
-            cached_doc.body = unicode(conn.read(), 'utf-8')
+            doc.time_retrieved = datetime.utcnow()
-            cached_doc.time_retrieved = datetime.utcnow()
 
+            error = 0
             try:
                 # Parse the response via minidom
-                dom = minidom.parseString(cached_doc.body.encode('utf-8'))
+                dom = minidom.parseString(doc.body.encode('utf-8'))
-            except xml.parsers.expat.ExpatError:
+            except:
-                cached_doc.cached_until = datetime.utcnow()
+                doc.cached_until = datetime.utcnow()
             else:
-                cached_doc.cached_until = dom.getElementsByTagName('cachedUntil')[0].childNodes[0].nodeValue
+                doc.cached_until = dom.getElementsByTagName('cachedUntil')[0].childNodes[0].nodeValue
                 enode = dom.getElementsByTagName('error')
                 if enode:
                    error = enode[0].getAttribute('code')
-                else:
-                   error = 0
 
             # If we have a error in the ignored error list use the cached doc, otherwise return the new doc
-            if not doc or (not error or not error in ROLLBACK_ERRORS):
+            if not error or not error in ROLLBACK_ERRORS:
-                cached_doc.save()
+                doc.save()
-                doc = self.get(id=cached_doc.pk)
+                doc = self.get(id=doc.pk)
 
             # If this is user related, write a log instance
             if params and params.get('userid', None):
@@ -103,7 +109,7 @@ class CachedDocumentManager(models.Manager):
                 except:
                     pass
                 else:
-                    ApiAccessLog(userid=v, service='Unknown', time_access=cached_doc.time_retrieved, document=url).save()
+                    ApiAccessLog(userid=v, service='Unknown', time_access=doc.time_retrieved, document=url).save()
 
         return doc
 
@@ -111,7 +117,7 @@ class CachedDocument(models.Model):
     """
     This is a cached XML document from the EVE API.
     """
-    url_path = models.CharField(max_length=255)
+    url_path = models.CharField(max_length=255, unique=True)
     body = models.TextField()
     time_retrieved = models.DateTimeField(blank=True, null=True)
     cached_until = models.DateTimeField(blank=True, null=True)