simple LRU cache for SQL databases

git-svn-id: http://svn.roundup-tracker.org/svnroot/roundup/trunk@1197 57a73879-2fb5-44c3-a270-3262357dd7e2

diff --git a/roundup/backends/rdbms_common.py b/roundup/backends/rdbms_common.py
index da0cf9f890dfc59cb0713f63692cb25454bca995..d90653fed69f86bc2f2f1a55e36ea5aa1f3f7fc6 100644 (file)
--- a/roundup/backends/rdbms_common.py
+++ b/roundup/backends/rdbms_common.py
@@ -1,4 +1,4 @@
-# $Id: rdbms_common.py,v 1.5 2002-09-19 03:56:20 richard Exp $
+# $Id: rdbms_common.py,v 1.6 2002-09-19 05:30:25 richard Exp $
 
 # standard python modules
 import sys, os, time, re, errno, weakref, copy
@@ -13,7 +13,16 @@ from blobfiles import FileStorage
 from roundup.indexer import Indexer
 from sessions import Sessions
 
+# number of rows to keep in memory
+ROW_CACHE_SIZE = 100
+
 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
+    ''' Wrapper around an SQL database that presents a hyperdb interface.
+
+        - some functionality is specific to the actual SQL database, hence
+          the sql_* methods that are NotImplemented
+        - we keep a cache of the latest ROW_CACHE_SIZE row fetches.
+    '''
     # flag to set on retired entries
     RETIRED_FLAG = '__hyperdb_retired'
 
@@ -30,6 +39,11 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database):
         # additional transaction support for external files and the like
         self.transactions = []
 
+        # keep a cache of the N most recently retrieved rows of any kind
+        # (classname, nodeid) = row
+        self.cache = {}
+        self.cache_lru = []
+
         # open a connection to the database, creating the "conn" attribute
         self.open_connection()
 
@@ -453,6 +467,13 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database):
                 if not node.has_key(col):
                     node[col] = None
 
+        # clear this node out of the cache if it's in there
+        key = (classname, nodeid)
+        if self.cache.has_key(key):
+            del self.cache[key]
+            self.cache_lru.remove(key)
+
+        # make the node data safe for the DB
         node = self.serialise(classname, node)
 
         # make sure the ordering is correct for column name -> column value
@@ -483,6 +504,13 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database):
         '''
         if __debug__:
             print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
+
+        # clear this node out of the cache if it's in there
+        key = (classname, nodeid)
+        if self.cache.has_key(key):
+            del self.cache[key]
+            self.cache_lru.remove(key)
+
         node = self.serialise(classname, node)
 
         cl = self.classes[classname]
@@ -531,6 +559,16 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database):
         '''
         if __debug__:
             print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid)
+
+        # see if we have this node cached
+        key = (classname, nodeid)
+        if self.cache.has_key(key):
+            # push us back to the top of the LRU
+            self.cache_lru.remove(key)
+            self.cache_lry.insert(0, key)
+            # return the cached information
+            return self.cache[key]
+
         # figure the columns we're fetching
         cl = self.classes[classname]
         cols, mls = self.determine_columns(cl.properties.items())
@@ -559,7 +597,17 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database):
             # extract the first column from the result
             node[col] = [x[0] for x in cursor.fetchall()]
 
-        return self.unserialise(classname, node)
+        # un-dbificate the node data
+        node = self.unserialise(classname, node)
+
+        # save off in the cache
+        key = (classname, nodeid)
+        self.cache[key] = node
+       # update the LRU
+       self.cache_lru.insert(0, key)
+       del self.cache[self.cache_lru.pop()]
+
+        return node
 
     def destroynode(self, classname, nodeid):
         '''Remove a node from the database. Called exclusively by the
@@ -572,6 +620,10 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database):
         if not self.hasnode(classname, nodeid):
             raise IndexError, '%s has no node %s'%(classname, nodeid)
 
+        # see if we have this node cached
+        if self.cache.has_key((classname, nodeid)):
+            del self.cache[(classname, nodeid)]
+
         # see if there's any obvious commit actions that we should get rid of
         for entry in self.transactions[:]:
             if entry[1][:2] == (classname, nodeid):
@@ -1636,6 +1688,8 @@ class Class(hyperdb.Class):
                     if '-1' in v:
                         v.remove('-1')
                         xtra = ' or _%s is NULL'%k
+                    else:
+                        xtra = ''
                     s = ','.join([a for x in v])
                     where.append('(_%s in (%s)%s)'%(k, s, xtra))
                     args = args + v