From: richard <richard@57a73879-2fb5-44c3-a270-3262357dd7e2>
Date: Wed, 15 May 2002 06:21:21 +0000 (+0000)
Subject:  . node caching now works, and gives a small boost in performance
X-Git-Url: https://git.tokkee.org/?a=commitdiff_plain;h=781ae10691a77f819f1271aa864c37517c9ed9ef;p=roundup.git

 . node caching now works, and gives a small boost in performance

As a part of this, I cleaned up the DEBUG output and implemented TRACE
output (HYPERDBTRACE='file to trace to') with checkpoints at the start of
CGI requests. Run roundup with python -O to skip all the DEBUG/TRACE stuff
(using if __debug__ which is compiled out with -O)


git-svn-id: http://svn.roundup-tracker.org/svnroot/roundup/trunk@732 57a73879-2fb5-44c3-a270-3262357dd7e2
---

diff --git a/CHANGES.txt b/CHANGES.txt
index ddfb7c8..bf0b32b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -42,6 +42,7 @@ Fixed:
    (thanks dman)
  . fixed some sorting issues that were breaking some unit tests under py2.2
  . mailgw test output dir was confusing the init test (but only on 2.2 *shrug*)
+ . node caching now works, and gives a small boost in performance
 
 
 2002-03-25 - 0.4.1
diff --git a/roundup/backends/back_anydbm.py b/roundup/backends/back_anydbm.py
index be4bd4e..0ee8fbb 100644
--- a/roundup/backends/back_anydbm.py
+++ b/roundup/backends/back_anydbm.py
@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-#$Id: back_anydbm.py,v 1.33 2002-04-24 10:38:26 rochecompaan Exp $
+#$Id: back_anydbm.py,v 1.34 2002-05-15 06:21:21 richard Exp $
 '''
 This module defines a backend that saves the hyperdatabase in a database
 chosen by anydbm. It is guaranteed to always be available in python
@@ -73,14 +73,14 @@ class Database(FileStorage, hyperdb.Database):
     def __getattr__(self, classname):
         """A convenient way of calling self.getclass(classname)."""
         if self.classes.has_key(classname):
-            if hyperdb.DEBUG:
-                print '__getattr__', (self, classname)
+            if __debug__:
+                print >>hyperdb.DEBUG, '__getattr__', (self, classname)
             return self.classes[classname]
         raise AttributeError, classname
 
     def addclass(self, cl):
-        if hyperdb.DEBUG:
-            print 'addclass', (self, cl)
+        if __debug__:
+            print >>hyperdb.DEBUG, 'addclass', (self, cl)
         cn = cl.classname
         if self.classes.has_key(cn):
             raise ValueError, cn
@@ -88,8 +88,8 @@ class Database(FileStorage, hyperdb.Database):
 
     def getclasses(self):
         """Return a list of the names of all existing classes."""
-        if hyperdb.DEBUG:
-            print 'getclasses', (self,)
+        if __debug__:
+            print >>hyperdb.DEBUG, 'getclasses', (self,)
         l = self.classes.keys()
         l.sort()
         return l
@@ -99,8 +99,8 @@ class Database(FileStorage, hyperdb.Database):
 
         If 'classname' is not a valid class name, a KeyError is raised.
         """
-        if hyperdb.DEBUG:
-            print 'getclass', (self, classname)
+        if __debug__:
+            print >>hyperdb.DEBUG, 'getclass', (self, classname)
         return self.classes[classname]
 
     #
@@ -109,8 +109,8 @@ class Database(FileStorage, hyperdb.Database):
     def clear(self):
         '''Delete all database contents
         '''
-        if hyperdb.DEBUG:
-            print 'clear', (self,)
+        if __debug__:
+            print >>hyperdb.DEBUG, 'clear', (self,)
         for cn in self.classes.keys():
             for dummy in 'nodes', 'journals':
                 path = os.path.join(self.dir, 'journals.%s'%cn)
@@ -123,16 +123,16 @@ class Database(FileStorage, hyperdb.Database):
         ''' grab a connection to the class db that will be used for
             multiple actions
         '''
-        if hyperdb.DEBUG:
-            print 'getclassdb', (self, classname, mode)
+        if __debug__:
+            print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode)
         return self._opendb('nodes.%s'%classname, mode)
 
     def _opendb(self, name, mode):
         '''Low-level database opener that gets around anydbm/dbm
            eccentricities.
         '''
-        if hyperdb.DEBUG:
-            print '_opendb', (self, name, mode)
+        if __debug__:
+            print >>hyperdb.DEBUG, '_opendb', (self, name, mode)
 
         # determine which DB wrote the class file
         db_type = ''
@@ -148,8 +148,8 @@ class Database(FileStorage, hyperdb.Database):
 
         # new database? let anydbm pick the best dbm
         if not db_type:
-            if hyperdb.DEBUG:
-                print "_opendb anydbm.open(%r, 'n')"%path
+            if __debug__:
+                print >>hyperdb.DEBUG, "_opendb anydbm.open(%r, 'n')"%path
             return anydbm.open(path, 'n')
 
         # open the database with the correct module
@@ -159,8 +159,9 @@ class Database(FileStorage, hyperdb.Database):
             raise hyperdb.DatabaseError, \
                 "Couldn't open database - the required module '%s'"\
                 "is not available"%db_type
-        if hyperdb.DEBUG:
-            print "_opendb %r.open(%r, %r)"%(db_type, path, mode)
+        if __debug__:
+            print >>hyperdb.DEBUG, "_opendb %r.open(%r, %r)"%(db_type, path,
+                mode)
         return dbm.open(path, mode)
 
     def _lockdb(self, name):
@@ -194,8 +195,8 @@ class Database(FileStorage, hyperdb.Database):
     def addnode(self, classname, nodeid, node):
         ''' add the specified node to its class's db
         '''
-        if hyperdb.DEBUG:
-            print 'addnode', (self, classname, nodeid, node)
+        if __debug__:
+            print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
         self.newnodes.setdefault(classname, {})[nodeid] = 1
         self.cache.setdefault(classname, {})[nodeid] = node
         self.savenode(classname, nodeid, node)
@@ -203,8 +204,8 @@ class Database(FileStorage, hyperdb.Database):
     def setnode(self, classname, nodeid, node):
         ''' change the specified node
         '''
-        if hyperdb.DEBUG:
-            print 'setnode', (self, classname, nodeid, node)
+        if __debug__:
+            print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
         self.dirtynodes.setdefault(classname, {})[nodeid] = 1
 
         # can't set without having already loaded the node
@@ -214,20 +215,26 @@ class Database(FileStorage, hyperdb.Database):
     def savenode(self, classname, nodeid, node):
         ''' perform the saving of data specified by the set/addnode
         '''
-        if hyperdb.DEBUG:
-            print 'savenode', (self, classname, nodeid, node)
+        if __debug__:
+            print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node)
         self.transactions.append((self._doSaveNode, (classname, nodeid, node)))
 
     def getnode(self, classname, nodeid, db=None, cache=1):
         ''' get a node from the database
         '''
-        if hyperdb.DEBUG:
-            print 'getnode', (self, classname, nodeid, db)
+        if __debug__:
+            print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid, db)
         if cache:
             # try the cache
-            cache = self.cache.setdefault(classname, {})
-            if cache.has_key(nodeid):
-                return cache[nodeid]
+            cache_dict = self.cache.setdefault(classname, {})
+            if cache_dict.has_key(nodeid):
+                if __debug__:
+                    print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
+                        nodeid)
+                return cache_dict[nodeid]
+
+        if __debug__:
+            print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid)
 
         # get from the database and save in the cache
         if db is None:
@@ -241,21 +248,26 @@ class Database(FileStorage, hyperdb.Database):
         # reverse the serialisation
         res = self.unserialise(classname, res)
 
-        # store off in the cache
+        # store off in the cache dict
         if cache:
-            cache[nodeid] = res
+            cache_dict[nodeid] = res
 
         return res
 
     def hasnode(self, classname, nodeid, db=None):
         ''' determine if the database has a given node
         '''
-        if hyperdb.DEBUG:
-            print 'hasnode', (self, classname, nodeid, db)
+        if __debug__:
+            print >>hyperdb.DEBUG, 'hasnode', (self, classname, nodeid, db)
+
         # try the cache
         cache = self.cache.setdefault(classname, {})
         if cache.has_key(nodeid):
+            if __debug__:
+                print >>hyperdb.TRACE, 'has %s %s cached'%(classname, nodeid)
             return 1
+        if __debug__:
+            print >>hyperdb.TRACE, 'has %s %s'%(classname, nodeid)
 
         # not in the cache - check the database
         if db is None:
@@ -264,8 +276,8 @@ class Database(FileStorage, hyperdb.Database):
         return res
 
     def countnodes(self, classname, db=None):
-        if hyperdb.DEBUG:
-            print 'countnodes', (self, classname, db)
+        if __debug__:
+            print >>hyperdb.DEBUG, 'countnodes', (self, classname, db)
         # include the new nodes not saved to the DB yet
         count = len(self.newnodes.get(classname, {}))
 
@@ -276,8 +288,8 @@ class Database(FileStorage, hyperdb.Database):
         return count
 
     def getnodeids(self, classname, db=None):
-        if hyperdb.DEBUG:
-            print 'getnodeids', (self, classname, db)
+        if __debug__:
+            print >>hyperdb.DEBUG, 'getnodeids', (self, classname, db)
         # start off with the new nodes
         res = self.newnodes.get(classname, {}).keys()
 
@@ -302,16 +314,17 @@ class Database(FileStorage, hyperdb.Database):
             'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
             'retire' -- 'params' is None
         '''
-        if hyperdb.DEBUG:
-            print 'addjournal', (self, classname, nodeid, action, params)
+        if __debug__:
+            print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid,
+                action, params)
         self.transactions.append((self._doSaveJournal, (classname, nodeid,
             action, params)))
 
     def getjournal(self, classname, nodeid):
         ''' get the journal for id
         '''
-        if hyperdb.DEBUG:
-            print 'getjournal', (self, classname, nodeid)
+        if __debug__:
+            print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid)
         # attempt to open the journal - in some rare cases, the journal may
         # not exist
         try:
@@ -330,8 +343,8 @@ class Database(FileStorage, hyperdb.Database):
 
     def pack(self, pack_before):
         ''' delete all journal entries before 'pack_before' '''
-        if hyperdb.DEBUG:
-            print 'packjournal', (self, pack_before)
+        if __debug__:
+            print >>hyperdb.DEBUG, 'packjournal', (self, pack_before)
 
         pack_before = pack_before.get_tuple()
 
@@ -383,8 +396,8 @@ class Database(FileStorage, hyperdb.Database):
     def commit(self):
         ''' Commit the current transactions.
         '''
-        if hyperdb.DEBUG:
-            print 'commit', (self,)
+        if __debug__:
+            print >>hyperdb.DEBUG, 'commit', (self,)
         # TODO: lock the DB
 
         # keep a handle to all the database files opened
@@ -407,8 +420,9 @@ class Database(FileStorage, hyperdb.Database):
         self.transactions = []
 
     def _doSaveNode(self, classname, nodeid, node):
-        if hyperdb.DEBUG:
-            print '_doSaveNode', (self, classname, nodeid, node)
+        if __debug__:
+            print >>hyperdb.DEBUG, '_doSaveNode', (self, classname, nodeid,
+                node)
 
         # get the database handle
         db_name = 'nodes.%s'%classname
@@ -429,8 +443,8 @@ class Database(FileStorage, hyperdb.Database):
         entry = (nodeid, date.Date().get_tuple(), self.journaltag, action,
             params)
 
-        if hyperdb.DEBUG:
-            print '_doSaveJournal', entry
+        if __debug__:
+            print >>hyperdb.DEBUG, '_doSaveJournal', entry
 
         # get the database handle
         db_name = 'journals.%s'%classname
@@ -457,8 +471,8 @@ class Database(FileStorage, hyperdb.Database):
     def rollback(self):
         ''' Reverse all actions from the current transaction.
         '''
-        if hyperdb.DEBUG:
-            print 'rollback', (self, )
+        if __debug__:
+            print >>hyperdb.DEBUG, 'rollback', (self, )
         for method, args in self.transactions:
             # delete temporary files
             if method == self._doStoreFile:
@@ -471,6 +485,9 @@ class Database(FileStorage, hyperdb.Database):
 
 #
 #$Log: not supported by cvs2svn $
+#Revision 1.33  2002/04/24 10:38:26  rochecompaan
+#All database files are now created group readable and writable.
+#
 #Revision 1.32  2002/04/15 23:25:15  richard
 #. node ids are now generated from a lockable store - no more race conditions
 #
diff --git a/roundup/backends/back_bsddb.py b/roundup/backends/back_bsddb.py
index ea5e1b7..6f8edd7 100644
--- a/roundup/backends/back_bsddb.py
+++ b/roundup/backends/back_bsddb.py
@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-#$Id: back_bsddb.py,v 1.17 2002-04-03 05:54:31 richard Exp $
+#$Id: back_bsddb.py,v 1.18 2002-05-15 06:21:21 richard Exp $
 '''
 This module defines a backend that saves the hyperdatabase in BSDDB.
 '''
@@ -55,18 +55,18 @@ class Database(back_anydbm.Database):
         '''Low-level database opener that gets around anydbm/dbm
            eccentricities.
         '''
-        if hyperdb.DEBUG:
-            print self, '_opendb', (self, name, mode)
+        if __debug__:
+            print >>hyperdb.DEBUG, self, '_opendb', (self, name, mode)
         # determine which DB wrote the class file
         path = os.path.join(os.getcwd(), self.dir, name)
         if not os.path.exists(path):
-            if hyperdb.DEBUG:
-                print "_opendb bsddb.open(%r, 'n')"%path
+            if __debug__:
+                print >>hyperdb.DEBUG, "_opendb bsddb.open(%r, 'n')"%path
             return bsddb.btopen(path, 'n')
 
         # open the database with the correct module
-        if hyperdb.DEBUG:
-            print "_opendb bsddb.open(%r, %r)"%(path, mode)
+        if __debug__:
+            print >>hyperdb.DEBUG, "_opendb bsddb.open(%r, %r)"%(path, mode)
         return bsddb.btopen(path, mode)
 
     #
@@ -102,8 +102,8 @@ class Database(back_anydbm.Database):
         entry = (nodeid, date.Date().get_tuple(), self.journaltag, action,
             params)
 
-        if hyperdb.DEBUG:
-            print '_doSaveJournal', entry
+        if __debug__:
+            print >>hyperdb.DEBUG, '_doSaveJournal', entry
 
         db = bsddb.btopen(os.path.join(self.dir, 'journals.%s'%classname), 'c')
 
@@ -119,6 +119,14 @@ class Database(back_anydbm.Database):
 
 #
 #$Log: not supported by cvs2svn $
+#Revision 1.17  2002/04/03 05:54:31  richard
+#Fixed serialisation problem by moving the serialisation step out of the
+#hyperdb.Class (get, set) into the hyperdb.Database.
+#
+#Also fixed htmltemplate after the showid changes I made yesterday.
+#
+#Unit tests for all of the above written.
+#
 #Revision 1.16  2002/02/27 03:40:59  richard
 #Ran it through pychecker, made fixes
 #
diff --git a/roundup/cgi_client.py b/roundup/cgi_client.py
index 9675bba..9bf6b28 100644
--- a/roundup/cgi_client.py
+++ b/roundup/cgi_client.py
@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-# $Id: cgi_client.py,v 1.118 2002-05-12 23:46:33 richard Exp $
+# $Id: cgi_client.py,v 1.119 2002-05-15 06:21:21 richard Exp $
 
 __doc__ = """
 WWW request handler (also used in the stand-alone server).
@@ -47,6 +47,7 @@ class Client:
     '''
 
     def __init__(self, instance, request, env, form=None):
+        hyperdb.traceMark()
         self.instance = instance
         self.request = request
         self.env = env
@@ -1382,6 +1383,9 @@ def parsePropsFromForm(db, cl, form, nodeid=0):
 
 #
 # $Log: not supported by cvs2svn $
+# Revision 1.118  2002/05/12 23:46:33  richard
+# ehem, part 2
+#
 # Revision 1.117  2002/05/12 23:42:29  richard
 # ehem
 #
diff --git a/roundup/hyperdb.py b/roundup/hyperdb.py
index 51cee62..5761e1d 100644
--- a/roundup/hyperdb.py
+++ b/roundup/hyperdb.py
@@ -15,19 +15,35 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-# $Id: hyperdb.py,v 1.63 2002-04-15 23:25:15 richard Exp $
+# $Id: hyperdb.py,v 1.64 2002-05-15 06:21:21 richard Exp $
 
 __doc__ = """
 Hyperdatabase implementation, especially field types.
 """
 
 # standard python modules
-import re, string, weakref, os
+import re, string, weakref, os, time
 
 # roundup modules
 import date, password
 
+# configure up the DEBUG and TRACE captures
+class Sink:
+    def write(self, content):
+        pass
 DEBUG = os.environ.get('HYPERDBDEBUG', '')
+if DEBUG and __debug__:
+    DEBUG = open(DEBUG, 'a')
+else:
+    DEBUG = Sink()
+TRACE = os.environ.get('HYPERDBTRACE', '')
+if TRACE and __debug__:
+    TRACE = open(TRACE, 'w')
+else:
+    TRACE = Sink()
+def traceMark():
+    print >>TRACE, '**MARK', time.ctime()
+del Sink
 
 #
 # Types
@@ -175,7 +191,8 @@ transaction.
         '''Copy the node contents, converting non-marshallable data into
            marshallable data.
         '''
-        if DEBUG: print 'serialise', classname, node
+        if __debug__:
+            print >>DEBUG, 'serialise', classname, node
         properties = self.getclass(classname).getprops()
         d = {}
         for k, v in node.items():
@@ -206,7 +223,8 @@ transaction.
     def unserialise(self, classname, node):
         '''Decode the marshalled node data
         '''
-        if DEBUG: print 'unserialise', classname, node
+        if __debug__:
+            print >>DEBUG, 'unserialise', classname, node
         properties = self.getclass(classname).getprops()
         d = {}
         for k, v in node.items():
@@ -1127,6 +1145,12 @@ def Choice(name, db, *options):
 
 #
 # $Log: not supported by cvs2svn $
+# Revision 1.63  2002/04/15 23:25:15  richard
+# . node ids are now generated from a lockable store - no more race conditions
+#
+# We're using the portalocker code by Jonathan Feinberg that was contributed
+# to the ASPN Python cookbook. This gives us locking across Unix and Windows.
+#
 # Revision 1.62  2002/04/03 07:05:50  richard
 # d'oh! killed retirement of nodes :(
 # all better now...