From 781ae10691a77f819f1271aa864c37517c9ed9ef Mon Sep 17 00:00:00 2001 From: richard Date: Wed, 15 May 2002 06:21:21 +0000 Subject: [PATCH] . node caching now works, and gives a small boost in performance As a part of this, I cleaned up the DEBUG output and implemented TRACE output (HYPERDBTRACE='file to trace to') with checkpoints at the start of CGI requests. Run roundup with python -O to skip all the DEBUG/TRACE stuff (using if __debug__ which is compiled out with -O) git-svn-id: http://svn.roundup-tracker.org/svnroot/roundup/trunk@732 57a73879-2fb5-44c3-a270-3262357dd7e2 --- CHANGES.txt | 1 + roundup/backends/back_anydbm.py | 121 ++++++++++++++++++-------------- roundup/backends/back_bsddb.py | 26 ++++--- roundup/cgi_client.py | 6 +- roundup/hyperdb.py | 32 +++++++-- 5 files changed, 120 insertions(+), 66 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index ddfb7c8..bf0b32b 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -42,6 +42,7 @@ Fixed: (thanks dman) . fixed some sorting issues that were breaking some unit tests under py2.2 . mailgw test output dir was confusing the init test (but only on 2.2 *shrug*) + . node caching now works, and gives a small boost in performance 2002-03-25 - 0.4.1 diff --git a/roundup/backends/back_anydbm.py b/roundup/backends/back_anydbm.py index be4bd4e..0ee8fbb 100644 --- a/roundup/backends/back_anydbm.py +++ b/roundup/backends/back_anydbm.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -#$Id: back_anydbm.py,v 1.33 2002-04-24 10:38:26 rochecompaan Exp $ +#$Id: back_anydbm.py,v 1.34 2002-05-15 06:21:21 richard Exp $ ''' This module defines a backend that saves the hyperdatabase in a database chosen by anydbm. It is guaranteed to always be available in python @@ -73,14 +73,14 @@ class Database(FileStorage, hyperdb.Database): def __getattr__(self, classname): """A convenient way of calling self.getclass(classname).""" if self.classes.has_key(classname): - if hyperdb.DEBUG: - print '__getattr__', (self, classname) + if __debug__: + print >>hyperdb.DEBUG, '__getattr__', (self, classname) return self.classes[classname] raise AttributeError, classname def addclass(self, cl): - if hyperdb.DEBUG: - print 'addclass', (self, cl) + if __debug__: + print >>hyperdb.DEBUG, 'addclass', (self, cl) cn = cl.classname if self.classes.has_key(cn): raise ValueError, cn @@ -88,8 +88,8 @@ class Database(FileStorage, hyperdb.Database): def getclasses(self): """Return a list of the names of all existing classes.""" - if hyperdb.DEBUG: - print 'getclasses', (self,) + if __debug__: + print >>hyperdb.DEBUG, 'getclasses', (self,) l = self.classes.keys() l.sort() return l @@ -99,8 +99,8 @@ class Database(FileStorage, hyperdb.Database): If 'classname' is not a valid class name, a KeyError is raised. """ - if hyperdb.DEBUG: - print 'getclass', (self, classname) + if __debug__: + print >>hyperdb.DEBUG, 'getclass', (self, classname) return self.classes[classname] # @@ -109,8 +109,8 @@ class Database(FileStorage, hyperdb.Database): def clear(self): '''Delete all database contents ''' - if hyperdb.DEBUG: - print 'clear', (self,) + if __debug__: + print >>hyperdb.DEBUG, 'clear', (self,) for cn in self.classes.keys(): for dummy in 'nodes', 'journals': path = os.path.join(self.dir, 'journals.%s'%cn) @@ -123,16 +123,16 @@ class Database(FileStorage, hyperdb.Database): ''' grab a connection to the class db that will be used for multiple actions ''' - if hyperdb.DEBUG: - print 'getclassdb', (self, classname, mode) + if __debug__: + print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode) return self._opendb('nodes.%s'%classname, mode) def _opendb(self, name, mode): '''Low-level database opener that gets around anydbm/dbm eccentricities. ''' - if hyperdb.DEBUG: - print '_opendb', (self, name, mode) + if __debug__: + print >>hyperdb.DEBUG, '_opendb', (self, name, mode) # determine which DB wrote the class file db_type = '' @@ -148,8 +148,8 @@ class Database(FileStorage, hyperdb.Database): # new database? let anydbm pick the best dbm if not db_type: - if hyperdb.DEBUG: - print "_opendb anydbm.open(%r, 'n')"%path + if __debug__: + print >>hyperdb.DEBUG, "_opendb anydbm.open(%r, 'n')"%path return anydbm.open(path, 'n') # open the database with the correct module @@ -159,8 +159,9 @@ class Database(FileStorage, hyperdb.Database): raise hyperdb.DatabaseError, \ "Couldn't open database - the required module '%s'"\ "is not available"%db_type - if hyperdb.DEBUG: - print "_opendb %r.open(%r, %r)"%(db_type, path, mode) + if __debug__: + print >>hyperdb.DEBUG, "_opendb %r.open(%r, %r)"%(db_type, path, + mode) return dbm.open(path, mode) def _lockdb(self, name): @@ -194,8 +195,8 @@ class Database(FileStorage, hyperdb.Database): def addnode(self, classname, nodeid, node): ''' add the specified node to its class's db ''' - if hyperdb.DEBUG: - print 'addnode', (self, classname, nodeid, node) + if __debug__: + print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node) self.newnodes.setdefault(classname, {})[nodeid] = 1 self.cache.setdefault(classname, {})[nodeid] = node self.savenode(classname, nodeid, node) @@ -203,8 +204,8 @@ class Database(FileStorage, hyperdb.Database): def setnode(self, classname, nodeid, node): ''' change the specified node ''' - if hyperdb.DEBUG: - print 'setnode', (self, classname, nodeid, node) + if __debug__: + print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node) self.dirtynodes.setdefault(classname, {})[nodeid] = 1 # can't set without having already loaded the node @@ -214,20 +215,26 @@ class Database(FileStorage, hyperdb.Database): def savenode(self, classname, nodeid, node): ''' perform the saving of data specified by the set/addnode ''' - if hyperdb.DEBUG: - print 'savenode', (self, classname, nodeid, node) + if __debug__: + print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node) self.transactions.append((self._doSaveNode, (classname, nodeid, node))) def getnode(self, classname, nodeid, db=None, cache=1): ''' get a node from the database ''' - if hyperdb.DEBUG: - print 'getnode', (self, classname, nodeid, db) + if __debug__: + print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid, db) if cache: # try the cache - cache = self.cache.setdefault(classname, {}) - if cache.has_key(nodeid): - return cache[nodeid] + cache_dict = self.cache.setdefault(classname, {}) + if cache_dict.has_key(nodeid): + if __debug__: + print >>hyperdb.TRACE, 'get %s %s cached'%(classname, + nodeid) + return cache_dict[nodeid] + + if __debug__: + print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid) # get from the database and save in the cache if db is None: @@ -241,21 +248,26 @@ class Database(FileStorage, hyperdb.Database): # reverse the serialisation res = self.unserialise(classname, res) - # store off in the cache + # store off in the cache dict if cache: - cache[nodeid] = res + cache_dict[nodeid] = res return res def hasnode(self, classname, nodeid, db=None): ''' determine if the database has a given node ''' - if hyperdb.DEBUG: - print 'hasnode', (self, classname, nodeid, db) + if __debug__: + print >>hyperdb.DEBUG, 'hasnode', (self, classname, nodeid, db) + # try the cache cache = self.cache.setdefault(classname, {}) if cache.has_key(nodeid): + if __debug__: + print >>hyperdb.TRACE, 'has %s %s cached'%(classname, nodeid) return 1 + if __debug__: + print >>hyperdb.TRACE, 'has %s %s'%(classname, nodeid) # not in the cache - check the database if db is None: @@ -264,8 +276,8 @@ class Database(FileStorage, hyperdb.Database): return res def countnodes(self, classname, db=None): - if hyperdb.DEBUG: - print 'countnodes', (self, classname, db) + if __debug__: + print >>hyperdb.DEBUG, 'countnodes', (self, classname, db) # include the new nodes not saved to the DB yet count = len(self.newnodes.get(classname, {})) @@ -276,8 +288,8 @@ class Database(FileStorage, hyperdb.Database): return count def getnodeids(self, classname, db=None): - if hyperdb.DEBUG: - print 'getnodeids', (self, classname, db) + if __debug__: + print >>hyperdb.DEBUG, 'getnodeids', (self, classname, db) # start off with the new nodes res = self.newnodes.get(classname, {}).keys() @@ -302,16 +314,17 @@ class Database(FileStorage, hyperdb.Database): 'link' or 'unlink' -- 'params' is (classname, nodeid, propname) 'retire' -- 'params' is None ''' - if hyperdb.DEBUG: - print 'addjournal', (self, classname, nodeid, action, params) + if __debug__: + print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid, + action, params) self.transactions.append((self._doSaveJournal, (classname, nodeid, action, params))) def getjournal(self, classname, nodeid): ''' get the journal for id ''' - if hyperdb.DEBUG: - print 'getjournal', (self, classname, nodeid) + if __debug__: + print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid) # attempt to open the journal - in some rare cases, the journal may # not exist try: @@ -330,8 +343,8 @@ class Database(FileStorage, hyperdb.Database): def pack(self, pack_before): ''' delete all journal entries before 'pack_before' ''' - if hyperdb.DEBUG: - print 'packjournal', (self, pack_before) + if __debug__: + print >>hyperdb.DEBUG, 'packjournal', (self, pack_before) pack_before = pack_before.get_tuple() @@ -383,8 +396,8 @@ class Database(FileStorage, hyperdb.Database): def commit(self): ''' Commit the current transactions. ''' - if hyperdb.DEBUG: - print 'commit', (self,) + if __debug__: + print >>hyperdb.DEBUG, 'commit', (self,) # TODO: lock the DB # keep a handle to all the database files opened @@ -407,8 +420,9 @@ class Database(FileStorage, hyperdb.Database): self.transactions = [] def _doSaveNode(self, classname, nodeid, node): - if hyperdb.DEBUG: - print '_doSaveNode', (self, classname, nodeid, node) + if __debug__: + print >>hyperdb.DEBUG, '_doSaveNode', (self, classname, nodeid, + node) # get the database handle db_name = 'nodes.%s'%classname @@ -429,8 +443,8 @@ class Database(FileStorage, hyperdb.Database): entry = (nodeid, date.Date().get_tuple(), self.journaltag, action, params) - if hyperdb.DEBUG: - print '_doSaveJournal', entry + if __debug__: + print >>hyperdb.DEBUG, '_doSaveJournal', entry # get the database handle db_name = 'journals.%s'%classname @@ -457,8 +471,8 @@ class Database(FileStorage, hyperdb.Database): def rollback(self): ''' Reverse all actions from the current transaction. ''' - if hyperdb.DEBUG: - print 'rollback', (self, ) + if __debug__: + print >>hyperdb.DEBUG, 'rollback', (self, ) for method, args in self.transactions: # delete temporary files if method == self._doStoreFile: @@ -471,6 +485,9 @@ class Database(FileStorage, hyperdb.Database): # #$Log: not supported by cvs2svn $ +#Revision 1.33 2002/04/24 10:38:26 rochecompaan +#All database files are now created group readable and writable. +# #Revision 1.32 2002/04/15 23:25:15 richard #. node ids are now generated from a lockable store - no more race conditions # diff --git a/roundup/backends/back_bsddb.py b/roundup/backends/back_bsddb.py index ea5e1b7..6f8edd7 100644 --- a/roundup/backends/back_bsddb.py +++ b/roundup/backends/back_bsddb.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -#$Id: back_bsddb.py,v 1.17 2002-04-03 05:54:31 richard Exp $ +#$Id: back_bsddb.py,v 1.18 2002-05-15 06:21:21 richard Exp $ ''' This module defines a backend that saves the hyperdatabase in BSDDB. ''' @@ -55,18 +55,18 @@ class Database(back_anydbm.Database): '''Low-level database opener that gets around anydbm/dbm eccentricities. ''' - if hyperdb.DEBUG: - print self, '_opendb', (self, name, mode) + if __debug__: + print >>hyperdb.DEBUG, self, '_opendb', (self, name, mode) # determine which DB wrote the class file path = os.path.join(os.getcwd(), self.dir, name) if not os.path.exists(path): - if hyperdb.DEBUG: - print "_opendb bsddb.open(%r, 'n')"%path + if __debug__: + print >>hyperdb.DEBUG, "_opendb bsddb.open(%r, 'n')"%path return bsddb.btopen(path, 'n') # open the database with the correct module - if hyperdb.DEBUG: - print "_opendb bsddb.open(%r, %r)"%(path, mode) + if __debug__: + print >>hyperdb.DEBUG, "_opendb bsddb.open(%r, %r)"%(path, mode) return bsddb.btopen(path, mode) # @@ -102,8 +102,8 @@ class Database(back_anydbm.Database): entry = (nodeid, date.Date().get_tuple(), self.journaltag, action, params) - if hyperdb.DEBUG: - print '_doSaveJournal', entry + if __debug__: + print >>hyperdb.DEBUG, '_doSaveJournal', entry db = bsddb.btopen(os.path.join(self.dir, 'journals.%s'%classname), 'c') @@ -119,6 +119,14 @@ class Database(back_anydbm.Database): # #$Log: not supported by cvs2svn $ +#Revision 1.17 2002/04/03 05:54:31 richard +#Fixed serialisation problem by moving the serialisation step out of the +#hyperdb.Class (get, set) into the hyperdb.Database. +# +#Also fixed htmltemplate after the showid changes I made yesterday. +# +#Unit tests for all of the above written. +# #Revision 1.16 2002/02/27 03:40:59 richard #Ran it through pychecker, made fixes # diff --git a/roundup/cgi_client.py b/roundup/cgi_client.py index 9675bba..9bf6b28 100644 --- a/roundup/cgi_client.py +++ b/roundup/cgi_client.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: cgi_client.py,v 1.118 2002-05-12 23:46:33 richard Exp $ +# $Id: cgi_client.py,v 1.119 2002-05-15 06:21:21 richard Exp $ __doc__ = """ WWW request handler (also used in the stand-alone server). @@ -47,6 +47,7 @@ class Client: ''' def __init__(self, instance, request, env, form=None): + hyperdb.traceMark() self.instance = instance self.request = request self.env = env @@ -1382,6 +1383,9 @@ def parsePropsFromForm(db, cl, form, nodeid=0): # # $Log: not supported by cvs2svn $ +# Revision 1.118 2002/05/12 23:46:33 richard +# ehem, part 2 +# # Revision 1.117 2002/05/12 23:42:29 richard # ehem # diff --git a/roundup/hyperdb.py b/roundup/hyperdb.py index 51cee62..5761e1d 100644 --- a/roundup/hyperdb.py +++ b/roundup/hyperdb.py @@ -15,19 +15,35 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: hyperdb.py,v 1.63 2002-04-15 23:25:15 richard Exp $ +# $Id: hyperdb.py,v 1.64 2002-05-15 06:21:21 richard Exp $ __doc__ = """ Hyperdatabase implementation, especially field types. """ # standard python modules -import re, string, weakref, os +import re, string, weakref, os, time # roundup modules import date, password +# configure up the DEBUG and TRACE captures +class Sink: + def write(self, content): + pass DEBUG = os.environ.get('HYPERDBDEBUG', '') +if DEBUG and __debug__: + DEBUG = open(DEBUG, 'a') +else: + DEBUG = Sink() +TRACE = os.environ.get('HYPERDBTRACE', '') +if TRACE and __debug__: + TRACE = open(TRACE, 'w') +else: + TRACE = Sink() +def traceMark(): + print >>TRACE, '**MARK', time.ctime() +del Sink # # Types @@ -175,7 +191,8 @@ transaction. '''Copy the node contents, converting non-marshallable data into marshallable data. ''' - if DEBUG: print 'serialise', classname, node + if __debug__: + print >>DEBUG, 'serialise', classname, node properties = self.getclass(classname).getprops() d = {} for k, v in node.items(): @@ -206,7 +223,8 @@ transaction. def unserialise(self, classname, node): '''Decode the marshalled node data ''' - if DEBUG: print 'unserialise', classname, node + if __debug__: + print >>DEBUG, 'unserialise', classname, node properties = self.getclass(classname).getprops() d = {} for k, v in node.items(): @@ -1127,6 +1145,12 @@ def Choice(name, db, *options): # # $Log: not supported by cvs2svn $ +# Revision 1.63 2002/04/15 23:25:15 richard +# . node ids are now generated from a lockable store - no more race conditions +# +# We're using the portalocker code by Jonathan Feinberg that was contributed +# to the ASPN Python cookbook. This gives us locking across Unix and Windows. +# # Revision 1.62 2002/04/03 07:05:50 richard # d'oh! killed retirement of nodes :( # all better now... -- 2.30.2