From 8bcea6bf45dbbaa8d48c7ef104264f67f6d9e774 Mon Sep 17 00:00:00 2001 From: richard Date: Tue, 21 May 2002 05:52:11 +0000 Subject: [PATCH] Well whadya know, bsddb3 works again. The backend is implemented _exactly_ the same as bsddb - so there's no using its transaction or locking support. It'd be nice to use those some day I suppose. git-svn-id: http://svn.roundup-tracker.org/svnroot/roundup/trunk@739 57a73879-2fb5-44c3-a270-3262357dd7e2 --- CHANGES.txt | 2 +- roundup/backends/back_bsddb3.py | 189 ++++++++++---------------------- test/test_db.py | 20 ++-- 3 files changed, 69 insertions(+), 142 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 5a42844..b20519b 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -44,7 +44,7 @@ Fixed: . fixed some sorting issues that were breaking some unit tests under py2.2 . mailgw test output dir was confusing the init test (but only on 2.2 *shrug*) . node caching now works, and gives a small boost in performance - + . bsddb3 backend now works, reinstating 2002-03-25 - 0.4.1 Feature: diff --git a/roundup/backends/back_bsddb3.py b/roundup/backends/back_bsddb3.py index 5892a2c..821203b 100644 --- a/roundup/backends/back_bsddb3.py +++ b/roundup/backends/back_bsddb3.py @@ -15,70 +15,28 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -#$Id: back_bsddb3.py,v 1.11 2002-01-14 02:20:15 richard Exp $ +#$Id: back_bsddb3.py,v 1.12 2002-05-21 05:52:11 richard Exp $ import bsddb3, os, marshal -from roundup import hyperdb, date, password +from roundup import hyperdb, date + +# these classes are so similar, we just use the anydbm methods +import back_anydbm # # Now the database # -class Database(hyperdb.Database): +class Database(back_anydbm.Database): """A database for storing records containing flexible data types.""" - - def __init__(self, config, journaltag=None): - """Open a hyperdatabase given a specifier to some storage. - - The 'storagelocator' is obtained from config.DATABASE. - The meaning of 'storagelocator' depends on the particular - implementation of the hyperdatabase. It could be a file name, - a directory path, a socket descriptor for a connection to a - database over the network, etc. - - The 'journaltag' is a token that will be attached to the journal - entries for any edits done on the database. If 'journaltag' is - None, the database is opened in read-only mode: the Class.create(), - Class.set(), and Class.retire() methods are disabled. - """ - self.config, self.journaltag = config, journaltag - self.dir = config.DATABASE - self.classes = {} - - # - # Classes - # - def __getattr__(self, classname): - """A convenient way of calling self.getclass(classname).""" - return self.classes[classname] - - def addclass(self, cl): - cn = cl.classname - if self.classes.has_key(cn): - raise ValueError, cn - self.classes[cn] = cl - - def getclasses(self): - """Return a list of the names of all existing classes.""" - l = self.classes.keys() - l.sort() - return l - - def getclass(self, classname): - """Get the Class object representing a particular class. - - If 'classname' is not a valid class name, a KeyError is raised. - """ - return self.classes[classname] - # # Class DBs # def clear(self): for cn in self.classes.keys(): db = os.path.join(self.dir, 'nodes.%s'%cn) - bsddb3.btopen(db, 'c') + bsddb3.btopen(db, 'n') db = os.path.join(self.dir, 'journals.%s'%cn) - bsddb3.btopen(db, 'c') + bsddb3.btopen(db, 'n') def getclassdb(self, classname, mode='r'): ''' grab a connection to the class db that will be used for @@ -88,73 +46,29 @@ class Database(hyperdb.Database): if os.path.exists(path): return bsddb3.btopen(path, mode) else: - return bsddb3.btopen(path, 'c') + return bsddb3.btopen(path, 'n') - # - # Nodes - # - def addnode(self, classname, nodeid, node): - ''' add the specified node to its class's db + def _opendb(self, name, mode): + '''Low-level database opener that gets around anydbm/dbm + eccentricities. ''' - db = self.getclassdb(classname, 'c') - # now save the marshalled data - db[nodeid] = marshal.dumps(node) - db.close() - setnode = addnode - - def getnode(self, classname, nodeid, cldb=None): - ''' add the specified node to its class's db - ''' - db = cldb or self.getclassdb(classname) - if not db.has_key(nodeid): - raise IndexError, nodeid - res = marshal.loads(db[nodeid]) - if not cldb: db.close() - return res - - def hasnode(self, classname, nodeid, cldb=None): - ''' add the specified node to its class's db - ''' - db = cldb or self.getclassdb(classname) - res = db.has_key(nodeid) - if not cldb: db.close() - return res - - def countnodes(self, classname, cldb=None): - db = cldb or self.getclassdb(classname) - return len(db.keys()) - if not cldb: db.close() - return res - - def getnodeids(self, classname, cldb=None): - db = cldb or self.getclassdb(classname) - res = db.keys() - if not cldb: db.close() - return res + if __debug__: + print >>hyperdb.DEBUG, self, '_opendb', (self, name, mode) + # determine which DB wrote the class file + path = os.path.join(os.getcwd(), self.dir, name) + if not os.path.exists(path): + if __debug__: + print >>hyperdb.DEBUG, "_opendb bsddb3.open(%r, 'n')"%path + return bsddb3.btopen(path, 'n') + + # open the database with the correct module + if __debug__: + print >>hyperdb.DEBUG, "_opendb bsddb3.open(%r, %r)"%(path, mode) + return bsddb3.btopen(path, mode) # # Journal # - def addjournal(self, classname, nodeid, action, params): - ''' Journal the Action - 'action' may be: - - 'create' or 'set' -- 'params' is a dictionary of property values - 'link' or 'unlink' -- 'params' is (classname, nodeid, propname) - 'retire' -- 'params' is None - ''' - entry = (nodeid, date.Date().get_tuple(), self.journaltag, action, - params) - db = bsddb3.btopen(os.path.join(self.dir, 'journals.%s'%classname), 'c') - if db.has_key(nodeid): - s = db[nodeid] - l = marshal.loads(db[nodeid]) - l.append(entry) - else: - l = [entry] - db[nodeid] = marshal.dumps(l) - db.close() - def getjournal(self, classname, nodeid): ''' get the journal for id ''' @@ -163,46 +77,53 @@ class Database(hyperdb.Database): try: db = bsddb3.btopen(os.path.join(self.dir, 'journals.%s'%classname), 'r') - except bsddb3.error, error: - if error.args[0] != 2: raise + except bsddb3.NoSuchFileError: return [] # mor handling of bad journals if not db.has_key(nodeid): return [] journal = marshal.loads(db[nodeid]) res = [] for entry in journal: - (nodeid, date_stamp, self.journaltag, action, params) = entry + (nodeid, date_stamp, user, action, params) = entry date_obj = date.Date(date_stamp) - res.append((nodeid, date_obj, self.journaltag, action, params)) + res.append((nodeid, date_obj, user, action, params)) db.close() return res - def close(self): - ''' Close the Database - we must release the circular refs so that - we can be del'ed and the underlying bsddb connections closed - cleanly. - ''' - self.classes = {} + def _doSaveJournal(self, classname, nodeid, action, params): + # serialise first + if action in ('set', 'create'): + params = self.serialise(classname, params) + entry = (nodeid, date.Date().get_tuple(), self.journaltag, action, + params) - # - # Basic transaction support - # - # TODO: well, write these methods (and then use them in other code) - def register_action(self): - ''' Register an action to the transaction undo log - ''' + if __debug__: + print >>hyperdb.DEBUG, '_doSaveJournal', entry - def commit(self): - ''' Commit the current transaction, start a new one - ''' + db = bsddb3.btopen(os.path.join(self.dir, 'journals.%s'%classname), 'c') - def rollback(self): - ''' Reverse all actions from the current transaction - ''' + if db.has_key(nodeid): + s = db[nodeid] + l = marshal.loads(s) + l.append(entry) + else: + l = [entry] + + db[nodeid] = marshal.dumps(l) + db.close() # #$Log: not supported by cvs2svn $ +#Revision 1.11 2002/01/14 02:20:15 richard +# . changed all config accesses so they access either the instance or the +# config attriubute on the db. This means that all config is obtained from +# instance_config instead of the mish-mash of classes. This will make +# switching to a ConfigParser setup easier too, I hope. +# +#At a minimum, this makes migration a _little_ easier (a lot easier in the +#0.5.0 switch, I hope!) +# #Revision 1.10 2001/11/21 02:34:18 richard #Added a target version field to the extended issue schema # diff --git a/test/test_db.py b/test/test_db.py index b09a44c..670d3ac 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: test_db.py,v 1.21 2002-04-15 23:25:15 richard Exp $ +# $Id: test_db.py,v 1.22 2002-05-21 05:52:11 richard Exp $ import unittest, os, shutil @@ -366,17 +366,23 @@ def suite(): except: print 'bsddb module not found, skipping bsddb DBTestCase' -# try: -# import bsddb3 -# l.append(unittest.makeSuite(bsddb3DBTestCase, 'test')) -# l.append(unittest.makeSuite(bsddb3ReadOnlyDBTestCase, 'test')) -# except: -# print 'bsddb3 module not found, skipping bsddb3 DBTestCase' + try: + import bsddb3 + l.append(unittest.makeSuite(bsddb3DBTestCase, 'test')) + l.append(unittest.makeSuite(bsddb3ReadOnlyDBTestCase, 'test')) + except: + print 'bsddb3 module not found, skipping bsddb3 DBTestCase' return unittest.TestSuite(l) # # $Log: not supported by cvs2svn $ +# Revision 1.21 2002/04/15 23:25:15 richard +# . node ids are now generated from a lockable store - no more race conditions +# +# We're using the portalocker code by Jonathan Feinberg that was contributed +# to the ASPN Python cookbook. This gives us locking across Unix and Windows. +# # Revision 1.20 2002/04/03 05:54:31 richard # Fixed serialisation problem by moving the serialisation step out of the # hyperdb.Class (get, set) into the hyperdb.Database. -- 2.30.2