X-Git-Url: https://git.tokkee.org/?a=blobdiff_plain;ds=sidebyside;f=roundup%2Fbackends%2Fback_anydbm.py;h=5f30b1ad6cffb3d2511a2d17070c84cdce4a9dd1;hb=e4c89796238027912e80acdab4c8c7a67fc21a9e;hp=b6226aac7de490ad1ef17b067a53638f0bf0ef9a;hpb=52c4073ff97774929896dfc1c347c42c78f46dbb;p=roundup.git diff --git a/roundup/backends/back_anydbm.py b/roundup/backends/back_anydbm.py index b6226aa..5f30b1a 100644 --- a/roundup/backends/back_anydbm.py +++ b/roundup/backends/back_anydbm.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -#$Id: back_anydbm.py,v 1.20 2001-12-18 15:30:34 rochecompaan Exp $ +#$Id: back_anydbm.py,v 1.28 2002-02-16 09:14:17 richard Exp $ ''' This module defines a backend that saves the hyperdatabase in a database chosen by anydbm. It is guaranteed to always be available in python @@ -26,8 +26,6 @@ serious bugs, and is not available) import whichdb, anydbm, os, marshal from roundup import hyperdb, date, password -DEBUG=os.environ.get('HYPERDBDEBUG', '') - # # Now the database # @@ -40,9 +38,10 @@ class Database(hyperdb.Database): . perhaps detect write collisions (related to above)? """ - def __init__(self, storagelocator, journaltag=None): + def __init__(self, config, journaltag=None): """Open a hyperdatabase given a specifier to some storage. + The 'storagelocator' is obtained from config.DATABASE. The meaning of 'storagelocator' depends on the particular implementation of the hyperdatabase. It could be a file name, a directory path, a socket descriptor for a connection to a @@ -53,7 +52,8 @@ class Database(hyperdb.Database): None, the database is opened in read-only mode: the Class.create(), Class.set(), and Class.retire() methods are disabled. """ - self.dir, self.journaltag = storagelocator, journaltag + self.config, self.journaltag = config, journaltag + self.dir = config.DATABASE self.classes = {} self.cache = {} # cache of nodes loaded or created self.dirtynodes = {} # keep track of the dirty nodes by class @@ -69,13 +69,13 @@ class Database(hyperdb.Database): def __getattr__(self, classname): """A convenient way of calling self.getclass(classname).""" if self.classes.has_key(classname): - if DEBUG: + if hyperdb.DEBUG: print '__getattr__', (self, classname) return self.classes[classname] raise AttributeError, classname def addclass(self, cl): - if DEBUG: + if hyperdb.DEBUG: print 'addclass', (self, cl) cn = cl.classname if self.classes.has_key(cn): @@ -84,7 +84,7 @@ class Database(hyperdb.Database): def getclasses(self): """Return a list of the names of all existing classes.""" - if DEBUG: + if hyperdb.DEBUG: print 'getclasses', (self,) l = self.classes.keys() l.sort() @@ -95,7 +95,7 @@ class Database(hyperdb.Database): If 'classname' is not a valid class name, a KeyError is raised. """ - if DEBUG: + if hyperdb.DEBUG: print 'getclass', (self, classname) return self.classes[classname] @@ -105,7 +105,7 @@ class Database(hyperdb.Database): def clear(self): '''Delete all database contents ''' - if DEBUG: + if hyperdb.DEBUG: print 'clear', (self,) for cn in self.classes.keys(): for type in 'nodes', 'journals': @@ -119,7 +119,7 @@ class Database(hyperdb.Database): ''' grab a connection to the class db that will be used for multiple actions ''' - if DEBUG: + if hyperdb.DEBUG: print 'getclassdb', (self, classname, mode) return self._opendb('nodes.%s'%classname, mode) @@ -127,7 +127,7 @@ class Database(hyperdb.Database): '''Low-level database opener that gets around anydbm/dbm eccentricities. ''' - if DEBUG: + if hyperdb.DEBUG: print '_opendb', (self, name, mode) # determine which DB wrote the class file db_type = '' @@ -143,7 +143,7 @@ class Database(hyperdb.Database): # new database? let anydbm pick the best dbm if not db_type: - if DEBUG: + if hyperdb.DEBUG: print "_opendb anydbm.open(%r, 'n')"%path return anydbm.open(path, 'n') @@ -154,7 +154,7 @@ class Database(hyperdb.Database): raise hyperdb.DatabaseError, \ "Couldn't open database - the required module '%s'"\ "is not available"%db_type - if DEBUG: + if hyperdb.DEBUG: print "_opendb %r.open(%r, %r)"%(db_type, path, mode) return dbm.open(path, mode) @@ -164,7 +164,7 @@ class Database(hyperdb.Database): def addnode(self, classname, nodeid, node): ''' add the specified node to its class's db ''' - if DEBUG: + if hyperdb.DEBUG: print 'addnode', (self, classname, nodeid, node) self.newnodes.setdefault(classname, {})[nodeid] = 1 self.cache.setdefault(classname, {})[nodeid] = node @@ -173,7 +173,7 @@ class Database(hyperdb.Database): def setnode(self, classname, nodeid, node): ''' change the specified node ''' - if DEBUG: + if hyperdb.DEBUG: print 'setnode', (self, classname, nodeid, node) self.dirtynodes.setdefault(classname, {})[nodeid] = 1 # can't set without having already loaded the node @@ -183,34 +183,36 @@ class Database(hyperdb.Database): def savenode(self, classname, nodeid, node): ''' perform the saving of data specified by the set/addnode ''' - if DEBUG: + if hyperdb.DEBUG: print 'savenode', (self, classname, nodeid, node) self.transactions.append((self._doSaveNode, (classname, nodeid, node))) - def getnode(self, classname, nodeid, db=None): + def getnode(self, classname, nodeid, db=None, cache=1): ''' get a node from the database ''' - if DEBUG: - print 'getnode', (self, classname, nodeid, cldb) - # try the cache - cache = self.cache.setdefault(classname, {}) - if cache.has_key(nodeid): - return cache[nodeid] + if hyperdb.DEBUG: + print 'getnode', (self, classname, nodeid, db) + if cache: + # try the cache + cache = self.cache.setdefault(classname, {}) + if cache.has_key(nodeid): + return cache[nodeid] # get from the database and save in the cache if db is None: db = self.getclassdb(classname) if not db.has_key(nodeid): - raise IndexError, nodeid + raise IndexError, "no such %s %s"%(classname, nodeid) res = marshal.loads(db[nodeid]) - cache[nodeid] = res + if cache: + cache[nodeid] = res return res def hasnode(self, classname, nodeid, db=None): ''' determine if the database has a given node ''' - if DEBUG: - print 'hasnode', (self, classname, nodeid, cldb) + if hyperdb.DEBUG: + print 'hasnode', (self, classname, nodeid, db) # try the cache cache = self.cache.setdefault(classname, {}) if cache.has_key(nodeid): @@ -223,8 +225,8 @@ class Database(hyperdb.Database): return res def countnodes(self, classname, db=None): - if DEBUG: - print 'countnodes', (self, classname, cldb) + if hyperdb.DEBUG: + print 'countnodes', (self, classname, db) # include the new nodes not saved to the DB yet count = len(self.newnodes.get(classname, {})) @@ -235,7 +237,7 @@ class Database(hyperdb.Database): return count def getnodeids(self, classname, db=None): - if DEBUG: + if hyperdb.DEBUG: print 'getnodeids', (self, classname, db) # start off with the new nodes res = self.newnodes.get(classname, {}).keys() @@ -290,7 +292,7 @@ class Database(hyperdb.Database): 'link' or 'unlink' -- 'params' is (classname, nodeid, propname) 'retire' -- 'params' is None ''' - if DEBUG: + if hyperdb.DEBUG: print 'addjournal', (self, classname, nodeid, action, params) self.transactions.append((self._doSaveJournal, (classname, nodeid, action, params))) @@ -298,7 +300,7 @@ class Database(hyperdb.Database): def getjournal(self, classname, nodeid): ''' get the journal for id ''' - if DEBUG: + if hyperdb.DEBUG: print 'getjournal', (self, classname, nodeid) # attempt to open the journal - in some rare cases, the journal may # not exist @@ -311,11 +313,59 @@ class Database(hyperdb.Database): journal = marshal.loads(db[nodeid]) res = [] for entry in journal: - (nodeid, date_stamp, self.journaltag, action, params) = entry + (nodeid, date_stamp, user, action, params) = entry date_obj = date.Date(date_stamp) - res.append((nodeid, date_obj, self.journaltag, action, params)) + res.append((nodeid, date_obj, user, action, params)) return res + def pack(self, pack_before): + ''' delete all journal entries before 'pack_before' ''' + if hyperdb.DEBUG: + print 'packjournal', (self, pack_before) + + pack_before = pack_before.get_tuple() + + classes = self.getclasses() + + # TODO: factor this out to method - we're already doing it in + # _opendb. + db_type = '' + path = os.path.join(os.getcwd(), self.dir, classes[0]) + if os.path.exists(path): + db_type = whichdb.whichdb(path) + if not db_type: + raise hyperdb.DatabaseError, "Couldn't identify database type" + elif os.path.exists(path+'.db'): + db_type = 'dbm' + + for classname in classes: + db_name = 'journals.%s'%classname + db = self._opendb(db_name, 'w') + + for key in db.keys(): + journal = marshal.loads(db[key]) + l = [] + last_set_entry = None + for entry in journal: + (nodeid, date_stamp, self.journaltag, action, + params) = entry + if date_stamp > pack_before or action == 'create': + l.append(entry) + elif action == 'set': + # grab the last set entry to keep information on + # activity + last_set_entry = entry + if last_set_entry: + date_stamp = last_set_entry[1] + # if the last set entry was made after the pack date + # then it is already in the list + if date_stamp < pack_before: + l.append(last_set_entry) + db[key] = marshal.dumps(l) + if db_type == 'gdbm': + db.reorganize() + db.close() + # # Basic transaction support @@ -323,7 +373,7 @@ class Database(hyperdb.Database): def commit(self): ''' Commit the current transactions. ''' - if DEBUG: + if hyperdb.DEBUG: print 'commit', (self,) # TODO: lock the DB @@ -347,7 +397,7 @@ class Database(hyperdb.Database): self.transactions = [] def _doSaveNode(self, classname, nodeid, node): - if DEBUG: + if hyperdb.DEBUG: print '_doSaveNode', (self, classname, nodeid, node) # get the database handle @@ -361,10 +411,10 @@ class Database(hyperdb.Database): db[nodeid] = marshal.dumps(node) def _doSaveJournal(self, classname, nodeid, action, params): - if DEBUG: - print '_doSaveJournal', (self, classname, nodeid, action, params) entry = (nodeid, date.Date().get_tuple(), self.journaltag, action, params) + if hyperdb.DEBUG: + print '_doSaveJournal', entry # get the database handle db_name = 'journals.%s'%classname @@ -389,12 +439,13 @@ class Database(hyperdb.Database): def rollback(self): ''' Reverse all actions from the current transaction. ''' - if DEBUG: + if hyperdb.DEBUG: print 'rollback', (self, ) for method, args in self.transactions: # delete temporary files if method == self._doStoreFile: - os.remove(args[0]+".tmp") + if os.path.exists(args[0]+".tmp"): + os.remove(args[0]+".tmp") self.cache = {} self.dirtynodes = {} self.newnodes = {} @@ -402,6 +453,61 @@ class Database(hyperdb.Database): # #$Log: not supported by cvs2svn $ +#Revision 1.27 2002/01/22 07:21:13 richard +#. fixed back_bsddb so it passed the journal tests +# +#... it didn't seem happy using the back_anydbm _open method, which is odd. +#Yet another occurrance of whichdb not being able to recognise older bsddb +#databases. Yadda yadda. Made the HYPERDBDEBUG stuff more sane in the +#process. +# +#Revision 1.26 2002/01/22 05:18:38 rochecompaan +#last_set_entry was referenced before assignment +# +#Revision 1.25 2002/01/22 05:06:08 rochecompaan +#We need to keep the last 'set' entry in the journal to preserve +#information on 'activity' for nodes. +# +#Revision 1.24 2002/01/21 16:33:20 rochecompaan +#You can now use the roundup-admin tool to pack the database +# +#Revision 1.23 2002/01/18 04:32:04 richard +#Rollback was breaking because a message hadn't actually been written to the file. Needs +#more investigation. +# +#Revision 1.22 2002/01/14 02:20:15 richard +# . changed all config accesses so they access either the instance or the +# config attriubute on the db. This means that all config is obtained from +# instance_config instead of the mish-mash of classes. This will make +# switching to a ConfigParser setup easier too, I hope. +# +#At a minimum, this makes migration a _little_ easier (a lot easier in the +#0.5.0 switch, I hope!) +# +#Revision 1.21 2002/01/02 02:31:38 richard +#Sorry for the huge checkin message - I was only intending to implement #496356 +#but I found a number of places where things had been broken by transactions: +# . modified ROUNDUPDBSENDMAILDEBUG to be SENDMAILDEBUG and hold a filename +# for _all_ roundup-generated smtp messages to be sent to. +# . the transaction cache had broken the roundupdb.Class set() reactors +# . newly-created author users in the mailgw weren't being committed to the db +# +#Stuff that made it into CHANGES.txt (ie. the stuff I was actually working +#on when I found that stuff :): +# . #496356 ] Use threading in messages +# . detectors were being registered multiple times +# . added tests for mailgw +# . much better attaching of erroneous messages in the mail gateway +# +#Revision 1.20 2001/12/18 15:30:34 rochecompaan +#Fixed bugs: +# . Fixed file creation and retrieval in same transaction in anydbm +# backend +# . Cgi interface now renders new issue after issue creation +# . Could not set issue status to resolved through cgi interface +# . Mail gateway was changing status back to 'chatting' if status was +# omitted as an argument +# #Revision 1.19 2001/12/17 03:52:48 richard #Implemented file store rollback. As a bonus, the hyperdb is now capable of #storing more than one file per node - if a property name is supplied,