From: richard Date: Mon, 23 Jul 2001 08:20:44 +0000 (+0000) Subject: Moved over to using marshal in the bsddb and anydbm backends. X-Git-Url: https://git.tokkee.org/?a=commitdiff_plain;h=34b717cd104a21808c81fe2ab23706c5c49b6eb3;p=roundup.git Moved over to using marshal in the bsddb and anydbm backends. roundup-admin now has a "freshen" command that'll load/save all nodes (not retired - mod hyperdb.Class.list() so it lists retired nodes) git-svn-id: http://svn.roundup-tracker.org/svnroot/roundup/trunk@52 57a73879-2fb5-44c3-a270-3262357dd7e2 --- diff --git a/roundup-admin b/roundup-admin index 1068301..16c38ba 100755 --- a/roundup-admin +++ b/roundup-admin @@ -1,6 +1,6 @@ #! /usr/bin/python -# $Id: roundup-admin,v 1.1 2001-07-23 03:46:48 richard Exp $ +# $Id: roundup-admin,v 1.2 2001-07-23 08:20:44 richard Exp $ import sys if int(sys.version[0]) < 2: @@ -254,6 +254,16 @@ def main(): classname, nodeid = roundupdb.splitDesignator(designator) db.getclass(classname).retire(nodeid) + elif command == 'freshen': + n, db = determineLogin(instance, argv, n) + for classname, cl in db.classes.items(): + properties = cl.properties.keys() + for nodeid in cl.list(): + node = {} + for name in properties: + node[name] = cl.get(nodeid, name) + db.setnode(classname, nodeid, node) + else: print "Unknown command '%s'"%command usage() @@ -267,6 +277,9 @@ if __name__ == '__main__': # # $Log: not supported by cvs2svn $ +# Revision 1.1 2001/07/23 03:46:48 richard +# moving the bin files to facilitate out-of-the-boxness +# # Revision 1.1 2001/07/22 11:15:45 richard # More Grande Splite stuff # diff --git a/roundup/backends/back_anydbm.py b/roundup/backends/back_anydbm.py index 91f93b4..23beeb0 100644 --- a/roundup/backends/back_anydbm.py +++ b/roundup/backends/back_anydbm.py @@ -1,6 +1,6 @@ -#$Id: back_anydbm.py,v 1.1 2001-07-23 07:22:13 richard Exp $ +#$Id: back_anydbm.py,v 1.2 2001-07-23 08:20:44 richard Exp $ -import anydbm, os, cPickle +import anydbm, os, marshal from roundup import hyperdb, date # @@ -68,11 +68,24 @@ class Database(hyperdb.Database): path = os.path.join(os.getcwd(), self.dir, 'nodes.%s'%classname) return anydbm.open(path, mode) + # + # Nodes + # def addnode(self, classname, nodeid, node): ''' add the specified node to its class's db ''' db = self.getclassdb(classname, 'c') - db[nodeid] = cPickle.dumps(node, 1) + + # convert the instance data to builtin types + properties = self.classes[classname].properties + for key in properties.keys(): + if properties[key].isDateType: + node[key] = node[key].get_tuple() + elif properties[key].isIntervalType: + node[key] = node[key].get_tuple() + + # now save the marshalled data + db[nodeid] = marshal.dumps(node) db.close() setnode = addnode @@ -82,7 +95,16 @@ class Database(hyperdb.Database): db = cldb or self.getclassdb(classname) if not db.has_key(nodeid): raise IndexError, nodeid - res = cPickle.loads(db[nodeid]) + res = marshal.loads(db[nodeid]) + + # convert the marshalled data to instances + properties = self.classes[classname].properties + for key in res.keys(): + if properties[key].isDateType: + res[key] = date.Date(res[key]) + elif properties[key].isIntervalType: + res[key] = date.Interval(res[key]) + if not cldb: db.close() return res @@ -117,22 +139,35 @@ class Database(hyperdb.Database): 'link' or 'unlink' -- 'params' is (classname, nodeid, propname) 'retire' -- 'params' is None ''' - entry = (nodeid, date.Date(), self.journaltag, action, params) + entry = (nodeid, date.Date().get_tuple(), self.journaltag, action, + params) db = anydbm.open(os.path.join(self.dir, 'journals.%s'%classname), 'c') if db.has_key(nodeid): s = db[nodeid] - l = cPickle.loads(db[nodeid]) + l = marshal.loads(db[nodeid]) l.append(entry) else: l = [entry] - db[nodeid] = cPickle.dumps(l) + db[nodeid] = marshal.dumps(l) db.close() def getjournal(self, classname, nodeid): ''' get the journal for id ''' - db = anydbm.open(os.path.join(self.dir, 'journals.%s'%classname), 'r') - res = cPickle.loads(db[nodeid]) + # attempt to open the journal - in some rare cases, the journal may + # not exist + try: + db = anydbm.open(os.path.join(self.dir, 'journals.%s'%classname), + 'r') + except anydbm.open, error: + if error.args[0] != 2: raise + return [] + journal = marshal.loads(db[nodeid]) + res = [] + for entry in journal: + (nodeid, date_stamp, self.journaltag, action, params) = entry + date_obj = date.Date(date_stamp) + res.append((nodeid, date_obj, self.journaltag, action, params)) db.close() return res @@ -162,7 +197,4 @@ class Database(hyperdb.Database): # #$Log: not supported by cvs2svn $ -#Revision 1.1 2001/07/23 07:15:57 richard -#Moved the backends into the backends package. Anydbm hasn't been tested at all. -# # diff --git a/roundup/backends/back_bsddb.py b/roundup/backends/back_bsddb.py index 7b4f96e..2c948c6 100644 --- a/roundup/backends/back_bsddb.py +++ b/roundup/backends/back_bsddb.py @@ -1,8 +1,6 @@ -#$Id: back_bsddb.py,v 1.2 2001-07-23 07:56:05 richard Exp $ +#$Id: back_bsddb.py,v 1.3 2001-07-23 08:20:44 richard Exp $ import bsddb, os, marshal -# handle the older cPickle'd data -import cPickle from roundup import hyperdb, date # @@ -77,14 +75,17 @@ class Database(hyperdb.Database): ''' add the specified node to its class's db ''' db = self.getclassdb(classname, 'c') + # convert the instance data to builtin types properties = self.classes[classname].properties - for key in res.keys(): + for key in properties.keys(): if properties[key].isDateType: - res[key] = res[key].get_tuple() + node[key] = node[key].get_tuple() elif properties[key].isIntervalType: - res[key] = res[key].get_tuple() - db[nodeid] = marshal.dumps(node, 1) + node[key] = node[key].get_tuple() + + # now save the marshalled data + db[nodeid] = marshal.dumps(node) db.close() setnode = addnode @@ -94,20 +95,15 @@ class Database(hyperdb.Database): db = cldb or self.getclassdb(classname) if not db.has_key(nodeid): raise IndexError, nodeid - try: - res = marshal.loads(db[nodeid]) - # convert the marshalled data to instances - properties = self.classes[classname].properties - for key in res.keys(): - if properties[key].isDateType: - res[key] = date.Date(res[key]) - elif properties[key].isIntervalType: - res[key] = date.Interval(res[key]) - except ValueError, message: - if str(message) != 'bad marshal data': - raise - # handle the older cPickle'd data - res = cPickle.loads(db[nodeid]) + res = marshal.loads(db[nodeid]) + + # convert the marshalled data to instances + properties = self.classes[classname].properties + for key in res.keys(): + if properties[key].isDateType: + res[key] = date.Date(res[key]) + elif properties[key].isIntervalType: + res[key] = date.Interval(res[key]) if not cldb: db.close() return res @@ -143,7 +139,7 @@ class Database(hyperdb.Database): 'link' or 'unlink' -- 'params' is (classname, nodeid, propname) 'retire' -- 'params' is None ''' - entry = (nodeid, date.Date().journal_tuple(), self.journaltag, action, + entry = (nodeid, date.Date().get_tuple(), self.journaltag, action, params) db = bsddb.btopen(os.path.join(self.dir, 'journals.%s'%classname), 'c') if db.has_key(nodeid): @@ -170,7 +166,7 @@ class Database(hyperdb.Database): res = [] for entry in journal: (nodeid, date_stamp, self.journaltag, action, params) = entry - date_obj = date.Date(set=date_stamp) + date_obj = date.Date(date_stamp) res.append((nodeid, date_obj, self.journaltag, action, params)) db.close() return res @@ -201,6 +197,9 @@ class Database(hyperdb.Database): # #$Log: not supported by cvs2svn $ +#Revision 1.2 2001/07/23 07:56:05 richard +#Storing only marshallable data in the db - no nasty pickled class references. +# #Revision 1.1 2001/07/23 07:22:13 richard #*sigh* some databases have _foo.so as their underlying implementation. #This time for sure, Rocky.