From 34d0f9a7351e474939203bb06ca98f100625b2b2 Mon Sep 17 00:00:00 2001 From: richard Date: Wed, 3 Apr 2002 05:54:31 +0000 Subject: [PATCH] Fixed serialisation problem by moving the serialisation step out of the hyperdb.Class (get, set) into the hyperdb.Database. Also fixed htmltemplate after the showid changes I made yesterday. Unit tests for all of the above written. git-svn-id: http://svn.roundup-tracker.org/svnroot/roundup/trunk@690 57a73879-2fb5-44c3-a270-3262357dd7e2 --- CHANGES.txt | 3 +- roundup/backends/back_anydbm.py | 24 ++++++++++- roundup/backends/back_bsddb.py | 15 ++++++- roundup/htmltemplate.py | 16 +++++-- roundup/hyperdb.py | 76 ++++++++++++++++++++------------- test/test_db.py | 29 +++++++++++-- test/test_htmltemplate.py | 14 +++++- 7 files changed, 135 insertions(+), 42 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 1f3b81b..fe640b1 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,7 +1,7 @@ This file contains the changes to the Roundup system over time. The entries are given with the most recent entry first. -0.4.2 +2002-04-?? 0.4.2 Feature: . link() htmltemplate function now has a "showid" option for links and multilinks. When true, it only displays the linked node id as the anchor @@ -18,6 +18,7 @@ Feature: Fixed: . stop sending blank (whitespace-only) notes + . cleanup of serialisation for database storage 2002-03-25 - 0.4.1 diff --git a/roundup/backends/back_anydbm.py b/roundup/backends/back_anydbm.py index df287d8..65607c8 100644 --- a/roundup/backends/back_anydbm.py +++ b/roundup/backends/back_anydbm.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -#$Id: back_anydbm.py,v 1.30 2002-02-27 03:40:59 richard Exp $ +#$Id: back_anydbm.py,v 1.31 2002-04-03 05:54:31 richard Exp $ ''' This module defines a backend that saves the hyperdatabase in a database chosen by anydbm. It is guaranteed to always be available in python @@ -177,6 +177,7 @@ class Database(FileStorage, hyperdb.Database): if hyperdb.DEBUG: print 'setnode', (self, classname, nodeid, node) self.dirtynodes.setdefault(classname, {})[nodeid] = 1 + # can't set without having already loaded the node self.cache[classname][nodeid] = node self.savenode(classname, nodeid, node) @@ -204,9 +205,17 @@ class Database(FileStorage, hyperdb.Database): db = self.getclassdb(classname) if not db.has_key(nodeid): raise IndexError, "no such %s %s"%(classname, nodeid) + + # decode res = marshal.loads(db[nodeid]) + + # reverse the serialisation + res = self.unserialise(classname, res) + + # store off in the cache if cache: cache[nodeid] = res + return res def hasnode(self, classname, nodeid, db=None): @@ -380,11 +389,17 @@ class Database(FileStorage, hyperdb.Database): db = self.databases[db_name] = self.getclassdb(classname, 'c') # now save the marshalled data - db[nodeid] = marshal.dumps(node) + db[nodeid] = marshal.dumps(self.serialise(classname, node)) def _doSaveJournal(self, classname, nodeid, action, params): + # serialise first + if action in ('set', 'create'): + params = self.serialise(classname, params) + + # create the journal entry entry = (nodeid, date.Date().get_tuple(), self.journaltag, action, params) + if hyperdb.DEBUG: print '_doSaveJournal', entry @@ -397,11 +412,13 @@ class Database(FileStorage, hyperdb.Database): # now insert the journal entry if db.has_key(nodeid): + # append to existing s = db[nodeid] l = marshal.loads(s) l.append(entry) else: l = [entry] + db[nodeid] = marshal.dumps(l) def _doStoreFile(self, name, **databases): @@ -425,6 +442,9 @@ class Database(FileStorage, hyperdb.Database): # #$Log: not supported by cvs2svn $ +#Revision 1.30 2002/02/27 03:40:59 richard +#Ran it through pychecker, made fixes +# #Revision 1.29 2002/02/25 14:34:31 grubert # . use blobfiles in back_anydbm which is used in back_bsddb. # change test_db as dirlist does not work for subdirectories. diff --git a/roundup/backends/back_bsddb.py b/roundup/backends/back_bsddb.py index 973e7bc..ea5e1b7 100644 --- a/roundup/backends/back_bsddb.py +++ b/roundup/backends/back_bsddb.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -#$Id: back_bsddb.py,v 1.16 2002-02-27 03:40:59 richard Exp $ +#$Id: back_bsddb.py,v 1.17 2002-04-03 05:54:31 richard Exp $ ''' This module defines a backend that saves the hyperdatabase in BSDDB. ''' @@ -95,20 +95,33 @@ class Database(back_anydbm.Database): return res def _doSaveJournal(self, classname, nodeid, action, params): + # serialise first + if action in ('set', 'create'): + params = self.serialise(classname, params) + entry = (nodeid, date.Date().get_tuple(), self.journaltag, action, params) + + if hyperdb.DEBUG: + print '_doSaveJournal', entry + db = bsddb.btopen(os.path.join(self.dir, 'journals.%s'%classname), 'c') + if db.has_key(nodeid): s = db[nodeid] l = marshal.loads(s) l.append(entry) else: l = [entry] + db[nodeid] = marshal.dumps(l) db.close() # #$Log: not supported by cvs2svn $ +#Revision 1.16 2002/02/27 03:40:59 richard +#Ran it through pychecker, made fixes +# #Revision 1.15 2002/02/16 09:15:33 richard #forgot to patch bsddb backend too # diff --git a/roundup/htmltemplate.py b/roundup/htmltemplate.py index 59ce26c..0355328 100644 --- a/roundup/htmltemplate.py +++ b/roundup/htmltemplate.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: htmltemplate.py,v 1.85 2002-04-02 01:40:58 richard Exp $ +# $Id: htmltemplate.py,v 1.86 2002-04-03 05:54:31 richard Exp $ __doc__ = """ Template engine. @@ -357,10 +357,11 @@ class TemplateFunctions: linkvalue = cgi.escape(linkcl.get(value, k)) if showid: label = value - title = ' title="%s"'%linkvalue - # note ... this should be urllib.quote(linkcl.get(value, k)) + title = ' title="%s"'%linkvalue + # note ... this should be urllib.quote(linkcl.get(value, k)) else: label = linkvalue + title = '' if is_download: return '%s'%(linkname, value, linkvalue, title, label) @@ -376,9 +377,10 @@ class TemplateFunctions: if showid: label = value title = ' title="%s"'%linkvalue - # note ... this should be urllib.quote(linkcl.get(value, k)) + # note ... this should be urllib.quote(linkcl.get(value, k)) else: label = linkvalue + title = '' if is_download: l.append('%s'%(linkname, value, linkvalue, title, label)) @@ -1126,6 +1128,12 @@ class NewItemTemplate(TemplateFunctions): # # $Log: not supported by cvs2svn $ +# Revision 1.85 2002/04/02 01:40:58 richard +# . link() htmltemplate function now has a "showid" option for links and +# multilinks. When true, it only displays the linked node id as the anchor +# text. The link value is displayed as a tooltip using the title anchor +# attribute. +# # Revision 1.84 2002/03/29 19:41:48 rochecompaan # . Fixed display of mutlilink properties when using the template # functions, menu and plain. diff --git a/roundup/hyperdb.py b/roundup/hyperdb.py index 3d3c97b..7319f1a 100644 --- a/roundup/hyperdb.py +++ b/roundup/hyperdb.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: hyperdb.py,v 1.59 2002-03-12 22:52:26 richard Exp $ +# $Id: hyperdb.py,v 1.60 2002-04-03 05:54:31 richard Exp $ __doc__ = """ Hyperdatabase implementation, especially field types. @@ -171,11 +171,51 @@ transaction. ''' raise NotImplementedError + def serialise(self, classname, node): + '''Copy the node contents, converting non-marshallable data into + marshallable data. + ''' + if DEBUG: print 'serialise', classname, node + properties = self.getclass(classname).getprops() + d = {} + for k, v in node.items(): + prop = properties[k] + + if isinstance(prop, Password): + d[k] = str(v) + elif isinstance(prop, Date) and v is not None: + d[k] = v.get_tuple() + elif isinstance(prop, Interval) and v is not None: + d[k] = v.get_tuple() + else: + d[k] = v + return d + def setnode(self, classname, nodeid, node): '''Change the specified node. ''' raise NotImplementedError + def unserialise(self, classname, node): + '''Decode the marshalled node data + ''' + if DEBUG: print 'unserialise', classname, node + properties = self.getclass(classname).getprops() + d = {} + for k, v in node.items(): + prop = properties[k] + if isinstance(prop, Date) and v is not None: + d[k] = date.Date(v) + elif isinstance(prop, Interval) and v is not None: + d[k] = date.Interval(v) + elif isinstance(prop, Password): + p = password.Password() + p.unpack(v) + d[k] = p + else: + d[k] = v + return d + def getnode(self, classname, nodeid, db=None, cache=1): '''Get a node from the database. ''' @@ -396,17 +436,6 @@ class Class: # TODO: None isn't right here, I think... propvalues[key] = None - # convert all data to strings - for key, prop in self.properties.items(): - if isinstance(prop, Date): - if propvalues[key] is not None: - propvalues[key] = propvalues[key].get_tuple() - elif isinstance(prop, Interval): - if propvalues[key] is not None: - propvalues[key] = propvalues[key].get_tuple() - elif isinstance(prop, Password): - propvalues[key] = str(propvalues[key]) - # done self.db.addnode(self.classname, newid, propvalues) self.db.addjournal(self.classname, newid, 'create', propvalues) @@ -443,20 +472,6 @@ class Class: else: return default - # possibly convert the marshalled data to instances - if isinstance(prop, Date): - if d[propname] is None: - return None - return date.Date(d[propname]) - elif isinstance(prop, Interval): - if d[propname] is None: - return None - return date.Interval(d[propname]) - elif isinstance(prop, Password): - p = password.Password() - p.unpack(d[propname]) - return p - return d[propname] # XXX not in spec @@ -605,17 +620,17 @@ class Class: elif isinstance(prop, Password): if not isinstance(value, password.Password): raise TypeError, 'new property "%s" not a Password'% key - propvalues[key] = value = str(value) + propvalues[key] = value elif value is not None and isinstance(prop, Date): if not isinstance(value, date.Date): raise TypeError, 'new property "%s" not a Date'% key - propvalues[key] = value = value.get_tuple() + propvalues[key] = value elif value is not None and isinstance(prop, Interval): if not isinstance(value, date.Interval): raise TypeError, 'new property "%s" not an Interval'% key - propvalues[key] = value = value.get_tuple() + propvalues[key] = value node[key] = value @@ -1097,6 +1112,9 @@ def Choice(name, db, *options): # # $Log: not supported by cvs2svn $ +# Revision 1.59 2002/03/12 22:52:26 richard +# more pychecker warnings removed +# # Revision 1.58 2002/02/27 03:23:16 richard # Ran it through pychecker, made fixes # diff --git a/test/test_db.py b/test/test_db.py index dc08709..19dcd16 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -15,14 +15,14 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: test_db.py,v 1.19 2002-02-25 14:34:31 grubert Exp $ +# $Id: test_db.py,v 1.20 2002-04-03 05:54:31 richard Exp $ import unittest, os, shutil from roundup.hyperdb import String, Password, Link, Multilink, Date, \ Interval, Class, DatabaseError from roundup.roundupdb import FileClass -from roundup import date +from roundup import date, password def setupSchema(db, create): status = Class(db, "status", name=String()) @@ -84,7 +84,11 @@ class anydbmDBTestCase(MyTestCase): a = self.db.issue.get('5', "deadline") self.db.issue.set('5', deadline=date.Date()) - self.assertNotEqual(a, self.db.issue.get('5', "deadline")) + b = self.db.issue.get('5', "deadline") + self.db.commit() + self.assertNotEqual(a, b) + self.assertNotEqual(b, date.Date('1970-1-1 00:00:00')) + self.db.issue.set('5', deadline=date.Date()) a = self.db.issue.get('5', "foo") self.db.issue.set('5', foo=date.Interval('-1d')) @@ -98,6 +102,17 @@ class anydbmDBTestCase(MyTestCase): self.db.status.history('1') self.db.status.history('2') + def testSerialisation(self): + self.db.issue.create(title="spam", status='1', + deadline=date.Date(), foo=date.Interval('-1d')) + self.db.commit() + assert isinstance(self.db.issue.get('1', 'deadline'), date.Date) + assert isinstance(self.db.issue.get('1', 'foo'), date.Interval) + self.db.user.create(username="fozzy", + password=password.Password('t. bear')) + self.db.commit() + assert isinstance(self.db.user.get('1', 'password'), password.Password) + def testTransactions(self): # remember the number of items we started num_issues = len(self.db.issue.list()) @@ -324,7 +339,8 @@ class bsddb3ReadOnlyDBTestCase(anydbmReadOnlyDBTestCase): def suite(): - l = [unittest.makeSuite(anydbmDBTestCase, 'test'), + l = [ + unittest.makeSuite(anydbmDBTestCase, 'test'), unittest.makeSuite(anydbmReadOnlyDBTestCase, 'test') ] @@ -346,6 +362,11 @@ def suite(): # # $Log: not supported by cvs2svn $ +# Revision 1.19 2002/02/25 14:34:31 grubert +# . use blobfiles in back_anydbm which is used in back_bsddb. +# change test_db as dirlist does not work for subdirectories. +# ATTENTION: blobfiles now creates subdirectories for files. +# # Revision 1.18 2002/01/22 07:21:13 richard # . fixed back_bsddb so it passed the journal tests # diff --git a/test/test_htmltemplate.py b/test/test_htmltemplate.py index 3fd109c..f216db7 100644 --- a/test/test_htmltemplate.py +++ b/test/test_htmltemplate.py @@ -8,7 +8,7 @@ # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # -# $Id: test_htmltemplate.py,v 1.12 2002-03-29 19:41:48 rochecompaan Exp $ +# $Id: test_htmltemplate.py,v 1.13 2002-04-03 05:54:31 richard Exp $ import unittest, cgi, time @@ -234,10 +234,18 @@ class NodeCase(unittest.TestCase): self.assertEqual(self.tf.do_link('link'), 'the key1') + def testLink_link_id(self): + self.assertEqual(self.tf.do_link('link', showid=1), + '1') + def testLink_multilink(self): self.assertEqual(self.tf.do_link('multilink'), 'the key1, the key2') + def testLink_multilink_id(self): + self.assertEqual(self.tf.do_link('multilink', showid=1), + '1, 2') + # def do_count(self, property, **args): def testCount_nonlinks(self): s = _('[Count: not a Multilink]') @@ -344,6 +352,10 @@ def suite(): # # $Log: not supported by cvs2svn $ +# Revision 1.12 2002/03/29 19:41:48 rochecompaan +# . Fixed display of mutlilink properties when using the template +# functions, menu and plain. +# # Revision 1.11 2002/02/21 23:11:45 richard # . fixed some problems in date calculations (calendar.py doesn't handle over- # and under-flow). Also, hour/minute/second intervals may now be more than -- 2.30.2