From 17678a45e1deb4cb61965b6eb0861a84e63aeb4b Mon Sep 17 00:00:00 2001 From: richard Date: Fri, 14 Nov 2003 00:11:19 +0000 Subject: [PATCH] - Fixed retirement of items in rdbms imports (sf bug 841355) - Fixed bug in looking up journal of newly-created items in *dbm backends git-svn-id: http://svn.roundup-tracker.org/svnroot/roundup/trunk@1988 57a73879-2fb5-44c3-a270-3262357dd7e2 --- CHANGES.txt | 2 ++ roundup/backends/back_anydbm.py | 8 +++-- roundup/backends/back_bsddb.py | 9 +++-- roundup/backends/back_bsddb3.py | 5 ++- roundup/backends/back_metakit.py | 4 +-- roundup/backends/back_postgresql.py | 2 +- roundup/backends/rdbms_common.py | 10 +++--- test/db_test_base.py | 55 ++++++++++++++++++++++++++++- test/test_anydbm.py | 7 ++-- test/test_bsddb.py | 8 +++-- test/test_bsddb3.py | 8 +++-- test/test_metakit.py | 8 +++-- test/test_mysql.py | 9 +++-- test/test_postgresql.py | 7 +++- test/test_sqlite.py | 8 +++-- 15 files changed, 118 insertions(+), 32 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 0f35bd0..8c1c007 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -61,6 +61,8 @@ Fixed: Date +/- Interval now works, and Date - Date also works (produces an Interval. - Handle socket timeout exception (thanks Marcus Priesch) +- Fixed retirement of items in rdbms imports (sf bug 841355) +- Fixed bug in looking up journal of newly-created items in *dbm backends 2003-09-29 0.6.2 diff --git a/roundup/backends/back_anydbm.py b/roundup/backends/back_anydbm.py index 61f2226..d2897ad 100644 --- a/roundup/backends/back_anydbm.py +++ b/roundup/backends/back_anydbm.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -#$Id: back_anydbm.py,v 1.130 2003-11-10 03:56:39 richard Exp $ +#$Id: back_anydbm.py,v 1.131 2003-11-14 00:11:18 richard Exp $ ''' This module defines a backend that saves the hyperdatabase in a database chosen by anydbm. It is guaranteed to always be available in python @@ -516,7 +516,11 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): if str(error) == "need 'c' or 'n' flag to open new db": raise IndexError, 'no such %s %s'%(classname, nodeid) elif error.args[0] != 2: + # this isn't a "not found" error, be alarmed! raise + if res: + # we have unsaved journal entries, return them + return res raise IndexError, 'no such %s %s'%(classname, nodeid) try: journal = marshal.loads(db[nodeid]) @@ -936,7 +940,7 @@ class Class(hyperdb.Class): l.append(repr(value)) # append retired flag - l.append(self.is_retired(nodeid)) + l.append(repr(self.is_retired(nodeid))) return l diff --git a/roundup/backends/back_bsddb.py b/roundup/backends/back_bsddb.py index 2b2a504..37abdd8 100644 --- a/roundup/backends/back_bsddb.py +++ b/roundup/backends/back_bsddb.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -#$Id: back_bsddb.py,v 1.27 2003-09-08 20:39:18 jlgijsbers Exp $ +#$Id: back_bsddb.py,v 1.28 2003-11-14 00:11:18 richard Exp $ ''' This module defines a backend that saves the hyperdatabase in BSDDB. ''' @@ -105,7 +105,12 @@ class Database(Database): db = bsddb.btopen(os.path.join(self.dir, 'journals.%s'%classname), 'r') except bsddb.error, error: - if error.args[0] != 2: raise + if error.args[0] != 2: + # this isn't a "not found" error, be alarmed! + raise + if res: + # we have unsaved journal entries, return them + return res raise IndexError, 'no such %s %s'%(classname, nodeid) # more handling of bad journals if not db.has_key(nodeid): diff --git a/roundup/backends/back_bsddb3.py b/roundup/backends/back_bsddb3.py index 2c01d57..444fe65 100644 --- a/roundup/backends/back_bsddb3.py +++ b/roundup/backends/back_bsddb3.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -#$Id: back_bsddb3.py,v 1.20 2003-09-08 20:39:18 jlgijsbers Exp $ +#$Id: back_bsddb3.py,v 1.21 2003-11-14 00:11:18 richard Exp $ ''' This module defines a backend that saves the hyperdatabase in BSDDB3. ''' @@ -105,6 +105,9 @@ class Database(Database): db = bsddb3.btopen(os.path.join(self.dir, 'journals.%s'%classname), 'r') except bsddb3._db.DBNoSuchFileError: + if res: + # we have unsaved journal entries, return them + return res raise IndexError, 'no such %s %s'%(classname, nodeid) # more handling of bad journals if not db.has_key(nodeid): diff --git a/roundup/backends/back_metakit.py b/roundup/backends/back_metakit.py index a612f8b..5303526 100755 --- a/roundup/backends/back_metakit.py +++ b/roundup/backends/back_metakit.py @@ -1,4 +1,4 @@ -# $Id: back_metakit.py,v 1.52 2003-11-12 03:42:12 richard Exp $ +# $Id: back_metakit.py,v 1.53 2003-11-14 00:11:18 richard Exp $ ''' Metakit backend for Roundup, originally by Gordon McMillan. @@ -1173,7 +1173,7 @@ class Class: l.append(repr(value)) # append retired flag - l.append(self.is_retired(nodeid)) + l.append(repr(self.is_retired(nodeid))) return l diff --git a/roundup/backends/back_postgresql.py b/roundup/backends/back_postgresql.py index 9559ee4..5a28fa7 100644 --- a/roundup/backends/back_postgresql.py +++ b/roundup/backends/back_postgresql.py @@ -21,7 +21,7 @@ class Database(rdbms_common.Database): try: self.conn = psycopg.connect(**db) except psycopg.OperationalError, message: - raise DatabaseError, message + raise hyperdb.DatabaseError, message self.cursor = self.conn.cursor() diff --git a/roundup/backends/rdbms_common.py b/roundup/backends/rdbms_common.py index 1e7db19..627ecec 100644 --- a/roundup/backends/rdbms_common.py +++ b/roundup/backends/rdbms_common.py @@ -1,4 +1,4 @@ -# $Id: rdbms_common.py,v 1.69 2003-11-12 03:42:12 richard Exp $ +# $Id: rdbms_common.py,v 1.70 2003-11-14 00:11:19 richard Exp $ ''' Relational database (SQL) backend common code. Basics: @@ -1236,7 +1236,7 @@ class Class(hyperdb.Class): elif isinstance(proptype, hyperdb.Password): value = str(value) l.append(repr(value)) - l.append(self.is_retired(nodeid)) + l.append(repr(self.is_retired(nodeid))) return l def import_list(self, propnames, proplist): @@ -1293,6 +1293,9 @@ class Class(hyperdb.Class): if newid is None: newid = self.db.newid(self.classname) + # add the node and journal + self.db.addnode(self.classname, newid, d) + # retire? if retire: # use the arg for __retired__ to cope with any odd database type @@ -1303,9 +1306,6 @@ class Class(hyperdb.Class): print >>hyperdb.DEBUG, 'retire', (self, sql, newid) self.db.cursor.execute(sql, (1, newid)) - # add the node and journal - self.db.addnode(self.classname, newid, d) - # extract the extraneous journalling gumpf and nuke it if d.has_key('creator'): creator = d['creator'] diff --git a/test/db_test_base.py b/test/db_test_base.py index 97c01cb..57d6e2b 100644 --- a/test/db_test_base.py +++ b/test/db_test_base.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: db_test_base.py,v 1.7 2003-11-12 03:42:13 richard Exp $ +# $Id: db_test_base.py,v 1.8 2003-11-14 00:11:19 richard Exp $ import unittest, os, shutil, errno, imp, sys, time, pprint @@ -744,6 +744,59 @@ class DBTest(MyTestCase): # XXX add sorting tests for other types # XXX test auditors and reactors + def testImportExport(self): + # use the filtering setup to create a bunch of items + ae, filt = self.filteringSetup() + self.db.user.retire('3') + self.db.issue.retire('2') + + # grab snapshot of the current database + orig = {} + for cn,klass in self.db.classes.items(): + cl = orig[cn] = {} + for id in klass.list(): + it = cl[id] = {} + for name in klass.getprops().keys(): + it[name] = klass.get(id, name) + + # grab the export + export = {} + for cn,klass in self.db.classes.items(): + names = klass.getprops().keys() + cl = export[cn] = [names+['is retired']] + for id in klass.getnodeids(): + cl.append(klass.export_list(names, id)) + + # shut down this db and nuke it + self.db.close() + self.nuke_database() + + # open a new, empty database + os.makedirs(config.DATABASE + '/files') + self.db = self.module.Database(config, 'admin') + setupSchema(self.db, 0, self.module) + + # import + for cn, items in export.items(): + klass = self.db.classes[cn] + names = items[0] + for itemprops in items[1:]: + klass.import_list(names, itemprops) + + # grab snapshot of the current database + for cn, items in orig.items(): + klass = self.db.classes[cn] + # ensure retired items are retired :) + l = items.keys(); l.sort() + m = klass.list(); m.sort() + ae(l, m) + for id, props in items.items(): + for name, value in props.items(): + ae(klass.get(id, name), value) + + # make sure the retired items are actually imported + ae(self.db.user.get('3', 'username'), 'blop') + ae(self.db.issue.get('2', 'title'), 'issue two') class ROTest(MyTestCase): def setUp(self): diff --git a/test/test_anydbm.py b/test/test_anydbm.py index 84d3108..77c57b8 100644 --- a/test/test_anydbm.py +++ b/test/test_anydbm.py @@ -15,15 +15,18 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: test_anydbm.py,v 1.1 2003-10-25 22:53:26 richard Exp $ +# $Id: test_anydbm.py,v 1.2 2003-11-14 00:11:19 richard Exp $ import unittest, os, shutil, time -from db_test_base import DBTest, ROTest, SchemaTest, ClassicInitTest +from db_test_base import DBTest, ROTest, SchemaTest, ClassicInitTest, config class anydbmOpener: from roundup.backends import anydbm as module + def nuke_database(self): + shutil.rmtree(config.DATABASE) + class anydbmDBTest(anydbmOpener, DBTest): pass diff --git a/test/test_bsddb.py b/test/test_bsddb.py index 45059b9..24cde20 100644 --- a/test/test_bsddb.py +++ b/test/test_bsddb.py @@ -15,18 +15,20 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: test_bsddb.py,v 1.1 2003-10-25 22:53:26 richard Exp $ +# $Id: test_bsddb.py,v 1.2 2003-11-14 00:11:19 richard Exp $ import unittest, os, shutil, time -from db_test_base import DBTest, ROTest, SchemaTest, \ - ClassicInitTest +from db_test_base import DBTest, ROTest, SchemaTest, ClassicInitTest, config from roundup import backends class bsddbOpener: if hasattr(backends, 'bsddb'): from roundup.backends import bsddb as module + def nuke_database(self): + shutil.rmtree(config.DATABASE) + class bsddbDBTest(bsddbOpener, DBTest): pass diff --git a/test/test_bsddb3.py b/test/test_bsddb3.py index 16f55e7..2387cda 100644 --- a/test/test_bsddb3.py +++ b/test/test_bsddb3.py @@ -15,18 +15,20 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: test_bsddb3.py,v 1.1 2003-10-25 22:53:26 richard Exp $ +# $Id: test_bsddb3.py,v 1.2 2003-11-14 00:11:19 richard Exp $ import unittest, os, shutil, time -from db_test_base import DBTest, ROTest, SchemaTest, \ - ClassicInitTest +from db_test_base import DBTest, ROTest, SchemaTest, ClassicInitTest, config from roundup import backends class bsddb3Opener: if hasattr(backends, 'bsddb3'): from roundup.backends import bsddb3 as module + def nuke_database(self): + shutil.rmtree(config.DATABASE) + class bsddb3DBTest(bsddb3Opener, DBTest): pass diff --git a/test/test_metakit.py b/test/test_metakit.py index 4d39642..8596154 100644 --- a/test/test_metakit.py +++ b/test/test_metakit.py @@ -15,12 +15,11 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: test_metakit.py,v 1.1 2003-10-25 22:53:26 richard Exp $ +# $Id: test_metakit.py,v 1.2 2003-11-14 00:11:19 richard Exp $ import unittest, os, shutil, time, weakref -from db_test_base import DBTest, ROTest, SchemaTest, \ - ClassicInitTest +from db_test_base import DBTest, ROTest, SchemaTest, ClassicInitTest, config from roundup import backends @@ -29,6 +28,9 @@ class metakitOpener: from roundup.backends import metakit as module module._instances = weakref.WeakValueDictionary() + def nuke_database(self): + shutil.rmtree(config.DATABASE) + class metakitDBTest(metakitOpener, DBTest): def testTransactions(self): # remember the number of items we started diff --git a/test/test_mysql.py b/test/test_mysql.py index 724eefc..1388d47 100644 --- a/test/test_mysql.py +++ b/test/test_mysql.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: test_mysql.py,v 1.5 2003-11-11 11:19:18 richard Exp $ +# $Id: test_mysql.py,v 1.6 2003-11-14 00:11:19 richard Exp $ import unittest, os, shutil, time, imp @@ -42,6 +42,9 @@ class mysqlOpener: def tearDown(self): self.db.close() + self.nuke_database() + + def nuke_database(self): self.module.db_nuke(config) class mysqlDBTest(mysqlOpener, DBTest): @@ -53,7 +56,7 @@ class mysqlROTest(mysqlOpener, ROTest): class mysqlSchemaTest(mysqlOpener, SchemaTest): pass -class mysqlClassicInitTest(ClassicInitTest): +class mysqlClassicInitTest(mysqlOpener, ClassicInitTest): backend = 'mysql' extra_config = ''' MYSQL_DBHOST = 'localhost' @@ -66,7 +69,7 @@ MYSQL_DATABASE = (MYSQL_DBHOST, MYSQL_DBUSER, MYSQL_DBPASSWORD, MYSQL_DBNAME) from roundup.backends import mysql as module def tearDown(self): ClassicInitTest.tearDown(self) - self.module.db_nuke(config) + self.nuke_database() def test_suite(): suite = unittest.TestSuite() diff --git a/test/test_postgresql.py b/test/test_postgresql.py index c7a957d..5841f84 100644 --- a/test/test_postgresql.py +++ b/test/test_postgresql.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: test_postgresql.py,v 1.3 2003-11-11 11:19:18 richard Exp $ +# $Id: test_postgresql.py,v 1.4 2003-11-14 00:11:19 richard Exp $ import sys, unittest, os, shutil, time, popen2 @@ -68,7 +68,12 @@ class postgresqlOpener: db_create() def tearDown(self): + self.nuke_database() + + def nuke_database(self): + # clear out the database - easiest way is to nuke and re-created it db_nuke() + db_create() class postgresqlDBTest(postgresqlOpener, DBTest): def setUp(self): diff --git a/test/test_sqlite.py b/test/test_sqlite.py index bf0cc22..19e4277 100644 --- a/test/test_sqlite.py +++ b/test/test_sqlite.py @@ -15,18 +15,20 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: test_sqlite.py,v 1.2 2003-11-02 08:44:17 richard Exp $ +# $Id: test_sqlite.py,v 1.3 2003-11-14 00:11:19 richard Exp $ import unittest, os, shutil, time -from db_test_base import DBTest, ROTest, SchemaTest, \ - ClassicInitTest +from db_test_base import DBTest, ROTest, SchemaTest, ClassicInitTest, config class sqliteOpener: from roundup import backends if hasattr(backends, 'sqlite'): from roundup.backends import sqlite as module + def nuke_database(self): + shutil.rmtree(config.DATABASE) + class sqliteDBTest(sqliteOpener, DBTest): pass -- 2.30.2