From e36034954b3a18a3ff48acbeedb501b27a6367c4 Mon Sep 17 00:00:00 2001 From: richard Date: Thu, 26 Sep 2002 03:04:24 +0000 Subject: [PATCH] added Class.find() unit test, fixed implementations git-svn-id: http://svn.roundup-tracker.org/svnroot/roundup/trunk@1248 57a73879-2fb5-44c3-a270-3262357dd7e2 --- CHANGES.txt | 1 + roundup/backends/back_anydbm.py | 27 +++++------- roundup/backends/back_gadfly.py | 29 ++++++++++--- roundup/backends/rdbms_common.py | 70 ++++++++++++++++++++++++-------- test/test_db.py | 49 +++++++++++++++++++--- test/test_mailgw.py | 47 ++++++++++++++++++++- 6 files changed, 177 insertions(+), 46 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 003f1b8..ecaf45f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -36,6 +36,7 @@ are given with the most recent entry first. - handle stupid mailers that QUOTE their Re; 'Re: "[issue1] bla blah"' - giving a user a Role that doesn't exist doesn't break stuff any more - revamped user guide, customisation guide, added maintenance guide +- merge Zope Collector #580 fix from ZPT CVS trunk 2002-09-13 0.5.0 beta2 diff --git a/roundup/backends/back_anydbm.py b/roundup/backends/back_anydbm.py index 1348798..ea2533a 100644 --- a/roundup/backends/back_anydbm.py +++ b/roundup/backends/back_anydbm.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -#$Id: back_anydbm.py,v 1.85 2002-09-24 01:59:28 richard Exp $ +#$Id: back_anydbm.py,v 1.86 2002-09-26 03:04:24 richard Exp $ ''' This module defines a backend that saves the hyperdatabase in a database chosen by anydbm. It is guaranteed to always be available in python @@ -524,6 +524,7 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): if __debug__: print >>hyperdb.DEBUG, 'packjournal', (self, pack_before) + pack_before = pack_before.serialise() for classname in self.getclasses(): # get the journal db db_name = 'journals.%s'%classname @@ -540,21 +541,10 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): # unpack the entry (nodeid, date_stamp, self.journaltag, action, params) = entry - date_stamp = date.Date(date_stamp) # if the entry is after the pack date, _or_ the initial # create entry, then it stays if date_stamp > pack_before or action == 'create': l.append(entry) - elif action == 'set': - # grab the last set entry to keep information on - # activity - last_set_entry = entry - if last_set_entry: - date_stamp = last_set_entry[1] - # if the last set entry was made after the pack date - # then it is already in the list - if date_stamp < pack_before: - l.append(last_set_entry) db[key] = marshal.dumps(l) if db_type == 'gdbm': db.reorganize() @@ -1443,14 +1433,17 @@ class Class(hyperdb.Class): def find(self, **propspec): '''Get the ids of nodes in this class which link to the given nodes. - 'propspec' consists of keyword args propname={nodeid:1,} - 'propname' must be the name of a property in this class, or a - KeyError is raised. That property must be a Link or Multilink - property, or a TypeError is raised. + 'propspec' consists of keyword args propname=nodeid or + propname={nodeid:1, } + 'propname' must be the name of a property in this class, or a + KeyError is raised. That property must be a Link or + Multilink property, or a TypeError is raised. Any node in this class whose 'propname' property links to any of the nodeids will be returned. Used by the full text indexing, which knows - that "foo" occurs in msg1, msg3 and file7, so we have hits on these issues: + that "foo" occurs in msg1, msg3 and file7, so we have hits on these + issues: + db.issue.find(messages={'1':1,'3':1}, files={'7':1}) ''' propspec = propspec.items() diff --git a/roundup/backends/back_gadfly.py b/roundup/backends/back_gadfly.py index e14a12c..1b9b05b 100644 --- a/roundup/backends/back_gadfly.py +++ b/roundup/backends/back_gadfly.py @@ -1,4 +1,4 @@ -# $Id: back_gadfly.py,v 1.26 2002-09-24 01:59:28 richard Exp $ +# $Id: back_gadfly.py,v 1.27 2002-09-26 03:04:24 richard Exp $ __doc__ = ''' About Gadfly ============ @@ -47,14 +47,23 @@ used. ''' -from roundup.backends.rdbms_common import * +# standard python modules +import sys, os, time, re, errno, weakref, copy + +# roundup modules +from roundup import hyperdb, date, password, roundupdb, security +from roundup.hyperdb import String, Password, Date, Interval, Link, \ + Multilink, DatabaseError, Boolean, Number + +# basic RDBMS backen implementation +from roundup.backends import rdbms_common # the all-important gadfly :) import gadfly import gadfly.client import gadfly.database -class Database(Database): +class Database(rdbms_common.Database): # char to use for positional arguments arg = '?' @@ -238,10 +247,18 @@ class GadflyClass: # return the IDs return [row[0] for row in l] -class Class(GadflyClass, Class): + def find(self, **propspec): + ''' Overload to filter out duplicates in the result + ''' + d = {} + for k in rdbms_common.Class.find(self, **propspec): + d[k] = 1 + return d.keys() + +class Class(GadflyClass, rdbms_common.Class): pass -class IssueClass(GadflyClass, IssueClass): +class IssueClass(GadflyClass, rdbms_common.IssueClass): pass -class FileClass(GadflyClass, FileClass): +class FileClass(GadflyClass, rdbms_common.FileClass): pass diff --git a/roundup/backends/rdbms_common.py b/roundup/backends/rdbms_common.py index f0acaee..7c48bf5 100644 --- a/roundup/backends/rdbms_common.py +++ b/roundup/backends/rdbms_common.py @@ -1,4 +1,4 @@ -# $Id: rdbms_common.py,v 1.18 2002-09-25 05:27:29 richard Exp $ +# $Id: rdbms_common.py,v 1.19 2002-09-26 03:04:24 richard Exp $ # standard python modules import sys, os, time, re, errno, weakref, copy @@ -1458,11 +1458,13 @@ class Class(hyperdb.Class): if self.db.journaltag is None: raise DatabaseError, 'Database open read-only' - sql = 'update _%s set __retired__=1 where id=%s'%(self.classname, - self.db.arg) + # use the arg for __retired__ to cope with any odd database type + # conversion (hello, sqlite) + sql = 'update _%s set __retired__=%s where id=%s'%(self.classname, + self.db.arg, self.db.arg) if __debug__: print >>hyperdb.DEBUG, 'retire', (self, sql, nodeid) - self.db.cursor.execute(sql, (nodeid,)) + self.db.cursor.execute(sql, (1, nodeid)) def is_retired(self, nodeid): '''Return true if the node is rerired @@ -1570,10 +1572,11 @@ class Class(hyperdb.Class): if not self.key: raise TypeError, 'No key property set for class %s'%self.classname - sql = '''select id from _%s where _%s=%s - and __retired__ != '1' '''%(self.classname, self.key, - self.db.arg) - self.db.sql(sql, (keyvalue,)) + # use the arg to handle any odd database type conversion (hello, + # sqlite) + sql = "select id from _%s where _%s=%s and __retired__ <> %s"%( + self.classname, self.key, self.db.arg, self.db.arg) + self.db.sql(sql, (keyvalue, 1)) # see if there was a result that's not retired row = self.db.sql_fetchone() @@ -1587,7 +1590,8 @@ class Class(hyperdb.Class): def find(self, **propspec): '''Get the ids of nodes in this class which link to the given nodes. - 'propspec' consists of keyword args propname={nodeid:1,} + 'propspec' consists of keyword args propname=nodeid or + propname={nodeid:1, } 'propname' must be the name of a property in this class, or a KeyError is raised. That property must be a Link or Multilink property, or a TypeError is raised. @@ -1601,17 +1605,51 @@ class Class(hyperdb.Class): ''' if __debug__: print >>hyperdb.DEBUG, 'find', (self, propspec) + + # shortcut if not propspec: return [] - queries = [] - tables = [] + + # validate the args + props = self.getprops() + propspec = propspec.items() + for propname, nodeids in propspec: + # check the prop is OK + prop = props[propname] + if not isinstance(prop, Link) and not isinstance(prop, Multilink): + raise TypeError, "'%s' not a Link/Multilink property"%propname + + # first, links + where = [] allvalues = () - for prop, values in propspec.items(): - allvalues += tuple(values.keys()) - a = self.db.arg + a = self.db.arg + for prop, values in propspec: + if not isinstance(props[prop], hyperdb.Link): + continue + if type(values) is type(''): + allvalues += (values,) + where.append('_%s = %s'%(prop, a)) + else: + allvalues += tuple(values.keys()) + where.append('_%s in (%s)'%(prop, ','.join([a]*len(values)))) + tables = [] + if where: + tables.append('select id as nodeid from _%s where %s'%( + self.classname, ' and '.join(where))) + + # now multilinks + for prop, values in propspec: + if not isinstance(props[prop], hyperdb.Multilink): + continue + if type(values) is type(''): + allvalues += (values,) + s = a + else: + allvalues += tuple(values.keys()) + s = ','.join([a]*len(values)) tables.append('select nodeid from %s_%s where linkid in (%s)'%( - self.classname, prop, ','.join([a for x in values.keys()]))) - sql = '\nintersect\n'.join(tables) + self.classname, prop, s)) + sql = '\nunion\n'.join(tables) self.db.sql(sql, allvalues) l = [x[0] for x in self.db.sql_fetchall()] if __debug__: diff --git a/test/test_db.py b/test/test_db.py index 315ab2f..bd1b20b 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: test_db.py,v 1.55 2002-09-24 01:59:44 richard Exp $ +# $Id: test_db.py,v 1.56 2002-09-26 03:04:24 richard Exp $ import unittest, os, shutil, time @@ -429,20 +429,20 @@ class anydbmDBTestCase(MyTestCase): self.db.commit() # sleep for at least a second, then get a date to pack at - time.sleep(2) + time.sleep(1) pack_before = date.Date('.') - time.sleep(2) - # one more entry + # wait another second and add one more entry + time.sleep(1) self.db.issue.set(id, status='3') self.db.commit() + jlen = len(self.db.getjournal('issue', id)) # pack self.db.pack(pack_before) - journal = self.db.getjournal('issue', id) # we should have the create and last set entries now - self.assertEqual(2, len(journal)) + self.assertEqual(jlen-1, len(self.db.getjournal('issue', id))) def testIDGeneration(self): id1 = self.db.issue.create(title="spam", status='1') @@ -487,6 +487,36 @@ class anydbmDBTestCase(MyTestCase): self.assertEquals(self.db.indexer.search(['flebble'], self.db.issue), {'1': {}}) + # + # searching tests follow + # + def testFind(self): + self.db.user.create(username='test') + ids = [] + ids.append(self.db.issue.create(status="1", nosy=['1'])) + oddid = self.db.issue.create(status="2", nosy=['2']) + ids.append(self.db.issue.create(status="1", nosy=['1','2'])) + self.db.issue.create(status="3", nosy=['1']) + ids.sort() + + # should match first and third + got = self.db.issue.find(status='1') + got.sort() + self.assertEqual(got, ids) + + # none + self.assertEqual(self.db.issue.find(status='4'), []) + + # should match first three + got = self.db.issue.find(status='1', nosy='2') + got.sort() + ids.append(oddid) + ids.sort() + self.assertEqual(got, ids) + + # none + self.assertEqual(self.db.issue.find(status='4', nosy='3'), []) + def testStringFind(self): ids = [] ids.append(self.db.issue.create(title="spam")) @@ -629,6 +659,13 @@ class gadflyDBTestCase(anydbmDBTestCase): id2 = self.db.issue.create(title="eggs", status='2') self.assertNotEqual(id1, id2) + def testFilteringString(self): + ae, filt = self.filteringSetup() + ae(filt(None, {'title': 'issue one'}, ('+','id'), (None,None)), ['1']) + # XXX gadfly can't do substring LIKE searches + #ae(filt(None, {'title': 'issue'}, ('+','id'), (None,None)), + # ['1','2','3']) + class gadflyReadOnlyDBTestCase(anydbmReadOnlyDBTestCase): def setUp(self): from roundup.backends import gadfly diff --git a/test/test_mailgw.py b/test/test_mailgw.py index 71293a5..af81ce6 100644 --- a/test/test_mailgw.py +++ b/test/test_mailgw.py @@ -8,7 +8,7 @@ # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # -# $Id: test_mailgw.py,v 1.31 2002-09-20 05:08:00 richard Exp $ +# $Id: test_mailgw.py,v 1.32 2002-09-26 03:04:24 richard Exp $ import unittest, cStringIO, tempfile, os, shutil, errno, imp, sys, difflib @@ -765,6 +765,51 @@ mary added the comment: A message with first part encoded (encoded oe =F6) +---------- +status: unread -> chatting +_________________________________________________________________________ +"Roundup issue tracker" +http://your.tracker.url.example/issue1 +_________________________________________________________________________ +''') + + def testFollowupStupidQuoting(self): + self.doNewIssue() + + message = cStringIO.StringIO('''Content-Type: text/plain; + charset="iso-8859-1" +From: richard +To: issue_tracker@your.tracker.email.domain.example +Message-Id: +In-Reply-To: +Subject: Re: "[issue1] Testing... " + +This is a followup +''') + handler = self.instance.MailGW(self.instance, self.db) + handler.trapExceptions = 0 + handler.main(message) + + self.compareStrings(open(os.environ['SENDMAILDEBUG']).read(), +'''FROM: roundup-admin@your.tracker.email.domain.example +TO: chef@bork.bork.bork +Content-Type: text/plain +Subject: [issue1] Testing... +To: chef@bork.bork.bork +From: "richard" +Reply-To: "Roundup issue tracker" +MIME-Version: 1.0 +Message-Id: +In-Reply-To: +X-Roundup-Name: Roundup issue tracker +Content-Transfer-Encoding: quoted-printable + + +richard added the comment: + +This is a followup + + ---------- status: unread -> chatting _________________________________________________________________________ -- 2.30.2