From d7fe6d4ffb59f8d803adf3f9c8344a81668bf64e Mon Sep 17 00:00:00 2001 From: richard Date: Wed, 26 Feb 2003 23:42:54 +0000 Subject: [PATCH] fixed export/import of retired nodes (sf bug 685273) git-svn-id: http://svn.roundup-tracker.org/svnroot/roundup/trunk@1553 57a73879-2fb5-44c3-a270-3262357dd7e2 --- CHANGES.txt | 57 +++++++++++----------- TODO.txt | 5 -- roundup/admin.py | 9 ++-- roundup/backends/back_anydbm.py | 14 ++++-- roundup/backends/back_metakit.py | 83 ++++++++++++++++++++++++++++---- roundup/backends/rdbms_common.py | 13 ++++- 6 files changed, 132 insertions(+), 49 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index ceac852..6d75a61 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -2,34 +2,15 @@ This file contains the changes to the Roundup system over time. The entries are given with the most recent entry first. 2003-??-?? 0.6.0 +Fixed: - better hyperlinking in web message texts (sf bug 669777) -- support setting of properties on message and file through web and - email interface (thanks John Rouillard) -- allow additional control over the roundupdb email sending (explicit - cc addresses, different from address and different nosy list property) - (thanks John Rouillard) -- applied patch for nicer history display (sf feature 638280) -- cleaning old unused sessions only once per hour, not on every cgi - request. It is greatly improves web interface performance, especially - on trackers under high load -- added mysql backend (see doc/mysql.txt for details) -- fixes to CGI form handling -- switch metakit to use "compressed" multilink journal change representation - fixed bug in metakit unlink journalling -- metakit now handles "unset" for most types (not Number and Boolean) -- fixed bug in metakit search-by-ID - applied unicode patch. All data is stored in utf-8. Incoming messages converted from any encoding to utf-8, outgoing messages are encoded according to rfc2822 (sf bug 568873) - fixed cookie path to use TRACKER_WEB (sf bug 667020) (thanks Nathaniel Smith for helping chase it down and Luke Opperman for confirming fix) -- added ability to display localized dates in web interface. User input is - convered to GMT (see doc/upgrading.txt). -- added a form to show a specific issue - fixed layout issues with forms in sidebar -- more proper sorting/grouping on mulitilink properties. Sorting is performed - not only by number of links, but also by links itself. This makes usable - grouping e.g. by topic multilink - fixed templating filter function arguments (sf bug 678911) - fixed multiselect in searching (sf bug 676874) - fixed parsing of content-disposition filenames (sf bug 675116) @@ -42,12 +23,36 @@ are given with the most recent entry first. - added warning filter for "FutureWarning: hex/oct constants > sys.maxint will return positive values..." (literal 0xffff0000 in portalocker.py) - fixed ZPT code generating SyntaxWarning for assignment to None -- add "ago" to intervals in the past (sf bug 679232) -- clarified licensing -- another attempt to fix cookie misbehaviour - customise cookie name using - tracker name - fixed error in indexargs_url (thanks Patrick Ohly) - fixed getnode (sf bug 684531) +- open static files using binary mode (sf bug 693208) +- fixed deja-vu bug 692910 +- don't display "Editing" on read-only pages (sf bug 651967) +- re-worked detectors initialisation - woohoo, no more cross-importing! +- fixed export/import of retired nodes (sf bug 685273) + +Feature: +- support setting of properties on message and file through web and + email interface (thanks John Rouillard) +- allow additional control over the roundupdb email sending (explicit + cc addresses, different from address and different nosy list property) + (thanks John Rouillard) +- applied patch for nicer history display (sf feature 638280) +- cleaning old unused sessions only once per hour, not on every cgi + request. It is greatly improves web interface performance, especially + on trackers under high load +- added mysql backend (see doc/mysql.txt for details) +- switch metakit to use "compressed" multilink journal change representation +- metakit now handles "unset" for most types (not Number and Boolean) +- fixed bug in metakit search-by-ID +- added ability to display localized dates in web interface. User input is + convered to GMT (see doc/upgrading.txt). +- added a form to show a specific issue +- more proper sorting/grouping on mulitilink properties. Sorting is performed + not only by number of links, but also by links itself. This makes usable + grouping e.g. by topic multilink +- add "ago" to intervals in the past (sf bug 679232) +- clarified licensing - included UN*X manual pages from Bastian Kleineidam - implemented extension to form parsing to allow editing of multiple items and creation of multiple items (but only one per class) @@ -56,7 +61,6 @@ are given with the most recent entry first. (e.g. images). They are accessible naturally: _file/images/img.gif - altered Class.create() and FileClass.create() methods to make "content" property available in auditors -- re-worked detectors initialisation - woohoo, no more cross-importing! - can now configure CC to author only for messages creating issues (sf feature 625808) - registration is now a two-step process, with confirmation from the email @@ -64,9 +68,6 @@ are given with the most recent entry first. - added support for last-modified and if-modified-since headers for static file serving - added Node.get() method -- open static files using binary mode (sf bug 693208) -- fixed deja-vu bug 692910 -- don't display "Editing" on read-only pages (sf bug 651967) 2003-??-?? 0.5.6 diff --git a/TODO.txt b/TODO.txt index 6f64a41..1e40489 100644 --- a/TODO.txt +++ b/TODO.txt @@ -7,17 +7,12 @@ done, it's moved to the CHANGES file. State Component Description ======= ========= ============================================================ pending example meta/parent bug implementation (feature request #506815) -pending example replace the "extended" example with a "help desk" one, and - rename "classic" to "bug tracker" pending example script for retrieval of "mbox" archive of all messages pending hyperdb range searching of values (dates in particular). Filter specifies {property: (comparison function, value)} comparison functions: lt, le, eq, ge, gt. eq and [value, value, ...] implies "in" pending hyperdb migrate "id" property to be Number type -pending hyperdb multilink sorting by length is dumb -pending hyperdb lastchangedby auto-property giving last user to change an - item pending tracker split instance.open() into open() and login() pending mailgw allow commands (feature request #556996) like "help", "dump issue123" (would send all info about diff --git a/roundup/admin.py b/roundup/admin.py index 94e3846..8b90e46 100644 --- a/roundup/admin.py +++ b/roundup/admin.py @@ -16,7 +16,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: admin.py,v 1.38 2003-02-25 10:19:31 richard Exp $ +# $Id: admin.py,v 1.39 2003-02-26 23:42:49 richard Exp $ '''Administration commands for maintaining Roundup trackers. ''' @@ -922,8 +922,11 @@ Command help: propnames.sort() print >> f, p.join(propnames) - # all nodes for this class - for nodeid in cl.list(): + # all nodes for this class (not using list() 'cos it doesn't + # include retired nodes) + + for nodeid in self.db.getnodeids(classname): + # get the regular props print >>f, p.join(cl.export_list(propnames, nodeid)) return 0 diff --git a/roundup/backends/back_anydbm.py b/roundup/backends/back_anydbm.py index 71ffdb0..dfacf2f 100644 --- a/roundup/backends/back_anydbm.py +++ b/roundup/backends/back_anydbm.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -#$Id: back_anydbm.py,v 1.105 2003-02-25 10:19:31 richard Exp $ +#$Id: back_anydbm.py,v 1.106 2003-02-26 23:42:50 richard Exp $ ''' This module defines a backend that saves the hyperdatabase in a database chosen by anydbm. It is guaranteed to always be available in python @@ -922,6 +922,10 @@ class Class(hyperdb.Class): elif isinstance(proptype, hyperdb.Password): value = str(value) l.append(repr(value)) + + # append retired flag + l.append(self.is_retired(nodeid)) + return l def import_list(self, propnames, proplist): @@ -963,6 +967,10 @@ class Class(hyperdb.Class): value = pwd d[propname] = value + # check retired flag + if int(proplist[-1]): + d[self.db.RETIRED_FLAG] = 1 + # add the node and journal self.db.addnode(self.classname, newid, d) @@ -1325,10 +1333,10 @@ class Class(hyperdb.Class): self.fireReactors('retire', nodeid, None) - def is_retired(self, nodeid): + def is_retired(self, nodeid, cldb=None): '''Return true if the node is retired. ''' - node = self.db.getnode(cn, nodeid, cldb) + node = self.db.getnode(self.classname, nodeid, cldb) if node.has_key(self.db.RETIRED_FLAG): return 1 return 0 diff --git a/roundup/backends/back_metakit.py b/roundup/backends/back_metakit.py index 234a817..3cdcca8 100755 --- a/roundup/backends/back_metakit.py +++ b/roundup/backends/back_metakit.py @@ -38,6 +38,8 @@ import locking _dbs = {} def Database(config, journaltag=None): + ''' Only have a single instance of the Database class for each instance + ''' db = _dbs.get(config.DATABASE, None) if db is None or db._db is None: db = _Database(config, journaltag) @@ -81,8 +83,10 @@ class _Database(hyperdb.Database, roundupdb.Database): if self.journaltag is None: return None + # try to set the curuserid from the journaltag try: - self.curuserid = x = int(self.classes['user'].lookup(self.journaltag)) + x = int(self.classes['user'].lookup(self.journaltag)) + self.curuserid = x except KeyError: if self.journaltag == 'admin': self.curuserid = x = 1 @@ -91,6 +95,7 @@ class _Database(hyperdb.Database, roundupdb.Database): return x elif classname == 'transactions': return self.dirty + # fall back on the classes return self.getclass(classname) def getclass(self, classname): try: @@ -100,6 +105,7 @@ class _Database(hyperdb.Database, roundupdb.Database): def getclasses(self): return self.classes.keys() # --- end of ping's spec + # --- exposed methods def commit(self): if self.dirty: @@ -182,9 +188,10 @@ class _Database(hyperdb.Database, roundupdb.Database): #usernm = userclass.get(str(row.user), 'username') dt = date.Date(time.gmtime(row.date)) #rslt.append((nodeid, dt, usernm, _actionnames[row.action], params)) - rslt.append((nodeid, dt, str(row.user), _actionnames[row.action], params)) + rslt.append((nodeid, dt, str(row.user), _actionnames[row.action], + params)) return rslt - + def destroyjournal(self, tablenm, nodeid): nodeid = int(nodeid) tblid = self.tables.find(name=tablenm) @@ -215,13 +222,22 @@ class _Database(hyperdb.Database, roundupdb.Database): # --- internal def __open(self): + ''' Open the metakit database + ''' + # make the database dir if it doesn't exist if not os.path.exists(self.config.DATABASE): os.makedirs(self.config.DATABASE) + + # figure the file names self.dbnm = db = os.path.join(self.config.DATABASE, 'tracker.mk4') lockfilenm = db[:-3]+'lck' + + # get the database lock self.lockfile = locking.acquire_lock(lockfilenm) self.lockfile.write(str(os.getpid())) self.lockfile.flush() + + # see if the schema has changed since last db access self.fastopen = 0 if os.path.exists(db): dbtm = os.path.getmtime(db) @@ -236,18 +252,28 @@ class _Database(hyperdb.Database, roundupdb.Database): else: # can't find schemamod - must be frozen self.fastopen = 1 + + # open the db db = metakit.storage(db, 1) hist = db.view('history') tables = db.view('tables') if not self.fastopen: + # create the database if it's brand new if not hist.structure(): hist = db.getas('history[tableid:I,nodeid:I,date:I,user:I,action:I,params:B]') if not tables.structure(): tables = db.getas('tables[name:S]') db.commit() + + # we now have an open, initialised database self.tables = tables self.hist = hist return db + + def setid(self, classname, maxid): + ''' No-op in metakit + ''' + pass _STRINGTYPE = type('') _LISTTYPE = type([]) @@ -632,10 +658,12 @@ class Class: ndx = view.find(id=int(nodeid)) if ndx < 0: raise KeyError, "nodeid %s not found" % nodeid + row = view[ndx] oldvalues = self.uncommitted.setdefault(row.id, {}) oldval = oldvalues['_isdel'] = row._isdel row._isdel = 1 + if self.do_journal: self.db.addjournal(self.classname, nodeid, _RETIRE, {}) if self.keyname: @@ -646,6 +674,16 @@ class Class: self.db.dirty = 1 self.fireReactors('retire', nodeid, None) + def is_retired(self, nodeid): + view = self.getview(1) + # node must exist & not be retired + id = int(nodeid) + ndx = view.find(id=id) + if ndx < 0: + raise IndexError, "%s has no node %s" % (self.classname, nodeid) + row = view[ndx] + return row._isdel + def history(self, nodeid): if not self.do_journal: raise ValueError, 'Journalling is disabled for this class' @@ -796,6 +834,7 @@ class Class: raise ValueError, "%s is already a property of %s"%(key, self.classname) self.ruprops.update(properties) + # Class structure has changed self.db.fastopen = 0 view = self.__getview() self.db.commit() @@ -1037,6 +1076,10 @@ class Class: elif isinstance(proptype, hyperdb.Password): value = str(value) l.append(repr(value)) + + # append retired flag + l.append(self.is_retired(nodeid)) + return l def import_list(self, propnames, proplist): @@ -1064,11 +1107,24 @@ class Class: value = int(calendar.timegm(value)) elif isinstance(prop, hyperdb.Interval): value = str(date.Interval(value)) + elif isinstance(prop, hyperdb.Number): + value = int(value) + elif isinstance(prop, hyperdb.Boolean): + value = int(value) + elif isinstance(prop, hyperdb.Link): + value = int(value) + elif isinstance(prop, hyperdb.Multilink): + value = map(int, value) d[propname] = value - view.append(d) - creator = d.get('creator', None) - creation = d.get('creation', None) - self.db.addjournal(self.classname, newid, 'create', {}, creator, + # is the item retired? + if int(proplist[-1]): + d['_isdel'] = 1 + # XXX this is BROKEN for reasons I don't understand! + ndx = view.append(d) + + creator = d.get('creator', 0) + creation = d.get('creation', 0) + self.db.addjournal(self.classname, newid, _CREATE, {}, creator, creation) return newid @@ -1101,11 +1157,20 @@ class Class: self.rbactions.append(action) # --- internal def __getview(self): + ''' Find the interface for a specific Class in the hyperdb. + + This method checks to see whether the schema has changed and + re-works the underlying metakit structure if it has. + ''' db = self.db._db view = db.view(self.classname) mkprops = view.structure() + + # if we have structure in the database, and the structure hasn't + # changed if mkprops and self.db.fastopen: return view.ordered(1) + # is the definition the same? for nm, rutyp in self.ruprops.items(): for mkprop in mkprops: @@ -1136,7 +1201,7 @@ class Class: return self.db._db.view(self.classname).ordered(1) def getindexview(self, RW=0): return self.db._db.view("_%s" % self.classname).ordered(1) - + def _fetchML(sv): l = [] for row in sv: @@ -1155,7 +1220,7 @@ def _fetchPW(s): return p def _fetchLink(n): - ''' Return None if the string is empty ?otherwise ensure it's a string? + ''' Return None if the link is 0 - otherwise strify it. ''' return n and str(n) or None diff --git a/roundup/backends/rdbms_common.py b/roundup/backends/rdbms_common.py index 5b6aab4..9fa79f7 100644 --- a/roundup/backends/rdbms_common.py +++ b/roundup/backends/rdbms_common.py @@ -1,4 +1,4 @@ -# $Id: rdbms_common.py,v 1.35 2003-02-25 10:19:32 richard Exp $ +# $Id: rdbms_common.py,v 1.36 2003-02-26 23:42:54 richard Exp $ ''' Relational database (SQL) backend common code. Basics: @@ -1143,6 +1143,7 @@ class Class(hyperdb.Class): elif isinstance(proptype, hyperdb.Password): value = str(value) l.append(repr(value)) + l.append(self.is_retired(nodeid)) return l def import_list(self, propnames, proplist): @@ -1184,6 +1185,16 @@ class Class(hyperdb.Class): value = pwd d[propname] = value + # retire? + if int(proplist[-1]): + # use the arg for __retired__ to cope with any odd database type + # conversion (hello, sqlite) + sql = 'update _%s set __retired__=%s where id=%s'%(self.classname, + self.db.arg, self.db.arg) + if __debug__: + print >>hyperdb.DEBUG, 'retire', (self, sql, newid) + self.db.cursor.execute(sql, (1, newid)) + # add the node and journal self.db.addnode(self.classname, newid, d) -- 2.30.2