X-Git-Url: https://git.tokkee.org/?a=blobdiff_plain;f=roundup%2Fbackends%2Fback_gadfly.py;h=8b01acc1cefdade654fafa7302e93c27084253da;hb=9c438c70783bcdc1f659796bc95404848cd61786;hp=ec62ba03a63f02e243d5dbcf1d7f49b88f9a1737;hpb=9639606db9eee8cc7e58ba5422848fe3a1a01a6d;p=roundup.git diff --git a/roundup/backends/back_gadfly.py b/roundup/backends/back_gadfly.py index ec62ba0..8b01acc 100644 --- a/roundup/backends/back_gadfly.py +++ b/roundup/backends/back_gadfly.py @@ -1,4 +1,4 @@ -# $Id: back_gadfly.py,v 1.1 2002-08-22 07:56:51 richard Exp $ +# $Id: back_gadfly.py,v 1.15 2002-09-10 00:11:50 richard Exp $ __doc__ = ''' About Gadfly ============ @@ -19,7 +19,7 @@ intermediate tables. Journals are stored adjunct to the per-class tables. -Table columns for properties have "_" prepended so the names can't +Table names and columns have "_" prepended so the names can't clash with restricted names (like "order"). Retirement is determined by the __retired__ column being true. @@ -48,7 +48,7 @@ used. ''' # standard python modules -import sys, os, time, re, errno, weakref +import sys, os, time, re, errno, weakref, copy # roundup modules from roundup import hyperdb, date, password, roundupdb, security @@ -57,7 +57,8 @@ from roundup.hyperdb import String, Password, Date, Interval, Link, \ # the all-important gadfly :) import gadfly -from gadfly import client +import gadfly.client +import gadfly.database # support from blobfiles import FileStorage @@ -78,6 +79,9 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): self.sessions = Sessions(self.config) self.security = security.Security(self) + # additional transaction support for external files and the like + self.transactions = [] + db = config.GADFLY_DATABASE if len(db) == 2: # ensure files are group readable and writable @@ -98,22 +102,26 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): cursor.execute('select schema from schema') self.database_schema = cursor.fetchone()[0] else: - self.conn = client.gfclient(*db) + self.conn = gadfly.client.gfclient(*db) cursor = self.conn.cursor() cursor.execute('select schema from schema') self.database_schema = cursor.fetchone()[0] + def __repr__(self): + return ''%id(self) + def post_init(self): ''' Called once the schema initialisation has finished. We should now confirm that the schema defined by our "classes" attribute actually matches the schema in the database. ''' + # now detect changes in the schema for classname, spec in self.classes.items(): if self.database_schema.has_key(classname): dbspec = self.database_schema[classname] - self.update_class(spec.schema(), dbspec) - self.database_schema[classname] = dbspec + self.update_class(spec, dbspec) + self.database_schema[classname] = spec.schema() else: self.create_class(spec) self.database_schema[classname] = spec.schema() @@ -122,12 +130,24 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): if not self.classes.has_key(classname): self.drop_class(classname) - # commit any changes + # update the database version of the schema cursor = self.conn.cursor() cursor.execute('delete from schema') cursor.execute('insert into schema values (?)', (self.database_schema,)) + + # reindex the db if necessary + if self.indexer.should_reindex(): + self.reindex() + + # commit self.conn.commit() + def reindex(self): + for klass in self.classes.values(): + for nodeid in klass.list(): + klass.index(nodeid) + self.indexer.save_index() + def determine_columns(self, spec): ''' Figure the column names and multilink properties from the spec ''' @@ -145,10 +165,89 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): def update_class(self, spec, dbspec): ''' Determine the differences between the current spec and the database version of the spec, and update where necessary + + NOTE that this doesn't work for adding/deleting properties! + ... until gadfly grows an ALTER TABLE command, it's not going to! ''' - if spec == dbspec: + spec_schema = spec.schema() + if spec_schema == dbspec: return - raise NotImplementedError + if __debug__: + print >>hyperdb.DEBUG, 'update_class FIRING' + + # key property changed? + if dbspec[0] != spec_schema[0]: + if __debug__: + print >>hyperdb.DEBUG, 'update_class setting keyprop', `spec[0]` + # XXX turn on indexing for the key property + + # dict 'em up + spec_propnames,spec_props = [],{} + for propname,prop in spec_schema[1]: + spec_propnames.append(propname) + spec_props[propname] = prop + dbspec_propnames,dbspec_props = [],{} + for propname,prop in dbspec[1]: + dbspec_propnames.append(propname) + dbspec_props[propname] = prop + + # we're going to need one of these + cursor = self.conn.cursor() + + # now compare + for propname in spec_propnames: + prop = spec_props[propname] + if __debug__: + print >>hyperdb.DEBUG, 'update_class ...', `prop` + if dbspec_props.has_key(propname) and prop==dbspec_props[propname]: + continue + if __debug__: + print >>hyperdb.DEBUG, 'update_class', `prop` + + if not dbspec_props.has_key(propname): + # add the property + if isinstance(prop, Multilink): + sql = 'create table %s_%s (linkid varchar, nodeid '\ + 'varchar)'%(spec.classname, prop) + if __debug__: + print >>hyperdb.DEBUG, 'update_class', (self, sql) + cursor.execute(sql) + else: + # XXX gadfly doesn't have an ALTER TABLE command + raise NotImplementedError + sql = 'alter table _%s add column (_%s varchar)'%( + spec.classname, propname) + if __debug__: + print >>hyperdb.DEBUG, 'update_class', (self, sql) + cursor.execute(sql) + else: + # modify the property + if __debug__: + print >>hyperdb.DEBUG, 'update_class NOOP' + pass # NOOP in gadfly + + # and the other way - only worry about deletions here + for propname in dbspec_propnames: + prop = dbspec_props[propname] + if spec_props.has_key(propname): + continue + if __debug__: + print >>hyperdb.DEBUG, 'update_class', `prop` + + # delete the property + if isinstance(prop, Multilink): + sql = 'drop table %s_%s'%(spec.classname, prop) + if __debug__: + print >>hyperdb.DEBUG, 'update_class', (self, sql) + cursor.execute(sql) + else: + # XXX gadfly doesn't have an ALTER TABLE command + raise NotImplementedError + sql = 'alter table _%s delete column _%s'%(spec.classname, + propname) + if __debug__: + print >>hyperdb.DEBUG, 'update_class', (self, sql) + cursor.execute(sql) def create_class(self, spec): ''' Create a database table according to the given spec. @@ -163,7 +262,7 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): # create the base table cols = ','.join(['%s varchar'%x for x in cols]) - sql = 'create table %s (%s)'%(spec.classname, cols) + sql = 'create table _%s (%s)'%(spec.classname, cols) if __debug__: print >>hyperdb.DEBUG, 'create_class', (self, sql) cursor.execute(sql) @@ -203,7 +302,7 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): mls.append(col) cursor = self.conn.cursor() - sql = 'drop table %s'%spec.classname + sql = 'drop table _%s'%spec.classname if __debug__: print >>hyperdb.DEBUG, 'drop_class', (self, sql) cursor.execute(sql) @@ -269,7 +368,7 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): print >>hyperdb.DEBUG, 'clear', (self,) cursor = self.conn.cursor() for cn in self.classes.keys(): - sql = 'delete from %s'%cn + sql = 'delete from _%s'%cn if __debug__: print >>hyperdb.DEBUG, 'clear', (self, sql) cursor.execute(sql) @@ -315,6 +414,8 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): def addnode(self, classname, nodeid, node): ''' Add the specified node to its class's db. ''' + if __debug__: + print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node) # gadfly requires values for all non-multilink columns cl = self.classes[classname] cols, mls = self.determine_columns(cl) @@ -334,7 +435,7 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): # perform the inserts cursor = self.conn.cursor() - sql = 'insert into %s (%s) values (%s)'%(classname, cols, s) + sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s) if __debug__: print >>hyperdb.DEBUG, 'addnode', (self, sql, vals) cursor.execute(sql, vals) @@ -349,9 +450,14 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): print >>hyperdb.DEBUG, 'addnode', (self, sql, vals) cursor.execute(sql, vals) - def setnode(self, classname, nodeid, node): + # make sure we do the commit-time extra stuff for this node + self.transactions.append((self.doSaveNode, (classname, nodeid, node))) + + def setnode(self, classname, nodeid, node, multilink_changes): ''' Change the specified node. ''' + if __debug__: + print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node) node = self.serialise(classname, node) cl = self.classes[classname] @@ -368,39 +474,62 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): # make sure the ordering is correct for column name -> column value vals = tuple([node[col[1:]] for col in cols]) - s = ','.join(['?' for x in cols]) + s = ','.join(['%s=?'%x for x in cols]) cols = ','.join(cols) # perform the update cursor = self.conn.cursor() - sql = 'update %s (%s) values (%s)'%(classname, cols, s) + sql = 'update _%s set %s'%(classname, s) if __debug__: print >>hyperdb.DEBUG, 'setnode', (self, sql, vals) cursor.execute(sql, vals) # now the fun bit, updating the multilinks ;) - # XXX TODO XXX + for col, (add, remove) in multilink_changes.items(): + tn = '%s_%s'%(classname, col) + if add: + sql = 'insert into %s (nodeid, linkid) values (?,?)'%tn + vals = [(nodeid, addid) for addid in add] + if __debug__: + print >>hyperdb.DEBUG, 'setnode (add)', (self, sql, vals) + cursor.execute(sql, vals) + if remove: + sql = 'delete from %s where nodeid=? and linkid=?'%tn + vals = [(nodeid, removeid) for removeid in remove] + if __debug__: + print >>hyperdb.DEBUG, 'setnode (rem)', (self, sql, vals) + cursor.execute(sql, vals) + + # make sure we do the commit-time extra stuff for this node + self.transactions.append((self.doSaveNode, (classname, nodeid, node))) def getnode(self, classname, nodeid): ''' Get a node from the database. ''' + if __debug__: + print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid) # figure the columns we're fetching cl = self.classes[classname] cols, mls = self.determine_columns(cl) - cols = ','.join(cols) + scols = ','.join(cols) # perform the basic property fetch cursor = self.conn.cursor() - sql = 'select %s from %s where id=?'%(cols, classname) + sql = 'select %s from _%s where id=?'%(scols, classname) if __debug__: print >>hyperdb.DEBUG, 'getnode', (self, sql, nodeid) cursor.execute(sql, (nodeid,)) - values = cursor.fetchone() + try: + values = cursor.fetchone() + except gadfly.database.error, message: + if message == 'no more results': + raise IndexError, 'no such %s node %s'%(classname, nodeid) + raise # make up the node node = {} for col in range(len(cols)): - node[col] = values[col] + node[cols[col][1:]] = values[col] # now the multilinks for col in mls: @@ -414,6 +543,29 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): return self.unserialise(classname, node) + def destroynode(self, classname, nodeid): + '''Remove a node from the database. Called exclusively by the + destroy() method on Class. + ''' + if __debug__: + print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid) + + # make sure the node exists + if not self.hasnode(classname, nodeid): + raise IndexError, '%s has no node %s'%(classname, nodeid) + + # see if there's any obvious commit actions that we should get rid of + for entry in self.transactions[:]: + if entry[1][:2] == (classname, nodeid): + self.transactions.remove(entry) + + # now do the SQL + cursor = self.conn.cursor() + sql = 'delete from _%s where id=?'%(classname) + if __debug__: + print >>hyperdb.DEBUG, 'destroynode', (self, sql, nodeid) + cursor.execute(sql, (nodeid,)) + def serialise(self, classname, node): '''Copy the node contents, converting non-marshallable data into marshallable data. @@ -475,7 +627,7 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): ''' Determine if the database has a given node. ''' cursor = self.conn.cursor() - sql = 'select count(*) from %s where nodeid=?'%classname + sql = 'select count(*) from _%s where id=?'%classname if __debug__: print >>hyperdb.DEBUG, 'hasnode', (self, sql, nodeid) cursor.execute(sql, (nodeid,)) @@ -485,20 +637,26 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): ''' Count the number of nodes that exist for a particular Class. ''' cursor = self.conn.cursor() - sql = 'select count(*) from %s'%classname + sql = 'select count(*) from _%s'%classname if __debug__: print >>hyperdb.DEBUG, 'countnodes', (self, sql) cursor.execute(sql) return cursor.fetchone()[0] - def getnodeids(self, classname): + def getnodeids(self, classname, retired=0): ''' Retrieve all the ids of the nodes for a particular Class. + + Set retired=None to get all nodes. Otherwise it'll get all the + retired or non-retired nodes, depending on the flag. ''' cursor = self.conn.cursor() - sql = 'select id from %s'%classname + # flip the sense of the flag if we don't want all of them + if retired is not None: + retired = not retired + sql = 'select id from _%s where __retired__ <> ?'%classname if __debug__: - print >>hyperdb.DEBUG, 'getnodeids', (self, sql) - cursor.execute(sql) + print >>hyperdb.DEBUG, 'getnodeids', (self, sql, retired) + cursor.execute(sql, (retired,)) return [x[0] for x in cursor.fetchall()] def addjournal(self, classname, nodeid, action, params): @@ -548,6 +706,11 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): def getjournal(self, classname, nodeid): ''' get the journal for id ''' + # make sure the node exists + if not self.hasnode(classname, nodeid): + raise IndexError, '%s has no node %s'%(classname, nodeid) + + # now get the journal entries cols = ','.join('nodeid date tag action params'.split()) cursor = self.conn.cursor() sql = 'select %s from %s__journal where nodeid=?'%(cols, classname) @@ -560,8 +723,7 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): return res def pack(self, pack_before): - ''' Pack the database, removing all journal entries before the - "pack_before" date. + ''' Delete all journal entries except "create" before 'pack_before'. ''' # get a 'yyyymmddhhmmss' version of the date date_stamp = pack_before.serialise() @@ -569,7 +731,8 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): # do the delete cursor = self.conn.cursor() for classname in self.classes.keys(): - sql = 'delete from %s__journal where date'create'"%classname if __debug__: print >>hyperdb.DEBUG, 'pack', (self, sql, date_stamp) cursor.execute(sql, (date_stamp,)) @@ -580,16 +743,53 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): Save all data changed since the database was opened or since the last commit() or rollback(). ''' + if __debug__: + print >>hyperdb.DEBUG, 'commit', (self,) + + # commit gadfly self.conn.commit() + # now, do all the other transaction stuff + reindex = {} + for method, args in self.transactions: + reindex[method(*args)] = 1 + + # reindex the nodes that request it + for classname, nodeid in filter(None, reindex.keys()): + print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid) + self.getclass(classname).index(nodeid) + + # save the indexer state + self.indexer.save_index() + + # clear out the transactions + self.transactions = [] + def rollback(self): ''' Reverse all actions from the current transaction. Undo all the changes made since the database was opened or the last commit() or rollback() was performed. ''' + if __debug__: + print >>hyperdb.DEBUG, 'rollback', (self,) + + # roll back gadfly self.conn.rollback() + # roll back "other" transaction stuff + for method, args in self.transactions: + # delete temporary files + if method == self.doStoreFile: + self.rollbackStoreFile(*args) + self.transactions = [] + + def doSaveNode(self, classname, nodeid, node): + ''' dummy that just generates a reindex event + ''' + # return the classname, nodeid so we reindex this content + return (classname, nodeid) + # # The base Class class # @@ -723,8 +923,8 @@ class Class(hyperdb.Class): l = [] for entry in value: if type(entry) != type(''): - raise ValueError, '"%s" link value (%s) must be '\ - 'String'%(key, value) + raise ValueError, '"%s" multilink value (%r) '\ + 'must contain Strings'%(key, value) # if it isn't a number, it's a key if not num_re.match(entry): try: @@ -836,7 +1036,11 @@ class Class(hyperdb.Class): name = self.db.getjournal(self.classname, nodeid)[0][2] else: return None - return self.db.user.lookup(name) + try: + return self.db.user.lookup(name) + except KeyError: + # the journaltag user doesn't exist any more + return None # get the property (raises KeyErorr if invalid) prop = self.properties[propname] @@ -845,7 +1049,7 @@ class Class(hyperdb.Class): d = self.db.getnode(self.classname, nodeid) #, cache=cache) if not d.has_key(propname): - if default is _marker: + if default is self._marker: if isinstance(prop, Multilink): return [] else: @@ -853,6 +1057,10 @@ class Class(hyperdb.Class): else: return default + # don't pass our list to other code + if isinstance(prop, Multilink): + return d[propname][:] + return d[propname] def getnode(self, nodeid, cache=1): @@ -886,7 +1094,198 @@ class Class(hyperdb.Class): If the value of a Link or Multilink property contains an invalid node id, a ValueError is raised. ''' - raise NotImplementedError + if not propvalues: + return propvalues + + if propvalues.has_key('creation') or propvalues.has_key('activity'): + raise KeyError, '"creation" and "activity" are reserved' + + if propvalues.has_key('id'): + raise KeyError, '"id" is reserved' + + if self.db.journaltag is None: + raise DatabaseError, 'Database open read-only' + + self.fireAuditors('set', nodeid, propvalues) + # Take a copy of the node dict so that the subsequent set + # operation doesn't modify the oldvalues structure. + # XXX used to try the cache here first + oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid)) + + node = self.db.getnode(self.classname, nodeid) + if self.is_retired(nodeid): + raise IndexError + num_re = re.compile('^\d+$') + + # if the journal value is to be different, store it in here + journalvalues = {} + + # remember the add/remove stuff for multilinks, making it easier + # for the Database layer to do its stuff + multilink_changes = {} + + for propname, value in propvalues.items(): + # check to make sure we're not duplicating an existing key + if propname == self.key and node[propname] != value: + try: + self.lookup(value) + except KeyError: + pass + else: + raise ValueError, 'node with key "%s" exists'%value + + # this will raise the KeyError if the property isn't valid + # ... we don't use getprops() here because we only care about + # the writeable properties. + prop = self.properties[propname] + + # if the value's the same as the existing value, no sense in + # doing anything + if node.has_key(propname) and value == node[propname]: + del propvalues[propname] + continue + + # do stuff based on the prop type + if isinstance(prop, Link): + link_class = prop.classname + # if it isn't a number, it's a key + if value is not None and not isinstance(value, type('')): + raise ValueError, 'property "%s" link value be a string'%( + propname) + if isinstance(value, type('')) and not num_re.match(value): + try: + value = self.db.classes[link_class].lookup(value) + except (TypeError, KeyError): + raise IndexError, 'new property "%s": %s not a %s'%( + propname, value, prop.classname) + + if (value is not None and + not self.db.getclass(link_class).hasnode(value)): + raise IndexError, '%s has no node %s'%(link_class, value) + + if self.do_journal and prop.do_journal: + # register the unlink with the old linked node + if node[propname] is not None: + self.db.addjournal(link_class, node[propname], 'unlink', + (self.classname, nodeid, propname)) + + # register the link with the newly linked node + if value is not None: + self.db.addjournal(link_class, value, 'link', + (self.classname, nodeid, propname)) + + elif isinstance(prop, Multilink): + if type(value) != type([]): + raise TypeError, 'new property "%s" not a list of'\ + ' ids'%propname + link_class = self.properties[propname].classname + l = [] + for entry in value: + # if it isn't a number, it's a key + if type(entry) != type(''): + raise ValueError, 'new property "%s" link value ' \ + 'must be a string'%propname + if not num_re.match(entry): + try: + entry = self.db.classes[link_class].lookup(entry) + except (TypeError, KeyError): + raise IndexError, 'new property "%s": %s not a %s'%( + propname, entry, + self.properties[propname].classname) + l.append(entry) + value = l + propvalues[propname] = value + + # figure the journal entry for this property + add = [] + remove = [] + + # handle removals + if node.has_key(propname): + l = node[propname] + else: + l = [] + for id in l[:]: + if id in value: + continue + # register the unlink with the old linked node + if self.do_journal and self.properties[propname].do_journal: + self.db.addjournal(link_class, id, 'unlink', + (self.classname, nodeid, propname)) + l.remove(id) + remove.append(id) + + # handle additions + for id in value: + if not self.db.getclass(link_class).hasnode(id): + raise IndexError, '%s has no node %s'%(link_class, id) + if id in l: + continue + # register the link with the newly linked node + if self.do_journal and self.properties[propname].do_journal: + self.db.addjournal(link_class, id, 'link', + (self.classname, nodeid, propname)) + l.append(id) + add.append(id) + + # figure the journal entry + l = [] + if add: + l.append(('+', add)) + if remove: + l.append(('-', remove)) + multilink_changes[propname] = (add, remove) + if l: + journalvalues[propname] = tuple(l) + + elif isinstance(prop, String): + if value is not None and type(value) != type(''): + raise TypeError, 'new property "%s" not a string'%propname + + elif isinstance(prop, Password): + if not isinstance(value, password.Password): + raise TypeError, 'new property "%s" not a Password'%propname + propvalues[propname] = value + + elif value is not None and isinstance(prop, Date): + if not isinstance(value, date.Date): + raise TypeError, 'new property "%s" not a Date'% propname + propvalues[propname] = value + + elif value is not None and isinstance(prop, Interval): + if not isinstance(value, date.Interval): + raise TypeError, 'new property "%s" not an '\ + 'Interval'%propname + propvalues[propname] = value + + elif value is not None and isinstance(prop, Number): + try: + float(value) + except ValueError: + raise TypeError, 'new property "%s" not numeric'%propname + + elif value is not None and isinstance(prop, Boolean): + try: + int(value) + except ValueError: + raise TypeError, 'new property "%s" not boolean'%propname + + node[propname] = value + + # nothing to do? + if not propvalues: + return propvalues + + # do the set, and journal it + self.db.setnode(self.classname, nodeid, node, multilink_changes) + + if self.do_journal: + propvalues.update(journalvalues) + self.db.addjournal(self.classname, nodeid, 'set', propvalues) + + self.fireReactors('set', nodeid, oldvalues) + + return propvalues def retire(self, nodeid): '''Retire a node. @@ -897,8 +1296,11 @@ class Class(hyperdb.Class): Retired nodes are not returned by the find(), list(), or lookup() methods, and other nodes may reuse the values of their key properties. ''' + if self.db.journaltag is None: + raise DatabaseError, 'Database open read-only' + cursor = self.db.conn.cursor() - sql = 'update %s set __retired__=1 where id=?'%self.classname + sql = 'update _%s set __retired__=1 where id=?'%self.classname if __debug__: print >>hyperdb.DEBUG, 'retire', (self, sql, nodeid) cursor.execute(sql, (nodeid,)) @@ -907,7 +1309,7 @@ class Class(hyperdb.Class): '''Return true if the node is rerired ''' cursor = self.db.conn.cursor() - sql = 'select __retired__ from %s where id=?'%self.classname + sql = 'select __retired__ from _%s where id=?'%self.classname if __debug__: print >>hyperdb.DEBUG, 'is_retired', (self, sql, nodeid) cursor.execute(sql, (nodeid,)) @@ -930,7 +1332,9 @@ class Class(hyperdb.Class): entries. It will no longer be available, and will generally break code if there are any references to the node. ''' - raise NotImplementedError + if self.db.journaltag is None: + raise DatabaseError, 'Database open read-only' + self.db.destroynode(self.classname, nodeid) def history(self, nodeid): '''Retrieve the journal of edits on a particular node. @@ -945,7 +1349,9 @@ class Class(hyperdb.Class): 'date' is a Timestamp object specifying the time of the change and 'tag' is the journaltag specified when the database was opened. ''' - raise NotImplementedError + if not self.do_journal: + raise ValueError, 'Journalling is disabled for this class' + return self.db.getjournal(self.classname, nodeid) # Locating nodes: def hasnode(self, nodeid): @@ -980,7 +1386,19 @@ class Class(hyperdb.Class): 3. "title" property 4. first property from the sorted property name list ''' - raise NotImplementedError + k = self.getkey() + if k: + return k + props = self.getprops() + if props.has_key('name'): + return 'name' + elif props.has_key('title'): + return 'title' + if default_to_id: + return 'id' + props = props.keys() + props.sort() + return props[0] def lookup(self, keyvalue): '''Locate a particular node by its key property and return its id. @@ -994,7 +1412,7 @@ class Class(hyperdb.Class): raise TypeError, 'No key property set' cursor = self.db.conn.cursor() - sql = 'select id from %s where _%s=?'%(self.classname, self.key) + sql = 'select id from _%s where _%s=?'%(self.classname, self.key) if __debug__: print >>hyperdb.DEBUG, 'lookup', (self, sql, keyvalue) cursor.execute(sql, (keyvalue,)) @@ -1022,15 +1440,126 @@ class Class(hyperdb.Class): db.issue.find(messages={'1':1,'3':1}, files={'7':1}) ''' - raise NotImplementedError + if __debug__: + print >>hyperdb.DEBUG, 'find', (self, propspec) + if not propspec: + return [] + queries = [] + tables = [] + allvalues = () + for prop, values in propspec.items(): + allvalues += tuple(values.keys()) + tables.append('select nodeid from %s_%s where linkid in (%s)'%( + self.classname, prop, ','.join(['?' for x in values.keys()]))) + sql = '\nintersect\n'.join(tables) + if __debug__: + print >>hyperdb.DEBUG, 'find', (self, sql, allvalues) + cursor = self.db.conn.cursor() + cursor.execute(sql, allvalues) + try: + l = [x[0] for x in cursor.fetchall()] + except gadfly.database.error, message: + if message == 'no more results': + l = [] + raise + if __debug__: + print >>hyperdb.DEBUG, 'find ... ', l + return l + + def list(self): + ''' Return a list of the ids of the active nodes in this class. + ''' + return self.db.getnodeids(self.classname, retired=0) - def filter(self, search_matches, filterspec, sort, group, - num_re = re.compile('^\d+$')): + def filter(self, search_matches, filterspec, sort, group): ''' Return a list of the ids of the active nodes in this class that match the 'filter' spec, sorted by the group spec and then the sort spec + + "filterspec" is {propname: value(s)} + "sort" and "group" are (dir, prop) where dir is '+', '-' or None + and prop is a prop name or None + "search_matches" is {nodeid: marker} ''' - raise NotImplementedError + cn = self.classname + + # figure the WHERE clause from the filterspec + props = self.getprops() + frum = ['_'+cn] + where = [] + args = [] + for k, v in filterspec.items(): + propclass = props[k] + if isinstance(propclass, Multilink): + tn = '%s_%s'%(cn, k) + frum.append(tn) + if isinstance(v, type([])): + s = ','.join(['?' for x in v]) + where.append('id=%s.nodeid and %s.linkid in (%s)'%(tn,tn,s)) + args = args + v + else: + where.append('id=%s.nodeid and %s.linkid = ?'%(tn, tn)) + args.append(v) + else: + if isinstance(v, type([])): + s = ','.join(['?' for x in v]) + where.append('_%s in (%s)'%(k, s)) + args = args + v + else: + where.append('_%s=?'%k) + args.append(v) + + # add results of full text search + if search_matches is not None: + v = search_matches.keys() + s = ','.join(['?' for x in v]) + where.append('id in (%s)'%s) + args = args + v + + # figure the order by clause + orderby = [] + ordercols = [] + if sort[0] is not None and sort[1] is not None: + if sort[0] != '-': + orderby.append('_'+sort[1]) + ordercols.append(sort[1]) + else: + orderby.append('_'+sort[1]+' desc') + ordercols.append(sort[1]) + + # figure the group by clause + groupby = [] + groupcols = [] + if group[0] is not None and group[1] is not None: + if group[0] != '-': + groupby.append('_'+group[1]) + groupcols.append(group[1]) + else: + groupby.append('_'+group[1]+' desc') + groupcols.append(group[1]) + + # construct the SQL + frum = ','.join(frum) + where = ' and '.join(where) + cols = ['id'] + if orderby: + cols = cols + ordercols + order = ' order by %s'%(','.join(orderby)) + else: + order = '' + if groupby: + cols = cols + groupcols + group = ' group by %s'%(','.join(groupby)) + else: + group = '' + cols = ','.join(cols) + sql = 'select %s from %s where %s%s%s'%(cols, frum, where, order, + group) + args = tuple(args) + if __debug__: + print >>hyperdb.DEBUG, 'find', (self, sql, args) + cursor = self.db.conn.cursor() + cursor.execute(sql, args) def count(self): '''Get the number of nodes in this class. @@ -1166,7 +1695,7 @@ class FileClass(Class): # BUG: by catching this we donot see an error in the log. return 'ERROR reading file: %s%s\n%s\n%s'%( self.classname, nodeid, poss_msg, strerror) - if default is not _marker: + if default is not self._marker: return Class.get(self, nodeid, propname, default, cache=cache) else: return Class.get(self, nodeid, propname, cache=cache) @@ -1220,404 +1749,9 @@ class IssueClass(Class, roundupdb.IssueClass): if not properties.has_key('files'): properties['files'] = hyperdb.Multilink("file") if not properties.has_key('nosy'): - properties['nosy'] = hyperdb.Multilink("user") + # note: journalling is turned off as it really just wastes + # space. this behaviour may be overridden in an instance + properties['nosy'] = hyperdb.Multilink("user", do_journal="no") if not properties.has_key('superseder'): properties['superseder'] = hyperdb.Multilink(classname) Class.__init__(self, db, classname, **properties) - -# -# $Log: not supported by cvs2svn $ -# Revision 1.80 2002/08/16 04:28:13 richard -# added is_retired query to Class -# -# Revision 1.79 2002/07/29 23:30:14 richard -# documentation reorg post-new-security -# -# Revision 1.78 2002/07/21 03:26:37 richard -# Gordon, does this help? -# -# Revision 1.77 2002/07/18 11:27:47 richard -# ws -# -# Revision 1.76 2002/07/18 11:17:30 gmcm -# Add Number and Boolean types to hyperdb. -# Add conversion cases to web, mail & admin interfaces. -# Add storage/serialization cases to back_anydbm & back_metakit. -# -# Revision 1.75 2002/07/14 02:05:53 richard -# . all storage-specific code (ie. backend) is now implemented by the backends -# -# Revision 1.74 2002/07/10 00:24:10 richard -# braino -# -# Revision 1.73 2002/07/10 00:19:48 richard -# Added explicit closing of backend database handles. -# -# Revision 1.72 2002/07/09 21:53:38 gmcm -# Optimize Class.find so that the propspec can contain a set of ids to match. -# This is used by indexer.search so it can do just one find for all the index matches. -# This was already confusing code, but for common terms (lots of index matches), -# it is enormously faster. -# -# Revision 1.71 2002/07/09 03:02:52 richard -# More indexer work: -# - all String properties may now be indexed too. Currently there's a bit of -# "issue" specific code in the actual searching which needs to be -# addressed. In a nutshell: -# + pass 'indexme="yes"' as a String() property initialisation arg, eg: -# file = FileClass(db, "file", name=String(), type=String(), -# comment=String(indexme="yes")) -# + the comment will then be indexed and be searchable, with the results -# related back to the issue that the file is linked to -# - as a result of this work, the FileClass has a default MIME type that may -# be overridden in a subclass, or by the use of a "type" property as is -# done in the default templates. -# - the regeneration of the indexes (if necessary) is done once the schema is -# set up in the dbinit. -# -# Revision 1.70 2002/06/27 12:06:20 gmcm -# Improve an error message. -# -# Revision 1.69 2002/06/17 23:15:29 richard -# Can debug to stdout now -# -# Revision 1.68 2002/06/11 06:52:03 richard -# . #564271 ] find() and new properties -# -# Revision 1.67 2002/06/11 05:02:37 richard -# . #565979 ] code error in hyperdb.Class.find -# -# Revision 1.66 2002/05/25 07:16:24 rochecompaan -# Merged search_indexing-branch with HEAD -# -# Revision 1.65 2002/05/22 04:12:05 richard -# . applied patch #558876 ] cgi client customization -# ... with significant additions and modifications ;) -# - extended handling of ML assignedto to all places it's handled -# - added more NotFound info -# -# Revision 1.64 2002/05/15 06:21:21 richard -# . node caching now works, and gives a small boost in performance -# -# As a part of this, I cleaned up the DEBUG output and implemented TRACE -# output (HYPERDBTRACE='file to trace to') with checkpoints at the start of -# CGI requests. Run roundup with python -O to skip all the DEBUG/TRACE stuff -# (using if __debug__ which is compiled out with -O) -# -# Revision 1.63 2002/04/15 23:25:15 richard -# . node ids are now generated from a lockable store - no more race conditions -# -# We're using the portalocker code by Jonathan Feinberg that was contributed -# to the ASPN Python cookbook. This gives us locking across Unix and Windows. -# -# Revision 1.62 2002/04/03 07:05:50 richard -# d'oh! killed retirement of nodes :( -# all better now... -# -# Revision 1.61 2002/04/03 06:11:51 richard -# Fix for old databases that contain properties that don't exist any more. -# -# Revision 1.60 2002/04/03 05:54:31 richard -# Fixed serialisation problem by moving the serialisation step out of the -# hyperdb.Class (get, set) into the hyperdb.Database. -# -# Also fixed htmltemplate after the showid changes I made yesterday. -# -# Unit tests for all of the above written. -# -# Revision 1.59.2.2 2002/04/20 13:23:33 rochecompaan -# We now have a separate search page for nodes. Search links for -# different classes can be customized in instance_config similar to -# index links. -# -# Revision 1.59.2.1 2002/04/19 19:54:42 rochecompaan -# cgi_client.py -# removed search link for the time being -# moved rendering of matches to htmltemplate -# hyperdb.py -# filtering of nodes on full text search incorporated in filter method -# roundupdb.py -# added paramater to call of filter method -# roundup_indexer.py -# added search method to RoundupIndexer class -# -# Revision 1.59 2002/03/12 22:52:26 richard -# more pychecker warnings removed -# -# Revision 1.58 2002/02/27 03:23:16 richard -# Ran it through pychecker, made fixes -# -# Revision 1.57 2002/02/20 05:23:24 richard -# Didn't accomodate new values for new properties -# -# Revision 1.56 2002/02/20 05:05:28 richard -# . Added simple editing for classes that don't define a templated interface. -# - access using the admin "class list" interface -# - limited to admin-only -# - requires the csv module from object-craft (url given if it's missing) -# -# Revision 1.55 2002/02/15 07:27:12 richard -# Oops, precedences around the way w0rng. -# -# Revision 1.54 2002/02/15 07:08:44 richard -# . Alternate email addresses are now available for users. See the MIGRATION -# file for info on how to activate the feature. -# -# Revision 1.53 2002/01/22 07:21:13 richard -# . fixed back_bsddb so it passed the journal tests -# -# ... it didn't seem happy using the back_anydbm _open method, which is odd. -# Yet another occurrance of whichdb not being able to recognise older bsddb -# databases. Yadda yadda. Made the HYPERDBDEBUG stuff more sane in the -# process. -# -# Revision 1.52 2002/01/21 16:33:19 rochecompaan -# You can now use the roundup-admin tool to pack the database -# -# Revision 1.51 2002/01/21 03:01:29 richard -# brief docco on the do_journal argument -# -# Revision 1.50 2002/01/19 13:16:04 rochecompaan -# Journal entries for link and multilink properties can now be switched on -# or off. -# -# Revision 1.49 2002/01/16 07:02:57 richard -# . lots of date/interval related changes: -# - more relaxed date format for input -# -# Revision 1.48 2002/01/14 06:32:34 richard -# . #502951 ] adding new properties to old database -# -# Revision 1.47 2002/01/14 02:20:15 richard -# . changed all config accesses so they access either the instance or the -# config attriubute on the db. This means that all config is obtained from -# instance_config instead of the mish-mash of classes. This will make -# switching to a ConfigParser setup easier too, I hope. -# -# At a minimum, this makes migration a _little_ easier (a lot easier in the -# 0.5.0 switch, I hope!) -# -# Revision 1.46 2002/01/07 10:42:23 richard -# oops -# -# Revision 1.45 2002/01/02 04:18:17 richard -# hyperdb docstrings -# -# Revision 1.44 2002/01/02 02:31:38 richard -# Sorry for the huge checkin message - I was only intending to implement #496356 -# but I found a number of places where things had been broken by transactions: -# . modified ROUNDUPDBSENDMAILDEBUG to be SENDMAILDEBUG and hold a filename -# for _all_ roundup-generated smtp messages to be sent to. -# . the transaction cache had broken the roundupdb.Class set() reactors -# . newly-created author users in the mailgw weren't being committed to the db -# -# Stuff that made it into CHANGES.txt (ie. the stuff I was actually working -# on when I found that stuff :): -# . #496356 ] Use threading in messages -# . detectors were being registered multiple times -# . added tests for mailgw -# . much better attaching of erroneous messages in the mail gateway -# -# Revision 1.43 2001/12/20 06:13:24 rochecompaan -# Bugs fixed: -# . Exception handling in hyperdb for strings-that-look-like numbers got -# lost somewhere -# . Internet Explorer submits full path for filename - we now strip away -# the path -# Features added: -# . Link and multilink properties are now displayed sorted in the cgi -# interface -# -# Revision 1.42 2001/12/16 10:53:37 richard -# take a copy of the node dict so that the subsequent set -# operation doesn't modify the oldvalues structure -# -# Revision 1.41 2001/12/15 23:47:47 richard -# Cleaned up some bare except statements -# -# Revision 1.40 2001/12/14 23:42:57 richard -# yuck, a gdbm instance tests false :( -# I've left the debugging code in - it should be removed one day if we're ever -# _really_ anal about performace :) -# -# Revision 1.39 2001/12/02 05:06:16 richard -# . We now use weakrefs in the Classes to keep the database reference, so -# the close() method on the database is no longer needed. -# I bumped the minimum python requirement up to 2.1 accordingly. -# . #487480 ] roundup-server -# . #487476 ] INSTALL.txt -# -# I also cleaned up the change message / post-edit stuff in the cgi client. -# There's now a clearly marked "TODO: append the change note" where I believe -# the change note should be added there. The "changes" list will obviously -# have to be modified to be a dict of the changes, or somesuch. -# -# More testing needed. -# -# Revision 1.38 2001/12/01 07:17:50 richard -# . We now have basic transaction support! Information is only written to -# the database when the commit() method is called. Only the anydbm -# backend is modified in this way - neither of the bsddb backends have been. -# The mail, admin and cgi interfaces all use commit (except the admin tool -# doesn't have a commit command, so interactive users can't commit...) -# . Fixed login/registration forwarding the user to the right page (or not, -# on a failure) -# -# Revision 1.37 2001/11/28 21:55:35 richard -# . login_action and newuser_action return values were being ignored -# . Woohoo! Found that bloody re-login bug that was killing the mail -# gateway. -# (also a minor cleanup in hyperdb) -# -# Revision 1.36 2001/11/27 03:16:09 richard -# Another place that wasn't handling missing properties. -# -# Revision 1.35 2001/11/22 15:46:42 jhermann -# Added module docstrings to all modules. -# -# Revision 1.34 2001/11/21 04:04:43 richard -# *sigh* more missing value handling -# -# Revision 1.33 2001/11/21 03:40:54 richard -# more new property handling -# -# Revision 1.32 2001/11/21 03:11:28 richard -# Better handling of new properties. -# -# Revision 1.31 2001/11/12 22:01:06 richard -# Fixed issues with nosy reaction and author copies. -# -# Revision 1.30 2001/11/09 10:11:08 richard -# . roundup-admin now handles all hyperdb exceptions -# -# Revision 1.29 2001/10/27 00:17:41 richard -# Made Class.stringFind() do caseless matching. -# -# Revision 1.28 2001/10/21 04:44:50 richard -# bug #473124: UI inconsistency with Link fields. -# This also prompted me to fix a fairly long-standing usability issue - -# that of being able to turn off certain filters. -# -# Revision 1.27 2001/10/20 23:44:27 richard -# Hyperdatabase sorts strings-that-look-like-numbers as numbers now. -# -# Revision 1.26 2001/10/16 03:48:01 richard -# admin tool now complains if a "find" is attempted with a non-link property. -# -# Revision 1.25 2001/10/11 00:17:51 richard -# Reverted a change in hyperdb so the default value for missing property -# values in a create() is None and not '' (the empty string.) This obviously -# breaks CSV import/export - the string 'None' will be created in an -# export/import operation. -# -# Revision 1.24 2001/10/10 03:54:57 richard -# Added database importing and exporting through CSV files. -# Uses the csv module from object-craft for exporting if it's available. -# Requires the csv module for importing. -# -# Revision 1.23 2001/10/09 23:58:10 richard -# Moved the data stringification up into the hyperdb.Class class' get, set -# and create methods. This means that the data is also stringified for the -# journal call, and removes duplication of code from the backends. The -# backend code now only sees strings. -# -# Revision 1.22 2001/10/09 07:25:59 richard -# Added the Password property type. See "pydoc roundup.password" for -# implementation details. Have updated some of the documentation too. -# -# Revision 1.21 2001/10/05 02:23:24 richard -# . roundup-admin create now prompts for property info if none is supplied -# on the command-line. -# . hyperdb Class getprops() method may now return only the mutable -# properties. -# . Login now uses cookies, which makes it a whole lot more flexible. We can -# now support anonymous user access (read-only, unless there's an -# "anonymous" user, in which case write access is permitted). Login -# handling has been moved into cgi_client.Client.main() -# . The "extended" schema is now the default in roundup init. -# . The schemas have had their page headings modified to cope with the new -# login handling. Existing installations should copy the interfaces.py -# file from the roundup lib directory to their instance home. -# . Incorrectly had a Bizar Software copyright on the cgitb.py module from -# Ping - has been removed. -# . Fixed a whole bunch of places in the CGI interface where we should have -# been returning Not Found instead of throwing an exception. -# . Fixed a deviation from the spec: trying to modify the 'id' property of -# an item now throws an exception. -# -# Revision 1.20 2001/10/04 02:12:42 richard -# Added nicer command-line item adding: passing no arguments will enter an -# interactive more which asks for each property in turn. While I was at it, I -# fixed an implementation problem WRT the spec - I wasn't raising a -# ValueError if the key property was missing from a create(). Also added a -# protected=boolean argument to getprops() so we can list only the mutable -# properties (defaults to yes, which lists the immutables). -# -# Revision 1.19 2001/08/29 04:47:18 richard -# Fixed CGI client change messages so they actually include the properties -# changed (again). -# -# Revision 1.18 2001/08/16 07:34:59 richard -# better CGI text searching - but hidden filter fields are disappearing... -# -# Revision 1.17 2001/08/16 06:59:58 richard -# all searches use re now - and they're all case insensitive -# -# Revision 1.16 2001/08/15 23:43:18 richard -# Fixed some isFooTypes that I missed. -# Refactored some code in the CGI code. -# -# Revision 1.15 2001/08/12 06:32:36 richard -# using isinstance(blah, Foo) now instead of isFooType -# -# Revision 1.14 2001/08/07 00:24:42 richard -# stupid typo -# -# Revision 1.13 2001/08/07 00:15:51 richard -# Added the copyright/license notice to (nearly) all files at request of -# Bizar Software. -# -# Revision 1.12 2001/08/02 06:38:17 richard -# Roundupdb now appends "mailing list" information to its messages which -# include the e-mail address and web interface address. Templates may -# override this in their db classes to include specific information (support -# instructions, etc). -# -# Revision 1.11 2001/08/01 04:24:21 richard -# mailgw was assuming certain properties existed on the issues being created. -# -# Revision 1.10 2001/07/30 02:38:31 richard -# get() now has a default arg - for migration only. -# -# Revision 1.9 2001/07/29 09:28:23 richard -# Fixed sorting by clicking on column headings. -# -# Revision 1.8 2001/07/29 08:27:40 richard -# Fixed handling of passed-in values in form elements (ie. during a -# drill-down) -# -# Revision 1.7 2001/07/29 07:01:39 richard -# Added vim command to all source so that we don't get no steenkin' tabs :) -# -# Revision 1.6 2001/07/29 05:36:14 richard -# Cleanup of the link label generation. -# -# Revision 1.5 2001/07/29 04:05:37 richard -# Added the fabricated property "id". -# -# Revision 1.4 2001/07/27 06:25:35 richard -# Fixed some of the exceptions so they're the right type. -# Removed the str()-ification of node ids so we don't mask oopsy errors any -# more. -# -# Revision 1.3 2001/07/27 05:17:14 richard -# just some comments -# -# Revision 1.2 2001/07/22 12:09:32 richard -# Final commit of Grande Splite -# -# Revision 1.1 2001/07/22 11:58:35 richard -# More Grande Splite -# -# -# vim: set filetype=python ts=4 sw=4 et si