From 15a5ffa06710ead2fbf5136caf16545961899b70 Mon Sep 17 00:00:00 2001 From: richard Date: Sun, 14 Jul 2002 02:05:54 +0000 Subject: [PATCH] . all storage-specific code (ie. backend) is now implemented by the backends git-svn-id: http://svn.roundup-tracker.org/svnroot/roundup/trunk@870 57a73879-2fb5-44c3-a270-3262357dd7e2 --- CHANGES.txt | 2 +- doc/upgrading.txt | 2 + roundup/backends/back_anydbm.py | 1096 +++++++++++++++++++++++++- roundup/backends/back_bsddb.py | 14 +- roundup/backends/back_bsddb3.py | 9 +- roundup/hyperdb.py | 748 ++---------------- roundup/init.py | 9 +- roundup/roundupdb.py | 254 +----- roundup/templates/classic/dbinit.py | 37 +- roundup/templates/extended/dbinit.py | 36 +- test/test_db.py | 68 +- test/test_init.py | 12 +- test/test_schema.py | 27 +- 13 files changed, 1298 insertions(+), 1016 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index e751d80..3558ca1 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -37,7 +37,7 @@ Feature: . added sorting of checklist HTML display . switched to using a session-based web login . made mailgw handle set and modify operations on multilinks (bug #579094) - + . all storage-specific code (ie. backend) is now implemented by the backends 2002-06-24 0.4.2 Fixed: diff --git a/doc/upgrading.txt b/doc/upgrading.txt index 2be9d30..e768d85 100644 --- a/doc/upgrading.txt +++ b/doc/upgrading.txt @@ -16,6 +16,8 @@ Migrating from 0.4.x to 0.5.0 TODO: mention stuff about indexing TODO: mention that the dbinit needs the db.post_init() method call for reindexing +TODO: dbinit now imports classes from selct_db +TODO: select_db needs fixing to include Class, FileClass and IssueClass Migrating from 0.4.1 to 0.4.2 diff --git a/roundup/backends/back_anydbm.py b/roundup/backends/back_anydbm.py index 4e1ee10..0acff4a 100644 --- a/roundup/backends/back_anydbm.py +++ b/roundup/backends/back_anydbm.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -#$Id: back_anydbm.py,v 1.43 2002-07-10 06:30:30 richard Exp $ +#$Id: back_anydbm.py,v 1.44 2002-07-14 02:05:53 richard Exp $ ''' This module defines a backend that saves the hyperdatabase in a database chosen by anydbm. It is guaranteed to always be available in python @@ -23,16 +23,18 @@ versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several serious bugs, and is not available) ''' -import whichdb, anydbm, os, marshal -from roundup import hyperdb, date +import whichdb, anydbm, os, marshal, re, weakref, string, copy +from roundup import hyperdb, date, password, roundupdb from blobfiles import FileStorage from roundup.indexer import Indexer from locking import acquire_lock, release_lock +from roundup.hyperdb import String, Password, Date, Interval, Link, \ + Multilink, DatabaseError # # Now the database # -class Database(FileStorage, hyperdb.Database): +class Database(FileStorage, hyperdb.Database, roundupdb.Database): """A database for storing records containing flexible data types. Transaction stuff TODO: @@ -268,6 +270,63 @@ class Database(FileStorage, hyperdb.Database): return res + def serialise(self, classname, node): + '''Copy the node contents, converting non-marshallable data into + marshallable data. + ''' + if __debug__: + print >>hyperdb.DEBUG, 'serialise', classname, node + properties = self.getclass(classname).getprops() + d = {} + for k, v in node.items(): + # if the property doesn't exist, or is the "retired" flag then + # it won't be in the properties dict + if not properties.has_key(k): + d[k] = v + continue + + # get the property spec + prop = properties[k] + + if isinstance(prop, Password): + d[k] = str(v) + elif isinstance(prop, Date) and v is not None: + d[k] = v.get_tuple() + elif isinstance(prop, Interval) and v is not None: + d[k] = v.get_tuple() + else: + d[k] = v + return d + + def unserialise(self, classname, node): + '''Decode the marshalled node data + ''' + if __debug__: + print >>hyperdb.DEBUG, 'unserialise', classname, node + properties = self.getclass(classname).getprops() + d = {} + for k, v in node.items(): + # if the property doesn't exist, or is the "retired" flag then + # it won't be in the properties dict + if not properties.has_key(k): + d[k] = v + continue + + # get the property spec + prop = properties[k] + + if isinstance(prop, Date) and v is not None: + d[k] = date.Date(v) + elif isinstance(prop, Interval) and v is not None: + d[k] = date.Interval(v) + elif isinstance(prop, Password): + p = password.Password() + p.unpack(v) + d[k] = p + else: + d[k] = v + return d + def hasnode(self, classname, nodeid, db=None): ''' determine if the database has a given node ''' @@ -509,8 +568,1037 @@ class Database(FileStorage, hyperdb.Database): self.newnodes = {} self.transactions = [] +_marker = [] +class Class(hyperdb.Class): + """The handle to a particular class of nodes in a hyperdatabase.""" + + def __init__(self, db, classname, **properties): + """Create a new class with a given name and property specification. + + 'classname' must not collide with the name of an existing class, + or a ValueError is raised. The keyword arguments in 'properties' + must map names to property objects, or a TypeError is raised. + """ + if (properties.has_key('creation') or properties.has_key('activity') + or properties.has_key('creator')): + raise ValueError, '"creation", "activity" and "creator" are '\ + 'reserved' + + self.classname = classname + self.properties = properties + self.db = weakref.proxy(db) # use a weak ref to avoid circularity + self.key = '' + + # do the db-related init stuff + db.addclass(self) + + self.auditors = {'create': [], 'set': [], 'retire': []} + self.reactors = {'create': [], 'set': [], 'retire': []} + + def __repr__(self): + '''Slightly more useful representation + ''' + return ''%self.classname + + # Editing nodes: + + def create(self, **propvalues): + """Create a new node of this class and return its id. + + The keyword arguments in 'propvalues' map property names to values. + + The values of arguments must be acceptable for the types of their + corresponding properties or a TypeError is raised. + + If this class has a key property, it must be present and its value + must not collide with other key strings or a ValueError is raised. + + Any other properties on this class that are missing from the + 'propvalues' dictionary are set to None. + + If an id in a link or multilink property does not refer to a valid + node, an IndexError is raised. + + These operations trigger detectors and can be vetoed. Attempts + to modify the "creation" or "activity" properties cause a KeyError. + """ + if propvalues.has_key('id'): + raise KeyError, '"id" is reserved' + + if self.db.journaltag is None: + raise DatabaseError, 'Database open read-only' + + if propvalues.has_key('creation') or propvalues.has_key('activity'): + raise KeyError, '"creation" and "activity" are reserved' + + self.fireAuditors('create', None, propvalues) + + # new node's id + newid = self.db.newid(self.classname) + + # validate propvalues + num_re = re.compile('^\d+$') + for key, value in propvalues.items(): + if key == self.key: + try: + self.lookup(value) + except KeyError: + pass + else: + raise ValueError, 'node with key "%s" exists'%value + + # try to handle this property + try: + prop = self.properties[key] + except KeyError: + raise KeyError, '"%s" has no property "%s"'%(self.classname, + key) + + if isinstance(prop, Link): + if type(value) != type(''): + raise ValueError, 'link value must be String' + link_class = self.properties[key].classname + # if it isn't a number, it's a key + if not num_re.match(value): + try: + value = self.db.classes[link_class].lookup(value) + except (TypeError, KeyError): + raise IndexError, 'new property "%s": %s not a %s'%( + key, value, link_class) + elif not self.db.hasnode(link_class, value): + raise IndexError, '%s has no node %s'%(link_class, value) + + # save off the value + propvalues[key] = value + + # register the link with the newly linked node + if self.properties[key].do_journal: + self.db.addjournal(link_class, value, 'link', + (self.classname, newid, key)) + + elif isinstance(prop, Multilink): + if type(value) != type([]): + raise TypeError, 'new property "%s" not a list of ids'%key + + # clean up and validate the list of links + link_class = self.properties[key].classname + l = [] + for entry in value: + if type(entry) != type(''): + raise ValueError, '"%s" link value (%s) must be '\ + 'String'%(key, value) + # if it isn't a number, it's a key + if not num_re.match(entry): + try: + entry = self.db.classes[link_class].lookup(entry) + except (TypeError, KeyError): + raise IndexError, 'new property "%s": %s not a %s'%( + key, entry, self.properties[key].classname) + l.append(entry) + value = l + propvalues[key] = value + + # handle additions + for id in value: + if not self.db.hasnode(link_class, id): + raise IndexError, '%s has no node %s'%(link_class, id) + # register the link with the newly linked node + if self.properties[key].do_journal: + self.db.addjournal(link_class, id, 'link', + (self.classname, newid, key)) + + elif isinstance(prop, String): + if type(value) != type(''): + raise TypeError, 'new property "%s" not a string'%key + + elif isinstance(prop, Password): + if not isinstance(value, password.Password): + raise TypeError, 'new property "%s" not a Password'%key + + elif isinstance(prop, Date): + if value is not None and not isinstance(value, date.Date): + raise TypeError, 'new property "%s" not a Date'%key + + elif isinstance(prop, Interval): + if value is not None and not isinstance(value, date.Interval): + raise TypeError, 'new property "%s" not an Interval'%key + + # make sure there's data where there needs to be + for key, prop in self.properties.items(): + if propvalues.has_key(key): + continue + if key == self.key: + raise ValueError, 'key property "%s" is required'%key + if isinstance(prop, Multilink): + propvalues[key] = [] + else: + # TODO: None isn't right here, I think... + propvalues[key] = None + + # done + self.db.addnode(self.classname, newid, propvalues) + self.db.addjournal(self.classname, newid, 'create', propvalues) + + self.fireReactors('create', newid, None) + + return newid + + def get(self, nodeid, propname, default=_marker, cache=1): + """Get the value of a property on an existing node of this class. + + 'nodeid' must be the id of an existing node of this class or an + IndexError is raised. 'propname' must be the name of a property + of this class or a KeyError is raised. + + 'cache' indicates whether the transaction cache should be queried + for the node. If the node has been modified and you need to + determine what its values prior to modification are, you need to + set cache=0. + + Attempts to get the "creation" or "activity" properties should + do the right thing. + """ + if propname == 'id': + return nodeid + + if propname == 'creation': + journal = self.db.getjournal(self.classname, nodeid) + if journal: + return self.db.getjournal(self.classname, nodeid)[0][1] + else: + # on the strange chance that there's no journal + return date.Date() + if propname == 'activity': + journal = self.db.getjournal(self.classname, nodeid) + if journal: + return self.db.getjournal(self.classname, nodeid)[-1][1] + else: + # on the strange chance that there's no journal + return date.Date() + if propname == 'creator': + journal = self.db.getjournal(self.classname, nodeid) + if journal: + name = self.db.getjournal(self.classname, nodeid)[0][2] + else: + return None + return self.db.user.lookup(name) + + # get the property (raises KeyErorr if invalid) + prop = self.properties[propname] + + # get the node's dict + d = self.db.getnode(self.classname, nodeid, cache=cache) + + if not d.has_key(propname): + if default is _marker: + if isinstance(prop, Multilink): + return [] + else: + # TODO: None isn't right here, I think... + return None + else: + return default + + return d[propname] + + # XXX not in spec + def getnode(self, nodeid, cache=1): + ''' Return a convenience wrapper for the node. + + 'nodeid' must be the id of an existing node of this class or an + IndexError is raised. + + 'cache' indicates whether the transaction cache should be queried + for the node. If the node has been modified and you need to + determine what its values prior to modification are, you need to + set cache=0. + ''' + return Node(self, nodeid, cache=cache) + + def set(self, nodeid, **propvalues): + """Modify a property on an existing node of this class. + + 'nodeid' must be the id of an existing node of this class or an + IndexError is raised. + + Each key in 'propvalues' must be the name of a property of this + class or a KeyError is raised. + + All values in 'propvalues' must be acceptable types for their + corresponding properties or a TypeError is raised. + + If the value of the key property is set, it must not collide with + other key strings or a ValueError is raised. + + If the value of a Link or Multilink property contains an invalid + node id, a ValueError is raised. + + These operations trigger detectors and can be vetoed. Attempts + to modify the "creation" or "activity" properties cause a KeyError. + """ + if not propvalues: + return + + if propvalues.has_key('creation') or propvalues.has_key('activity'): + raise KeyError, '"creation" and "activity" are reserved' + + if propvalues.has_key('id'): + raise KeyError, '"id" is reserved' + + if self.db.journaltag is None: + raise DatabaseError, 'Database open read-only' + + self.fireAuditors('set', nodeid, propvalues) + # Take a copy of the node dict so that the subsequent set + # operation doesn't modify the oldvalues structure. + try: + # try not using the cache initially + oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid, + cache=0)) + except IndexError: + # this will be needed if somone does a create() and set() + # with no intervening commit() + oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid)) + + node = self.db.getnode(self.classname, nodeid) + if node.has_key(self.db.RETIRED_FLAG): + raise IndexError + num_re = re.compile('^\d+$') + for key, value in propvalues.items(): + # check to make sure we're not duplicating an existing key + if key == self.key and node[key] != value: + try: + self.lookup(value) + except KeyError: + pass + else: + raise ValueError, 'node with key "%s" exists'%value + + # this will raise the KeyError if the property isn't valid + # ... we don't use getprops() here because we only care about + # the writeable properties. + prop = self.properties[key] + + # if the value's the same as the existing value, no sense in + # doing anything + if node.has_key(key) and value == node[key]: + del propvalues[key] + continue + + # do stuff based on the prop type + if isinstance(prop, Link): + link_class = self.properties[key].classname + # if it isn't a number, it's a key + if type(value) != type(''): + raise ValueError, 'link value must be String' + if not num_re.match(value): + try: + value = self.db.classes[link_class].lookup(value) + except (TypeError, KeyError): + raise IndexError, 'new property "%s": %s not a %s'%( + key, value, self.properties[key].classname) + + if not self.db.hasnode(link_class, value): + raise IndexError, '%s has no node %s'%(link_class, value) + + if self.properties[key].do_journal: + # register the unlink with the old linked node + if node[key] is not None: + self.db.addjournal(link_class, node[key], 'unlink', + (self.classname, nodeid, key)) + + # register the link with the newly linked node + if value is not None: + self.db.addjournal(link_class, value, 'link', + (self.classname, nodeid, key)) + + elif isinstance(prop, Multilink): + if type(value) != type([]): + raise TypeError, 'new property "%s" not a list of ids'%key + link_class = self.properties[key].classname + l = [] + for entry in value: + # if it isn't a number, it's a key + if type(entry) != type(''): + raise ValueError, 'new property "%s" link value ' \ + 'must be a string'%key + if not num_re.match(entry): + try: + entry = self.db.classes[link_class].lookup(entry) + except (TypeError, KeyError): + raise IndexError, 'new property "%s": %s not a %s'%( + key, entry, self.properties[key].classname) + l.append(entry) + value = l + propvalues[key] = value + + # handle removals + if node.has_key(key): + l = node[key] + else: + l = [] + for id in l[:]: + if id in value: + continue + # register the unlink with the old linked node + if self.properties[key].do_journal: + self.db.addjournal(link_class, id, 'unlink', + (self.classname, nodeid, key)) + l.remove(id) + + # handle additions + for id in value: + if not self.db.hasnode(link_class, id): + raise IndexError, '%s has no node %s'%( + link_class, id) + if id in l: + continue + # register the link with the newly linked node + if self.properties[key].do_journal: + self.db.addjournal(link_class, id, 'link', + (self.classname, nodeid, key)) + l.append(id) + + elif isinstance(prop, String): + if value is not None and type(value) != type(''): + raise TypeError, 'new property "%s" not a string'%key + + elif isinstance(prop, Password): + if not isinstance(value, password.Password): + raise TypeError, 'new property "%s" not a Password'% key + propvalues[key] = value + + elif value is not None and isinstance(prop, Date): + if not isinstance(value, date.Date): + raise TypeError, 'new property "%s" not a Date'% key + propvalues[key] = value + + elif value is not None and isinstance(prop, Interval): + if not isinstance(value, date.Interval): + raise TypeError, 'new property "%s" not an Interval'% key + propvalues[key] = value + + node[key] = value + + # nothing to do? + if not propvalues: + return + + # do the set, and journal it + self.db.setnode(self.classname, nodeid, node) + self.db.addjournal(self.classname, nodeid, 'set', propvalues) + + self.fireReactors('set', nodeid, oldvalues) + + def retire(self, nodeid): + """Retire a node. + + The properties on the node remain available from the get() method, + and the node's id is never reused. + + Retired nodes are not returned by the find(), list(), or lookup() + methods, and other nodes may reuse the values of their key properties. + + These operations trigger detectors and can be vetoed. Attempts + to modify the "creation" or "activity" properties cause a KeyError. + """ + if self.db.journaltag is None: + raise DatabaseError, 'Database open read-only' + + self.fireAuditors('retire', nodeid, None) + + node = self.db.getnode(self.classname, nodeid) + node[self.db.RETIRED_FLAG] = 1 + self.db.setnode(self.classname, nodeid, node) + self.db.addjournal(self.classname, nodeid, 'retired', None) + + self.fireReactors('retire', nodeid, None) + + def history(self, nodeid): + """Retrieve the journal of edits on a particular node. + + 'nodeid' must be the id of an existing node of this class or an + IndexError is raised. + + The returned list contains tuples of the form + + (date, tag, action, params) + + 'date' is a Timestamp object specifying the time of the change and + 'tag' is the journaltag specified when the database was opened. + """ + return self.db.getjournal(self.classname, nodeid) + + # Locating nodes: + def hasnode(self, nodeid): + '''Determine if the given nodeid actually exists + ''' + return self.db.hasnode(self.classname, nodeid) + + def setkey(self, propname): + """Select a String property of this class to be the key property. + + 'propname' must be the name of a String property of this class or + None, or a TypeError is raised. The values of the key property on + all existing nodes must be unique or a ValueError is raised. + """ + # TODO: validate that the property is a String! + self.key = propname + + def getkey(self): + """Return the name of the key property for this class or None.""" + return self.key + + def labelprop(self, default_to_id=0): + ''' Return the property name for a label for the given node. + + This method attempts to generate a consistent label for the node. + It tries the following in order: + 1. key property + 2. "name" property + 3. "title" property + 4. first property from the sorted property name list + ''' + k = self.getkey() + if k: + return k + props = self.getprops() + if props.has_key('name'): + return 'name' + elif props.has_key('title'): + return 'title' + if default_to_id: + return 'id' + props = props.keys() + props.sort() + return props[0] + + # TODO: set up a separate index db file for this? profile? + def lookup(self, keyvalue): + """Locate a particular node by its key property and return its id. + + If this class has no key property, a TypeError is raised. If the + 'keyvalue' matches one of the values for the key property among + the nodes in this class, the matching node's id is returned; + otherwise a KeyError is raised. + """ + cldb = self.db.getclassdb(self.classname) + try: + for nodeid in self.db.getnodeids(self.classname, cldb): + node = self.db.getnode(self.classname, nodeid, cldb) + if node.has_key(self.db.RETIRED_FLAG): + continue + if node[self.key] == keyvalue: + cldb.close() + return nodeid + finally: + cldb.close() + raise KeyError, keyvalue + + # XXX: change from spec - allows multiple props to match + def find(self, **propspec): + """Get the ids of nodes in this class which link to the given nodes. + + 'propspec' consists of keyword args propname={nodeid:1,} + 'propname' must be the name of a property in this class, or a + KeyError is raised. That property must be a Link or Multilink + property, or a TypeError is raised. + + Any node in this class whose 'propname' property links to any of the + nodeids will be returned. Used by the full text indexing, which knows + that "foo" occurs in msg1, msg3 and file7, so we have hits on these issues: + db.issue.find(messages={'1':1,'3':1}, files={'7':1}) + """ + propspec = propspec.items() + for propname, nodeids in propspec: + # check the prop is OK + prop = self.properties[propname] + if not isinstance(prop, Link) and not isinstance(prop, Multilink): + raise TypeError, "'%s' not a Link/Multilink property"%propname + #XXX edit is expensive and of questionable use + #for nodeid in nodeids: + # if not self.db.hasnode(prop.classname, nodeid): + # raise ValueError, '%s has no node %s'%(prop.classname, nodeid) + + # ok, now do the find + cldb = self.db.getclassdb(self.classname) + l = [] + try: + for id in self.db.getnodeids(self.classname, db=cldb): + node = self.db.getnode(self.classname, id, db=cldb) + if node.has_key(self.db.RETIRED_FLAG): + continue + for propname, nodeids in propspec: + # can't test if the node doesn't have this property + if not node.has_key(propname): + continue + if type(nodeids) is type(''): + nodeids = {nodeids:1} + prop = self.properties[propname] + value = node[propname] + if isinstance(prop, Link) and nodeids.has_key(value): + l.append(id) + break + elif isinstance(prop, Multilink): + hit = 0 + for v in value: + if nodeids.has_key(v): + l.append(id) + hit = 1 + break + if hit: + break + finally: + cldb.close() + return l + + def stringFind(self, **requirements): + """Locate a particular node by matching a set of its String + properties in a caseless search. + + If the property is not a String property, a TypeError is raised. + + The return is a list of the id of all nodes that match. + """ + for propname in requirements.keys(): + prop = self.properties[propname] + if isinstance(not prop, String): + raise TypeError, "'%s' not a String property"%propname + requirements[propname] = requirements[propname].lower() + l = [] + cldb = self.db.getclassdb(self.classname) + try: + for nodeid in self.db.getnodeids(self.classname, cldb): + node = self.db.getnode(self.classname, nodeid, cldb) + if node.has_key(self.db.RETIRED_FLAG): + continue + for key, value in requirements.items(): + if node[key] and node[key].lower() != value: + break + else: + l.append(nodeid) + finally: + cldb.close() + return l + + def list(self): + """Return a list of the ids of the active nodes in this class.""" + l = [] + cn = self.classname + cldb = self.db.getclassdb(cn) + try: + for nodeid in self.db.getnodeids(cn, cldb): + node = self.db.getnode(cn, nodeid, cldb) + if node.has_key(self.db.RETIRED_FLAG): + continue + l.append(nodeid) + finally: + cldb.close() + l.sort() + return l + + # XXX not in spec + def filter(self, search_matches, filterspec, sort, group, + num_re = re.compile('^\d+$')): + ''' Return a list of the ids of the active nodes in this class that + match the 'filter' spec, sorted by the group spec and then the + sort spec + ''' + cn = self.classname + + # optimise filterspec + l = [] + props = self.getprops() + for k, v in filterspec.items(): + propclass = props[k] + if isinstance(propclass, Link): + if type(v) is not type([]): + v = [v] + # replace key values with node ids + u = [] + link_class = self.db.classes[propclass.classname] + for entry in v: + if entry == '-1': entry = None + elif not num_re.match(entry): + try: + entry = link_class.lookup(entry) + except (TypeError,KeyError): + raise ValueError, 'property "%s": %s not a %s'%( + k, entry, self.properties[k].classname) + u.append(entry) + + l.append((0, k, u)) + elif isinstance(propclass, Multilink): + if type(v) is not type([]): + v = [v] + # replace key values with node ids + u = [] + link_class = self.db.classes[propclass.classname] + for entry in v: + if not num_re.match(entry): + try: + entry = link_class.lookup(entry) + except (TypeError,KeyError): + raise ValueError, 'new property "%s": %s not a %s'%( + k, entry, self.properties[k].classname) + u.append(entry) + l.append((1, k, u)) + elif isinstance(propclass, String): + # simple glob searching + v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v) + v = v.replace('?', '.') + v = v.replace('*', '.*?') + l.append((2, k, re.compile(v, re.I))) + else: + l.append((6, k, v)) + filterspec = l + + # now, find all the nodes that are active and pass filtering + l = [] + cldb = self.db.getclassdb(cn) + try: + for nodeid in self.db.getnodeids(cn, cldb): + node = self.db.getnode(cn, nodeid, cldb) + if node.has_key(self.db.RETIRED_FLAG): + continue + # apply filter + for t, k, v in filterspec: + # this node doesn't have this property, so reject it + if not node.has_key(k): break + + if t == 0 and node[k] not in v: + # link - if this node'd property doesn't appear in the + # filterspec's nodeid list, skip it + break + elif t == 1: + # multilink - if any of the nodeids required by the + # filterspec aren't in this node's property, then skip + # it + for value in v: + if value not in node[k]: + break + else: + continue + break + elif t == 2 and (node[k] is None or not v.search(node[k])): + # RE search + break + elif t == 6 and node[k] != v: + # straight value comparison for the other types + break + else: + l.append((nodeid, node)) + finally: + cldb.close() + l.sort() + + # filter based on full text search + if search_matches is not None: + k = [] + l_debug = [] + for v in l: + l_debug.append(v[0]) + if search_matches.has_key(v[0]): + k.append(v) + l = k + + # optimise sort + m = [] + for entry in sort: + if entry[0] != '-': + m.append(('+', entry)) + else: + m.append((entry[0], entry[1:])) + sort = m + + # optimise group + m = [] + for entry in group: + if entry[0] != '-': + m.append(('+', entry)) + else: + m.append((entry[0], entry[1:])) + group = m + # now, sort the result + def sortfun(a, b, sort=sort, group=group, properties=self.getprops(), + db = self.db, cl=self): + a_id, an = a + b_id, bn = b + # sort by group and then sort + for list in group, sort: + for dir, prop in list: + # sorting is class-specific + propclass = properties[prop] + + # handle the properties that might be "faked" + # also, handle possible missing properties + try: + if not an.has_key(prop): + an[prop] = cl.get(a_id, prop) + av = an[prop] + except KeyError: + # the node doesn't have a value for this property + if isinstance(propclass, Multilink): av = [] + else: av = '' + try: + if not bn.has_key(prop): + bn[prop] = cl.get(b_id, prop) + bv = bn[prop] + except KeyError: + # the node doesn't have a value for this property + if isinstance(propclass, Multilink): bv = [] + else: bv = '' + + # String and Date values are sorted in the natural way + if isinstance(propclass, String): + # clean up the strings + if av and av[0] in string.uppercase: + av = an[prop] = av.lower() + if bv and bv[0] in string.uppercase: + bv = bn[prop] = bv.lower() + if (isinstance(propclass, String) or + isinstance(propclass, Date)): + # it might be a string that's really an integer + try: + av = int(av) + bv = int(bv) + except: + pass + if dir == '+': + r = cmp(av, bv) + if r != 0: return r + elif dir == '-': + r = cmp(bv, av) + if r != 0: return r + + # Link properties are sorted according to the value of + # the "order" property on the linked nodes if it is + # present; or otherwise on the key string of the linked + # nodes; or finally on the node ids. + elif isinstance(propclass, Link): + link = db.classes[propclass.classname] + if av is None and bv is not None: return -1 + if av is not None and bv is None: return 1 + if av is None and bv is None: continue + if link.getprops().has_key('order'): + if dir == '+': + r = cmp(link.get(av, 'order'), + link.get(bv, 'order')) + if r != 0: return r + elif dir == '-': + r = cmp(link.get(bv, 'order'), + link.get(av, 'order')) + if r != 0: return r + elif link.getkey(): + key = link.getkey() + if dir == '+': + r = cmp(link.get(av, key), link.get(bv, key)) + if r != 0: return r + elif dir == '-': + r = cmp(link.get(bv, key), link.get(av, key)) + if r != 0: return r + else: + if dir == '+': + r = cmp(av, bv) + if r != 0: return r + elif dir == '-': + r = cmp(bv, av) + if r != 0: return r + + # Multilink properties are sorted according to how many + # links are present. + elif isinstance(propclass, Multilink): + if dir == '+': + r = cmp(len(av), len(bv)) + if r != 0: return r + elif dir == '-': + r = cmp(len(bv), len(av)) + if r != 0: return r + # end for dir, prop in list: + # end for list in sort, group: + # if all else fails, compare the ids + return cmp(a[0], b[0]) + + l.sort(sortfun) + return [i[0] for i in l] + + def count(self): + """Get the number of nodes in this class. + + If the returned integer is 'numnodes', the ids of all the nodes + in this class run from 1 to numnodes, and numnodes+1 will be the + id of the next node to be created in this class. + """ + return self.db.countnodes(self.classname) + + # Manipulating properties: + + def getprops(self, protected=1): + """Return a dictionary mapping property names to property objects. + If the "protected" flag is true, we include protected properties - + those which may not be modified. + + In addition to the actual properties on the node, these + methods provide the "creation" and "activity" properties. If the + "protected" flag is true, we include protected properties - those + which may not be modified. + """ + d = self.properties.copy() + if protected: + d['id'] = String() + d['creation'] = hyperdb.Date() + d['activity'] = hyperdb.Date() + d['creator'] = hyperdb.Link("user") + return d + + def addprop(self, **properties): + """Add properties to this class. + + The keyword arguments in 'properties' must map names to property + objects, or a TypeError is raised. None of the keys in 'properties' + may collide with the names of existing properties, or a ValueError + is raised before any properties have been added. + """ + for key in properties.keys(): + if self.properties.has_key(key): + raise ValueError, key + self.properties.update(properties) + + def index(self, nodeid): + '''Add (or refresh) the node to search indexes + ''' + # find all the String properties that have indexme + for prop, propclass in self.getprops().items(): + if isinstance(propclass, String) and propclass.indexme: + # and index them under (classname, nodeid, property) + self.db.indexer.add_text((self.classname, nodeid, prop), + str(self.get(nodeid, prop))) + + # + # Detector interface + # + def audit(self, event, detector): + """Register a detector + """ + l = self.auditors[event] + if detector not in l: + self.auditors[event].append(detector) + + def fireAuditors(self, action, nodeid, newvalues): + """Fire all registered auditors. + """ + for audit in self.auditors[action]: + audit(self.db, self, nodeid, newvalues) + + def react(self, event, detector): + """Register a detector + """ + l = self.reactors[event] + if detector not in l: + self.reactors[event].append(detector) + + def fireReactors(self, action, nodeid, oldvalues): + """Fire all registered reactors. + """ + for react in self.reactors[action]: + react(self.db, self, nodeid, oldvalues) + +class FileClass(Class): + '''This class defines a large chunk of data. To support this, it has a + mandatory String property "content" which is typically saved off + externally to the hyperdb. + + The default MIME type of this data is defined by the + "default_mime_type" class attribute, which may be overridden by each + node if the class defines a "type" String property. + ''' + default_mime_type = 'text/plain' + + def create(self, **propvalues): + ''' snaffle the file propvalue and store in a file + ''' + content = propvalues['content'] + del propvalues['content'] + newid = Class.create(self, **propvalues) + self.db.storefile(self.classname, newid, None, content) + return newid + + def get(self, nodeid, propname, default=_marker, cache=1): + ''' trap the content propname and get it from the file + ''' + + poss_msg = 'Possibly a access right configuration problem.' + if propname == 'content': + try: + return self.db.getfile(self.classname, nodeid, None) + except IOError, (strerror): + # BUG: by catching this we donot see an error in the log. + return 'ERROR reading file: %s%s\n%s\n%s'%( + self.classname, nodeid, poss_msg, strerror) + if default is not _marker: + return Class.get(self, nodeid, propname, default, cache=cache) + else: + return Class.get(self, nodeid, propname, cache=cache) + + def getprops(self, protected=1): + ''' In addition to the actual properties on the node, these methods + provide the "content" property. If the "protected" flag is true, + we include protected properties - those which may not be + modified. + ''' + d = Class.getprops(self, protected=protected).copy() + if protected: + d['content'] = hyperdb.String() + return d + + def index(self, nodeid): + ''' Index the node in the search index. + + We want to index the content in addition to the normal String + property indexing. + ''' + # perform normal indexing + Class.index(self, nodeid) + + # get the content to index + content = self.get(nodeid, 'content') + + # figure the mime type + if self.properties.has_key('type'): + mime_type = self.get(nodeid, 'type') + else: + mime_type = self.default_mime_type + + # and index! + self.db.indexer.add_text((self.classname, nodeid, 'content'), content, + mime_type) + +# XXX deviation from spec - was called ItemClass +class IssueClass(Class, roundupdb.IssueClass): + # Overridden methods: + def __init__(self, db, classname, **properties): + """The newly-created class automatically includes the "messages", + "files", "nosy", and "superseder" properties. If the 'properties' + dictionary attempts to specify any of these properties or a + "creation" or "activity" property, a ValueError is raised. + """ + if not properties.has_key('title'): + properties['title'] = hyperdb.String(indexme='yes') + if not properties.has_key('messages'): + properties['messages'] = hyperdb.Multilink("msg") + if not properties.has_key('files'): + properties['files'] = hyperdb.Multilink("file") + if not properties.has_key('nosy'): + properties['nosy'] = hyperdb.Multilink("user") + if not properties.has_key('superseder'): + properties['superseder'] = hyperdb.Multilink(classname) + Class.__init__(self, db, classname, **properties) + # #$Log: not supported by cvs2svn $ +#Revision 1.43 2002/07/10 06:30:30 richard +#...except of course it's nice to use valid Python syntax +# #Revision 1.42 2002/07/10 06:21:38 richard #Be extra safe # diff --git a/roundup/backends/back_bsddb.py b/roundup/backends/back_bsddb.py index 6f8edd7..530a691 100644 --- a/roundup/backends/back_bsddb.py +++ b/roundup/backends/back_bsddb.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -#$Id: back_bsddb.py,v 1.18 2002-05-15 06:21:21 richard Exp $ +#$Id: back_bsddb.py,v 1.19 2002-07-14 02:05:53 richard Exp $ ''' This module defines a backend that saves the hyperdatabase in BSDDB. ''' @@ -24,12 +24,12 @@ import bsddb, os, marshal from roundup import hyperdb, date # these classes are so similar, we just use the anydbm methods -import back_anydbm +from back_anydbm import Database, Class, FileClass, IssueClass # # Now the database # -class Database(back_anydbm.Database): +class Database(Database): """A database for storing records containing flexible data types.""" # # Class DBs @@ -119,6 +119,14 @@ class Database(back_anydbm.Database): # #$Log: not supported by cvs2svn $ +#Revision 1.18 2002/05/15 06:21:21 richard +# . node caching now works, and gives a small boost in performance +# +#As a part of this, I cleaned up the DEBUG output and implemented TRACE +#output (HYPERDBTRACE='file to trace to') with checkpoints at the start of +#CGI requests. Run roundup with python -O to skip all the DEBUG/TRACE stuff +#(using if __debug__ which is compiled out with -O) +# #Revision 1.17 2002/04/03 05:54:31 richard #Fixed serialisation problem by moving the serialisation step out of the #hyperdb.Class (get, set) into the hyperdb.Database. diff --git a/roundup/backends/back_bsddb3.py b/roundup/backends/back_bsddb3.py index 287b2ab..bce357a 100644 --- a/roundup/backends/back_bsddb3.py +++ b/roundup/backends/back_bsddb3.py @@ -15,18 +15,18 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -#$Id: back_bsddb3.py,v 1.13 2002-07-08 06:41:03 richard Exp $ +#$Id: back_bsddb3.py,v 1.14 2002-07-14 02:05:54 richard Exp $ import bsddb3, os, marshal from roundup import hyperdb, date # these classes are so similar, we just use the anydbm methods -import back_anydbm +from back_anydbm import Database, Class, FileClass, IssueClass # # Now the database # -class Database(back_anydbm.Database): +class Database(Database): """A database for storing records containing flexible data types.""" # # Class DBs @@ -115,6 +115,9 @@ class Database(back_anydbm.Database): # #$Log: not supported by cvs2svn $ +#Revision 1.13 2002/07/08 06:41:03 richard +#Was reopening the database with 'n'. +# #Revision 1.12 2002/05/21 05:52:11 richard #Well whadya know, bsddb3 works again. #The backend is implemented _exactly_ the same as bsddb - so there's no diff --git a/roundup/hyperdb.py b/roundup/hyperdb.py index af70c2d..3f6429f 100644 --- a/roundup/hyperdb.py +++ b/roundup/hyperdb.py @@ -15,14 +15,14 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: hyperdb.py,v 1.74 2002-07-10 00:24:10 richard Exp $ +# $Id: hyperdb.py,v 1.75 2002-07-14 02:05:53 richard Exp $ __doc__ = """ Hyperdatabase implementation, especially field types. """ # standard python modules -import sys, re, string, weakref, os, time +import sys, os, time, re # roundup modules import date, password @@ -110,15 +110,26 @@ class Multilink: ' more useful for dumps ' return '<%s to "%s">'%(self.__class__, self.classname) -class DatabaseError(ValueError): - '''Error to be raised when there is some problem in the database code - ''' +# +# Support for splitting designators +# +class DesignatorError(ValueError): pass - +def splitDesignator(designator, dre=re.compile(r'([^\d]+)(\d+)')): + ''' Take a foo123 and return ('foo', 123) + ''' + m = dre.match(designator) + if m is None: + raise DesignatorError, '"%s" not a node designator'%designator + return m.group(1), m.group(2) # # the base Database class # +class DatabaseError(ValueError): + '''Error to be raised when there is some problem in the database code + ''' + pass class Database: '''A database for storing records containing flexible data types. @@ -137,6 +148,13 @@ in-database value is returned in preference to the in-transaction value. This is necessary to determine if any values have changed during a transaction. + +Implementation +-------------- + +All methods except __repr__ and getnode must be implemented by a +concrete backend Class. + ''' # flag to set on retired entries @@ -203,29 +221,7 @@ transaction. '''Copy the node contents, converting non-marshallable data into marshallable data. ''' - if __debug__: - print >>DEBUG, 'serialise', classname, node - properties = self.getclass(classname).getprops() - d = {} - for k, v in node.items(): - # if the property doesn't exist, or is the "retired" flag then - # it won't be in the properties dict - if not properties.has_key(k): - d[k] = v - continue - - # get the property spec - prop = properties[k] - - if isinstance(prop, Password): - d[k] = str(v) - elif isinstance(prop, Date) and v is not None: - d[k] = v.get_tuple() - elif isinstance(prop, Interval) and v is not None: - d[k] = v.get_tuple() - else: - d[k] = v - return d + return node def setnode(self, classname, nodeid, node): '''Change the specified node. @@ -235,31 +231,7 @@ transaction. def unserialise(self, classname, node): '''Decode the marshalled node data ''' - if __debug__: - print >>DEBUG, 'unserialise', classname, node - properties = self.getclass(classname).getprops() - d = {} - for k, v in node.items(): - # if the property doesn't exist, or is the "retired" flag then - # it won't be in the properties dict - if not properties.has_key(k): - d[k] = v - continue - - # get the property spec - prop = properties[k] - - if isinstance(prop, Date) and v is not None: - d[k] = date.Date(v) - elif isinstance(prop, Interval) and v is not None: - d[k] = date.Interval(v) - elif isinstance(prop, Password): - p = password.Password() - p.unpack(v) - d[k] = p - else: - d[k] = v - return d + return node def getnode(self, classname, nodeid, db=None, cache=1): '''Get a node from the database. @@ -330,12 +302,15 @@ transaction. ''' raise NotImplementedError -_marker = [] # # The base Class class # class Class: - """The handle to a particular class of nodes in a hyperdatabase.""" + """ The handle to a particular class of nodes in a hyperdatabase. + + All methods except __repr__ and getnode must be implemented by a + concrete backend Class. + """ def __init__(self, db, classname, **properties): """Create a new class with a given name and property specification. @@ -344,13 +319,7 @@ class Class: or a ValueError is raised. The keyword arguments in 'properties' must map names to property objects, or a TypeError is raised. """ - self.classname = classname - self.properties = properties - self.db = weakref.proxy(db) # use a weak ref to avoid circularity - self.key = '' - - # do the db-related init stuff - db.addclass(self) + raise NotImplementedError def __repr__(self): '''Slightly more useful representation @@ -376,118 +345,9 @@ class Class: If an id in a link or multilink property does not refer to a valid node, an IndexError is raised. """ - if propvalues.has_key('id'): - raise KeyError, '"id" is reserved' - - if self.db.journaltag is None: - raise DatabaseError, 'Database open read-only' - - # new node's id - newid = self.db.newid(self.classname) - - # validate propvalues - num_re = re.compile('^\d+$') - for key, value in propvalues.items(): - if key == self.key: - try: - self.lookup(value) - except KeyError: - pass - else: - raise ValueError, 'node with key "%s" exists'%value - - # try to handle this property - try: - prop = self.properties[key] - except KeyError: - raise KeyError, '"%s" has no property "%s"'%(self.classname, - key) - - if isinstance(prop, Link): - if type(value) != type(''): - raise ValueError, 'link value must be String' - link_class = self.properties[key].classname - # if it isn't a number, it's a key - if not num_re.match(value): - try: - value = self.db.classes[link_class].lookup(value) - except (TypeError, KeyError): - raise IndexError, 'new property "%s": %s not a %s'%( - key, value, link_class) - elif not self.db.hasnode(link_class, value): - raise IndexError, '%s has no node %s'%(link_class, value) - - # save off the value - propvalues[key] = value - - # register the link with the newly linked node - if self.properties[key].do_journal: - self.db.addjournal(link_class, value, 'link', - (self.classname, newid, key)) - - elif isinstance(prop, Multilink): - if type(value) != type([]): - raise TypeError, 'new property "%s" not a list of ids'%key - - # clean up and validate the list of links - link_class = self.properties[key].classname - l = [] - for entry in value: - if type(entry) != type(''): - raise ValueError, '"%s" link value (%s) must be String' % (key, value) - # if it isn't a number, it's a key - if not num_re.match(entry): - try: - entry = self.db.classes[link_class].lookup(entry) - except (TypeError, KeyError): - raise IndexError, 'new property "%s": %s not a %s'%( - key, entry, self.properties[key].classname) - l.append(entry) - value = l - propvalues[key] = value - - # handle additions - for id in value: - if not self.db.hasnode(link_class, id): - raise IndexError, '%s has no node %s'%(link_class, id) - # register the link with the newly linked node - if self.properties[key].do_journal: - self.db.addjournal(link_class, id, 'link', - (self.classname, newid, key)) - - elif isinstance(prop, String): - if type(value) != type(''): - raise TypeError, 'new property "%s" not a string'%key - - elif isinstance(prop, Password): - if not isinstance(value, password.Password): - raise TypeError, 'new property "%s" not a Password'%key - - elif isinstance(prop, Date): - if value is not None and not isinstance(value, date.Date): - raise TypeError, 'new property "%s" not a Date'%key - - elif isinstance(prop, Interval): - if value is not None and not isinstance(value, date.Interval): - raise TypeError, 'new property "%s" not an Interval'%key - - # make sure there's data where there needs to be - for key, prop in self.properties.items(): - if propvalues.has_key(key): - continue - if key == self.key: - raise ValueError, 'key property "%s" is required'%key - if isinstance(prop, Multilink): - propvalues[key] = [] - else: - # TODO: None isn't right here, I think... - propvalues[key] = None - - # done - self.db.addnode(self.classname, newid, propvalues) - self.db.addjournal(self.classname, newid, 'create', propvalues) - return newid + raise NotImplementedError + _marker = [] def get(self, nodeid, propname, default=_marker, cache=1): """Get the value of a property on an existing node of this class. @@ -500,26 +360,7 @@ class Class: determine what its values prior to modification are, you need to set cache=0. """ - if propname == 'id': - return nodeid - - # get the property (raises KeyErorr if invalid) - prop = self.properties[propname] - - # get the node's dict - d = self.db.getnode(self.classname, nodeid, cache=cache) - - if not d.has_key(propname): - if default is _marker: - if isinstance(prop, Multilink): - return [] - else: - # TODO: None isn't right here, I think... - return None - else: - return default - - return d[propname] + raise NotImplementedError # XXX not in spec def getnode(self, nodeid, cache=1): @@ -553,142 +394,7 @@ class Class: If the value of a Link or Multilink property contains an invalid node id, a ValueError is raised. """ - if not propvalues: - return - - if propvalues.has_key('id'): - raise KeyError, '"id" is reserved' - - if self.db.journaltag is None: - raise DatabaseError, 'Database open read-only' - - node = self.db.getnode(self.classname, nodeid) - if node.has_key(self.db.RETIRED_FLAG): - raise IndexError - num_re = re.compile('^\d+$') - for key, value in propvalues.items(): - # check to make sure we're not duplicating an existing key - if key == self.key and node[key] != value: - try: - self.lookup(value) - except KeyError: - pass - else: - raise ValueError, 'node with key "%s" exists'%value - - # this will raise the KeyError if the property isn't valid - # ... we don't use getprops() here because we only care about - # the writeable properties. - prop = self.properties[key] - - # if the value's the same as the existing value, no sense in - # doing anything - if node.has_key(key) and value == node[key]: - del propvalues[key] - continue - - # do stuff based on the prop type - if isinstance(prop, Link): - link_class = self.properties[key].classname - # if it isn't a number, it's a key - if type(value) != type(''): - raise ValueError, 'link value must be String' - if not num_re.match(value): - try: - value = self.db.classes[link_class].lookup(value) - except (TypeError, KeyError): - raise IndexError, 'new property "%s": %s not a %s'%( - key, value, self.properties[key].classname) - - if not self.db.hasnode(link_class, value): - raise IndexError, '%s has no node %s'%(link_class, value) - - if self.properties[key].do_journal: - # register the unlink with the old linked node - if node[key] is not None: - self.db.addjournal(link_class, node[key], 'unlink', - (self.classname, nodeid, key)) - - # register the link with the newly linked node - if value is not None: - self.db.addjournal(link_class, value, 'link', - (self.classname, nodeid, key)) - - elif isinstance(prop, Multilink): - if type(value) != type([]): - raise TypeError, 'new property "%s" not a list of ids'%key - link_class = self.properties[key].classname - l = [] - for entry in value: - # if it isn't a number, it's a key - if type(entry) != type(''): - raise ValueError, 'new property "%s" link value ' \ - 'must be a string'%key - if not num_re.match(entry): - try: - entry = self.db.classes[link_class].lookup(entry) - except (TypeError, KeyError): - raise IndexError, 'new property "%s": %s not a %s'%( - key, entry, self.properties[key].classname) - l.append(entry) - value = l - propvalues[key] = value - - # handle removals - if node.has_key(key): - l = node[key] - else: - l = [] - for id in l[:]: - if id in value: - continue - # register the unlink with the old linked node - if self.properties[key].do_journal: - self.db.addjournal(link_class, id, 'unlink', - (self.classname, nodeid, key)) - l.remove(id) - - # handle additions - for id in value: - if not self.db.hasnode(link_class, id): - raise IndexError, '%s has no node %s'%( - link_class, id) - if id in l: - continue - # register the link with the newly linked node - if self.properties[key].do_journal: - self.db.addjournal(link_class, id, 'link', - (self.classname, nodeid, key)) - l.append(id) - - elif isinstance(prop, String): - if value is not None and type(value) != type(''): - raise TypeError, 'new property "%s" not a string'%key - - elif isinstance(prop, Password): - if not isinstance(value, password.Password): - raise TypeError, 'new property "%s" not a Password'% key - propvalues[key] = value - - elif value is not None and isinstance(prop, Date): - if not isinstance(value, date.Date): - raise TypeError, 'new property "%s" not a Date'% key - propvalues[key] = value - - elif value is not None and isinstance(prop, Interval): - if not isinstance(value, date.Interval): - raise TypeError, 'new property "%s" not an Interval'% key - propvalues[key] = value - - node[key] = value - - # nothing to do? - if not propvalues: - return - - # do the set, and journal it - self.db.setnode(self.classname, nodeid, node) - self.db.addjournal(self.classname, nodeid, 'set', propvalues) + raise NotImplementedError def retire(self, nodeid): """Retire a node. @@ -699,12 +405,7 @@ class Class: Retired nodes are not returned by the find(), list(), or lookup() methods, and other nodes may reuse the values of their key properties. """ - if self.db.journaltag is None: - raise DatabaseError, 'Database open read-only' - node = self.db.getnode(self.classname, nodeid) - node[self.db.RETIRED_FLAG] = 1 - self.db.setnode(self.classname, nodeid, node) - self.db.addjournal(self.classname, nodeid, 'retired', None) + raise NotImplementedError def history(self, nodeid): """Retrieve the journal of edits on a particular node. @@ -719,13 +420,13 @@ class Class: 'date' is a Timestamp object specifying the time of the change and 'tag' is the journaltag specified when the database was opened. """ - return self.db.getjournal(self.classname, nodeid) + raise NotImplementedError # Locating nodes: def hasnode(self, nodeid): '''Determine if the given nodeid actually exists ''' - return self.db.hasnode(self.classname, nodeid) + raise NotImplementedError def setkey(self, propname): """Select a String property of this class to be the key property. @@ -734,12 +435,11 @@ class Class: None, or a TypeError is raised. The values of the key property on all existing nodes must be unique or a ValueError is raised. """ - # TODO: validate that the property is a String! - self.key = propname + raise NotImplementedError def getkey(self): """Return the name of the key property for this class or None.""" - return self.key + raise NotImplementedError def labelprop(self, default_to_id=0): ''' Return the property name for a label for the given node. @@ -751,21 +451,8 @@ class Class: 3. "title" property 4. first property from the sorted property name list ''' - k = self.getkey() - if k: - return k - props = self.getprops() - if props.has_key('name'): - return 'name' - elif props.has_key('title'): - return 'title' - if default_to_id: - return 'id' - props = props.keys() - props.sort() - return props[0] - - # TODO: set up a separate index db file for this? profile? + raise NotImplementedError + def lookup(self, keyvalue): """Locate a particular node by its key property and return its id. @@ -774,18 +461,7 @@ class Class: the nodes in this class, the matching node's id is returned; otherwise a KeyError is raised. """ - cldb = self.db.getclassdb(self.classname) - try: - for nodeid in self.db.getnodeids(self.classname, cldb): - node = self.db.getnode(self.classname, nodeid, cldb) - if node.has_key(self.db.RETIRED_FLAG): - continue - if node[self.key] == keyvalue: - cldb.close() - return nodeid - finally: - cldb.close() - raise KeyError, keyvalue + raise NotImplementedError # XXX: change from spec - allows multiple props to match def find(self, **propspec): @@ -798,96 +474,12 @@ class Class: Any node in this class whose 'propname' property links to any of the nodeids will be returned. Used by the full text indexing, which knows - that "foo" occurs in msg1, msg3 and file7, so we have hits on these issues: - db.issue.find(messages={'1':1,'3':1}, files={'7':1}) - """ - propspec = propspec.items() - for propname, nodeids in propspec: - # check the prop is OK - prop = self.properties[propname] - if not isinstance(prop, Link) and not isinstance(prop, Multilink): - raise TypeError, "'%s' not a Link/Multilink property"%propname - #XXX edit is expensive and of questionable use - #for nodeid in nodeids: - # if not self.db.hasnode(prop.classname, nodeid): - # raise ValueError, '%s has no node %s'%(prop.classname, nodeid) - - # ok, now do the find - cldb = self.db.getclassdb(self.classname) - l = [] - try: - for id in self.db.getnodeids(self.classname, db=cldb): - node = self.db.getnode(self.classname, id, db=cldb) - if node.has_key(self.db.RETIRED_FLAG): - continue - for propname, nodeids in propspec: - # can't test if the node doesn't have this property - if not node.has_key(propname): - continue - if type(nodeids) is type(''): - nodeids = {nodeids:1} - prop = self.properties[propname] - value = node[propname] - if isinstance(prop, Link) and nodeids.has_key(value): - l.append(id) - break - elif isinstance(prop, Multilink): - hit = 0 - for v in value: - if nodeids.has_key(v): - l.append(id) - hit = 1 - break - if hit: - break - finally: - cldb.close() - return l + that "foo" occurs in msg1, msg3 and file7, so we have hits on these + issues: - def stringFind(self, **requirements): - """Locate a particular node by matching a set of its String - properties in a caseless search. - - If the property is not a String property, a TypeError is raised. - - The return is a list of the id of all nodes that match. + db.issue.find(messages={'1':1,'3':1}, files={'7':1}) """ - for propname in requirements.keys(): - prop = self.properties[propname] - if isinstance(not prop, String): - raise TypeError, "'%s' not a String property"%propname - requirements[propname] = requirements[propname].lower() - l = [] - cldb = self.db.getclassdb(self.classname) - try: - for nodeid in self.db.getnodeids(self.classname, cldb): - node = self.db.getnode(self.classname, nodeid, cldb) - if node.has_key(self.db.RETIRED_FLAG): - continue - for key, value in requirements.items(): - if node[key] and node[key].lower() != value: - break - else: - l.append(nodeid) - finally: - cldb.close() - return l - - def list(self): - """Return a list of the ids of the active nodes in this class.""" - l = [] - cn = self.classname - cldb = self.db.getclassdb(cn) - try: - for nodeid in self.db.getnodeids(cn, cldb): - node = self.db.getnode(cn, nodeid, cldb) - if node.has_key(self.db.RETIRED_FLAG): - continue - l.append(nodeid) - finally: - cldb.close() - l.sort() - return l + raise NotImplementedError # XXX not in spec def filter(self, search_matches, filterspec, sort, group, @@ -896,223 +488,7 @@ class Class: match the 'filter' spec, sorted by the group spec and then the sort spec ''' - cn = self.classname - - # optimise filterspec - l = [] - props = self.getprops() - for k, v in filterspec.items(): - propclass = props[k] - if isinstance(propclass, Link): - if type(v) is not type([]): - v = [v] - # replace key values with node ids - u = [] - link_class = self.db.classes[propclass.classname] - for entry in v: - if entry == '-1': entry = None - elif not num_re.match(entry): - try: - entry = link_class.lookup(entry) - except (TypeError,KeyError): - raise ValueError, 'property "%s": %s not a %s'%( - k, entry, self.properties[k].classname) - u.append(entry) - - l.append((0, k, u)) - elif isinstance(propclass, Multilink): - if type(v) is not type([]): - v = [v] - # replace key values with node ids - u = [] - link_class = self.db.classes[propclass.classname] - for entry in v: - if not num_re.match(entry): - try: - entry = link_class.lookup(entry) - except (TypeError,KeyError): - raise ValueError, 'new property "%s": %s not a %s'%( - k, entry, self.properties[k].classname) - u.append(entry) - l.append((1, k, u)) - elif isinstance(propclass, String): - # simple glob searching - v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v) - v = v.replace('?', '.') - v = v.replace('*', '.*?') - l.append((2, k, re.compile(v, re.I))) - else: - l.append((6, k, v)) - filterspec = l - - # now, find all the nodes that are active and pass filtering - l = [] - cldb = self.db.getclassdb(cn) - try: - for nodeid in self.db.getnodeids(cn, cldb): - node = self.db.getnode(cn, nodeid, cldb) - if node.has_key(self.db.RETIRED_FLAG): - continue - # apply filter - for t, k, v in filterspec: - # this node doesn't have this property, so reject it - if not node.has_key(k): break - - if t == 0 and node[k] not in v: - # link - if this node'd property doesn't appear in the - # filterspec's nodeid list, skip it - break - elif t == 1: - # multilink - if any of the nodeids required by the - # filterspec aren't in this node's property, then skip - # it - for value in v: - if value not in node[k]: - break - else: - continue - break - elif t == 2 and (node[k] is None or not v.search(node[k])): - # RE search - break - elif t == 6 and node[k] != v: - # straight value comparison for the other types - break - else: - l.append((nodeid, node)) - finally: - cldb.close() - l.sort() - - # filter based on full text search - if search_matches is not None: - k = [] - l_debug = [] - for v in l: - l_debug.append(v[0]) - if search_matches.has_key(v[0]): - k.append(v) - l = k - - # optimise sort - m = [] - for entry in sort: - if entry[0] != '-': - m.append(('+', entry)) - else: - m.append((entry[0], entry[1:])) - sort = m - - # optimise group - m = [] - for entry in group: - if entry[0] != '-': - m.append(('+', entry)) - else: - m.append((entry[0], entry[1:])) - group = m - # now, sort the result - def sortfun(a, b, sort=sort, group=group, properties=self.getprops(), - db = self.db, cl=self): - a_id, an = a - b_id, bn = b - # sort by group and then sort - for list in group, sort: - for dir, prop in list: - # sorting is class-specific - propclass = properties[prop] - - # handle the properties that might be "faked" - # also, handle possible missing properties - try: - if not an.has_key(prop): - an[prop] = cl.get(a_id, prop) - av = an[prop] - except KeyError: - # the node doesn't have a value for this property - if isinstance(propclass, Multilink): av = [] - else: av = '' - try: - if not bn.has_key(prop): - bn[prop] = cl.get(b_id, prop) - bv = bn[prop] - except KeyError: - # the node doesn't have a value for this property - if isinstance(propclass, Multilink): bv = [] - else: bv = '' - - # String and Date values are sorted in the natural way - if isinstance(propclass, String): - # clean up the strings - if av and av[0] in string.uppercase: - av = an[prop] = av.lower() - if bv and bv[0] in string.uppercase: - bv = bn[prop] = bv.lower() - if (isinstance(propclass, String) or - isinstance(propclass, Date)): - # it might be a string that's really an integer - try: - av = int(av) - bv = int(bv) - except: - pass - if dir == '+': - r = cmp(av, bv) - if r != 0: return r - elif dir == '-': - r = cmp(bv, av) - if r != 0: return r - - # Link properties are sorted according to the value of - # the "order" property on the linked nodes if it is - # present; or otherwise on the key string of the linked - # nodes; or finally on the node ids. - elif isinstance(propclass, Link): - link = db.classes[propclass.classname] - if av is None and bv is not None: return -1 - if av is not None and bv is None: return 1 - if av is None and bv is None: continue - if link.getprops().has_key('order'): - if dir == '+': - r = cmp(link.get(av, 'order'), - link.get(bv, 'order')) - if r != 0: return r - elif dir == '-': - r = cmp(link.get(bv, 'order'), - link.get(av, 'order')) - if r != 0: return r - elif link.getkey(): - key = link.getkey() - if dir == '+': - r = cmp(link.get(av, key), link.get(bv, key)) - if r != 0: return r - elif dir == '-': - r = cmp(link.get(bv, key), link.get(av, key)) - if r != 0: return r - else: - if dir == '+': - r = cmp(av, bv) - if r != 0: return r - elif dir == '-': - r = cmp(bv, av) - if r != 0: return r - - # Multilink properties are sorted according to how many - # links are present. - elif isinstance(propclass, Multilink): - if dir == '+': - r = cmp(len(av), len(bv)) - if r != 0: return r - elif dir == '-': - r = cmp(len(bv), len(av)) - if r != 0: return r - # end for dir, prop in list: - # end for list in sort, group: - # if all else fails, compare the ids - return cmp(a[0], b[0]) - - l.sort(sortfun) - return [i[0] for i in l] + raise NotImplementedError def count(self): """Get the number of nodes in this class. @@ -1121,18 +497,15 @@ class Class: in this class run from 1 to numnodes, and numnodes+1 will be the id of the next node to be created in this class. """ - return self.db.countnodes(self.classname) + raise NotImplementedError # Manipulating properties: - def getprops(self, protected=1): """Return a dictionary mapping property names to property objects. If the "protected" flag is true, we include protected properties - - those which may not be modified.""" - d = self.properties.copy() - if protected: - d['id'] = String() - return d + those which may not be modified. + """ + raise NotImplementedError def addprop(self, **properties): """Add properties to this class. @@ -1142,20 +515,12 @@ class Class: may collide with the names of existing properties, or a ValueError is raised before any properties have been added. """ - for key in properties.keys(): - if self.properties.has_key(key): - raise ValueError, key - self.properties.update(properties) + raise NotImplementedError def index(self, nodeid): '''Add (or refresh) the node to search indexes ''' - # find all the String properties that have indexme - for prop, propclass in self.getprops().items(): - if isinstance(propclass, String) and propclass.indexme: - # and index them under (classname, nodeid, property) - self.db.indexer.add_text((self.classname, nodeid, prop), - str(self.get(nodeid, prop))) + raise NotImplementedError # XXX not in spec class Node: @@ -1215,6 +580,9 @@ def Choice(name, db, *options): # # $Log: not supported by cvs2svn $ +# Revision 1.74 2002/07/10 00:24:10 richard +# braino +# # Revision 1.73 2002/07/10 00:19:48 richard # Added explicit closing of backend database handles. # diff --git a/roundup/init.py b/roundup/init.py index fe2cde8..9214fb6 100644 --- a/roundup/init.py +++ b/roundup/init.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: init.py,v 1.19 2002-05-23 01:14:20 richard Exp $ +# $Id: init.py,v 1.20 2002-07-14 02:05:53 richard Exp $ __doc__ = """ Init (create) a roundup instance. @@ -98,7 +98,8 @@ def install(instance_home, template, backend): # now select database db = '''# WARNING: DO NOT EDIT THIS FILE!!! -from roundup.backends.back_%s import Database'''%backend +from roundup.backends.back_%s import Database, Class, FileClass, IssueClass +'''%backend open(os.path.join(instance_home, 'select_db.py'), 'w').write(db) @@ -113,6 +114,10 @@ def initialise(instance_home, adminpw): # # $Log: not supported by cvs2svn $ +# Revision 1.19 2002/05/23 01:14:20 richard +# . split instance initialisation into two steps, allowing config changes +# before the database is initialised. +# # Revision 1.18 2001/11/22 15:46:42 jhermann # Added module docstrings to all modules. # diff --git a/roundup/roundupdb.py b/roundup/roundupdb.py index dc181a4..57e678d 100644 --- a/roundup/roundupdb.py +++ b/roundup/roundupdb.py @@ -15,13 +15,13 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: roundupdb.py,v 1.61 2002-07-09 04:19:09 richard Exp $ +# $Id: roundupdb.py,v 1.62 2002-07-14 02:05:53 richard Exp $ __doc__ = """ Extending hyperdb with types specific to issue-tracking. """ -import re, os, smtplib, socket, copy, time, random +import re, os, smtplib, socket, time, random import MimeWriter, cStringIO import base64, quopri, mimetypes # if available, use the 'email' module, otherwise fallback to 'rfc822' @@ -30,22 +30,12 @@ try : except ImportError : from rfc822 import dump_address_pair as straddr -import hyperdb, date +import hyperdb # set to indicate to roundup not to actually _send_ email # this var must contain a file to write the mail to SENDMAILDEBUG = os.environ.get('SENDMAILDEBUG', '') -class DesignatorError(ValueError): - pass -def splitDesignator(designator, dre=re.compile(r'([^\d]+)(\d+)')): - ''' Take a foo123 and return ('foo', 123) - ''' - m = dre.match(designator) - if m is None: - raise DesignatorError, '"%s" not a node designator'%designator - return m.group(1), m.group(2) - def extractUserFromList(userClass, users): '''Given a list of users, try to extract the first non-anonymous user @@ -102,200 +92,6 @@ class Database: else: return 0 -_marker = [] -# XXX: added the 'creator' faked attribute -class Class(hyperdb.Class): - # Overridden methods: - def __init__(self, db, classname, **properties): - if (properties.has_key('creation') or properties.has_key('activity') - or properties.has_key('creator')): - raise ValueError, '"creation", "activity" and "creator" are reserved' - hyperdb.Class.__init__(self, db, classname, **properties) - self.auditors = {'create': [], 'set': [], 'retire': []} - self.reactors = {'create': [], 'set': [], 'retire': []} - - def create(self, **propvalues): - """These operations trigger detectors and can be vetoed. Attempts - to modify the "creation" or "activity" properties cause a KeyError. - """ - if propvalues.has_key('creation') or propvalues.has_key('activity'): - raise KeyError, '"creation" and "activity" are reserved' - self.fireAuditors('create', None, propvalues) - nodeid = hyperdb.Class.create(self, **propvalues) - self.fireReactors('create', nodeid, None) - return nodeid - - def set(self, nodeid, **propvalues): - """These operations trigger detectors and can be vetoed. Attempts - to modify the "creation" or "activity" properties cause a KeyError. - """ - if propvalues.has_key('creation') or propvalues.has_key('activity'): - raise KeyError, '"creation" and "activity" are reserved' - self.fireAuditors('set', nodeid, propvalues) - # Take a copy of the node dict so that the subsequent set - # operation doesn't modify the oldvalues structure. - try: - # try not using the cache initially - oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid, - cache=0)) - except IndexError: - # this will be needed if somone does a create() and set() - # with no intervening commit() - oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid)) - hyperdb.Class.set(self, nodeid, **propvalues) - self.fireReactors('set', nodeid, oldvalues) - - def retire(self, nodeid): - """These operations trigger detectors and can be vetoed. Attempts - to modify the "creation" or "activity" properties cause a KeyError. - """ - self.fireAuditors('retire', nodeid, None) - hyperdb.Class.retire(self, nodeid) - self.fireReactors('retire', nodeid, None) - - def get(self, nodeid, propname, default=_marker, cache=1): - """Attempts to get the "creation" or "activity" properties should - do the right thing. - """ - if propname == 'creation': - journal = self.db.getjournal(self.classname, nodeid) - if journal: - return self.db.getjournal(self.classname, nodeid)[0][1] - else: - # on the strange chance that there's no journal - return date.Date() - if propname == 'activity': - journal = self.db.getjournal(self.classname, nodeid) - if journal: - return self.db.getjournal(self.classname, nodeid)[-1][1] - else: - # on the strange chance that there's no journal - return date.Date() - if propname == 'creator': - journal = self.db.getjournal(self.classname, nodeid) - if journal: - name = self.db.getjournal(self.classname, nodeid)[0][2] - else: - return None - return self.db.user.lookup(name) - if default is not _marker: - return hyperdb.Class.get(self, nodeid, propname, default, - cache=cache) - else: - return hyperdb.Class.get(self, nodeid, propname, cache=cache) - - def getprops(self, protected=1): - """In addition to the actual properties on the node, these - methods provide the "creation" and "activity" properties. If the - "protected" flag is true, we include protected properties - those - which may not be modified. - """ - d = hyperdb.Class.getprops(self, protected=protected).copy() - if protected: - d['creation'] = hyperdb.Date() - d['activity'] = hyperdb.Date() - d['creator'] = hyperdb.Link("user") - return d - - # - # Detector interface - # - def audit(self, event, detector): - """Register a detector - """ - l = self.auditors[event] - if detector not in l: - self.auditors[event].append(detector) - - def fireAuditors(self, action, nodeid, newvalues): - """Fire all registered auditors. - """ - for audit in self.auditors[action]: - audit(self.db, self, nodeid, newvalues) - - def react(self, event, detector): - """Register a detector - """ - l = self.reactors[event] - if detector not in l: - self.reactors[event].append(detector) - - def fireReactors(self, action, nodeid, oldvalues): - """Fire all registered reactors. - """ - for react in self.reactors[action]: - react(self.db, self, nodeid, oldvalues) - -class FileClass(Class): - '''This class defines a large chunk of data. To support this, it has a - mandatory String property "content" which is typically saved off - externally to the hyperdb. - - The default MIME type of this data is defined by the - "default_mime_type" class attribute, which may be overridden by each - node if the class defines a "type" String property. - ''' - default_mime_type = 'text/plain' - - def create(self, **propvalues): - ''' snaffle the file propvalue and store in a file - ''' - content = propvalues['content'] - del propvalues['content'] - newid = Class.create(self, **propvalues) - self.db.storefile(self.classname, newid, None, content) - return newid - - def get(self, nodeid, propname, default=_marker, cache=1): - ''' trap the content propname and get it from the file - ''' - - poss_msg = 'Possibly a access right configuration problem.' - if propname == 'content': - try: - return self.db.getfile(self.classname, nodeid, None) - except IOError, (strerror): - # BUG: by catching this we donot see an error in the log. - return 'ERROR reading file: %s%s\n%s\n%s'%( - self.classname, nodeid, poss_msg, strerror) - if default is not _marker: - return Class.get(self, nodeid, propname, default, cache=cache) - else: - return Class.get(self, nodeid, propname, cache=cache) - - def getprops(self, protected=1): - ''' In addition to the actual properties on the node, these methods - provide the "content" property. If the "protected" flag is true, - we include protected properties - those which may not be - modified. - ''' - d = Class.getprops(self, protected=protected).copy() - if protected: - d['content'] = hyperdb.String() - return d - - def index(self, nodeid): - ''' Index the node in the search index. - - We want to index the content in addition to the normal String - property indexing. - ''' - # perform normal indexing - Class.index(self, nodeid) - - # get the content to index - content = self.get(nodeid, 'content') - - # figure the mime type - if self.properties.has_key('type'): - mime_type = self.get(nodeid, 'type') - else: - mime_type = self.default_mime_type - - # and index! - self.db.indexer.add_text((self.classname, nodeid, 'content'), content, - mime_type) - class MessageSendError(RuntimeError): pass @@ -303,29 +99,19 @@ class DetectorError(RuntimeError): pass # XXX deviation from spec - was called ItemClass -class IssueClass(Class): - - # Overridden methods: - - def __init__(self, db, classname, **properties): - """The newly-created class automatically includes the "messages", - "files", "nosy", and "superseder" properties. If the 'properties' - dictionary attempts to specify any of these properties or a - "creation" or "activity" property, a ValueError is raised.""" - if not properties.has_key('title'): +class IssueClass: + """ This class is intended to be mixed-in with a hyperdb backend + implementation. The backend should provide a mechanism that + enforces the title, messages, files, nosy and superseder + properties: properties['title'] = hyperdb.String(indexme='yes') - if not properties.has_key('messages'): properties['messages'] = hyperdb.Multilink("msg") - if not properties.has_key('files'): properties['files'] = hyperdb.Multilink("file") - if not properties.has_key('nosy'): properties['nosy'] = hyperdb.Multilink("user") - if not properties.has_key('superseder'): properties['superseder'] = hyperdb.Multilink(classname) - Class.__init__(self, db, classname, **properties) + """ # New methods: - def addmessage(self, nodeid, summary, text): """Add a message to an issue's mail spool. @@ -553,15 +339,16 @@ class IssueClass(Class): # simplistic check to see if the url is valid, # then append a trailing slash if it is missing base = self.db.config.ISSUE_TRACKER_WEB - if not isinstance( base , type('') ) or not base.startswith( "http://" ) : - base = "Configuration Error: ISSUE_TRACKER_WEB isn't a fully-qualified URL" + if not isinstance(base , type('')) or not base.startswith('http://'): + base = "Configuration Error: ISSUE_TRACKER_WEB isn't a " \ + "fully-qualified URL" elif base[-1] != '/' : base += '/' web = base + 'issue'+ nodeid # ensure the email address is properly quoted - email = straddr( (self.db.config.INSTANCE_NAME , - self.db.config.ISSUE_TRACKER_EMAIL) ) + email = straddr((self.db.config.INSTANCE_NAME, + self.db.config.ISSUE_TRACKER_EMAIL)) line = '_' * max(len(web), len(email)) return '%s\n%s\n%s\n%s'%(line, email, web, line) @@ -608,12 +395,10 @@ class IssueClass(Class): def generateChangeNote(self, nodeid, oldvalues): """Generate a change note that lists property changes """ - if __debug__ : - if not isinstance( oldvalues , type({}) ) : - raise TypeError( - "'oldvalues' must be dict-like, not %s." - % str(type(oldvalues)) ) + if not isinstance(oldvalues, type({})) : + raise TypeError("'oldvalues' must be dict-like, not %s."% + type(oldvalues)) cn = self.classname cl = self.db.classes[cn] @@ -691,6 +476,11 @@ class IssueClass(Class): # # $Log: not supported by cvs2svn $ +# Revision 1.61 2002/07/09 04:19:09 richard +# Added reindex command to roundup-admin. +# Fixed reindex on first access. +# Also fixed reindexing of entries that change. +# # Revision 1.60 2002/07/09 03:02:52 richard # More indexer work: # - all String properties may now be indexed too. Currently there's a bit of diff --git a/roundup/templates/classic/dbinit.py b/roundup/templates/classic/dbinit.py index f6c1829..03fbeac 100644 --- a/roundup/templates/classic/dbinit.py +++ b/roundup/templates/classic/dbinit.py @@ -15,32 +15,15 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: dbinit.py,v 1.18 2002-07-09 03:02:53 richard Exp $ +# $Id: dbinit.py,v 1.19 2002-07-14 02:05:54 richard Exp $ import os import instance_config -from roundup import roundupdb -import select_db +from select_db import Database, Class, FileClass, IssueClass -from roundup.roundupdb import Class, FileClass - -class Database(roundupdb.Database, select_db.Database): - ''' Creates a hybrid database from: - . the selected database back-end from select_db - . the roundup extensions from roundupdb - ''' - pass - -class IssueClass(roundupdb.IssueClass): - ''' issues need the email information - ''' - pass - - def open(name=None): ''' as from the roundupdb method openDB - ''' from roundup.hyperdb import String, Password, Date, Link, Multilink @@ -143,6 +126,22 @@ def init(adminpw): # # $Log: not supported by cvs2svn $ +# Revision 1.18 2002/07/09 03:02:53 richard +# More indexer work: +# - all String properties may now be indexed too. Currently there's a bit of +# "issue" specific code in the actual searching which needs to be +# addressed. In a nutshell: +# + pass 'indexme="yes"' as a String() property initialisation arg, eg: +# file = FileClass(db, "file", name=String(), type=String(), +# comment=String(indexme="yes")) +# + the comment will then be indexed and be searchable, with the results +# related back to the issue that the file is linked to +# - as a result of this work, the FileClass has a default MIME type that may +# be overridden in a subclass, or by the use of a "type" property as is +# done in the default templates. +# - the regeneration of the indexes (if necessary) is done once the schema is +# set up in the dbinit. +# # Revision 1.17 2002/05/24 04:03:23 richard # Added commentage to the dbinit files to help people with their # customisation. diff --git a/roundup/templates/extended/dbinit.py b/roundup/templates/extended/dbinit.py index fb8f875..0ada216 100644 --- a/roundup/templates/extended/dbinit.py +++ b/roundup/templates/extended/dbinit.py @@ -15,29 +15,13 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: dbinit.py,v 1.22 2002-07-09 03:02:53 richard Exp $ +# $Id: dbinit.py,v 1.23 2002-07-14 02:05:54 richard Exp $ import os import instance_config -from roundup import roundupdb -import select_db +from select_db import Database, Class, FileClass, IssueClass -from roundup.roundupdb import Class, FileClass - -class Database(roundupdb.Database, select_db.Database): - ''' Creates a hybrid database from: - . the selected database back-end from select_db - . the roundup extensions from roundupdb - ''' - pass - -class IssueClass(roundupdb.IssueClass): - ''' issues need the email information - ''' - pass - - def open(name=None): ''' as from the roundupdb method openDB @@ -195,6 +179,22 @@ def init(adminpw): # # $Log: not supported by cvs2svn $ +# Revision 1.22 2002/07/09 03:02:53 richard +# More indexer work: +# - all String properties may now be indexed too. Currently there's a bit of +# "issue" specific code in the actual searching which needs to be +# addressed. In a nutshell: +# + pass 'indexme="yes"' as a String() property initialisation arg, eg: +# file = FileClass(db, "file", name=String(), type=String(), +# comment=String(indexme="yes")) +# + the comment will then be indexed and be searchable, with the results +# related back to the issue that the file is linked to +# - as a result of this work, the FileClass has a default MIME type that may +# be overridden in a subclass, or by the use of a "type" property as is +# done in the default templates. +# - the regeneration of the indexes (if necessary) is done once the schema is +# set up in the dbinit. +# # Revision 1.21 2002/05/24 04:03:23 richard # Added commentage to the dbinit files to help people with their # customisation. diff --git a/test/test_db.py b/test/test_db.py index 3b9c9ba..e8a5b6b 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -15,23 +15,22 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: test_db.py,v 1.26 2002-07-11 01:11:03 richard Exp $ +# $Id: test_db.py,v 1.27 2002-07-14 02:05:54 richard Exp $ import unittest, os, shutil from roundup.hyperdb import String, Password, Link, Multilink, Date, \ - Interval, DatabaseError, Class -from roundup.roundupdb import FileClass + Interval, DatabaseError from roundup import date, password from roundup.indexer import Indexer -def setupSchema(db, create, Class, FileClass): - status = Class(db, "status", name=String()) +def setupSchema(db, create, module): + status = module.Class(db, "status", name=String()) status.setkey("name") - user = Class(db, "user", username=String(), password=Password()) - file = FileClass(db, "file", name=String(), type=String(), + user = module.Class(db, "user", username=String(), password=Password()) + file = module.FileClass(db, "file", name=String(), type=String(), comment=String(indexme="yes")) - issue = Class(db, "issue", title=String(indexme="yes"), + issue = module.IssueClass(db, "issue", title=String(indexme="yes"), status=Link("status"), nosy=Multilink("user"), deadline=Date(), foo=Interval(), files=Multilink("file")) db.post_init() @@ -69,9 +68,9 @@ class anydbmDBTestCase(MyTestCase): shutil.rmtree(config.DATABASE) os.makedirs(config.DATABASE + '/files') self.db = anydbm.Database(config, 'test') - setupSchema(self.db, 1, Class, FileClass) + setupSchema(self.db, 1, anydbm) self.db2 = anydbm.Database(config, 'test') - setupSchema(self.db2, 0, Class, FileClass) + setupSchema(self.db2, 0, anydbm) def testStringChange(self): self.db.issue.create(title="spam", status='1') @@ -117,8 +116,9 @@ class anydbmDBTestCase(MyTestCase): props = self.db.issue.getprops() keys = props.keys() keys.sort() - self.assertEqual(keys, ['deadline', 'files', 'fixer', 'foo', 'id', - 'nosy', 'status', 'title']) + self.assertEqual(keys, ['activity', 'creation', 'creator', 'deadline', + 'files', 'fixer', 'foo', 'id', 'messages', 'nosy', 'status', + 'superseder', 'title']) self.assertEqual(self.db.issue.get('1', "fixer"), None) def testRetire(self): @@ -251,8 +251,8 @@ class anydbmDBTestCase(MyTestCase): self.assertEqual(action, 'create') keys = params.keys() keys.sort() - self.assertEqual(keys, ['deadline', 'files', 'fixer', 'foo', 'nosy', - 'status', 'title']) + self.assertEqual(keys, ['deadline', 'files', 'fixer', 'foo', + 'messages', 'nosy', 'status', 'superseder', 'title']) self.assertEqual(None,params['deadline']) self.assertEqual(None,params['fixer']) self.assertEqual(None,params['foo']) @@ -347,11 +347,11 @@ class anydbmReadOnlyDBTestCase(MyTestCase): shutil.rmtree(config.DATABASE) os.makedirs(config.DATABASE + '/files') db = anydbm.Database(config, 'test') - setupSchema(db, 1, Class, FileClass) + setupSchema(db, 1, anydbm) self.db = anydbm.Database(config) - setupSchema(self.db, 0, Class, FileClass) + setupSchema(self.db, 0, anydbm) self.db2 = anydbm.Database(config, 'test') - setupSchema(self.db2, 0, Class, FileClass) + setupSchema(self.db2, 0, anydbm) def testExceptions(self): ' make sure exceptions are raised on writes to a read-only db ' @@ -372,9 +372,9 @@ class bsddbDBTestCase(anydbmDBTestCase): shutil.rmtree(config.DATABASE) os.makedirs(config.DATABASE + '/files') self.db = bsddb.Database(config, 'test') - setupSchema(self.db, 1, Class, FileClass) + setupSchema(self.db, 1, bsddb) self.db2 = bsddb.Database(config, 'test') - setupSchema(self.db2, 0, Class, FileClass) + setupSchema(self.db2, 0, bsddb) class bsddbReadOnlyDBTestCase(anydbmReadOnlyDBTestCase): def setUp(self): @@ -384,11 +384,11 @@ class bsddbReadOnlyDBTestCase(anydbmReadOnlyDBTestCase): shutil.rmtree(config.DATABASE) os.makedirs(config.DATABASE + '/files') db = bsddb.Database(config, 'test') - setupSchema(db, 1, Class, FileClass) + setupSchema(db, 1, bsddb) self.db = bsddb.Database(config) - setupSchema(self.db, 0, Class, FileClass) + setupSchema(self.db, 0, bsddb) self.db2 = bsddb.Database(config, 'test') - setupSchema(self.db2, 0, Class, FileClass) + setupSchema(self.db2, 0, bsddb) class bsddb3DBTestCase(anydbmDBTestCase): @@ -399,9 +399,9 @@ class bsddb3DBTestCase(anydbmDBTestCase): shutil.rmtree(config.DATABASE) os.makedirs(config.DATABASE + '/files') self.db = bsddb3.Database(config, 'test') - setupSchema(self.db, 1, Class, FileClass) + setupSchema(self.db, 1, bsddb3) self.db2 = bsddb3.Database(config, 'test') - setupSchema(self.db2, 0, Class, FileClass) + setupSchema(self.db2, 0, bsddb3) class bsddb3ReadOnlyDBTestCase(anydbmReadOnlyDBTestCase): def setUp(self): @@ -411,11 +411,11 @@ class bsddb3ReadOnlyDBTestCase(anydbmReadOnlyDBTestCase): shutil.rmtree(config.DATABASE) os.makedirs(config.DATABASE + '/files') db = bsddb3.Database(config, 'test') - setupSchema(db, 1, Class, FileClass) + setupSchema(db, 1, bsddb3) self.db = bsddb3.Database(config) - setupSchema(self.db, 0, Class, FileClass) + setupSchema(self.db, 0, bsddb3) self.db2 = bsddb3.Database(config, 'test') - setupSchema(self.db2, 0, Class, FileClass) + setupSchema(self.db2, 0, bsddb3) class metakitDBTestCase(anydbmDBTestCase): @@ -428,9 +428,9 @@ class metakitDBTestCase(anydbmDBTestCase): shutil.rmtree(config.DATABASE) os.makedirs(config.DATABASE + '/files') self.db = metakit.Database(config, 'test') - setupSchema(self.db, 1, metakit.Class, metakit.FileClass) + setupSchema(self.db, 1, metakit) self.db2 = metakit.Database(config, 'test') - setupSchema(self.db2, 0, metakit.Class, metakit.FileClass) + setupSchema(self.db2, 0, metakit) def testTransactions(self): # remember the number of items we started @@ -480,11 +480,11 @@ class metakitReadOnlyDBTestCase(anydbmReadOnlyDBTestCase): shutil.rmtree(config.DATABASE) os.makedirs(config.DATABASE + '/files') db = metakit.Database(config, 'test') - setupSchema(db, 1, metakit.Class, metakit.FileClass) + setupSchema(db, 1, metakit) self.db = metakit.Database(config) - setupSchema(self.db, 0, metakit.Class, metakit.FileClass) + setupSchema(self.db, 0, metakit) self.db2 = metakit.Database(config, 'test') - setupSchema(self.db2, 0, metakit.Class, metakit.FileClass) + setupSchema(self.db2, 0, metakit) def suite(): l = [ @@ -517,6 +517,10 @@ def suite(): # # $Log: not supported by cvs2svn $ +# Revision 1.26 2002/07/11 01:11:03 richard +# Added metakit backend to the db tests and fixed the more easily fixable test +# failures. +# # Revision 1.25 2002/07/09 04:19:09 richard # Added reindex command to roundup-admin. # Fixed reindex on first access. diff --git a/test/test_init.py b/test/test_init.py index 09b3029..1a9520d 100644 --- a/test/test_init.py +++ b/test/test_init.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: test_init.py,v 1.12 2002-07-11 01:13:13 richard Exp $ +# $Id: test_init.py,v 1.13 2002-07-14 02:05:54 richard Exp $ import unittest, os, shutil, errno, imp, sys @@ -125,8 +125,11 @@ class metakitExtendedTestCase(ExtendedTestCase): backend = 'metakit' def suite(): - l = [unittest.makeSuite(ClassicTestCase, 'test'), - unittest.makeSuite(ExtendedTestCase, 'test')] + l = [ + unittest.makeSuite(ClassicTestCase, 'test'), + unittest.makeSuite(ExtendedTestCase, 'test') + ] + try: import bsddb l.append(unittest.makeSuite(bsddbClassicTestCase, 'test')) @@ -152,6 +155,9 @@ def suite(): # # $Log: not supported by cvs2svn $ +# Revision 1.12 2002/07/11 01:13:13 richard +# *** empty log message *** +# # Revision 1.11 2002/07/11 01:12:34 richard # Forgot to add to init tests # diff --git a/test/test_schema.py b/test/test_schema.py index 88ac024..89a78ce 100644 --- a/test/test_schema.py +++ b/test/test_schema.py @@ -15,13 +15,13 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: test_schema.py,v 1.7 2002-01-14 02:20:15 richard Exp $ +# $Id: test_schema.py,v 1.8 2002-07-14 02:05:54 richard Exp $ import unittest, os, shutil -from roundup.backends import anydbm +from roundup.backends import back_anydbm from roundup.hyperdb import String, Password, Link, Multilink, Date, \ - Interval, Class + Interval class config: DATABASE='_test_dir' @@ -39,20 +39,18 @@ class config: class SchemaTestCase(unittest.TestCase): def setUp(self): - class Database(anydbm.Database): - pass # remove previous test, ignore errors if os.path.exists(config.DATABASE): shutil.rmtree(config.DATABASE) os.makedirs(config.DATABASE + '/files') - self.db = Database(config, 'test') + self.db = back_anydbm.Database(config, 'test') self.db.clear() def tearDown(self): shutil.rmtree('_test_dir') def testA_Status(self): - status = Class(self.db, "status", name=String()) + status = back_anydbm.Class(self.db, "status", name=String()) self.assert_(status, 'no class object generated') status.setkey("name") val = status.create(name="unread") @@ -74,11 +72,13 @@ class SchemaTestCase(unittest.TestCase): self.assertEqual(val, ['1', '2', '4'], 'blah') def testB_Issue(self): - issue = Class(self.db, "issue", title=String(), status=Link("status")) + issue = back_anydbm.Class(self.db, "issue", title=String(), + status=Link("status")) self.assert_(issue, 'no class object returned') def testC_User(self): - user = Class(self.db, "user", username=String(), password=Password()) + user = back_anydbm.Class(self.db, "user", username=String(), + password=Password()) self.assert_(user, 'no class object returned') user.setkey("username") @@ -89,6 +89,15 @@ def suite(): # # $Log: not supported by cvs2svn $ +# Revision 1.7 2002/01/14 02:20:15 richard +# . changed all config accesses so they access either the instance or the +# config attriubute on the db. This means that all config is obtained from +# instance_config instead of the mish-mash of classes. This will make +# switching to a ConfigParser setup easier too, I hope. +# +# At a minimum, this makes migration a _little_ easier (a lot easier in the +# 0.5.0 switch, I hope!) +# # Revision 1.6 2001/12/03 21:33:39 richard # Fixes so the tests use commit and not close # -- 2.30.2