X-Git-Url: https://git.tokkee.org/?a=blobdiff_plain;f=roundup%2Fhyperdb.py;h=4bac87fc63039f2f258e2a26737990063e101f3b;hb=c9e46f5700bb319acc389498e7ec904a4ac7ed11;hp=b4c06039c4259eac0e6a569cbb40a65879cd1d83;hpb=276e0395a0dfc3223c8941fe3b8218a28f1433d0;p=roundup.git diff --git a/roundup/hyperdb.py b/roundup/hyperdb.py index b4c0603..4bac87f 100644 --- a/roundup/hyperdb.py +++ b/roundup/hyperdb.py @@ -15,66 +15,132 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: hyperdb.py,v 1.47 2002-01-14 02:20:15 richard Exp $ +# $Id: hyperdb.py,v 1.96 2004-02-11 23:55:08 richard Exp $ -__doc__ = """ -Hyperdatabase implementation, especially field types. +"""Hyperdatabase implementation, especially field types. """ +__docformat__ = 'restructuredtext' # standard python modules -import cPickle, re, string, weakref +import sys, os, time, re # roundup modules import date, password +# configure up the DEBUG and TRACE captures +class Sink: + def write(self, content): + pass +DEBUG = os.environ.get('HYPERDBDEBUG', '') +if DEBUG and __debug__: + if DEBUG == 'stdout': + DEBUG = sys.stdout + else: + DEBUG = open(DEBUG, 'a') +else: + DEBUG = Sink() +TRACE = os.environ.get('HYPERDBTRACE', '') +if TRACE and __debug__: + if TRACE == 'stdout': + TRACE = sys.stdout + else: + TRACE = open(TRACE, 'w') +else: + TRACE = Sink() +def traceMark(): + print >>TRACE, '**MARK', time.ctime() +del Sink # # Types # class String: """An object designating a String property.""" + def __init__(self, indexme='no'): + self.indexme = indexme == 'yes' def __repr__(self): + ' more useful for dumps ' return '<%s>'%self.__class__ class Password: """An object designating a Password property.""" def __repr__(self): + ' more useful for dumps ' return '<%s>'%self.__class__ class Date: """An object designating a Date property.""" def __repr__(self): + ' more useful for dumps ' return '<%s>'%self.__class__ class Interval: """An object designating an Interval property.""" def __repr__(self): + ' more useful for dumps ' return '<%s>'%self.__class__ class Link: """An object designating a Link property that links to a node in a specified class.""" - def __init__(self, classname): + def __init__(self, classname, do_journal='yes'): + ''' Default is to not journal link and unlink events + ''' self.classname = classname + self.do_journal = do_journal == 'yes' def __repr__(self): + ' more useful for dumps ' return '<%s to "%s">'%(self.__class__, self.classname) class Multilink: """An object designating a Multilink property that links to nodes in a specified class. + + "classname" indicates the class to link to + + "do_journal" indicates whether the linked-to nodes should have + 'link' and 'unlink' events placed in their journal """ - def __init__(self, classname): + def __init__(self, classname, do_journal='yes'): + ''' Default is to not journal link and unlink events + ''' self.classname = classname + self.do_journal = do_journal == 'yes' def __repr__(self): + ' more useful for dumps ' return '<%s to "%s">'%(self.__class__, self.classname) -class DatabaseError(ValueError): +class Boolean: + """An object designating a boolean property""" + def __repr__(self): + 'more useful for dumps' + return '<%s>' % self.__class__ + +class Number: + """An object designating a numeric property""" + def __repr__(self): + 'more useful for dumps' + return '<%s>' % self.__class__ +# +# Support for splitting designators +# +class DesignatorError(ValueError): pass - +def splitDesignator(designator, dre=re.compile(r'([^\d]+)(\d+)')): + ''' Take a foo123 and return ('foo', 123) + ''' + m = dre.match(designator) + if m is None: + raise DesignatorError, '"%s" not a node designator'%designator + return m.group(1), m.group(2) # # the base Database class # +class DatabaseError(ValueError): + '''Error to be raised when there is some problem in the database code + ''' + pass class Database: '''A database for storing records containing flexible data types. @@ -93,12 +159,17 @@ in-database value is returned in preference to the in-transaction value. This is necessary to determine if any values have changed during a transaction. + +Implementation +-------------- + +All methods except __repr__ must be implemented by a concrete backend Database. + ''' # flag to set on retired entries RETIRED_FLAG = '__hyperdb_retired' - # XXX deviates from spec: storagelocator is obtained from the config def __init__(self, config, journaltag=None): """Open a hyperdatabase given a specifier to some storage. @@ -115,6 +186,18 @@ transaction. """ raise NotImplementedError + def post_init(self): + """Called once the schema initialisation has finished. + If 'refresh' is true, we want to rebuild the backend + structures. + """ + raise NotImplementedError + + def refresh_database(self): + """Called to indicate that the backend should rebuild all tables + and structures. Not called in normal usage.""" + raise NotImplementedError + def __getattr__(self, classname): """A convenient way of calling self.getclass(classname).""" raise NotImplementedError @@ -147,35 +230,43 @@ transaction. raise NotImplementedError def addnode(self, classname, nodeid, node): - '''Add the specified node to its class's db. - ''' + """Add the specified node to its class's db. + """ raise NotImplementedError + def serialise(self, classname, node): + '''Copy the node contents, converting non-marshallable data into + marshallable data. + ''' + return node + def setnode(self, classname, nodeid, node): '''Change the specified node. ''' raise NotImplementedError - def getnode(self, classname, nodeid, db=None, cache=1): + def unserialise(self, classname, node): + '''Decode the marshalled node data + ''' + return node + + def getnode(self, classname, nodeid): '''Get a node from the database. + + 'cache' exists for backwards compatibility, and is not used. ''' raise NotImplementedError - def hasnode(self, classname, nodeid, db=None): + def hasnode(self, classname, nodeid): '''Determine if the database has a given node. ''' raise NotImplementedError - def countnodes(self, classname, db=None): + def countnodes(self, classname): '''Count the number of nodes that exist for a particular Class. ''' raise NotImplementedError - def getnodeids(self, classname, db=None): - '''Retrieve all the ids of the nodes for a particular Class. - ''' - raise NotImplementedError - def storefile(self, classname, nodeid, property, content): '''Store the content of the file in the database. @@ -204,6 +295,11 @@ transaction. ''' raise NotImplementedError + def pack(self, pack_before): + ''' pack the database + ''' + raise NotImplementedError + def commit(self): ''' Commit the current transactions. @@ -220,12 +316,15 @@ transaction. ''' raise NotImplementedError -_marker = [] # # The base Class class # class Class: - """The handle to a particular class of nodes in a hyperdatabase.""" + """ The handle to a particular class of nodes in a hyperdatabase. + + All methods except __repr__ and getnode must be implemented by a + concrete backend Class. + """ def __init__(self, db, classname, **properties): """Create a new class with a given name and property specification. @@ -234,16 +333,12 @@ class Class: or a ValueError is raised. The keyword arguments in 'properties' must map names to property objects, or a TypeError is raised. """ - self.classname = classname - self.properties = properties - self.db = weakref.proxy(db) # use a weak ref to avoid circularity - self.key = '' - - # do the db-related init stuff - db.addclass(self) + raise NotImplementedError def __repr__(self): - return ''%self.classname + '''Slightly more useful representation + ''' + return ''%self.classname # Editing nodes: @@ -264,122 +359,9 @@ class Class: If an id in a link or multilink property does not refer to a valid node, an IndexError is raised. """ - if propvalues.has_key('id'): - raise KeyError, '"id" is reserved' - - if self.db.journaltag is None: - raise DatabaseError, 'Database open read-only' - - # new node's id - newid = str(self.count() + 1) - - # validate propvalues - num_re = re.compile('^\d+$') - for key, value in propvalues.items(): - if key == self.key: - try: - self.lookup(value) - except KeyError: - pass - else: - raise ValueError, 'node with key "%s" exists'%value - - # try to handle this property - try: - prop = self.properties[key] - except KeyError: - raise KeyError, '"%s" has no property "%s"'%(self.classname, - key) - - if isinstance(prop, Link): - if type(value) != type(''): - raise ValueError, 'link value must be String' - link_class = self.properties[key].classname - # if it isn't a number, it's a key - if not num_re.match(value): - try: - value = self.db.classes[link_class].lookup(value) - except (TypeError, KeyError): - raise IndexError, 'new property "%s": %s not a %s'%( - key, value, link_class) - elif not self.db.hasnode(link_class, value): - raise IndexError, '%s has no node %s'%(link_class, value) - - # save off the value - propvalues[key] = value - - # register the link with the newly linked node - self.db.addjournal(link_class, value, 'link', - (self.classname, newid, key)) - - elif isinstance(prop, Multilink): - if type(value) != type([]): - raise TypeError, 'new property "%s" not a list of ids'%key - link_class = self.properties[key].classname - l = [] - for entry in value: - if type(entry) != type(''): - raise ValueError, 'link value must be String' - # if it isn't a number, it's a key - if not num_re.match(entry): - try: - entry = self.db.classes[link_class].lookup(entry) - except (TypeError, KeyError): - raise IndexError, 'new property "%s": %s not a %s'%( - key, entry, self.properties[key].classname) - l.append(entry) - value = l - propvalues[key] = value - - # handle additions - for id in value: - if not self.db.hasnode(link_class, id): - raise IndexError, '%s has no node %s'%(link_class, id) - # register the link with the newly linked node - self.db.addjournal(link_class, id, 'link', - (self.classname, newid, key)) - - elif isinstance(prop, String): - if type(value) != type(''): - raise TypeError, 'new property "%s" not a string'%key - - elif isinstance(prop, Password): - if not isinstance(value, password.Password): - raise TypeError, 'new property "%s" not a Password'%key - - elif isinstance(prop, Date): - if not isinstance(value, date.Date): - raise TypeError, 'new property "%s" not a Date'%key - - elif isinstance(prop, Interval): - if not isinstance(value, date.Interval): - raise TypeError, 'new property "%s" not an Interval'%key - - # make sure there's data where there needs to be - for key, prop in self.properties.items(): - if propvalues.has_key(key): - continue - if key == self.key: - raise ValueError, 'key property "%s" is required'%key - if isinstance(prop, Multilink): - propvalues[key] = [] - else: - propvalues[key] = None - - # convert all data to strings - for key, prop in self.properties.items(): - if isinstance(prop, Date): - propvalues[key] = propvalues[key].get_tuple() - elif isinstance(prop, Interval): - propvalues[key] = propvalues[key].get_tuple() - elif isinstance(prop, Password): - propvalues[key] = str(propvalues[key]) - - # done - self.db.addnode(self.classname, newid, propvalues) - self.db.addjournal(self.classname, newid, 'create', propvalues) - return newid + raise NotImplementedError + _marker = [] def get(self, nodeid, propname, default=_marker, cache=1): """Get the value of a property on an existing node of this class. @@ -387,47 +369,25 @@ class Class: IndexError is raised. 'propname' must be the name of a property of this class or a KeyError is raised. - 'cache' indicates whether the transaction cache should be queried - for the node. If the node has been modified and you need to - determine what its values prior to modification are, you need to - set cache=0. + 'cache' exists for backwards compatibility, and is not used. """ - if propname == 'id': - return nodeid - - # get the node's dict - d = self.db.getnode(self.classname, nodeid, cache=cache) - if not d.has_key(propname) and default is not _marker: - return default - - # get the value - prop = self.properties[propname] - - # possibly convert the marshalled data to instances - if isinstance(prop, Date): - return date.Date(d[propname]) - elif isinstance(prop, Interval): - return date.Interval(d[propname]) - elif isinstance(prop, Password): - p = password.Password() - p.unpack(d[propname]) - return p - - return d[propname] + raise NotImplementedError - # XXX not in spec - def getnode(self, nodeid, cache=1): + # not in spec + def getnode(self, nodeid): ''' Return a convenience wrapper for the node. 'nodeid' must be the id of an existing node of this class or an IndexError is raised. - 'cache' indicates whether the transaction cache should be queried - for the node. If the node has been modified and you need to - determine what its values prior to modification are, you need to - set cache=0. + 'cache' exists for backwards compatibility, and is not used. + ''' + return Node(self, nodeid) + + def getnodeids(self, retired=None): + '''Retrieve all the ids of the nodes for a particular Class. ''' - return Node(self, nodeid, cache=cache) + raise NotImplementedError def set(self, nodeid, **propvalues): """Modify a property on an existing node of this class. @@ -447,122 +407,7 @@ class Class: If the value of a Link or Multilink property contains an invalid node id, a ValueError is raised. """ - if not propvalues: - return - - if propvalues.has_key('id'): - raise KeyError, '"id" is reserved' - - if self.db.journaltag is None: - raise DatabaseError, 'Database open read-only' - - node = self.db.getnode(self.classname, nodeid) - if node.has_key(self.db.RETIRED_FLAG): - raise IndexError - num_re = re.compile('^\d+$') - for key, value in propvalues.items(): - # check to make sure we're not duplicating an existing key - if key == self.key and node[key] != value: - try: - self.lookup(value) - except KeyError: - pass - else: - raise ValueError, 'node with key "%s" exists'%value - - # this will raise the KeyError if the property isn't valid - # ... we don't use getprops() here because we only care about - # the writeable properties. - prop = self.properties[key] - - if isinstance(prop, Link): - link_class = self.properties[key].classname - # if it isn't a number, it's a key - if type(value) != type(''): - raise ValueError, 'link value must be String' - if not num_re.match(value): - try: - value = self.db.classes[link_class].lookup(value) - except (TypeError, KeyError): - raise IndexError, 'new property "%s": %s not a %s'%( - key, value, self.properties[key].classname) - - if not self.db.hasnode(link_class, value): - raise IndexError, '%s has no node %s'%(link_class, value) - - # register the unlink with the old linked node - if node[key] is not None: - self.db.addjournal(link_class, node[key], 'unlink', - (self.classname, nodeid, key)) - - # register the link with the newly linked node - if value is not None: - self.db.addjournal(link_class, value, 'link', - (self.classname, nodeid, key)) - - elif isinstance(prop, Multilink): - if type(value) != type([]): - raise TypeError, 'new property "%s" not a list of ids'%key - link_class = self.properties[key].classname - l = [] - for entry in value: - # if it isn't a number, it's a key - if type(entry) != type(''): - raise ValueError, 'link value must be String' - if not num_re.match(entry): - try: - entry = self.db.classes[link_class].lookup(entry) - except (TypeError, KeyError): - raise IndexError, 'new property "%s": %s not a %s'%( - key, entry, self.properties[key].classname) - l.append(entry) - value = l - propvalues[key] = value - - #handle removals - l = node[key] - for id in l[:]: - if id in value: - continue - # register the unlink with the old linked node - self.db.addjournal(link_class, id, 'unlink', - (self.classname, nodeid, key)) - l.remove(id) - - # handle additions - for id in value: - if not self.db.hasnode(link_class, id): - raise IndexError, '%s has no node %s'%(link_class, id) - if id in l: - continue - # register the link with the newly linked node - self.db.addjournal(link_class, id, 'link', - (self.classname, nodeid, key)) - l.append(id) - - elif isinstance(prop, String): - if value is not None and type(value) != type(''): - raise TypeError, 'new property "%s" not a string'%key - - elif isinstance(prop, Password): - if not isinstance(value, password.Password): - raise TypeError, 'new property "%s" not a Password'% key - propvalues[key] = value = str(value) - - elif isinstance(prop, Date): - if not isinstance(value, date.Date): - raise TypeError, 'new property "%s" not a Date'% key - propvalues[key] = value = value.get_tuple() - - elif isinstance(prop, Interval): - if not isinstance(value, date.Interval): - raise TypeError, 'new property "%s" not an Interval'% key - propvalues[key] = value = value.get_tuple() - - node[key] = value - - self.db.setnode(self.classname, nodeid, node) - self.db.addjournal(self.classname, nodeid, 'set', propvalues) + raise NotImplementedError def retire(self, nodeid): """Retire a node. @@ -573,12 +418,40 @@ class Class: Retired nodes are not returned by the find(), list(), or lookup() methods, and other nodes may reuse the values of their key properties. """ - if self.db.journaltag is None: - raise DatabaseError, 'Database open read-only' - node = self.db.getnode(self.classname, nodeid) - node[self.db.RETIRED_FLAG] = 1 - self.db.setnode(self.classname, nodeid, node) - self.db.addjournal(self.classname, nodeid, 'retired', None) + raise NotImplementedError + + def restore(self, nodeid): + '''Restpre a retired node. + + Make node available for all operations like it was before retirement. + ''' + raise NotImplementedError + + def is_retired(self, nodeid): + '''Return true if the node is rerired + ''' + raise NotImplementedError + + def destroy(self, nodeid): + """Destroy a node. + + WARNING: this method should never be used except in extremely rare + situations where there could never be links to the node being + deleted + + WARNING: use retire() instead + + WARNING: the properties of this node will not be available ever again + + WARNING: really, use retire() instead + + Well, I think that's enough warnings. This method exists mostly to + support the session storage of the cgi interface. + + The node is completely removed from the hyperdb, including all journal + entries. It will no longer be available, and will generally break code + if there are any references to the node. + """ def history(self, nodeid): """Retrieve the journal of edits on a particular node. @@ -593,9 +466,13 @@ class Class: 'date' is a Timestamp object specifying the time of the change and 'tag' is the journaltag specified when the database was opened. """ - return self.db.getjournal(self.classname, nodeid) + raise NotImplementedError # Locating nodes: + def hasnode(self, nodeid): + '''Determine if the given nodeid actually exists + ''' + raise NotImplementedError def setkey(self, propname): """Select a String property of this class to be the key property. @@ -604,38 +481,25 @@ class Class: None, or a TypeError is raised. The values of the key property on all existing nodes must be unique or a ValueError is raised. """ - # TODO: validate that the property is a String! - self.key = propname + raise NotImplementedError def getkey(self): """Return the name of the key property for this class or None.""" - return self.key + raise NotImplementedError def labelprop(self, default_to_id=0): - ''' Return the property name for a label for the given node. + """Return the property name for a label for the given node. This method attempts to generate a consistent label for the node. It tries the following in order: - 1. key property - 2. "name" property - 3. "title" property - 4. first property from the sorted property name list - ''' - k = self.getkey() - if k: - return k - props = self.getprops() - if props.has_key('name'): - return 'name' - elif props.has_key('title'): - return 'title' - if default_to_id: - return 'id' - props = props.keys() - props.sort() - return props[0] - - # TODO: set up a separate index db file for this? profile? + + 1. key property + 2. "name" property + 3. "title" property + 4. first property from the sorted property name list + """ + raise NotImplementedError + def lookup(self, keyvalue): """Locate a particular node by its key property and return its id. @@ -644,300 +508,43 @@ class Class: the nodes in this class, the matching node's id is returned; otherwise a KeyError is raised. """ - cldb = self.db.getclassdb(self.classname) - for nodeid in self.db.getnodeids(self.classname, cldb): - node = self.db.getnode(self.classname, nodeid, cldb) - if node.has_key(self.db.RETIRED_FLAG): - continue - if node[self.key] == keyvalue: - return nodeid - raise KeyError, keyvalue - - # XXX: change from spec - allows multiple props to match - def find(self, **propspec): - """Get the ids of nodes in this class which link to a given node. + raise NotImplementedError - 'propspec' consists of keyword args propname=nodeid - 'propname' must be the name of a property in this class, or a - KeyError is raised. That property must be a Link or Multilink - property, or a TypeError is raised. + def find(self, **propspec): + """Get the ids of nodes in this class which link to the given nodes. - 'nodeid' must be the id of an existing node in the class linked - to by the given property, or an IndexError is raised. - """ - propspec = propspec.items() - for propname, nodeid in propspec: - # check the prop is OK - prop = self.properties[propname] - if not isinstance(prop, Link) and not isinstance(prop, Multilink): - raise TypeError, "'%s' not a Link/Multilink property"%propname - if not self.db.hasnode(prop.classname, nodeid): - raise ValueError, '%s has no node %s'%(prop.classname, nodeid) - - # ok, now do the find - cldb = self.db.getclassdb(self.classname) - l = [] - for id in self.db.getnodeids(self.classname, cldb): - node = self.db.getnode(self.classname, id, cldb) - if node.has_key(self.db.RETIRED_FLAG): - continue - for propname, nodeid in propspec: - property = node[propname] - if isinstance(prop, Link) and nodeid == property: - l.append(id) - elif isinstance(prop, Multilink) and nodeid in property: - l.append(id) - return l + 'propspec' consists of keyword args propname={nodeid:1,} + 'propname' must be the name of a property in this class, or a + KeyError is raised. That property must be a Link or Multilink + property, or a TypeError is raised. - def stringFind(self, **requirements): - """Locate a particular node by matching a set of its String - properties in a caseless search. + Any node in this class whose 'propname' property links to any of the + nodeids will be returned. Used by the full text indexing, which knows + that "foo" occurs in msg1, msg3 and file7, so we have hits on these + issues: - If the property is not a String property, a TypeError is raised. - - The return is a list of the id of all nodes that match. + db.issue.find(messages={'1':1,'3':1}, files={'7':1}) """ - for propname in requirements.keys(): - prop = self.properties[propname] - if isinstance(not prop, String): - raise TypeError, "'%s' not a String property"%propname - requirements[propname] = requirements[propname].lower() - l = [] - cldb = self.db.getclassdb(self.classname) - for nodeid in self.db.getnodeids(self.classname, cldb): - node = self.db.getnode(self.classname, nodeid, cldb) - if node.has_key(self.db.RETIRED_FLAG): - continue - for key, value in requirements.items(): - if node[key] and node[key].lower() != value: - break - else: - l.append(nodeid) - return l + raise NotImplementedError - def list(self): - """Return a list of the ids of the active nodes in this class.""" - l = [] - cn = self.classname - cldb = self.db.getclassdb(cn) - for nodeid in self.db.getnodeids(cn, cldb): - node = self.db.getnode(cn, nodeid, cldb) - if node.has_key(self.db.RETIRED_FLAG): - continue - l.append(nodeid) - l.sort() - return l + def filter(self, search_matches, filterspec, sort=(None,None), + group=(None,None)): + """Return a list of the ids of the active nodes in this class that + match the 'filter' spec, sorted by the group spec and then the + sort spec. - # XXX not in spec - def filter(self, filterspec, sort, group, num_re = re.compile('^\d+$')): - ''' Return a list of the ids of the active nodes in this class that - match the 'filter' spec, sorted by the group spec and then the - sort spec - ''' - cn = self.classname + "filterspec" is {propname: value(s)} - # optimise filterspec - l = [] - props = self.getprops() - for k, v in filterspec.items(): - propclass = props[k] - if isinstance(propclass, Link): - if type(v) is not type([]): - v = [v] - # replace key values with node ids - u = [] - link_class = self.db.classes[propclass.classname] - for entry in v: - if entry == '-1': entry = None - elif not num_re.match(entry): - try: - entry = link_class.lookup(entry) - except (TypeError,KeyError): - raise ValueError, 'property "%s": %s not a %s'%( - k, entry, self.properties[k].classname) - u.append(entry) - - l.append((0, k, u)) - elif isinstance(propclass, Multilink): - if type(v) is not type([]): - v = [v] - # replace key values with node ids - u = [] - link_class = self.db.classes[propclass.classname] - for entry in v: - if not num_re.match(entry): - try: - entry = link_class.lookup(entry) - except (TypeError,KeyError): - raise ValueError, 'new property "%s": %s not a %s'%( - k, entry, self.properties[k].classname) - u.append(entry) - l.append((1, k, u)) - elif isinstance(propclass, String): - # simple glob searching - v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v) - v = v.replace('?', '.') - v = v.replace('*', '.*?') - l.append((2, k, re.compile(v, re.I))) - else: - l.append((6, k, v)) - filterspec = l + "sort" and "group" are (dir, prop) where dir is '+', '-' or None + and prop is a prop name or None - # now, find all the nodes that are active and pass filtering - l = [] - cldb = self.db.getclassdb(cn) - for nodeid in self.db.getnodeids(cn, cldb): - node = self.db.getnode(cn, nodeid, cldb) - if node.has_key(self.db.RETIRED_FLAG): - continue - # apply filter - for t, k, v in filterspec: - # this node doesn't have this property, so reject it - if not node.has_key(k): break - - if t == 0 and node[k] not in v: - # link - if this node'd property doesn't appear in the - # filterspec's nodeid list, skip it - break - elif t == 1: - # multilink - if any of the nodeids required by the - # filterspec aren't in this node's property, then skip - # it - for value in v: - if value not in node[k]: - break - else: - continue - break - elif t == 2 and not v.search(node[k]): - # RE search - break - elif t == 6 and node[k] != v: - # straight value comparison for the other types - break - else: - l.append((nodeid, node)) - l.sort() - - # optimise sort - m = [] - for entry in sort: - if entry[0] != '-': - m.append(('+', entry)) - else: - m.append((entry[0], entry[1:])) - sort = m - - # optimise group - m = [] - for entry in group: - if entry[0] != '-': - m.append(('+', entry)) - else: - m.append((entry[0], entry[1:])) - group = m - # now, sort the result - def sortfun(a, b, sort=sort, group=group, properties=self.getprops(), - db = self.db, cl=self): - a_id, an = a - b_id, bn = b - # sort by group and then sort - for list in group, sort: - for dir, prop in list: - # sorting is class-specific - propclass = properties[prop] - - # handle the properties that might be "faked" - # also, handle possible missing properties - try: - if not an.has_key(prop): - an[prop] = cl.get(a_id, prop) - av = an[prop] - except KeyError: - # the node doesn't have a value for this property - if isinstance(propclass, Multilink): av = [] - else: av = '' - try: - if not bn.has_key(prop): - bn[prop] = cl.get(b_id, prop) - bv = bn[prop] - except KeyError: - # the node doesn't have a value for this property - if isinstance(propclass, Multilink): bv = [] - else: bv = '' - - # String and Date values are sorted in the natural way - if isinstance(propclass, String): - # clean up the strings - if av and av[0] in string.uppercase: - av = an[prop] = av.lower() - if bv and bv[0] in string.uppercase: - bv = bn[prop] = bv.lower() - if (isinstance(propclass, String) or - isinstance(propclass, Date)): - # it might be a string that's really an integer - try: - av = int(av) - bv = int(bv) - except: - pass - if dir == '+': - r = cmp(av, bv) - if r != 0: return r - elif dir == '-': - r = cmp(bv, av) - if r != 0: return r - - # Link properties are sorted according to the value of - # the "order" property on the linked nodes if it is - # present; or otherwise on the key string of the linked - # nodes; or finally on the node ids. - elif isinstance(propclass, Link): - link = db.classes[propclass.classname] - if av is None and bv is not None: return -1 - if av is not None and bv is None: return 1 - if av is None and bv is None: continue - if link.getprops().has_key('order'): - if dir == '+': - r = cmp(link.get(av, 'order'), - link.get(bv, 'order')) - if r != 0: return r - elif dir == '-': - r = cmp(link.get(bv, 'order'), - link.get(av, 'order')) - if r != 0: return r - elif link.getkey(): - key = link.getkey() - if dir == '+': - r = cmp(link.get(av, key), link.get(bv, key)) - if r != 0: return r - elif dir == '-': - r = cmp(link.get(bv, key), link.get(av, key)) - if r != 0: return r - else: - if dir == '+': - r = cmp(av, bv) - if r != 0: return r - elif dir == '-': - r = cmp(bv, av) - if r != 0: return r - - # Multilink properties are sorted according to how many - # links are present. - elif isinstance(propclass, Multilink): - if dir == '+': - r = cmp(len(av), len(bv)) - if r != 0: return r - elif dir == '-': - r = cmp(len(bv), len(av)) - if r != 0: return r - # end for dir, prop in list: - # end for list in sort, group: - # if all else fails, compare the ids - return cmp(a[0], b[0]) - - l.sort(sortfun) - return [i[0] for i in l] + "search_matches" is {nodeid: marker} + + The filter must match all properties specificed - but if the + property value to match is a list, any one of the values in the + list may match for that property to match. + """ + raise NotImplementedError def count(self): """Get the number of nodes in this class. @@ -946,18 +553,15 @@ class Class: in this class run from 1 to numnodes, and numnodes+1 will be the id of the next node to be created in this class. """ - return self.db.countnodes(self.classname) + raise NotImplementedError # Manipulating properties: - def getprops(self, protected=1): """Return a dictionary mapping property names to property objects. If the "protected" flag is true, we include protected properties - - those which may not be modified.""" - d = self.properties.copy() - if protected: - d['id'] = String() - return d + those which may not be modified. + """ + raise NotImplementedError def addprop(self, **properties): """Add properties to this class. @@ -967,39 +571,224 @@ class Class: may collide with the names of existing properties, or a ValueError is raised before any properties have been added. """ - for key in properties.keys(): - if self.properties.has_key(key): - raise ValueError, key - self.properties.update(properties) + raise NotImplementedError + + def index(self, nodeid): + '''Add (or refresh) the node to search indexes + ''' + raise NotImplementedError + + def safeget(self, nodeid, propname, default=None): + """Safely get the value of a property on an existing node of this class. + + Return 'default' if the node doesn't exist. + """ + try: + return self.get(nodeid, propname) + except IndexError: + return default + +class HyperdbValueError(ValueError): + ''' Error converting a raw value into a Hyperdb value ''' + pass +def convertLinkValue(db, propname, prop, value, idre=re.compile('\d+')): + ''' Convert the link value (may be id or key value) to an id value. ''' + linkcl = db.classes[prop.classname] + if not idre.match(value): + if linkcl.getkey(): + try: + value = linkcl.lookup(value) + except KeyError, message: + raise HyperdbValueError, 'property %s: %r is not a %s.'%( + propname, value, prop.classname) + else: + raise HyperdbValueError, 'you may only enter ID values '\ + 'for property %s'%propname + return value + +def fixNewlines(text): + """ Homogenise line endings. + + Different web clients send different line ending values, but + other systems (eg. email) don't necessarily handle those line + endings. Our solution is to convert all line endings to LF. + """ + text = text.replace('\r\n', '\n') + return text.replace('\r', '\n') + +def rawToHyperdb(db, klass, itemid, propname, value, + pwre=re.compile(r'{(\w+)}(.+)')): + ''' Convert the raw (user-input) value to a hyperdb-storable value. The + value is for the "propname" property on itemid (may be None for a + new item) of "klass" in "db". + + The value is usually a string, but in the case of multilink inputs + it may be either a list of strings or a string with comma-separated + values. + ''' + properties = klass.getprops() + + # ensure it's a valid property name + propname = propname.strip() + try: + proptype = properties[propname] + except KeyError: + raise HyperdbValueError, '%r is not a property of %s'%(propname, + klass.classname) + + # if we got a string, strip it now + if isinstance(value, type('')): + value = value.strip() + + # convert the input value to a real property value + if isinstance(proptype, String): + # fix the CRLF/CR -> LF stuff + value = fixNewlines(value) + if isinstance(proptype, Password): + m = pwre.match(value) + if m: + # password is being given to us encrypted + p = password.Password() + p.scheme = m.group(1) + if p.scheme not in 'SHA crypt plaintext'.split(): + raise HyperdbValueError, 'property %s: unknown encryption '\ + 'scheme %r'%(propname, p.scheme) + p.password = m.group(2) + value = p + else: + try: + value = password.Password(value) + except password.PasswordValueError, message: + raise HyperdbValueError, 'property %s: %s'%(propname, message) + elif isinstance(proptype, Date): + try: + tz = db.getUserTimezone() + value = date.Date(value).local(tz) + except ValueError, message: + raise HyperdbValueError, 'property %s: %r is an invalid '\ + 'date (%s)'%(propname, value, message) + elif isinstance(proptype, Interval): + try: + value = date.Interval(value) + except ValueError, message: + raise HyperdbValueError, 'property %s: %r is an invalid '\ + 'date interval (%s)'%(propname, value, message) + elif isinstance(proptype, Link): + if value == '-1' or not value: + value = None + else: + value = convertLinkValue(db, propname, proptype, value) + + elif isinstance(proptype, Multilink): + # get the current item value if it's not a new item + if itemid and not itemid.startswith('-'): + curvalue = klass.get(itemid, propname) + else: + curvalue = [] + + # if the value is a comma-separated string then split it now + if isinstance(value, type('')): + value = value.split(',') + + # handle each add/remove in turn + # keep an extra list for all items that are + # definitely in the new list (in case of e.g. + # =A,+B, which should replace the old + # list with A,B) + set = 1 + newvalue = [] + for item in value: + item = item.strip() + + # skip blanks + if not item: continue + + # handle +/- + remove = 0 + if item.startswith('-'): + remove = 1 + item = item[1:] + set = 0 + elif item.startswith('+'): + item = item[1:] + set = 0 + + # look up the value + itemid = convertLinkValue(db, propname, proptype, item) + + # perform the add/remove + if remove: + try: + curvalue.remove(itemid) + except ValueError: + raise HyperdbValueError, 'property %s: %r is not ' \ + 'currently an element'%(propname, item) + else: + newvalue.append(itemid) + if itemid not in curvalue: + curvalue.append(itemid) + + # that's it, set the new Multilink property value, + # or overwrite it completely + if set: + value = newvalue + else: + value = curvalue + + # TODO: one day, we'll switch to numeric ids and this will be + # unnecessary :( + value = [int(x) for x in value] + value.sort() + value = [str(x) for x in value] + elif isinstance(proptype, Boolean): + value = value.strip() + value = value.lower() in ('yes', 'true', 'on', '1') + elif isinstance(proptype, Number): + value = value.strip() + try: + value = float(value) + except ValueError: + raise HyperdbValueError, 'property %s: %r is not a number'%( + propname, value) + return value + +class FileClass: + ''' A class that requires the "content" property and stores it on + disk. + ''' + pass -# XXX not in spec class Node: ''' A convenience wrapper for the given node ''' def __init__(self, cl, nodeid, cache=1): self.__dict__['cl'] = cl self.__dict__['nodeid'] = nodeid - self.__dict__['cache'] = cache def keys(self, protected=1): return self.cl.getprops(protected=protected).keys() def values(self, protected=1): l = [] for name in self.cl.getprops(protected=protected).keys(): - l.append(self.cl.get(self.nodeid, name, cache=self.cache)) + l.append(self.cl.get(self.nodeid, name)) return l def items(self, protected=1): l = [] for name in self.cl.getprops(protected=protected).keys(): - l.append((name, self.cl.get(self.nodeid, name, cache=self.cache))) + l.append((name, self.cl.get(self.nodeid, name))) return l def has_key(self, name): return self.cl.getprops().has_key(name) + def get(self, name, default=None): + if self.has_key(name): + return self[name] + else: + return default def __getattr__(self, name): if self.__dict__.has_key(name): return self.__dict__[name] try: - return self.cl.get(self.nodeid, name, cache=self.cache) + return self.cl.get(self.nodeid, name) except KeyError, value: # we trap this but re-raise it as AttributeError - all other # exceptions should pass through untrapped @@ -1007,7 +796,7 @@ class Node: # nope, no such attribute raise AttributeError, str(value) def __getitem__(self, name): - return self.cl.get(self.nodeid, name, cache=self.cache) + return self.cl.get(self.nodeid, name) def __setattr__(self, name, value): try: return self.cl.set(self.nodeid, **{name: value}) @@ -1021,235 +810,12 @@ class Node: return self.cl.retire(self.nodeid) -def Choice(name, *options): - cl = Class(db, name, name=hyperdb.String(), order=hyperdb.String()) +def Choice(name, db, *options): + '''Quick helper to create a simple class with choices + ''' + cl = Class(db, name, name=String(), order=String()) for i in range(len(options)): - cl.create(name=option[i], order=i) + cl.create(name=options[i], order=i) return hyperdb.Link(name) -# -# $Log: not supported by cvs2svn $ -# Revision 1.46 2002/01/07 10:42:23 richard -# oops -# -# Revision 1.45 2002/01/02 04:18:17 richard -# hyperdb docstrings -# -# Revision 1.44 2002/01/02 02:31:38 richard -# Sorry for the huge checkin message - I was only intending to implement #496356 -# but I found a number of places where things had been broken by transactions: -# . modified ROUNDUPDBSENDMAILDEBUG to be SENDMAILDEBUG and hold a filename -# for _all_ roundup-generated smtp messages to be sent to. -# . the transaction cache had broken the roundupdb.Class set() reactors -# . newly-created author users in the mailgw weren't being committed to the db -# -# Stuff that made it into CHANGES.txt (ie. the stuff I was actually working -# on when I found that stuff :): -# . #496356 ] Use threading in messages -# . detectors were being registered multiple times -# . added tests for mailgw -# . much better attaching of erroneous messages in the mail gateway -# -# Revision 1.43 2001/12/20 06:13:24 rochecompaan -# Bugs fixed: -# . Exception handling in hyperdb for strings-that-look-like numbers got -# lost somewhere -# . Internet Explorer submits full path for filename - we now strip away -# the path -# Features added: -# . Link and multilink properties are now displayed sorted in the cgi -# interface -# -# Revision 1.42 2001/12/16 10:53:37 richard -# take a copy of the node dict so that the subsequent set -# operation doesn't modify the oldvalues structure -# -# Revision 1.41 2001/12/15 23:47:47 richard -# Cleaned up some bare except statements -# -# Revision 1.40 2001/12/14 23:42:57 richard -# yuck, a gdbm instance tests false :( -# I've left the debugging code in - it should be removed one day if we're ever -# _really_ anal about performace :) -# -# Revision 1.39 2001/12/02 05:06:16 richard -# . We now use weakrefs in the Classes to keep the database reference, so -# the close() method on the database is no longer needed. -# I bumped the minimum python requirement up to 2.1 accordingly. -# . #487480 ] roundup-server -# . #487476 ] INSTALL.txt -# -# I also cleaned up the change message / post-edit stuff in the cgi client. -# There's now a clearly marked "TODO: append the change note" where I believe -# the change note should be added there. The "changes" list will obviously -# have to be modified to be a dict of the changes, or somesuch. -# -# More testing needed. -# -# Revision 1.38 2001/12/01 07:17:50 richard -# . We now have basic transaction support! Information is only written to -# the database when the commit() method is called. Only the anydbm -# backend is modified in this way - neither of the bsddb backends have been. -# The mail, admin and cgi interfaces all use commit (except the admin tool -# doesn't have a commit command, so interactive users can't commit...) -# . Fixed login/registration forwarding the user to the right page (or not, -# on a failure) -# -# Revision 1.37 2001/11/28 21:55:35 richard -# . login_action and newuser_action return values were being ignored -# . Woohoo! Found that bloody re-login bug that was killing the mail -# gateway. -# (also a minor cleanup in hyperdb) -# -# Revision 1.36 2001/11/27 03:16:09 richard -# Another place that wasn't handling missing properties. -# -# Revision 1.35 2001/11/22 15:46:42 jhermann -# Added module docstrings to all modules. -# -# Revision 1.34 2001/11/21 04:04:43 richard -# *sigh* more missing value handling -# -# Revision 1.33 2001/11/21 03:40:54 richard -# more new property handling -# -# Revision 1.32 2001/11/21 03:11:28 richard -# Better handling of new properties. -# -# Revision 1.31 2001/11/12 22:01:06 richard -# Fixed issues with nosy reaction and author copies. -# -# Revision 1.30 2001/11/09 10:11:08 richard -# . roundup-admin now handles all hyperdb exceptions -# -# Revision 1.29 2001/10/27 00:17:41 richard -# Made Class.stringFind() do caseless matching. -# -# Revision 1.28 2001/10/21 04:44:50 richard -# bug #473124: UI inconsistency with Link fields. -# This also prompted me to fix a fairly long-standing usability issue - -# that of being able to turn off certain filters. -# -# Revision 1.27 2001/10/20 23:44:27 richard -# Hyperdatabase sorts strings-that-look-like-numbers as numbers now. -# -# Revision 1.26 2001/10/16 03:48:01 richard -# admin tool now complains if a "find" is attempted with a non-link property. -# -# Revision 1.25 2001/10/11 00:17:51 richard -# Reverted a change in hyperdb so the default value for missing property -# values in a create() is None and not '' (the empty string.) This obviously -# breaks CSV import/export - the string 'None' will be created in an -# export/import operation. -# -# Revision 1.24 2001/10/10 03:54:57 richard -# Added database importing and exporting through CSV files. -# Uses the csv module from object-craft for exporting if it's available. -# Requires the csv module for importing. -# -# Revision 1.23 2001/10/09 23:58:10 richard -# Moved the data stringification up into the hyperdb.Class class' get, set -# and create methods. This means that the data is also stringified for the -# journal call, and removes duplication of code from the backends. The -# backend code now only sees strings. -# -# Revision 1.22 2001/10/09 07:25:59 richard -# Added the Password property type. See "pydoc roundup.password" for -# implementation details. Have updated some of the documentation too. -# -# Revision 1.21 2001/10/05 02:23:24 richard -# . roundup-admin create now prompts for property info if none is supplied -# on the command-line. -# . hyperdb Class getprops() method may now return only the mutable -# properties. -# . Login now uses cookies, which makes it a whole lot more flexible. We can -# now support anonymous user access (read-only, unless there's an -# "anonymous" user, in which case write access is permitted). Login -# handling has been moved into cgi_client.Client.main() -# . The "extended" schema is now the default in roundup init. -# . The schemas have had their page headings modified to cope with the new -# login handling. Existing installations should copy the interfaces.py -# file from the roundup lib directory to their instance home. -# . Incorrectly had a Bizar Software copyright on the cgitb.py module from -# Ping - has been removed. -# . Fixed a whole bunch of places in the CGI interface where we should have -# been returning Not Found instead of throwing an exception. -# . Fixed a deviation from the spec: trying to modify the 'id' property of -# an item now throws an exception. -# -# Revision 1.20 2001/10/04 02:12:42 richard -# Added nicer command-line item adding: passing no arguments will enter an -# interactive more which asks for each property in turn. While I was at it, I -# fixed an implementation problem WRT the spec - I wasn't raising a -# ValueError if the key property was missing from a create(). Also added a -# protected=boolean argument to getprops() so we can list only the mutable -# properties (defaults to yes, which lists the immutables). -# -# Revision 1.19 2001/08/29 04:47:18 richard -# Fixed CGI client change messages so they actually include the properties -# changed (again). -# -# Revision 1.18 2001/08/16 07:34:59 richard -# better CGI text searching - but hidden filter fields are disappearing... -# -# Revision 1.17 2001/08/16 06:59:58 richard -# all searches use re now - and they're all case insensitive -# -# Revision 1.16 2001/08/15 23:43:18 richard -# Fixed some isFooTypes that I missed. -# Refactored some code in the CGI code. -# -# Revision 1.15 2001/08/12 06:32:36 richard -# using isinstance(blah, Foo) now instead of isFooType -# -# Revision 1.14 2001/08/07 00:24:42 richard -# stupid typo -# -# Revision 1.13 2001/08/07 00:15:51 richard -# Added the copyright/license notice to (nearly) all files at request of -# Bizar Software. -# -# Revision 1.12 2001/08/02 06:38:17 richard -# Roundupdb now appends "mailing list" information to its messages which -# include the e-mail address and web interface address. Templates may -# override this in their db classes to include specific information (support -# instructions, etc). -# -# Revision 1.11 2001/08/01 04:24:21 richard -# mailgw was assuming certain properties existed on the issues being created. -# -# Revision 1.10 2001/07/30 02:38:31 richard -# get() now has a default arg - for migration only. -# -# Revision 1.9 2001/07/29 09:28:23 richard -# Fixed sorting by clicking on column headings. -# -# Revision 1.8 2001/07/29 08:27:40 richard -# Fixed handling of passed-in values in form elements (ie. during a -# drill-down) -# -# Revision 1.7 2001/07/29 07:01:39 richard -# Added vim command to all source so that we don't get no steenkin' tabs :) -# -# Revision 1.6 2001/07/29 05:36:14 richard -# Cleanup of the link label generation. -# -# Revision 1.5 2001/07/29 04:05:37 richard -# Added the fabricated property "id". -# -# Revision 1.4 2001/07/27 06:25:35 richard -# Fixed some of the exceptions so they're the right type. -# Removed the str()-ification of node ids so we don't mask oopsy errors any -# more. -# -# Revision 1.3 2001/07/27 05:17:14 richard -# just some comments -# -# Revision 1.2 2001/07/22 12:09:32 richard -# Final commit of Grande Splite -# -# Revision 1.1 2001/07/22 11:58:35 richard -# More Grande Splite -# -# # vim: set filetype=python ts=4 sw=4 et si