![[tokkee]](http://tokkee.org/images/avatar.png){kind=avatar}
diff --git a/roundup/hyperdb.py b/roundup/hyperdb.py
index 391c6f3c1b80a411dd6d4ebc12a8974c3ed07ad4..0ed036060d468d1a4fbd1336fb6db4c942cae431 100644 (file)
--- a/roundup/hyperdb.py
+++ b/roundup/hyperdb.py
# FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
# BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
# SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
-#
-# $Id: hyperdb.py,v 1.27 2001-10-20 23:44:27 richard Exp $
+#
+
+"""Hyperdatabase implementation, especially field types.
+"""
+__docformat__ = 'restructuredtext'
# standard python modules
-import cPickle, re, string
+import os, re, shutil, weakref
+# Python 2.3 ... 2.6 compatibility:
+from roundup.anypy.sets_ import set
# roundup modules
import date, password
-
+from support import ensureParentsExist, PrioList, sorted, reversed
+from roundup.i18n import _
#
# Types
#
-class String:
- """An object designating a String property."""
+class _Type(object):
+ """A roundup property type."""
+ def __init__(self, required=False, default_value = None):
+ self.required = required
+ self.__default_value = default_value
def __repr__(self):
- return '<%s>'%self.__class__
+ ' more useful for dumps '
+ return '<%s.%s>'%(self.__class__.__module__, self.__class__.__name__)
+ def get_default_value(self):
+ """The default value when creating a new instance of this property."""
+ return self.__default_value
+ def sort_repr (self, cls, val, name):
+ """Representation used for sorting. This should be a python
+ built-in type, otherwise sorting will take ages. Note that
+ individual backends may chose to use something different for
+ sorting as long as the outcome is the same.
+ """
+ return val
-class Password:
+class String(_Type):
+ """An object designating a String property."""
+ def __init__(self, indexme='no', required=False, default_value = ""):
+ super(String, self).__init__(required, default_value)
+ self.indexme = indexme == 'yes'
+ def from_raw(self, value, propname='', **kw):
+ """fix the CRLF/CR -> LF stuff"""
+ if propname == 'content':
+ # Why oh why wasn't the FileClass content property a File
+ # type from the beginning?
+ return value
+ return fixNewlines(value)
+ def sort_repr (self, cls, val, name):
+ if not val:
+ return val
+ if name == 'id':
+ return int(val)
+ return val.lower()
+
+class Password(_Type):
"""An object designating a Password property."""
- def __repr__(self):
- return '<%s>'%self.__class__
+ def from_raw(self, value, **kw):
+ if not value:
+ return None
+ try:
+ return password.Password(encrypted=value, strict=True)
+ except password.PasswordValueError, message:
+ raise HyperdbValueError, \
+ _('property %s: %s')%(kw['propname'], message)
-class Date:
- """An object designating a Date property."""
- def __repr__(self):
- return '<%s>'%self.__class__
+ def sort_repr (self, cls, val, name):
+ if not val:
+ return val
+ return str(val)
-class Interval:
+class Date(_Type):
+ """An object designating a Date property."""
+ def __init__(self, offset=None, required=False, default_value = None):
+ super(Date, self).__init__(required = required,
+ default_value = default_value)
+ self._offset = offset
+ def offset(self, db):
+ if self._offset is not None:
+ return self._offset
+ return db.getUserTimezone()
+ def from_raw(self, value, db, **kw):
+ try:
+ value = date.Date(value, self.offset(db))
+ except ValueError, message:
+ raise HyperdbValueError, _('property %s: %r is an invalid '\
+ 'date (%s)')%(kw['propname'], value, message)
+ return value
+ def range_from_raw(self, value, db):
+ """return Range value from given raw value with offset correction"""
+ return date.Range(value, date.Date, offset=self.offset(db))
+ def sort_repr (self, cls, val, name):
+ if not val:
+ return val
+ return str(val)
+
+class Interval(_Type):
"""An object designating an Interval property."""
+ def from_raw(self, value, **kw):
+ try:
+ value = date.Interval(value)
+ except ValueError, message:
+ raise HyperdbValueError, _('property %s: %r is an invalid '\
+ 'date interval (%s)')%(kw['propname'], value, message)
+ return value
+ def sort_repr (self, cls, val, name):
+ if not val:
+ return val
+ return val.as_seconds()
+
+class _Pointer(_Type):
+ """An object designating a Pointer property that links or multilinks
+ to a node in a specified class."""
+ def __init__(self, classname, do_journal='yes', required=False,
+ default_value = None):
+ """ Default is to journal link and unlink events
+ """
+ super(_Pointer, self).__init__(required, default_value)
+ self.classname = classname
+ self.do_journal = do_journal == 'yes'
def __repr__(self):
- return '<%s>'%self.__class__
+ """more useful for dumps. But beware: This is also used in schema
+ storage in SQL backends!
+ """
+ return '<%s.%s to "%s">'%(self.__class__.__module__,
+ self.__class__.__name__, self.classname)
-class Link:
+class Link(_Pointer):
"""An object designating a Link property that links to a
node in a specified class."""
- def __init__(self, classname):
- self.classname = classname
- def __repr__(self):
- return '<%s to "%s">'%(self.__class__, self.classname)
-
-class Multilink:
+ def from_raw(self, value, db, propname, **kw):
+ if value == '-1' or not value:
+ value = None
+ else:
+ value = convertLinkValue(db, propname, self, value)
+ return value
+ def sort_repr (self, cls, val, name):
+ if not val:
+ return val
+ op = cls.labelprop()
+ if op == 'id':
+ return int(cls.get(val, op))
+ return cls.get(val, op)
+
+class Multilink(_Pointer):
"""An object designating a Multilink property that links
to nodes in a specified class.
+
+ "classname" indicates the class to link to
+
+ "do_journal" indicates whether the linked-to nodes should have
+ 'link' and 'unlink' events placed in their journal
"""
- def __init__(self, classname):
- self.classname = classname
- def __repr__(self):
- return '<%s to "%s">'%(self.__class__, self.classname)
-class DatabaseError(ValueError):
+ def __init__(self, classname, do_journal = 'yes', required = False):
+
+ super(Multilink, self).__init__(classname,
+ do_journal,
+ required = required,
+ default_value = [])
+
+ def from_raw(self, value, db, klass, propname, itemid, **kw):
+ if not value:
+ return []
+
+ # get the current item value if it's not a new item
+ if itemid and not itemid.startswith('-'):
+ curvalue = klass.get(itemid, propname)
+ else:
+ curvalue = []
+
+ # if the value is a comma-separated string then split it now
+ if isinstance(value, type('')):
+ value = value.split(',')
+
+ # handle each add/remove in turn
+ # keep an extra list for all items that are
+ # definitely in the new list (in case of e.g.
+ # <propname>=A,+B, which should replace the old
+ # list with A,B)
+ do_set = 1
+ newvalue = []
+ for item in value:
+ item = item.strip()
+
+ # skip blanks
+ if not item: continue
+
+ # handle +/-
+ remove = 0
+ if item.startswith('-'):
+ remove = 1
+ item = item[1:]
+ do_set = 0
+ elif item.startswith('+'):
+ item = item[1:]
+ do_set = 0
+
+ # look up the value
+ itemid = convertLinkValue(db, propname, self, item)
+
+ # perform the add/remove
+ if remove:
+ try:
+ curvalue.remove(itemid)
+ except ValueError:
+ raise HyperdbValueError, _('property %s: %r is not ' \
+ 'currently an element')%(propname, item)
+ else:
+ newvalue.append(itemid)
+ if itemid not in curvalue:
+ curvalue.append(itemid)
+
+ # that's it, set the new Multilink property value,
+ # or overwrite it completely
+ if do_set:
+ value = newvalue
+ else:
+ value = curvalue
+
+ # TODO: one day, we'll switch to numeric ids and this will be
+ # unnecessary :(
+ value = [int(x) for x in value]
+ value.sort()
+ value = [str(x) for x in value]
+ return value
+
+ def sort_repr (self, cls, val, name):
+ if not val:
+ return val
+ op = cls.labelprop()
+ if op == 'id':
+ return [int(cls.get(v, op)) for v in val]
+ return [cls.get(v, op) for v in val]
+
+class Boolean(_Type):
+ """An object designating a boolean property"""
+ def from_raw(self, value, **kw):
+ value = value.strip()
+ # checked is a common HTML checkbox value
+ value = value.lower() in ('checked', 'yes', 'true', 'on', '1')
+ return value
+
+class Number(_Type):
+ """An object designating a numeric property"""
+ def from_raw(self, value, **kw):
+ value = value.strip()
+ try:
+ value = float(value)
+ except ValueError:
+ raise HyperdbValueError, _('property %s: %r is not a number')%(
+ kw['propname'], value)
+ return value
+#
+# Support for splitting designators
+#
+class DesignatorError(ValueError):
pass
+def splitDesignator(designator, dre=re.compile(r'([^\d]+)(\d+)')):
+ """ Take a foo123 and return ('foo', 123)
+ """
+ m = dre.match(designator)
+ if m is None:
+ raise DesignatorError, _('"%s" not a node designator')%designator
+ return m.group(1), m.group(2)
+
+class Proptree(object):
+ """ Simple tree data structure for optimizing searching of
+ properties. Each node in the tree represents a roundup Class
+ Property that has to be navigated for finding the given search
+ or sort properties. The need_for attribute is used for
+ distinguishing nodes in the tree used for sorting, searching or
+ retrieval: The attribute is a dictionary containing one or several
+ of the values 'sort', 'search', 'retrieve'.
+
+ The Proptree is also used for transitively searching attributes for
+ backends that do not support transitive search (e.g. anydbm). The
+ _val attribute with set_val is used for this.
+ """
+
+ def __init__(self, db, cls, name, props, parent=None, retr=False):
+ self.db = db
+ self.name = name
+ self.props = props
+ self.parent = parent
+ self._val = None
+ self.has_values = False
+ self.cls = cls
+ self.classname = None
+ self.uniqname = None
+ self.children = []
+ self.sortattr = []
+ self.propdict = {}
+ self.need_for = {'search' : True}
+ self.sort_direction = None
+ self.sort_ids = None
+ self.sort_ids_needed = False
+ self.sort_result = None
+ self.attr_sort_done = False
+ self.tree_sort_done = False
+ self.propclass = None
+ self.orderby = []
+ self.sql_idx = None # index of retrieved column in sql result
+ if parent:
+ self.root = parent.root
+ self.depth = parent.depth + 1
+ else:
+ self.root = self
+ self.seqno = 1
+ self.depth = 0
+ self.need_for['sort'] = True
+ self.id = self.root.seqno
+ self.root.seqno += 1
+ if self.cls:
+ self.classname = self.cls.classname
+ self.uniqname = '%s%s' % (self.cls.classname, self.id)
+ if not self.parent:
+ self.uniqname = self.cls.classname
+ if retr:
+ self.append_retr_props()
+
+ def append(self, name, need_for='search', retr=False):
+ """Append a property to self.children. Will create a new
+ propclass for the child.
+ """
+ if name in self.propdict:
+ pt = self.propdict[name]
+ pt.need_for[need_for] = True
+ if retr and isinstance(pt.propclass, Link):
+ pt.append_retr_props()
+ return pt
+ propclass = self.props[name]
+ cls = None
+ props = None
+ if isinstance(propclass, (Link, Multilink)):
+ cls = self.db.getclass(propclass.classname)
+ props = cls.getprops()
+ child = self.__class__(self.db, cls, name, props, parent = self)
+ child.need_for = {need_for : True}
+ child.propclass = propclass
+ self.children.append(child)
+ self.propdict[name] = child
+ if retr and isinstance(child.propclass, Link):
+ child.append_retr_props()
+ return child
+
+ def append_retr_props(self):
+ """Append properties for retrieval."""
+ for name, prop in self.cls.getprops(protected=1).iteritems():
+ if isinstance(prop, Multilink):
+ continue
+ self.append(name, need_for='retrieve')
+
+ def compute_sort_done(self, mlseen=False):
+ """ Recursively check if attribute is needed for sorting
+ ('sort' in self.need_for) or all children have tree_sort_done set and
+ sort_ids_needed unset: set self.tree_sort_done if one of the conditions
+ holds. Also remove sort_ids_needed recursively once having seen a
+ Multilink.
+ """
+ if isinstance (self.propclass, Multilink):
+ mlseen = True
+ if mlseen:
+ self.sort_ids_needed = False
+ self.tree_sort_done = True
+ for p in self.children:
+ p.compute_sort_done(mlseen)
+ if not p.tree_sort_done:
+ self.tree_sort_done = False
+ if 'sort' not in self.need_for:
+ self.tree_sort_done = True
+ if mlseen:
+ self.tree_sort_done = False
+
+ def ancestors(self):
+ p = self
+ while p.parent:
+ yield p
+ p = p.parent
+
+ def search(self, search_matches=None, sort=True):
+ """ Recursively search for the given properties in a proptree.
+ Once all properties are non-transitive, the search generates a
+ simple _filter call which does the real work
+ """
+ filterspec = {}
+ for p in self.children:
+ if 'search' in p.need_for:
+ if p.children:
+ p.search(sort = False)
+ filterspec[p.name] = p.val
+ self.val = self.cls._filter(search_matches, filterspec, sort and self)
+ return self.val
+
+ def sort (self, ids=None):
+ """ Sort ids by the order information stored in self. With
+ optimisations: Some order attributes may be precomputed (by the
+ backend) and some properties may already be sorted.
+ """
+ if ids is None:
+ ids = self.val
+ if self.sortattr and [s for s in self.sortattr if not s.attr_sort_done]:
+ return self._searchsort(ids, True, True)
+ return ids
+
+ def sortable_children(self, intermediate=False):
+ """ All children needed for sorting. If intermediate is True,
+ intermediate nodes (not being a sort attribute) are returned,
+ too.
+ """
+ return [p for p in self.children
+ if 'sort' in p.need_for and (intermediate or p.sort_direction)]
+
+ def __iter__(self):
+ """ Yield nodes in depth-first order -- visited nodes first """
+ for p in self.children:
+ yield p
+ for c in p:
+ yield c
+
+ def _get (self, ids):
+ """Lookup given ids -- possibly a list of list. We recurse until
+ we have a list of ids.
+ """
+ if not ids:
+ return ids
+ if isinstance (ids[0], list):
+ cids = [self._get(i) for i in ids]
+ else:
+ cids = [i and self.parent.cls.get(i, self.name) for i in ids]
+ if self.sortattr:
+ cids = [self._searchsort(i, False, True) for i in cids]
+ return cids
+
+ def _searchsort(self, ids=None, update=True, dosort=True):
+ """ Recursively compute the sort attributes. Note that ids
+ may be a deeply nested list of lists of ids if several
+ multilinks are encountered on the way from the root to an
+ individual attribute. We make sure that everything is properly
+ sorted on the way up. Note that the individual backend may
+ already have precomputed self.result or self.sort_ids. In this
+ case we do nothing for existing sa.result and recurse further if
+ self.sort_ids is available.
+
+ Yech, Multilinks: This gets especially complicated if somebody
+ sorts by different attributes of the same multilink (or
+ transitively across several multilinks). My use-case is sorting
+ by issue.messages.author and (reverse) by issue.messages.date.
+ In this case we sort the messages by author and date and use
+ this sorted list twice for sorting issues. This means that
+ issues are sorted by author and then by the time of the messages
+ *of this author*. Probably what the user intends in that case,
+ so we do *not* use two sorted lists of messages, one sorted by
+ author and one sorted by date for sorting issues.
+ """
+ for pt in self.sortable_children(intermediate = True):
+ # ids can be an empty list
+ if pt.tree_sort_done or not ids:
+ continue
+ if pt.sort_ids: # cached or computed by backend
+ cids = pt.sort_ids
+ else:
+ cids = pt._get(ids)
+ if pt.sort_direction and not pt.sort_result:
+ sortrep = pt.propclass.sort_repr
+ pt.sort_result = pt._sort_repr(sortrep, cids)
+ pt.sort_ids = cids
+ if pt.children:
+ pt._searchsort(cids, update, False)
+ if self.sortattr and dosort:
+ ids = self._sort(ids)
+ if not update:
+ for pt in self.sortable_children(intermediate = True):
+ pt.sort_ids = None
+ for pt in self.sortattr:
+ pt.sort_result = None
+ return ids
+
+ def _set_val(self, val):
+ """Check if self._val is already defined. If yes, we compute the
+ intersection of the old and the new value(s)
+ """
+ if self.has_values:
+ v = self._val
+ if not isinstance(self._val, type([])):
+ v = [self._val]
+ vals = set(v)
+ vals.intersection_update(val)
+ self._val = [v for v in vals]
+ else:
+ self._val = val
+ self.has_values = True
+
+ val = property(lambda self: self._val, _set_val)
+
+ def _sort(self, val):
+ """Finally sort by the given sortattr.sort_result. Note that we
+ do not sort by attrs having attr_sort_done set. The caller is
+ responsible for setting attr_sort_done only for trailing
+ attributes (otherwise the sort order is wrong). Since pythons
+ sort is stable, we can sort already sorted lists without
+ destroying the sort-order for items that compare equal with the
+ current sort.
+
+ Sorting-Strategy: We sort repeatedly by different sort-keys from
+ right to left. Since pythons sort is stable, we can safely do
+ that. An optimisation is a "run-length encoding" of the
+ sort-directions: If several sort attributes sort in the same
+ direction we can combine them into a single sort. Note that
+ repeated sorting is probably more efficient than using
+ compare-methods in python due to the overhead added by compare
+ methods.
+ """
+ if not val:
+ return val
+ sortattr = []
+ directions = []
+ dir_idx = []
+ idx = 0
+ curdir = None
+ for sa in self.sortattr:
+ if sa.attr_sort_done:
+ break
+ if sortattr:
+ assert len(sortattr[0]) == len(sa.sort_result)
+ sortattr.append (sa.sort_result)
+ if curdir != sa.sort_direction:
+ dir_idx.append (idx)
+ directions.append (sa.sort_direction)
+ curdir = sa.sort_direction
+ idx += 1
+ sortattr.append (val)
+ sortattr = zip (*sortattr)
+ for dir, i in reversed(zip(directions, dir_idx)):
+ rev = dir == '-'
+ sortattr = sorted (sortattr, key = lambda x:x[i:idx], reverse = rev)
+ idx = i
+ return [x[-1] for x in sortattr]
+
+ def _sort_repr(self, sortrep, ids):
+ """Call sortrep for given ids -- possibly a list of list. We
+ recurse until we have a list of ids.
+ """
+ if not ids:
+ return ids
+ if isinstance (ids[0], list):
+ res = [self._sort_repr(sortrep, i) for i in ids]
+ else:
+ res = [sortrep(self.cls, i, self.name) for i in ids]
+ return res
+ def __repr__(self):
+ r = ["proptree:" + self.name]
+ for n in self:
+ r.append("proptree:" + " " * n.depth + n.name)
+ return '\n'.join(r)
+ __str__ = __repr__
#
# the base Database class
#
+class DatabaseError(ValueError):
+ """Error to be raised when there is some problem in the database code
+ """
+ pass
class Database:
+ """A database for storing records containing flexible data types.
+
+This class defines a hyperdatabase storage layer, which the Classes use to
+store their data.
+
+
+Transactions
+------------
+The Database should support transactions through the commit() and
+rollback() methods. All other Database methods should be transaction-aware,
+using data from the current transaction before looking up the database.
+
+An implementation must provide an override for the get() method so that the
+in-database value is returned in preference to the in-transaction value.
+This is necessary to determine if any values have changed during a
+transaction.
+
+
+Implementation
+--------------
+
+All methods except __repr__ must be implemented by a concrete backend Database.
+
+"""
+
# flag to set on retired entries
RETIRED_FLAG = '__hyperdb_retired'
+ BACKEND_MISSING_STRING = None
+ BACKEND_MISSING_NUMBER = None
+ BACKEND_MISSING_BOOLEAN = None
+
+ def __init__(self, config, journaltag=None):
+ """Open a hyperdatabase given a specifier to some storage.
+
+ The 'storagelocator' is obtained from config.DATABASE.
+ The meaning of 'storagelocator' depends on the particular
+ implementation of the hyperdatabase. It could be a file name,
+ a directory path, a socket descriptor for a connection to a
+ database over the network, etc.
+
+ The 'journaltag' is a token that will be attached to the journal
+ entries for any edits done on the database. If 'journaltag' is
+ None, the database is opened in read-only mode: the Class.create(),
+ Class.set(), and Class.retire() methods are disabled.
+ """
+ raise NotImplementedError
+
+ def post_init(self):
+ """Called once the schema initialisation has finished.
+ If 'refresh' is true, we want to rebuild the backend
+ structures.
+ """
+ raise NotImplementedError
+
+ def refresh_database(self):
+ """Called to indicate that the backend should rebuild all tables
+ and structures. Not called in normal usage."""
+ raise NotImplementedError
+
+ def __getattr__(self, classname):
+ """A convenient way of calling self.getclass(classname)."""
+ raise NotImplementedError
+
+ def addclass(self, cl):
+ """Add a Class to the hyperdatabase.
+ """
+ raise NotImplementedError
+
+ def getclasses(self):
+ """Return a list of the names of all existing classes."""
+ raise NotImplementedError
+
+ def getclass(self, classname):
+ """Get the Class object representing a particular class.
+
+ If 'classname' is not a valid class name, a KeyError is raised.
+ """
+ raise NotImplementedError
+
+ def clear(self):
+ """Delete all database contents.
+ """
+ raise NotImplementedError
+
+ def getclassdb(self, classname, mode='r'):
+ """Obtain a connection to the class db that will be used for
+ multiple actions.
+ """
+ raise NotImplementedError
+
+ def addnode(self, classname, nodeid, node):
+ """Add the specified node to its class's db.
+ """
+ raise NotImplementedError
+
+ def serialise(self, classname, node):
+ """Copy the node contents, converting non-marshallable data into
+ marshallable data.
+ """
+ return node
+
+ def setnode(self, classname, nodeid, node):
+ """Change the specified node.
+ """
+ raise NotImplementedError
+
+ def unserialise(self, classname, node):
+ """Decode the marshalled node data
+ """
+ return node
+
+ def getnode(self, classname, nodeid):
+ """Get a node from the database.
+
+ 'cache' exists for backwards compatibility, and is not used.
+ """
+ raise NotImplementedError
+
+ def hasnode(self, classname, nodeid):
+ """Determine if the database has a given node.
+ """
+ raise NotImplementedError
+
+ def countnodes(self, classname):
+ """Count the number of nodes that exist for a particular Class.
+ """
+ raise NotImplementedError
+
+ def storefile(self, classname, nodeid, property, content):
+ """Store the content of the file in the database.
+
+ The property may be None, in which case the filename does not
+ indicate which property is being saved.
+ """
+ raise NotImplementedError
+
+ def getfile(self, classname, nodeid, property):
+ """Get the content of the file in the database.
+ """
+ raise NotImplementedError
+
+ def addjournal(self, classname, nodeid, action, params):
+ """ Journal the Action
+ 'action' may be:
+
+ 'create' or 'set' -- 'params' is a dictionary of property values
+ 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
+ 'retire' -- 'params' is None
+ """
+ raise NotImplementedError
+
+ def getjournal(self, classname, nodeid):
+ """ get the journal for id
+ """
+ raise NotImplementedError
+
+ def pack(self, pack_before):
+ """ pack the database
+ """
+ raise NotImplementedError
+
+ def commit(self):
+ """ Commit the current transactions.
+
+ Save all data changed since the database was opened or since the
+ last commit() or rollback().
+
+ fail_ok indicates that the commit is allowed to fail. This is used
+ in the web interface when committing cleaning of the session
+ database. We don't care if there's a concurrency issue there.
+
+ The only backend this seems to affect is postgres.
+ """
+ raise NotImplementedError
+
+ def rollback(self):
+ """ Reverse all actions from the current transaction.
+
+ Undo all the changes made since the database was opened or the last
+ commit() or rollback() was performed.
+ """
+ raise NotImplementedError
+
+ def close(self):
+ """Close the database.
+
+ This method must be called at the end of processing.
+
+ """
+
+def iter_roles(roles):
+ ''' handle the text processing of turning the roles list
+ into something python can use more easily
+ '''
+ if not roles or not roles.strip():
+ raise StopIteration, "Empty roles given"
+ for role in [x.lower().strip() for x in roles.split(',')]:
+ yield role
+
-_marker = []
#
# The base Class class
#
class Class:
- """The handle to a particular class of nodes in a hyperdatabase."""
+ """ The handle to a particular class of nodes in a hyperdatabase.
+
+ All methods except __repr__ and getnode must be implemented by a
+ concrete backend Class.
+ """
def __init__(self, db, classname, **properties):
"""Create a new class with a given name and property specification.
or a ValueError is raised. The keyword arguments in 'properties'
must map names to property objects, or a TypeError is raised.
"""
+ for name in 'creation activity creator actor'.split():
+ if properties.has_key(name):
+ raise ValueError, '"creation", "activity", "creator" and '\
+ '"actor" are reserved'
+
self.classname = classname
self.properties = properties
- self.db = db
+ self.db = weakref.proxy(db) # use a weak ref to avoid circularity
self.key = ''
+ # should we journal changes (default yes)
+ self.do_journal = 1
+
# do the db-related init stuff
db.addclass(self)
+ actions = "create set retire restore".split()
+ self.auditors = dict([(a, PrioList()) for a in actions])
+ self.reactors = dict([(a, PrioList()) for a in actions])
+
+ def __repr__(self):
+ """Slightly more useful representation
+ """
+ return '<hyperdb.Class "%s">'%self.classname
+
# Editing nodes:
def create(self, **propvalues):
The values of arguments must be acceptable for the types of their
corresponding properties or a TypeError is raised.
-
+
If this class has a key property, it must be present and its value
must not collide with other key strings or a ValueError is raised.
-
+
Any other properties on this class that are missing from the
'propvalues' dictionary are set to None.
-
+
If an id in a link or multilink property does not refer to a valid
node, an IndexError is raised.
"""
- if propvalues.has_key('id'):
- raise KeyError, '"id" is reserved'
-
- if self.db.journaltag is None:
- raise DatabaseError, 'Database open read-only'
+ raise NotImplementedError
- # new node's id
- newid = str(self.count() + 1)
-
- # validate propvalues
- num_re = re.compile('^\d+$')
- for key, value in propvalues.items():
- if key == self.key:
- try:
- self.lookup(value)
- except KeyError:
- pass
- else:
- raise ValueError, 'node with key "%s" exists'%value
-
- # try to handle this property
- try:
- prop = self.properties[key]
- except KeyError:
- raise KeyError, '"%s" has no property "%s"'%(self.classname,
- key)
-
- if isinstance(prop, Link):
- if type(value) != type(''):
- raise ValueError, 'link value must be String'
- link_class = self.properties[key].classname
- # if it isn't a number, it's a key
- if not num_re.match(value):
- try:
- value = self.db.classes[link_class].lookup(value)
- except:
- raise IndexError, 'new property "%s": %s not a %s'%(
- key, value, self.properties[key].classname)
- propvalues[key] = value
- if not self.db.hasnode(link_class, value):
- raise IndexError, '%s has no node %s'%(link_class, value)
-
- # register the link with the newly linked node
- self.db.addjournal(link_class, value, 'link',
- (self.classname, newid, key))
-
- elif isinstance(prop, Multilink):
- if type(value) != type([]):
- raise TypeError, 'new property "%s" not a list of ids'%key
- link_class = self.properties[key].classname
- l = []
- for entry in value:
- if type(entry) != type(''):
- raise ValueError, 'link value must be String'
- # if it isn't a number, it's a key
- if not num_re.match(entry):
- try:
- entry = self.db.classes[link_class].lookup(entry)
- except:
- raise IndexError, 'new property "%s": %s not a %s'%(
- key, entry, self.properties[key].classname)
- l.append(entry)
- value = l
- propvalues[key] = value
-
- # handle additions
- for id in value:
- if not self.db.hasnode(link_class, id):
- raise IndexError, '%s has no node %s'%(link_class, id)
- # register the link with the newly linked node
- self.db.addjournal(link_class, id, 'link',
- (self.classname, newid, key))
-
- elif isinstance(prop, String):
- if type(value) != type(''):
- raise TypeError, 'new property "%s" not a string'%key
-
- elif isinstance(prop, Password):
- if not isinstance(value, password.Password):
- raise TypeError, 'new property "%s" not a Password'%key
-
- elif isinstance(prop, Date):
- if not isinstance(value, date.Date):
- raise TypeError, 'new property "%s" not a Date'%key
-
- elif isinstance(prop, Interval):
- if not isinstance(value, date.Interval):
- raise TypeError, 'new property "%s" not an Interval'%key
-
- # make sure there's data where there needs to be
- for key, prop in self.properties.items():
- if propvalues.has_key(key):
- continue
- if key == self.key:
- raise ValueError, 'key property "%s" is required'%key
- if isinstance(prop, Multilink):
- propvalues[key] = []
- else:
- propvalues[key] = None
-
- # convert all data to strings
- for key, prop in self.properties.items():
- if isinstance(prop, Date):
- propvalues[key] = propvalues[key].get_tuple()
- elif isinstance(prop, Interval):
- propvalues[key] = propvalues[key].get_tuple()
- elif isinstance(prop, Password):
- propvalues[key] = str(propvalues[key])
-
- # done
- self.db.addnode(self.classname, newid, propvalues)
- self.db.addjournal(self.classname, newid, 'create', propvalues)
- return newid
-
- def get(self, nodeid, propname, default=_marker):
+ _marker = []
+ def get(self, nodeid, propname, default=_marker, cache=1):
"""Get the value of a property on an existing node of this class.
'nodeid' must be the id of an existing node of this class or an
IndexError is raised. 'propname' must be the name of a property
of this class or a KeyError is raised.
+
+ 'cache' exists for backwards compatibility, and is not used.
"""
- d = self.db.getnode(self.classname, nodeid)
-
- # convert the marshalled data to instances
- for key, prop in self.properties.items():
- if isinstance(prop, Date):
- d[key] = date.Date(d[key])
- elif isinstance(prop, Interval):
- d[key] = date.Interval(d[key])
- elif isinstance(prop, Password):
- p = password.Password()
- p.unpack(d[key])
- d[key] = p
-
- if propname == 'id':
- return nodeid
- if not d.has_key(propname) and default is not _marker:
- return default
- return d[propname]
+ raise NotImplementedError
- # XXX not in spec
+ # not in spec
def getnode(self, nodeid):
- ''' Return a convenience wrapper for the node
- '''
+ """ Return a convenience wrapper for the node.
+
+ 'nodeid' must be the id of an existing node of this class or an
+ IndexError is raised.
+
+ 'cache' exists for backwards compatibility, and is not used.
+ """
return Node(self, nodeid)
+ def getnodeids(self, retired=None):
+ """Retrieve all the ids of the nodes for a particular Class.
+ """
+ raise NotImplementedError
+
def set(self, nodeid, **propvalues):
"""Modify a property on an existing node of this class.
-
+
'nodeid' must be the id of an existing node of this class or an
IndexError is raised.
If the value of a Link or Multilink property contains an invalid
node id, a ValueError is raised.
"""
- if not propvalues:
- return
-
- if propvalues.has_key('id'):
- raise KeyError, '"id" is reserved'
-
- if self.db.journaltag is None:
- raise DatabaseError, 'Database open read-only'
-
- node = self.db.getnode(self.classname, nodeid)
- if node.has_key(self.db.RETIRED_FLAG):
- raise IndexError
- num_re = re.compile('^\d+$')
- for key, value in propvalues.items():
- if not node.has_key(key):
- raise KeyError, key
-
- # check to make sure we're not duplicating an existing key
- if key == self.key and node[key] != value:
- try:
- self.lookup(value)
- except KeyError:
- pass
- else:
- raise ValueError, 'node with key "%s" exists'%value
-
- prop = self.properties[key]
-
- if isinstance(prop, Link):
- link_class = self.properties[key].classname
- # if it isn't a number, it's a key
- if type(value) != type(''):
- raise ValueError, 'link value must be String'
- if not num_re.match(value):
- try:
- value = self.db.classes[link_class].lookup(value)
- except:
- raise IndexError, 'new property "%s": %s not a %s'%(
- key, value, self.properties[key].classname)
-
- if not self.db.hasnode(link_class, value):
- raise IndexError, '%s has no node %s'%(link_class, value)
-
- # register the unlink with the old linked node
- if node[key] is not None:
- self.db.addjournal(link_class, node[key], 'unlink',
- (self.classname, nodeid, key))
-
- # register the link with the newly linked node
- if value is not None:
- self.db.addjournal(link_class, value, 'link',
- (self.classname, nodeid, key))
-
- elif isinstance(prop, Multilink):
- if type(value) != type([]):
- raise TypeError, 'new property "%s" not a list of ids'%key
- link_class = self.properties[key].classname
- l = []
- for entry in value:
- # if it isn't a number, it's a key
- if type(entry) != type(''):
- raise ValueError, 'link value must be String'
- if not num_re.match(entry):
- try:
- entry = self.db.classes[link_class].lookup(entry)
- except:
- raise IndexError, 'new property "%s": %s not a %s'%(
- key, entry, self.properties[key].classname)
- l.append(entry)
- value = l
- propvalues[key] = value
-
- #handle removals
- l = node[key]
- for id in l[:]:
- if id in value:
- continue
- # register the unlink with the old linked node
- self.db.addjournal(link_class, id, 'unlink',
- (self.classname, nodeid, key))
- l.remove(id)
-
- # handle additions
- for id in value:
- if not self.db.hasnode(link_class, id):
- raise IndexError, '%s has no node %s'%(link_class, id)
- if id in l:
- continue
- # register the link with the newly linked node
- self.db.addjournal(link_class, id, 'link',
- (self.classname, nodeid, key))
- l.append(id)
-
- elif isinstance(prop, String):
- if value is not None and type(value) != type(''):
- raise TypeError, 'new property "%s" not a string'%key
-
- elif isinstance(prop, Password):
- if not isinstance(value, password.Password):
- raise TypeError, 'new property "%s" not a Password'% key
- propvalues[key] = value = str(value)
-
- elif isinstance(prop, Date):
- if not isinstance(value, date.Date):
- raise TypeError, 'new property "%s" not a Date'% key
- propvalues[key] = value = value.get_tuple()
-
- elif isinstance(prop, Interval):
- if not isinstance(value, date.Interval):
- raise TypeError, 'new property "%s" not an Interval'% key
- propvalues[key] = value = value.get_tuple()
-
- node[key] = value
-
- self.db.setnode(self.classname, nodeid, node)
- self.db.addjournal(self.classname, nodeid, 'set', propvalues)
+ raise NotImplementedError
def retire(self, nodeid):
"""Retire a node.
-
+
The properties on the node remain available from the get() method,
and the node's id is never reused.
-
+
Retired nodes are not returned by the find(), list(), or lookup()
methods, and other nodes may reuse the values of their key properties.
"""
- if self.db.journaltag is None:
- raise DatabaseError, 'Database open read-only'
- node = self.db.getnode(self.classname, nodeid)
- node[self.db.RETIRED_FLAG] = 1
- self.db.setnode(self.classname, nodeid, node)
- self.db.addjournal(self.classname, nodeid, 'retired', None)
+ raise NotImplementedError
+
+ def restore(self, nodeid):
+ """Restpre a retired node.
+
+ Make node available for all operations like it was before retirement.
+ """
+ raise NotImplementedError
+
+ def is_retired(self, nodeid):
+ """Return true if the node is rerired
+ """
+ raise NotImplementedError
+
+ def destroy(self, nodeid):
+ """Destroy a node.
+
+ WARNING: this method should never be used except in extremely rare
+ situations where there could never be links to the node being
+ deleted
+
+ WARNING: use retire() instead
+
+ WARNING: the properties of this node will not be available ever again
+
+ WARNING: really, use retire() instead
+
+ Well, I think that's enough warnings. This method exists mostly to
+ support the session storage of the cgi interface.
+
+ The node is completely removed from the hyperdb, including all journal
+ entries. It will no longer be available, and will generally break code
+ if there are any references to the node.
+ """
def history(self, nodeid):
"""Retrieve the journal of edits on a particular node.
'date' is a Timestamp object specifying the time of the change and
'tag' is the journaltag specified when the database was opened.
"""
+ if not self.do_journal:
+ raise ValueError('Journalling is disabled for this class')
return self.db.getjournal(self.classname, nodeid)
# Locating nodes:
+ def hasnode(self, nodeid):
+ """Determine if the given nodeid actually exists
+ """
+ raise NotImplementedError
def setkey(self, propname):
"""Select a String property of this class to be the key property.
None, or a TypeError is raised. The values of the key property on
all existing nodes must be unique or a ValueError is raised.
"""
- # TODO: validate that the property is a String!
- self.key = propname
+ raise NotImplementedError
+
+ def setlabelprop(self, labelprop):
+ """Set the label property. Used for override of labelprop
+ resolution order.
+ """
+ if labelprop not in self.getprops():
+ raise ValueError, _("Not a property name: %s") % labelprop
+ self._labelprop = labelprop
+
+ def setorderprop(self, orderprop):
+ """Set the order property. Used for override of orderprop
+ resolution order
+ """
+ if orderprop not in self.getprops():
+ raise ValueError, _("Not a property name: %s") % orderprop
+ self._orderprop = orderprop
def getkey(self):
"""Return the name of the key property for this class or None."""
- return self.key
+ raise NotImplementedError
def labelprop(self, default_to_id=0):
- ''' Return the property name for a label for the given node.
+ """Return the property name for a label for the given node.
This method attempts to generate a consistent label for the node.
It tries the following in order:
- 1. key property
- 2. "name" property
- 3. "title" property
- 4. first property from the sorted property name list
- '''
+
+ 0. self._labelprop if set
+ 1. key property
+ 2. "name" property
+ 3. "title" property
+ 4. first property from the sorted property name list
+ """
+ if hasattr(self, '_labelprop'):
+ return self._labelprop
k = self.getkey()
if k:
return k
props.sort()
return props[0]
- # TODO: set up a separate index db file for this? profile?
+ def orderprop(self):
+ """Return the property name to use for sorting for the given node.
+
+ This method computes the property for sorting.
+ It tries the following in order:
+
+ 0. self._orderprop if set
+ 1. "order" property
+ 2. self.labelprop()
+ """
+
+ if hasattr(self, '_orderprop'):
+ return self._orderprop
+ props = self.getprops()
+ if props.has_key('order'):
+ return 'order'
+ return self.labelprop()
+
def lookup(self, keyvalue):
"""Locate a particular node by its key property and return its id.
the nodes in this class, the matching node's id is returned;
otherwise a KeyError is raised.
"""
- cldb = self.db.getclassdb(self.classname)
- for nodeid in self.db.getnodeids(self.classname, cldb):
- node = self.db.getnode(self.classname, nodeid, cldb)
- if node.has_key(self.db.RETIRED_FLAG):
- continue
- if node[self.key] == keyvalue:
- return nodeid
- cldb.close()
- raise KeyError, keyvalue
+ raise NotImplementedError
- # XXX: change from spec - allows multiple props to match
def find(self, **propspec):
- """Get the ids of nodes in this class which link to a given node.
-
- 'propspec' consists of keyword args propname=nodeid
- 'propname' must be the name of a property in this class, or a
- KeyError is raised. That property must be a Link or Multilink
- property, or a TypeError is raised.
-
- 'nodeid' must be the id of an existing node in the class linked
- to by the given property, or an IndexError is raised.
- """
- propspec = propspec.items()
- for propname, nodeid in propspec:
- # check the prop is OK
- prop = self.properties[propname]
- if not isinstance(prop, Link) and not isinstance(prop, Multilink):
- raise TypeError, "'%s' not a Link/Multilink property"%propname
- if not self.db.hasnode(prop.classname, nodeid):
- raise ValueError, '%s has no node %s'%(link_class, nodeid)
-
- # ok, now do the find
- cldb = self.db.getclassdb(self.classname)
- l = []
- for id in self.db.getnodeids(self.classname, cldb):
- node = self.db.getnode(self.classname, id, cldb)
- if node.has_key(self.db.RETIRED_FLAG):
- continue
- for propname, nodeid in propspec:
- property = node[propname]
- if isinstance(prop, Link) and nodeid == property:
- l.append(id)
- elif isinstance(prop, Multilink) and nodeid in property:
- l.append(id)
- cldb.close()
- return l
-
- def stringFind(self, **requirements):
- """Locate a particular node by matching a set of its String properties.
-
- If the property is not a String property, a TypeError is raised.
-
- The return is a list of the id of all nodes that match.
- """
- for propname in requirements.keys():
- prop = self.properties[propname]
- if isinstance(not prop, String):
- raise TypeError, "'%s' not a String property"%propname
- l = []
- cldb = self.db.getclassdb(self.classname)
- for nodeid in self.db.getnodeids(self.classname, cldb):
- node = self.db.getnode(self.classname, nodeid, cldb)
- if node.has_key(self.db.RETIRED_FLAG):
- continue
- for key, value in requirements.items():
- if node[key] != value:
- break
- else:
- l.append(nodeid)
- cldb.close()
- return l
+ """Get the ids of nodes in this class which link to the given nodes.
- def list(self):
- """Return a list of the ids of the active nodes in this class."""
- l = []
- cn = self.classname
- cldb = self.db.getclassdb(cn)
- for nodeid in self.db.getnodeids(cn, cldb):
- node = self.db.getnode(cn, nodeid, cldb)
- if node.has_key(self.db.RETIRED_FLAG):
- continue
- l.append(nodeid)
- l.sort()
- cldb.close()
- return l
+ 'propspec' consists of keyword args propname={nodeid:1,}
+ 'propname' must be the name of a property in this class, or a
+ KeyError is raised. That property must be a Link or Multilink
+ property, or a TypeError is raised.
- # XXX not in spec
- def filter(self, filterspec, sort, group, num_re = re.compile('^\d+$')):
- ''' Return a list of the ids of the active nodes in this class that
- match the 'filter' spec, sorted by the group spec and then the
- sort spec
- '''
- cn = self.classname
+ Any node in this class whose 'propname' property links to any of the
+ nodeids will be returned. Used by the full text indexing, which knows
+ that "foo" occurs in msg1, msg3 and file7, so we have hits on these
+ issues:
- # optimise filterspec
- l = []
- props = self.getprops()
- for k, v in filterspec.items():
- propclass = props[k]
- if isinstance(propclass, Link):
- if type(v) is not type([]):
- v = [v]
- # replace key values with node ids
- u = []
- link_class = self.db.classes[propclass.classname]
- for entry in v:
- if not num_re.match(entry):
- try:
- entry = link_class.lookup(entry)
- except:
- raise ValueError, 'new property "%s": %s not a %s'%(
- k, entry, self.properties[k].classname)
- u.append(entry)
-
- l.append((0, k, u))
- elif isinstance(propclass, Multilink):
- if type(v) is not type([]):
- v = [v]
- # replace key values with node ids
- u = []
- link_class = self.db.classes[propclass.classname]
- for entry in v:
- if not num_re.match(entry):
- try:
- entry = link_class.lookup(entry)
- except:
- raise ValueError, 'new property "%s": %s not a %s'%(
- k, entry, self.properties[k].classname)
- u.append(entry)
- l.append((1, k, u))
- elif isinstance(propclass, String):
- # simple glob searching
- v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
- v = v.replace('?', '.')
- v = v.replace('*', '.*?')
- l.append((2, k, re.compile(v, re.I)))
- else:
- l.append((6, k, v))
- filterspec = l
+ db.issue.find(messages={'1':1,'3':1}, files={'7':1})
+ """
+ raise NotImplementedError
- # now, find all the nodes that are active and pass filtering
- l = []
- cldb = self.db.getclassdb(cn)
- for nodeid in self.db.getnodeids(cn, cldb):
- node = self.db.getnode(cn, nodeid, cldb)
- if node.has_key(self.db.RETIRED_FLAG):
+ def _filter(self, search_matches, filterspec, sort=(None,None),
+ group=(None,None)):
+ """For some backends this implements the non-transitive
+ search, for more information see the filter method.
+ """
+ raise NotImplementedError
+
+ def _proptree(self, filterspec, sortattr=[], retr=False):
+ """Build a tree of all transitive properties in the given
+ filterspec.
+ If we retrieve (retr is True) linked items we don't follow
+ across multilinks. We also don't follow if the searched value
+ can contain NULL values.
+ """
+ proptree = Proptree(self.db, self, '', self.getprops(), retr=retr)
+ for key, v in filterspec.iteritems():
+ keys = key.split('.')
+ p = proptree
+ mlseen = False
+ for k in keys:
+ if isinstance (p.propclass, Multilink):
+ mlseen = True
+ isnull = v == '-1' or v is None
+ nullin = isinstance(v, type([])) and ('-1' in v or None in v)
+ r = retr and not mlseen and not isnull and not nullin
+ p = p.append(k, retr=r)
+ p.val = v
+ multilinks = {}
+ for s in sortattr:
+ keys = s[1].split('.')
+ p = proptree
+ mlseen = False
+ for k in keys:
+ if isinstance (p.propclass, Multilink):
+ mlseen = True
+ r = retr and not mlseen
+ p = p.append(k, need_for='sort', retr=r)
+ if isinstance (p.propclass, Multilink):
+ multilinks[p] = True
+ if p.cls:
+ p = p.append(p.cls.orderprop(), need_for='sort')
+ if p.sort_direction: # if an orderprop is also specified explicitly
continue
- # apply filter
- for t, k, v in filterspec:
- if t == 0 and node[k] not in v:
- # link - if this node'd property doesn't appear in the
- # filterspec's nodeid list, skip it
- break
- elif t == 1:
- # multilink - if any of the nodeids required by the
- # filterspec aren't in this node's property, then skip
- # it
- for value in v:
- if value not in node[k]:
- break
- else:
- continue
- break
- elif t == 2 and not v.search(node[k]):
- # RE search
- break
- elif t == 6 and node[k] != v:
- # straight value comparison for the other types
- break
- else:
- l.append((nodeid, node))
- l.sort()
- cldb.close()
-
- # optimise sort
- m = []
- for entry in sort:
- if entry[0] != '-':
- m.append(('+', entry))
- else:
- m.append((entry[0], entry[1:]))
- sort = m
-
- # optimise group
- m = []
- for entry in group:
- if entry[0] != '-':
- m.append(('+', entry))
- else:
- m.append((entry[0], entry[1:]))
- group = m
- # now, sort the result
- def sortfun(a, b, sort=sort, group=group, properties=self.getprops(),
- db = self.db, cl=self):
- a_id, an = a
- b_id, bn = b
- # sort by group and then sort
- for list in group, sort:
- for dir, prop in list:
- # handle the properties that might be "faked"
- if not an.has_key(prop):
- an[prop] = cl.get(a_id, prop)
- av = an[prop]
- if not bn.has_key(prop):
- bn[prop] = cl.get(b_id, prop)
- bv = bn[prop]
-
- # sorting is class-specific
- propclass = properties[prop]
-
- # String and Date values are sorted in the natural way
- if isinstance(propclass, String):
- # clean up the strings
- if av and av[0] in string.uppercase:
- av = an[prop] = av.lower()
- if bv and bv[0] in string.uppercase:
- bv = bn[prop] = bv.lower()
- if (isinstance(propclass, String) or
- isinstance(propclass, Date)):
- # it might be a string that's really an integer
- try:
- av = int(av)
- bv = int(bv)
- except:
- pass
- if dir == '+':
- r = cmp(av, bv)
- if r != 0: return r
- elif dir == '-':
- r = cmp(bv, av)
- if r != 0: return r
-
- # Link properties are sorted according to the value of
- # the "order" property on the linked nodes if it is
- # present; or otherwise on the key string of the linked
- # nodes; or finally on the node ids.
- elif isinstance(propclass, Link):
- link = db.classes[propclass.classname]
- if av is None and bv is not None: return -1
- if av is not None and bv is None: return 1
- if av is None and bv is None: return 0
- if link.getprops().has_key('order'):
- if dir == '+':
- r = cmp(link.get(av, 'order'),
- link.get(bv, 'order'))
- if r != 0: return r
- elif dir == '-':
- r = cmp(link.get(bv, 'order'),
- link.get(av, 'order'))
- if r != 0: return r
- elif link.getkey():
- key = link.getkey()
- if dir == '+':
- r = cmp(link.get(av, key), link.get(bv, key))
- if r != 0: return r
- elif dir == '-':
- r = cmp(link.get(bv, key), link.get(av, key))
- if r != 0: return r
- else:
- if dir == '+':
- r = cmp(av, bv)
- if r != 0: return r
- elif dir == '-':
- r = cmp(bv, av)
- if r != 0: return r
-
- # Multilink properties are sorted according to how many
- # links are present.
- elif isinstance(propclass, Multilink):
- if dir == '+':
- r = cmp(len(av), len(bv))
- if r != 0: return r
- elif dir == '-':
- r = cmp(len(bv), len(av))
- if r != 0: return r
- # end for dir, prop in list:
- # end for list in sort, group:
- # if all else fails, compare the ids
- return cmp(a[0], b[0])
-
- l.sort(sortfun)
- return [i[0] for i in l]
+ p.sort_direction = s[0]
+ proptree.sortattr.append (p)
+ for p in multilinks.iterkeys():
+ sattr = {}
+ for c in p:
+ if c.sort_direction:
+ sattr [c] = True
+ for sa in proptree.sortattr:
+ if sa in sattr:
+ p.sortattr.append (sa)
+ return proptree
+
+ def get_transitive_prop(self, propname_path, default = None):
+ """Expand a transitive property (individual property names
+ separated by '.' into a new property at the end of the path. If
+ one of the names does not refer to a valid property, we return
+ None.
+ Example propname_path (for class issue): "messages.author"
+ """
+ props = self.db.getclass(self.classname).getprops()
+ for k in propname_path.split('.'):
+ try:
+ prop = props[k]
+ except (KeyError, TypeError):
+ return default
+ cl = getattr(prop, 'classname', None)
+ props = None
+ if cl:
+ props = self.db.getclass(cl).getprops()
+ return prop
+
+ def _sortattr(self, sort=[], group=[]):
+ """Build a single list of sort attributes in the correct order
+ with sanity checks (no duplicate properties) included. Always
+ sort last by id -- if id is not already in sortattr.
+ """
+ seen = {}
+ sortattr = []
+ for srt in group, sort:
+ if not isinstance(srt, list):
+ srt = [srt]
+ for s in srt:
+ if s[1] and s[1] not in seen:
+ sortattr.append((s[0] or '+', s[1]))
+ seen[s[1]] = True
+ if 'id' not in seen :
+ sortattr.append(('+', 'id'))
+ return sortattr
+
+ def filter(self, search_matches, filterspec, sort=[], group=[]):
+ """Return a list of the ids of the active nodes in this class that
+ match the 'filter' spec, sorted by the group spec and then the
+ sort spec.
+
+ "filterspec" is {propname: value(s)}
+
+ "sort" and "group" are [(dir, prop), ...] where dir is '+', '-'
+ or None and prop is a prop name or None. Note that for
+ backward-compatibility reasons a single (dir, prop) tuple is
+ also allowed.
+
+ "search_matches" is a container type
+
+ The filter must match all properties specificed. If the property
+ value to match is a list:
+
+ 1. String properties must match all elements in the list, and
+ 2. Other properties must match any of the elements in the list.
+
+ Note that now the propname in filterspec and prop in a
+ sort/group spec may be transitive, i.e., it may contain
+ properties of the form link.link.link.name, e.g. you can search
+ for all issues where a message was added by a certain user in
+ the last week with a filterspec of
+ {'messages.author' : '42', 'messages.creation' : '.-1w;'}
+
+ Implementation note:
+ This implements a non-optimized version of Transitive search
+ using _filter implemented in a backend class. A more efficient
+ version can be implemented in the individual backends -- e.g.,
+ an SQL backend will want to create a single SQL statement and
+ override the filter method instead of implementing _filter.
+ """
+ sortattr = self._sortattr(sort = sort, group = group)
+ proptree = self._proptree(filterspec, sortattr)
+ proptree.search(search_matches)
+ return proptree.sort()
+
+ # non-optimized filter_iter, a backend may chose to implement a
+ # better version that provides a real iterator that pre-fills the
+ # cache for each id returned. Note that the filter_iter doesn't
+ # promise to correctly sort by multilink (which isn't sane to do
+ # anyway).
+ filter_iter = filter
def count(self):
"""Get the number of nodes in this class.
in this class run from 1 to numnodes, and numnodes+1 will be the
id of the next node to be created in this class.
"""
- return self.db.countnodes(self.classname)
+ raise NotImplementedError
# Manipulating properties:
-
def getprops(self, protected=1):
"""Return a dictionary mapping property names to property objects.
If the "protected" flag is true, we include protected properties -
- those which may not be modified."""
- d = self.properties.copy()
- if protected:
- d['id'] = String()
- return d
+ those which may not be modified.
+ """
+ raise NotImplementedError
+
+ def get_required_props(self, propnames = []):
+ """Return a dict of property names mapping to property objects.
+ All properties that have the "required" flag set will be
+ returned in addition to all properties in the propnames
+ parameter.
+ """
+ props = self.getprops(protected = False)
+ pdict = dict([(p, props[p]) for p in propnames])
+ pdict.update([(k, v) for k, v in props.iteritems() if v.required])
+ return pdict
def addprop(self, **properties):
"""Add properties to this class.
may collide with the names of existing properties, or a ValueError
is raised before any properties have been added.
"""
- for key in properties.keys():
- if self.properties.has_key(key):
- raise ValueError, key
- self.properties.update(properties)
+ raise NotImplementedError
+
+ def index(self, nodeid):
+ """Add (or refresh) the node to search indexes"""
+ raise NotImplementedError
+
+ #
+ # Detector interface
+ #
+ def audit(self, event, detector, priority = 100):
+ """Register an auditor detector"""
+ self.auditors[event].append((priority, detector.__name__, detector))
+
+ def fireAuditors(self, event, nodeid, newvalues):
+ """Fire all registered auditors"""
+ for prio, name, audit in self.auditors[event]:
+ audit(self.db, self, nodeid, newvalues)
+
+ def react(self, event, detector, priority = 100):
+ """Register a reactor detector"""
+ self.reactors[event].append((priority, detector.__name__, detector))
+
+ def fireReactors(self, event, nodeid, oldvalues):
+ """Fire all registered reactors"""
+ for prio, name, react in self.reactors[event]:
+ react(self.db, self, nodeid, oldvalues)
+
+ #
+ # import / export support
+ #
+ def export_propnames(self):
+ """List the property names for export from this Class"""
+ propnames = self.getprops().keys()
+ propnames.sort()
+ return propnames
+
+ def import_journals(self, entries):
+ """Import a class's journal.
+
+ Uses setjournal() to set the journal for each item.
+ Strategy for import: Sort first by id, then import journals for
+ each id, this way the memory footprint is a lot smaller than the
+ initial implementation which stored everything in a big hash by
+ id and then proceeded to import journals for each id."""
+ properties = self.getprops()
+ a = []
+ for l in entries:
+ # first element in sorted list is the (numeric) id
+ # in python2.4 and up we would use sorted with a key...
+ a.append ((int (l [0].strip ("'")), l))
+ a.sort ()
+
+
+ last = 0
+ r = []
+ for n, l in a:
+ nodeid, jdate, user, action, params = map(eval, l)
+ assert (str(n) == nodeid)
+ if n != last:
+ if r:
+ self.db.setjournal(self.classname, str(last), r)
+ last = n
+ r = []
+
+ if action == 'set':
+ for propname, value in params.iteritems():
+ prop = properties[propname]
+ if value is None:
+ pass
+ elif isinstance(prop, Date):
+ value = date.Date(value)
+ elif isinstance(prop, Interval):
+ value = date.Interval(value)
+ elif isinstance(prop, Password):
+ value = password.JournalPassword(encrypted=value)
+ params[propname] = value
+ elif action == 'create' and params:
+ # old tracker with data stored in the create!
+ params = {}
+ r.append((nodeid, date.Date(jdate), user, action, params))
+ if r:
+ self.db.setjournal(self.classname, nodeid, r)
+
+ #
+ # convenience methods
+ #
+ def get_roles(self, nodeid):
+ """Return iterator for all roles for this nodeid.
+
+ Yields string-processed roles.
+ This method can be overridden to provide a hook where we can
+ insert other permission models (e.g. get roles from database)
+ In standard schemas only a user has a roles property but
+ this may be different in customized schemas.
+ Note that this is the *central place* where role
+ processing happens!
+ """
+ node = self.db.getnode(self.classname, nodeid)
+ return iter_roles(node['roles'])
+
+ def has_role(self, nodeid, *roles):
+ '''See if this node has any roles that appear in roles.
+
+ For convenience reasons we take a list.
+ In standard schemas only a user has a roles property but
+ this may be different in customized schemas.
+ '''
+ roles = dict.fromkeys ([r.strip().lower() for r in roles])
+ for role in self.get_roles(nodeid):
+ if role in roles:
+ return True
+ return False
+
+
+class HyperdbValueError(ValueError):
+ """ Error converting a raw value into a Hyperdb value """
+ pass
+
+def convertLinkValue(db, propname, prop, value, idre=re.compile('^\d+$')):
+ """ Convert the link value (may be id or key value) to an id value. """
+ linkcl = db.classes[prop.classname]
+ if not idre.match(value):
+ if linkcl.getkey():
+ try:
+ value = linkcl.lookup(value)
+ except KeyError, message:
+ raise HyperdbValueError, _('property %s: %r is not a %s.')%(
+ propname, value, prop.classname)
+ else:
+ raise HyperdbValueError, _('you may only enter ID values '\
+ 'for property %s')%propname
+ return value
+
+def fixNewlines(text):
+ """ Homogenise line endings.
+
+ Different web clients send different line ending values, but
+ other systems (eg. email) don't necessarily handle those line
+ endings. Our solution is to convert all line endings to LF.
+ """
+ text = text.replace('\r\n', '\n')
+ return text.replace('\r', '\n')
+
+def rawToHyperdb(db, klass, itemid, propname, value, **kw):
+ """ Convert the raw (user-input) value to a hyperdb-storable value. The
+ value is for the "propname" property on itemid (may be None for a
+ new item) of "klass" in "db".
+
+ The value is usually a string, but in the case of multilink inputs
+ it may be either a list of strings or a string with comma-separated
+ values.
+ """
+ properties = klass.getprops()
+
+ # ensure it's a valid property name
+ propname = propname.strip()
+ try:
+ proptype = properties[propname]
+ except KeyError:
+ raise HyperdbValueError, _('%r is not a property of %s')%(propname,
+ klass.classname)
+
+ # if we got a string, strip it now
+ if isinstance(value, type('')):
+ value = value.strip()
+
+ # convert the input value to a real property value
+ value = proptype.from_raw(value, db=db, klass=klass,
+ propname=propname, itemid=itemid, **kw)
+
+ return value
+
+class FileClass:
+ """ A class that requires the "content" property and stores it on
+ disk.
+ """
+ default_mime_type = 'text/plain'
+
+ def __init__(self, db, classname, **properties):
+ """The newly-created class automatically includes the "content"
+ property.
+ """
+ if not properties.has_key('content'):
+ properties['content'] = String(indexme='yes')
+
+ def export_propnames(self):
+ """ Don't export the "content" property
+ """
+ propnames = self.getprops().keys()
+ propnames.remove('content')
+ propnames.sort()
+ return propnames
+
+ def exportFilename(self, dirname, nodeid):
+ subdir_filename = self.db.subdirFilename(self.classname, nodeid)
+ return os.path.join(dirname, self.classname+'-files', subdir_filename)
+
+ def export_files(self, dirname, nodeid):
+ """ Export the "content" property as a file, not csv column
+ """
+ source = self.db.filename(self.classname, nodeid)
+
+ dest = self.exportFilename(dirname, nodeid)
+ ensureParentsExist(dest)
+ shutil.copyfile(source, dest)
+
+ def import_files(self, dirname, nodeid):
+ """ Import the "content" property as a file
+ """
+ source = self.exportFilename(dirname, nodeid)
+ dest = self.db.filename(self.classname, nodeid, create=1)
+ ensureParentsExist(dest)
+ shutil.copyfile(source, dest)
+
+ mime_type = None
+ props = self.getprops()
+ if props.has_key('type'):
+ mime_type = self.get(nodeid, 'type')
+ if not mime_type:
+ mime_type = self.default_mime_type
+ if props['content'].indexme:
+ self.db.indexer.add_text((self.classname, nodeid, 'content'),
+ self.get(nodeid, 'content'), mime_type)
-# XXX not in spec
class Node:
- ''' A convenience wrapper for the given node
- '''
- def __init__(self, cl, nodeid):
+ """ A convenience wrapper for the given node
+ """
+ def __init__(self, cl, nodeid, cache=1):
self.__dict__['cl'] = cl
self.__dict__['nodeid'] = nodeid
def keys(self, protected=1):
return l
def has_key(self, name):
return self.cl.getprops().has_key(name)
+ def get(self, name, default=None):
+ if self.has_key(name):
+ return self[name]
+ else:
+ return default
def __getattr__(self, name):
if self.__dict__.has_key(name):
return self.__dict__[name]
try:
return self.cl.get(self.nodeid, name)
except KeyError, value:
- raise AttributeError, str(value)
+ # we trap this but re-raise it as AttributeError - all other
+ # exceptions should pass through untrapped
+ pass
+ # nope, no such attribute
+ raise AttributeError, str(value)
def __getitem__(self, name):
return self.cl.get(self.nodeid, name)
def __setattr__(self, name, value):
return self.cl.retire(self.nodeid)
-def Choice(name, *options):
- cl = Class(db, name, name=hyperdb.String(), order=hyperdb.String())
+def Choice(name, db, *options):
+ """Quick helper to create a simple class with choices
+ """
+ cl = Class(db, name, name=String(), order=String())
for i in range(len(options)):
- cl.create(name=option[i], order=i)
- return hyperdb.Link(name)
+ cl.create(name=options[i], order=i)
+ return Link(name)
-#
-# $Log: not supported by cvs2svn $
-# Revision 1.26 2001/10/16 03:48:01 richard
-# admin tool now complains if a "find" is attempted with a non-link property.
-#
-# Revision 1.25 2001/10/11 00:17:51 richard
-# Reverted a change in hyperdb so the default value for missing property
-# values in a create() is None and not '' (the empty string.) This obviously
-# breaks CSV import/export - the string 'None' will be created in an
-# export/import operation.
-#
-# Revision 1.24 2001/10/10 03:54:57 richard
-# Added database importing and exporting through CSV files.
-# Uses the csv module from object-craft for exporting if it's available.
-# Requires the csv module for importing.
-#
-# Revision 1.23 2001/10/09 23:58:10 richard
-# Moved the data stringification up into the hyperdb.Class class' get, set
-# and create methods. This means that the data is also stringified for the
-# journal call, and removes duplication of code from the backends. The
-# backend code now only sees strings.
-#
-# Revision 1.22 2001/10/09 07:25:59 richard
-# Added the Password property type. See "pydoc roundup.password" for
-# implementation details. Have updated some of the documentation too.
-#
-# Revision 1.21 2001/10/05 02:23:24 richard
-# . roundup-admin create now prompts for property info if none is supplied
-# on the command-line.
-# . hyperdb Class getprops() method may now return only the mutable
-# properties.
-# . Login now uses cookies, which makes it a whole lot more flexible. We can
-# now support anonymous user access (read-only, unless there's an
-# "anonymous" user, in which case write access is permitted). Login
-# handling has been moved into cgi_client.Client.main()
-# . The "extended" schema is now the default in roundup init.
-# . The schemas have had their page headings modified to cope with the new
-# login handling. Existing installations should copy the interfaces.py
-# file from the roundup lib directory to their instance home.
-# . Incorrectly had a Bizar Software copyright on the cgitb.py module from
-# Ping - has been removed.
-# . Fixed a whole bunch of places in the CGI interface where we should have
-# been returning Not Found instead of throwing an exception.
-# . Fixed a deviation from the spec: trying to modify the 'id' property of
-# an item now throws an exception.
-#
-# Revision 1.20 2001/10/04 02:12:42 richard
-# Added nicer command-line item adding: passing no arguments will enter an
-# interactive more which asks for each property in turn. While I was at it, I
-# fixed an implementation problem WRT the spec - I wasn't raising a
-# ValueError if the key property was missing from a create(). Also added a
-# protected=boolean argument to getprops() so we can list only the mutable
-# properties (defaults to yes, which lists the immutables).
-#
-# Revision 1.19 2001/08/29 04:47:18 richard
-# Fixed CGI client change messages so they actually include the properties
-# changed (again).
-#
-# Revision 1.18 2001/08/16 07:34:59 richard
-# better CGI text searching - but hidden filter fields are disappearing...
-#
-# Revision 1.17 2001/08/16 06:59:58 richard
-# all searches use re now - and they're all case insensitive
-#
-# Revision 1.16 2001/08/15 23:43:18 richard
-# Fixed some isFooTypes that I missed.
-# Refactored some code in the CGI code.
-#
-# Revision 1.15 2001/08/12 06:32:36 richard
-# using isinstance(blah, Foo) now instead of isFooType
-#
-# Revision 1.14 2001/08/07 00:24:42 richard
-# stupid typo
-#
-# Revision 1.13 2001/08/07 00:15:51 richard
-# Added the copyright/license notice to (nearly) all files at request of
-# Bizar Software.
-#
-# Revision 1.12 2001/08/02 06:38:17 richard
-# Roundupdb now appends "mailing list" information to its messages which
-# include the e-mail address and web interface address. Templates may
-# override this in their db classes to include specific information (support
-# instructions, etc).
-#
-# Revision 1.11 2001/08/01 04:24:21 richard
-# mailgw was assuming certain properties existed on the issues being created.
-#
-# Revision 1.10 2001/07/30 02:38:31 richard
-# get() now has a default arg - for migration only.
-#
-# Revision 1.9 2001/07/29 09:28:23 richard
-# Fixed sorting by clicking on column headings.
-#
-# Revision 1.8 2001/07/29 08:27:40 richard
-# Fixed handling of passed-in values in form elements (ie. during a
-# drill-down)
-#
-# Revision 1.7 2001/07/29 07:01:39 richard
-# Added vim command to all source so that we don't get no steenkin' tabs :)
-#
-# Revision 1.6 2001/07/29 05:36:14 richard
-# Cleanup of the link label generation.
-#
-# Revision 1.5 2001/07/29 04:05:37 richard
-# Added the fabricated property "id".
-#
-# Revision 1.4 2001/07/27 06:25:35 richard
-# Fixed some of the exceptions so they're the right type.
-# Removed the str()-ification of node ids so we don't mask oopsy errors any
-# more.
-#
-# Revision 1.3 2001/07/27 05:17:14 richard
-# just some comments
-#
-# Revision 1.2 2001/07/22 12:09:32 richard
-# Final commit of Grande Splite
-#
-# Revision 1.1 2001/07/22 11:58:35 richard
-# More Grande Splite
-#
-#
-# vim: set filetype=python ts=4 sw=4 et si
+# vim: set filetype=python sts=4 sw=4 et si :