index e27344772d11724afde9bc06ea21ebdd9b72b8b0..ed99a5bf44d06ed24e55ce8b5240bcf4772c6310 100644 (file)
# BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
# SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
#
-#$Id: back_anydbm.py,v 1.55 2002-07-30 08:22:38 richard Exp $
+#$Id: back_anydbm.py,v 1.64 2002-08-22 07:57:11 richard Exp $
'''
This module defines a backend that saves the hyperdatabase in a database
chosen by anydbm. It is guaranteed to always be available in python
# Now the database
#
class Database(FileStorage, hyperdb.Database, roundupdb.Database):
- """A database for storing records containing flexible data types.
+ '''A database for storing records containing flexible data types.
Transaction stuff TODO:
. check the timestamp of the class file and nuke the cache if it's
modified. Do some sort of conflict checking on the dirty stuff.
. perhaps detect write collisions (related to above)?
- """
+ '''
def __init__(self, config, journaltag=None):
- """Open a hyperdatabase given a specifier to some storage.
+ '''Open a hyperdatabase given a specifier to some storage.
The 'storagelocator' is obtained from config.DATABASE.
The meaning of 'storagelocator' depends on the particular
entries for any edits done on the database. If 'journaltag' is
None, the database is opened in read-only mode: the Class.create(),
Class.set(), and Class.retire() methods are disabled.
- """
+ '''
self.config, self.journaltag = config, journaltag
self.dir = config.DATABASE
self.classes = {}
os.umask(0002)
def post_init(self):
- """Called once the schema initialisation has finished."""
+ '''Called once the schema initialisation has finished.'''
# reindex the db if necessary
if self.indexer.should_reindex():
self.reindex()
# Classes
#
def __getattr__(self, classname):
- """A convenient way of calling self.getclass(classname)."""
+ '''A convenient way of calling self.getclass(classname).'''
if self.classes.has_key(classname):
if __debug__:
print >>hyperdb.DEBUG, '__getattr__', (self, classname)
self.classes[cn] = cl
def getclasses(self):
- """Return a list of the names of all existing classes."""
+ '''Return a list of the names of all existing classes.'''
if __debug__:
print >>hyperdb.DEBUG, 'getclasses', (self,)
l = self.classes.keys()
return l
def getclass(self, classname):
- """Get the Class object representing a particular class.
+ '''Get the Class object representing a particular class.
If 'classname' is not a valid class name, a KeyError is raised.
- """
+ '''
if __debug__:
print >>hyperdb.DEBUG, 'getclass', (self, classname)
return self.classes[classname]
release_lock(lock)
return newid
+ def setid(self, classname, setid):
+ ''' Set the id counter: used during import of database
+ '''
+ # open the ids DB - create if if doesn't exist
+ lock = self.lockdb('_ids')
+ db = self.opendb('_ids', 'c')
+ db[classname] = str(setid)
+ db.close()
+ release_lock(lock)
+
#
# Nodes
#
if isinstance(prop, Password):
d[k] = str(v)
elif isinstance(prop, Date) and v is not None:
- d[k] = v.get_tuple()
+ d[k] = v.serialise()
elif isinstance(prop, Interval) and v is not None:
- d[k] = v.get_tuple()
+ d[k] = v.serialise()
else:
d[k] = v
return d
if __debug__:
print >>hyperdb.DEBUG, 'packjournal', (self, pack_before)
- pack_before = pack_before.get_tuple()
-
classes = self.getclasses()
# figure the class db type
for entry in journal:
(nodeid, date_stamp, self.journaltag, action,
params) = entry
+ date_stamp = date.Date(date_stamp)
if date_stamp > pack_before or action == 'create':
l.append(entry)
elif action == 'set':
return self.databases[db_name]
def doSaveJournal(self, classname, nodeid, action, params):
- # serialise first
- if action in ('set', 'create'):
- params = self.serialise(classname, params)
+ # handle supply of the special journalling parameters (usually
+ # supplied on importing an existing database)
+ if isinstance(params, type({})):
+ if params.has_key('creator'):
+ journaltag = self.user.get(params['creator'], 'username')
+ del params['creator']
+ else:
+ journaltag = self.journaltag
+ if params.has_key('created'):
+ journaldate = params['created'].serialise()
+ del params['created']
+ else:
+ journaldate = date.Date().serialise()
+ if params.has_key('activity'):
+ del params['activity']
+
+ # serialise the parameters now
+ if action in ('set', 'create'):
+ params = self.serialise(classname, params)
+ else:
+ journaltag = self.journaltag
+ journaldate = date.Date().serialise()
# create the journal entry
- entry = (nodeid, date.Date().get_tuple(), self.journaltag, action,
- params)
+ entry = (nodeid, journaldate, journaltag, action, params)
if __debug__:
print >>hyperdb.DEBUG, 'doSaveJournal', entry
_marker = []
class Class(hyperdb.Class):
- """The handle to a particular class of nodes in a hyperdatabase."""
+ '''The handle to a particular class of nodes in a hyperdatabase.'''
def __init__(self, db, classname, **properties):
- """Create a new class with a given name and property specification.
+ '''Create a new class with a given name and property specification.
'classname' must not collide with the name of an existing class,
or a ValueError is raised. The keyword arguments in 'properties'
must map names to property objects, or a TypeError is raised.
- """
+ '''
if (properties.has_key('creation') or properties.has_key('activity')
or properties.has_key('creator')):
raise ValueError, '"creation", "activity" and "creator" are '\
# Editing nodes:
def create(self, **propvalues):
- """Create a new node of this class and return its id.
+ '''Create a new node of this class and return its id.
The keyword arguments in 'propvalues' map property names to values.
These operations trigger detectors and can be vetoed. Attempts
to modify the "creation" or "activity" properties cause a KeyError.
- """
+ '''
if propvalues.has_key('id'):
raise KeyError, '"id" is reserved'
raise KeyError, '"%s" has no property "%s"'%(self.classname,
key)
- if isinstance(prop, Link):
+ if value is not None and isinstance(prop, Link):
if type(value) != type(''):
raise ValueError, 'link value must be String'
link_class = self.properties[key].classname
return newid
+ def export_list(self, propnames, nodeid):
+ ''' Export a node - generate a list of CSV-able data in the order
+ specified by propnames for the given node.
+ '''
+ properties = self.getprops()
+ l = []
+ for prop in propnames:
+ proptype = properties[prop]
+ value = self.get(nodeid, prop)
+ # "marshal" data where needed
+ if isinstance(proptype, hyperdb.Date):
+ value = value.get_tuple()
+ elif isinstance(proptype, hyperdb.Interval):
+ value = value.get_tuple()
+ elif isinstance(proptype, hyperdb.Password):
+ value = str(value)
+ l.append(repr(value))
+ return l
+
+ def import_list(self, propnames, proplist):
+ ''' Import a node - all information including "id" is present and
+ should not be sanity checked. Triggers are not triggered. The
+ journal should be initialised using the "creator" and "created"
+ information.
+
+ Return the nodeid of the node imported.
+ '''
+ if self.db.journaltag is None:
+ raise DatabaseError, 'Database open read-only'
+ properties = self.getprops()
+
+ # make the new node's property map
+ d = {}
+ for i in range(len(propnames)):
+ # Use eval to reverse the repr() used to output the CSV
+ value = eval(proplist[i])
+
+ # Figure the property for this column
+ propname = propnames[i]
+ prop = properties[propname]
+
+ # "unmarshal" where necessary
+ if propname == 'id':
+ newid = value
+ continue
+ elif isinstance(prop, hyperdb.Date):
+ value = date.Date(value)
+ elif isinstance(prop, hyperdb.Interval):
+ value = date.Interval(value)
+ elif isinstance(prop, hyperdb.Password):
+ pwd = password.Password()
+ pwd.unpack(value)
+ value = pwd
+ if value is not None:
+ d[propname] = value
+
+ # add
+ self.db.addnode(self.classname, newid, d)
+ self.db.addjournal(self.classname, newid, 'create', d)
+ return newid
+
def get(self, nodeid, propname, default=_marker, cache=1):
- """Get the value of a property on an existing node of this class.
+ '''Get the value of a property on an existing node of this class.
'nodeid' must be the id of an existing node of this class or an
IndexError is raised. 'propname' must be the name of a property
Attempts to get the "creation" or "activity" properties should
do the right thing.
- """
+ '''
if propname == 'id':
return nodeid
return Node(self, nodeid, cache=cache)
def set(self, nodeid, **propvalues):
- """Modify a property on an existing node of this class.
+ '''Modify a property on an existing node of this class.
'nodeid' must be the id of an existing node of this class or an
IndexError is raised.
These operations trigger detectors and can be vetoed. Attempts
to modify the "creation" or "activity" properties cause a KeyError.
- """
+ '''
if not propvalues:
- return
+ return propvalues
if propvalues.has_key('creation') or propvalues.has_key('activity'):
raise KeyError, '"creation" and "activity" are reserved'
# do stuff based on the prop type
if isinstance(prop, Link):
- link_class = self.properties[propname].classname
+ link_class = prop.classname
# if it isn't a number, it's a key
- if type(value) != type(''):
- raise ValueError, 'link value must be String'
- if not num_re.match(value):
+ if value is not None and not isinstance(value, type('')):
+ raise ValueError, 'property "%s" link value be a string'%(
+ propname)
+ if isinstance(value, type('')) and not num_re.match(value):
try:
value = self.db.classes[link_class].lookup(value)
except (TypeError, KeyError):
raise IndexError, 'new property "%s": %s not a %s'%(
- propname, value, self.properties[propname].classname)
+ propname, value, prop.classname)
- if not self.db.getclass(link_class).hasnode(value):
+ if (value is not None and
+ not self.db.getclass(link_class).hasnode(value)):
raise IndexError, '%s has no node %s'%(link_class, value)
- if self.do_journal and self.properties[propname].do_journal:
+ if self.do_journal and prop.do_journal:
# register the unlink with the old linked node
if node[propname] is not None:
self.db.addjournal(link_class, node[propname], 'unlink',
# figure the journal entry
l = []
if add:
- l.append(('add', add))
+ l.append(('+', add))
if remove:
- l.append(('remove', remove))
+ l.append(('-', remove))
if l:
journalvalues[propname] = tuple(l)
# nothing to do?
if not propvalues:
- return
+ return propvalues
# do the set, and journal it
self.db.setnode(self.classname, nodeid, node)
self.fireReactors('set', nodeid, oldvalues)
+ return propvalues
+
def retire(self, nodeid):
- """Retire a node.
+ '''Retire a node.
The properties on the node remain available from the get() method,
and the node's id is never reused.
These operations trigger detectors and can be vetoed. Attempts
to modify the "creation" or "activity" properties cause a KeyError.
- """
+ '''
if self.db.journaltag is None:
raise DatabaseError, 'Database open read-only'
self.fireReactors('retire', nodeid, None)
+ def is_retired(self, nodeid):
+ '''Return true if the node is retired.
+ '''
+ node = self.db.getnode(cn, nodeid, cldb)
+ if node.has_key(self.db.RETIRED_FLAG):
+ return 1
+ return 0
+
def destroy(self, nodeid):
- """Destroy a node.
+ '''Destroy a node.
WARNING: this method should never be used except in extremely rare
situations where there could never be links to the node being
Well, I think that's enough warnings. This method exists mostly to
support the session storage of the cgi interface.
- """
+ '''
if self.db.journaltag is None:
raise DatabaseError, 'Database open read-only'
self.db.destroynode(self.classname, nodeid)
def history(self, nodeid):
- """Retrieve the journal of edits on a particular node.
+ '''Retrieve the journal of edits on a particular node.
'nodeid' must be the id of an existing node of this class or an
IndexError is raised.
'date' is a Timestamp object specifying the time of the change and
'tag' is the journaltag specified when the database was opened.
- """
+ '''
if not self.do_journal:
raise ValueError, 'Journalling is disabled for this class'
return self.db.getjournal(self.classname, nodeid)
return self.db.hasnode(self.classname, nodeid)
def setkey(self, propname):
- """Select a String property of this class to be the key property.
+ '''Select a String property of this class to be the key property.
'propname' must be the name of a String property of this class or
None, or a TypeError is raised. The values of the key property on
all existing nodes must be unique or a ValueError is raised. If the
property doesn't exist, KeyError is raised.
- """
+ '''
prop = self.getprops()[propname]
if not isinstance(prop, String):
raise TypeError, 'key properties must be String'
self.key = propname
def getkey(self):
- """Return the name of the key property for this class or None."""
+ '''Return the name of the key property for this class or None.'''
return self.key
def labelprop(self, default_to_id=0):
# TODO: set up a separate index db file for this? profile?
def lookup(self, keyvalue):
- """Locate a particular node by its key property and return its id.
+ '''Locate a particular node by its key property and return its id.
If this class has no key property, a TypeError is raised. If the
'keyvalue' matches one of the values for the key property among
the nodes in this class, the matching node's id is returned;
otherwise a KeyError is raised.
- """
+ '''
+ if not self.key:
+ raise TypeError, 'No key property set'
cldb = self.db.getclassdb(self.classname)
try:
for nodeid in self.db.getnodeids(self.classname, cldb):
# XXX: change from spec - allows multiple props to match
def find(self, **propspec):
- """Get the ids of nodes in this class which link to the given nodes.
+ '''Get the ids of nodes in this class which link to the given nodes.
'propspec' consists of keyword args propname={nodeid:1,}
'propname' must be the name of a property in this class, or a
nodeids will be returned. Used by the full text indexing, which knows
that "foo" occurs in msg1, msg3 and file7, so we have hits on these issues:
db.issue.find(messages={'1':1,'3':1}, files={'7':1})
- """
+ '''
propspec = propspec.items()
for propname, nodeids in propspec:
# check the prop is OK
return l
def stringFind(self, **requirements):
- """Locate a particular node by matching a set of its String
+ '''Locate a particular node by matching a set of its String
properties in a caseless search.
If the property is not a String property, a TypeError is raised.
The return is a list of the id of all nodes that match.
- """
+ '''
for propname in requirements.keys():
prop = self.properties[propname]
if isinstance(not prop, String):
return l
def list(self):
- """Return a list of the ids of the active nodes in this class."""
+ ''' Return a list of the ids of the active nodes in this class.
+ '''
l = []
cn = self.classname
cldb = self.db.getclassdb(cn)
l.sort()
return l
- # XXX not in spec
def filter(self, search_matches, filterspec, sort, group,
num_re = re.compile('^\d+$')):
''' Return a list of the ids of the active nodes in this class that
match the 'filter' spec, sorted by the group spec and then the
- sort spec
+ sort spec.
+
+ "filterspec" is {propname: value(s)}
+ "sort" is ['+propname', '-propname', 'propname', ...]
+ "group is ['+propname', '-propname', 'propname', ...]
'''
cn = self.classname
# optimise filterspec
l = []
props = self.getprops()
+ LINK = 0
+ MULTILINK = 1
+ STRING = 2
+ OTHER = 6
for k, v in filterspec.items():
propclass = props[k]
if isinstance(propclass, Link):
k, entry, self.properties[k].classname)
u.append(entry)
- l.append((0, k, u))
+ l.append((LINK, k, u))
elif isinstance(propclass, Multilink):
if type(v) is not type([]):
v = [v]
raise ValueError, 'new property "%s": %s not a %s'%(
k, entry, self.properties[k].classname)
u.append(entry)
- l.append((1, k, u))
+ l.append((MULTILINK, k, u))
elif isinstance(propclass, String):
# simple glob searching
v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
v = v.replace('?', '.')
v = v.replace('*', '.*?')
- l.append((2, k, re.compile(v, re.I)))
+ l.append((STRING, k, re.compile(v, re.I)))
elif isinstance(propclass, Boolean):
if type(v) is type(''):
bv = v.lower() in ('yes', 'true', 'on', '1')
else:
bv = v
- l.append((6, k, bv))
+ l.append((OTHER, k, bv))
elif isinstance(propclass, Number):
- l.append((6, k, int(v)))
+ l.append((OTHER, k, int(v)))
else:
- l.append((6, k, v))
+ l.append((OTHER, k, v))
filterspec = l
# now, find all the nodes that are active and pass filtering
l = []
cldb = self.db.getclassdb(cn)
try:
+ # TODO: only full-scan once (use items())
for nodeid in self.db.getnodeids(cn, cldb):
node = self.db.getnode(cn, nodeid, cldb)
if node.has_key(self.db.RETIRED_FLAG):
continue
# apply filter
for t, k, v in filterspec:
- # this node doesn't have this property, so reject it
- if not node.has_key(k): break
+ # make sure the node has the property
+ if not node.has_key(k):
+ # this node doesn't have this property, so reject it
+ break
- if t == 0 and node[k] not in v:
- # link - if this node'd property doesn't appear in the
+ # now apply the property filter
+ if t == LINK:
+ # link - if this node's property doesn't appear in the
# filterspec's nodeid list, skip it
- break
- elif t == 1:
+ if node[k] not in v:
+ break
+ elif t == MULTILINK:
# multilink - if any of the nodeids required by the
# filterspec aren't in this node's property, then skip
# it
- for value in v:
- if value not in node[k]:
+ have = node[k]
+ for want in v:
+ if want not in have:
break
else:
continue
break
- elif t == 2 and (node[k] is None or not v.search(node[k])):
+ elif t == STRING:
# RE search
- break
- elif t == 6 and node[k] != v:
+ if node[k] is None or not v.search(node[k]):
+ break
+ elif t == OTHER:
# straight value comparison for the other types
- break
+ if node[k] != v:
+ break
else:
l.append((nodeid, node))
finally:
# filter based on full text search
if search_matches is not None:
k = []
- l_debug = []
for v in l:
- l_debug.append(v[0])
if search_matches.has_key(v[0]):
k.append(v)
l = k
return [i[0] for i in l]
def count(self):
- """Get the number of nodes in this class.
+ '''Get the number of nodes in this class.
If the returned integer is 'numnodes', the ids of all the nodes
in this class run from 1 to numnodes, and numnodes+1 will be the
id of the next node to be created in this class.
- """
+ '''
return self.db.countnodes(self.classname)
# Manipulating properties:
def getprops(self, protected=1):
- """Return a dictionary mapping property names to property objects.
+ '''Return a dictionary mapping property names to property objects.
If the "protected" flag is true, we include protected properties -
those which may not be modified.
methods provide the "creation" and "activity" properties. If the
"protected" flag is true, we include protected properties - those
which may not be modified.
- """
+ '''
d = self.properties.copy()
if protected:
d['id'] = String()
return d
def addprop(self, **properties):
- """Add properties to this class.
+ '''Add properties to this class.
The keyword arguments in 'properties' must map names to property
objects, or a TypeError is raised. None of the keys in 'properties'
may collide with the names of existing properties, or a ValueError
is raised before any properties have been added.
- """
+ '''
for key in properties.keys():
if self.properties.has_key(key):
raise ValueError, key
# Detector interface
#
def audit(self, event, detector):
- """Register a detector
- """
+ '''Register a detector
+ '''
l = self.auditors[event]
if detector not in l:
self.auditors[event].append(detector)
def fireAuditors(self, action, nodeid, newvalues):
- """Fire all registered auditors.
- """
+ '''Fire all registered auditors.
+ '''
for audit in self.auditors[action]:
audit(self.db, self, nodeid, newvalues)
def react(self, event, detector):
- """Register a detector
- """
+ '''Register a detector
+ '''
l = self.reactors[event]
if detector not in l:
self.reactors[event].append(detector)
def fireReactors(self, action, nodeid, oldvalues):
- """Fire all registered reactors.
- """
+ '''Fire all registered reactors.
+ '''
for react in self.reactors[action]:
react(self.db, self, nodeid, oldvalues)
self.db.storefile(self.classname, newid, None, content)
return newid
+ def import_list(self, propnames, proplist):
+ ''' Trap the "content" property...
+ '''
+ # dupe this list so we don't affect others
+ propnames = propnames[:]
+
+ # extract the "content" property from the proplist
+ i = propnames.index('content')
+ content = proplist[i]
+ del propnames[i]
+ del proplist[i]
+
+ # do the normal import
+ newid = Class.import_list(self, propnames, proplist)
+
+ # save off the "content" file
+ self.db.storefile(self.classname, newid, None, content)
+ return newid
+
def get(self, nodeid, propname, default=_marker, cache=1):
''' trap the content propname and get it from the file
'''
class IssueClass(Class, roundupdb.IssueClass):
# Overridden methods:
def __init__(self, db, classname, **properties):
- """The newly-created class automatically includes the "messages",
+ '''The newly-created class automatically includes the "messages",
"files", "nosy", and "superseder" properties. If the 'properties'
dictionary attempts to specify any of these properties or a
"creation" or "activity" property, a ValueError is raised.
- """
+ '''
if not properties.has_key('title'):
properties['title'] = hyperdb.String(indexme='yes')
if not properties.has_key('messages'):
#
#$Log: not supported by cvs2svn $
+#Revision 1.63 2002/08/22 04:42:28 richard
+#use more robust date stamp comparisons in pack(), make journal smaller too
+#
+#Revision 1.62 2002/08/21 07:07:27 richard
+#In preparing to turn back on link/unlink journal events (by default these
+#are turned off) I've:
+#- fixed back_anydbm so it can journal those events again (had broken it
+# with recent changes)
+#- changed the serialisation format for dates and intervals to use a
+# numbers-only (and sign for Intervals) string instead of tuple-of-ints.
+# Much smaller.
+#
+#Revision 1.61 2002/08/19 02:53:27 richard
+#full database export and import is done
+#
+#Revision 1.60 2002/08/19 00:23:19 richard
+#handle "unset" initial Link values (!)
+#
+#Revision 1.59 2002/08/16 04:28:13 richard
+#added is_retired query to Class
+#
+#Revision 1.58 2002/08/01 15:06:24 gmcm
+#Use same regex to split search terms as used to index text.
+#Fix to back_metakit for not changing journaltag on reopen.
+#Fix htmltemplate's do_link so [No <whatever>] strings are href'd.
+#Fix bogus "nosy edited ok" msg - the **d syntax does NOT share d between caller and callee.
+#
+#Revision 1.57 2002/07/31 23:57:36 richard
+# . web forms may now unset Link values (like assignedto)
+#
+#Revision 1.56 2002/07/31 22:04:33 richard
+#cleanup
+#
+#Revision 1.55 2002/07/30 08:22:38 richard
+#Session storage in the hyperdb was horribly, horribly inefficient. We use
+#a simple anydbm wrapper now - which could be overridden by the metakit
+#backend or RDB backend if necessary.
+#Much, much better.
+#
#Revision 1.54 2002/07/26 08:26:59 richard
#Very close now. The cgi and mailgw now use the new security API. The two
#templates have been migrated to that setup. Lots of unit tests. Still some