![[tokkee]](http://tokkee.org/images/avatar.png){kind=avatar}
index 195c8d06f5a35d5322c4b59fc62af89468bae7d9..933738dbc6d6a436c398502f4d574fb6fefc4a9c 100644 (file)
# BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
# SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
#
-#$Id: back_anydbm.py,v 1.45 2002-07-14 04:03:14 richard Exp $
+#$Id: back_anydbm.py,v 1.52 2002-07-19 03:36:34 richard Exp $
'''
This module defines a backend that saves the hyperdatabase in a database
chosen by anydbm. It is guaranteed to always be available in python
from roundup.indexer import Indexer
from locking import acquire_lock, release_lock
from roundup.hyperdb import String, Password, Date, Interval, Link, \
- Multilink, DatabaseError
+ Multilink, DatabaseError, Boolean, Number
#
# Now the database
self.cache = {} # cache of nodes loaded or created
self.dirtynodes = {} # keep track of the dirty nodes by class
self.newnodes = {} # keep track of the new nodes by class
+ self.destroyednodes = {}# keep track of the destroyed nodes by class
self.transactions = []
self.indexer = Indexer(self.dir)
# ensure files are group readable and writable
'''
if __debug__:
print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode)
- return self._opendb('nodes.%s'%classname, mode)
+ return self.opendb('nodes.%s'%classname, mode)
- def _opendb(self, name, mode):
- '''Low-level database opener that gets around anydbm/dbm
- eccentricities.
+ def determine_db_type(self, path):
+ ''' determine which DB wrote the class file
'''
- if __debug__:
- print >>hyperdb.DEBUG, '_opendb', (self, name, mode)
-
- # determine which DB wrote the class file
db_type = ''
- path = os.path.join(os.getcwd(), self.dir, name)
if os.path.exists(path):
db_type = whichdb.whichdb(path)
if not db_type:
# if the path ends in '.db', it's a dbm database, whether
# anydbm says it's dbhash or not!
db_type = 'dbm'
+ return db_type
+
+ def opendb(self, name, mode):
+ '''Low-level database opener that gets around anydbm/dbm
+ eccentricities.
+ '''
+ if __debug__:
+ print >>hyperdb.DEBUG, 'opendb', (self, name, mode)
+
+ # figure the class db type
+ path = os.path.join(os.getcwd(), self.dir, name)
+ db_type = self.determine_db_type(path)
# new database? let anydbm pick the best dbm
if not db_type:
if __debug__:
- print >>hyperdb.DEBUG, "_opendb anydbm.open(%r, 'n')"%path
+ print >>hyperdb.DEBUG, "opendb anydbm.open(%r, 'n')"%path
return anydbm.open(path, 'n')
# open the database with the correct module
"Couldn't open database - the required module '%s'"\
" is not available"%db_type
if __debug__:
- print >>hyperdb.DEBUG, "_opendb %r.open(%r, %r)"%(db_type, path,
+ print >>hyperdb.DEBUG, "opendb %r.open(%r, %r)"%(db_type, path,
mode)
return dbm.open(path, mode)
- def _lockdb(self, name):
+ def lockdb(self, name):
''' Lock a database file
'''
path = os.path.join(os.getcwd(), self.dir, '%s.lock'%name)
''' Generate a new id for the given class
'''
# open the ids DB - create if if doesn't exist
- lock = self._lockdb('_ids')
- db = self._opendb('_ids', 'c')
+ lock = self.lockdb('_ids')
+ db = self.opendb('_ids', 'c')
if db.has_key(classname):
newid = db[classname] = str(int(db[classname]) + 1)
else:
'''
if __debug__:
print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node)
- self.transactions.append((self._doSaveNode, (classname, nodeid, node)))
+ self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
def getnode(self, classname, nodeid, db=None, cache=1):
''' get a node from the database
if not db.has_key(nodeid):
raise IndexError, "no such %s %s"%(classname, nodeid)
+ # check the uncommitted, destroyed nodes
+ if (self.destroyednodes.has_key(classname) and
+ self.destroyednodes[classname].has_key(nodeid)):
+ raise IndexError, "no such %s %s"%(classname, nodeid)
+
# decode
res = marshal.loads(db[nodeid])
return res
+ def destroynode(self, classname, nodeid):
+ '''Remove a node from the database. Called exclusively by the
+ destroy() method on Class.
+ '''
+ if __debug__:
+ print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
+
+ # remove from cache and newnodes if it's there
+ if (self.cache.has_key(classname) and
+ self.cache[classname].has_key(nodeid)):
+ del self.cache[classname][nodeid]
+ if (self.newnodes.has_key(classname) and
+ self.newnodes[classname].has_key(nodeid)):
+ del self.newnodes[classname][nodeid]
+
+ # see if there's any obvious commit actions that we should get rid of
+ for entry in self.transactions[:]:
+ if entry[1][:2] == (classname, nodeid):
+ self.transactions.remove(entry)
+
+ # add to the destroyednodes map
+ self.destroyednodes.setdefault(classname, {})[nodeid] = 1
+
+ # add the destroy commit action
+ self.transactions.append((self.doDestroyNode, (classname, nodeid)))
+
def serialise(self, classname, node):
'''Copy the node contents, converting non-marshallable data into
marshallable data.
def countnodes(self, classname, db=None):
if __debug__:
print >>hyperdb.DEBUG, 'countnodes', (self, classname, db)
- # include the new nodes not saved to the DB yet
- count = len(self.newnodes.get(classname, {}))
+
+ count = 0
+
+ # include the uncommitted nodes
+ if self.newnodes.has_key(classname):
+ count += len(self.newnodes[classname])
+ if self.destroyednodes.has_key(classname):
+ count -= len(self.destroyednodes[classname])
# and count those in the DB
if db is None:
def getnodeids(self, classname, db=None):
if __debug__:
print >>hyperdb.DEBUG, 'getnodeids', (self, classname, db)
+
+ res = []
+
# start off with the new nodes
- res = self.newnodes.get(classname, {}).keys()
+ if self.newnodes.has_key(classname):
+ res += self.newnodes[classname].keys()
if db is None:
db = self.getclassdb(classname)
res = res + db.keys()
+
+ # remove the uncommitted, destroyed nodes
+ if self.destroyednodes.has_key(classname):
+ for nodeid in self.destroyednodes[classname].keys():
+ if db.has_key(nodeid):
+ res.remove(nodeid)
+
return res
if __debug__:
print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid,
action, params)
- self.transactions.append((self._doSaveJournal, (classname, nodeid,
+ self.transactions.append((self.doSaveJournal, (classname, nodeid,
action, params)))
def getjournal(self, classname, nodeid):
''' get the journal for id
+
+ Raise IndexError if the node doesn't exist (as per history()'s
+ API)
'''
if __debug__:
print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid)
# attempt to open the journal - in some rare cases, the journal may
# not exist
try:
- db = self._opendb('journals.%s'%classname, 'r')
+ db = self.opendb('journals.%s'%classname, 'r')
except anydbm.error, error:
- if str(error) == "need 'c' or 'n' flag to open new db": return []
- elif error.args[0] != 2: raise
- return []
+ if str(error) == "need 'c' or 'n' flag to open new db":
+ raise IndexError, 'no such %s %s'%(classname, nodeid)
+ elif error.args[0] != 2:
+ raise
+ raise IndexError, 'no such %s %s'%(classname, nodeid)
try:
journal = marshal.loads(db[nodeid])
except KeyError:
db.close()
- raise KeyError, 'no such %s %s'%(classname, nodeid)
+ raise IndexError, 'no such %s %s'%(classname, nodeid)
db.close()
res = []
- for entry in journal:
- (nodeid, date_stamp, user, action, params) = entry
- date_obj = date.Date(date_stamp)
- res.append((nodeid, date_obj, user, action, params))
+ for nodeid, date_stamp, user, action, params in journal:
+ res.append((nodeid, date.Date(date_stamp), user, action, params))
return res
def pack(self, pack_before):
classes = self.getclasses()
- # TODO: factor this out to method - we're already doing it in
- # _opendb.
- db_type = ''
- path = os.path.join(os.getcwd(), self.dir, classes[0])
- if os.path.exists(path):
- db_type = whichdb.whichdb(path)
- if not db_type:
- raise hyperdb.DatabaseError, "Couldn't identify database type"
- elif os.path.exists(path+'.db'):
- db_type = 'dbm'
+ # figure the class db type
for classname in classes:
db_name = 'journals.%s'%classname
- db = self._opendb(db_name, 'w')
+ path = os.path.join(os.getcwd(), self.dir, classname)
+ db_type = self.determine_db_type(path)
+ db = self.opendb(db_name, 'w')
for key in db.keys():
journal = marshal.loads(db[key])
self.cache = {}
self.dirtynodes = {}
self.newnodes = {}
+ self.destroyednodes = {}
self.transactions = []
- def _doSaveNode(self, classname, nodeid, node):
+ def getCachedClassDB(self, classname):
+ ''' get the class db, looking in our cache of databases for commit
+ '''
+ # get the database handle
+ db_name = 'nodes.%s'%classname
+ if not self.databases.has_key(db_name):
+ self.databases[db_name] = self.getclassdb(classname, 'c')
+ return self.databases[db_name]
+
+ def doSaveNode(self, classname, nodeid, node):
if __debug__:
- print >>hyperdb.DEBUG, '_doSaveNode', (self, classname, nodeid,
+ print >>hyperdb.DEBUG, 'doSaveNode', (self, classname, nodeid,
node)
- # get the database handle
- db_name = 'nodes.%s'%classname
- if self.databases.has_key(db_name):
- db = self.databases[db_name]
- else:
- db = self.databases[db_name] = self.getclassdb(classname, 'c')
+ db = self.getCachedClassDB(classname)
# now save the marshalled data
db[nodeid] = marshal.dumps(self.serialise(classname, node))
# return the classname, nodeid so we reindex this content
return (classname, nodeid)
- def _doSaveJournal(self, classname, nodeid, action, params):
+ def getCachedJournalDB(self, classname):
+ ''' get the journal db, looking in our cache of databases for commit
+ '''
+ # get the database handle
+ db_name = 'journals.%s'%classname
+ if not self.databases.has_key(db_name):
+ self.databases[db_name] = self.opendb(db_name, 'c')
+ return self.databases[db_name]
+
+ def doSaveJournal(self, classname, nodeid, action, params):
# serialise first
if action in ('set', 'create'):
params = self.serialise(classname, params)
params)
if __debug__:
- print >>hyperdb.DEBUG, '_doSaveJournal', entry
+ print >>hyperdb.DEBUG, 'doSaveJournal', entry
- # get the database handle
- db_name = 'journals.%s'%classname
- if self.databases.has_key(db_name):
- db = self.databases[db_name]
- else:
- db = self.databases[db_name] = self._opendb(db_name, 'c')
+ db = self.getCachedJournalDB(classname)
# now insert the journal entry
if db.has_key(nodeid):
db[nodeid] = marshal.dumps(l)
+ def doDestroyNode(self, classname, nodeid):
+ if __debug__:
+ print >>hyperdb.DEBUG, 'doDestroyNode', (self, classname, nodeid)
+
+ # delete from the class database
+ db = self.getCachedClassDB(classname)
+ if db.has_key(nodeid):
+ del db[nodeid]
+
+ # delete from the database
+ db = self.getCachedJournalDB(classname)
+ if db.has_key(nodeid):
+ del db[nodeid]
+
+ # return the classname, nodeid so we reindex this content
+ return (classname, nodeid)
+
def rollback(self):
''' Reverse all actions from the current transaction.
'''
print >>hyperdb.DEBUG, 'rollback', (self, )
for method, args in self.transactions:
# delete temporary files
- if method == self._doStoreFile:
- self._rollbackStoreFile(*args)
+ if method == self.doStoreFile:
+ self.rollbackStoreFile(*args)
self.cache = {}
self.dirtynodes = {}
self.newnodes = {}
+ self.destroyednodes = {}
self.transactions = []
_marker = []
if value is not None and not isinstance(value, date.Interval):
raise TypeError, 'new property "%s" not an Interval'%key
+ elif value is not None and isinstance(prop, Number):
+ try:
+ float(value)
+ except ValueError:
+ raise TypeError, 'new property "%s" not numeric'%key
+
+ elif value is not None and isinstance(prop, Boolean):
+ try:
+ int(value)
+ except ValueError:
+ raise TypeError, 'new property "%s" not boolean'%key
+
# make sure there's data where there needs to be
for key, prop in self.properties.items():
if propvalues.has_key(key):
if isinstance(prop, Multilink):
propvalues[key] = []
else:
- # TODO: None isn't right here, I think...
propvalues[key] = None
# done
if isinstance(prop, Multilink):
return []
else:
- # TODO: None isn't right here, I think...
return None
else:
return default
if node.has_key(self.db.RETIRED_FLAG):
raise IndexError
num_re = re.compile('^\d+$')
- for key, value in propvalues.items():
+
+ # if the journal value is to be different, store it in here
+ journalvalues = {}
+
+ for propname, value in propvalues.items():
# check to make sure we're not duplicating an existing key
- if key == self.key and node[key] != value:
+ if propname == self.key and node[propname] != value:
try:
self.lookup(value)
except KeyError:
# this will raise the KeyError if the property isn't valid
# ... we don't use getprops() here because we only care about
# the writeable properties.
- prop = self.properties[key]
+ prop = self.properties[propname]
# if the value's the same as the existing value, no sense in
# doing anything
- if node.has_key(key) and value == node[key]:
- del propvalues[key]
+ if node.has_key(propname) and value == node[propname]:
+ del propvalues[propname]
continue
# do stuff based on the prop type
if isinstance(prop, Link):
- link_class = self.properties[key].classname
+ link_class = self.properties[propname].classname
# if it isn't a number, it's a key
if type(value) != type(''):
raise ValueError, 'link value must be String'
value = self.db.classes[link_class].lookup(value)
except (TypeError, KeyError):
raise IndexError, 'new property "%s": %s not a %s'%(
- key, value, self.properties[key].classname)
+ propname, value, self.properties[propname].classname)
if not self.db.hasnode(link_class, value):
raise IndexError, '%s has no node %s'%(link_class, value)
- if self.do_journal and self.properties[key].do_journal:
+ if self.do_journal and self.properties[propname].do_journal:
# register the unlink with the old linked node
- if node[key] is not None:
- self.db.addjournal(link_class, node[key], 'unlink',
- (self.classname, nodeid, key))
+ if node[propname] is not None:
+ self.db.addjournal(link_class, node[propname], 'unlink',
+ (self.classname, nodeid, propname))
# register the link with the newly linked node
if value is not None:
self.db.addjournal(link_class, value, 'link',
- (self.classname, nodeid, key))
+ (self.classname, nodeid, propname))
elif isinstance(prop, Multilink):
if type(value) != type([]):
- raise TypeError, 'new property "%s" not a list of ids'%key
- link_class = self.properties[key].classname
+ raise TypeError, 'new property "%s" not a list of'\
+ ' ids'%propname
+ link_class = self.properties[propname].classname
l = []
for entry in value:
# if it isn't a number, it's a key
if type(entry) != type(''):
raise ValueError, 'new property "%s" link value ' \
- 'must be a string'%key
+ 'must be a string'%propname
if not num_re.match(entry):
try:
entry = self.db.classes[link_class].lookup(entry)
except (TypeError, KeyError):
raise IndexError, 'new property "%s": %s not a %s'%(
- key, entry, self.properties[key].classname)
+ propname, entry,
+ self.properties[propname].classname)
l.append(entry)
value = l
- propvalues[key] = value
+ propvalues[propname] = value
+
+ # figure the journal entry for this property
+ add = []
+ remove = []
# handle removals
- if node.has_key(key):
- l = node[key]
+ if node.has_key(propname):
+ l = node[propname]
else:
l = []
for id in l[:]:
if id in value:
continue
# register the unlink with the old linked node
- if self.do_journal and self.properties[key].do_journal:
+ if self.do_journal and self.properties[propname].do_journal:
self.db.addjournal(link_class, id, 'unlink',
- (self.classname, nodeid, key))
+ (self.classname, nodeid, propname))
l.remove(id)
+ remove.append(id)
# handle additions
for id in value:
if not self.db.hasnode(link_class, id):
- raise IndexError, '%s has no node %s'%(
- link_class, id)
+ raise IndexError, '%s has no node %s'%(link_class, id)
if id in l:
continue
# register the link with the newly linked node
- if self.do_journal and self.properties[key].do_journal:
+ if self.do_journal and self.properties[propname].do_journal:
self.db.addjournal(link_class, id, 'link',
- (self.classname, nodeid, key))
+ (self.classname, nodeid, propname))
l.append(id)
+ add.append(id)
+
+ # figure the journal entry
+ l = []
+ if add:
+ l.append(('add', add))
+ if remove:
+ l.append(('remove', remove))
+ if l:
+ journalvalues[propname] = tuple(l)
elif isinstance(prop, String):
if value is not None and type(value) != type(''):
- raise TypeError, 'new property "%s" not a string'%key
+ raise TypeError, 'new property "%s" not a string'%propname
elif isinstance(prop, Password):
if not isinstance(value, password.Password):
- raise TypeError, 'new property "%s" not a Password'% key
- propvalues[key] = value
+ raise TypeError, 'new property "%s" not a Password'%propname
+ propvalues[propname] = value
elif value is not None and isinstance(prop, Date):
if not isinstance(value, date.Date):
- raise TypeError, 'new property "%s" not a Date'% key
- propvalues[key] = value
+ raise TypeError, 'new property "%s" not a Date'% propname
+ propvalues[propname] = value
elif value is not None and isinstance(prop, Interval):
if not isinstance(value, date.Interval):
- raise TypeError, 'new property "%s" not an Interval'% key
- propvalues[key] = value
+ raise TypeError, 'new property "%s" not an '\
+ 'Interval'%propname
+ propvalues[propname] = value
- node[key] = value
+ elif value is not None and isinstance(prop, Number):
+ try:
+ float(value)
+ except ValueError:
+ raise TypeError, 'new property "%s" not numeric'%propname
+
+ elif value is not None and isinstance(prop, Boolean):
+ try:
+ int(value)
+ except ValueError:
+ raise TypeError, 'new property "%s" not boolean'%propname
+
+ node[propname] = value
# nothing to do?
if not propvalues:
# do the set, and journal it
self.db.setnode(self.classname, nodeid, node)
+
if self.do_journal:
+ propvalues.update(journalvalues)
self.db.addjournal(self.classname, nodeid, 'set', propvalues)
self.fireReactors('set', nodeid, oldvalues)
self.fireReactors('retire', nodeid, None)
+ def destroy(self, nodeid):
+ """Destroy a node.
+
+ WARNING: this method should never be used except in extremely rare
+ situations where there could never be links to the node being
+ deleted
+ WARNING: use retire() instead
+ WARNING: the properties of this node will not be available ever again
+ WARNING: really, use retire() instead
+
+ Well, I think that's enough warnings. This method exists mostly to
+ support the session storage of the cgi interface.
+ """
+ if self.db.journaltag is None:
+ raise DatabaseError, 'Database open read-only'
+ self.db.destroynode(self.classname, nodeid)
+
def history(self, nodeid):
"""Retrieve the journal of edits on a particular node.
'propname' must be the name of a String property of this class or
None, or a TypeError is raised. The values of the key property on
- all existing nodes must be unique or a ValueError is raised.
+ all existing nodes must be unique or a ValueError is raised. If the
+ property doesn't exist, KeyError is raised.
"""
- # TODO: validate that the property is a String!
+ prop = self.getprops()[propname]
+ if not isinstance(prop, String):
+ raise TypeError, 'key properties must be String'
self.key = propname
def getkey(self):
v = v.replace('?', '.')
v = v.replace('*', '.*?')
l.append((2, k, re.compile(v, re.I)))
+ elif isinstance(propclass, Boolean):
+ if type(v) is type(''):
+ bv = v.lower() in ('yes', 'true', 'on', '1')
+ else:
+ bv = v
+ l.append((6, k, bv))
+ elif isinstance(propclass, Number):
+ l.append((6, k, int(v)))
else:
l.append((6, k, v))
filterspec = l
elif dir == '-':
r = cmp(len(bv), len(av))
if r != 0: return r
+ elif isinstance(propclass, Number) or isinstance(propclass, Boolean):
+ if dir == '+':
+ r = cmp(av, bv)
+ elif dir == '-':
+ r = cmp(bv, av)
+
# end for dir, prop in list:
# end for list in sort, group:
# if all else fails, compare the ids
# find all the String properties that have indexme
for prop, propclass in self.getprops().items():
if isinstance(propclass, String) and propclass.indexme:
- # and index them under (classname, nodeid, property)
- self.db.indexer.add_text((self.classname, nodeid, prop),
- str(self.get(nodeid, prop)))
+ try:
+ value = str(self.get(nodeid, prop))
+ except IndexError:
+ # node no longer exists - entry should be removed
+ self.db.indexer.purge_entry((self.classname, nodeid, prop))
+ else:
+ # and index them under (classname, nodeid, property)
+ self.db.indexer.add_text((self.classname, nodeid, prop),
+ value)
#
# Detector interface
#
#$Log: not supported by cvs2svn $
+#Revision 1.51 2002/07/18 23:07:08 richard
+#Unit tests and a few fixes.
+#
+#Revision 1.50 2002/07/18 11:50:58 richard
+#added tests for number type too
+#
+#Revision 1.49 2002/07/18 11:41:10 richard
+#added tests for boolean type, and fixes to anydbm backend
+#
+#Revision 1.48 2002/07/18 11:17:31 gmcm
+#Add Number and Boolean types to hyperdb.
+#Add conversion cases to web, mail & admin interfaces.
+#Add storage/serialization cases to back_anydbm & back_metakit.
+#
+#Revision 1.47 2002/07/14 23:18:20 richard
+#. fixed the journal bloat from multilink changes - we just log the add or
+# remove operations, not the whole list
+#
+#Revision 1.46 2002/07/14 06:06:34 richard
+#Did some old TODOs
+#
+#Revision 1.45 2002/07/14 04:03:14 richard
+#Implemented a switch to disable journalling for a Class. CGI session
+#database now uses it.
+#
#Revision 1.44 2002/07/14 02:05:53 richard
#. all storage-specific code (ie. backend) is now implemented by the backends
#