![[tokkee]](http://tokkee.org/images/avatar.png){kind=avatar}
index 4d3ad98387e3298a1038f59f9f445b44cea6b642..845b1dc30aea7ce48b8ce0ecbd7040ce8ca67165 100644 (file)
# FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
# BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
# SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
-#
-#$Id: back_anydbm.py,v 1.136 2004-03-12 05:36:26 richard Exp $
-'''This module defines a backend that saves the hyperdatabase in a
+#
+"""This module defines a backend that saves the hyperdatabase in a
database chosen by anydbm. It is guaranteed to always be available in python
versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
serious bugs, and is not available)
-'''
+"""
__docformat__ = 'restructuredtext'
-try:
- import anydbm, sys
- # dumbdbm only works in python 2.1.2+
- if sys.version_info < (2,1,2):
- import dumbdbm
- assert anydbm._defaultmod != dumbdbm
- del dumbdbm
-except AssertionError:
- print "WARNING: you should upgrade to python 2.1.3"
-
-import whichdb, os, marshal, re, weakref, string, copy
-from roundup import hyperdb, date, password, roundupdb, security
-from blobfiles import FileStorage
-from sessions import Sessions, OneTimeKeys
-from roundup.indexer import Indexer
+import os, marshal, re, weakref, string, copy, time, shutil, logging
+
+from roundup.anypy.dbm_ import anydbm, whichdb, key_in
+
+from roundup import hyperdb, date, password, roundupdb, security, support
+from roundup.support import reversed
from roundup.backends import locking
-from roundup.hyperdb import String, Password, Date, Interval, Link, \
- Multilink, DatabaseError, Boolean, Number, Node
-from roundup.date import Range
+from roundup.i18n import _
+
+from roundup.backends.blobfiles import FileStorage
+from roundup.backends.sessions_dbm import Sessions, OneTimeKeys
+
+try:
+ from roundup.backends.indexer_xapian import Indexer
+except ImportError:
+ from roundup.backends.indexer_dbm import Indexer
+
+def db_exists(config):
+ # check for the user db
+ for db in 'nodes.user nodes.user.db'.split():
+ if os.path.exists(os.path.join(config.DATABASE, db)):
+ return 1
+ return 0
+
+def db_nuke(config):
+ shutil.rmtree(config.DATABASE)
+
+class Binary:
+
+ def __init__(self, x, y):
+ self.x = x
+ self.y = y
+
+ def visit(self, visitor):
+ self.x.visit(visitor)
+ self.y.visit(visitor)
+
+class Unary:
+
+ def __init__(self, x):
+ self.x = x
+
+ def generate(self, atom):
+ return atom(self)
+
+ def visit(self, visitor):
+ self.x.visit(visitor)
+
+class Equals(Unary):
+
+ def evaluate(self, v):
+ return self.x in v
+
+ def visit(self, visitor):
+ visitor(self)
+
+class Not(Unary):
+
+ def evaluate(self, v):
+ return not self.x.evaluate(v)
+
+ def generate(self, atom):
+ return "NOT(%s)" % self.x.generate(atom)
+
+class Or(Binary):
+
+ def evaluate(self, v):
+ return self.x.evaluate(v) or self.y.evaluate(v)
+
+ def generate(self, atom):
+ return "(%s)OR(%s)" % (
+ self.x.generate(atom),
+ self.y.generate(atom))
+
+class And(Binary):
+
+ def evaluate(self, v):
+ return self.x.evaluate(v) and self.y.evaluate(v)
+
+ def generate(self, atom):
+ return "(%s)AND(%s)" % (
+ self.x.generate(atom),
+ self.y.generate(atom))
+
+def compile_expression(opcodes):
+
+ stack = []
+ push, pop = stack.append, stack.pop
+ for opcode in opcodes:
+ if opcode == -2: push(Not(pop()))
+ elif opcode == -3: push(And(pop(), pop()))
+ elif opcode == -4: push(Or(pop(), pop()))
+ else: push(Equals(opcode))
+
+ return pop()
+
+class Expression:
+
+ def __init__(self, v):
+ try:
+ opcodes = [int(x) for x in v]
+ if min(opcodes) >= -1: raise ValueError()
+
+ compiled = compile_expression(opcodes)
+ self.evaluate = lambda x: compiled.evaluate([int(y) for y in x])
+ except:
+ self.evaluate = lambda x: bool(set(x) & set(v))
#
# Now the database
#
class Database(FileStorage, hyperdb.Database, roundupdb.Database):
- '''A database for storing records containing flexible data types.
+ """A database for storing records containing flexible data types.
Transaction stuff TODO:
-
+
- check the timestamp of the class file and nuke the cache if it's
modified. Do some sort of conflict checking on the dirty stuff.
- perhaps detect write collisions (related to above)?
- '''
+ """
def __init__(self, config, journaltag=None):
- '''Open a hyperdatabase given a specifier to some storage.
+ """Open a hyperdatabase given a specifier to some storage.
The 'storagelocator' is obtained from config.DATABASE.
The meaning of 'storagelocator' depends on the particular
entries for any edits done on the database. If 'journaltag' is
None, the database is opened in read-only mode: the Class.create(),
Class.set(), Class.retire(), and Class.restore() methods are
- disabled.
- '''
+ disabled.
+ """
+ FileStorage.__init__(self, config.UMASK)
self.config, self.journaltag = config, journaltag
self.dir = config.DATABASE
self.classes = {}
self.cache = {} # cache of nodes loaded or created
+ self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0,
+ 'filtering': 0}
self.dirtynodes = {} # keep track of the dirty nodes by class
self.newnodes = {} # keep track of the new nodes by class
self.destroyednodes = {}# keep track of the destroyed nodes by class
self.transactions = []
- self.indexer = Indexer(self.dir)
- self.sessions = Sessions(self.config)
- self.otks = OneTimeKeys(self.config)
+ self.indexer = Indexer(self)
self.security = security.Security(self)
- # ensure files are group readable and writable
- os.umask(0002)
+ os.umask(config.UMASK)
# lock it
lockfilenm = os.path.join(self.dir, 'lock')
self.lockfile.flush()
def post_init(self):
- '''Called once the schema initialisation has finished.
- '''
+ """Called once the schema initialisation has finished.
+ """
# reindex the db if necessary
if self.indexer.should_reindex():
self.reindex()
"""
self.reindex()
- def reindex(self):
- for klass in self.classes.values():
- for nodeid in klass.list():
- klass.index(nodeid)
+ def getSessionManager(self):
+ return Sessions(self)
+
+ def getOTKManager(self):
+ return OneTimeKeys(self)
+
+ def reindex(self, classname=None, show_progress=False):
+ if classname:
+ classes = [self.getclass(classname)]
+ else:
+ classes = self.classes.values()
+ for klass in classes:
+ if show_progress:
+ for nodeid in support.Progress('Reindex %s'%klass.classname,
+ klass.list()):
+ klass.index(nodeid)
+ else:
+ for nodeid in klass.list():
+ klass.index(nodeid)
self.indexer.save_index()
def __repr__(self):
- return '<back_anydbm instance at %x>'%id(self)
+ return '<back_anydbm instance at %x>'%id(self)
#
# Classes
#
def __getattr__(self, classname):
- '''A convenient way of calling self.getclass(classname).'''
- if self.classes.has_key(classname):
- if __debug__:
- print >>hyperdb.DEBUG, '__getattr__', (self, classname)
+ """A convenient way of calling self.getclass(classname)."""
+ if classname in self.classes:
return self.classes[classname]
raise AttributeError, classname
def addclass(self, cl):
- if __debug__:
- print >>hyperdb.DEBUG, 'addclass', (self, cl)
cn = cl.classname
- if self.classes.has_key(cn):
+ if cn in self.classes:
raise ValueError, cn
self.classes[cn] = cl
# add default Edit and View permissions
+ self.security.addPermission(name="Create", klass=cn,
+ description="User is allowed to create "+cn)
self.security.addPermission(name="Edit", klass=cn,
description="User is allowed to edit "+cn)
self.security.addPermission(name="View", klass=cn,
description="User is allowed to access "+cn)
+ self.security.addPermission(name="Retire", klass=cn,
+ description="User is allowed to retire "+cn)
def getclasses(self):
- '''Return a list of the names of all existing classes.'''
- if __debug__:
- print >>hyperdb.DEBUG, 'getclasses', (self,)
+ """Return a list of the names of all existing classes."""
l = self.classes.keys()
l.sort()
return l
def getclass(self, classname):
- '''Get the Class object representing a particular class.
+ """Get the Class object representing a particular class.
If 'classname' is not a valid class name, a KeyError is raised.
- '''
- if __debug__:
- print >>hyperdb.DEBUG, 'getclass', (self, classname)
+ """
try:
return self.classes[classname]
except KeyError:
- raise KeyError, 'There is no class called "%s"'%classname
+ raise KeyError('There is no class called "%s"'%classname)
#
# Class DBs
#
def clear(self):
- '''Delete all database contents
- '''
- if __debug__:
- print >>hyperdb.DEBUG, 'clear', (self,)
- for cn in self.classes.keys():
+ """Delete all database contents
+ """
+ logging.getLogger('roundup.hyperdb').info('clear')
+ for cn in self.classes:
for dummy in 'nodes', 'journals':
path = os.path.join(self.dir, 'journals.%s'%cn)
if os.path.exists(path):
os.remove(path)
elif os.path.exists(path+'.db'): # dbm appends .db
os.remove(path+'.db')
+ # reset id sequences
+ path = os.path.join(os.getcwd(), self.dir, '_ids')
+ if os.path.exists(path):
+ os.remove(path)
+ elif os.path.exists(path+'.db'): # dbm appends .db
+ os.remove(path+'.db')
def getclassdb(self, classname, mode='r'):
- ''' grab a connection to the class db that will be used for
+ """ grab a connection to the class db that will be used for
multiple actions
- '''
- if __debug__:
- print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode)
+ """
return self.opendb('nodes.%s'%classname, mode)
def determine_db_type(self, path):
- ''' determine which DB wrote the class file
- '''
+ """ determine which DB wrote the class file
+ """
db_type = ''
if os.path.exists(path):
- db_type = whichdb.whichdb(path)
+ db_type = whichdb(path)
if not db_type:
- raise DatabaseError, "Couldn't identify database type"
+ raise hyperdb.DatabaseError(_("Couldn't identify database type"))
elif os.path.exists(path+'.db'):
# if the path ends in '.db', it's a dbm database, whether
# anydbm says it's dbhash or not!
return db_type
def opendb(self, name, mode):
- '''Low-level database opener that gets around anydbm/dbm
+ """Low-level database opener that gets around anydbm/dbm
eccentricities.
- '''
- if __debug__:
- print >>hyperdb.DEBUG, 'opendb', (self, name, mode)
-
+ """
# figure the class db type
path = os.path.join(os.getcwd(), self.dir, name)
db_type = self.determine_db_type(path)
# new database? let anydbm pick the best dbm
- if not db_type:
+ # in Python 3+ the "dbm" ("anydbm" to us) module already uses the
+ # whichdb() function to do this
+ if not db_type or hasattr(anydbm, 'whichdb'):
if __debug__:
- print >>hyperdb.DEBUG, "opendb anydbm.open(%r, 'c')"%path
+ logging.getLogger('roundup.hyperdb').debug(
+ "opendb anydbm.open(%r, 'c')"%path)
return anydbm.open(path, 'c')
- # open the database with the correct module
+ # in Python <3 it anydbm was a little dumb so manually open the
+ # database with the correct module
try:
dbm = __import__(db_type)
except ImportError:
- raise DatabaseError, \
- "Couldn't open database - the required module '%s'"\
- " is not available"%db_type
+ raise hyperdb.DatabaseError(_("Couldn't open database - the "
+ "required module '%s' is not available")%db_type)
if __debug__:
- print >>hyperdb.DEBUG, "opendb %r.open(%r, %r)"%(db_type, path,
- mode)
+ logging.getLogger('roundup.hyperdb').debug(
+ "opendb %r.open(%r, %r)"%(db_type, path, mode))
return dbm.open(path, mode)
#
# Node IDs
#
def newid(self, classname):
- ''' Generate a new id for the given class
- '''
+ """ Generate a new id for the given class
+ """
# open the ids DB - create if if doesn't exist
db = self.opendb('_ids', 'c')
- if db.has_key(classname):
+ if key_in(db, classname):
newid = db[classname] = str(int(db[classname]) + 1)
else:
# the count() bit is transitional - older dbs won't start at 1
return newid
def setid(self, classname, setid):
- ''' Set the id counter: used during import of database
- '''
+ """ Set the id counter: used during import of database
+ """
# open the ids DB - create if if doesn't exist
db = self.opendb('_ids', 'c')
db[classname] = str(setid)
# Nodes
#
def addnode(self, classname, nodeid, node):
- ''' add the specified node to its class's db
- '''
- if __debug__:
- print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
-
+ """ add the specified node to its class's db
+ """
# we'll be supplied these props if we're doing an import
- if not node.has_key('creator'):
+ if 'creator' not in node:
# add in the "calculated" properties (dupe so we don't affect
# calling code's node assumptions)
node = node.copy()
node['creator'] = self.getuid()
+ node['actor'] = self.getuid()
node['creation'] = node['activity'] = date.Date()
self.newnodes.setdefault(classname, {})[nodeid] = 1
self.savenode(classname, nodeid, node)
def setnode(self, classname, nodeid, node):
- ''' change the specified node
- '''
- if __debug__:
- print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
+ """ change the specified node
+ """
self.dirtynodes.setdefault(classname, {})[nodeid] = 1
- # update the activity time (dupe so we don't affect
- # calling code's node assumptions)
- node = node.copy()
- node['activity'] = date.Date()
-
# can't set without having already loaded the node
self.cache[classname][nodeid] = node
self.savenode(classname, nodeid, node)
def savenode(self, classname, nodeid, node):
- ''' perform the saving of data specified by the set/addnode
- '''
+ """ perform the saving of data specified by the set/addnode
+ """
if __debug__:
- print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node)
+ logging.getLogger('roundup.hyperdb').debug(
+ 'save %s%s %r'%(classname, nodeid, node))
self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
def getnode(self, classname, nodeid, db=None, cache=1):
- ''' get a node from the database
+ """ get a node from the database
Note the "cache" parameter is not used, and exists purely for
backward compatibility!
- '''
- if __debug__:
- print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid, db)
-
+ """
# try the cache
cache_dict = self.cache.setdefault(classname, {})
- if cache_dict.has_key(nodeid):
+ if nodeid in cache_dict:
if __debug__:
- print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
- nodeid)
+ logging.getLogger('roundup.hyperdb').debug(
+ 'get %s%s cached'%(classname, nodeid))
+ self.stats['cache_hits'] += 1
return cache_dict[nodeid]
if __debug__:
- print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid)
+ self.stats['cache_misses'] += 1
+ start_t = time.time()
+ logging.getLogger('roundup.hyperdb').debug(
+ 'get %s%s'%(classname, nodeid))
# get from the database and save in the cache
if db is None:
db = self.getclassdb(classname)
- if not db.has_key(nodeid):
- raise IndexError, "no such %s %s"%(classname, nodeid)
+ if not key_in(db, nodeid):
+ raise IndexError("no such %s %s"%(classname, nodeid))
# check the uncommitted, destroyed nodes
- if (self.destroyednodes.has_key(classname) and
- self.destroyednodes[classname].has_key(nodeid)):
- raise IndexError, "no such %s %s"%(classname, nodeid)
+ if (classname in self.destroyednodes and
+ nodeid in self.destroyednodes[classname]):
+ raise IndexError("no such %s %s"%(classname, nodeid))
# decode
res = marshal.loads(db[nodeid])
if cache:
cache_dict[nodeid] = res
+ if __debug__:
+ self.stats['get_items'] += (time.time() - start_t)
+
return res
def destroynode(self, classname, nodeid):
- '''Remove a node from the database. Called exclusively by the
+ """Remove a node from the database. Called exclusively by the
destroy() method on Class.
- '''
- if __debug__:
- print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
+ """
+ logging.getLogger('roundup.hyperdb').info(
+ 'destroy %s%s'%(classname, nodeid))
# remove from cache and newnodes if it's there
- if (self.cache.has_key(classname) and
- self.cache[classname].has_key(nodeid)):
+ if (classname in self.cache and nodeid in self.cache[classname]):
del self.cache[classname][nodeid]
- if (self.newnodes.has_key(classname) and
- self.newnodes[classname].has_key(nodeid)):
+ if (classname in self.newnodes and nodeid in self.newnodes[classname]):
del self.newnodes[classname][nodeid]
# see if there's any obvious commit actions that we should get rid of
# add the destroy commit action
self.transactions.append((self.doDestroyNode, (classname, nodeid)))
+ self.transactions.append((FileStorage.destroy, (self, classname, nodeid)))
def serialise(self, classname, node):
- '''Copy the node contents, converting non-marshallable data into
+ """Copy the node contents, converting non-marshallable data into
marshallable data.
- '''
- if __debug__:
- print >>hyperdb.DEBUG, 'serialise', classname, node
+ """
properties = self.getclass(classname).getprops()
d = {}
- for k, v in node.items():
- # if the property doesn't exist, or is the "retired" flag then
- # it won't be in the properties dict
- if not properties.has_key(k):
+ for k, v in node.iteritems():
+ if k == self.RETIRED_FLAG:
d[k] = v
continue
+ # if the property doesn't exist then we really don't care
+ if k not in properties:
+ continue
+
# get the property spec
prop = properties[k]
- if isinstance(prop, Password) and v is not None:
+ if isinstance(prop, hyperdb.Password) and v is not None:
d[k] = str(v)
- elif isinstance(prop, Date) and v is not None:
+ elif isinstance(prop, hyperdb.Date) and v is not None:
d[k] = v.serialise()
- elif isinstance(prop, Interval) and v is not None:
+ elif isinstance(prop, hyperdb.Interval) and v is not None:
d[k] = v.serialise()
else:
d[k] = v
return d
def unserialise(self, classname, node):
- '''Decode the marshalled node data
- '''
- if __debug__:
- print >>hyperdb.DEBUG, 'unserialise', classname, node
+ """Decode the marshalled node data
+ """
properties = self.getclass(classname).getprops()
d = {}
- for k, v in node.items():
+ for k, v in node.iteritems():
# if the property doesn't exist, or is the "retired" flag then
# it won't be in the properties dict
- if not properties.has_key(k):
+ if k not in properties:
d[k] = v
continue
# get the property spec
prop = properties[k]
- if isinstance(prop, Date) and v is not None:
+ if isinstance(prop, hyperdb.Date) and v is not None:
d[k] = date.Date(v)
- elif isinstance(prop, Interval) and v is not None:
+ elif isinstance(prop, hyperdb.Interval) and v is not None:
d[k] = date.Interval(v)
- elif isinstance(prop, Password) and v is not None:
- p = password.Password()
- p.unpack(v)
- d[k] = p
+ elif isinstance(prop, hyperdb.Password) and v is not None:
+ d[k] = password.Password(encrypted=v)
else:
d[k] = v
return d
def hasnode(self, classname, nodeid, db=None):
- ''' determine if the database has a given node
- '''
- if __debug__:
- print >>hyperdb.DEBUG, 'hasnode', (self, classname, nodeid, db)
-
+ """ determine if the database has a given node
+ """
# try the cache
cache = self.cache.setdefault(classname, {})
- if cache.has_key(nodeid):
- if __debug__:
- print >>hyperdb.TRACE, 'has %s %s cached'%(classname, nodeid)
+ if nodeid in cache:
return 1
- if __debug__:
- print >>hyperdb.TRACE, 'has %s %s'%(classname, nodeid)
# not in the cache - check the database
if db is None:
db = self.getclassdb(classname)
- res = db.has_key(nodeid)
- return res
+ return key_in(db, nodeid)
def countnodes(self, classname, db=None):
- if __debug__:
- print >>hyperdb.DEBUG, 'countnodes', (self, classname, db)
-
count = 0
# include the uncommitted nodes
- if self.newnodes.has_key(classname):
+ if classname in self.newnodes:
count += len(self.newnodes[classname])
- if self.destroyednodes.has_key(classname):
+ if classname in self.destroyednodes:
count -= len(self.destroyednodes[classname])
# and count those in the DB
if db is None:
db = self.getclassdb(classname)
- count = count + len(db.keys())
- return count
+ return count + len(db)
#
#
def addjournal(self, classname, nodeid, action, params, creator=None,
creation=None):
- ''' Journal the Action
+ """ Journal the Action
'action' may be:
'create' or 'set' -- 'params' is a dictionary of property values
'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
'retire' -- 'params' is None
- '''
+
+ 'creator' -- the user performing the action, which defaults to
+ the current user.
+ """
if __debug__:
- print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid,
- action, params, creator, creation)
+ logging.getLogger('roundup.hyperdb').debug(
+ 'addjournal %s%s %s %r %s %r'%(classname,
+ nodeid, action, params, creator, creation))
+ if creator is None:
+ creator = self.getuid()
self.transactions.append((self.doSaveJournal, (classname, nodeid,
action, params, creator, creation)))
+ def setjournal(self, classname, nodeid, journal):
+ """Set the journal to the "journal" list."""
+ if __debug__:
+ logging.getLogger('roundup.hyperdb').debug(
+ 'setjournal %s%s %r'%(classname, nodeid, journal))
+ self.transactions.append((self.doSetJournal, (classname, nodeid,
+ journal)))
+
+ def fix_journal(self, classname, journal):
+ """ fix password entries to correct type """
+ pwprops = {}
+ for pn, prop in self.getclass(classname).properties.iteritems():
+ if isinstance(prop, hyperdb.Password):
+ pwprops [pn] = 1
+ if not pwprops:
+ return journal
+ for j in journal:
+ if j[3] == 'set':
+ for k, v in j[4].items():
+ if k in pwprops and j[4][k]:
+ j[4][k] = password.JournalPassword(j[4][k])
+ return journal
+
def getjournal(self, classname, nodeid):
- ''' get the journal for id
+ """ get the journal for id
Raise IndexError if the node doesn't exist (as per history()'s
API)
- '''
- if __debug__:
- print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid)
-
+ """
# our journal result
res = []
db = self.opendb('journals.%s'%classname, 'r')
except anydbm.error, error:
if str(error) == "need 'c' or 'n' flag to open new db":
- raise IndexError, 'no such %s %s'%(classname, nodeid)
+ raise IndexError('no such %s %s'%(classname, nodeid))
elif error.args[0] != 2:
# this isn't a "not found" error, be alarmed!
raise
if res:
# we have unsaved journal entries, return them
- return res
- raise IndexError, 'no such %s %s'%(classname, nodeid)
+ return self.fix_journal (classname, res)
+ raise IndexError('no such %s %s'%(classname, nodeid))
try:
journal = marshal.loads(db[nodeid])
except KeyError:
db.close()
if res:
# we have some unsaved journal entries, be happy!
- return res
- raise IndexError, 'no such %s %s'%(classname, nodeid)
+ return self.fix_journal (classname, res)
+ raise IndexError('no such %s %s'%(classname, nodeid))
db.close()
# add all the saved journal entries for this node
for nodeid, date_stamp, user, action, params in journal:
res.append((nodeid, date.Date(date_stamp), user, action, params))
- return res
+ return self.fix_journal (classname, res)
def pack(self, pack_before):
- ''' Delete all journal entries except "create" before 'pack_before'.
- '''
- if __debug__:
- print >>hyperdb.DEBUG, 'packjournal', (self, pack_before)
-
+ """ Delete all journal entries except "create" before 'pack_before'.
+ """
pack_before = pack_before.serialise()
for classname in self.getclasses():
+ packed = 0
# get the journal db
db_name = 'journals.%s'%classname
path = os.path.join(os.getcwd(), self.dir, classname)
last_set_entry = None
for entry in journal:
# unpack the entry
- (nodeid, date_stamp, self.journaltag, action,
+ (nodeid, date_stamp, self.journaltag, action,
params) = entry
# if the entry is after the pack date, _or_ the initial
# create entry, then it stays
if date_stamp > pack_before or action == 'create':
l.append(entry)
+ else:
+ packed += 1
db[key] = marshal.dumps(l)
+
+ logging.getLogger('roundup.hyperdb').info(
+ 'packed %d %s items'%(packed, classname))
+
if db_type == 'gdbm':
db.reorganize()
db.close()
-
+
#
# Basic transaction support
#
- def commit(self):
- ''' Commit the current transactions.
- '''
- if __debug__:
- print >>hyperdb.DEBUG, 'commit', (self,)
+ def commit(self, fail_ok=False):
+ """ Commit the current transactions.
+
+ Save all data changed since the database was opened or since the
+ last commit() or rollback().
+
+ fail_ok indicates that the commit is allowed to fail. This is used
+ in the web interface when committing cleaning of the session
+ database. We don't care if there's a concurrency issue there.
+
+ The only backend this seems to affect is postgres.
+ """
+ logging.getLogger('roundup.hyperdb').info('commit %s transactions'%(
+ len(self.transactions)))
# keep a handle to all the database files opened
self.databases = {}
reindex[method(*args)] = 1
finally:
# make sure we close all the database files
- for db in self.databases.values():
+ for db in self.databases.itervalues():
db.close()
del self.databases
+ # clear the transactions list now so the blobfile implementation
+ # doesn't think there's still pending file commits when it tries
+ # to access the file data
+ self.transactions = []
+
# reindex the nodes that request it
- for classname, nodeid in filter(None, reindex.keys()):
- print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
+ for classname, nodeid in [k for k in reindex if k]:
self.getclass(classname).index(nodeid)
# save the indexer state
self.transactions = []
def getCachedClassDB(self, classname):
- ''' get the class db, looking in our cache of databases for commit
- '''
+ """ get the class db, looking in our cache of databases for commit
+ """
# get the database handle
db_name = 'nodes.%s'%classname
- if not self.databases.has_key(db_name):
+ if db_name not in self.databases:
self.databases[db_name] = self.getclassdb(classname, 'c')
return self.databases[db_name]
def doSaveNode(self, classname, nodeid, node):
- if __debug__:
- print >>hyperdb.DEBUG, 'doSaveNode', (self, classname, nodeid,
- node)
-
db = self.getCachedClassDB(classname)
# now save the marshalled data
return (classname, nodeid)
def getCachedJournalDB(self, classname):
- ''' get the journal db, looking in our cache of databases for commit
- '''
+ """ get the journal db, looking in our cache of databases for commit
+ """
# get the database handle
db_name = 'journals.%s'%classname
- if not self.databases.has_key(db_name):
+ if db_name not in self.databases:
self.databases[db_name] = self.opendb(db_name, 'c')
return self.databases[db_name]
# handle supply of the special journalling parameters (usually
# supplied on importing an existing database)
- if creator:
- journaltag = creator
- else:
- journaltag = self.getuid()
+ journaltag = creator
if creation:
journaldate = creation.serialise()
else:
# create the journal entry
entry = (nodeid, journaldate, journaltag, action, params)
- if __debug__:
- print >>hyperdb.DEBUG, 'doSaveJournal', entry
-
db = self.getCachedJournalDB(classname)
# now insert the journal entry
- if db.has_key(nodeid):
+ if key_in(db, nodeid):
# append to existing
s = db[nodeid]
l = marshal.loads(s)
db[nodeid] = marshal.dumps(l)
- def doDestroyNode(self, classname, nodeid):
- if __debug__:
- print >>hyperdb.DEBUG, 'doDestroyNode', (self, classname, nodeid)
+ def doSetJournal(self, classname, nodeid, journal):
+ l = []
+ for nodeid, journaldate, journaltag, action, params in journal:
+ # serialise the parameters now if necessary
+ if isinstance(params, type({})):
+ if action in ('set', 'create'):
+ params = self.serialise(classname, params)
+ journaldate = journaldate.serialise()
+ l.append((nodeid, journaldate, journaltag, action, params))
+ db = self.getCachedJournalDB(classname)
+ db[nodeid] = marshal.dumps(l)
+ def doDestroyNode(self, classname, nodeid):
# delete from the class database
db = self.getCachedClassDB(classname)
- if db.has_key(nodeid):
+ if key_in(db, nodeid):
del db[nodeid]
# delete from the database
db = self.getCachedJournalDB(classname)
- if db.has_key(nodeid):
+ if key_in(db, nodeid):
del db[nodeid]
- # return the classname, nodeid so we reindex this content
- return (classname, nodeid)
-
def rollback(self):
- ''' Reverse all actions from the current transaction.
- '''
- if __debug__:
- print >>hyperdb.DEBUG, 'rollback', (self, )
+ """ Reverse all actions from the current transaction.
+ """
+ logging.getLogger('roundup.hyperdb').info('rollback %s transactions'%(
+ len(self.transactions)))
+
for method, args in self.transactions:
# delete temporary files
if method == self.doStoreFile:
self.transactions = []
def close(self):
- ''' Nothing to do
- '''
+ """ Nothing to do
+ """
if self.lockfile is not None:
locking.release_lock(self.lockfile)
- if self.lockfile is not None:
self.lockfile.close()
self.lockfile = None
_marker = []
class Class(hyperdb.Class):
- '''The handle to a particular class of nodes in a hyperdatabase.'''
-
- def __init__(self, db, classname, **properties):
- '''Create a new class with a given name and property specification.
-
- 'classname' must not collide with the name of an existing class,
- or a ValueError is raised. The keyword arguments in 'properties'
- must map names to property objects, or a TypeError is raised.
- '''
- if (properties.has_key('creation') or properties.has_key('activity')
- or properties.has_key('creator')):
- raise ValueError, '"creation", "activity" and "creator" are '\
- 'reserved'
-
- self.classname = classname
- self.properties = properties
- self.db = weakref.proxy(db) # use a weak ref to avoid circularity
- self.key = ''
-
- # should we journal changes (default yes)
- self.do_journal = 1
-
- # do the db-related init stuff
- db.addclass(self)
-
- self.auditors = {'create': [], 'set': [], 'retire': [], 'restore': []}
- self.reactors = {'create': [], 'set': [], 'retire': [], 'restore': []}
+ """The handle to a particular class of nodes in a hyperdatabase."""
def enableJournalling(self):
- '''Turn journalling on for this class
- '''
+ """Turn journalling on for this class
+ """
self.do_journal = 1
def disableJournalling(self):
- '''Turn journalling off for this class
- '''
+ """Turn journalling off for this class
+ """
self.do_journal = 0
# Editing nodes:
def create(self, **propvalues):
- '''Create a new node of this class and return its id.
+ """Create a new node of this class and return its id.
The keyword arguments in 'propvalues' map property names to values.
The values of arguments must be acceptable for the types of their
corresponding properties or a TypeError is raised.
-
+
If this class has a key property, it must be present and its value
must not collide with other key strings or a ValueError is raised.
-
+
Any other properties on this class that are missing from the
'propvalues' dictionary are set to None.
-
+
If an id in a link or multilink property does not refer to a valid
node, an IndexError is raised.
These operations trigger detectors and can be vetoed. Attempts
to modify the "creation" or "activity" properties cause a KeyError.
- '''
+ """
+ if self.db.journaltag is None:
+ raise hyperdb.DatabaseError(_('Database open read-only'))
self.fireAuditors('create', None, propvalues)
newid = self.create_inner(**propvalues)
self.fireReactors('create', newid, None)
return newid
def create_inner(self, **propvalues):
- ''' Called by create, in-between the audit and react calls.
- '''
- if propvalues.has_key('id'):
- raise KeyError, '"id" is reserved'
+ """ Called by create, in-between the audit and react calls.
+ """
+ if 'id' in propvalues:
+ raise KeyError('"id" is reserved')
if self.db.journaltag is None:
- raise DatabaseError, 'Database open read-only'
+ raise hyperdb.DatabaseError(_('Database open read-only'))
- if propvalues.has_key('creation') or propvalues.has_key('activity'):
- raise KeyError, '"creation" and "activity" are reserved'
+ if 'creation' in propvalues or 'activity' in propvalues:
+ raise KeyError('"creation" and "activity" are reserved')
# new node's id
newid = self.db.newid(self.classname)
# validate propvalues
num_re = re.compile('^\d+$')
- for key, value in propvalues.items():
+ for key, value in propvalues.iteritems():
if key == self.key:
try:
self.lookup(value)
except KeyError:
pass
else:
- raise ValueError, 'node with key "%s" exists'%value
+ raise ValueError('node with key "%s" exists'%value)
# try to handle this property
try:
prop = self.properties[key]
except KeyError:
- raise KeyError, '"%s" has no property "%s"'%(self.classname,
- key)
+ raise KeyError('"%s" has no property "%s"'%(self.classname,
+ key))
- if value is not None and isinstance(prop, Link):
+ if value is not None and isinstance(prop, hyperdb.Link):
if type(value) != type(''):
- raise ValueError, 'link value must be String'
+ raise ValueError('link value must be String')
link_class = self.properties[key].classname
# if it isn't a number, it's a key
if not num_re.match(value):
try:
value = self.db.classes[link_class].lookup(value)
except (TypeError, KeyError):
- raise IndexError, 'new property "%s": %s not a %s'%(
- key, value, link_class)
+ raise IndexError('new property "%s": %s not a %s'%(
+ key, value, link_class))
elif not self.db.getclass(link_class).hasnode(value):
- raise IndexError, '%s has no node %s'%(link_class, value)
+ raise IndexError('%s has no node %s'%(link_class,
+ value))
# save off the value
propvalues[key] = value
self.db.addjournal(link_class, value, 'link',
(self.classname, newid, key))
- elif isinstance(prop, Multilink):
- if type(value) != type([]):
- raise TypeError, 'new property "%s" not a list of ids'%key
+ elif isinstance(prop, hyperdb.Multilink):
+ if value is None:
+ value = []
+ if not hasattr(value, '__iter__'):
+ raise TypeError('new property "%s" not an iterable of ids'%key)
# clean up and validate the list of links
link_class = self.properties[key].classname
l = []
for entry in value:
if type(entry) != type(''):
- raise ValueError, '"%s" multilink value (%r) '\
- 'must contain Strings'%(key, value)
+ raise ValueError('"%s" multilink value (%r) '\
+ 'must contain Strings'%(key, value))
# if it isn't a number, it's a key
if not num_re.match(entry):
try:
entry = self.db.classes[link_class].lookup(entry)
except (TypeError, KeyError):
- raise IndexError, 'new property "%s": %s not a %s'%(
- key, entry, self.properties[key].classname)
+ raise IndexError('new property "%s": %s not a %s'%(
+ key, entry, self.properties[key].classname))
l.append(entry)
value = l
propvalues[key] = value
# handle additions
for nodeid in value:
if not self.db.getclass(link_class).hasnode(nodeid):
- raise IndexError, '%s has no node %s'%(link_class,
- nodeid)
+ raise IndexError('%s has no node %s'%(link_class,
+ nodeid))
# register the link with the newly linked node
if self.do_journal and self.properties[key].do_journal:
self.db.addjournal(link_class, nodeid, 'link',
(self.classname, newid, key))
- elif isinstance(prop, String):
+ elif isinstance(prop, hyperdb.String):
if type(value) != type('') and type(value) != type(u''):
- raise TypeError, 'new property "%s" not a string'%key
+ raise TypeError('new property "%s" not a string'%key)
+ if prop.indexme:
+ self.db.indexer.add_text((self.classname, newid, key),
+ value)
- elif isinstance(prop, Password):
+ elif isinstance(prop, hyperdb.Password):
if not isinstance(value, password.Password):
- raise TypeError, 'new property "%s" not a Password'%key
+ raise TypeError('new property "%s" not a Password'%key)
- elif isinstance(prop, Date):
+ elif isinstance(prop, hyperdb.Date):
if value is not None and not isinstance(value, date.Date):
- raise TypeError, 'new property "%s" not a Date'%key
+ raise TypeError('new property "%s" not a Date'%key)
- elif isinstance(prop, Interval):
+ elif isinstance(prop, hyperdb.Interval):
if value is not None and not isinstance(value, date.Interval):
- raise TypeError, 'new property "%s" not an Interval'%key
+ raise TypeError('new property "%s" not an Interval'%key)
- elif value is not None and isinstance(prop, Number):
+ elif value is not None and isinstance(prop, hyperdb.Number):
try:
float(value)
except ValueError:
- raise TypeError, 'new property "%s" not numeric'%key
+ raise TypeError('new property "%s" not numeric'%key)
- elif value is not None and isinstance(prop, Boolean):
+ elif value is not None and isinstance(prop, hyperdb.Boolean):
try:
int(value)
except ValueError:
- raise TypeError, 'new property "%s" not boolean'%key
+ raise TypeError('new property "%s" not boolean'%key)
# make sure there's data where there needs to be
- for key, prop in self.properties.items():
- if propvalues.has_key(key):
+ for key, prop in self.properties.iteritems():
+ if key in propvalues:
continue
if key == self.key:
- raise ValueError, 'key property "%s" is required'%key
- if isinstance(prop, Multilink):
+ raise ValueError('key property "%s" is required'%key)
+ if isinstance(prop, hyperdb.Multilink):
propvalues[key] = []
- else:
- propvalues[key] = None
# done
self.db.addnode(self.classname, newid, propvalues)
return newid
- def export_list(self, propnames, nodeid):
- ''' Export a node - generate a list of CSV-able data in the order
- specified by propnames for the given node.
- '''
- properties = self.getprops()
- l = []
- for prop in propnames:
- proptype = properties[prop]
- value = self.get(nodeid, prop)
- # "marshal" data where needed
- if value is None:
- pass
- elif isinstance(proptype, hyperdb.Date):
- value = value.get_tuple()
- elif isinstance(proptype, hyperdb.Interval):
- value = value.get_tuple()
- elif isinstance(proptype, hyperdb.Password):
- value = str(value)
- l.append(repr(value))
-
- # append retired flag
- l.append(repr(self.is_retired(nodeid)))
-
- return l
-
- def import_list(self, propnames, proplist):
- ''' Import a node - all information including "id" is present and
- should not be sanity checked. Triggers are not triggered. The
- journal should be initialised using the "creator" and "created"
- information.
-
- Return the nodeid of the node imported.
- '''
- if self.db.journaltag is None:
- raise DatabaseError, 'Database open read-only'
- properties = self.getprops()
-
- # make the new node's property map
- d = {}
- newid = None
- for i in range(len(propnames)):
- # Figure the property for this column
- propname = propnames[i]
-
- # Use eval to reverse the repr() used to output the CSV
- value = eval(proplist[i])
-
- # "unmarshal" where necessary
- if propname == 'id':
- newid = value
- continue
- elif propname == 'is retired':
- # is the item retired?
- if int(value):
- d[self.db.RETIRED_FLAG] = 1
- continue
- elif value is None:
- d[propname] = None
- continue
-
- prop = properties[propname]
- if isinstance(prop, hyperdb.Date):
- value = date.Date(value)
- elif isinstance(prop, hyperdb.Interval):
- value = date.Interval(value)
- elif isinstance(prop, hyperdb.Password):
- pwd = password.Password()
- pwd.unpack(value)
- value = pwd
- d[propname] = value
-
- # get a new id if necessary
- if newid is None:
- newid = self.db.newid(self.classname)
-
- # add the node and journal
- self.db.addnode(self.classname, newid, d)
-
- # extract the journalling stuff and nuke it
- if d.has_key('creator'):
- creator = d['creator']
- del d['creator']
- else:
- creator = None
- if d.has_key('creation'):
- creation = d['creation']
- del d['creation']
- else:
- creation = None
- if d.has_key('activity'):
- del d['activity']
- self.db.addjournal(self.classname, newid, 'create', {}, creator,
- creation)
- return newid
-
def get(self, nodeid, propname, default=_marker, cache=1):
- '''Get the value of a property on an existing node of this class.
+ """Get the value of a property on an existing node of this class.
'nodeid' must be the id of an existing node of this class or an
IndexError is raised. 'propname' must be the name of a property
Attempts to get the "creation" or "activity" properties should
do the right thing.
- '''
+ """
if propname == 'id':
return nodeid
# check for one of the special props
if propname == 'creation':
- if d.has_key('creation'):
+ if 'creation' in d:
return d['creation']
if not self.do_journal:
- raise ValueError, 'Journalling is disabled for this class'
+ raise ValueError('Journalling is disabled for this class')
journal = self.db.getjournal(self.classname, nodeid)
if journal:
- return self.db.getjournal(self.classname, nodeid)[0][1]
+ return journal[0][1]
else:
# on the strange chance that there's no journal
return date.Date()
if propname == 'activity':
- if d.has_key('activity'):
+ if 'activity' in d:
return d['activity']
if not self.do_journal:
- raise ValueError, 'Journalling is disabled for this class'
+ raise ValueError('Journalling is disabled for this class')
journal = self.db.getjournal(self.classname, nodeid)
if journal:
return self.db.getjournal(self.classname, nodeid)[-1][1]
# on the strange chance that there's no journal
return date.Date()
if propname == 'creator':
- if d.has_key('creator'):
+ if 'creator' in d:
return d['creator']
if not self.do_journal:
- raise ValueError, 'Journalling is disabled for this class'
+ raise ValueError('Journalling is disabled for this class')
journal = self.db.getjournal(self.classname, nodeid)
if journal:
num_re = re.compile('^\d+$')
- value = self.db.getjournal(self.classname, nodeid)[0][2]
+ value = journal[0][2]
+ if num_re.match(value):
+ return value
+ else:
+ # old-style "username" journal tag
+ try:
+ return self.db.user.lookup(value)
+ except KeyError:
+ # user's been retired, return admin
+ return '1'
+ else:
+ return self.db.getuid()
+ if propname == 'actor':
+ if 'actor' in d:
+ return d['actor']
+ if not self.do_journal:
+ raise ValueError('Journalling is disabled for this class')
+ journal = self.db.getjournal(self.classname, nodeid)
+ if journal:
+ num_re = re.compile('^\d+$')
+ value = journal[-1][2]
if num_re.match(value):
return value
else:
# get the property (raises KeyErorr if invalid)
prop = self.properties[propname]
- if not d.has_key(propname):
+ if propname not in d:
if default is _marker:
- if isinstance(prop, Multilink):
+ if isinstance(prop, hyperdb.Multilink):
return []
else:
return None
return default
# return a dupe of the list so code doesn't get confused
- if isinstance(prop, Multilink):
+ if isinstance(prop, hyperdb.Multilink):
return d[propname][:]
return d[propname]
def set(self, nodeid, **propvalues):
- '''Modify a property on an existing node of this class.
-
+ """Modify a property on an existing node of this class.
+
'nodeid' must be the id of an existing node of this class or an
IndexError is raised.
These operations trigger detectors and can be vetoed. Attempts
to modify the "creation" or "activity" properties cause a KeyError.
- '''
+ """
+ if self.db.journaltag is None:
+ raise hyperdb.DatabaseError(_('Database open read-only'))
+
+ self.fireAuditors('set', nodeid, propvalues)
+ oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
+ for name, prop in self.getprops(protected=0).iteritems():
+ if name in oldvalues:
+ continue
+ if isinstance(prop, hyperdb.Multilink):
+ oldvalues[name] = []
+ else:
+ oldvalues[name] = None
+ propvalues = self.set_inner(nodeid, **propvalues)
+ self.fireReactors('set', nodeid, oldvalues)
+ return propvalues
+
+ def set_inner(self, nodeid, **propvalues):
+ """ Called by set, in-between the audit and react calls.
+ """
if not propvalues:
return propvalues
- if propvalues.has_key('creation') or propvalues.has_key('activity'):
+ if 'creation' in propvalues or 'activity' in propvalues:
raise KeyError, '"creation" and "activity" are reserved'
- if propvalues.has_key('id'):
+ if 'id' in propvalues:
raise KeyError, '"id" is reserved'
if self.db.journaltag is None:
- raise DatabaseError, 'Database open read-only'
-
- self.fireAuditors('set', nodeid, propvalues)
- # Take a copy of the node dict so that the subsequent set
- # operation doesn't modify the oldvalues structure.
- oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
+ raise hyperdb.DatabaseError(_('Database open read-only'))
node = self.db.getnode(self.classname, nodeid)
- if node.has_key(self.db.RETIRED_FLAG):
+ if self.db.RETIRED_FLAG in node:
raise IndexError
num_re = re.compile('^\d+$')
# if the journal value is to be different, store it in here
journalvalues = {}
- for propname, value in propvalues.items():
+ # list() propvalues 'cos it might be modified by the loop
+ for propname, value in list(propvalues.items()):
# check to make sure we're not duplicating an existing key
if propname == self.key and node[propname] != value:
try:
except KeyError:
pass
else:
- raise ValueError, 'node with key "%s" exists'%value
+ raise ValueError('node with key "%s" exists'%value)
# this will raise the KeyError if the property isn't valid
# ... we don't use getprops() here because we only care about
try:
prop = self.properties[propname]
except KeyError:
- raise KeyError, '"%s" has no property named "%s"'%(
- self.classname, propname)
+ raise KeyError('"%s" has no property named "%s"'%(
+ self.classname, propname))
# if the value's the same as the existing value, no sense in
# doing anything
journalvalues[propname] = current
# do stuff based on the prop type
- if isinstance(prop, Link):
+ if isinstance(prop, hyperdb.Link):
link_class = prop.classname
# if it isn't a number, it's a key
if value is not None and not isinstance(value, type('')):
- raise ValueError, 'property "%s" link value be a string'%(
- propname)
+ raise ValueError('property "%s" link value be a string'%(
+ propname))
if isinstance(value, type('')) and not num_re.match(value):
try:
value = self.db.classes[link_class].lookup(value)
except (TypeError, KeyError):
- raise IndexError, 'new property "%s": %s not a %s'%(
- propname, value, prop.classname)
+ raise IndexError('new property "%s": %s not a %s'%(
+ propname, value, prop.classname))
if (value is not None and
not self.db.getclass(link_class).hasnode(value)):
- raise IndexError, '%s has no node %s'%(link_class, value)
+ raise IndexError('%s has no node %s'%(link_class,
+ value))
if self.do_journal and prop.do_journal:
# register the unlink with the old linked node
- if node.has_key(propname) and node[propname] is not None:
+ if propname in node and node[propname] is not None:
self.db.addjournal(link_class, node[propname], 'unlink',
(self.classname, nodeid, propname))
self.db.addjournal(link_class, value, 'link',
(self.classname, nodeid, propname))
- elif isinstance(prop, Multilink):
- if type(value) != type([]):
- raise TypeError, 'new property "%s" not a list of'\
- ' ids'%propname
+ elif isinstance(prop, hyperdb.Multilink):
+ if value is None:
+ value = []
+ if not hasattr(value, '__iter__'):
+ raise TypeError('new property "%s" not an iterable of'
+ ' ids'%propname)
link_class = self.properties[propname].classname
l = []
for entry in value:
# if it isn't a number, it's a key
if type(entry) != type(''):
- raise ValueError, 'new property "%s" link value ' \
- 'must be a string'%propname
+ raise ValueError('new property "%s" link value '
+ 'must be a string'%propname)
if not num_re.match(entry):
try:
entry = self.db.classes[link_class].lookup(entry)
except (TypeError, KeyError):
- raise IndexError, 'new property "%s": %s not a %s'%(
+ raise IndexError('new property "%s": %s not a %s'%(
propname, entry,
- self.properties[propname].classname)
+ self.properties[propname].classname))
l.append(entry)
value = l
propvalues[propname] = value
remove = []
# handle removals
- if node.has_key(propname):
+ if propname in node:
l = node[propname]
else:
l = []
# handle additions
for id in value:
if not self.db.getclass(link_class).hasnode(id):
- raise IndexError, '%s has no node %s'%(link_class, id)
+ raise IndexError('%s has no node %s'%(link_class,
+ id))
if id in l:
continue
# register the link with the newly linked node
if l:
journalvalues[propname] = tuple(l)
- elif isinstance(prop, String):
+ elif isinstance(prop, hyperdb.String):
if value is not None and type(value) != type('') and type(value) != type(u''):
- raise TypeError, 'new property "%s" not a string'%propname
+ raise TypeError('new property "%s" not a '
+ 'string'%propname)
+ if prop.indexme:
+ self.db.indexer.add_text((self.classname, nodeid, propname),
+ value)
- elif isinstance(prop, Password):
+ elif isinstance(prop, hyperdb.Password):
if not isinstance(value, password.Password):
- raise TypeError, 'new property "%s" not a Password'%propname
+ raise TypeError('new property "%s" not a '
+ 'Password'%propname)
propvalues[propname] = value
+ journalvalues[propname] = \
+ current and password.JournalPassword(current)
- elif value is not None and isinstance(prop, Date):
+ elif value is not None and isinstance(prop, hyperdb.Date):
if not isinstance(value, date.Date):
- raise TypeError, 'new property "%s" not a Date'% propname
+ raise TypeError('new property "%s" not a '
+ 'Date'%propname)
propvalues[propname] = value
- elif value is not None and isinstance(prop, Interval):
+ elif value is not None and isinstance(prop, hyperdb.Interval):
if not isinstance(value, date.Interval):
- raise TypeError, 'new property "%s" not an '\
- 'Interval'%propname
+ raise TypeError('new property "%s" not an '
+ 'Interval'%propname)
propvalues[propname] = value
- elif value is not None and isinstance(prop, Number):
+ elif value is not None and isinstance(prop, hyperdb.Number):
try:
float(value)
except ValueError:
- raise TypeError, 'new property "%s" not numeric'%propname
+ raise TypeError('new property "%s" not '
+ 'numeric'%propname)
- elif value is not None and isinstance(prop, Boolean):
+ elif value is not None and isinstance(prop, hyperdb.Boolean):
try:
int(value)
except ValueError:
- raise TypeError, 'new property "%s" not boolean'%propname
+ raise TypeError('new property "%s" not '
+ 'boolean'%propname)
node[propname] = value
if not propvalues:
return propvalues
+ # update the activity time
+ node['activity'] = date.Date()
+ node['actor'] = self.db.getuid()
+
# do the set, and journal it
self.db.setnode(self.classname, nodeid, node)
if self.do_journal:
self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
- self.fireReactors('set', nodeid, oldvalues)
-
- return propvalues
+ return propvalues
def retire(self, nodeid):
- '''Retire a node.
-
+ """Retire a node.
+
The properties on the node remain available from the get() method,
and the node's id is never reused.
-
+
Retired nodes are not returned by the find(), list(), or lookup()
methods, and other nodes may reuse the values of their key properties.
These operations trigger detectors and can be vetoed. Attempts
to modify the "creation" or "activity" properties cause a KeyError.
- '''
+ """
if self.db.journaltag is None:
- raise DatabaseError, 'Database open read-only'
+ raise hyperdb.DatabaseError(_('Database open read-only'))
self.fireAuditors('retire', nodeid, None)
self.fireReactors('retire', nodeid, None)
def restore(self, nodeid):
- '''Restpre a retired node.
+ """Restpre a retired node.
Make node available for all operations like it was before retirement.
- '''
+ """
if self.db.journaltag is None:
- raise DatabaseError, 'Database open read-only'
+ raise hyperdb.DatabaseError(_('Database open read-only'))
node = self.db.getnode(self.classname, nodeid)
# check if key property was overrided
except KeyError:
pass
else:
- raise KeyError, "Key property (%s) of retired node clashes with \
- existing one (%s)" % (key, node[key])
+ raise KeyError("Key property (%s) of retired node clashes "
+ "with existing one (%s)" % (key, node[key]))
# Now we can safely restore node
self.fireAuditors('restore', nodeid, None)
del node[self.db.RETIRED_FLAG]
self.fireReactors('restore', nodeid, None)
def is_retired(self, nodeid, cldb=None):
- '''Return true if the node is retired.
- '''
+ """Return true if the node is retired.
+ """
node = self.db.getnode(self.classname, nodeid, cldb)
- if node.has_key(self.db.RETIRED_FLAG):
+ if self.db.RETIRED_FLAG in node:
return 1
return 0
def destroy(self, nodeid):
- '''Destroy a node.
+ """Destroy a node.
WARNING: this method should never be used except in extremely rare
situations where there could never be links to the node being
Well, I think that's enough warnings. This method exists mostly to
support the session storage of the cgi interface.
- '''
+ """
if self.db.journaltag is None:
- raise DatabaseError, 'Database open read-only'
+ raise hyperdb.DatabaseError(_('Database open read-only'))
self.db.destroynode(self.classname, nodeid)
- def history(self, nodeid):
- '''Retrieve the journal of edits on a particular node.
-
- 'nodeid' must be the id of an existing node of this class or an
- IndexError is raised.
-
- The returned list contains tuples of the form
-
- (nodeid, date, tag, action, params)
-
- 'date' is a Timestamp object specifying the time of the change and
- 'tag' is the journaltag specified when the database was opened.
- '''
- if not self.do_journal:
- raise ValueError, 'Journalling is disabled for this class'
- return self.db.getjournal(self.classname, nodeid)
-
# Locating nodes:
def hasnode(self, nodeid):
- '''Determine if the given nodeid actually exists
- '''
+ """Determine if the given nodeid actually exists
+ """
return self.db.hasnode(self.classname, nodeid)
def setkey(self, propname):
- '''Select a String property of this class to be the key property.
+ """Select a String property of this class to be the key property.
'propname' must be the name of a String property of this class or
None, or a TypeError is raised. The values of the key property on
all existing nodes must be unique or a ValueError is raised. If the
property doesn't exist, KeyError is raised.
- '''
+ """
prop = self.getprops()[propname]
- if not isinstance(prop, String):
- raise TypeError, 'key properties must be String'
+ if not isinstance(prop, hyperdb.String):
+ raise TypeError('key properties must be String')
self.key = propname
def getkey(self):
- '''Return the name of the key property for this class or None.'''
+ """Return the name of the key property for this class or None."""
return self.key
- def labelprop(self, default_to_id=0):
- '''Return the property name for a label for the given node.
-
- This method attempts to generate a consistent label for the node.
- It tries the following in order:
-
- 1. key property
- 2. "name" property
- 3. "title" property
- 4. first property from the sorted property name list
- '''
- k = self.getkey()
- if k:
- return k
- props = self.getprops()
- if props.has_key('name'):
- return 'name'
- elif props.has_key('title'):
- return 'title'
- if default_to_id:
- return 'id'
- props = props.keys()
- props.sort()
- return props[0]
-
# TODO: set up a separate index db file for this? profile?
def lookup(self, keyvalue):
- '''Locate a particular node by its key property and return its id.
+ """Locate a particular node by its key property and return its id.
If this class has no key property, a TypeError is raised. If the
'keyvalue' matches one of the values for the key property among
the nodes in this class, the matching node's id is returned;
otherwise a KeyError is raised.
- '''
+ """
if not self.key:
- raise TypeError, 'No key property set for class %s'%self.classname
+ raise TypeError('No key property set for '
+ 'class %s'%self.classname)
cldb = self.db.getclassdb(self.classname)
try:
for nodeid in self.getnodeids(cldb):
node = self.db.getnode(self.classname, nodeid, cldb)
- if node.has_key(self.db.RETIRED_FLAG):
+ if self.db.RETIRED_FLAG in node:
+ continue
+ if self.key not in node:
continue
if node[self.key] == keyvalue:
return nodeid
finally:
cldb.close()
- raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
- keyvalue, self.classname)
+ raise KeyError('No key (%s) value "%s" for "%s"'%(self.key,
+ keyvalue, self.classname))
# change from spec - allows multiple props to match
def find(self, **propspec):
- '''Get the ids of items in this class which link to the given items.
+ """Get the ids of nodes in this class which link to the given nodes.
- 'propspec' consists of keyword args propname=itemid or
- propname={itemid:1, }
+ 'propspec' consists of keyword args propname=nodeid or
+ propname={nodeid:1, }
'propname' must be the name of a property in this class, or a
KeyError is raised. That property must be a Link or
Multilink property, or a TypeError is raised.
- Any item in this class whose 'propname' property links to any of the
- itemids will be returned. Used by the full text indexing, which knows
- that "foo" occurs in msg1, msg3 and file7, so we have hits on these
- issues:
+ Any node in this class whose 'propname' property links to any of
+ the nodeids will be returned. Examples::
+ db.issue.find(messages='1')
db.issue.find(messages={'1':1,'3':1}, files={'7':1})
- '''
- propspec = propspec.items()
- for propname, itemids in propspec:
+ """
+ for propname, itemids in propspec.iteritems():
# check the prop is OK
prop = self.properties[propname]
- if not isinstance(prop, Link) and not isinstance(prop, Multilink):
- raise TypeError, "'%s' not a Link/Multilink property"%propname
+ if not isinstance(prop, hyperdb.Link) and not isinstance(prop, hyperdb.Multilink):
+ raise TypeError("'%s' not a Link/Multilink "
+ "property"%propname)
# ok, now do the find
cldb = self.db.getclassdb(self.classname)
try:
for id in self.getnodeids(db=cldb):
item = self.db.getnode(self.classname, id, db=cldb)
- if item.has_key(self.db.RETIRED_FLAG):
+ if self.db.RETIRED_FLAG in item:
continue
- for propname, itemids in propspec:
- # can't test if the item doesn't have this property
- if not item.has_key(propname):
- continue
+ for propname, itemids in propspec.iteritems():
if type(itemids) is not type({}):
itemids = {itemids:1}
+ # special case if the item doesn't have this property
+ if propname not in item:
+ if None in itemids:
+ l.append(id)
+ break
+ continue
+
# grab the property definition and its value on this item
prop = self.properties[propname]
value = item[propname]
- if isinstance(prop, Link) and itemids.has_key(value):
+ if isinstance(prop, hyperdb.Link) and value in itemids:
l.append(id)
break
- elif isinstance(prop, Multilink):
+ elif isinstance(prop, hyperdb.Multilink):
hit = 0
for v in value:
- if itemids.has_key(v):
+ if v in itemids:
l.append(id)
hit = 1
break
return l
def stringFind(self, **requirements):
- '''Locate a particular node by matching a set of its String
+ """Locate a particular node by matching a set of its String
properties in a caseless search.
If the property is not a String property, a TypeError is raised.
-
+
The return is a list of the id of all nodes that match.
- '''
- for propname in requirements.keys():
+ """
+ for propname in requirements:
prop = self.properties[propname]
- if not isinstance(prop, String):
- raise TypeError, "'%s' not a String property"%propname
+ if not isinstance(prop, hyperdb.String):
+ raise TypeError("'%s' not a String property"%propname)
requirements[propname] = requirements[propname].lower()
l = []
cldb = self.db.getclassdb(self.classname)
try:
for nodeid in self.getnodeids(cldb):
node = self.db.getnode(self.classname, nodeid, cldb)
- if node.has_key(self.db.RETIRED_FLAG):
+ if self.db.RETIRED_FLAG in node:
continue
- for key, value in requirements.items():
- if not node.has_key(key):
+ for key, value in requirements.iteritems():
+ if key not in node:
break
if node[key] is None or node[key].lower() != value:
break
return l
def list(self):
- ''' Return a list of the ids of the active nodes in this class.
- '''
+ """ Return a list of the ids of the active nodes in this class.
+ """
l = []
cn = self.classname
cldb = self.db.getclassdb(cn)
try:
for nodeid in self.getnodeids(cldb):
node = self.db.getnode(cn, nodeid, cldb)
- if node.has_key(self.db.RETIRED_FLAG):
+ if self.db.RETIRED_FLAG in node:
continue
l.append(nodeid)
finally:
l.sort()
return l
- def getnodeids(self, db=None):
- ''' Return a list of ALL nodeids
- '''
- if __debug__:
- print >>hyperdb.DEBUG, 'getnodeids', (self, self.classname, db)
+ def getnodeids(self, db=None, retired=None):
+ """ Return a list of ALL nodeids
+ Set retired=None to get all nodes. Otherwise it'll get all the
+ retired or non-retired nodes, depending on the flag.
+ """
res = []
# start off with the new nodes
- if self.db.newnodes.has_key(self.classname):
- res += self.db.newnodes[self.classname].keys()
+ if self.classname in self.db.newnodes:
+ res.extend(self.db.newnodes[self.classname])
+ must_close = False
if db is None:
db = self.db.getclassdb(self.classname)
- res = res + db.keys()
+ must_close = True
+ try:
+ res.extend(db.keys())
- # remove the uncommitted, destroyed nodes
- if self.db.destroyednodes.has_key(self.classname):
- for nodeid in self.db.destroyednodes[self.classname].keys():
- if db.has_key(nodeid):
- res.remove(nodeid)
+ # remove the uncommitted, destroyed nodes
+ if self.classname in self.db.destroyednodes:
+ for nodeid in self.db.destroyednodes[self.classname]:
+ if key_in(db, nodeid):
+ res.remove(nodeid)
+ # check retired flag
+ if retired is False or retired is True:
+ l = []
+ for nodeid in res:
+ node = self.db.getnode(self.classname, nodeid, db)
+ is_ret = self.db.RETIRED_FLAG in node
+ if retired == is_ret:
+ l.append(nodeid)
+ res = l
+ finally:
+ if must_close:
+ db.close()
return res
- def filter(self, search_matches, filterspec, sort=(None,None),
- group=(None,None), num_re = re.compile('^\d+$')):
+ def _filter(self, search_matches, filterspec, proptree,
+ num_re = re.compile('^\d+$')):
"""Return a list of the ids of the active nodes in this class that
match the 'filter' spec, sorted by the group spec and then the
sort spec.
"sort" and "group" are (dir, prop) where dir is '+', '-' or None
and prop is a prop name or None
- "search_matches" is {nodeid: marker}
+ "search_matches" is a sequence type or None
- The filter must match all properties specificed - but if the
- property value to match is a list, any one of the values in the
- list may match for that property to match. Unless the property
- is a Multilink, in which case the item's property list must
- match the filterspec list.
+ The filter must match all properties specificed. If the property
+ value to match is a list:
+
+ 1. String properties must match all elements in the list, and
+ 2. Other properties must match any of the elements in the list.
"""
+ if __debug__:
+ start_t = time.time()
+
cn = self.classname
# optimise filterspec
l = []
props = self.getprops()
- LINK = 0
- MULTILINK = 1
- STRING = 2
- DATE = 3
- INTERVAL = 4
- OTHER = 6
-
- timezone = self.db.getUserTimezone()
- for k, v in filterspec.items():
+ LINK = 'spec:link'
+ MULTILINK = 'spec:multilink'
+ STRING = 'spec:string'
+ DATE = 'spec:date'
+ INTERVAL = 'spec:interval'
+ OTHER = 'spec:other'
+
+ for k, v in filterspec.iteritems():
propclass = props[k]
- if isinstance(propclass, Link):
+ if isinstance(propclass, hyperdb.Link):
if type(v) is not type([]):
v = [v]
u = []
entry = None
u.append(entry)
l.append((LINK, k, u))
- elif isinstance(propclass, Multilink):
+ elif isinstance(propclass, hyperdb.Multilink):
# the value -1 is a special "not set" sentinel
if v in ('-1', ['-1']):
v = []
elif type(v) is not type([]):
v = [v]
l.append((MULTILINK, k, v))
- elif isinstance(propclass, String) and k != 'id':
+ elif isinstance(propclass, hyperdb.String) and k != 'id':
if type(v) is not type([]):
v = [v]
- m = []
for v in v:
# simple glob searching
v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
v = v.replace('?', '.')
v = v.replace('*', '.*?')
- m.append(v)
- m = re.compile('(%s)'%('|'.join(m)), re.I)
- l.append((STRING, k, m))
- elif isinstance(propclass, Date):
+ l.append((STRING, k, re.compile(v, re.I)))
+ elif isinstance(propclass, hyperdb.Date):
try:
- date_rng = Range(v, date.Date, offset=timezone)
+ date_rng = propclass.range_from_raw(v, self.db)
l.append((DATE, k, date_rng))
except ValueError:
# If range creation fails - ignore that search parameter
pass
- elif isinstance(propclass, Interval):
+ elif isinstance(propclass, hyperdb.Interval):
try:
- intv_rng = Range(v, date.Interval)
+ intv_rng = date.Range(v, date.Interval)
l.append((INTERVAL, k, intv_rng))
except ValueError:
# If range creation fails - ignore that search parameter
pass
-
- elif isinstance(propclass, Boolean):
- if type(v) is type(''):
- bv = v.lower() in ('yes', 'true', 'on', '1')
- else:
- bv = v
+
+ elif isinstance(propclass, hyperdb.Boolean):
+ if type(v) == type(""):
+ v = v.split(',')
+ if type(v) != type([]):
+ v = [v]
+ bv = []
+ for val in v:
+ if type(val) is type(''):
+ bv.append(propclass.from_raw (val))
+ else:
+ bv.append(val)
l.append((OTHER, k, bv))
- elif isinstance(propclass, Number):
- l.append((OTHER, k, int(v)))
- else:
- l.append((OTHER, k, v))
+
+ elif k == 'id':
+ if type(v) != type([]):
+ v = v.split(',')
+ l.append((OTHER, k, [str(int(val)) for val in v]))
+
+ elif isinstance(propclass, hyperdb.Number):
+ if type(v) != type([]):
+ try :
+ v = v.split(',')
+ except AttributeError :
+ v = [v]
+ l.append((OTHER, k, [float(val) for val in v]))
+
filterspec = l
# now, find all the nodes that are active and pass filtering
- l = []
+ matches = []
cldb = self.db.getclassdb(cn)
+ t = 0
try:
# TODO: only full-scan once (use items())
for nodeid in self.getnodeids(cldb):
node = self.db.getnode(cn, nodeid, cldb)
- if node.has_key(self.db.RETIRED_FLAG):
+ if self.db.RETIRED_FLAG in node:
continue
# apply filter
for t, k, v in filterspec:
# handle the id prop
- if k == 'id' and v == nodeid:
+ if k == 'id':
+ if nodeid not in v:
+ break
continue
- # make sure the node has the property
- if not node.has_key(k):
- # this node doesn't have this property, so reject it
- break
+ # get the node value
+ nv = node.get(k, None)
+
+ match = 0
# now apply the property filter
if t == LINK:
# link - if this node's property doesn't appear in the
# filterspec's nodeid list, skip it
- if node[k] not in v:
- break
+ match = nv in v
elif t == MULTILINK:
# multilink - if any of the nodeids required by the
# filterspec aren't in this node's property, then skip
# it
- have = node[k]
- # check for matching the absence of multilink values
- if not v and have:
- break
+ nv = node.get(k, [])
- # othewise, make sure this node has each of the
- # required values
- for want in v:
- if want not in have:
- break
+ # check for matching the absence of multilink values
+ if not v:
+ match = not nv
else:
- continue
- break
+ # otherwise, make sure this node has each of the
+ # required values
+ expr = Expression(v)
+ if expr.evaluate(nv): match = 1
elif t == STRING:
- if node[k] is None:
- break
+ if nv is None:
+ nv = ''
# RE search
- if not v.search(node[k]):
- break
+ match = v.search(nv)
elif t == DATE or t == INTERVAL:
- if node[k] is None:
- break
- if v.to_value:
- if not (v.from_value <= node[k] and v.to_value >= node[k]):
- break
+ if nv is None:
+ match = v is None
else:
- if not (v.from_value <= node[k]):
- break
+ if v.to_value:
+ if v.from_value <= nv and v.to_value >= nv:
+ match = 1
+ else:
+ if v.from_value <= nv:
+ match = 1
elif t == OTHER:
# straight value comparison for the other types
- if node[k] != v:
- break
+ match = nv in v
+ if not match:
+ break
else:
- l.append((nodeid, node))
+ matches.append([nodeid, node])
+
+ # filter based on full text search
+ if search_matches is not None:
+ k = []
+ for v in matches:
+ if v[0] in search_matches:
+ k.append(v)
+ matches = k
+
+ # add sorting information to the proptree
+ JPROPS = {'actor':1, 'activity':1, 'creator':1, 'creation':1}
+ children = []
+ if proptree:
+ children = proptree.sortable_children()
+ for pt in children:
+ dir = pt.sort_direction
+ prop = pt.name
+ assert (dir and prop)
+ propclass = props[prop]
+ pt.sort_ids = []
+ is_pointer = isinstance(propclass,(hyperdb.Link,
+ hyperdb.Multilink))
+ if not is_pointer:
+ pt.sort_result = []
+ try:
+ # cache the opened link class db, if needed.
+ lcldb = None
+ # cache the linked class items too
+ lcache = {}
+
+ for entry in matches:
+ itemid = entry[-2]
+ item = entry[-1]
+ # handle the properties that might be "faked"
+ # also, handle possible missing properties
+ try:
+ v = item[prop]
+ except KeyError:
+ if prop in JPROPS:
+ # force lookup of the special journal prop
+ v = self.get(itemid, prop)
+ else:
+ # the node doesn't have a value for this
+ # property
+ v = None
+ if isinstance(propclass, hyperdb.Multilink):
+ v = []
+ if prop == 'id':
+ v = int (itemid)
+ pt.sort_ids.append(v)
+ if not is_pointer:
+ pt.sort_result.append(v)
+ continue
+
+ # missing (None) values are always sorted first
+ if v is None:
+ pt.sort_ids.append(v)
+ if not is_pointer:
+ pt.sort_result.append(v)
+ continue
+
+ if isinstance(propclass, hyperdb.Link):
+ lcn = propclass.classname
+ link = self.db.classes[lcn]
+ key = link.orderprop()
+ child = pt.propdict[key]
+ if key!='id':
+ if v not in lcache:
+ # open the link class db if it's not already
+ if lcldb is None:
+ lcldb = self.db.getclassdb(lcn)
+ lcache[v] = self.db.getnode(lcn, v, lcldb)
+ r = lcache[v][key]
+ child.propdict[key].sort_ids.append(r)
+ else:
+ child.propdict[key].sort_ids.append(v)
+ pt.sort_ids.append(v)
+ if not is_pointer:
+ r = propclass.sort_repr(pt.parent.cls, v, pt.name)
+ pt.sort_result.append(r)
+ finally:
+ # if we opened the link class db, close it now
+ if lcldb is not None:
+ lcldb.close()
+ del lcache
finally:
cldb.close()
- l.sort()
-
- # filter based on full text search
- if search_matches is not None:
- k = []
- for v in l:
- if search_matches.has_key(v[0]):
- k.append(v)
- l = k
-
- # now, sort the result
- def sortfun(a, b, sort=sort, group=group, properties=self.getprops(),
- db = self.db, cl=self):
- a_id, an = a
- b_id, bn = b
- # sort by group and then sort
- for dir, prop in group, sort:
- if dir is None or prop is None: continue
-
- # sorting is class-specific
- propclass = properties[prop]
-
- # handle the properties that might be "faked"
- # also, handle possible missing properties
- try:
- if not an.has_key(prop):
- an[prop] = cl.get(a_id, prop)
- av = an[prop]
- except KeyError:
- # the node doesn't have a value for this property
- if isinstance(propclass, Multilink): av = []
- else: av = ''
- try:
- if not bn.has_key(prop):
- bn[prop] = cl.get(b_id, prop)
- bv = bn[prop]
- except KeyError:
- # the node doesn't have a value for this property
- if isinstance(propclass, Multilink): bv = []
- else: bv = ''
-
- # String and Date values are sorted in the natural way
- if isinstance(propclass, String):
- # clean up the strings
- if av and av[0] in string.uppercase:
- av = av.lower()
- if bv and bv[0] in string.uppercase:
- bv = bv.lower()
- if (isinstance(propclass, String) or
- isinstance(propclass, Date)):
- # it might be a string that's really an integer
- try:
- av = int(av)
- bv = int(bv)
- except:
- pass
- if dir == '+':
- r = cmp(av, bv)
- if r != 0: return r
- elif dir == '-':
- r = cmp(bv, av)
- if r != 0: return r
-
- # Link properties are sorted according to the value of
- # the "order" property on the linked nodes if it is
- # present; or otherwise on the key string of the linked
- # nodes; or finally on the node ids.
- elif isinstance(propclass, Link):
- link = db.classes[propclass.classname]
- if av is None and bv is not None: return -1
- if av is not None and bv is None: return 1
- if av is None and bv is None: continue
- if link.getprops().has_key('order'):
- if dir == '+':
- r = cmp(link.get(av, 'order'),
- link.get(bv, 'order'))
- if r != 0: return r
- elif dir == '-':
- r = cmp(link.get(bv, 'order'),
- link.get(av, 'order'))
- if r != 0: return r
- elif link.getkey():
- key = link.getkey()
- if dir == '+':
- r = cmp(link.get(av, key), link.get(bv, key))
- if r != 0: return r
- elif dir == '-':
- r = cmp(link.get(bv, key), link.get(av, key))
- if r != 0: return r
- else:
- if dir == '+':
- r = cmp(av, bv)
- if r != 0: return r
- elif dir == '-':
- r = cmp(bv, av)
- if r != 0: return r
- else:
- # all other types just compare
- if dir == '+':
- r = cmp(av, bv)
- elif dir == '-':
- r = cmp(bv, av)
- if r != 0: return r
-
- # end for dir, prop in sort, group:
- # if all else fails, compare the ids
- return cmp(a[0], b[0])
-
- l.sort(sortfun)
- return [i[0] for i in l]
+ # pull the id out of the individual entries
+ matches = [entry[-2] for entry in matches]
+ if __debug__:
+ self.db.stats['filtering'] += (time.time() - start_t)
+ return matches
def count(self):
- '''Get the number of nodes in this class.
+ """Get the number of nodes in this class.
If the returned integer is 'numnodes', the ids of all the nodes
in this class run from 1 to numnodes, and numnodes+1 will be the
id of the next node to be created in this class.
- '''
+ """
return self.db.countnodes(self.classname)
# Manipulating properties:
def getprops(self, protected=1):
- '''Return a dictionary mapping property names to property objects.
+ """Return a dictionary mapping property names to property objects.
If the "protected" flag is true, we include protected properties -
those which may not be modified.
methods provide the "creation" and "activity" properties. If the
"protected" flag is true, we include protected properties - those
which may not be modified.
- '''
+ """
d = self.properties.copy()
if protected:
- d['id'] = String()
+ d['id'] = hyperdb.String()
d['creation'] = hyperdb.Date()
d['activity'] = hyperdb.Date()
d['creator'] = hyperdb.Link('user')
+ d['actor'] = hyperdb.Link('user')
return d
def addprop(self, **properties):
- '''Add properties to this class.
+ """Add properties to this class.
The keyword arguments in 'properties' must map names to property
objects, or a TypeError is raised. None of the keys in 'properties'
may collide with the names of existing properties, or a ValueError
is raised before any properties have been added.
- '''
- for key in properties.keys():
- if self.properties.has_key(key):
- raise ValueError, key
+ """
+ for key in properties:
+ if key in self.properties:
+ raise ValueError(key)
self.properties.update(properties)
def index(self, nodeid):
- '''Add (or refresh) the node to search indexes
- '''
+ """ Add (or refresh) the node to search indexes """
# find all the String properties that have indexme
- for prop, propclass in self.getprops().items():
- if isinstance(propclass, String) and propclass.indexme:
+ for prop, propclass in self.getprops().iteritems():
+ if isinstance(propclass, hyperdb.String) and propclass.indexme:
+ # index them under (classname, nodeid, property)
try:
value = str(self.get(nodeid, prop))
except IndexError:
- # node no longer exists - entry should be removed
- self.db.indexer.purge_entry((self.classname, nodeid, prop))
- else:
- # and index them under (classname, nodeid, property)
- self.db.indexer.add_text((self.classname, nodeid, prop),
- value)
+ # node has been destroyed
+ continue
+ self.db.indexer.add_text((self.classname, nodeid, prop), value)
#
- # Detector interface
+ # import / export support
#
- def audit(self, event, detector):
- '''Register a detector
- '''
- l = self.auditors[event]
- if detector not in l:
- self.auditors[event].append(detector)
-
- def fireAuditors(self, action, nodeid, newvalues):
- '''Fire all registered auditors.
- '''
- for audit in self.auditors[action]:
- audit(self.db, self, nodeid, newvalues)
-
- def react(self, event, detector):
- '''Register a detector
- '''
- l = self.reactors[event]
- if detector not in l:
- self.reactors[event].append(detector)
-
- def fireReactors(self, action, nodeid, oldvalues):
- '''Fire all registered reactors.
- '''
- for react in self.reactors[action]:
- react(self.db, self, nodeid, oldvalues)
-
-class FileClass(Class, hyperdb.FileClass):
- '''This class defines a large chunk of data. To support this, it has a
+ def export_list(self, propnames, nodeid):
+ """ Export a node - generate a list of CSV-able data in the order
+ specified by propnames for the given node.
+ """
+ properties = self.getprops()
+ l = []
+ for prop in propnames:
+ proptype = properties[prop]
+ value = self.get(nodeid, prop)
+ # "marshal" data where needed
+ if value is None:
+ pass
+ elif isinstance(proptype, hyperdb.Date):
+ value = value.get_tuple()
+ elif isinstance(proptype, hyperdb.Interval):
+ value = value.get_tuple()
+ elif isinstance(proptype, hyperdb.Password):
+ value = str(value)
+ l.append(repr(value))
+
+ # append retired flag
+ l.append(repr(self.is_retired(nodeid)))
+
+ return l
+
+ def import_list(self, propnames, proplist):
+ """ Import a node - all information including "id" is present and
+ should not be sanity checked. Triggers are not triggered. The
+ journal should be initialised using the "creator" and "created"
+ information.
+
+ Return the nodeid of the node imported.
+ """
+ if self.db.journaltag is None:
+ raise hyperdb.DatabaseError(_('Database open read-only'))
+ properties = self.getprops()
+
+ # make the new node's property map
+ d = {}
+ newid = None
+ for i in range(len(propnames)):
+ # Figure the property for this column
+ propname = propnames[i]
+
+ # Use eval to reverse the repr() used to output the CSV
+ value = eval(proplist[i])
+
+ # "unmarshal" where necessary
+ if propname == 'id':
+ newid = value
+ continue
+ elif propname == 'is retired':
+ # is the item retired?
+ if int(value):
+ d[self.db.RETIRED_FLAG] = 1
+ continue
+ elif value is None:
+ d[propname] = None
+ continue
+
+ prop = properties[propname]
+ if isinstance(prop, hyperdb.Date):
+ value = date.Date(value)
+ elif isinstance(prop, hyperdb.Interval):
+ value = date.Interval(value)
+ elif isinstance(prop, hyperdb.Password):
+ value = password.Password(encrypted=value)
+ d[propname] = value
+
+ # get a new id if necessary
+ if newid is None:
+ newid = self.db.newid(self.classname)
+
+ # add the node and journal
+ self.db.addnode(self.classname, newid, d)
+ return newid
+
+ def export_journals(self):
+ """Export a class's journal - generate a list of lists of
+ CSV-able data:
+
+ nodeid, date, user, action, params
+
+ No heading here - the columns are fixed.
+ """
+ properties = self.getprops()
+ r = []
+ for nodeid in self.getnodeids():
+ for nodeid, date, user, action, params in self.history(nodeid):
+ date = date.get_tuple()
+ if action == 'set':
+ export_data = {}
+ for propname, value in params.iteritems():
+ if propname not in properties:
+ # property no longer in the schema
+ continue
+
+ prop = properties[propname]
+ # make sure the params are eval()'able
+ if value is None:
+ pass
+ elif isinstance(prop, hyperdb.Date):
+ # this is a hack - some dates are stored as strings
+ if not isinstance(value, type('')):
+ value = value.get_tuple()
+ elif isinstance(prop, hyperdb.Interval):
+ # hack too - some intervals are stored as strings
+ if not isinstance(value, type('')):
+ value = value.get_tuple()
+ elif isinstance(prop, hyperdb.Password):
+ value = str(value)
+ export_data[propname] = value
+ params = export_data
+ r.append([repr(nodeid), repr(date), repr(user),
+ repr(action), repr(params)])
+ return r
+
+class FileClass(hyperdb.FileClass, Class):
+ """This class defines a large chunk of data. To support this, it has a
mandatory String property "content" which is typically saved off
externally to the hyperdb.
The default MIME type of this data is defined by the
"default_mime_type" class attribute, which may be overridden by each
node if the class defines a "type" String property.
- '''
- default_mime_type = 'text/plain'
+ """
+ def __init__(self, db, classname, **properties):
+ """The newly-created class automatically includes the "content"
+ and "type" properties.
+ """
+ if 'content' not in properties:
+ properties['content'] = hyperdb.String(indexme='yes')
+ if 'type' not in properties:
+ properties['type'] = hyperdb.String()
+ Class.__init__(self, db, classname, **properties)
def create(self, **propvalues):
- ''' Snarf the "content" propvalue and store in a file
- '''
+ """ Snarf the "content" propvalue and store in a file
+ """
# we need to fire the auditors now, or the content property won't
# be in propvalues for the auditors to play with
self.fireAuditors('create', None, propvalues)
content = propvalues['content']
del propvalues['content']
- # do the database create
- newid = Class.create_inner(self, **propvalues)
+ # make sure we have a MIME type
+ mime_type = propvalues.get('type', self.default_mime_type)
- # fire reactors
- self.fireReactors('create', newid, None)
+ # do the database create
+ newid = self.create_inner(**propvalues)
# store off the content as a file
self.db.storefile(self.classname, newid, None, content)
- return newid
- def import_list(self, propnames, proplist):
- ''' Trap the "content" property...
- '''
- # dupe this list so we don't affect others
- propnames = propnames[:]
-
- # extract the "content" property from the proplist
- i = propnames.index('content')
- content = eval(proplist[i])
- del propnames[i]
- del proplist[i]
-
- # do the normal import
- newid = Class.import_list(self, propnames, proplist)
+ # fire reactors
+ self.fireReactors('create', newid, None)
- # save off the "content" file
- self.db.storefile(self.classname, newid, None, content)
return newid
def get(self, nodeid, propname, default=_marker, cache=1):
- ''' Trap the content propname and get it from the file
+ """ Trap the content propname and get it from the file
'cache' exists for backwards compatibility, and is not used.
- '''
+ """
poss_msg = 'Possibly an access right configuration problem.'
if propname == 'content':
try:
return self.db.getfile(self.classname, nodeid, None)
- except IOError, (strerror):
- # XXX by catching this we donot see an error in the log.
+ except IOError, strerror:
+ # XXX by catching this we don't see an error in the log.
return 'ERROR reading file: %s%s\n%s\n%s'%(
self.classname, nodeid, poss_msg, strerror)
if default is not _marker:
else:
return Class.get(self, nodeid, propname)
- def getprops(self, protected=1):
- ''' In addition to the actual properties on the node, these methods
- provide the "content" property. If the "protected" flag is true,
- we include protected properties - those which may not be
- modified.
- '''
- d = Class.getprops(self, protected=protected).copy()
- d['content'] = hyperdb.String()
- return d
+ def set(self, itemid, **propvalues):
+ """ Snarf the "content" propvalue and update it in a file
+ """
+ self.fireAuditors('set', itemid, propvalues)
- def index(self, nodeid):
- ''' Index the node in the search index.
+ # create the oldvalues dict - fill in any missing values
+ oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid))
+ for name, prop in self.getprops(protected=0).iteritems():
+ if name in oldvalues:
+ continue
+ if isinstance(prop, hyperdb.Multilink):
+ oldvalues[name] = []
+ else:
+ oldvalues[name] = None
- We want to index the content in addition to the normal String
- property indexing.
- '''
- # perform normal indexing
- Class.index(self, nodeid)
+ # now remove the content property so it's not stored in the db
+ content = None
+ if 'content' in propvalues:
+ content = propvalues['content']
+ del propvalues['content']
+
+ # do the database update
+ propvalues = self.set_inner(itemid, **propvalues)
+
+ # do content?
+ if content:
+ # store and possibly index
+ self.db.storefile(self.classname, itemid, None, content)
+ if self.properties['content'].indexme:
+ mime_type = self.get(itemid, 'type', self.default_mime_type)
+ self.db.indexer.add_text((self.classname, itemid, 'content'),
+ content, mime_type)
+ propvalues['content'] = content
- # get the content to index
- content = self.get(nodeid, 'content')
+ # fire reactors
+ self.fireReactors('set', itemid, oldvalues)
+ return propvalues
- # figure the mime type
- if self.properties.has_key('type'):
- mime_type = self.get(nodeid, 'type')
- else:
- mime_type = self.default_mime_type
+ def index(self, nodeid):
+ """ Add (or refresh) the node to search indexes.
- # and index!
- self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
- mime_type)
+ Use the content-type property for the content property.
+ """
+ # find all the String properties that have indexme
+ for prop, propclass in self.getprops().iteritems():
+ if prop == 'content' and propclass.indexme:
+ mime_type = self.get(nodeid, 'type', self.default_mime_type)
+ self.db.indexer.add_text((self.classname, nodeid, 'content'),
+ str(self.get(nodeid, 'content')), mime_type)
+ elif isinstance(propclass, hyperdb.String) and propclass.indexme:
+ # index them under (classname, nodeid, property)
+ try:
+ value = str(self.get(nodeid, prop))
+ except IndexError:
+ # node has been destroyed
+ continue
+ self.db.indexer.add_text((self.classname, nodeid, prop), value)
# deviation from spec - was called ItemClass
class IssueClass(Class, roundupdb.IssueClass):
# Overridden methods:
def __init__(self, db, classname, **properties):
- '''The newly-created class automatically includes the "messages",
+ """The newly-created class automatically includes the "messages",
"files", "nosy", and "superseder" properties. If the 'properties'
dictionary attempts to specify any of these properties or a
"creation" or "activity" property, a ValueError is raised.
- '''
- if not properties.has_key('title'):
+ """
+ if 'title' not in properties:
properties['title'] = hyperdb.String(indexme='yes')
- if not properties.has_key('messages'):
+ if 'messages' not in properties:
properties['messages'] = hyperdb.Multilink("msg")
- if not properties.has_key('files'):
+ if 'files' not in properties:
properties['files'] = hyperdb.Multilink("file")
- if not properties.has_key('nosy'):
+ if 'nosy' not in properties:
# note: journalling is turned off as it really just wastes
# space. this behaviour may be overridden in an instance
properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
- if not properties.has_key('superseder'):
+ if 'superseder' not in properties:
properties['superseder'] = hyperdb.Multilink(classname)
Class.__init__(self, db, classname, **properties)
-#
+# vim: set et sts=4 sw=4 :