index 2a338a0bb7ba250a088606c5c0ab9c454726638c..681698b389d29fe04aef13b101c938d224745e18 100644 (file)
# BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
# SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
#
-#$Id: back_anydbm.py,v 1.73 2002-09-10 00:11:49 richard Exp $
+#$Id: back_anydbm.py,v 1.100 2003-02-06 05:43:47 richard Exp $
'''
This module defines a backend that saves the hyperdatabase in a database
chosen by anydbm. It is guaranteed to always be available in python
from blobfiles import FileStorage
from sessions import Sessions
from roundup.indexer import Indexer
-from locking import acquire_lock, release_lock
+from roundup.backends import locking
from roundup.hyperdb import String, Password, Date, Interval, Link, \
Multilink, DatabaseError, Boolean, Number
# ensure files are group readable and writable
os.umask(0002)
+ # lock it
+ lockfilenm = os.path.join(self.dir, 'lock')
+ self.lockfile = locking.acquire_lock(lockfilenm)
+ self.lockfile.write(str(os.getpid()))
+ self.lockfile.flush()
+
def post_init(self):
- '''Called once the schema initialisation has finished.'''
+ ''' Called once the schema initialisation has finished.
+ '''
# reindex the db if necessary
if self.indexer.should_reindex():
self.reindex()
+ # figure the "curuserid"
+ if self.journaltag is None:
+ self.curuserid = None
+ elif self.journaltag == 'admin':
+ # admin user may not exist, but always has ID 1
+ self.curuserid = '1'
+ else:
+ self.curuserid = self.user.lookup(self.journaltag)
+
def reindex(self):
for klass in self.classes.values():
for nodeid in klass.list():
'''
if __debug__:
print >>hyperdb.DEBUG, 'getclass', (self, classname)
- return self.classes[classname]
+ try:
+ return self.classes[classname]
+ except KeyError:
+ raise KeyError, 'There is no class called "%s"'%classname
#
# Class DBs
if os.path.exists(path):
db_type = whichdb.whichdb(path)
if not db_type:
- raise hyperdb.DatabaseError, "Couldn't identify database type"
+ raise DatabaseError, "Couldn't identify database type"
elif os.path.exists(path+'.db'):
# if the path ends in '.db', it's a dbm database, whether
# anydbm says it's dbhash or not!
try:
dbm = __import__(db_type)
except ImportError:
- raise hyperdb.DatabaseError, \
+ raise DatabaseError, \
"Couldn't open database - the required module '%s'"\
" is not available"%db_type
if __debug__:
mode)
return dbm.open(path, mode)
- def lockdb(self, name):
- ''' Lock a database file
- '''
- path = os.path.join(os.getcwd(), self.dir, '%s.lock'%name)
- return acquire_lock(path)
-
#
# Node IDs
#
''' Generate a new id for the given class
'''
# open the ids DB - create if if doesn't exist
- lock = self.lockdb('_ids')
db = self.opendb('_ids', 'c')
if db.has_key(classname):
newid = db[classname] = str(int(db[classname]) + 1)
newid = str(self.getclass(classname).count()+1)
db[classname] = newid
db.close()
- release_lock(lock)
return newid
def setid(self, classname, setid):
''' Set the id counter: used during import of database
'''
# open the ids DB - create if if doesn't exist
- lock = self.lockdb('_ids')
db = self.opendb('_ids', 'c')
db[classname] = str(setid)
db.close()
- release_lock(lock)
#
# Nodes
'''
if __debug__:
print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
+
+ # we'll be supplied these props if we're doing an import
+ if not node.has_key('creator'):
+ # add in the "calculated" properties (dupe so we don't affect
+ # calling code's node assumptions)
+ node = node.copy()
+ node['creator'] = self.curuserid
+ node['creation'] = node['activity'] = date.Date()
+
self.newnodes.setdefault(classname, {})[nodeid] = 1
self.cache.setdefault(classname, {})[nodeid] = node
self.savenode(classname, nodeid, node)
print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
self.dirtynodes.setdefault(classname, {})[nodeid] = 1
+ # update the activity time (dupe so we don't affect
+ # calling code's node assumptions)
+ node = node.copy()
+ node['activity'] = date.Date()
+
# can't set without having already loaded the node
self.cache[classname][nodeid] = node
self.savenode(classname, nodeid, node)
# get the property spec
prop = properties[k]
- if isinstance(prop, Password):
+ if isinstance(prop, Password) and v is not None:
d[k] = str(v)
elif isinstance(prop, Date) and v is not None:
d[k] = v.serialise()
d[k] = date.Date(v)
elif isinstance(prop, Interval) and v is not None:
d[k] = date.Interval(v)
- elif isinstance(prop, Password):
+ elif isinstance(prop, Password) and v is not None:
p = password.Password()
p.unpack(v)
d[k] = p
#
# Journal
#
- def addjournal(self, classname, nodeid, action, params):
+ def addjournal(self, classname, nodeid, action, params, creator=None,
+ creation=None):
''' Journal the Action
'action' may be:
'''
if __debug__:
print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid,
- action, params)
+ action, params, creator, creation)
self.transactions.append((self.doSaveJournal, (classname, nodeid,
- action, params)))
+ action, params, creator, creation)))
def getjournal(self, classname, nodeid):
''' get the journal for id
if __debug__:
print >>hyperdb.DEBUG, 'packjournal', (self, pack_before)
+ pack_before = pack_before.serialise()
for classname in self.getclasses():
# get the journal db
db_name = 'journals.%s'%classname
# unpack the entry
(nodeid, date_stamp, self.journaltag, action,
params) = entry
- date_stamp = date.Date(date_stamp)
# if the entry is after the pack date, _or_ the initial
# create entry, then it stays
if date_stamp > pack_before or action == 'create':
l.append(entry)
- elif action == 'set':
- # grab the last set entry to keep information on
- # activity
- last_set_entry = entry
- if last_set_entry:
- date_stamp = last_set_entry[1]
- # if the last set entry was made after the pack date
- # then it is already in the list
- if date_stamp < pack_before:
- l.append(last_set_entry)
db[key] = marshal.dumps(l)
if db_type == 'gdbm':
db.reorganize()
'''
if __debug__:
print >>hyperdb.DEBUG, 'commit', (self,)
- # TODO: lock the DB
# keep a handle to all the database files opened
self.databases = {}
for db in self.databases.values():
db.close()
del self.databases
- # TODO: unlock the DB
# reindex the nodes that request it
for classname, nodeid in filter(None, reindex.keys()):
# save the indexer state
self.indexer.save_index()
+ self.clearCache()
+
+ def clearCache(self):
# all transactions committed, back to normal
self.cache = {}
self.dirtynodes = {}
self.databases[db_name] = self.opendb(db_name, 'c')
return self.databases[db_name]
- def doSaveJournal(self, classname, nodeid, action, params):
- # handle supply of the special journalling parameters (usually
- # supplied on importing an existing database)
+ def doSaveJournal(self, classname, nodeid, action, params, creator,
+ creation):
+ # serialise the parameters now if necessary
if isinstance(params, type({})):
- if params.has_key('creator'):
- journaltag = self.user.get(params['creator'], 'username')
- del params['creator']
- else:
- journaltag = self.journaltag
- if params.has_key('created'):
- journaldate = params['created'].serialise()
- del params['created']
- else:
- journaldate = date.Date().serialise()
- if params.has_key('activity'):
- del params['activity']
-
- # serialise the parameters now
if action in ('set', 'create'):
params = self.serialise(classname, params)
+
+ # handle supply of the special journalling parameters (usually
+ # supplied on importing an existing database)
+ if creator:
+ journaltag = creator
+ else:
+ journaltag = self.curuserid
+ if creation:
+ journaldate = creation.serialise()
else:
- journaltag = self.journaltag
journaldate = date.Date().serialise()
# create the journal entry
self.destroyednodes = {}
self.transactions = []
+ def close(self):
+ ''' Nothing to do
+ '''
+ if self.lockfile is not None:
+ locking.release_lock(self.lockfile)
+ if self.lockfile is not None:
+ self.lockfile.close()
+ self.lockfile = None
+
_marker = []
class Class(hyperdb.Class):
'''The handle to a particular class of nodes in a hyperdatabase.'''
(self.classname, newid, key))
elif isinstance(prop, String):
- if type(value) != type(''):
+ if type(value) != type('') and type(value) != type(u''):
raise TypeError, 'new property "%s" not a string'%key
elif isinstance(prop, Password):
# done
self.db.addnode(self.classname, newid, propvalues)
if self.do_journal:
- self.db.addjournal(self.classname, newid, 'create', propvalues)
+ self.db.addjournal(self.classname, newid, 'create', {})
self.fireReactors('create', newid, None)
proptype = properties[prop]
value = self.get(nodeid, prop)
# "marshal" data where needed
- if isinstance(proptype, hyperdb.Date):
+ if value is None:
+ pass
+ elif isinstance(proptype, hyperdb.Date):
value = value.get_tuple()
elif isinstance(proptype, hyperdb.Interval):
value = value.get_tuple()
if propname == 'id':
newid = value
continue
+ elif value is None:
+ # don't set Nones
+ continue
elif isinstance(prop, hyperdb.Date):
value = date.Date(value)
elif isinstance(prop, hyperdb.Interval):
pwd = password.Password()
pwd.unpack(value)
value = pwd
- if value is not None:
- d[propname] = value
+ d[propname] = value
- # add
+ # add the node and journal
self.db.addnode(self.classname, newid, d)
- self.db.addjournal(self.classname, newid, 'create', d)
+
+ # extract the journalling stuff and nuke it
+ if d.has_key('creator'):
+ creator = d['creator']
+ del d['creator']
+ else:
+ creator = None
+ if d.has_key('creation'):
+ creation = d['creation']
+ del d['creation']
+ else:
+ creation = None
+ if d.has_key('activity'):
+ del d['activity']
+ self.db.addjournal(self.classname, newid, 'create', {}, creator,
+ creation)
return newid
def get(self, nodeid, propname, default=_marker, cache=1):
if propname == 'id':
return nodeid
+ # get the node's dict
+ d = self.db.getnode(self.classname, nodeid, cache=cache)
+
+ # check for one of the special props
if propname == 'creation':
+ if d.has_key('creation'):
+ return d['creation']
if not self.do_journal:
raise ValueError, 'Journalling is disabled for this class'
journal = self.db.getjournal(self.classname, nodeid)
# on the strange chance that there's no journal
return date.Date()
if propname == 'activity':
+ if d.has_key('activity'):
+ return d['activity']
if not self.do_journal:
raise ValueError, 'Journalling is disabled for this class'
journal = self.db.getjournal(self.classname, nodeid)
# on the strange chance that there's no journal
return date.Date()
if propname == 'creator':
+ if d.has_key('creator'):
+ return d['creator']
if not self.do_journal:
raise ValueError, 'Journalling is disabled for this class'
journal = self.db.getjournal(self.classname, nodeid)
if journal:
- name = self.db.getjournal(self.classname, nodeid)[0][2]
+ num_re = re.compile('^\d+$')
+ value = self.db.getjournal(self.classname, nodeid)[0][2]
+ if num_re.match(value):
+ return value
+ else:
+ # old-style "username" journal tag
+ try:
+ return self.db.user.lookup(value)
+ except KeyError:
+ # user's been retired, return admin
+ return '1'
else:
- return None
- try:
- return self.db.user.lookup(name)
- except KeyError:
- # the journaltag user doesn't exist any more
- return None
+ return self.db.curuserid
# get the property (raises KeyErorr if invalid)
prop = self.properties[propname]
- # get the node's dict
- d = self.db.getnode(self.classname, nodeid, cache=cache)
-
if not d.has_key(propname):
if default is _marker:
if isinstance(prop, Multilink):
return d[propname]
- # XXX not in spec
+ # not in spec
def getnode(self, nodeid, cache=1):
''' Return a convenience wrapper for the node.
# this will raise the KeyError if the property isn't valid
# ... we don't use getprops() here because we only care about
# the writeable properties.
- prop = self.properties[propname]
+ try:
+ prop = self.properties[propname]
+ except KeyError:
+ raise KeyError, '"%s" has no property named "%s"'%(
+ self.classname, propname)
# if the value's the same as the existing value, no sense in
# doing anything
- if node.has_key(propname) and value == node[propname]:
+ current = node.get(propname, None)
+ if value == current:
del propvalues[propname]
continue
+ journalvalues[propname] = current
# do stuff based on the prop type
if isinstance(prop, Link):
if self.do_journal and prop.do_journal:
# register the unlink with the old linked node
- if node[propname] is not None:
+ if node.has_key(propname) and node[propname] is not None:
self.db.addjournal(link_class, node[propname], 'unlink',
(self.classname, nodeid, propname))
journalvalues[propname] = tuple(l)
elif isinstance(prop, String):
- if value is not None and type(value) != type(''):
+ if value is not None and type(value) != type('') and type(value) != type(u''):
raise TypeError, 'new property "%s" not a string'%propname
elif isinstance(prop, Password):
self.db.setnode(self.classname, nodeid, node)
if self.do_journal:
- propvalues.update(journalvalues)
- self.db.addjournal(self.classname, nodeid, 'set', propvalues)
+ self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
self.fireReactors('set', nodeid, oldvalues)
def destroy(self, nodeid):
'''Destroy a node.
-
+
WARNING: this method should never be used except in extremely rare
situations where there could never be links to the node being
deleted
The returned list contains tuples of the form
- (date, tag, action, params)
+ (nodeid, date, tag, action, params)
'date' is a Timestamp object specifying the time of the change and
'tag' is the journaltag specified when the database was opened.
otherwise a KeyError is raised.
'''
if not self.key:
- raise TypeError, 'No key property set'
+ raise TypeError, 'No key property set for class %s'%self.classname
cldb = self.db.getclassdb(self.classname)
try:
for nodeid in self.db.getnodeids(self.classname, cldb):
if node.has_key(self.db.RETIRED_FLAG):
continue
if node[self.key] == keyvalue:
- cldb.close()
return nodeid
finally:
cldb.close()
- raise KeyError, keyvalue
+ raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
+ keyvalue, self.classname)
- # XXX: change from spec - allows multiple props to match
+ # change from spec - allows multiple props to match
def find(self, **propspec):
'''Get the ids of nodes in this class which link to the given nodes.
- 'propspec' consists of keyword args propname={nodeid:1,}
- 'propname' must be the name of a property in this class, or a
- KeyError is raised. That property must be a Link or Multilink
- property, or a TypeError is raised.
+ 'propspec' consists of keyword args propname=nodeid or
+ propname={nodeid:1, }
+ 'propname' must be the name of a property in this class, or a
+ KeyError is raised. That property must be a Link or
+ Multilink property, or a TypeError is raised.
Any node in this class whose 'propname' property links to any of the
nodeids will be returned. Used by the full text indexing, which knows
- that "foo" occurs in msg1, msg3 and file7, so we have hits on these issues:
+ that "foo" occurs in msg1, msg3 and file7, so we have hits on these
+ issues:
+
db.issue.find(messages={'1':1,'3':1}, files={'7':1})
'''
propspec = propspec.items()
if node.has_key(self.db.RETIRED_FLAG):
continue
for key, value in requirements.items():
+ if not node.has_key(key):
+ break
if node[key] is None or node[key].lower() != value:
break
else:
l.sort()
return l
- def filter(self, search_matches, filterspec, sort, group,
- num_re = re.compile('^\d+$')):
+ def filter(self, search_matches, filterspec, sort=(None,None),
+ group=(None,None), num_re = re.compile('^\d+$')):
''' Return a list of the ids of the active nodes in this class that
match the 'filter' spec, sorted by the group spec and then the
sort spec.
"sort" and "group" are (dir, prop) where dir is '+', '-' or None
and prop is a prop name or None
"search_matches" is {nodeid: marker}
+
+ The filter must match all properties specificed - but if the
+ property value to match is a list, any one of the values in the
+ list may match for that property to match.
'''
cn = self.classname
k, entry, self.properties[k].classname)
u.append(entry)
l.append((MULTILINK, k, u))
- elif isinstance(propclass, String):
+ elif isinstance(propclass, String) and k != 'id':
# simple glob searching
v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
v = v.replace('?', '.')
else:
bv = v
l.append((OTHER, k, bv))
+ elif isinstance(propclass, Date):
+ l.append((OTHER, k, date.Date(v)))
+ elif isinstance(propclass, Interval):
+ l.append((OTHER, k, date.Interval(v)))
elif isinstance(propclass, Number):
l.append((OTHER, k, int(v)))
else:
continue
# apply filter
for t, k, v in filterspec:
+ # handle the id prop
+ if k == 'id' and v == nodeid:
+ continue
+
# make sure the node has the property
if not node.has_key(k):
# this node doesn't have this property, so reject it
if isinstance(propclass, String):
# clean up the strings
if av and av[0] in string.uppercase:
- av = an[prop] = av.lower()
+ av = av.lower()
if bv and bv[0] in string.uppercase:
- bv = bn[prop] = bv.lower()
+ bv = bv.lower()
if (isinstance(propclass, String) or
isinstance(propclass, Date)):
# it might be a string that's really an integer
# Multilink properties are sorted according to how many
# links are present.
elif isinstance(propclass, Multilink):
+ r = cmp(len(av), len(bv))
+ if r == 0:
+ # Compare contents of multilink property if lenghts is
+ # equal
+ r = cmp ('.'.join(av), '.'.join(bv))
if dir == '+':
- r = cmp(len(av), len(bv))
- if r != 0: return r
+ return r
elif dir == '-':
- r = cmp(len(bv), len(av))
- if r != 0: return r
+ return -r
elif isinstance(propclass, Number) or isinstance(propclass, Boolean):
if dir == '+':
r = cmp(av, bv)
d['id'] = String()
d['creation'] = hyperdb.Date()
d['activity'] = hyperdb.Date()
- d['creator'] = hyperdb.Link("user")
+ d['creator'] = hyperdb.Link('user')
return d
def addprop(self, **properties):
# extract the "content" property from the proplist
i = propnames.index('content')
- content = proplist[i]
+ content = eval(proplist[i])
del propnames[i]
del proplist[i]
def get(self, nodeid, propname, default=_marker, cache=1):
''' trap the content propname and get it from the file
'''
-
- poss_msg = 'Possibly a access right configuration problem.'
+ poss_msg = 'Possibly an access right configuration problem.'
if propname == 'content':
try:
return self.db.getfile(self.classname, nodeid, None)
except IOError, (strerror):
- # BUG: by catching this we donot see an error in the log.
+ # XXX by catching this we donot see an error in the log.
return 'ERROR reading file: %s%s\n%s\n%s'%(
self.classname, nodeid, poss_msg, strerror)
if default is not _marker:
modified.
'''
d = Class.getprops(self, protected=protected).copy()
- if protected:
- d['content'] = hyperdb.String()
+ d['content'] = hyperdb.String()
return d
def index(self, nodeid):
self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
mime_type)
-# XXX deviation from spec - was called ItemClass
+# deviation from spec - was called ItemClass
class IssueClass(Class, roundupdb.IssueClass):
# Overridden methods:
def __init__(self, db, classname, **properties):