X-Git-Url: https://git.tokkee.org/?a=blobdiff_plain;f=roundup%2Fbackends%2Fback_anydbm.py;h=cfda7577f1f07bd20858c75dcd85551419e3312d;hb=6aa8d96236e1f272e29768542bcf7b9ed965360e;hp=27d8ae76274a7b76da22504724d925a5dac46ae5;hpb=7cd224cf545e2c71aef8116f4e3bdd97fdfa3d10;p=roundup.git diff --git a/roundup/backends/back_anydbm.py b/roundup/backends/back_anydbm.py index 27d8ae7..cfda757 100644 --- a/roundup/backends/back_anydbm.py +++ b/roundup/backends/back_anydbm.py @@ -14,23 +14,48 @@ # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. -# -#$Id: back_anydbm.py,v 1.61 2002-08-19 02:53:27 richard Exp $ -''' -This module defines a backend that saves the hyperdatabase in a database -chosen by anydbm. It is guaranteed to always be available in python +# +"""This module defines a backend that saves the hyperdatabase in a +database chosen by anydbm. It is guaranteed to always be available in python versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several serious bugs, and is not available) -''' +""" +__docformat__ = 'restructuredtext' + +try: + import anydbm, sys + # dumbdbm only works in python 2.1.2+ + if sys.version_info < (2,1,2): + import dumbdbm + assert anydbm._defaultmod != dumbdbm + del dumbdbm +except AssertionError: + print "WARNING: you should upgrade to python 2.1.3" + +import whichdb, os, marshal, re, weakref, string, copy, time, shutil, logging + +from roundup import hyperdb, date, password, roundupdb, security, support +from roundup.support import reversed +from roundup.backends import locking +from roundup.i18n import _ -import whichdb, anydbm, os, marshal, re, weakref, string, copy -from roundup import hyperdb, date, password, roundupdb, security from blobfiles import FileStorage -from sessions import Sessions -from roundup.indexer import Indexer -from locking import acquire_lock, release_lock -from roundup.hyperdb import String, Password, Date, Interval, Link, \ - Multilink, DatabaseError, Boolean, Number +from sessions_dbm import Sessions, OneTimeKeys + +try: + from indexer_xapian import Indexer +except ImportError: + from indexer_dbm import Indexer + +def db_exists(config): + # check for the user db + for db in 'nodes.user nodes.user.db'.split(): + if os.path.exists(os.path.join(config.DATABASE, db)): + return 1 + return 0 + +def db_nuke(config): + shutil.rmtree(config.DATABASE) # # Now the database @@ -39,10 +64,10 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): """A database for storing records containing flexible data types. Transaction stuff TODO: - . check the timestamp of the class file and nuke the cache if it's - modified. Do some sort of conflict checking on the dirty stuff. - . perhaps detect write collisions (related to above)? + - check the timestamp of the class file and nuke the cache if it's + modified. Do some sort of conflict checking on the dirty stuff. + - perhaps detect write collisions (related to above)? """ def __init__(self, config, journaltag=None): """Open a hyperdatabase given a specifier to some storage. @@ -56,36 +81,65 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): The 'journaltag' is a token that will be attached to the journal entries for any edits done on the database. If 'journaltag' is None, the database is opened in read-only mode: the Class.create(), - Class.set(), and Class.retire() methods are disabled. + Class.set(), Class.retire(), and Class.restore() methods are + disabled. """ + FileStorage.__init__(self, config.UMASK) self.config, self.journaltag = config, journaltag self.dir = config.DATABASE self.classes = {} self.cache = {} # cache of nodes loaded or created + self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0, + 'filtering': 0} self.dirtynodes = {} # keep track of the dirty nodes by class self.newnodes = {} # keep track of the new nodes by class self.destroyednodes = {}# keep track of the destroyed nodes by class self.transactions = [] - self.indexer = Indexer(self.dir) - self.sessions = Sessions(self.config) + self.indexer = Indexer(self) self.security = security.Security(self) - # ensure files are group readable and writable - os.umask(0002) + os.umask(config.UMASK) + + # lock it + lockfilenm = os.path.join(self.dir, 'lock') + self.lockfile = locking.acquire_lock(lockfilenm) + self.lockfile.write(str(os.getpid())) + self.lockfile.flush() def post_init(self): - """Called once the schema initialisation has finished.""" + """Called once the schema initialisation has finished. + """ # reindex the db if necessary if self.indexer.should_reindex(): self.reindex() - def reindex(self): - for klass in self.classes.values(): - for nodeid in klass.list(): - klass.index(nodeid) + def refresh_database(self): + """Rebuild the database + """ + self.reindex() + + def getSessionManager(self): + return Sessions(self) + + def getOTKManager(self): + return OneTimeKeys(self) + + def reindex(self, classname=None, show_progress=False): + if classname: + classes = [self.getclass(classname)] + else: + classes = self.classes.values() + for klass in classes: + if show_progress: + for nodeid in support.Progress('Reindex %s'%klass.classname, + klass.list()): + klass.index(nodeid) + else: + for nodeid in klass.list(): + klass.index(nodeid) self.indexer.save_index() def __repr__(self): - return ''%id(self) + return ''%id(self) # # Classes @@ -93,23 +147,25 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): def __getattr__(self, classname): """A convenient way of calling self.getclass(classname).""" if self.classes.has_key(classname): - if __debug__: - print >>hyperdb.DEBUG, '__getattr__', (self, classname) return self.classes[classname] raise AttributeError, classname def addclass(self, cl): - if __debug__: - print >>hyperdb.DEBUG, 'addclass', (self, cl) cn = cl.classname if self.classes.has_key(cn): raise ValueError, cn self.classes[cn] = cl + # add default Edit and View permissions + self.security.addPermission(name="Create", klass=cn, + description="User is allowed to create "+cn) + self.security.addPermission(name="Edit", klass=cn, + description="User is allowed to edit "+cn) + self.security.addPermission(name="View", klass=cn, + description="User is allowed to access "+cn) + def getclasses(self): """Return a list of the names of all existing classes.""" - if __debug__: - print >>hyperdb.DEBUG, 'getclasses', (self,) l = self.classes.keys() l.sort() return l @@ -119,18 +175,18 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): If 'classname' is not a valid class name, a KeyError is raised. """ - if __debug__: - print >>hyperdb.DEBUG, 'getclass', (self, classname) - return self.classes[classname] + try: + return self.classes[classname] + except KeyError: + raise KeyError, 'There is no class called "%s"'%classname # # Class DBs # def clear(self): - '''Delete all database contents - ''' - if __debug__: - print >>hyperdb.DEBUG, 'clear', (self,) + """Delete all database contents + """ + logging.getLogger('hyperdb').info('clear') for cn in self.classes.keys(): for dummy in 'nodes', 'journals': path = os.path.join(self.dir, 'journals.%s'%cn) @@ -138,23 +194,28 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): os.remove(path) elif os.path.exists(path+'.db'): # dbm appends .db os.remove(path+'.db') + # reset id sequences + path = os.path.join(os.getcwd(), self.dir, '_ids') + if os.path.exists(path): + os.remove(path) + elif os.path.exists(path+'.db'): # dbm appends .db + os.remove(path+'.db') def getclassdb(self, classname, mode='r'): - ''' grab a connection to the class db that will be used for + """ grab a connection to the class db that will be used for multiple actions - ''' - if __debug__: - print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode) + """ return self.opendb('nodes.%s'%classname, mode) def determine_db_type(self, path): - ''' determine which DB wrote the class file - ''' + """ determine which DB wrote the class file + """ db_type = '' if os.path.exists(path): db_type = whichdb.whichdb(path) if not db_type: - raise hyperdb.DatabaseError, "Couldn't identify database type" + raise hyperdb.DatabaseError, \ + _("Couldn't identify database type") elif os.path.exists(path+'.db'): # if the path ends in '.db', it's a dbm database, whether # anydbm says it's dbhash or not! @@ -162,12 +223,9 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): return db_type def opendb(self, name, mode): - '''Low-level database opener that gets around anydbm/dbm + """Low-level database opener that gets around anydbm/dbm eccentricities. - ''' - if __debug__: - print >>hyperdb.DEBUG, 'opendb', (self, name, mode) - + """ # figure the class db type path = os.path.join(os.getcwd(), self.dir, name) db_type = self.determine_db_type(path) @@ -175,35 +233,28 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): # new database? let anydbm pick the best dbm if not db_type: if __debug__: - print >>hyperdb.DEBUG, "opendb anydbm.open(%r, 'n')"%path - return anydbm.open(path, 'n') + logging.getLogger('hyperdb').debug("opendb anydbm.open(%r, 'c')"%path) + return anydbm.open(path, 'c') # open the database with the correct module try: dbm = __import__(db_type) except ImportError: raise hyperdb.DatabaseError, \ - "Couldn't open database - the required module '%s'"\ - " is not available"%db_type + _("Couldn't open database - the required module '%s'"\ + " is not available")%db_type if __debug__: - print >>hyperdb.DEBUG, "opendb %r.open(%r, %r)"%(db_type, path, - mode) + logging.getLogger('hyperdb').debug("opendb %r.open(%r, %r)"%(db_type, path, + mode)) return dbm.open(path, mode) - def lockdb(self, name): - ''' Lock a database file - ''' - path = os.path.join(os.getcwd(), self.dir, '%s.lock'%name) - return acquire_lock(path) - # # Node IDs # def newid(self, classname): - ''' Generate a new id for the given class - ''' + """ Generate a new id for the given class + """ # open the ids DB - create if if doesn't exist - lock = self.lockdb('_ids') db = self.opendb('_ids', 'c') if db.has_key(classname): newid = db[classname] = str(int(db[classname]) + 1) @@ -212,36 +263,38 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): newid = str(self.getclass(classname).count()+1) db[classname] = newid db.close() - release_lock(lock) return newid def setid(self, classname, setid): - ''' Set the id counter: used during import of database - ''' + """ Set the id counter: used during import of database + """ # open the ids DB - create if if doesn't exist - lock = self.lockdb('_ids') db = self.opendb('_ids', 'c') db[classname] = str(setid) db.close() - release_lock(lock) # # Nodes # def addnode(self, classname, nodeid, node): - ''' add the specified node to its class's db - ''' - if __debug__: - print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node) + """ add the specified node to its class's db + """ + # we'll be supplied these props if we're doing an import + if not node.has_key('creator'): + # add in the "calculated" properties (dupe so we don't affect + # calling code's node assumptions) + node = node.copy() + node['creator'] = self.getuid() + node['actor'] = self.getuid() + node['creation'] = node['activity'] = date.Date() + self.newnodes.setdefault(classname, {})[nodeid] = 1 self.cache.setdefault(classname, {})[nodeid] = node self.savenode(classname, nodeid, node) def setnode(self, classname, nodeid, node): - ''' change the specified node - ''' - if __debug__: - print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node) + """ change the specified node + """ self.dirtynodes.setdefault(classname, {})[nodeid] = 1 # can't set without having already loaded the node @@ -249,28 +302,30 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): self.savenode(classname, nodeid, node) def savenode(self, classname, nodeid, node): - ''' perform the saving of data specified by the set/addnode - ''' + """ perform the saving of data specified by the set/addnode + """ if __debug__: - print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node) + logging.getLogger('hyperdb').debug('save %s%s %r'%(classname, nodeid, node)) self.transactions.append((self.doSaveNode, (classname, nodeid, node))) def getnode(self, classname, nodeid, db=None, cache=1): - ''' get a node from the database - ''' - if __debug__: - print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid, db) - if cache: - # try the cache - cache_dict = self.cache.setdefault(classname, {}) - if cache_dict.has_key(nodeid): - if __debug__: - print >>hyperdb.TRACE, 'get %s %s cached'%(classname, - nodeid) - return cache_dict[nodeid] + """ get a node from the database + + Note the "cache" parameter is not used, and exists purely for + backward compatibility! + """ + # try the cache + cache_dict = self.cache.setdefault(classname, {}) + if cache_dict.has_key(nodeid): + if __debug__: + logging.getLogger('hyperdb').debug('get %s%s cached'%(classname, nodeid)) + self.stats['cache_hits'] += 1 + return cache_dict[nodeid] if __debug__: - print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid) + self.stats['cache_misses'] += 1 + start_t = time.time() + logging.getLogger('hyperdb').debug('get %s%s'%(classname, nodeid)) # get from the database and save in the cache if db is None: @@ -293,14 +348,16 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): if cache: cache_dict[nodeid] = res + if __debug__: + self.stats['get_items'] += (time.time() - start_t) + return res def destroynode(self, classname, nodeid): - '''Remove a node from the database. Called exclusively by the + """Remove a node from the database. Called exclusively by the destroy() method on Class. - ''' - if __debug__: - print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid) + """ + logging.getLogger('hyperdb').info('destroy %s%s'%(classname, nodeid)) # remove from cache and newnodes if it's there if (self.cache.has_key(classname) and @@ -320,40 +377,39 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): # add the destroy commit action self.transactions.append((self.doDestroyNode, (classname, nodeid))) + self.transactions.append((FileStorage.destroy, (self, classname, nodeid))) def serialise(self, classname, node): - '''Copy the node contents, converting non-marshallable data into + """Copy the node contents, converting non-marshallable data into marshallable data. - ''' - if __debug__: - print >>hyperdb.DEBUG, 'serialise', classname, node + """ properties = self.getclass(classname).getprops() d = {} for k, v in node.items(): - # if the property doesn't exist, or is the "retired" flag then - # it won't be in the properties dict - if not properties.has_key(k): + if k == self.RETIRED_FLAG: d[k] = v continue + # if the property doesn't exist then we really don't care + if not properties.has_key(k): + continue + # get the property spec prop = properties[k] - if isinstance(prop, Password): + if isinstance(prop, hyperdb.Password) and v is not None: d[k] = str(v) - elif isinstance(prop, Date) and v is not None: - d[k] = v.get_tuple() - elif isinstance(prop, Interval) and v is not None: - d[k] = v.get_tuple() + elif isinstance(prop, hyperdb.Date) and v is not None: + d[k] = v.serialise() + elif isinstance(prop, hyperdb.Interval) and v is not None: + d[k] = v.serialise() else: d[k] = v return d def unserialise(self, classname, node): - '''Decode the marshalled node data - ''' - if __debug__: - print >>hyperdb.DEBUG, 'unserialise', classname, node + """Decode the marshalled node data + """ properties = self.getclass(classname).getprops() d = {} for k, v in node.items(): @@ -366,11 +422,11 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): # get the property spec prop = properties[k] - if isinstance(prop, Date) and v is not None: + if isinstance(prop, hyperdb.Date) and v is not None: d[k] = date.Date(v) - elif isinstance(prop, Interval) and v is not None: + elif isinstance(prop, hyperdb.Interval) and v is not None: d[k] = date.Interval(v) - elif isinstance(prop, Password): + elif isinstance(prop, hyperdb.Password) and v is not None: p = password.Password() p.unpack(v) d[k] = p @@ -379,19 +435,12 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): return d def hasnode(self, classname, nodeid, db=None): - ''' determine if the database has a given node - ''' - if __debug__: - print >>hyperdb.DEBUG, 'hasnode', (self, classname, nodeid, db) - + """ determine if the database has a given node + """ # try the cache cache = self.cache.setdefault(classname, {}) if cache.has_key(nodeid): - if __debug__: - print >>hyperdb.TRACE, 'has %s %s cached'%(classname, nodeid) return 1 - if __debug__: - print >>hyperdb.TRACE, 'has %s %s'%(classname, nodeid) # not in the cache - check the database if db is None: @@ -400,9 +449,6 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): return res def countnodes(self, classname, db=None): - if __debug__: - print >>hyperdb.DEBUG, 'countnodes', (self, classname, db) - count = 0 # include the uncommitted nodes @@ -417,28 +463,6 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): count = count + len(db.keys()) return count - def getnodeids(self, classname, db=None): - if __debug__: - print >>hyperdb.DEBUG, 'getnodeids', (self, classname, db) - - res = [] - - # start off with the new nodes - if self.newnodes.has_key(classname): - res += self.newnodes[classname].keys() - - if db is None: - db = self.getclassdb(classname) - res = res + db.keys() - - # remove the uncommitted, destroyed nodes - if self.destroyednodes.has_key(classname): - for nodeid in self.destroyednodes[classname].keys(): - if db.has_key(nodeid): - res.remove(nodeid) - - return res - # # Files - special node properties @@ -447,28 +471,58 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): # # Journal # - def addjournal(self, classname, nodeid, action, params): - ''' Journal the Action + def addjournal(self, classname, nodeid, action, params, creator=None, + creation=None): + """ Journal the Action 'action' may be: 'create' or 'set' -- 'params' is a dictionary of property values 'link' or 'unlink' -- 'params' is (classname, nodeid, propname) 'retire' -- 'params' is None - ''' + + 'creator' -- the user performing the action, which defaults to + the current user. + """ if __debug__: - print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid, - action, params) + logging.getLogger('hyperdb').debug('addjournal %s%s %s %r %s %r'%(classname, + nodeid, action, params, creator, creation)) + if creator is None: + creator = self.getuid() self.transactions.append((self.doSaveJournal, (classname, nodeid, - action, params))) + action, params, creator, creation))) + + def setjournal(self, classname, nodeid, journal): + """Set the journal to the "journal" list.""" + if __debug__: + logging.getLogger('hyperdb').debug('setjournal %s%s %r'%(classname, + nodeid, journal)) + self.transactions.append((self.doSetJournal, (classname, nodeid, + journal))) def getjournal(self, classname, nodeid): - ''' get the journal for id + """ get the journal for id Raise IndexError if the node doesn't exist (as per history()'s API) - ''' - if __debug__: - print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid) + """ + # our journal result + res = [] + + # add any journal entries for transactions not committed to the + # database + for method, args in self.transactions: + if method != self.doSaveJournal: + continue + (cache_classname, cache_nodeid, cache_action, cache_params, + cache_creator, cache_creation) = args + if cache_classname == classname and cache_nodeid == nodeid: + if not cache_creator: + cache_creator = self.getuid() + if not cache_creation: + cache_creation = date.Date() + res.append((cache_nodeid, cache_creation, cache_creator, + cache_action, cache_params)) + # attempt to open the journal - in some rare cases, the journal may # not exist try: @@ -477,93 +531,111 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): if str(error) == "need 'c' or 'n' flag to open new db": raise IndexError, 'no such %s %s'%(classname, nodeid) elif error.args[0] != 2: + # this isn't a "not found" error, be alarmed! raise + if res: + # we have unsaved journal entries, return them + return res raise IndexError, 'no such %s %s'%(classname, nodeid) try: journal = marshal.loads(db[nodeid]) except KeyError: db.close() + if res: + # we have some unsaved journal entries, be happy! + return res raise IndexError, 'no such %s %s'%(classname, nodeid) db.close() - res = [] + + # add all the saved journal entries for this node for nodeid, date_stamp, user, action, params in journal: res.append((nodeid, date.Date(date_stamp), user, action, params)) return res def pack(self, pack_before): - ''' delete all journal entries before 'pack_before' ''' - if __debug__: - print >>hyperdb.DEBUG, 'packjournal', (self, pack_before) - - pack_before = pack_before.get_tuple() - - classes = self.getclasses() - - # figure the class db type - - for classname in classes: + """ Delete all journal entries except "create" before 'pack_before'. + """ + pack_before = pack_before.serialise() + for classname in self.getclasses(): + packed = 0 + # get the journal db db_name = 'journals.%s'%classname path = os.path.join(os.getcwd(), self.dir, classname) db_type = self.determine_db_type(path) db = self.opendb(db_name, 'w') for key in db.keys(): + # get the journal for this db entry journal = marshal.loads(db[key]) l = [] last_set_entry = None for entry in journal: - (nodeid, date_stamp, self.journaltag, action, + # unpack the entry + (nodeid, date_stamp, self.journaltag, action, params) = entry + # if the entry is after the pack date, _or_ the initial + # create entry, then it stays if date_stamp > pack_before or action == 'create': l.append(entry) - elif action == 'set': - # grab the last set entry to keep information on - # activity - last_set_entry = entry - if last_set_entry: - date_stamp = last_set_entry[1] - # if the last set entry was made after the pack date - # then it is already in the list - if date_stamp < pack_before: - l.append(last_set_entry) + else: + packed += 1 db[key] = marshal.dumps(l) + + logging.getLogger('hyperdb').info('packed %d %s items'%(packed, + classname)) + if db_type == 'gdbm': db.reorganize() db.close() - + # # Basic transaction support # - def commit(self): - ''' Commit the current transactions. - ''' - if __debug__: - print >>hyperdb.DEBUG, 'commit', (self,) - # TODO: lock the DB + def commit(self, fail_ok=False): + """ Commit the current transactions. + + Save all data changed since the database was opened or since the + last commit() or rollback(). + + fail_ok indicates that the commit is allowed to fail. This is used + in the web interface when committing cleaning of the session + database. We don't care if there's a concurrency issue there. + + The only backend this seems to affect is postgres. + """ + logging.getLogger('hyperdb').info('commit %s transactions'%( + len(self.transactions))) # keep a handle to all the database files opened self.databases = {} - # now, do all the transactions - reindex = {} - for method, args in self.transactions: - reindex[method(*args)] = 1 - - # now close all the database files - for db in self.databases.values(): - db.close() - del self.databases - # TODO: unlock the DB + try: + # now, do all the transactions + reindex = {} + for method, args in self.transactions: + reindex[method(*args)] = 1 + finally: + # make sure we close all the database files + for db in self.databases.values(): + db.close() + del self.databases + + # clear the transactions list now so the blobfile implementation + # doesn't think there's still pending file commits when it tries + # to access the file data + self.transactions = [] # reindex the nodes that request it for classname, nodeid in filter(None, reindex.keys()): - print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid) self.getclass(classname).index(nodeid) # save the indexer state self.indexer.save_index() + self.clearCache() + + def clearCache(self): # all transactions committed, back to normal self.cache = {} self.dirtynodes = {} @@ -572,8 +644,8 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): self.transactions = [] def getCachedClassDB(self, classname): - ''' get the class db, looking in our cache of databases for commit - ''' + """ get the class db, looking in our cache of databases for commit + """ # get the database handle db_name = 'nodes.%s'%classname if not self.databases.has_key(db_name): @@ -581,10 +653,6 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): return self.databases[db_name] def doSaveNode(self, classname, nodeid, node): - if __debug__: - print >>hyperdb.DEBUG, 'doSaveNode', (self, classname, nodeid, - node) - db = self.getCachedClassDB(classname) # now save the marshalled data @@ -594,40 +662,32 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): return (classname, nodeid) def getCachedJournalDB(self, classname): - ''' get the journal db, looking in our cache of databases for commit - ''' + """ get the journal db, looking in our cache of databases for commit + """ # get the database handle db_name = 'journals.%s'%classname if not self.databases.has_key(db_name): self.databases[db_name] = self.opendb(db_name, 'c') return self.databases[db_name] - def doSaveJournal(self, classname, nodeid, action, params): + def doSaveJournal(self, classname, nodeid, action, params, creator, + creation): + # serialise the parameters now if necessary + if isinstance(params, type({})): + if action in ('set', 'create'): + params = self.serialise(classname, params) + # handle supply of the special journalling parameters (usually # supplied on importing an existing database) - if params.has_key('creator'): - journaltag = self.user.get(params['creator'], 'username') - del params['creator'] + journaltag = creator + if creation: + journaldate = creation.serialise() else: - journaltag = self.journaltag - if params.has_key('created'): - journaldate = params['created'].get_tuple() - del params['created'] - else: - journaldate = date.Date().get_tuple() - if params.has_key('activity'): - del params['activity'] - - # serialise the parameters now - if action in ('set', 'create'): - params = self.serialise(classname, params) + journaldate = date.Date().serialise() # create the journal entry entry = (nodeid, journaldate, journaltag, action, params) - if __debug__: - print >>hyperdb.DEBUG, 'doSaveJournal', entry - db = self.getCachedJournalDB(classname) # now insert the journal entry @@ -641,10 +701,19 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): db[nodeid] = marshal.dumps(l) - def doDestroyNode(self, classname, nodeid): - if __debug__: - print >>hyperdb.DEBUG, 'doDestroyNode', (self, classname, nodeid) + def doSetJournal(self, classname, nodeid, journal): + l = [] + for nodeid, journaldate, journaltag, action, params in journal: + # serialise the parameters now if necessary + if isinstance(params, type({})): + if action in ('set', 'create'): + params = self.serialise(classname, params) + journaldate = journaldate.serialise() + l.append((nodeid, journaldate, journaltag, action, params)) + db = self.getCachedJournalDB(classname) + db[nodeid] = marshal.dumps(l) + def doDestroyNode(self, classname, nodeid): # delete from the class database db = self.getCachedClassDB(classname) if db.has_key(nodeid): @@ -655,14 +724,12 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): if db.has_key(nodeid): del db[nodeid] - # return the classname, nodeid so we reindex this content - return (classname, nodeid) - def rollback(self): - ''' Reverse all actions from the current transaction. - ''' - if __debug__: - print >>hyperdb.DEBUG, 'rollback', (self, ) + """ Reverse all actions from the current transaction. + """ + logging.getLogger('hyperdb').info('rollback %s transactions'%( + len(self.transactions))) + for method, args in self.transactions: # delete temporary files if method == self.doStoreFile: @@ -673,44 +740,26 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): self.destroyednodes = {} self.transactions = [] + def close(self): + """ Nothing to do + """ + if self.lockfile is not None: + locking.release_lock(self.lockfile) + self.lockfile.close() + self.lockfile = None + _marker = [] class Class(hyperdb.Class): """The handle to a particular class of nodes in a hyperdatabase.""" - def __init__(self, db, classname, **properties): - """Create a new class with a given name and property specification. - - 'classname' must not collide with the name of an existing class, - or a ValueError is raised. The keyword arguments in 'properties' - must map names to property objects, or a TypeError is raised. - """ - if (properties.has_key('creation') or properties.has_key('activity') - or properties.has_key('creator')): - raise ValueError, '"creation", "activity" and "creator" are '\ - 'reserved' - - self.classname = classname - self.properties = properties - self.db = weakref.proxy(db) # use a weak ref to avoid circularity - self.key = '' - - # should we journal changes (default yes) - self.do_journal = 1 - - # do the db-related init stuff - db.addclass(self) - - self.auditors = {'create': [], 'set': [], 'retire': []} - self.reactors = {'create': [], 'set': [], 'retire': []} - def enableJournalling(self): - '''Turn journalling on for this class - ''' + """Turn journalling on for this class + """ self.do_journal = 1 def disableJournalling(self): - '''Turn journalling off for this class - ''' + """Turn journalling off for this class + """ self.do_journal = 0 # Editing nodes: @@ -722,30 +771,37 @@ class Class(hyperdb.Class): The values of arguments must be acceptable for the types of their corresponding properties or a TypeError is raised. - + If this class has a key property, it must be present and its value must not collide with other key strings or a ValueError is raised. - + Any other properties on this class that are missing from the 'propvalues' dictionary are set to None. - + If an id in a link or multilink property does not refer to a valid node, an IndexError is raised. These operations trigger detectors and can be vetoed. Attempts to modify the "creation" or "activity" properties cause a KeyError. """ + if self.db.journaltag is None: + raise hyperdb.DatabaseError, _('Database open read-only') + self.fireAuditors('create', None, propvalues) + newid = self.create_inner(**propvalues) + self.fireReactors('create', newid, None) + return newid + + def create_inner(self, **propvalues): + """ Called by create, in-between the audit and react calls. + """ if propvalues.has_key('id'): raise KeyError, '"id" is reserved' if self.db.journaltag is None: - raise DatabaseError, 'Database open read-only' + raise hyperdb.DatabaseError, _('Database open read-only') if propvalues.has_key('creation') or propvalues.has_key('activity'): raise KeyError, '"creation" and "activity" are reserved' - - self.fireAuditors('create', None, propvalues) - # new node's id newid = self.db.newid(self.classname) @@ -767,7 +823,7 @@ class Class(hyperdb.Class): raise KeyError, '"%s" has no property "%s"'%(self.classname, key) - if value is not None and isinstance(prop, Link): + if value is not None and isinstance(prop, hyperdb.Link): if type(value) != type(''): raise ValueError, 'link value must be String' link_class = self.properties[key].classname @@ -789,17 +845,19 @@ class Class(hyperdb.Class): self.db.addjournal(link_class, value, 'link', (self.classname, newid, key)) - elif isinstance(prop, Multilink): - if type(value) != type([]): - raise TypeError, 'new property "%s" not a list of ids'%key + elif isinstance(prop, hyperdb.Multilink): + if value is None: + value = [] + if not hasattr(value, '__iter__'): + raise TypeError, 'new property "%s" not an iterable of ids'%key # clean up and validate the list of links link_class = self.properties[key].classname l = [] for entry in value: if type(entry) != type(''): - raise ValueError, '"%s" link value (%s) must be '\ - 'String'%(key, value) + raise ValueError, '"%s" multilink value (%r) '\ + 'must contain Strings'%(key, value) # if it isn't a number, it's a key if not num_re.match(entry): try: @@ -821,29 +879,32 @@ class Class(hyperdb.Class): self.db.addjournal(link_class, nodeid, 'link', (self.classname, newid, key)) - elif isinstance(prop, String): - if type(value) != type(''): + elif isinstance(prop, hyperdb.String): + if type(value) != type('') and type(value) != type(u''): raise TypeError, 'new property "%s" not a string'%key + if prop.indexme: + self.db.indexer.add_text((self.classname, newid, key), + value) - elif isinstance(prop, Password): + elif isinstance(prop, hyperdb.Password): if not isinstance(value, password.Password): raise TypeError, 'new property "%s" not a Password'%key - elif isinstance(prop, Date): + elif isinstance(prop, hyperdb.Date): if value is not None and not isinstance(value, date.Date): raise TypeError, 'new property "%s" not a Date'%key - elif isinstance(prop, Interval): + elif isinstance(prop, hyperdb.Interval): if value is not None and not isinstance(value, date.Interval): raise TypeError, 'new property "%s" not an Interval'%key - elif value is not None and isinstance(prop, Number): + elif value is not None and isinstance(prop, hyperdb.Number): try: float(value) except ValueError: raise TypeError, 'new property "%s" not numeric'%key - elif value is not None and isinstance(prop, Boolean): + elif value is not None and isinstance(prop, hyperdb.Boolean): try: int(value) except ValueError: @@ -855,79 +916,14 @@ class Class(hyperdb.Class): continue if key == self.key: raise ValueError, 'key property "%s" is required'%key - if isinstance(prop, Multilink): + if isinstance(prop, hyperdb.Multilink): propvalues[key] = [] - else: - propvalues[key] = None # done self.db.addnode(self.classname, newid, propvalues) if self.do_journal: - self.db.addjournal(self.classname, newid, 'create', propvalues) - - self.fireReactors('create', newid, None) - - return newid - - def export_list(self, propnames, nodeid): - ''' Export a node - generate a list of CSV-able data in the order - specified by propnames for the given node. - ''' - properties = self.getprops() - l = [] - for prop in propnames: - proptype = properties[prop] - value = self.get(nodeid, prop) - # "marshal" data where needed - if isinstance(proptype, hyperdb.Date): - value = value.get_tuple() - elif isinstance(proptype, hyperdb.Interval): - value = value.get_tuple() - elif isinstance(proptype, hyperdb.Password): - value = str(value) - l.append(repr(value)) - return l - - def import_list(self, propnames, proplist): - ''' Import a node - all information including "id" is present and - should not be sanity checked. Triggers are not triggered. The - journal should be initialised using the "creator" and "created" - information. - - Return the nodeid of the node imported. - ''' - if self.db.journaltag is None: - raise DatabaseError, 'Database open read-only' - properties = self.getprops() - - # make the new node's property map - d = {} - for i in range(len(propnames)): - # Use eval to reverse the repr() used to output the CSV - value = eval(proplist[i]) - - # Figure the property for this column - propname = propnames[i] - prop = properties[propname] - - # "unmarshal" where necessary - if propname == 'id': - newid = value - continue - elif isinstance(prop, hyperdb.Date): - value = date.Date(value) - elif isinstance(prop, hyperdb.Interval): - value = date.Interval(value) - elif isinstance(prop, hyperdb.Password): - pwd = password.Password() - pwd.unpack(value) - value = pwd - if value is not None: - d[propname] = value + self.db.addjournal(self.classname, newid, 'create', {}) - # add - self.db.addnode(self.classname, newid, d) - self.db.addjournal(self.classname, newid, 'create', d) return newid def get(self, nodeid, propname, default=_marker, cache=1): @@ -937,10 +933,7 @@ class Class(hyperdb.Class): IndexError is raised. 'propname' must be the name of a property of this class or a KeyError is raised. - 'cache' indicates whether the transaction cache should be queried - for the node. If the node has been modified and you need to - determine what its values prior to modification are, you need to - set cache=0. + 'cache' exists for backward compatibility, and is not used. Attempts to get the "creation" or "activity" properties should do the right thing. @@ -948,16 +941,24 @@ class Class(hyperdb.Class): if propname == 'id': return nodeid + # get the node's dict + d = self.db.getnode(self.classname, nodeid) + + # check for one of the special props if propname == 'creation': + if d.has_key('creation'): + return d['creation'] if not self.do_journal: raise ValueError, 'Journalling is disabled for this class' journal = self.db.getjournal(self.classname, nodeid) if journal: - return self.db.getjournal(self.classname, nodeid)[0][1] + return journal[0][1] else: # on the strange chance that there's no journal return date.Date() if propname == 'activity': + if d.has_key('activity'): + return d['activity'] if not self.do_journal: raise ValueError, 'Journalling is disabled for this class' journal = self.db.getjournal(self.classname, nodeid) @@ -967,49 +968,67 @@ class Class(hyperdb.Class): # on the strange chance that there's no journal return date.Date() if propname == 'creator': + if d.has_key('creator'): + return d['creator'] + if not self.do_journal: + raise ValueError, 'Journalling is disabled for this class' + journal = self.db.getjournal(self.classname, nodeid) + if journal: + num_re = re.compile('^\d+$') + value = journal[0][2] + if num_re.match(value): + return value + else: + # old-style "username" journal tag + try: + return self.db.user.lookup(value) + except KeyError: + # user's been retired, return admin + return '1' + else: + return self.db.getuid() + if propname == 'actor': + if d.has_key('actor'): + return d['actor'] if not self.do_journal: raise ValueError, 'Journalling is disabled for this class' journal = self.db.getjournal(self.classname, nodeid) if journal: - name = self.db.getjournal(self.classname, nodeid)[0][2] + num_re = re.compile('^\d+$') + value = journal[-1][2] + if num_re.match(value): + return value + else: + # old-style "username" journal tag + try: + return self.db.user.lookup(value) + except KeyError: + # user's been retired, return admin + return '1' else: - return None - return self.db.user.lookup(name) + return self.db.getuid() # get the property (raises KeyErorr if invalid) prop = self.properties[propname] - # get the node's dict - d = self.db.getnode(self.classname, nodeid, cache=cache) - if not d.has_key(propname): if default is _marker: - if isinstance(prop, Multilink): + if isinstance(prop, hyperdb.Multilink): return [] else: return None else: return default - return d[propname] - - # XXX not in spec - def getnode(self, nodeid, cache=1): - ''' Return a convenience wrapper for the node. + # return a dupe of the list so code doesn't get confused + if isinstance(prop, hyperdb.Multilink): + return d[propname][:] - 'nodeid' must be the id of an existing node of this class or an - IndexError is raised. - - 'cache' indicates whether the transaction cache should be queried - for the node. If the node has been modified and you need to - determine what its values prior to modification are, you need to - set cache=0. - ''' - return Node(self, nodeid, cache=cache) + return d[propname] def set(self, nodeid, **propvalues): """Modify a property on an existing node of this class. - + 'nodeid' must be the id of an existing node of this class or an IndexError is raised. @@ -1028,6 +1047,25 @@ class Class(hyperdb.Class): These operations trigger detectors and can be vetoed. Attempts to modify the "creation" or "activity" properties cause a KeyError. """ + if self.db.journaltag is None: + raise hyperdb.DatabaseError, _('Database open read-only') + + self.fireAuditors('set', nodeid, propvalues) + oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid)) + for name,prop in self.getprops(protected=0).items(): + if oldvalues.has_key(name): + continue + if isinstance(prop, hyperdb.Multilink): + oldvalues[name] = [] + else: + oldvalues[name] = None + propvalues = self.set_inner(nodeid, **propvalues) + self.fireReactors('set', nodeid, oldvalues) + return propvalues + + def set_inner(self, nodeid, **propvalues): + """ Called by set, in-between the audit and react calls. + """ if not propvalues: return propvalues @@ -1038,19 +1076,7 @@ class Class(hyperdb.Class): raise KeyError, '"id" is reserved' if self.db.journaltag is None: - raise DatabaseError, 'Database open read-only' - - self.fireAuditors('set', nodeid, propvalues) - # Take a copy of the node dict so that the subsequent set - # operation doesn't modify the oldvalues structure. - try: - # try not using the cache initially - oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid, - cache=0)) - except IndexError: - # this will be needed if somone does a create() and set() - # with no intervening commit() - oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid)) + raise hyperdb.DatabaseError, _('Database open read-only') node = self.db.getnode(self.classname, nodeid) if node.has_key(self.db.RETIRED_FLAG): @@ -1073,16 +1099,22 @@ class Class(hyperdb.Class): # this will raise the KeyError if the property isn't valid # ... we don't use getprops() here because we only care about # the writeable properties. - prop = self.properties[propname] + try: + prop = self.properties[propname] + except KeyError: + raise KeyError, '"%s" has no property named "%s"'%( + self.classname, propname) # if the value's the same as the existing value, no sense in # doing anything - if node.has_key(propname) and value == node[propname]: + current = node.get(propname, None) + if value == current: del propvalues[propname] continue + journalvalues[propname] = current # do stuff based on the prop type - if isinstance(prop, Link): + if isinstance(prop, hyperdb.Link): link_class = prop.classname # if it isn't a number, it's a key if value is not None and not isinstance(value, type('')): @@ -1101,7 +1133,7 @@ class Class(hyperdb.Class): if self.do_journal and prop.do_journal: # register the unlink with the old linked node - if node[propname] is not None: + if node.has_key(propname) and node[propname] is not None: self.db.addjournal(link_class, node[propname], 'unlink', (self.classname, nodeid, propname)) @@ -1110,9 +1142,11 @@ class Class(hyperdb.Class): self.db.addjournal(link_class, value, 'link', (self.classname, nodeid, propname)) - elif isinstance(prop, Multilink): - if type(value) != type([]): - raise TypeError, 'new property "%s" not a list of'\ + elif isinstance(prop, hyperdb.Multilink): + if value is None: + value = [] + if not hasattr(value, '__iter__'): + raise TypeError, 'new property "%s" not an iterable of'\ ' ids'%propname link_class = self.properties[propname].classname l = [] @@ -1167,39 +1201,42 @@ class Class(hyperdb.Class): # figure the journal entry l = [] if add: - l.append(('add', add)) + l.append(('+', add)) if remove: - l.append(('remove', remove)) + l.append(('-', remove)) if l: journalvalues[propname] = tuple(l) - elif isinstance(prop, String): - if value is not None and type(value) != type(''): + elif isinstance(prop, hyperdb.String): + if value is not None and type(value) != type('') and type(value) != type(u''): raise TypeError, 'new property "%s" not a string'%propname + if prop.indexme: + self.db.indexer.add_text((self.classname, nodeid, propname), + value) - elif isinstance(prop, Password): + elif isinstance(prop, hyperdb.Password): if not isinstance(value, password.Password): raise TypeError, 'new property "%s" not a Password'%propname propvalues[propname] = value - elif value is not None and isinstance(prop, Date): + elif value is not None and isinstance(prop, hyperdb.Date): if not isinstance(value, date.Date): raise TypeError, 'new property "%s" not a Date'% propname propvalues[propname] = value - elif value is not None and isinstance(prop, Interval): + elif value is not None and isinstance(prop, hyperdb.Interval): if not isinstance(value, date.Interval): raise TypeError, 'new property "%s" not an '\ 'Interval'%propname propvalues[propname] = value - elif value is not None and isinstance(prop, Number): + elif value is not None and isinstance(prop, hyperdb.Number): try: float(value) except ValueError: raise TypeError, 'new property "%s" not numeric'%propname - elif value is not None and isinstance(prop, Boolean): + elif value is not None and isinstance(prop, hyperdb.Boolean): try: int(value) except ValueError: @@ -1211,23 +1248,24 @@ class Class(hyperdb.Class): if not propvalues: return propvalues + # update the activity time + node['activity'] = date.Date() + node['actor'] = self.db.getuid() + # do the set, and journal it self.db.setnode(self.classname, nodeid, node) if self.do_journal: - propvalues.update(journalvalues) - self.db.addjournal(self.classname, nodeid, 'set', propvalues) - - self.fireReactors('set', nodeid, oldvalues) + self.db.addjournal(self.classname, nodeid, 'set', journalvalues) - return propvalues + return propvalues def retire(self, nodeid): """Retire a node. - + The properties on the node remain available from the get() method, and the node's id is never reused. - + Retired nodes are not returned by the find(), list(), or lookup() methods, and other nodes may reuse the values of their key properties. @@ -1235,7 +1273,7 @@ class Class(hyperdb.Class): to modify the "creation" or "activity" properties cause a KeyError. """ if self.db.journaltag is None: - raise DatabaseError, 'Database open read-only' + raise hyperdb.DatabaseError, _('Database open read-only') self.fireAuditors('retire', nodeid, None) @@ -1247,29 +1285,59 @@ class Class(hyperdb.Class): self.fireReactors('retire', nodeid, None) - def is_retired(self, nodeid): - '''Return true if the node is retired. - ''' - node = self.db.getnode(cn, nodeid, cldb) + def restore(self, nodeid): + """Restpre a retired node. + + Make node available for all operations like it was before retirement. + """ + if self.db.journaltag is None: + raise hyperdb.DatabaseError, _('Database open read-only') + + node = self.db.getnode(self.classname, nodeid) + # check if key property was overrided + key = self.getkey() + try: + id = self.lookup(node[key]) + except KeyError: + pass + else: + raise KeyError, "Key property (%s) of retired node clashes with \ + existing one (%s)" % (key, node[key]) + # Now we can safely restore node + self.fireAuditors('restore', nodeid, None) + del node[self.db.RETIRED_FLAG] + self.db.setnode(self.classname, nodeid, node) + if self.do_journal: + self.db.addjournal(self.classname, nodeid, 'restored', None) + + self.fireReactors('restore', nodeid, None) + + def is_retired(self, nodeid, cldb=None): + """Return true if the node is retired. + """ + node = self.db.getnode(self.classname, nodeid, cldb) if node.has_key(self.db.RETIRED_FLAG): return 1 return 0 def destroy(self, nodeid): """Destroy a node. - + WARNING: this method should never be used except in extremely rare situations where there could never be links to the node being deleted + WARNING: use retire() instead + WARNING: the properties of this node will not be available ever again + WARNING: really, use retire() instead Well, I think that's enough warnings. This method exists mostly to support the session storage of the cgi interface. """ if self.db.journaltag is None: - raise DatabaseError, 'Database open read-only' + raise hyperdb.DatabaseError, _('Database open read-only') self.db.destroynode(self.classname, nodeid) def history(self, nodeid): @@ -1280,7 +1348,7 @@ class Class(hyperdb.Class): The returned list contains tuples of the form - (date, tag, action, params) + (nodeid, date, tag, action, params) 'date' is a Timestamp object specifying the time of the change and 'tag' is the journaltag specified when the database was opened. @@ -1291,8 +1359,8 @@ class Class(hyperdb.Class): # Locating nodes: def hasnode(self, nodeid): - '''Determine if the given nodeid actually exists - ''' + """Determine if the given nodeid actually exists + """ return self.db.hasnode(self.classname, nodeid) def setkey(self, propname): @@ -1304,7 +1372,7 @@ class Class(hyperdb.Class): property doesn't exist, KeyError is raised. """ prop = self.getprops()[propname] - if not isinstance(prop, String): + if not isinstance(prop, hyperdb.String): raise TypeError, 'key properties must be String' self.key = propname @@ -1312,30 +1380,6 @@ class Class(hyperdb.Class): """Return the name of the key property for this class or None.""" return self.key - def labelprop(self, default_to_id=0): - ''' Return the property name for a label for the given node. - - This method attempts to generate a consistent label for the node. - It tries the following in order: - 1. key property - 2. "name" property - 3. "title" property - 4. first property from the sorted property name list - ''' - k = self.getkey() - if k: - return k - props = self.getprops() - if props.has_key('name'): - return 'name' - elif props.has_key('title'): - return 'title' - if default_to_id: - return 'id' - props = props.keys() - props.sort() - return props[0] - # TODO: set up a separate index db file for this? profile? def lookup(self, keyvalue): """Locate a particular node by its key property and return its id. @@ -1345,63 +1389,75 @@ class Class(hyperdb.Class): the nodes in this class, the matching node's id is returned; otherwise a KeyError is raised. """ + if not self.key: + raise TypeError, 'No key property set for class %s'%self.classname cldb = self.db.getclassdb(self.classname) try: - for nodeid in self.db.getnodeids(self.classname, cldb): + for nodeid in self.getnodeids(cldb): node = self.db.getnode(self.classname, nodeid, cldb) if node.has_key(self.db.RETIRED_FLAG): continue + if not node.has_key(self.key): + continue if node[self.key] == keyvalue: - cldb.close() return nodeid finally: cldb.close() - raise KeyError, keyvalue + raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key, + keyvalue, self.classname) - # XXX: change from spec - allows multiple props to match + # change from spec - allows multiple props to match def find(self, **propspec): """Get the ids of nodes in this class which link to the given nodes. - 'propspec' consists of keyword args propname={nodeid:1,} - 'propname' must be the name of a property in this class, or a - KeyError is raised. That property must be a Link or Multilink - property, or a TypeError is raised. + 'propspec' consists of keyword args propname=nodeid or + propname={nodeid:1, } + 'propname' must be the name of a property in this class, or a + KeyError is raised. That property must be a Link or + Multilink property, or a TypeError is raised. + + Any node in this class whose 'propname' property links to any of + the nodeids will be returned. Examples:: - Any node in this class whose 'propname' property links to any of the - nodeids will be returned. Used by the full text indexing, which knows - that "foo" occurs in msg1, msg3 and file7, so we have hits on these issues: + db.issue.find(messages='1') db.issue.find(messages={'1':1,'3':1}, files={'7':1}) """ propspec = propspec.items() - for propname, nodeids in propspec: + for propname, itemids in propspec: # check the prop is OK prop = self.properties[propname] - if not isinstance(prop, Link) and not isinstance(prop, Multilink): + if not isinstance(prop, hyperdb.Link) and not isinstance(prop, hyperdb.Multilink): raise TypeError, "'%s' not a Link/Multilink property"%propname # ok, now do the find cldb = self.db.getclassdb(self.classname) l = [] try: - for id in self.db.getnodeids(self.classname, db=cldb): - node = self.db.getnode(self.classname, id, db=cldb) - if node.has_key(self.db.RETIRED_FLAG): + for id in self.getnodeids(db=cldb): + item = self.db.getnode(self.classname, id, db=cldb) + if item.has_key(self.db.RETIRED_FLAG): continue - for propname, nodeids in propspec: - # can't test if the node doesn't have this property - if not node.has_key(propname): + for propname, itemids in propspec: + if type(itemids) is not type({}): + itemids = {itemids:1} + + # special case if the item doesn't have this property + if not item.has_key(propname): + if itemids.has_key(None): + l.append(id) + break continue - if type(nodeids) is type(''): - nodeids = {nodeids:1} + + # grab the property definition and its value on this item prop = self.properties[propname] - value = node[propname] - if isinstance(prop, Link) and nodeids.has_key(value): + value = item[propname] + if isinstance(prop, hyperdb.Link) and itemids.has_key(value): l.append(id) break - elif isinstance(prop, Multilink): + elif isinstance(prop, hyperdb.Multilink): hit = 0 for v in value: - if nodeids.has_key(v): + if itemids.has_key(v): l.append(id) hit = 1 break @@ -1416,22 +1472,24 @@ class Class(hyperdb.Class): properties in a caseless search. If the property is not a String property, a TypeError is raised. - + The return is a list of the id of all nodes that match. """ for propname in requirements.keys(): prop = self.properties[propname] - if isinstance(not prop, String): + if not isinstance(prop, hyperdb.String): raise TypeError, "'%s' not a String property"%propname requirements[propname] = requirements[propname].lower() l = [] cldb = self.db.getclassdb(self.classname) try: - for nodeid in self.db.getnodeids(self.classname, cldb): + for nodeid in self.getnodeids(cldb): node = self.db.getnode(self.classname, nodeid, cldb) if node.has_key(self.db.RETIRED_FLAG): continue for key, value in requirements.items(): + if not node.has_key(key): + break if node[key] is None or node[key].lower() != value: break else: @@ -1441,12 +1499,13 @@ class Class(hyperdb.Class): return l def list(self): - """Return a list of the ids of the active nodes in this class.""" + """ Return a list of the ids of the active nodes in this class. + """ l = [] cn = self.classname cldb = self.db.getclassdb(cn) try: - for nodeid in self.db.getnodeids(cn, cldb): + for nodeid in self.getnodeids(cldb): node = self.db.getnode(cn, nodeid, cldb) if node.has_key(self.db.RETIRED_FLAG): continue @@ -1456,257 +1515,304 @@ class Class(hyperdb.Class): l.sort() return l - def filter(self, search_matches, filterspec, sort, group, + def getnodeids(self, db=None, retired=None): + """ Return a list of ALL nodeids + + Set retired=None to get all nodes. Otherwise it'll get all the + retired or non-retired nodes, depending on the flag. + """ + res = [] + + # start off with the new nodes + if self.db.newnodes.has_key(self.classname): + res += self.db.newnodes[self.classname].keys() + + must_close = False + if db is None: + db = self.db.getclassdb(self.classname) + must_close = True + try: + res = res + db.keys() + + # remove the uncommitted, destroyed nodes + if self.db.destroyednodes.has_key(self.classname): + for nodeid in self.db.destroyednodes[self.classname].keys(): + if db.has_key(nodeid): + res.remove(nodeid) + + # check retired flag + if retired is False or retired is True: + l = [] + for nodeid in res: + node = self.db.getnode(self.classname, nodeid, db) + is_ret = node.has_key(self.db.RETIRED_FLAG) + if retired == is_ret: + l.append(nodeid) + res = l + finally: + if must_close: + db.close() + return res + + def _filter(self, search_matches, filterspec, proptree, num_re = re.compile('^\d+$')): - ''' Return a list of the ids of the active nodes in this class that - match the 'filter' spec, sorted by the group spec and then the - sort spec. - - "filterspec" is {propname: value(s)} - "sort" is ['+propname', '-propname', 'propname', ...] - "group is ['+propname', '-propname', 'propname', ...] - ''' + """Return a list of the ids of the active nodes in this class that + match the 'filter' spec, sorted by the group spec and then the + sort spec. + + "filterspec" is {propname: value(s)} + + "sort" and "group" are (dir, prop) where dir is '+', '-' or None + and prop is a prop name or None + + "search_matches" is a sequence type or None + + The filter must match all properties specificed. If the property + value to match is a list: + + 1. String properties must match all elements in the list, and + 2. Other properties must match any of the elements in the list. + """ + if __debug__: + start_t = time.time() + cn = self.classname # optimise filterspec l = [] props = self.getprops() - LINK = 0 - MULTILINK = 1 - STRING = 2 - OTHER = 6 + LINK = 'spec:link' + MULTILINK = 'spec:multilink' + STRING = 'spec:string' + DATE = 'spec:date' + INTERVAL = 'spec:interval' + OTHER = 'spec:other' + for k, v in filterspec.items(): propclass = props[k] - if isinstance(propclass, Link): + if isinstance(propclass, hyperdb.Link): if type(v) is not type([]): v = [v] - # replace key values with node ids u = [] - link_class = self.db.classes[propclass.classname] for entry in v: - if entry == '-1': entry = None - elif not num_re.match(entry): - try: - entry = link_class.lookup(entry) - except (TypeError,KeyError): - raise ValueError, 'property "%s": %s not a %s'%( - k, entry, self.properties[k].classname) + # the value -1 is a special "not set" sentinel + if entry == '-1': + entry = None u.append(entry) - l.append((LINK, k, u)) - elif isinstance(propclass, Multilink): + elif isinstance(propclass, hyperdb.Multilink): + # the value -1 is a special "not set" sentinel + if v in ('-1', ['-1']): + v = [] + elif type(v) is not type([]): + v = [v] + l.append((MULTILINK, k, v)) + elif isinstance(propclass, hyperdb.String) and k != 'id': if type(v) is not type([]): v = [v] - # replace key values with node ids - u = [] - link_class = self.db.classes[propclass.classname] - for entry in v: - if not num_re.match(entry): - try: - entry = link_class.lookup(entry) - except (TypeError,KeyError): - raise ValueError, 'new property "%s": %s not a %s'%( - k, entry, self.properties[k].classname) - u.append(entry) - l.append((MULTILINK, k, u)) - elif isinstance(propclass, String): - # simple glob searching - v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v) - v = v.replace('?', '.') - v = v.replace('*', '.*?') - l.append((STRING, k, re.compile(v, re.I))) - elif isinstance(propclass, Boolean): - if type(v) is type(''): - bv = v.lower() in ('yes', 'true', 'on', '1') - else: - bv = v + for v in v: + # simple glob searching + v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v) + v = v.replace('?', '.') + v = v.replace('*', '.*?') + l.append((STRING, k, re.compile(v, re.I))) + elif isinstance(propclass, hyperdb.Date): + try: + date_rng = propclass.range_from_raw(v, self.db) + l.append((DATE, k, date_rng)) + except ValueError: + # If range creation fails - ignore that search parameter + pass + elif isinstance(propclass, hyperdb.Interval): + try: + intv_rng = date.Range(v, date.Interval) + l.append((INTERVAL, k, intv_rng)) + except ValueError: + # If range creation fails - ignore that search parameter + pass + + elif isinstance(propclass, hyperdb.Boolean): + if type(v) != type([]): + v = v.split(',') + bv = [] + for val in v: + if type(val) is type(''): + bv.append(val.lower() in ('yes', 'true', 'on', '1')) + else: + bv.append(val) l.append((OTHER, k, bv)) - elif isinstance(propclass, Number): - l.append((OTHER, k, int(v))) - else: - l.append((OTHER, k, v)) - filterspec = l + elif k == 'id': + if type(v) != type([]): + v = v.split(',') + l.append((OTHER, k, [str(int(val)) for val in v])) + + elif isinstance(propclass, hyperdb.Number): + if type(v) != type([]): + v = v.split(',') + l.append((OTHER, k, [float(val) for val in v])) + + filterspec = l + # now, find all the nodes that are active and pass filtering - l = [] + matches = [] cldb = self.db.getclassdb(cn) + t = 0 try: # TODO: only full-scan once (use items()) - for nodeid in self.db.getnodeids(cn, cldb): + for nodeid in self.getnodeids(cldb): node = self.db.getnode(cn, nodeid, cldb) if node.has_key(self.db.RETIRED_FLAG): continue # apply filter for t, k, v in filterspec: - # make sure the node has the property - if not node.has_key(k): - # this node doesn't have this property, so reject it - break + # handle the id prop + if k == 'id': + if nodeid not in v: + break + continue + + # get the node value + nv = node.get(k, None) + + match = 0 # now apply the property filter if t == LINK: # link - if this node's property doesn't appear in the # filterspec's nodeid list, skip it - if node[k] not in v: - break + match = nv in v elif t == MULTILINK: # multilink - if any of the nodeids required by the # filterspec aren't in this node's property, then skip # it - have = node[k] - for want in v: - if want not in have: - break + nv = node.get(k, []) + + # check for matching the absence of multilink values + if not v: + match = not nv else: - continue - break + # othewise, make sure this node has each of the + # required values + for want in v: + if want in nv: + match = 1 + break elif t == STRING: + if nv is None: + nv = '' # RE search - if node[k] is None or not v.search(node[k]): - break + match = v.search(nv) + elif t == DATE or t == INTERVAL: + if nv is None: + match = v is None + else: + if v.to_value: + if v.from_value <= nv and v.to_value >= nv: + match = 1 + else: + if v.from_value <= nv: + match = 1 elif t == OTHER: # straight value comparison for the other types - if node[k] != v: - break + match = nv in v + if not match: + break else: - l.append((nodeid, node)) + matches.append([nodeid, node]) + + # filter based on full text search + if search_matches is not None: + k = [] + for v in matches: + if v[0] in search_matches: + k.append(v) + matches = k + + # add sorting information to the proptree + JPROPS = {'actor':1, 'activity':1, 'creator':1, 'creation':1} + children = [] + if proptree: + children = proptree.sortable_children() + for pt in children: + dir = pt.sort_direction + prop = pt.name + assert (dir and prop) + propclass = props[prop] + pt.sort_ids = [] + is_pointer = isinstance(propclass,(hyperdb.Link, + hyperdb.Multilink)) + if not is_pointer: + pt.sort_result = [] + try: + # cache the opened link class db, if needed. + lcldb = None + # cache the linked class items too + lcache = {} + + for entry in matches: + itemid = entry[-2] + item = entry[-1] + # handle the properties that might be "faked" + # also, handle possible missing properties + try: + v = item[prop] + except KeyError: + if JPROPS.has_key(prop): + # force lookup of the special journal prop + v = self.get(itemid, prop) + else: + # the node doesn't have a value for this + # property + v = None + if isinstance(propclass, hyperdb.Multilink): + v = [] + if prop == 'id': + v = int (itemid) + pt.sort_ids.append(v) + if not is_pointer: + pt.sort_result.append(v) + continue + + # missing (None) values are always sorted first + if v is None: + pt.sort_ids.append(v) + if not is_pointer: + pt.sort_result.append(v) + continue + + if isinstance(propclass, hyperdb.Link): + lcn = propclass.classname + link = self.db.classes[lcn] + key = link.orderprop() + child = pt.propdict[key] + if key!='id': + if not lcache.has_key(v): + # open the link class db if it's not already + if lcldb is None: + lcldb = self.db.getclassdb(lcn) + lcache[v] = self.db.getnode(lcn, v, lcldb) + r = lcache[v][key] + child.propdict[key].sort_ids.append(r) + else: + child.propdict[key].sort_ids.append(v) + pt.sort_ids.append(v) + if not is_pointer: + r = propclass.sort_repr(pt.parent.cls, v, pt.name) + pt.sort_result.append(r) + finally: + # if we opened the link class db, close it now + if lcldb is not None: + lcldb.close() + del lcache finally: cldb.close() - l.sort() - # filter based on full text search - if search_matches is not None: - k = [] - for v in l: - if search_matches.has_key(v[0]): - k.append(v) - l = k - - # optimise sort - m = [] - for entry in sort: - if entry[0] != '-': - m.append(('+', entry)) - else: - m.append((entry[0], entry[1:])) - sort = m - - # optimise group - m = [] - for entry in group: - if entry[0] != '-': - m.append(('+', entry)) - else: - m.append((entry[0], entry[1:])) - group = m - # now, sort the result - def sortfun(a, b, sort=sort, group=group, properties=self.getprops(), - db = self.db, cl=self): - a_id, an = a - b_id, bn = b - # sort by group and then sort - for list in group, sort: - for dir, prop in list: - # sorting is class-specific - propclass = properties[prop] - - # handle the properties that might be "faked" - # also, handle possible missing properties - try: - if not an.has_key(prop): - an[prop] = cl.get(a_id, prop) - av = an[prop] - except KeyError: - # the node doesn't have a value for this property - if isinstance(propclass, Multilink): av = [] - else: av = '' - try: - if not bn.has_key(prop): - bn[prop] = cl.get(b_id, prop) - bv = bn[prop] - except KeyError: - # the node doesn't have a value for this property - if isinstance(propclass, Multilink): bv = [] - else: bv = '' - - # String and Date values are sorted in the natural way - if isinstance(propclass, String): - # clean up the strings - if av and av[0] in string.uppercase: - av = an[prop] = av.lower() - if bv and bv[0] in string.uppercase: - bv = bn[prop] = bv.lower() - if (isinstance(propclass, String) or - isinstance(propclass, Date)): - # it might be a string that's really an integer - try: - av = int(av) - bv = int(bv) - except: - pass - if dir == '+': - r = cmp(av, bv) - if r != 0: return r - elif dir == '-': - r = cmp(bv, av) - if r != 0: return r - - # Link properties are sorted according to the value of - # the "order" property on the linked nodes if it is - # present; or otherwise on the key string of the linked - # nodes; or finally on the node ids. - elif isinstance(propclass, Link): - link = db.classes[propclass.classname] - if av is None and bv is not None: return -1 - if av is not None and bv is None: return 1 - if av is None and bv is None: continue - if link.getprops().has_key('order'): - if dir == '+': - r = cmp(link.get(av, 'order'), - link.get(bv, 'order')) - if r != 0: return r - elif dir == '-': - r = cmp(link.get(bv, 'order'), - link.get(av, 'order')) - if r != 0: return r - elif link.getkey(): - key = link.getkey() - if dir == '+': - r = cmp(link.get(av, key), link.get(bv, key)) - if r != 0: return r - elif dir == '-': - r = cmp(link.get(bv, key), link.get(av, key)) - if r != 0: return r - else: - if dir == '+': - r = cmp(av, bv) - if r != 0: return r - elif dir == '-': - r = cmp(bv, av) - if r != 0: return r - - # Multilink properties are sorted according to how many - # links are present. - elif isinstance(propclass, Multilink): - if dir == '+': - r = cmp(len(av), len(bv)) - if r != 0: return r - elif dir == '-': - r = cmp(len(bv), len(av)) - if r != 0: return r - elif isinstance(propclass, Number) or isinstance(propclass, Boolean): - if dir == '+': - r = cmp(av, bv) - elif dir == '-': - r = cmp(bv, av) - - # end for dir, prop in list: - # end for list in sort, group: - # if all else fails, compare the ids - return cmp(a[0], b[0]) - - l.sort(sortfun) - return [i[0] for i in l] + # pull the id out of the individual entries + matches = [entry[-2] for entry in matches] + if __debug__: + self.db.stats['filtering'] += (time.time() - start_t) + return matches def count(self): """Get the number of nodes in this class. @@ -1731,10 +1837,11 @@ class Class(hyperdb.Class): """ d = self.properties.copy() if protected: - d['id'] = String() + d['id'] = hyperdb.String() d['creation'] = hyperdb.Date() d['activity'] = hyperdb.Date() - d['creator'] = hyperdb.Link("user") + d['creator'] = hyperdb.Link('user') + d['actor'] = hyperdb.Link('user') return d def addprop(self, **properties): @@ -1751,140 +1858,290 @@ class Class(hyperdb.Class): self.properties.update(properties) def index(self, nodeid): - '''Add (or refresh) the node to search indexes - ''' + """ Add (or refresh) the node to search indexes """ # find all the String properties that have indexme for prop, propclass in self.getprops().items(): - if isinstance(propclass, String) and propclass.indexme: + if isinstance(propclass, hyperdb.String) and propclass.indexme: + # index them under (classname, nodeid, property) try: value = str(self.get(nodeid, prop)) except IndexError: - # node no longer exists - entry should be removed - self.db.indexer.purge_entry((self.classname, nodeid, prop)) - else: - # and index them under (classname, nodeid, property) - self.db.indexer.add_text((self.classname, nodeid, prop), - value) + # node has been destroyed + continue + self.db.indexer.add_text((self.classname, nodeid, prop), value) # - # Detector interface + # import / export support # - def audit(self, event, detector): - """Register a detector + def export_list(self, propnames, nodeid): + """ Export a node - generate a list of CSV-able data in the order + specified by propnames for the given node. """ - l = self.auditors[event] - if detector not in l: - self.auditors[event].append(detector) + properties = self.getprops() + l = [] + for prop in propnames: + proptype = properties[prop] + value = self.get(nodeid, prop) + # "marshal" data where needed + if value is None: + pass + elif isinstance(proptype, hyperdb.Date): + value = value.get_tuple() + elif isinstance(proptype, hyperdb.Interval): + value = value.get_tuple() + elif isinstance(proptype, hyperdb.Password): + value = str(value) + l.append(repr(value)) - def fireAuditors(self, action, nodeid, newvalues): - """Fire all registered auditors. - """ - for audit in self.auditors[action]: - audit(self.db, self, nodeid, newvalues) + # append retired flag + l.append(repr(self.is_retired(nodeid))) + + return l + + def import_list(self, propnames, proplist): + """ Import a node - all information including "id" is present and + should not be sanity checked. Triggers are not triggered. The + journal should be initialised using the "creator" and "created" + information. - def react(self, event, detector): - """Register a detector + Return the nodeid of the node imported. """ - l = self.reactors[event] - if detector not in l: - self.reactors[event].append(detector) + if self.db.journaltag is None: + raise hyperdb.DatabaseError, _('Database open read-only') + properties = self.getprops() + + # make the new node's property map + d = {} + newid = None + for i in range(len(propnames)): + # Figure the property for this column + propname = propnames[i] + + # Use eval to reverse the repr() used to output the CSV + value = eval(proplist[i]) + + # "unmarshal" where necessary + if propname == 'id': + newid = value + continue + elif propname == 'is retired': + # is the item retired? + if int(value): + d[self.db.RETIRED_FLAG] = 1 + continue + elif value is None: + d[propname] = None + continue + + prop = properties[propname] + if isinstance(prop, hyperdb.Date): + value = date.Date(value) + elif isinstance(prop, hyperdb.Interval): + value = date.Interval(value) + elif isinstance(prop, hyperdb.Password): + pwd = password.Password() + pwd.unpack(value) + value = pwd + d[propname] = value + + # get a new id if necessary + if newid is None: + newid = self.db.newid(self.classname) + + # add the node and journal + self.db.addnode(self.classname, newid, d) + return newid - def fireReactors(self, action, nodeid, oldvalues): - """Fire all registered reactors. + def export_journals(self): + """Export a class's journal - generate a list of lists of + CSV-able data: + + nodeid, date, user, action, params + + No heading here - the columns are fixed. """ - for react in self.reactors[action]: - react(self.db, self, nodeid, oldvalues) + properties = self.getprops() + r = [] + for nodeid in self.getnodeids(): + for nodeid, date, user, action, params in self.history(nodeid): + date = date.get_tuple() + if action == 'set': + export_data = {} + for propname, value in params.items(): + if not properties.has_key(propname): + # property no longer in the schema + continue -class FileClass(Class): - '''This class defines a large chunk of data. To support this, it has a + prop = properties[propname] + # make sure the params are eval()'able + if value is None: + pass + elif isinstance(prop, hyperdb.Date): + # this is a hack - some dates are stored as strings + if not isinstance(value, type('')): + value = value.get_tuple() + elif isinstance(prop, hyperdb.Interval): + # hack too - some intervals are stored as strings + if not isinstance(value, type('')): + value = value.get_tuple() + elif isinstance(prop, hyperdb.Password): + value = str(value) + export_data[propname] = value + params = export_data + l = [nodeid, date, user, action, params] + r.append(map(repr, l)) + return r + + def import_journals(self, entries): + """Import a class's journal. + + Uses setjournal() to set the journal for each item.""" + properties = self.getprops() + d = {} + for l in entries: + l = map(eval, l) + nodeid, jdate, user, action, params = l + r = d.setdefault(nodeid, []) + if action == 'set': + for propname, value in params.items(): + prop = properties[propname] + if value is None: + pass + elif isinstance(prop, hyperdb.Date): + value = date.Date(value) + elif isinstance(prop, hyperdb.Interval): + value = date.Interval(value) + elif isinstance(prop, hyperdb.Password): + pwd = password.Password() + pwd.unpack(value) + value = pwd + params[propname] = value + r.append((nodeid, date.Date(jdate), user, action, params)) + + for nodeid, l in d.items(): + self.db.setjournal(self.classname, nodeid, l) + +class FileClass(hyperdb.FileClass, Class): + """This class defines a large chunk of data. To support this, it has a mandatory String property "content" which is typically saved off externally to the hyperdb. The default MIME type of this data is defined by the "default_mime_type" class attribute, which may be overridden by each node if the class defines a "type" String property. - ''' - default_mime_type = 'text/plain' + """ + def __init__(self, db, classname, **properties): + """The newly-created class automatically includes the "content" + and "type" properties. + """ + if not properties.has_key('content'): + properties['content'] = hyperdb.String(indexme='yes') + if not properties.has_key('type'): + properties['type'] = hyperdb.String() + Class.__init__(self, db, classname, **properties) def create(self, **propvalues): - ''' snaffle the file propvalue and store in a file - ''' + """ Snarf the "content" propvalue and store in a file + """ + # we need to fire the auditors now, or the content property won't + # be in propvalues for the auditors to play with + self.fireAuditors('create', None, propvalues) + + # now remove the content property so it's not stored in the db content = propvalues['content'] del propvalues['content'] - newid = Class.create(self, **propvalues) - self.db.storefile(self.classname, newid, None, content) - return newid - - def import_list(self, propnames, proplist): - ''' Trap the "content" property... - ''' - # dupe this list so we don't affect others - propnames = propnames[:] - # extract the "content" property from the proplist - i = propnames.index('content') - content = proplist[i] - del propnames[i] - del proplist[i] + # make sure we have a MIME type + mime_type = propvalues.get('type', self.default_mime_type) - # do the normal import - newid = Class.import_list(self, propnames, proplist) + # do the database create + newid = self.create_inner(**propvalues) - # save off the "content" file + # store off the content as a file self.db.storefile(self.classname, newid, None, content) + + # fire reactors + self.fireReactors('create', newid, None) + return newid def get(self, nodeid, propname, default=_marker, cache=1): - ''' trap the content propname and get it from the file - ''' + """ Trap the content propname and get it from the file - poss_msg = 'Possibly a access right configuration problem.' + 'cache' exists for backwards compatibility, and is not used. + """ + poss_msg = 'Possibly an access right configuration problem.' if propname == 'content': try: return self.db.getfile(self.classname, nodeid, None) except IOError, (strerror): - # BUG: by catching this we donot see an error in the log. + # XXX by catching this we don't see an error in the log. return 'ERROR reading file: %s%s\n%s\n%s'%( self.classname, nodeid, poss_msg, strerror) if default is not _marker: - return Class.get(self, nodeid, propname, default, cache=cache) + return Class.get(self, nodeid, propname, default) else: - return Class.get(self, nodeid, propname, cache=cache) + return Class.get(self, nodeid, propname) - def getprops(self, protected=1): - ''' In addition to the actual properties on the node, these methods - provide the "content" property. If the "protected" flag is true, - we include protected properties - those which may not be - modified. - ''' - d = Class.getprops(self, protected=protected).copy() - if protected: - d['content'] = hyperdb.String() - return d - - def index(self, nodeid): - ''' Index the node in the search index. - - We want to index the content in addition to the normal String - property indexing. - ''' - # perform normal indexing - Class.index(self, nodeid) + def set(self, itemid, **propvalues): + """ Snarf the "content" propvalue and update it in a file + """ + self.fireAuditors('set', itemid, propvalues) - # get the content to index - content = self.get(nodeid, 'content') + # create the oldvalues dict - fill in any missing values + oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid)) + for name,prop in self.getprops(protected=0).items(): + if oldvalues.has_key(name): + continue + if isinstance(prop, hyperdb.Multilink): + oldvalues[name] = [] + else: + oldvalues[name] = None + + # now remove the content property so it's not stored in the db + content = None + if propvalues.has_key('content'): + content = propvalues['content'] + del propvalues['content'] + + # do the database update + propvalues = self.set_inner(itemid, **propvalues) + + # do content? + if content: + # store and possibly index + self.db.storefile(self.classname, itemid, None, content) + if self.properties['content'].indexme: + mime_type = self.get(itemid, 'type', self.default_mime_type) + self.db.indexer.add_text((self.classname, itemid, 'content'), + content, mime_type) + propvalues['content'] = content + + # fire reactors + self.fireReactors('set', itemid, oldvalues) + return propvalues - # figure the mime type - if self.properties.has_key('type'): - mime_type = self.get(nodeid, 'type') - else: - mime_type = self.default_mime_type + def index(self, nodeid): + """ Add (or refresh) the node to search indexes. - # and index! - self.db.indexer.add_text((self.classname, nodeid, 'content'), content, - mime_type) + Use the content-type property for the content property. + """ + # find all the String properties that have indexme + for prop, propclass in self.getprops().items(): + if prop == 'content' and propclass.indexme: + mime_type = self.get(nodeid, 'type', self.default_mime_type) + self.db.indexer.add_text((self.classname, nodeid, 'content'), + str(self.get(nodeid, 'content')), mime_type) + elif isinstance(propclass, hyperdb.String) and propclass.indexme: + # index them under (classname, nodeid, property) + try: + value = str(self.get(nodeid, prop)) + except IndexError: + # node has been destroyed + continue + self.db.indexer.add_text((self.classname, nodeid, prop), value) -# XXX deviation from spec - was called ItemClass +# deviation from spec - was called ItemClass class IssueClass(Class, roundupdb.IssueClass): # Overridden methods: def __init__(self, db, classname, **properties): @@ -1900,326 +2157,11 @@ class IssueClass(Class, roundupdb.IssueClass): if not properties.has_key('files'): properties['files'] = hyperdb.Multilink("file") if not properties.has_key('nosy'): - properties['nosy'] = hyperdb.Multilink("user") + # note: journalling is turned off as it really just wastes + # space. this behaviour may be overridden in an instance + properties['nosy'] = hyperdb.Multilink("user", do_journal="no") if not properties.has_key('superseder'): properties['superseder'] = hyperdb.Multilink(classname) Class.__init__(self, db, classname, **properties) -# -#$Log: not supported by cvs2svn $ -#Revision 1.60 2002/08/19 00:23:19 richard -#handle "unset" initial Link values (!) -# -#Revision 1.59 2002/08/16 04:28:13 richard -#added is_retired query to Class -# -#Revision 1.58 2002/08/01 15:06:24 gmcm -#Use same regex to split search terms as used to index text. -#Fix to back_metakit for not changing journaltag on reopen. -#Fix htmltemplate's do_link so [No ] strings are href'd. -#Fix bogus "nosy edited ok" msg - the **d syntax does NOT share d between caller and callee. -# -#Revision 1.57 2002/07/31 23:57:36 richard -# . web forms may now unset Link values (like assignedto) -# -#Revision 1.56 2002/07/31 22:04:33 richard -#cleanup -# -#Revision 1.55 2002/07/30 08:22:38 richard -#Session storage in the hyperdb was horribly, horribly inefficient. We use -#a simple anydbm wrapper now - which could be overridden by the metakit -#backend or RDB backend if necessary. -#Much, much better. -# -#Revision 1.54 2002/07/26 08:26:59 richard -#Very close now. The cgi and mailgw now use the new security API. The two -#templates have been migrated to that setup. Lots of unit tests. Still some -#issue in the web form for editing Roles assigned to users. -# -#Revision 1.53 2002/07/25 07:14:06 richard -#Bugger it. Here's the current shape of the new security implementation. -#Still to do: -# . call the security funcs from cgi and mailgw -# . change shipped templates to include correct initialisation and remove -# the old config vars -#... that seems like a lot. The bulk of the work has been done though. Honest :) -# -#Revision 1.52 2002/07/19 03:36:34 richard -#Implemented the destroy() method needed by the session database (and possibly -#others). At the same time, I removed the leading underscores from the hyperdb -#methods that Really Didn't Need Them. -#The journal also raises IndexError now for all situations where there is a -#request for the journal of a node that doesn't have one. It used to return -#[] in _some_ situations, but not all. This _may_ break code, but the tests -#pass... -# -#Revision 1.51 2002/07/18 23:07:08 richard -#Unit tests and a few fixes. -# -#Revision 1.50 2002/07/18 11:50:58 richard -#added tests for number type too -# -#Revision 1.49 2002/07/18 11:41:10 richard -#added tests for boolean type, and fixes to anydbm backend -# -#Revision 1.48 2002/07/18 11:17:31 gmcm -#Add Number and Boolean types to hyperdb. -#Add conversion cases to web, mail & admin interfaces. -#Add storage/serialization cases to back_anydbm & back_metakit. -# -#Revision 1.47 2002/07/14 23:18:20 richard -#. fixed the journal bloat from multilink changes - we just log the add or -# remove operations, not the whole list -# -#Revision 1.46 2002/07/14 06:06:34 richard -#Did some old TODOs -# -#Revision 1.45 2002/07/14 04:03:14 richard -#Implemented a switch to disable journalling for a Class. CGI session -#database now uses it. -# -#Revision 1.44 2002/07/14 02:05:53 richard -#. all storage-specific code (ie. backend) is now implemented by the backends -# -#Revision 1.43 2002/07/10 06:30:30 richard -#...except of course it's nice to use valid Python syntax -# -#Revision 1.42 2002/07/10 06:21:38 richard -#Be extra safe -# -#Revision 1.41 2002/07/10 00:21:45 richard -#explicit database closing -# -#Revision 1.40 2002/07/09 04:19:09 richard -#Added reindex command to roundup-admin. -#Fixed reindex on first access. -#Also fixed reindexing of entries that change. -# -#Revision 1.39 2002/07/09 03:02:52 richard -#More indexer work: -#- all String properties may now be indexed too. Currently there's a bit of -# "issue" specific code in the actual searching which needs to be -# addressed. In a nutshell: -# + pass 'indexme="yes"' as a String() property initialisation arg, eg: -# file = FileClass(db, "file", name=String(), type=String(), -# comment=String(indexme="yes")) -# + the comment will then be indexed and be searchable, with the results -# related back to the issue that the file is linked to -#- as a result of this work, the FileClass has a default MIME type that may -# be overridden in a subclass, or by the use of a "type" property as is -# done in the default templates. -#- the regeneration of the indexes (if necessary) is done once the schema is -# set up in the dbinit. -# -#Revision 1.38 2002/07/08 06:58:15 richard -#cleaned up the indexer code: -# - it splits more words out (much simpler, faster splitter) -# - removed code we'll never use (roundup.roundup_indexer has the full -# implementation, and replaces roundup.indexer) -# - only index text/plain and rfc822/message (ideas for other text formats to -# index are welcome) -# - added simple unit test for indexer. Needs more tests for regression. -# -#Revision 1.37 2002/06/20 23:52:35 richard -#More informative error message -# -#Revision 1.36 2002/06/19 03:07:19 richard -#Moved the file storage commit into blobfiles where it belongs. -# -#Revision 1.35 2002/05/25 07:16:24 rochecompaan -#Merged search_indexing-branch with HEAD -# -#Revision 1.34 2002/05/15 06:21:21 richard -# . node caching now works, and gives a small boost in performance -# -#As a part of this, I cleaned up the DEBUG output and implemented TRACE -#output (HYPERDBTRACE='file to trace to') with checkpoints at the start of -#CGI requests. Run roundup with python -O to skip all the DEBUG/TRACE stuff -#(using if __debug__ which is compiled out with -O) -# -#Revision 1.33 2002/04/24 10:38:26 rochecompaan -#All database files are now created group readable and writable. -# -#Revision 1.32 2002/04/15 23:25:15 richard -#. node ids are now generated from a lockable store - no more race conditions -# -#We're using the portalocker code by Jonathan Feinberg that was contributed -#to the ASPN Python cookbook. This gives us locking across Unix and Windows. -# -#Revision 1.31 2002/04/03 05:54:31 richard -#Fixed serialisation problem by moving the serialisation step out of the -#hyperdb.Class (get, set) into the hyperdb.Database. -# -#Also fixed htmltemplate after the showid changes I made yesterday. -# -#Unit tests for all of the above written. -# -#Revision 1.30.2.1 2002/04/03 11:55:57 rochecompaan -# . Added feature #526730 - search for messages capability -# -#Revision 1.30 2002/02/27 03:40:59 richard -#Ran it through pychecker, made fixes -# -#Revision 1.29 2002/02/25 14:34:31 grubert -# . use blobfiles in back_anydbm which is used in back_bsddb. -# change test_db as dirlist does not work for subdirectories. -# ATTENTION: blobfiles now creates subdirectories for files. -# -#Revision 1.28 2002/02/16 09:14:17 richard -# . #514854 ] History: "User" is always ticket creator -# -#Revision 1.27 2002/01/22 07:21:13 richard -#. fixed back_bsddb so it passed the journal tests -# -#... it didn't seem happy using the back_anydbm _open method, which is odd. -#Yet another occurrance of whichdb not being able to recognise older bsddb -#databases. Yadda yadda. Made the HYPERDBDEBUG stuff more sane in the -#process. -# -#Revision 1.26 2002/01/22 05:18:38 rochecompaan -#last_set_entry was referenced before assignment -# -#Revision 1.25 2002/01/22 05:06:08 rochecompaan -#We need to keep the last 'set' entry in the journal to preserve -#information on 'activity' for nodes. -# -#Revision 1.24 2002/01/21 16:33:20 rochecompaan -#You can now use the roundup-admin tool to pack the database -# -#Revision 1.23 2002/01/18 04:32:04 richard -#Rollback was breaking because a message hadn't actually been written to the file. Needs -#more investigation. -# -#Revision 1.22 2002/01/14 02:20:15 richard -# . changed all config accesses so they access either the instance or the -# config attriubute on the db. This means that all config is obtained from -# instance_config instead of the mish-mash of classes. This will make -# switching to a ConfigParser setup easier too, I hope. -# -#At a minimum, this makes migration a _little_ easier (a lot easier in the -#0.5.0 switch, I hope!) -# -#Revision 1.21 2002/01/02 02:31:38 richard -#Sorry for the huge checkin message - I was only intending to implement #496356 -#but I found a number of places where things had been broken by transactions: -# . modified ROUNDUPDBSENDMAILDEBUG to be SENDMAILDEBUG and hold a filename -# for _all_ roundup-generated smtp messages to be sent to. -# . the transaction cache had broken the roundupdb.Class set() reactors -# . newly-created author users in the mailgw weren't being committed to the db -# -#Stuff that made it into CHANGES.txt (ie. the stuff I was actually working -#on when I found that stuff :): -# . #496356 ] Use threading in messages -# . detectors were being registered multiple times -# . added tests for mailgw -# . much better attaching of erroneous messages in the mail gateway -# -#Revision 1.20 2001/12/18 15:30:34 rochecompaan -#Fixed bugs: -# . Fixed file creation and retrieval in same transaction in anydbm -# backend -# . Cgi interface now renders new issue after issue creation -# . Could not set issue status to resolved through cgi interface -# . Mail gateway was changing status back to 'chatting' if status was -# omitted as an argument -# -#Revision 1.19 2001/12/17 03:52:48 richard -#Implemented file store rollback. As a bonus, the hyperdb is now capable of -#storing more than one file per node - if a property name is supplied, -#the file is called designator.property. -#I decided not to migrate the existing files stored over to the new naming -#scheme - the FileClass just doesn't specify the property name. -# -#Revision 1.18 2001/12/16 10:53:38 richard -#take a copy of the node dict so that the subsequent set -#operation doesn't modify the oldvalues structure -# -#Revision 1.17 2001/12/14 23:42:57 richard -#yuck, a gdbm instance tests false :( -#I've left the debugging code in - it should be removed one day if we're ever -#_really_ anal about performace :) -# -#Revision 1.16 2001/12/12 03:23:14 richard -#Cor blimey this anydbm/whichdb stuff is yecchy. Turns out that whichdb -#incorrectly identifies a dbm file as a dbhash file on my system. This has -#been submitted to the python bug tracker as issue #491888: -#https://sourceforge.net/tracker/index.php?func=detail&aid=491888&group_id=5470&atid=105470 -# -#Revision 1.15 2001/12/12 02:30:51 richard -#I fixed the problems with people whose anydbm was using the dbm module at the -#backend. It turns out the dbm module modifies the file name to append ".db" -#and my check to determine if we're opening an existing or new db just -#tested os.path.exists() on the filename. Well, no longer! We now perform a -#much better check _and_ cope with the anydbm implementation module changing -#too! -#I also fixed the backends __init__ so only ImportError is squashed. -# -#Revision 1.14 2001/12/10 22:20:01 richard -#Enabled transaction support in the bsddb backend. It uses the anydbm code -#where possible, only replacing methods where the db is opened (it uses the -#btree opener specifically.) -#Also cleaned up some change note generation. -#Made the backends package work with pydoc too. -# -#Revision 1.13 2001/12/02 05:06:16 richard -#. We now use weakrefs in the Classes to keep the database reference, so -# the close() method on the database is no longer needed. -# I bumped the minimum python requirement up to 2.1 accordingly. -#. #487480 ] roundup-server -#. #487476 ] INSTALL.txt -# -#I also cleaned up the change message / post-edit stuff in the cgi client. -#There's now a clearly marked "TODO: append the change note" where I believe -#the change note should be added there. The "changes" list will obviously -#have to be modified to be a dict of the changes, or somesuch. -# -#More testing needed. -# -#Revision 1.12 2001/12/01 07:17:50 richard -#. We now have basic transaction support! Information is only written to -# the database when the commit() method is called. Only the anydbm -# backend is modified in this way - neither of the bsddb backends have been. -# The mail, admin and cgi interfaces all use commit (except the admin tool -# doesn't have a commit command, so interactive users can't commit...) -#. Fixed login/registration forwarding the user to the right page (or not, -# on a failure) -# -#Revision 1.11 2001/11/21 02:34:18 richard -#Added a target version field to the extended issue schema -# -#Revision 1.10 2001/10/09 23:58:10 richard -#Moved the data stringification up into the hyperdb.Class class' get, set -#and create methods. This means that the data is also stringified for the -#journal call, and removes duplication of code from the backends. The -#backend code now only sees strings. -# -#Revision 1.9 2001/10/09 07:25:59 richard -#Added the Password property type. See "pydoc roundup.password" for -#implementation details. Have updated some of the documentation too. -# -#Revision 1.8 2001/09/29 13:27:00 richard -#CGI interfaces now spit up a top-level index of all the instances they can -#serve. -# -#Revision 1.7 2001/08/12 06:32:36 richard -#using isinstance(blah, Foo) now instead of isFooType -# -#Revision 1.6 2001/08/07 00:24:42 richard -#stupid typo -# -#Revision 1.5 2001/08/07 00:15:51 richard -#Added the copyright/license notice to (nearly) all files at request of -#Bizar Software. -# -#Revision 1.4 2001/07/30 01:41:36 richard -#Makes schema changes mucho easier. -# -#Revision 1.3 2001/07/25 01:23:07 richard -#Added the Roundup spec to the new documentation directory. -# -#Revision 1.2 2001/07/23 08:20:44 richard -#Moved over to using marshal in the bsddb and anydbm backends. -#roundup-admin now has a "freshen" command that'll load/save all nodes (not -# retired - mod hyperdb.Class.list() so it lists retired nodes) -# -# +# vim: set et sts=4 sw=4 :