X-Git-Url: https://git.tokkee.org/?a=blobdiff_plain;f=roundup%2Fbackends%2Fback_anydbm.py;h=979382f50533736619b38de15c4373adf8de69ba;hb=03694a1acfcf8704487c6c345de65a7940fd3583;hp=fa550fdc84b5dd9a3c6f1b0db62a1378c775d8bd;hpb=b72488541b2ffa5376e367c69bc6d10e899cc02f;p=roundup.git diff --git a/roundup/backends/back_anydbm.py b/roundup/backends/back_anydbm.py index fa550fd..979382f 100644 --- a/roundup/backends/back_anydbm.py +++ b/roundup/backends/back_anydbm.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -#$Id: back_anydbm.py,v 1.50 2002-07-18 11:50:58 richard Exp $ +#$Id: back_anydbm.py,v 1.57 2002-07-31 23:57:36 richard Exp $ ''' This module defines a backend that saves the hyperdatabase in a database chosen by anydbm. It is guaranteed to always be available in python @@ -24,8 +24,9 @@ serious bugs, and is not available) ''' import whichdb, anydbm, os, marshal, re, weakref, string, copy -from roundup import hyperdb, date, password, roundupdb +from roundup import hyperdb, date, password, roundupdb, security from blobfiles import FileStorage +from sessions import Sessions from roundup.indexer import Indexer from locking import acquire_lock, release_lock from roundup.hyperdb import String, Password, Date, Interval, Link, \ @@ -63,8 +64,11 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): self.cache = {} # cache of nodes loaded or created self.dirtynodes = {} # keep track of the dirty nodes by class self.newnodes = {} # keep track of the new nodes by class + self.destroyednodes = {}# keep track of the destroyed nodes by class self.transactions = [] self.indexer = Indexer(self.dir) + self.sessions = Sessions(self.config) + self.security = security.Security(self) # ensure files are group readable and writable os.umask(0002) @@ -141,7 +145,7 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): ''' if __debug__: print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode) - return self._opendb('nodes.%s'%classname, mode) + return self.opendb('nodes.%s'%classname, mode) def determine_db_type(self, path): ''' determine which DB wrote the class file @@ -157,12 +161,12 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): db_type = 'dbm' return db_type - def _opendb(self, name, mode): + def opendb(self, name, mode): '''Low-level database opener that gets around anydbm/dbm eccentricities. ''' if __debug__: - print >>hyperdb.DEBUG, '_opendb', (self, name, mode) + print >>hyperdb.DEBUG, 'opendb', (self, name, mode) # figure the class db type path = os.path.join(os.getcwd(), self.dir, name) @@ -171,7 +175,7 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): # new database? let anydbm pick the best dbm if not db_type: if __debug__: - print >>hyperdb.DEBUG, "_opendb anydbm.open(%r, 'n')"%path + print >>hyperdb.DEBUG, "opendb anydbm.open(%r, 'n')"%path return anydbm.open(path, 'n') # open the database with the correct module @@ -182,11 +186,11 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): "Couldn't open database - the required module '%s'"\ " is not available"%db_type if __debug__: - print >>hyperdb.DEBUG, "_opendb %r.open(%r, %r)"%(db_type, path, + print >>hyperdb.DEBUG, "opendb %r.open(%r, %r)"%(db_type, path, mode) return dbm.open(path, mode) - def _lockdb(self, name): + def lockdb(self, name): ''' Lock a database file ''' path = os.path.join(os.getcwd(), self.dir, '%s.lock'%name) @@ -199,8 +203,8 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): ''' Generate a new id for the given class ''' # open the ids DB - create if if doesn't exist - lock = self._lockdb('_ids') - db = self._opendb('_ids', 'c') + lock = self.lockdb('_ids') + db = self.opendb('_ids', 'c') if db.has_key(classname): newid = db[classname] = str(int(db[classname]) + 1) else: @@ -239,7 +243,7 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): ''' if __debug__: print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node) - self.transactions.append((self._doSaveNode, (classname, nodeid, node))) + self.transactions.append((self.doSaveNode, (classname, nodeid, node))) def getnode(self, classname, nodeid, db=None, cache=1): ''' get a node from the database @@ -264,6 +268,11 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): if not db.has_key(nodeid): raise IndexError, "no such %s %s"%(classname, nodeid) + # check the uncommitted, destroyed nodes + if (self.destroyednodes.has_key(classname) and + self.destroyednodes[classname].has_key(nodeid)): + raise IndexError, "no such %s %s"%(classname, nodeid) + # decode res = marshal.loads(db[nodeid]) @@ -276,6 +285,32 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): return res + def destroynode(self, classname, nodeid): + '''Remove a node from the database. Called exclusively by the + destroy() method on Class. + ''' + if __debug__: + print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid) + + # remove from cache and newnodes if it's there + if (self.cache.has_key(classname) and + self.cache[classname].has_key(nodeid)): + del self.cache[classname][nodeid] + if (self.newnodes.has_key(classname) and + self.newnodes[classname].has_key(nodeid)): + del self.newnodes[classname][nodeid] + + # see if there's any obvious commit actions that we should get rid of + for entry in self.transactions[:]: + if entry[1][:2] == (classname, nodeid): + self.transactions.remove(entry) + + # add to the destroyednodes map + self.destroyednodes.setdefault(classname, {})[nodeid] = 1 + + # add the destroy commit action + self.transactions.append((self.doDestroyNode, (classname, nodeid))) + def serialise(self, classname, node): '''Copy the node contents, converting non-marshallable data into marshallable data. @@ -357,8 +392,14 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): def countnodes(self, classname, db=None): if __debug__: print >>hyperdb.DEBUG, 'countnodes', (self, classname, db) - # include the new nodes not saved to the DB yet - count = len(self.newnodes.get(classname, {})) + + count = 0 + + # include the uncommitted nodes + if self.newnodes.has_key(classname): + count += len(self.newnodes[classname]) + if self.destroyednodes.has_key(classname): + count -= len(self.destroyednodes[classname]) # and count those in the DB if db is None: @@ -369,12 +410,23 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): def getnodeids(self, classname, db=None): if __debug__: print >>hyperdb.DEBUG, 'getnodeids', (self, classname, db) + + res = [] + # start off with the new nodes - res = self.newnodes.get(classname, {}).keys() + if self.newnodes.has_key(classname): + res += self.newnodes[classname].keys() if db is None: db = self.getclassdb(classname) res = res + db.keys() + + # remove the uncommitted, destroyed nodes + if self.destroyednodes.has_key(classname): + for nodeid in self.destroyednodes[classname].keys(): + if db.has_key(nodeid): + res.remove(nodeid) + return res @@ -396,33 +448,36 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): if __debug__: print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid, action, params) - self.transactions.append((self._doSaveJournal, (classname, nodeid, + self.transactions.append((self.doSaveJournal, (classname, nodeid, action, params))) def getjournal(self, classname, nodeid): ''' get the journal for id + + Raise IndexError if the node doesn't exist (as per history()'s + API) ''' if __debug__: print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid) # attempt to open the journal - in some rare cases, the journal may # not exist try: - db = self._opendb('journals.%s'%classname, 'r') + db = self.opendb('journals.%s'%classname, 'r') except anydbm.error, error: - if str(error) == "need 'c' or 'n' flag to open new db": return [] - elif error.args[0] != 2: raise - return [] + if str(error) == "need 'c' or 'n' flag to open new db": + raise IndexError, 'no such %s %s'%(classname, nodeid) + elif error.args[0] != 2: + raise + raise IndexError, 'no such %s %s'%(classname, nodeid) try: journal = marshal.loads(db[nodeid]) except KeyError: db.close() - raise KeyError, 'no such %s %s'%(classname, nodeid) + raise IndexError, 'no such %s %s'%(classname, nodeid) db.close() res = [] - for entry in journal: - (nodeid, date_stamp, user, action, params) = entry - date_obj = date.Date(date_stamp) - res.append((nodeid, date_obj, user, action, params)) + for nodeid, date_stamp, user, action, params in journal: + res.append((nodeid, date.Date(date_stamp), user, action, params)) return res def pack(self, pack_before): @@ -440,7 +495,7 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): db_name = 'journals.%s'%classname path = os.path.join(os.getcwd(), self.dir, classname) db_type = self.determine_db_type(path) - db = self._opendb(db_name, 'w') + db = self.opendb(db_name, 'w') for key in db.keys(): journal = marshal.loads(db[key]) @@ -503,19 +558,24 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): self.cache = {} self.dirtynodes = {} self.newnodes = {} + self.destroyednodes = {} self.transactions = [] - def _doSaveNode(self, classname, nodeid, node): + def getCachedClassDB(self, classname): + ''' get the class db, looking in our cache of databases for commit + ''' + # get the database handle + db_name = 'nodes.%s'%classname + if not self.databases.has_key(db_name): + self.databases[db_name] = self.getclassdb(classname, 'c') + return self.databases[db_name] + + def doSaveNode(self, classname, nodeid, node): if __debug__: - print >>hyperdb.DEBUG, '_doSaveNode', (self, classname, nodeid, + print >>hyperdb.DEBUG, 'doSaveNode', (self, classname, nodeid, node) - # get the database handle - db_name = 'nodes.%s'%classname - if self.databases.has_key(db_name): - db = self.databases[db_name] - else: - db = self.databases[db_name] = self.getclassdb(classname, 'c') + db = self.getCachedClassDB(classname) # now save the marshalled data db[nodeid] = marshal.dumps(self.serialise(classname, node)) @@ -523,7 +583,16 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): # return the classname, nodeid so we reindex this content return (classname, nodeid) - def _doSaveJournal(self, classname, nodeid, action, params): + def getCachedJournalDB(self, classname): + ''' get the journal db, looking in our cache of databases for commit + ''' + # get the database handle + db_name = 'journals.%s'%classname + if not self.databases.has_key(db_name): + self.databases[db_name] = self.opendb(db_name, 'c') + return self.databases[db_name] + + def doSaveJournal(self, classname, nodeid, action, params): # serialise first if action in ('set', 'create'): params = self.serialise(classname, params) @@ -533,14 +602,9 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): params) if __debug__: - print >>hyperdb.DEBUG, '_doSaveJournal', entry + print >>hyperdb.DEBUG, 'doSaveJournal', entry - # get the database handle - db_name = 'journals.%s'%classname - if self.databases.has_key(db_name): - db = self.databases[db_name] - else: - db = self.databases[db_name] = self._opendb(db_name, 'c') + db = self.getCachedJournalDB(classname) # now insert the journal entry if db.has_key(nodeid): @@ -553,6 +617,23 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): db[nodeid] = marshal.dumps(l) + def doDestroyNode(self, classname, nodeid): + if __debug__: + print >>hyperdb.DEBUG, 'doDestroyNode', (self, classname, nodeid) + + # delete from the class database + db = self.getCachedClassDB(classname) + if db.has_key(nodeid): + del db[nodeid] + + # delete from the database + db = self.getCachedJournalDB(classname) + if db.has_key(nodeid): + del db[nodeid] + + # return the classname, nodeid so we reindex this content + return (classname, nodeid) + def rollback(self): ''' Reverse all actions from the current transaction. ''' @@ -560,11 +641,12 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): print >>hyperdb.DEBUG, 'rollback', (self, ) for method, args in self.transactions: # delete temporary files - if method == self._doStoreFile: - self._rollbackStoreFile(*args) + if method == self.doStoreFile: + self.rollbackStoreFile(*args) self.cache = {} self.dirtynodes = {} self.newnodes = {} + self.destroyednodes = {} self.transactions = [] _marker = [] @@ -672,7 +754,7 @@ class Class(hyperdb.Class): except (TypeError, KeyError): raise IndexError, 'new property "%s": %s not a %s'%( key, value, link_class) - elif not self.db.hasnode(link_class, value): + elif not self.db.getclass(link_class).hasnode(value): raise IndexError, '%s has no node %s'%(link_class, value) # save off the value @@ -706,12 +788,13 @@ class Class(hyperdb.Class): propvalues[key] = value # handle additions - for id in value: - if not self.db.hasnode(link_class, id): - raise IndexError, '%s has no node %s'%(link_class, id) + for nodeid in value: + if not self.db.getclass(link_class).hasnode(nodeid): + raise IndexError, '%s has no node %s'%(link_class, + nodeid) # register the link with the newly linked node if self.do_journal and self.properties[key].do_journal: - self.db.addjournal(link_class, id, 'link', + self.db.addjournal(link_class, nodeid, 'link', (self.classname, newid, key)) elif isinstance(prop, String): @@ -732,27 +815,15 @@ class Class(hyperdb.Class): elif value is not None and isinstance(prop, Number): try: - int(value) + float(value) except ValueError: - try: - float(value) - except ValueError: - raise TypeError, 'new property "%s" not numeric'%key + raise TypeError, 'new property "%s" not numeric'%key elif value is not None and isinstance(prop, Boolean): - if isinstance(value, type('')): - s = value.lower() - if s in ('0', 'false', 'no'): - value = 0 - elif s in ('1', 'true', 'yes'): - value = 1 - else: - raise TypeError, 'new property "%s" not boolean'%key - else: - try: - int(value) - except TypeError: - raise TypeError, 'new property "%s" not boolean'%key + try: + int(value) + except ValueError: + raise TypeError, 'new property "%s" not boolean'%key # make sure there's data where there needs to be for key, prop in self.properties.items(): @@ -927,21 +998,23 @@ class Class(hyperdb.Class): # do stuff based on the prop type if isinstance(prop, Link): - link_class = self.properties[propname].classname + link_class = prop.classname # if it isn't a number, it's a key - if type(value) != type(''): - raise ValueError, 'link value must be String' - if not num_re.match(value): + if value is not None and not isinstance(value, type('')): + raise ValueError, 'property "%s" link value be a string'%( + propname) + if isinstance(value, type('')) and not num_re.match(value): try: value = self.db.classes[link_class].lookup(value) except (TypeError, KeyError): raise IndexError, 'new property "%s": %s not a %s'%( - propname, value, self.properties[propname].classname) + propname, value, prop.classname) - if not self.db.hasnode(link_class, value): + if (value is not None and + not self.db.getclass(link_class).hasnode(value)): raise IndexError, '%s has no node %s'%(link_class, value) - if self.do_journal and self.properties[propname].do_journal: + if self.do_journal and prop.do_journal: # register the unlink with the old linked node if node[propname] is not None: self.db.addjournal(link_class, node[propname], 'unlink', @@ -995,7 +1068,7 @@ class Class(hyperdb.Class): # handle additions for id in value: - if not self.db.hasnode(link_class, id): + if not self.db.getclass(link_class).hasnode(id): raise IndexError, '%s has no node %s'%(link_class, id) if id in l: continue @@ -1037,30 +1110,15 @@ class Class(hyperdb.Class): elif value is not None and isinstance(prop, Number): try: - int(value) + float(value) except ValueError: - try: - float(value) - except ValueError: - raise TypeError, 'new property "%s" not '\ - 'numeric'%propname + raise TypeError, 'new property "%s" not numeric'%propname elif value is not None and isinstance(prop, Boolean): - if isinstance(value, type('')): - s = value.lower() - if s in ('0', 'false', 'no'): - value = 0 - elif s in ('1', 'true', 'yes'): - value = 1 - else: - raise TypeError, 'new property "%s" not '\ - 'boolean'%propname - else: - try: - int(value) - except ValueError: - raise TypeError, 'new property "%s" not '\ - 'boolean'%propname + try: + int(value) + except ValueError: + raise TypeError, 'new property "%s" not boolean'%propname node[propname] = value @@ -1102,6 +1160,23 @@ class Class(hyperdb.Class): self.fireReactors('retire', nodeid, None) + def destroy(self, nodeid): + """Destroy a node. + + WARNING: this method should never be used except in extremely rare + situations where there could never be links to the node being + deleted + WARNING: use retire() instead + WARNING: the properties of this node will not be available ever again + WARNING: really, use retire() instead + + Well, I think that's enough warnings. This method exists mostly to + support the session storage of the cgi interface. + """ + if self.db.journaltag is None: + raise DatabaseError, 'Database open read-only' + self.db.destroynode(self.classname, nodeid) + def history(self, nodeid): """Retrieve the journal of edits on a particular node. @@ -1208,10 +1283,6 @@ class Class(hyperdb.Class): prop = self.properties[propname] if not isinstance(prop, Link) and not isinstance(prop, Multilink): raise TypeError, "'%s' not a Link/Multilink property"%propname - #XXX edit is expensive and of questionable use - #for nodeid in nodeids: - # if not self.db.hasnode(prop.classname, nodeid): - # raise ValueError, '%s has no node %s'%(prop.classname, nodeid) # ok, now do the find cldb = self.db.getclassdb(self.classname) @@ -1266,7 +1337,7 @@ class Class(hyperdb.Class): if node.has_key(self.db.RETIRED_FLAG): continue for key, value in requirements.items(): - if node[key] and node[key].lower() != value: + if node[key] is None or node[key].lower() != value: break else: l.append(nodeid) @@ -1290,18 +1361,25 @@ class Class(hyperdb.Class): l.sort() return l - # XXX not in spec def filter(self, search_matches, filterspec, sort, group, num_re = re.compile('^\d+$')): ''' Return a list of the ids of the active nodes in this class that match the 'filter' spec, sorted by the group spec and then the - sort spec + sort spec. + + "filterspec" is {propname: value(s)} + "sort" is ['+propname', '-propname', 'propname', ...] + "group is ['+propname', '-propname', 'propname', ...] ''' cn = self.classname # optimise filterspec l = [] props = self.getprops() + LINK = 0 + MULTILINK = 1 + STRING = 2 + OTHER = 6 for k, v in filterspec.items(): propclass = props[k] if isinstance(propclass, Link): @@ -1320,7 +1398,7 @@ class Class(hyperdb.Class): k, entry, self.properties[k].classname) u.append(entry) - l.append((0, k, u)) + l.append((LINK, k, u)) elif isinstance(propclass, Multilink): if type(v) is not type([]): v = [v] @@ -1335,58 +1413,66 @@ class Class(hyperdb.Class): raise ValueError, 'new property "%s": %s not a %s'%( k, entry, self.properties[k].classname) u.append(entry) - l.append((1, k, u)) + l.append((MULTILINK, k, u)) elif isinstance(propclass, String): # simple glob searching v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v) v = v.replace('?', '.') v = v.replace('*', '.*?') - l.append((2, k, re.compile(v, re.I))) + l.append((STRING, k, re.compile(v, re.I))) elif isinstance(propclass, Boolean): if type(v) is type(''): bv = v.lower() in ('yes', 'true', 'on', '1') else: bv = v - l.append((6, k, bv)) + l.append((OTHER, k, bv)) elif isinstance(propclass, Number): - l.append((6, k, int(v))) + l.append((OTHER, k, int(v))) else: - l.append((6, k, v)) + l.append((OTHER, k, v)) filterspec = l # now, find all the nodes that are active and pass filtering l = [] cldb = self.db.getclassdb(cn) try: + # TODO: only full-scan once (use items()) for nodeid in self.db.getnodeids(cn, cldb): node = self.db.getnode(cn, nodeid, cldb) if node.has_key(self.db.RETIRED_FLAG): continue # apply filter for t, k, v in filterspec: - # this node doesn't have this property, so reject it - if not node.has_key(k): break + # make sure the node has the property + if not node.has_key(k): + # this node doesn't have this property, so reject it + break - if t == 0 and node[k] not in v: - # link - if this node'd property doesn't appear in the + # now apply the property filter + if t == LINK: + # link - if this node's property doesn't appear in the # filterspec's nodeid list, skip it - break - elif t == 1: + if node[k] not in v: + break + elif t == MULTILINK: # multilink - if any of the nodeids required by the # filterspec aren't in this node's property, then skip # it - for value in v: - if value not in node[k]: + have = node[k] + for want in v: + if want not in have: break else: continue break - elif t == 2 and (node[k] is None or not v.search(node[k])): + elif t == STRING: # RE search - break - elif t == 6 and node[k] != v: + if node[k] is None or not v.search(node[k]): + break + elif t == OTHER: # straight value comparison for the other types - break + if node[k] != v: + break else: l.append((nodeid, node)) finally: @@ -1396,9 +1482,7 @@ class Class(hyperdb.Class): # filter based on full text search if search_matches is not None: k = [] - l_debug = [] for v in l: - l_debug.append(v[0]) if search_matches.has_key(v[0]): k.append(v) l = k @@ -1577,9 +1661,15 @@ class Class(hyperdb.Class): # find all the String properties that have indexme for prop, propclass in self.getprops().items(): if isinstance(propclass, String) and propclass.indexme: - # and index them under (classname, nodeid, property) - self.db.indexer.add_text((self.classname, nodeid, prop), - str(self.get(nodeid, prop))) + try: + value = str(self.get(nodeid, prop)) + except IndexError: + # node no longer exists - entry should be removed + self.db.indexer.purge_entry((self.classname, nodeid, prop)) + else: + # and index them under (classname, nodeid, property) + self.db.indexer.add_text((self.classname, nodeid, prop), + value) # # Detector interface @@ -1703,6 +1793,43 @@ class IssueClass(Class, roundupdb.IssueClass): # #$Log: not supported by cvs2svn $ +#Revision 1.56 2002/07/31 22:04:33 richard +#cleanup +# +#Revision 1.55 2002/07/30 08:22:38 richard +#Session storage in the hyperdb was horribly, horribly inefficient. We use +#a simple anydbm wrapper now - which could be overridden by the metakit +#backend or RDB backend if necessary. +#Much, much better. +# +#Revision 1.54 2002/07/26 08:26:59 richard +#Very close now. The cgi and mailgw now use the new security API. The two +#templates have been migrated to that setup. Lots of unit tests. Still some +#issue in the web form for editing Roles assigned to users. +# +#Revision 1.53 2002/07/25 07:14:06 richard +#Bugger it. Here's the current shape of the new security implementation. +#Still to do: +# . call the security funcs from cgi and mailgw +# . change shipped templates to include correct initialisation and remove +# the old config vars +#... that seems like a lot. The bulk of the work has been done though. Honest :) +# +#Revision 1.52 2002/07/19 03:36:34 richard +#Implemented the destroy() method needed by the session database (and possibly +#others). At the same time, I removed the leading underscores from the hyperdb +#methods that Really Didn't Need Them. +#The journal also raises IndexError now for all situations where there is a +#request for the journal of a node that doesn't have one. It used to return +#[] in _some_ situations, but not all. This _may_ break code, but the tests +#pass... +# +#Revision 1.51 2002/07/18 23:07:08 richard +#Unit tests and a few fixes. +# +#Revision 1.50 2002/07/18 11:50:58 richard +#added tests for number type too +# #Revision 1.49 2002/07/18 11:41:10 richard #added tests for boolean type, and fixes to anydbm backend #