X-Git-Url: https://git.tokkee.org/?a=blobdiff_plain;f=roundup%2Fbackends%2Fback_metakit.py;h=c8e220d8cfdbbafe927eb65854264e25580665ad;hb=9f96bccb32cae3de034a552084ddad6c9f94b29a;hp=49028519361c754d4902b792cb4029c9554b6164;hpb=54c2e57dc78736b2d60cd7c66abc7ba2efc0e0a2;p=roundup.git diff --git a/roundup/backends/back_metakit.py b/roundup/backends/back_metakit.py index 4902851..c8e220d 100755 --- a/roundup/backends/back_metakit.py +++ b/roundup/backends/back_metakit.py @@ -1,36 +1,39 @@ -from roundup import hyperdb, date, password, roundupdb +from roundup import hyperdb, date, password, roundupdb, security import metakit +from sessions import Sessions import re, marshal, os, sys, weakref, time, calendar -from roundup.indexer import Indexer +from roundup import indexer +import locking -_instances = weakref.WeakValueDictionary() +_dbs = {} def Database(config, journaltag=None): - if _instances.has_key(id(config)): - db = _instances[id(config)] - old = db.journaltag + db = _dbs.get(config.DATABASE, None) + if db is None or db._db is None: + db = _Database(config, journaltag) + _dbs[config.DATABASE] = db + else: db.journaltag = journaltag try: delattr(db, 'curuserid') except AttributeError: pass - return db - else: - db = _Database(config, journaltag) - _instances[id(config)] = db - return db + return db class _Database(hyperdb.Database): def __init__(self, config, journaltag=None): self.config = config self.journaltag = journaltag self.classes = {} - self._classes = [] self.dirty = 0 - self.__RW = 0 + self.lockfile = None self._db = self.__open() - self.indexer = Indexer(self.config.DATABASE) + self.indexer = Indexer(self.config.DATABASE, self._db) + self.sessions = Sessions(self.config) + self.security = security.Security(self) + os.umask(0002) + def post_init(self): if self.indexer.should_reindex(): self.reindex() @@ -45,94 +48,150 @@ class _Database(hyperdb.Database): # --- defined in ping's spec def __getattr__(self, classname): if classname == 'curuserid': + if self.journaltag is None: + return None + try: self.curuserid = x = int(self.classes['user'].lookup(self.journaltag)) except KeyError: - x = 0 + if self.journaltag == 'admin': + self.curuserid = x = 1 + else: + x = 0 return x + elif classname == 'transactions': + return self.dirty return self.getclass(classname) def getclass(self, classname): - return self.classes[classname] + try: + return self.classes[classname] + except KeyError: + raise KeyError, 'There is no class called "%s"'%classname def getclasses(self): return self.classes.keys() # --- end of ping's spec # --- exposed methods def commit(self): if self.dirty: - if self.__RW: - self._db.commit() - for cl in self.classes.values(): - cl._commit() - self.indexer.save_index() - else: - raise RuntimeError, "metakit is open RO" + self._db.commit() + for cl in self.classes.values(): + cl._commit() + self.indexer.save_index() self.dirty = 0 def rollback(self): if self.dirty: for cl in self.classes.values(): cl._rollback() self._db.rollback() + self._db = None + self._db = metakit.storage(self.dbnm, 1) + self.hist = self._db.view('history') + self.tables = self._db.view('tables') + self.indexer.rollback() + self.indexer.datadb = self._db self.dirty = 0 + def clearCache(self): + for cl in self.classes.values(): + cl._commit() def clear(self): for cl in self.classes.values(): cl._clear() def hasnode(self, classname, nodeid): - return self.getclass(clasname).hasnode(nodeid) + return self.getclass(classname).hasnode(nodeid) def pack(self, pack_before): - pass + mindate = int(calendar.timegm(pack_before.get_tuple())) + i = 0 + while i < len(self.hist): + if self.hist[i].date < mindate and self.hist[i].action != _CREATE: + self.hist.delete(i) + else: + i = i + 1 def addclass(self, cl): self.classes[cl.classname] = cl - def addjournal(self, tablenm, nodeid, action, params): + if self.tables.find(name=cl.classname) < 0: + self.tables.append(name=cl.classname) + def addjournal(self, tablenm, nodeid, action, params, creator=None, + creation=None): tblid = self.tables.find(name=tablenm) if tblid == -1: tblid = self.tables.append(name=tablenm) + if creator is None: + creator = self.curuserid + else: + try: + creator = int(creator) + except TypeError: + creator = int(self.getclass('user').lookup(creator)) + if creation is None: + creation = int(time.time()) + elif isinstance(creation, date.Date): + creation = int(calendar.timegm(creation.get_tuple())) # tableid:I,nodeid:I,date:I,user:I,action:I,params:B self.hist.append(tableid=tblid, nodeid=int(nodeid), - date=int(time.time()), + date=creation, action=action, - user = self.curuserid, + user = creator, params = marshal.dumps(params)) - def gethistory(self, tablenm, nodeid): + def getjournal(self, tablenm, nodeid): rslt = [] tblid = self.tables.find(name=tablenm) if tblid == -1: return rslt q = self.hist.select(tableid=tblid, nodeid=int(nodeid)) + if len(q) == 0: + raise IndexError, "no history for id %s in %s" % (nodeid, tablenm) i = 0 - userclass = self.getclass('user') + #userclass = self.getclass('user') for row in q: try: params = marshal.loads(row.params) except ValueError: print "history couldn't unmarshal %r" % row.params params = {} - usernm = userclass.get(str(row.user), 'username') + #usernm = userclass.get(str(row.user), 'username') dt = date.Date(time.gmtime(row.date)) - rslt.append((i, dt, usernm, _actionnames[row.action], params)) - i += 1 + #rslt.append((nodeid, dt, usernm, _actionnames[row.action], params)) + rslt.append((nodeid, dt, str(row.user), _actionnames[row.action], params)) return rslt + def destroyjournal(self, tablenm, nodeid): + nodeid = int(nodeid) + tblid = self.tables.find(name=tablenm) + if tblid == -1: + return + i = 0 + hist = self.hist + while i < len(hist): + if hist[i].tableid == tblid and hist[i].nodeid == nodeid: + hist.delete(i) + else: + i = i + 1 + self.dirty = 1 + def close(self): - import time - now = time.time - start = now() for cl in self.classes.values(): cl.db = None - #self._db.rollback() - #print "pre-close cleanup of DB(%d) took %2.2f secs" % (self.__RW, now()-start) self._db = None - #print "close of DB(%d) took %2.2f secs" % (self.__RW, now()-start) + if self.lockfile is not None: + locking.release_lock(self.lockfile) + if _dbs.has_key(self.config.DATABASE): + del _dbs[self.config.DATABASE] + if self.lockfile is not None: + self.lockfile.close() + self.lockfile = None self.classes = {} - try: - del _instances[id(self.config)] - except KeyError: - pass - self.__RW = 0 - + self.indexer = None + # --- internal def __open(self): + if not os.path.exists(self.config.DATABASE): + os.makedirs(self.config.DATABASE) self.dbnm = db = os.path.join(self.config.DATABASE, 'tracker.mk4') + lockfilenm = db[:-3]+'lck' + self.lockfile = locking.acquire_lock(lockfilenm) + self.lockfile.write(str(os.getpid())) + self.lockfile.flush() self.fastopen = 0 if os.path.exists(db): dbtm = os.path.getmtime(db) @@ -147,11 +206,7 @@ class _Database(hyperdb.Database): else: # can't find schemamod - must be frozen self.fastopen = 1 - else: - self.__RW = 1 - if not self.fastopen: - self.__RW = 1 - db = metakit.storage(db, self.__RW) + db = metakit.storage(db, 1) hist = db.view('history') tables = db.view('tables') if not self.fastopen: @@ -159,24 +214,11 @@ class _Database(hyperdb.Database): hist = db.getas('history[tableid:I,nodeid:I,date:I,user:I,action:I,params:B]') if not tables.structure(): tables = db.getas('tables[name:S]') + db.commit() self.tables = tables self.hist = hist return db - def isReadOnly(self): - return self.__RW == 0 - def getWriteAccess(self): - if self.journaltag is not None and self.__RW == 0: - now = time.time - start = now() - self._db = None - #print "closing the file took %2.2f secs" % (now()-start) - start = now() - self._db = metakit.storage(self.dbnm, 1) - self.__RW = 1 - self.hist = self._db.view('history') - self.tables = self._db.view('tables') - #print "getting RW access took %2.2f secs" % (now()-start) - + _STRINGTYPE = type('') _LISTTYPE = type([]) _CREATE, _SET, _RETIRE, _LINK, _UNLINK = range(5) @@ -193,10 +235,11 @@ _marker = [] _ALLOWSETTINGPRIVATEPROPS = 0 -class Class: # no, I'm not going to subclass the existing! +class Class: privateprops = None def __init__(self, db, classname, **properties): - self.db = weakref.proxy(db) + #self.db = weakref.proxy(db) + self.db = db self.classname = classname self.keyname = None self.ruprops = properties @@ -216,6 +259,19 @@ class Class: # no, I'm not going to subclass the existing! self.properties = self.ruprops self.db.addclass(self) self.idcache = {} + + # default is to journal changes + self.do_journal = 1 + + def enableJournalling(self): + '''Turn journalling on for this class + ''' + self.do_journal = 1 + + def disableJournalling(self): + '''Turn journalling off for this class + ''' + self.do_journal = 0 # --- the roundup.Class methods def audit(self, event, detector): @@ -234,6 +290,7 @@ class Class: # no, I'm not going to subclass the existing! self.reactors[event].append(detector) # --- the hyperdb.Class methods def create(self, **propvalues): + self.fireAuditors('create', None, propvalues) rowdict = {} rowdict['id'] = newid = self.maxid self.maxid += 1 @@ -279,12 +336,14 @@ class Class: # no, I'm not going to subclass the existing! if propvalues.has_key('#ISNEW'): isnew = 1 del propvalues['#ISNEW'] + if not isnew: + self.fireAuditors('set', nodeid, propvalues) if not propvalues: - return + return propvalues if propvalues.has_key('id'): raise KeyError, '"id" is reserved' if self.db.journaltag is None: - raise DatabaseError, 'Database open read-only' + raise hyperdb.DatabaseError, 'Database open read-only' view = self.getview(1) # node must exist & not be retired id = int(nodeid) @@ -296,7 +355,7 @@ class Class: # no, I'm not going to subclass the existing! raise IndexError, "%s has no node %s" % (self.classname, nodeid) oldnode = self.uncommitted.setdefault(id, {}) changes = {} - + for key, value in propvalues.items(): # this will raise the KeyError if the property isn't valid # ... we don't use getprops() here because we only care about @@ -331,9 +390,14 @@ class Class: # no, I'm not going to subclass the existing! # do stuff based on the prop type if isinstance(prop, hyperdb.Link): link_class = prop.classname + # must be a string or None + if value is not None and not isinstance(value, type('')): + raise ValueError, 'property "%s" link value be a string'%( + key) + # Roundup sets to "unselected" by passing None + if value is None: + value = 0 # if it isn't a number, it's a key - if type(value) != _STRINGTYPE: - raise ValueError, 'link value must be String' try: int(value) except ValueError: @@ -343,26 +407,31 @@ class Class: # no, I'm not going to subclass the existing! raise IndexError, 'new property "%s": %s not a %s'%( key, value, prop.classname) - if not self.db.getclass(link_class).hasnode(value): + if (value is not None and + not self.db.getclass(link_class).hasnode(value)): raise IndexError, '%s has no node %s'%(link_class, value) setattr(row, key, int(value)) changes[key] = oldvalue - if prop.do_journal: + if self.do_journal and prop.do_journal: # register the unlink with the old linked node if oldvalue: - self.db.addjournal(link_class, value, _UNLINK, (self.classname, str(row.id), key)) + self.db.addjournal(link_class, value, _UNLINK, + (self.classname, str(row.id), key)) # register the link with the newly linked node if value: - self.db.addjournal(link_class, value, _LINK, (self.classname, str(row.id), key)) + self.db.addjournal(link_class, value, _LINK, + (self.classname, str(row.id), key)) elif isinstance(prop, hyperdb.Multilink): - if type(value) != _LISTTYPE: + if value is not None and type(value) != _LISTTYPE: raise TypeError, 'new property "%s" not a list of ids'%key link_class = prop.classname l = [] + if value is None: + value = [] for entry in value: if type(entry) != _STRINGTYPE: raise ValueError, 'new property "%s" link value ' \ @@ -385,8 +454,9 @@ class Class: # no, I'm not going to subclass the existing! if id not in value: rmvd.append(id) # register the unlink with the old linked node - if prop.do_journal: - self.db.addjournal(link_class, id, _UNLINK, (self.classname, str(row.id), key)) + if self.do_journal and prop.do_journal: + self.db.addjournal(link_class, id, _UNLINK, + (self.classname, str(row.id), key)) # handle additions adds = [] @@ -397,8 +467,9 @@ class Class: # no, I'm not going to subclass the existing! link_class, id) adds.append(id) # register the link with the newly linked node - if prop.do_journal: - self.db.addjournal(link_class, id, _LINK, (self.classname, str(row.id), key)) + if self.do_journal and prop.do_journal: + self.db.addjournal(link_class, id, _LINK, + (self.classname, str(row.id), key)) sv = getattr(row, key) i = 0 @@ -410,45 +481,79 @@ class Class: # no, I'm not going to subclass the existing! for id in adds: sv.append(fid=int(id)) changes[key] = oldvalue + if not rmvd and not adds: + del propvalues[key] - elif isinstance(prop, hyperdb.String): if value is not None and type(value) != _STRINGTYPE: raise TypeError, 'new property "%s" not a string'%key + if value is None: + value = '' setattr(row, key, value) changes[key] = oldvalue if hasattr(prop, 'isfilename') and prop.isfilename: propvalues[key] = os.path.basename(value) if prop.indexme: - self.db.indexer.add_text((self.classname, nodeid, key), value, 'text/plain') + self.db.indexer.add_text((self.classname, nodeid, key), + value, 'text/plain') elif isinstance(prop, hyperdb.Password): - if not isinstance(value, password.Password): + if value is not None and not isinstance(value, password.Password): raise TypeError, 'new property "%s" not a Password'% key + if value is None: + value = '' setattr(row, key, str(value)) changes[key] = str(oldvalue) propvalues[key] = str(value) - elif value is not None and isinstance(prop, hyperdb.Date): - if not isinstance(value, date.Date): + elif isinstance(prop, hyperdb.Date): + if value is not None and not isinstance(value, date.Date): raise TypeError, 'new property "%s" not a Date'% key - setattr(row, key, int(calendar.timegm(value.get_tuple()))) + if value is None: + setattr(row, key, 0) + else: + setattr(row, key, int(calendar.timegm(value.get_tuple()))) changes[key] = str(oldvalue) propvalues[key] = str(value) - elif value is not None and isinstance(prop, hyperdb.Interval): - if not isinstance(value, date.Interval): + elif isinstance(prop, hyperdb.Interval): + if value is not None and not isinstance(value, date.Interval): raise TypeError, 'new property "%s" not an Interval'% key - setattr(row, key, str(value)) + if value is None: + setattr(row, key, '') + else: + setattr(row, key, str(value)) changes[key] = str(oldvalue) propvalues[key] = str(value) + + elif isinstance(prop, hyperdb.Number): + if value is None: + value = 0 + try: + v = int(value) + except ValueError: + raise TypeError, "%s (%s) is not numeric" % (key, repr(value)) + setattr(row, key, v) + changes[key] = oldvalue + propvalues[key] = value + + elif isinstance(prop, hyperdb.Boolean): + if value is None: + bv = 0 + elif value not in (0,1): + raise TypeError, "%s (%s) is not boolean" % (key, repr(value)) + else: + bv = value + setattr(row, key, bv) + changes[key] = oldvalue + propvalues[key] = value oldnode[key] = oldvalue # nothing to do? if not propvalues: - return - if not row.activity: + return propvalues + if not propvalues.has_key('activity'): row.activity = int(time.time()) if isnew: if not row.creation: @@ -457,12 +562,20 @@ class Class: # no, I'm not going to subclass the existing! row.creator = self.db.curuserid self.db.dirty = 1 - if isnew: - self.db.addjournal(self.classname, nodeid, _CREATE, {}) - else: - self.db.addjournal(self.classname, nodeid, _SET, changes) + if self.do_journal: + if isnew: + self.db.addjournal(self.classname, nodeid, _CREATE, {}) + self.fireReactors('create', nodeid, None) + else: + self.db.addjournal(self.classname, nodeid, _SET, changes) + self.fireReactors('set', nodeid, oldnode) + return propvalues + def retire(self, nodeid): + if self.db.journaltag is None: + raise hyperdb.DatabaseError, 'Database open read-only' + self.fireAuditors('retire', nodeid, None) view = self.getview(1) ndx = view.find(id=int(nodeid)) if ndx < 0: @@ -471,31 +584,44 @@ class Class: # no, I'm not going to subclass the existing! oldvalues = self.uncommitted.setdefault(row.id, {}) oldval = oldvalues['_isdel'] = row._isdel row._isdel = 1 - self.db.addjournal(self.classname, nodeid, _RETIRE, {}) - iv = self.getindexview(1) - ndx = iv.find(k=getattr(row, self.keyname),i=row.id) - if ndx > -1: - iv.delete(ndx) + if self.do_journal: + self.db.addjournal(self.classname, nodeid, _RETIRE, {}) + if self.keyname: + iv = self.getindexview(1) + ndx = iv.find(k=getattr(row, self.keyname),i=row.id) + if ndx > -1: + iv.delete(ndx) self.db.dirty = 1 + self.fireReactors('retire', nodeid, None) def history(self, nodeid): - return self.db.gethistory(self.classname, nodeid) + if not self.do_journal: + raise ValueError, 'Journalling is disabled for this class' + return self.db.getjournal(self.classname, nodeid) def setkey(self, propname): if self.keyname: if propname == self.keyname: return raise ValueError, "%s already indexed on %s" % (self.classname, self.keyname) + prop = self.properties.get(propname, None) + if prop is None: + prop = self.privateprops.get(propname, None) + if prop is None: + raise KeyError, "no property %s" % propname + if not isinstance(prop, hyperdb.String): + raise TypeError, "%s is not a String" % propname # first setkey for this run self.keyname = propname iv = self.db._db.view('_%s' % self.classname) - if self.db.fastopen or iv.structure(): + if self.db.fastopen and iv.structure(): return # very first setkey ever + self.db.dirty = 1 iv = self.db._db.getas('_%s[k:S,i:I]' % self.classname) iv = iv.ordered(1) - #XXX # print "setkey building index" for row in self.getview(): iv.append(k=getattr(row, propname), i=row.id) + self.db.commit() def getkey(self): return self.keyname def lookup(self, keyvalue): @@ -512,6 +638,22 @@ class Class: # no, I'm not going to subclass the existing! if ndx > -1: return str(view[ndx].id) raise KeyError, keyvalue + + def destroy(self, id): + view = self.getview(1) + ndx = view.find(id=int(id)) + if ndx > -1: + if self.keyname: + keyvalue = getattr(view[ndx], self.keyname) + iv = self.getindexview(1) + if iv: + ivndx = iv.find(k=keyvalue) + if ivndx > -1: + iv.delete(ivndx) + view.delete(ndx) + self.db.destroyjournal(self.classname, id) + self.db.dirty = 1 + def find(self, **propspec): """Get the ids of nodes in this class which link to the given nodes. @@ -539,16 +681,24 @@ class Class: # no, I'm not going to subclass the existing! vws = [] for propname, ids in propspec: if type(ids) is _STRINGTYPE: - ids = {ids:1} + ids = {int(ids):1} + else: + d = {} + for id in ids.keys(): + d[int(id)] = 1 + ids = d prop = self.ruprops[propname] view = self.getview() if isinstance(prop, hyperdb.Multilink): - view = view.flatten(getattr(view, propname)) def ff(row, nm=propname, ids=ids): - return ids.has_key(str(row.fid)) + sv = getattr(row, nm) + for sr in sv: + if ids.has_key(sr.fid): + return 1 + return 0 else: def ff(row, nm=propname, ids=ids): - return ids.has_key(str(getattr(row, nm))) + return ids.has_key(getattr(row, nm)) ndxview = view.filter(ff) vws.append(ndxview.unique()) @@ -559,7 +709,7 @@ class Class: # no, I'm not going to subclass the existing! ndxview = vws[0] for v in vws[1:]: ndxview = ndxview.union(v) - view = view.remapwith(ndxview) + view = self.getview().remapwith(ndxview) rslt = [] for row in view: rslt.append(str(row.id)) @@ -584,13 +734,17 @@ class Class: # no, I'm not going to subclass the existing! if self.ruprops.has_key(key): raise ValueError, "%s is already a property of %s" % (key, self.classname) self.ruprops.update(properties) + self.db.fastopen = 0 view = self.__getview() + self.db.commit() # ---- end of ping's spec - def filter(self, search_matches, filterspec, sort, group): + def filter(self, search_matches, filterspec, sort=(None,None), + group=(None,None)): # search_matches is None or a set (dict of {nodeid: {propname:[nodeid,...]}}) # filterspec is a dict {propname:value} - # sort and group are lists of propnames - + # sort and group are (dir, prop) where dir is '+', '-' or None + # and prop is a prop name or None + where = {'_isdel':0} mlcriteria = {} regexes = {} @@ -638,6 +792,14 @@ class Class: # no, I'm not going to subclass the existing! regexes[propname] = re.compile(v, re.I) elif propname == 'id': where[propname] = int(value) + elif isinstance(prop, hyperdb.Boolean): + if type(value) is _STRINGTYPE: + bv = value.lower() in ('yes', 'true', 'on', '1') + else: + bv = value + where[propname] = bv + elif isinstance(prop, hyperdb.Number): + where[propname] = int(value) else: where[propname] = str(value) v = self.getview() @@ -689,20 +851,36 @@ class Class: # no, I'm not going to subclass the existing! if sort or group: sortspec = [] rev = [] - for propname in group + sort: + for dir, propname in group, sort: + if propname is None: continue isreversed = 0 - if propname[0] == '-': - propname = propname[1:] + if dir == '-': isreversed = 1 try: prop = getattr(v, propname) except AttributeError: - # I can't sort on 'activity', cause it's psuedo!! + print "MK has no property %s" % propname continue + propclass = self.ruprops.get(propname, None) + if propclass is None: + propclass = self.privateprops.get(propname, None) + if propclass is None: + print "Schema has no property %s" % propname + continue + if isinstance(propclass, hyperdb.Link): + linkclass = self.db.getclass(propclass.classname) + lv = linkclass.getview() + lv = lv.rename('id', propname) + v = v.join(lv, prop, 1) + if linkclass.getprops().has_key('order'): + propname = 'order' + else: + propname = linkclass.labelprop() + prop = getattr(v, propname) if isreversed: rev.append(prop) sortspec.append(prop) - v = v.sortrev(sortspec, rev)[:] #XXX Aaaabh + v = v.sortrev(sortspec, rev)[:] #XXX Metakit bug #print "filter sort at %s" % time.time() rslt = [] @@ -773,6 +951,59 @@ class Class: # no, I'm not going to subclass the existing! self.db.indexer.add_text((self.classname, nodeid, prop), str(self.get(nodeid, prop))) + def export_list(self, propnames, nodeid): + ''' Export a node - generate a list of CSV-able data in the order + specified by propnames for the given node. + ''' + properties = self.getprops() + l = [] + for prop in propnames: + proptype = properties[prop] + value = self.get(nodeid, prop) + # "marshal" data where needed + if value is None: + pass + elif isinstance(proptype, hyperdb.Date): + value = value.get_tuple() + elif isinstance(proptype, hyperdb.Interval): + value = value.get_tuple() + elif isinstance(proptype, hyperdb.Password): + value = str(value) + l.append(repr(value)) + return l + + def import_list(self, propnames, proplist): + ''' Import a node - all information including "id" is present and + should not be sanity checked. Triggers are not triggered. The + journal should be initialised using the "creator" and "creation" + information. + + Return the nodeid of the node imported. + ''' + if self.db.journaltag is None: + raise hyperdb.DatabaseError, 'Database open read-only' + properties = self.getprops() + + d = {} + view = self.getview(1) + for i in range(len(propnames)): + value = eval(proplist[i]) + propname = propnames[i] + prop = properties[propname] + if propname == 'id': + newid = value + value = int(value) + elif isinstance(prop, hyperdb.Date): + value = int(calendar.timegm(value)) + elif isinstance(prop, hyperdb.Interval): + value = str(date.Interval(value)) + d[propname] = value + view.append(d) + creator = d.get('creator', None) + creation = d.get('creation', None) + self.db.addjournal(self.classname, newid, 'create', {}, creator, creation) + return newid + # --- used by Database def _commit(self): """ called post commit of the DB. @@ -804,10 +1035,10 @@ class Class: # no, I'm not going to subclass the existing! def __getview(self): db = self.db._db view = db.view(self.classname) - if self.db.fastopen: + mkprops = view.structure() + if mkprops and self.db.fastopen: return view.ordered(1) # is the definition the same? - mkprops = view.structure() for nm, rutyp in self.ruprops.items(): for mkprop in mkprops: if mkprop.name == nm: @@ -815,10 +1046,8 @@ class Class: # no, I'm not going to subclass the existing! else: mkprop = None if mkprop is None: - #print "%s missing prop %s (%s)" % (self.classname, nm, rutyp.__class__.__name__) break if _typmap[rutyp.__class__] != mkprop.type: - #print "%s - prop %s (%s) has wrong mktyp (%s)" % (self.classname, nm, rutyp.__class__.__name__, mkprop.type) break else: return view.ordered(1) @@ -832,15 +1061,12 @@ class Class: # no, I'm not going to subclass the existing! if mktyp == 'V': s[-1] += ('[fid:I]') s.append('_isdel:I,activity:I,creation:I,creator:I]') - v = db.getas(','.join(s)) + v = self.db._db.getas(','.join(s)) + self.db.commit() return v.ordered(1) def getview(self, RW=0): - if RW and self.db.isReadOnly(): - self.db.getWriteAccess() return self.db._db.view(self.classname).ordered(1) def getindexview(self, RW=0): - if RW and self.db.isReadOnly(): - self.db.getWriteAccess() return self.db._db.view("_%s" % self.classname).ordered(1) def _fetchML(sv): @@ -867,6 +1093,9 @@ _converters = { hyperdb.Multilink : _fetchML, hyperdb.Interval : date.Interval, hyperdb.Password : _fetchPW, + hyperdb.Boolean : lambda n: n, + hyperdb.Number : lambda n: n, + hyperdb.String : str, } class FileName(hyperdb.String): @@ -880,6 +1109,8 @@ _typmap = { hyperdb.Multilink : 'V', hyperdb.Interval : 'S', hyperdb.Password : 'S', + hyperdb.Boolean : 'I', + hyperdb.Number : 'I', } class FileClass(Class): ' like Class but with a content property ' @@ -905,9 +1136,6 @@ class FileClass(Class): newid = Class.create(self, **propvalues) if not content: return newid - if content.startswith('/tracker/download.php?'): - self.set(newid, content='http://sourceforge.net'+content) - return newid nm = bnm = '%s%s' % (self.classname, newid) sd = str(int(int(newid) / 1000)) d = os.path.join(self.db.config.DATABASE, 'files', self.classname, sd) @@ -919,7 +1147,7 @@ class FileClass(Class): mimetype = propvalues.get('type', self.default_mime_type) self.db.indexer.add_text((self.classname, newid, 'content'), content, mimetype) def undo(fnm=nm, action1=os.remove, indexer=self.db.indexer): - remove(fnm) + action1(fnm) self.rollbackaction(undo) return newid def index(self, nodeid): @@ -930,11 +1158,8 @@ class FileClass(Class): self.db.indexer.add_text((self.classname, nodeid, 'content'), self.get(nodeid, 'content'), mimetype) -# Yuck - c&p to avoid getting hyperdb.Class -class IssueClass(Class): - +class IssueClass(Class, roundupdb.IssueClass): # Overridden methods: - def __init__(self, db, classname, **properties): """The newly-created class automatically includes the "messages", "files", "nosy", and "superseder" properties. If the 'properties' @@ -947,352 +1172,127 @@ class IssueClass(Class): if not properties.has_key('files'): properties['files'] = hyperdb.Multilink("file") if not properties.has_key('nosy'): - properties['nosy'] = hyperdb.Multilink("user") + # note: journalling is turned off as it really just wastes + # space. this behaviour may be overridden in an instance + properties['nosy'] = hyperdb.Multilink("user", do_journal="no") if not properties.has_key('superseder'): properties['superseder'] = hyperdb.Multilink(classname) Class.__init__(self, db, classname, **properties) - - # New methods: - - def addmessage(self, nodeid, summary, text): - """Add a message to an issue's mail spool. - - A new "msg" node is constructed using the current date, the user that - owns the database connection as the author, and the specified summary - text. - - The "files" and "recipients" fields are left empty. - - The given text is saved as the body of the message and the node is - appended to the "messages" field of the specified issue. - """ - - def nosymessage(self, nodeid, msgid, oldvalues): - """Send a message to the members of an issue's nosy list. - - The message is sent only to users on the nosy list who are not - already on the "recipients" list for the message. - These users are then added to the message's "recipients" list. - """ - users = self.db.user - messages = self.db.msg - - # figure the recipient ids - sendto = [] - r = {} - recipients = messages.get(msgid, 'recipients') - for recipid in messages.get(msgid, 'recipients'): - r[recipid] = 1 - - # figure the author's id, and indicate they've received the message - authid = messages.get(msgid, 'author') - - # possibly send the message to the author, as long as they aren't - # anonymous - if (self.db.config.MESSAGES_TO_AUTHOR == 'yes' and - users.get(authid, 'username') != 'anonymous'): - sendto.append(authid) - r[authid] = 1 - - # now figure the nosy people who weren't recipients - nosy = self.get(nodeid, 'nosy') - for nosyid in nosy: - # Don't send nosy mail to the anonymous user (that user - # shouldn't appear in the nosy list, but just in case they - # do...) - if users.get(nosyid, 'username') == 'anonymous': - continue - # make sure they haven't seen the message already - if not r.has_key(nosyid): - # send it to them - sendto.append(nosyid) - recipients.append(nosyid) - - # generate a change note - if oldvalues: - note = self.generateChangeNote(nodeid, oldvalues) - else: - note = self.generateCreateNote(nodeid) - - # we have new recipients - if sendto: - # map userids to addresses - sendto = [users.get(i, 'address') for i in sendto] - - # update the message's recipients list - messages.set(msgid, recipients=recipients) - - # send the message - self.send_message(nodeid, msgid, note, sendto) - - # XXX backwards compatibility - don't remove - sendmessage = nosymessage - - def send_message(self, nodeid, msgid, note, sendto): - '''Actually send the nominated message from this node to the sendto - recipients, with the note appended. - ''' - users = self.db.user - messages = self.db.msg - files = self.db.file - - # determine the messageid and inreplyto of the message - inreplyto = messages.get(msgid, 'inreplyto') - messageid = messages.get(msgid, 'messageid') - - # make up a messageid if there isn't one (web edit) - if not messageid: - # this is an old message that didn't get a messageid, so - # create one - messageid = "<%s.%s.%s%s@%s>"%(time.time(), random.random(), - self.classname, nodeid, self.db.config.MAIL_DOMAIN) - messages.set(msgid, messageid=messageid) - - # send an email to the people who missed out - cn = self.classname - title = self.get(nodeid, 'title') or '%s message copy'%cn - # figure author information - authid = messages.get(msgid, 'author') - authname = users.get(authid, 'realname') - if not authname: - authname = users.get(authid, 'username') - authaddr = users.get(authid, 'address') - if authaddr: - authaddr = ' <%s>'%authaddr - else: - authaddr = '' - - # make the message body - m = [''] - - # put in roundup's signature - if self.db.config.EMAIL_SIGNATURE_POSITION == 'top': - m.append(self.email_signature(nodeid, msgid)) - - # add author information - if len(self.get(nodeid,'messages')) == 1: - m.append("New submission from %s%s:"%(authname, authaddr)) - else: - m.append("%s%s added the comment:"%(authname, authaddr)) - m.append('') - - # add the content - m.append(messages.get(msgid, 'content')) - - # add the change note - if note: - m.append(note) - - # put in roundup's signature - if self.db.config.EMAIL_SIGNATURE_POSITION == 'bottom': - m.append(self.email_signature(nodeid, msgid)) - - # encode the content as quoted-printable - content = cStringIO.StringIO('\n'.join(m)) - content_encoded = cStringIO.StringIO() - quopri.encode(content, content_encoded, 0) - content_encoded = content_encoded.getvalue() - - # get the files for this message - message_files = messages.get(msgid, 'files') - - # make sure the To line is always the same (for testing mostly) - sendto.sort() - - # create the message - message = cStringIO.StringIO() - writer = MimeWriter.MimeWriter(message) - writer.addheader('Subject', '[%s%s] %s'%(cn, nodeid, title)) - writer.addheader('To', ', '.join(sendto)) - writer.addheader('From', '%s <%s>'%(authname, - self.db.config.ISSUE_TRACKER_EMAIL)) - writer.addheader('Reply-To', '%s <%s>'%(self.db.config.INSTANCE_NAME, - self.db.config.ISSUE_TRACKER_EMAIL)) - writer.addheader('MIME-Version', '1.0') - if messageid: - writer.addheader('Message-Id', messageid) - if inreplyto: - writer.addheader('In-Reply-To', inreplyto) - - # add a uniquely Roundup header to help filtering - writer.addheader('X-Roundup-Name', self.db.config.INSTANCE_NAME) - - # attach files - if message_files: - part = writer.startmultipartbody('mixed') - part = writer.nextpart() - part.addheader('Content-Transfer-Encoding', 'quoted-printable') - body = part.startbody('text/plain') - body.write(content_encoded) - for fileid in message_files: - name = files.get(fileid, 'name') - mime_type = files.get(fileid, 'type') - content = files.get(fileid, 'content') - part = writer.nextpart() - if mime_type == 'text/plain': - part.addheader('Content-Disposition', - 'attachment;\n filename="%s"'%name) - part.addheader('Content-Transfer-Encoding', '7bit') - body = part.startbody('text/plain') - body.write(content) - else: - # some other type, so encode it - if not mime_type: - # this should have been done when the file was saved - mime_type = mimetypes.guess_type(name)[0] - if mime_type is None: - mime_type = 'application/octet-stream' - part.addheader('Content-Disposition', - 'attachment;\n filename="%s"'%name) - part.addheader('Content-Transfer-Encoding', 'base64') - body = part.startbody(mime_type) - body.write(base64.encodestring(content)) - writer.lastpart() - else: - writer.addheader('Content-Transfer-Encoding', 'quoted-printable') - body = writer.startbody('text/plain') - body.write(content_encoded) - - # now try to send the message - if SENDMAILDEBUG: - open(SENDMAILDEBUG, 'w').write('FROM: %s\nTO: %s\n%s\n'%( - self.db.config.ADMIN_EMAIL, - ', '.join(sendto),message.getvalue())) - else: - try: - # send the message as admin so bounces are sent there - # instead of to roundup - smtp = smtplib.SMTP(self.db.config.MAILHOST) - smtp.sendmail(self.db.config.ADMIN_EMAIL, sendto, - message.getvalue()) - except socket.error, value: - raise MessageSendError, \ - "Couldn't send confirmation email: mailhost %s"%value - except smtplib.SMTPException, value: - raise MessageSendError, \ - "Couldn't send confirmation email: %s"%value - - def email_signature(self, nodeid, msgid): - ''' Add a signature to the e-mail with some useful information - ''' - web = self.db.config.ISSUE_TRACKER_WEB + 'issue'+ nodeid - email = '"%s" <%s>'%(self.db.config.INSTANCE_NAME, - self.db.config.ISSUE_TRACKER_EMAIL) - line = '_' * max(len(web), len(email)) - return '%s\n%s\n%s\n%s'%(line, email, web, line) - - def generateCreateNote(self, nodeid): - """Generate a create note that lists initial property values - """ - cn = self.classname - cl = self.db.classes[cn] - props = cl.getprops(protected=0) - - # list the values - m = [] - l = props.items() - l.sort() - for propname, prop in l: - value = cl.get(nodeid, propname, None) - # skip boring entries - if not value: +CURVERSION = 2 + +class Indexer(indexer.Indexer): + disallows = {'THE':1, 'THIS':1, 'ZZZ':1, 'THAT':1, 'WITH':1} + def __init__(self, path, datadb): + self.path = os.path.join(path, 'index.mk4') + self.db = metakit.storage(self.path, 1) + self.datadb = datadb + self.reindex = 0 + v = self.db.view('version') + if not v.structure(): + v = self.db.getas('version[vers:I]') + self.db.commit() + v.append(vers=CURVERSION) + self.reindex = 1 + elif v[0].vers != CURVERSION: + v[0].vers = CURVERSION + self.reindex = 1 + if self.reindex: + self.db.getas('ids[tblid:I,nodeid:I,propid:I,ignore:I]') + self.db.getas('index[word:S,hits[pos:I]]') + self.db.commit() + self.reindex = 1 + self.changed = 0 + self.propcache = {} + def force_reindex(self): + v = self.db.view('ids') + v[:] = [] + v = self.db.view('index') + v[:] = [] + self.db.commit() + self.reindex = 1 + def should_reindex(self): + return self.reindex + def _getprops(self, classname): + props = self.propcache.get(classname, None) + if props is None: + props = self.datadb.view(classname).structure() + props = [prop.name for prop in props] + self.propcache[classname] = props + return props + def _getpropid(self, classname, propname): + return self._getprops(classname).index(propname) + def _getpropname(self, classname, propid): + return self._getprops(classname)[propid] + + def add_text(self, identifier, text, mime_type='text/plain'): + if mime_type != 'text/plain': + return + classname, nodeid, property = identifier + tbls = self.datadb.view('tables') + tblid = tbls.find(name=classname) + if tblid < 0: + raise KeyError, "unknown class %r"%classname + nodeid = int(nodeid) + propid = self._getpropid(classname, property) + ids = self.db.view('ids') + oldpos = ids.find(tblid=tblid,nodeid=nodeid,propid=propid,ignore=0) + if oldpos > -1: + ids[oldpos].ignore = 1 + self.changed = 1 + pos = ids.append(tblid=tblid,nodeid=nodeid,propid=propid) + + wordlist = re.findall(r'\b\w{2,25}\b', text.upper()) + words = {} + for word in wordlist: + if not self.disallows.has_key(word): + words[word] = 1 + words = words.keys() + + index = self.db.view('index').ordered(1) + for word in words: + ndx = index.find(word=word) + if ndx < 0: + index.append(word=word) + ndx = index.find(word=word) + index[ndx].hits.append(pos=pos) + self.changed = 1 + + def find(self, wordlist): + hits = None + index = self.db.view('index').ordered(1) + for word in wordlist: + word = word.upper() + if not 2 < len(word) < 26: continue - if isinstance(prop, hyperdb.Link): - link = self.db.classes[prop.classname] - if value: - key = link.labelprop(default_to_id=1) - if key: - value = link.get(value, key) - else: - value = '' - elif isinstance(prop, hyperdb.Multilink): - if value is None: value = [] - l = [] - link = self.db.classes[prop.classname] - key = link.labelprop(default_to_id=1) - if key: - value = [link.get(entry, key) for entry in value] - value.sort() - value = ', '.join(value) - m.append('%s: %s'%(propname, value)) - m.insert(0, '----------') - m.insert(0, '') - return '\n'.join(m) - - def generateChangeNote(self, nodeid, oldvalues): - """Generate a change note that lists property changes - """ - cn = self.classname - cl = self.db.classes[cn] - changed = {} - props = cl.getprops(protected=0) - - # determine what changed - for key in oldvalues.keys(): - if key in ['files','messages']: continue - new_value = cl.get(nodeid, key) - # the old value might be non existent - try: - old_value = oldvalues[key] - if type(new_value) is type([]): - new_value.sort() - old_value.sort() - if new_value != old_value: - changed[key] = old_value - except: - changed[key] = new_value - - # list the changes - m = [] - l = changed.items() - l.sort() - for propname, oldvalue in l: - prop = props[propname] - value = cl.get(nodeid, propname, None) - if isinstance(prop, hyperdb.Link): - link = self.db.classes[prop.classname] - key = link.labelprop(default_to_id=1) - if key: - if value: - value = link.get(value, key) - else: - value = '' - if oldvalue: - oldvalue = link.get(oldvalue, key) - else: - oldvalue = '' - change = '%s -> %s'%(oldvalue, value) - elif isinstance(prop, hyperdb.Multilink): - change = '' - if value is None: value = [] - if oldvalue is None: oldvalue = [] - l = [] - link = self.db.classes[prop.classname] - key = link.labelprop(default_to_id=1) - # check for additions - for entry in value: - if entry in oldvalue: continue - if key: - l.append(link.get(entry, key)) - else: - l.append(entry) - if l: - change = '+%s'%(', '.join(l)) - l = [] - # check for removals - for entry in oldvalue: - if entry in value: continue - if key: - l.append(link.get(entry, key)) - else: - l.append(entry) - if l: - change += ' -%s'%(', '.join(l)) + ndx = index.find(word=word) + if ndx < 0: + return {} + if hits is None: + hits = index[ndx].hits else: - change = '%s -> %s'%(oldvalue, value) - m.append('%s: %s'%(propname, change)) - if m: - m.insert(0, '----------') - m.insert(0, '') - return '\n'.join(m) + hits = hits.intersect(index[ndx].hits) + if len(hits) == 0: + return {} + if hits is None: + return {} + rslt = {} + ids = self.db.view('ids').remapwith(hits) + tbls = self.datadb.view('tables') + for i in range(len(ids)): + hit = ids[i] + if not hit.ignore: + classname = tbls[hit.tblid].name + nodeid = str(hit.nodeid) + property = self._getpropname(classname, hit.propid) + rslt[i] = (classname, nodeid, property) + return rslt + def save_index(self): + if self.changed: + self.db.commit() + self.changed = 0 + def rollback(self): + if self.changed: + self.db.rollback() + self.db = metakit.storage(self.path, 1) + self.changed = 0