X-Git-Url: https://git.tokkee.org/?a=blobdiff_plain;f=roundup%2Fbackends%2Fback_metakit.py;h=d78b27f9f8034f176cf7c29718d4c42938faad02;hb=ac2e8aab0ebae2eff3fab1e482dba47184ec7447;hp=1c93af77592aa747a41b08f153f75cd3800c10d1;hpb=e2f11d886cb2651b550fd31ad215e3f4cafee6fa;p=roundup.git diff --git a/roundup/backends/back_metakit.py b/roundup/backends/back_metakit.py index 1c93af7..d78b27f 100755 --- a/roundup/backends/back_metakit.py +++ b/roundup/backends/back_metakit.py @@ -1,35 +1,37 @@ -from roundup import hyperdb, date, password, roundupdb +from roundup import hyperdb, date, password, roundupdb, security import metakit +from sessions import Sessions import re, marshal, os, sys, weakref, time, calendar -from roundup.indexer import Indexer +from roundup import indexer +import locking -_instances = weakref.WeakValueDictionary() +_dbs = {} def Database(config, journaltag=None): - if _instances.has_key(id(config)): - db = _instances[id(config)] - old = db.journaltag + db = _dbs.get(config.DATABASE, None) + if db is None or db._db is None: + db = _Database(config, journaltag) + _dbs[config.DATABASE] = db + else: db.journaltag = journaltag try: delattr(db, 'curuserid') except AttributeError: pass - return db - else: - db = _Database(config, journaltag) - _instances[id(config)] = db - return db + return db class _Database(hyperdb.Database): def __init__(self, config, journaltag=None): self.config = config self.journaltag = journaltag self.classes = {} - self._classes = [] self.dirty = 0 - self.__RW = 0 + self.lockfile = None self._db = self.__open() - self.indexer = Indexer(self.config.DATABASE) + self.indexer = Indexer(self.config.DATABASE, self._db) + self.sessions = Sessions(self.config) + self.security = security.Security(self) + os.umask(0002) def post_init(self): if self.indexer.should_reindex(): @@ -50,6 +52,8 @@ class _Database(hyperdb.Database): except KeyError: x = 0 return x + elif classname == 'transactions': + return self.dirty return self.getclass(classname) def getclass(self, classname): return self.classes[classname] @@ -59,13 +63,10 @@ class _Database(hyperdb.Database): # --- exposed methods def commit(self): if self.dirty: - if self.__RW: - self._db.commit() - for cl in self.classes.values(): - cl._commit() - self.indexer.save_index() - else: - raise RuntimeError, "metakit is open RO" + self._db.commit() + for cl in self.classes.values(): + cl._commit() + self.indexer.save_index() self.dirty = 0 def rollback(self): if self.dirty: @@ -77,11 +78,13 @@ class _Database(hyperdb.Database): for cl in self.classes.values(): cl._clear() def hasnode(self, classname, nodeid): - return self.getclass(clasname).hasnode(nodeid) + return self.getclass(classname).hasnode(nodeid) def pack(self, pack_before): pass def addclass(self, cl): self.classes[cl.classname] = cl + if self.tables.find(name=cl.classname) < 0: + self.tables.append(name=cl.classname) def addjournal(self, tablenm, nodeid, action, params): tblid = self.tables.find(name=tablenm) if tblid == -1: @@ -114,25 +117,22 @@ class _Database(hyperdb.Database): return rslt def close(self): - import time - now = time.time - start = now() for cl in self.classes.values(): cl.db = None - #self._db.rollback() - #print "pre-close cleanup of DB(%d) took %2.2f secs" % (self.__RW, now()-start) self._db = None - #print "close of DB(%d) took %2.2f secs" % (self.__RW, now()-start) + locking.release_lock(self.lockfile) + del _dbs[self.config.DATABASE] + self.lockfile.close() self.classes = {} - try: - del _instances[id(self.config)] - except KeyError: - pass - self.__RW = 0 - + self.indexer = None + # --- internal def __open(self): self.dbnm = db = os.path.join(self.config.DATABASE, 'tracker.mk4') + lockfilenm = db[:-3]+'lck' + self.lockfile = locking.acquire_lock(lockfilenm) + self.lockfile.write(str(os.getpid())) + self.lockfile.flush() self.fastopen = 0 if os.path.exists(db): dbtm = os.path.getmtime(db) @@ -147,11 +147,7 @@ class _Database(hyperdb.Database): else: # can't find schemamod - must be frozen self.fastopen = 1 - else: - self.__RW = 1 - if not self.fastopen: - self.__RW = 1 - db = metakit.storage(db, self.__RW) + db = metakit.storage(db, 1) hist = db.view('history') tables = db.view('tables') if not self.fastopen: @@ -162,21 +158,7 @@ class _Database(hyperdb.Database): self.tables = tables self.hist = hist return db - def isReadOnly(self): - return self.__RW == 0 - def getWriteAccess(self): - if self.journaltag is not None and self.__RW == 0: - #now = time.time - #start = now() - self._db = None - #print "closing the file took %2.2f secs" % (now()-start) - #start = now() - self._db = metakit.storage(self.dbnm, 1) - self.__RW = 1 - self.hist = self._db.view('history') - self.tables = self._db.view('tables') - #print "getting RW access took %2.2f secs" % (now()-start) - + _STRINGTYPE = type('') _LISTTYPE = type([]) _CREATE, _SET, _RETIRE, _LINK, _UNLINK = range(5) @@ -196,7 +178,8 @@ _ALLOWSETTINGPRIVATEPROPS = 0 class Class: privateprops = None def __init__(self, db, classname, **properties): - self.db = weakref.proxy(db) + #self.db = weakref.proxy(db) + self.db = db self.classname = classname self.keyname = None self.ruprops = properties @@ -247,6 +230,7 @@ class Class: self.reactors[event].append(detector) # --- the hyperdb.Class methods def create(self, **propvalues): + self.fireAuditors('create', None, propvalues) rowdict = {} rowdict['id'] = newid = self.maxid self.maxid += 1 @@ -292,8 +276,10 @@ class Class: if propvalues.has_key('#ISNEW'): isnew = 1 del propvalues['#ISNEW'] + if not isnew: + self.fireAuditors('set', nodeid, propvalues) if not propvalues: - return + return propvalues if propvalues.has_key('id'): raise KeyError, '"id" is reserved' if self.db.journaltag is None: @@ -344,9 +330,14 @@ class Class: # do stuff based on the prop type if isinstance(prop, hyperdb.Link): link_class = prop.classname + # must be a string or None + if value is not None and not isinstance(value, type('')): + raise ValueError, 'property "%s" link value be a string'%( + propname) + # Roundup sets to "unselected" by passing None + if value is None: + value = 0 # if it isn't a number, it's a key - if type(value) != _STRINGTYPE: - raise ValueError, 'link value must be String' try: int(value) except ValueError: @@ -356,7 +347,8 @@ class Class: raise IndexError, 'new property "%s": %s not a %s'%( key, value, prop.classname) - if not self.db.getclass(link_class).hasnode(value): + if (value is not None and + not self.db.getclass(link_class).hasnode(value)): raise IndexError, '%s has no node %s'%(link_class, value) setattr(row, key, int(value)) @@ -365,11 +357,13 @@ class Class: if self.do_journal and prop.do_journal: # register the unlink with the old linked node if oldvalue: - self.db.addjournal(link_class, value, _UNLINK, (self.classname, str(row.id), key)) + self.db.addjournal(link_class, value, _UNLINK, + (self.classname, str(row.id), key)) # register the link with the newly linked node if value: - self.db.addjournal(link_class, value, _LINK, (self.classname, str(row.id), key)) + self.db.addjournal(link_class, value, _LINK, + (self.classname, str(row.id), key)) elif isinstance(prop, hyperdb.Multilink): if type(value) != _LISTTYPE: @@ -423,6 +417,8 @@ class Class: for id in adds: sv.append(fid=int(id)) changes[key] = oldvalue + if not rmvd and not adds: + del propvalues[key] elif isinstance(prop, hyperdb.String): @@ -455,12 +451,23 @@ class Class: setattr(row, key, str(value)) changes[key] = str(oldvalue) propvalues[key] = str(value) + + elif value is not None and isinstance(prop, hyperdb.Number): + setattr(row, key, int(value)) + changes[key] = oldvalue + propvalues[key] = value + + elif value is not None and isinstance(prop, hyperdb.Boolean): + bv = value != 0 + setattr(row, key, bv) + changes[key] = oldvalue + propvalues[key] = value oldnode[key] = oldvalue # nothing to do? if not propvalues: - return + return propvalues if not propvalues.has_key('activity'): row.activity = int(time.time()) if isnew: @@ -473,10 +480,15 @@ class Class: if self.do_journal: if isnew: self.db.addjournal(self.classname, nodeid, _CREATE, {}) + self.fireReactors('create', nodeid, None) else: self.db.addjournal(self.classname, nodeid, _SET, changes) + self.fireReactors('set', nodeid, oldnode) + return propvalues + def retire(self, nodeid): + self.fireAuditors('retire', nodeid, None) view = self.getview(1) ndx = view.find(id=int(nodeid)) if ndx < 0: @@ -487,11 +499,13 @@ class Class: row._isdel = 1 if self.do_journal: self.db.addjournal(self.classname, nodeid, _RETIRE, {}) - iv = self.getindexview(1) - ndx = iv.find(k=getattr(row, self.keyname),i=row.id) - if ndx > -1: - iv.delete(ndx) + if self.keyname: + iv = self.getindexview(1) + ndx = iv.find(k=getattr(row, self.keyname),i=row.id) + if ndx > -1: + iv.delete(ndx) self.db.dirty = 1 + self.fireReactors('retire', nodeid, None) def history(self, nodeid): if not self.do_journal: raise ValueError, 'Journalling is disabled for this class' @@ -507,11 +521,9 @@ class Class: if self.db.fastopen and iv.structure(): return # very first setkey ever - self.db.getWriteAccess() self.db.dirty = 1 iv = self.db._db.getas('_%s[k:S,i:I]' % self.classname) iv = iv.ordered(1) - #XXX # print "setkey building index" for row in self.getview(): iv.append(k=getattr(row, propname), i=row.id) @@ -532,6 +544,20 @@ class Class: if ndx > -1: return str(view[ndx].id) raise KeyError, keyvalue + + def destroy(self, keyvalue): + #TODO clean this up once Richard's said how it should work + iv = self.getindexview() + if iv: + ndx = iv.find(k=keyvalue) + if ndx > -1: + id = iv[ndx].i + iv.delete(ndx) + view = self.getview() + ndx = view.find(id=id) + if ndx > -1: + view.delete(ndx) + def find(self, **propspec): """Get the ids of nodes in this class which link to the given nodes. @@ -604,7 +630,6 @@ class Class: if self.ruprops.has_key(key): raise ValueError, "%s is already a property of %s" % (key, self.classname) self.ruprops.update(properties) - self.db.getWriteAccess() self.db.fastopen = 0 view = self.__getview() self.db.commit() @@ -661,6 +686,14 @@ class Class: regexes[propname] = re.compile(v, re.I) elif propname == 'id': where[propname] = int(value) + elif isinstance(prop, hyperdb.Boolean): + if type(value) is _STRINGTYPE: + bv = value.lower() in ('yes', 'true', 'on', '1') + else: + bv = value + where[propname] = bv + elif isinstance(prop, hyperdb.Number): + where[propname] = int(value) else: where[propname] = str(value) v = self.getview() @@ -720,7 +753,24 @@ class Class: try: prop = getattr(v, propname) except AttributeError: + print "MK has no property %s" % propname continue + propclass = self.ruprops.get(propname, None) + if propclass is None: + propclass = self.privateprops.get(propname, None) + if propclass is None: + print "Schema has no property %s" % propname + continue + if isinstance(propclass, hyperdb.Link): + linkclass = self.db.getclass(propclass.classname) + lv = linkclass.getview() + lv = lv.rename('id', propname) + v = v.join(lv, prop, 1) + if linkclass.getprops().has_key('order'): + propname = 'order' + else: + propname = linkclass.labelprop() + prop = getattr(v, propname) if isreversed: rev.append(prop) sortspec.append(prop) @@ -837,16 +887,13 @@ class Class: else: mkprop = None if mkprop is None: - print "%s missing prop %s (%s)" % (self.classname, nm, rutyp.__class__.__name__) break if _typmap[rutyp.__class__] != mkprop.type: - print "%s - prop %s (%s) has wrong mktyp (%s)" % (self.classname, nm, rutyp.__class__.__name__, mkprop.type) break else: return view.ordered(1) # need to create or restructure the mk view # id comes first, so MK will order it for us - self.db.getWriteAccess() self.db.dirty = 1 s = ["%s[id:I" % self.classname] for nm, rutyp in self.ruprops.items(): @@ -859,12 +906,8 @@ class Class: self.db.commit() return v.ordered(1) def getview(self, RW=0): - if RW and self.db.isReadOnly(): - self.db.getWriteAccess() return self.db._db.view(self.classname).ordered(1) def getindexview(self, RW=0): - if RW and self.db.isReadOnly(): - self.db.getWriteAccess() return self.db._db.view("_%s" % self.classname).ordered(1) def _fetchML(sv): @@ -891,6 +934,9 @@ _converters = { hyperdb.Multilink : _fetchML, hyperdb.Interval : date.Interval, hyperdb.Password : _fetchPW, + hyperdb.Boolean : lambda n: n, + hyperdb.Number : lambda n: n, + hyperdb.String : str, } class FileName(hyperdb.String): @@ -904,6 +950,8 @@ _typmap = { hyperdb.Multilink : 'V', hyperdb.Interval : 'S', hyperdb.Password : 'S', + hyperdb.Boolean : 'I', + hyperdb.Number : 'I', } class FileClass(Class): ' like Class but with a content property ' @@ -929,9 +977,6 @@ class FileClass(Class): newid = Class.create(self, **propvalues) if not content: return newid - if content.startswith('/tracker/download.php?'): - self.set(newid, content='http://sourceforge.net'+content) - return newid nm = bnm = '%s%s' % (self.classname, newid) sd = str(int(int(newid) / 1000)) d = os.path.join(self.db.config.DATABASE, 'files', self.classname, sd) @@ -943,7 +988,7 @@ class FileClass(Class): mimetype = propvalues.get('type', self.default_mime_type) self.db.indexer.add_text((self.classname, newid, 'content'), content, mimetype) def undo(fnm=nm, action1=os.remove, indexer=self.db.indexer): - remove(fnm) + action1(fnm) self.rollbackaction(undo) return newid def index(self, nodeid): @@ -972,4 +1017,108 @@ class IssueClass(Class, roundupdb.IssueClass): if not properties.has_key('superseder'): properties['superseder'] = hyperdb.Multilink(classname) Class.__init__(self, db, classname, **properties) + +CURVERSION = 1 +class Indexer(indexer.Indexer): + disallows = {'THE':1, 'THIS':1, 'ZZZ':1, 'THAT':1, 'WITH':1} + def __init__(self, path, datadb): + self.db = metakit.storage(os.path.join(path, 'index.mk4'), 1) + self.datadb = datadb + self.reindex = 0 + v = self.db.view('version') + if not v.structure(): + v = self.db.getas('version[vers:I]') + self.db.commit() + v.append(vers=CURVERSION) + self.reindex = 1 + elif v[0].vers != CURVERSION: + v[0].vers = CURVERSION + self.reindex = 1 + if self.reindex: + self.db.getas('ids[tblid:I,nodeid:I,propid:I]') + self.db.getas('index[word:S,hits[pos:I]]') + self.db.commit() + self.reindex = 1 + self.changed = 0 + self.propcache = {} + def force_reindex(self): + v = self.db.view('ids') + v[:] = [] + v = self.db.view('index') + v[:] = [] + self.db.commit() + self.reindex = 1 + def should_reindex(self): + return self.reindex + def _getprops(self, classname): + props = self.propcache.get(classname, None) + if props is None: + props = self.datadb.view(classname).structure() + props = [prop.name for prop in props] + self.propcache[classname] = props + return props + def _getpropid(self, classname, propname): + return self._getprops(classname).index(propname) + def _getpropname(self, classname, propid): + return self._getprops(classname)[propid] + def add_text(self, identifier, text, mime_type='text/plain'): + if mime_type != 'text/plain': + return + classname, nodeid, property = identifier + tbls = self.datadb.view('tables') + tblid = tbls.find(name=classname) + if tblid < 0: + raise KeyError, "unknown class %r"%classname + nodeid = int(nodeid) + propid = self._getpropid(classname, property) + pos = self.db.view('ids').append(tblid=tblid,nodeid=nodeid,propid=propid) + + wordlist = re.findall(r'\b\w{3,25}\b', text) + words = {} + for word in wordlist: + word = word.upper() + if not self.disallows.has_key(word): + words[word] = 1 + words = words.keys() + + index = self.db.view('index').ordered(1) + for word in words: + ndx = index.find(word=word) + if ndx < 0: + ndx = index.append(word=word) + hits = index[ndx].hits + if len(hits)==0 or hits.find(pos=pos) < 0: + hits.append(pos=pos) + self.changed = 1 + def find(self, wordlist): + hits = None + index = self.db.view('index').ordered(1) + for word in wordlist: + if not 2 < len(word) < 26: + continue + ndx = index.find(word=word) + if ndx < 0: + return {} + if hits is None: + hits = index[ndx].hits + else: + hits = hits.intersect(index[ndx].hits) + if len(hits) == 0: + return {} + if hits is None: + return {} + rslt = {} + ids = self.db.view('ids').remapwith(hits) + tbls = self.datadb.view('tables') + for i in range(len(ids)): + hit = ids[i] + classname = tbls[hit.tblid].name + nodeid = str(hit.nodeid) + property = self._getpropname(classname, hit.propid) + rslt[i] = (classname, nodeid, property) + return rslt + def save_index(self): + if self.changed: + self.db.commit() + self.changed = 0