X-Git-Url: https://git.tokkee.org/?a=blobdiff_plain;f=roundup%2Fbackends%2Fback_metakit.py;h=c8e220d8cfdbbafe927eb65854264e25580665ad;hb=9f96bccb32cae3de034a552084ddad6c9f94b29a;hp=d78b27f9f8034f176cf7c29718d4c42938faad02;hpb=89591aaaaa98e0aa929cc1544bdc6b9348057f3c;p=roundup.git diff --git a/roundup/backends/back_metakit.py b/roundup/backends/back_metakit.py index d78b27f..c8e220d 100755 --- a/roundup/backends/back_metakit.py +++ b/roundup/backends/back_metakit.py @@ -33,6 +33,7 @@ class _Database(hyperdb.Database): self.security = security.Security(self) os.umask(0002) + def post_init(self): if self.indexer.should_reindex(): self.reindex() @@ -47,16 +48,25 @@ class _Database(hyperdb.Database): # --- defined in ping's spec def __getattr__(self, classname): if classname == 'curuserid': + if self.journaltag is None: + return None + try: self.curuserid = x = int(self.classes['user'].lookup(self.journaltag)) except KeyError: - x = 0 + if self.journaltag == 'admin': + self.curuserid = x = 1 + else: + x = 0 return x elif classname == 'transactions': return self.dirty return self.getclass(classname) def getclass(self, classname): - return self.classes[classname] + try: + return self.classes[classname] + except KeyError: + raise KeyError, 'There is no class called "%s"'%classname def getclasses(self): return self.classes.keys() # --- end of ping's spec @@ -73,61 +83,110 @@ class _Database(hyperdb.Database): for cl in self.classes.values(): cl._rollback() self._db.rollback() + self._db = None + self._db = metakit.storage(self.dbnm, 1) + self.hist = self._db.view('history') + self.tables = self._db.view('tables') + self.indexer.rollback() + self.indexer.datadb = self._db self.dirty = 0 + def clearCache(self): + for cl in self.classes.values(): + cl._commit() def clear(self): for cl in self.classes.values(): cl._clear() def hasnode(self, classname, nodeid): return self.getclass(classname).hasnode(nodeid) def pack(self, pack_before): - pass + mindate = int(calendar.timegm(pack_before.get_tuple())) + i = 0 + while i < len(self.hist): + if self.hist[i].date < mindate and self.hist[i].action != _CREATE: + self.hist.delete(i) + else: + i = i + 1 def addclass(self, cl): self.classes[cl.classname] = cl if self.tables.find(name=cl.classname) < 0: self.tables.append(name=cl.classname) - def addjournal(self, tablenm, nodeid, action, params): + def addjournal(self, tablenm, nodeid, action, params, creator=None, + creation=None): tblid = self.tables.find(name=tablenm) if tblid == -1: tblid = self.tables.append(name=tablenm) + if creator is None: + creator = self.curuserid + else: + try: + creator = int(creator) + except TypeError: + creator = int(self.getclass('user').lookup(creator)) + if creation is None: + creation = int(time.time()) + elif isinstance(creation, date.Date): + creation = int(calendar.timegm(creation.get_tuple())) # tableid:I,nodeid:I,date:I,user:I,action:I,params:B self.hist.append(tableid=tblid, nodeid=int(nodeid), - date=int(time.time()), + date=creation, action=action, - user = self.curuserid, + user = creator, params = marshal.dumps(params)) - def gethistory(self, tablenm, nodeid): + def getjournal(self, tablenm, nodeid): rslt = [] tblid = self.tables.find(name=tablenm) if tblid == -1: return rslt q = self.hist.select(tableid=tblid, nodeid=int(nodeid)) + if len(q) == 0: + raise IndexError, "no history for id %s in %s" % (nodeid, tablenm) i = 0 - userclass = self.getclass('user') + #userclass = self.getclass('user') for row in q: try: params = marshal.loads(row.params) except ValueError: print "history couldn't unmarshal %r" % row.params params = {} - usernm = userclass.get(str(row.user), 'username') + #usernm = userclass.get(str(row.user), 'username') dt = date.Date(time.gmtime(row.date)) - rslt.append((i, dt, usernm, _actionnames[row.action], params)) - i += 1 + #rslt.append((nodeid, dt, usernm, _actionnames[row.action], params)) + rslt.append((nodeid, dt, str(row.user), _actionnames[row.action], params)) return rslt + def destroyjournal(self, tablenm, nodeid): + nodeid = int(nodeid) + tblid = self.tables.find(name=tablenm) + if tblid == -1: + return + i = 0 + hist = self.hist + while i < len(hist): + if hist[i].tableid == tblid and hist[i].nodeid == nodeid: + hist.delete(i) + else: + i = i + 1 + self.dirty = 1 + def close(self): for cl in self.classes.values(): cl.db = None self._db = None - locking.release_lock(self.lockfile) - del _dbs[self.config.DATABASE] - self.lockfile.close() + if self.lockfile is not None: + locking.release_lock(self.lockfile) + if _dbs.has_key(self.config.DATABASE): + del _dbs[self.config.DATABASE] + if self.lockfile is not None: + self.lockfile.close() + self.lockfile = None self.classes = {} self.indexer = None # --- internal def __open(self): + if not os.path.exists(self.config.DATABASE): + os.makedirs(self.config.DATABASE) self.dbnm = db = os.path.join(self.config.DATABASE, 'tracker.mk4') lockfilenm = db[:-3]+'lck' self.lockfile = locking.acquire_lock(lockfilenm) @@ -155,6 +214,7 @@ class _Database(hyperdb.Database): hist = db.getas('history[tableid:I,nodeid:I,date:I,user:I,action:I,params:B]') if not tables.structure(): tables = db.getas('tables[name:S]') + db.commit() self.tables = tables self.hist = hist return db @@ -283,7 +343,7 @@ class Class: if propvalues.has_key('id'): raise KeyError, '"id" is reserved' if self.db.journaltag is None: - raise DatabaseError, 'Database open read-only' + raise hyperdb.DatabaseError, 'Database open read-only' view = self.getview(1) # node must exist & not be retired id = int(nodeid) @@ -295,7 +355,7 @@ class Class: raise IndexError, "%s has no node %s" % (self.classname, nodeid) oldnode = self.uncommitted.setdefault(id, {}) changes = {} - + for key, value in propvalues.items(): # this will raise the KeyError if the property isn't valid # ... we don't use getprops() here because we only care about @@ -333,7 +393,7 @@ class Class: # must be a string or None if value is not None and not isinstance(value, type('')): raise ValueError, 'property "%s" link value be a string'%( - propname) + key) # Roundup sets to "unselected" by passing None if value is None: value = 0 @@ -366,10 +426,12 @@ class Class: (self.classname, str(row.id), key)) elif isinstance(prop, hyperdb.Multilink): - if type(value) != _LISTTYPE: + if value is not None and type(value) != _LISTTYPE: raise TypeError, 'new property "%s" not a list of ids'%key link_class = prop.classname l = [] + if value is None: + value = [] for entry in value: if type(entry) != _STRINGTYPE: raise ValueError, 'new property "%s" link value ' \ @@ -393,7 +455,8 @@ class Class: rmvd.append(id) # register the unlink with the old linked node if self.do_journal and prop.do_journal: - self.db.addjournal(link_class, id, _UNLINK, (self.classname, str(row.id), key)) + self.db.addjournal(link_class, id, _UNLINK, + (self.classname, str(row.id), key)) # handle additions adds = [] @@ -405,7 +468,8 @@ class Class: adds.append(id) # register the link with the newly linked node if self.do_journal and prop.do_journal: - self.db.addjournal(link_class, id, _LINK, (self.classname, str(row.id), key)) + self.db.addjournal(link_class, id, _LINK, + (self.classname, str(row.id), key)) sv = getattr(row, key) i = 0 @@ -420,45 +484,66 @@ class Class: if not rmvd and not adds: del propvalues[key] - elif isinstance(prop, hyperdb.String): if value is not None and type(value) != _STRINGTYPE: raise TypeError, 'new property "%s" not a string'%key + if value is None: + value = '' setattr(row, key, value) changes[key] = oldvalue if hasattr(prop, 'isfilename') and prop.isfilename: propvalues[key] = os.path.basename(value) if prop.indexme: - self.db.indexer.add_text((self.classname, nodeid, key), value, 'text/plain') + self.db.indexer.add_text((self.classname, nodeid, key), + value, 'text/plain') elif isinstance(prop, hyperdb.Password): - if not isinstance(value, password.Password): + if value is not None and not isinstance(value, password.Password): raise TypeError, 'new property "%s" not a Password'% key + if value is None: + value = '' setattr(row, key, str(value)) changes[key] = str(oldvalue) propvalues[key] = str(value) - elif value is not None and isinstance(prop, hyperdb.Date): - if not isinstance(value, date.Date): + elif isinstance(prop, hyperdb.Date): + if value is not None and not isinstance(value, date.Date): raise TypeError, 'new property "%s" not a Date'% key - setattr(row, key, int(calendar.timegm(value.get_tuple()))) + if value is None: + setattr(row, key, 0) + else: + setattr(row, key, int(calendar.timegm(value.get_tuple()))) changes[key] = str(oldvalue) propvalues[key] = str(value) - elif value is not None and isinstance(prop, hyperdb.Interval): - if not isinstance(value, date.Interval): + elif isinstance(prop, hyperdb.Interval): + if value is not None and not isinstance(value, date.Interval): raise TypeError, 'new property "%s" not an Interval'% key - setattr(row, key, str(value)) + if value is None: + setattr(row, key, '') + else: + setattr(row, key, str(value)) changes[key] = str(oldvalue) propvalues[key] = str(value) - elif value is not None and isinstance(prop, hyperdb.Number): - setattr(row, key, int(value)) + elif isinstance(prop, hyperdb.Number): + if value is None: + value = 0 + try: + v = int(value) + except ValueError: + raise TypeError, "%s (%s) is not numeric" % (key, repr(value)) + setattr(row, key, v) changes[key] = oldvalue propvalues[key] = value - elif value is not None and isinstance(prop, hyperdb.Boolean): - bv = value != 0 + elif isinstance(prop, hyperdb.Boolean): + if value is None: + bv = 0 + elif value not in (0,1): + raise TypeError, "%s (%s) is not boolean" % (key, repr(value)) + else: + bv = value setattr(row, key, bv) changes[key] = oldvalue propvalues[key] = value @@ -488,6 +573,8 @@ class Class: return propvalues def retire(self, nodeid): + if self.db.journaltag is None: + raise hyperdb.DatabaseError, 'Database open read-only' self.fireAuditors('retire', nodeid, None) view = self.getview(1) ndx = view.find(id=int(nodeid)) @@ -509,12 +596,19 @@ class Class: def history(self, nodeid): if not self.do_journal: raise ValueError, 'Journalling is disabled for this class' - return self.db.gethistory(self.classname, nodeid) + return self.db.getjournal(self.classname, nodeid) def setkey(self, propname): if self.keyname: if propname == self.keyname: return raise ValueError, "%s already indexed on %s" % (self.classname, self.keyname) + prop = self.properties.get(propname, None) + if prop is None: + prop = self.privateprops.get(propname, None) + if prop is None: + raise KeyError, "no property %s" % propname + if not isinstance(prop, hyperdb.String): + raise TypeError, "%s is not a String" % propname # first setkey for this run self.keyname = propname iv = self.db._db.view('_%s' % self.classname) @@ -545,19 +639,21 @@ class Class: return str(view[ndx].id) raise KeyError, keyvalue - def destroy(self, keyvalue): - #TODO clean this up once Richard's said how it should work - iv = self.getindexview() - if iv: - ndx = iv.find(k=keyvalue) - if ndx > -1: - id = iv[ndx].i - iv.delete(ndx) - view = self.getview() - ndx = view.find(id=id) - if ndx > -1: - view.delete(ndx) - + def destroy(self, id): + view = self.getview(1) + ndx = view.find(id=int(id)) + if ndx > -1: + if self.keyname: + keyvalue = getattr(view[ndx], self.keyname) + iv = self.getindexview(1) + if iv: + ivndx = iv.find(k=keyvalue) + if ivndx > -1: + iv.delete(ivndx) + view.delete(ndx) + self.db.destroyjournal(self.classname, id) + self.db.dirty = 1 + def find(self, **propspec): """Get the ids of nodes in this class which link to the given nodes. @@ -585,16 +681,24 @@ class Class: vws = [] for propname, ids in propspec: if type(ids) is _STRINGTYPE: - ids = {ids:1} + ids = {int(ids):1} + else: + d = {} + for id in ids.keys(): + d[int(id)] = 1 + ids = d prop = self.ruprops[propname] view = self.getview() if isinstance(prop, hyperdb.Multilink): - view = view.flatten(getattr(view, propname)) def ff(row, nm=propname, ids=ids): - return ids.has_key(str(row.fid)) + sv = getattr(row, nm) + for sr in sv: + if ids.has_key(sr.fid): + return 1 + return 0 else: def ff(row, nm=propname, ids=ids): - return ids.has_key(str(getattr(row, nm))) + return ids.has_key(getattr(row, nm)) ndxview = view.filter(ff) vws.append(ndxview.unique()) @@ -605,7 +709,7 @@ class Class: ndxview = vws[0] for v in vws[1:]: ndxview = ndxview.union(v) - view = view.remapwith(ndxview) + view = self.getview().remapwith(ndxview) rslt = [] for row in view: rslt.append(str(row.id)) @@ -634,11 +738,13 @@ class Class: view = self.__getview() self.db.commit() # ---- end of ping's spec - def filter(self, search_matches, filterspec, sort, group): + def filter(self, search_matches, filterspec, sort=(None,None), + group=(None,None)): # search_matches is None or a set (dict of {nodeid: {propname:[nodeid,...]}}) # filterspec is a dict {propname:value} - # sort and group are lists of propnames - + # sort and group are (dir, prop) where dir is '+', '-' or None + # and prop is a prop name or None + where = {'_isdel':0} mlcriteria = {} regexes = {} @@ -745,10 +851,10 @@ class Class: if sort or group: sortspec = [] rev = [] - for propname in group + sort: + for dir, propname in group, sort: + if propname is None: continue isreversed = 0 - if propname[0] == '-': - propname = propname[1:] + if dir == '-': isreversed = 1 try: prop = getattr(v, propname) @@ -845,6 +951,59 @@ class Class: self.db.indexer.add_text((self.classname, nodeid, prop), str(self.get(nodeid, prop))) + def export_list(self, propnames, nodeid): + ''' Export a node - generate a list of CSV-able data in the order + specified by propnames for the given node. + ''' + properties = self.getprops() + l = [] + for prop in propnames: + proptype = properties[prop] + value = self.get(nodeid, prop) + # "marshal" data where needed + if value is None: + pass + elif isinstance(proptype, hyperdb.Date): + value = value.get_tuple() + elif isinstance(proptype, hyperdb.Interval): + value = value.get_tuple() + elif isinstance(proptype, hyperdb.Password): + value = str(value) + l.append(repr(value)) + return l + + def import_list(self, propnames, proplist): + ''' Import a node - all information including "id" is present and + should not be sanity checked. Triggers are not triggered. The + journal should be initialised using the "creator" and "creation" + information. + + Return the nodeid of the node imported. + ''' + if self.db.journaltag is None: + raise hyperdb.DatabaseError, 'Database open read-only' + properties = self.getprops() + + d = {} + view = self.getview(1) + for i in range(len(propnames)): + value = eval(proplist[i]) + propname = propnames[i] + prop = properties[propname] + if propname == 'id': + newid = value + value = int(value) + elif isinstance(prop, hyperdb.Date): + value = int(calendar.timegm(value)) + elif isinstance(prop, hyperdb.Interval): + value = str(date.Interval(value)) + d[propname] = value + view.append(d) + creator = d.get('creator', None) + creation = d.get('creation', None) + self.db.addjournal(self.classname, newid, 'create', {}, creator, creation) + return newid + # --- used by Database def _commit(self): """ called post commit of the DB. @@ -1013,17 +1172,20 @@ class IssueClass(Class, roundupdb.IssueClass): if not properties.has_key('files'): properties['files'] = hyperdb.Multilink("file") if not properties.has_key('nosy'): - properties['nosy'] = hyperdb.Multilink("user") + # note: journalling is turned off as it really just wastes + # space. this behaviour may be overridden in an instance + properties['nosy'] = hyperdb.Multilink("user", do_journal="no") if not properties.has_key('superseder'): properties['superseder'] = hyperdb.Multilink(classname) Class.__init__(self, db, classname, **properties) -CURVERSION = 1 +CURVERSION = 2 class Indexer(indexer.Indexer): disallows = {'THE':1, 'THIS':1, 'ZZZ':1, 'THAT':1, 'WITH':1} def __init__(self, path, datadb): - self.db = metakit.storage(os.path.join(path, 'index.mk4'), 1) + self.path = os.path.join(path, 'index.mk4') + self.db = metakit.storage(self.path, 1) self.datadb = datadb self.reindex = 0 v = self.db.view('version') @@ -1036,7 +1198,7 @@ class Indexer(indexer.Indexer): v[0].vers = CURVERSION self.reindex = 1 if self.reindex: - self.db.getas('ids[tblid:I,nodeid:I,propid:I]') + self.db.getas('ids[tblid:I,nodeid:I,propid:I,ignore:I]') self.db.getas('index[word:S,hits[pos:I]]') self.db.commit() self.reindex = 1 @@ -1062,6 +1224,7 @@ class Indexer(indexer.Indexer): return self._getprops(classname).index(propname) def _getpropname(self, classname, propid): return self._getprops(classname)[propid] + def add_text(self, identifier, text, mime_type='text/plain'): if mime_type != 'text/plain': return @@ -1072,12 +1235,16 @@ class Indexer(indexer.Indexer): raise KeyError, "unknown class %r"%classname nodeid = int(nodeid) propid = self._getpropid(classname, property) - pos = self.db.view('ids').append(tblid=tblid,nodeid=nodeid,propid=propid) + ids = self.db.view('ids') + oldpos = ids.find(tblid=tblid,nodeid=nodeid,propid=propid,ignore=0) + if oldpos > -1: + ids[oldpos].ignore = 1 + self.changed = 1 + pos = ids.append(tblid=tblid,nodeid=nodeid,propid=propid) - wordlist = re.findall(r'\b\w{3,25}\b', text) + wordlist = re.findall(r'\b\w{2,25}\b', text.upper()) words = {} for word in wordlist: - word = word.upper() if not self.disallows.has_key(word): words[word] = 1 words = words.keys() @@ -1086,15 +1253,16 @@ class Indexer(indexer.Indexer): for word in words: ndx = index.find(word=word) if ndx < 0: - ndx = index.append(word=word) - hits = index[ndx].hits - if len(hits)==0 or hits.find(pos=pos) < 0: - hits.append(pos=pos) - self.changed = 1 + index.append(word=word) + ndx = index.find(word=word) + index[ndx].hits.append(pos=pos) + self.changed = 1 + def find(self, wordlist): hits = None index = self.db.view('index').ordered(1) for word in wordlist: + word = word.upper() if not 2 < len(word) < 26: continue ndx = index.find(word=word) @@ -1113,12 +1281,18 @@ class Indexer(indexer.Indexer): tbls = self.datadb.view('tables') for i in range(len(ids)): hit = ids[i] - classname = tbls[hit.tblid].name - nodeid = str(hit.nodeid) - property = self._getpropname(classname, hit.propid) - rslt[i] = (classname, nodeid, property) + if not hit.ignore: + classname = tbls[hit.tblid].name + nodeid = str(hit.nodeid) + property = self._getpropname(classname, hit.propid) + rslt[i] = (classname, nodeid, property) return rslt def save_index(self): if self.changed: self.db.commit() self.changed = 0 + def rollback(self): + if self.changed: + self.db.rollback() + self.db = metakit.storage(self.path, 1) + self.changed = 0