![[tokkee]](http://tokkee.org/images/avatar.png){kind=avatar}
index 26bf2aa33a4b4d8b8c8f50d92cdbc32ebd44ab32..c8e220d8cfdbbafe927eb65854264e25580665ad 100755 (executable)
if db is None or db._db is None:
db = _Database(config, journaltag)
_dbs[config.DATABASE] = db
+ else:
+ db.journaltag = journaltag
+ try:
+ delattr(db, 'curuserid')
+ except AttributeError:
+ pass
return db
class _Database(hyperdb.Database):
self.security = security.Security(self)
os.umask(0002)
+
def post_init(self):
if self.indexer.should_reindex():
self.reindex()
# --- defined in ping's spec
def __getattr__(self, classname):
if classname == 'curuserid':
+ if self.journaltag is None:
+ return None
+
try:
self.curuserid = x = int(self.classes['user'].lookup(self.journaltag))
except KeyError:
- x = 0
+ if self.journaltag == 'admin':
+ self.curuserid = x = 1
+ else:
+ x = 0
return x
elif classname == 'transactions':
return self.dirty
return self.getclass(classname)
def getclass(self, classname):
- return self.classes[classname]
+ try:
+ return self.classes[classname]
+ except KeyError:
+ raise KeyError, 'There is no class called "%s"'%classname
def getclasses(self):
return self.classes.keys()
# --- end of ping's spec
for cl in self.classes.values():
cl._rollback()
self._db.rollback()
+ self._db = None
+ self._db = metakit.storage(self.dbnm, 1)
+ self.hist = self._db.view('history')
+ self.tables = self._db.view('tables')
+ self.indexer.rollback()
+ self.indexer.datadb = self._db
self.dirty = 0
+ def clearCache(self):
+ for cl in self.classes.values():
+ cl._commit()
def clear(self):
for cl in self.classes.values():
cl._clear()
def hasnode(self, classname, nodeid):
return self.getclass(classname).hasnode(nodeid)
def pack(self, pack_before):
- pass
+ mindate = int(calendar.timegm(pack_before.get_tuple()))
+ i = 0
+ while i < len(self.hist):
+ if self.hist[i].date < mindate and self.hist[i].action != _CREATE:
+ self.hist.delete(i)
+ else:
+ i = i + 1
def addclass(self, cl):
self.classes[cl.classname] = cl
if self.tables.find(name=cl.classname) < 0:
self.tables.append(name=cl.classname)
- def addjournal(self, tablenm, nodeid, action, params):
+ def addjournal(self, tablenm, nodeid, action, params, creator=None,
+ creation=None):
tblid = self.tables.find(name=tablenm)
if tblid == -1:
tblid = self.tables.append(name=tablenm)
+ if creator is None:
+ creator = self.curuserid
+ else:
+ try:
+ creator = int(creator)
+ except TypeError:
+ creator = int(self.getclass('user').lookup(creator))
+ if creation is None:
+ creation = int(time.time())
+ elif isinstance(creation, date.Date):
+ creation = int(calendar.timegm(creation.get_tuple()))
# tableid:I,nodeid:I,date:I,user:I,action:I,params:B
self.hist.append(tableid=tblid,
nodeid=int(nodeid),
- date=int(time.time()),
+ date=creation,
action=action,
- user = self.curuserid,
+ user = creator,
params = marshal.dumps(params))
- def gethistory(self, tablenm, nodeid):
+ def getjournal(self, tablenm, nodeid):
rslt = []
tblid = self.tables.find(name=tablenm)
if tblid == -1:
return rslt
q = self.hist.select(tableid=tblid, nodeid=int(nodeid))
+ if len(q) == 0:
+ raise IndexError, "no history for id %s in %s" % (nodeid, tablenm)
i = 0
- userclass = self.getclass('user')
+ #userclass = self.getclass('user')
for row in q:
try:
params = marshal.loads(row.params)
except ValueError:
print "history couldn't unmarshal %r" % row.params
params = {}
- usernm = userclass.get(str(row.user), 'username')
+ #usernm = userclass.get(str(row.user), 'username')
dt = date.Date(time.gmtime(row.date))
- rslt.append((i, dt, usernm, _actionnames[row.action], params))
- i += 1
+ #rslt.append((nodeid, dt, usernm, _actionnames[row.action], params))
+ rslt.append((nodeid, dt, str(row.user), _actionnames[row.action], params))
return rslt
+ def destroyjournal(self, tablenm, nodeid):
+ nodeid = int(nodeid)
+ tblid = self.tables.find(name=tablenm)
+ if tblid == -1:
+ return
+ i = 0
+ hist = self.hist
+ while i < len(hist):
+ if hist[i].tableid == tblid and hist[i].nodeid == nodeid:
+ hist.delete(i)
+ else:
+ i = i + 1
+ self.dirty = 1
+
def close(self):
for cl in self.classes.values():
cl.db = None
self._db = None
- locking.release_lock(self.lockfile)
- del _dbs[self.config.DATABASE]
- self.lockfile.close()
+ if self.lockfile is not None:
+ locking.release_lock(self.lockfile)
+ if _dbs.has_key(self.config.DATABASE):
+ del _dbs[self.config.DATABASE]
+ if self.lockfile is not None:
+ self.lockfile.close()
+ self.lockfile = None
self.classes = {}
self.indexer = None
# --- internal
def __open(self):
+ if not os.path.exists(self.config.DATABASE):
+ os.makedirs(self.config.DATABASE)
self.dbnm = db = os.path.join(self.config.DATABASE, 'tracker.mk4')
lockfilenm = db[:-3]+'lck'
self.lockfile = locking.acquire_lock(lockfilenm)
hist = db.getas('history[tableid:I,nodeid:I,date:I,user:I,action:I,params:B]')
if not tables.structure():
tables = db.getas('tables[name:S]')
+ db.commit()
self.tables = tables
self.hist = hist
return db
if not isnew:
self.fireAuditors('set', nodeid, propvalues)
if not propvalues:
- return
+ return propvalues
if propvalues.has_key('id'):
raise KeyError, '"id" is reserved'
if self.db.journaltag is None:
- raise DatabaseError, 'Database open read-only'
+ raise hyperdb.DatabaseError, 'Database open read-only'
view = self.getview(1)
# node must exist & not be retired
id = int(nodeid)
raise IndexError, "%s has no node %s" % (self.classname, nodeid)
oldnode = self.uncommitted.setdefault(id, {})
changes = {}
-
+
for key, value in propvalues.items():
# this will raise the KeyError if the property isn't valid
# ... we don't use getprops() here because we only care about
# must be a string or None
if value is not None and not isinstance(value, type('')):
raise ValueError, 'property "%s" link value be a string'%(
- propname)
+ key)
+ # Roundup sets to "unselected" by passing None
+ if value is None:
+ value = 0
# if it isn't a number, it's a key
try:
int(value)
(self.classname, str(row.id), key))
elif isinstance(prop, hyperdb.Multilink):
- if type(value) != _LISTTYPE:
+ if value is not None and type(value) != _LISTTYPE:
raise TypeError, 'new property "%s" not a list of ids'%key
link_class = prop.classname
l = []
+ if value is None:
+ value = []
for entry in value:
if type(entry) != _STRINGTYPE:
raise ValueError, 'new property "%s" link value ' \
rmvd.append(id)
# register the unlink with the old linked node
if self.do_journal and prop.do_journal:
- self.db.addjournal(link_class, id, _UNLINK, (self.classname, str(row.id), key))
+ self.db.addjournal(link_class, id, _UNLINK,
+ (self.classname, str(row.id), key))
# handle additions
adds = []
adds.append(id)
# register the link with the newly linked node
if self.do_journal and prop.do_journal:
- self.db.addjournal(link_class, id, _LINK, (self.classname, str(row.id), key))
+ self.db.addjournal(link_class, id, _LINK,
+ (self.classname, str(row.id), key))
sv = getattr(row, key)
i = 0
for id in adds:
sv.append(fid=int(id))
changes[key] = oldvalue
+ if not rmvd and not adds:
+ del propvalues[key]
-
elif isinstance(prop, hyperdb.String):
if value is not None and type(value) != _STRINGTYPE:
raise TypeError, 'new property "%s" not a string'%key
+ if value is None:
+ value = ''
setattr(row, key, value)
changes[key] = oldvalue
if hasattr(prop, 'isfilename') and prop.isfilename:
propvalues[key] = os.path.basename(value)
if prop.indexme:
- self.db.indexer.add_text((self.classname, nodeid, key), value, 'text/plain')
+ self.db.indexer.add_text((self.classname, nodeid, key),
+ value, 'text/plain')
elif isinstance(prop, hyperdb.Password):
- if not isinstance(value, password.Password):
+ if value is not None and not isinstance(value, password.Password):
raise TypeError, 'new property "%s" not a Password'% key
+ if value is None:
+ value = ''
setattr(row, key, str(value))
changes[key] = str(oldvalue)
propvalues[key] = str(value)
- elif value is not None and isinstance(prop, hyperdb.Date):
- if not isinstance(value, date.Date):
+ elif isinstance(prop, hyperdb.Date):
+ if value is not None and not isinstance(value, date.Date):
raise TypeError, 'new property "%s" not a Date'% key
- setattr(row, key, int(calendar.timegm(value.get_tuple())))
+ if value is None:
+ setattr(row, key, 0)
+ else:
+ setattr(row, key, int(calendar.timegm(value.get_tuple())))
changes[key] = str(oldvalue)
propvalues[key] = str(value)
- elif value is not None and isinstance(prop, hyperdb.Interval):
- if not isinstance(value, date.Interval):
+ elif isinstance(prop, hyperdb.Interval):
+ if value is not None and not isinstance(value, date.Interval):
raise TypeError, 'new property "%s" not an Interval'% key
- setattr(row, key, str(value))
+ if value is None:
+ setattr(row, key, '')
+ else:
+ setattr(row, key, str(value))
changes[key] = str(oldvalue)
propvalues[key] = str(value)
- elif value is not None and isinstance(prop, hyperdb.Number):
- setattr(row, key, int(value))
+ elif isinstance(prop, hyperdb.Number):
+ if value is None:
+ value = 0
+ try:
+ v = int(value)
+ except ValueError:
+ raise TypeError, "%s (%s) is not numeric" % (key, repr(value))
+ setattr(row, key, v)
changes[key] = oldvalue
propvalues[key] = value
- elif value is not None and isinstance(prop, hyperdb.Boolean):
- bv = value != 0
+ elif isinstance(prop, hyperdb.Boolean):
+ if value is None:
+ bv = 0
+ elif value not in (0,1):
+ raise TypeError, "%s (%s) is not boolean" % (key, repr(value))
+ else:
+ bv = value
setattr(row, key, bv)
changes[key] = oldvalue
propvalues[key] = value
# nothing to do?
if not propvalues:
- return
+ return propvalues
if not propvalues.has_key('activity'):
row.activity = int(time.time())
if isnew:
self.db.addjournal(self.classname, nodeid, _SET, changes)
self.fireReactors('set', nodeid, oldnode)
+ return propvalues
+
def retire(self, nodeid):
+ if self.db.journaltag is None:
+ raise hyperdb.DatabaseError, 'Database open read-only'
self.fireAuditors('retire', nodeid, None)
view = self.getview(1)
ndx = view.find(id=int(nodeid))
def history(self, nodeid):
if not self.do_journal:
raise ValueError, 'Journalling is disabled for this class'
- return self.db.gethistory(self.classname, nodeid)
+ return self.db.getjournal(self.classname, nodeid)
def setkey(self, propname):
if self.keyname:
if propname == self.keyname:
return
raise ValueError, "%s already indexed on %s" % (self.classname, self.keyname)
+ prop = self.properties.get(propname, None)
+ if prop is None:
+ prop = self.privateprops.get(propname, None)
+ if prop is None:
+ raise KeyError, "no property %s" % propname
+ if not isinstance(prop, hyperdb.String):
+ raise TypeError, "%s is not a String" % propname
# first setkey for this run
self.keyname = propname
iv = self.db._db.view('_%s' % self.classname)
return str(view[ndx].id)
raise KeyError, keyvalue
- def destroy(self, keyvalue):
- #TODO clean this up once Richard's said how it should work
- iv = self.getindexview()
- if iv:
- ndx = iv.find(k=keyvalue)
- if ndx > -1:
- id = iv[ndx].i
- iv.delete(ndx)
- view = self.getview()
- ndx = view.find(id=id)
- if ndx > -1:
- view.delete(ndx)
-
+ def destroy(self, id):
+ view = self.getview(1)
+ ndx = view.find(id=int(id))
+ if ndx > -1:
+ if self.keyname:
+ keyvalue = getattr(view[ndx], self.keyname)
+ iv = self.getindexview(1)
+ if iv:
+ ivndx = iv.find(k=keyvalue)
+ if ivndx > -1:
+ iv.delete(ivndx)
+ view.delete(ndx)
+ self.db.destroyjournal(self.classname, id)
+ self.db.dirty = 1
+
def find(self, **propspec):
"""Get the ids of nodes in this class which link to the given nodes.
vws = []
for propname, ids in propspec:
if type(ids) is _STRINGTYPE:
- ids = {ids:1}
+ ids = {int(ids):1}
+ else:
+ d = {}
+ for id in ids.keys():
+ d[int(id)] = 1
+ ids = d
prop = self.ruprops[propname]
view = self.getview()
if isinstance(prop, hyperdb.Multilink):
- view = view.flatten(getattr(view, propname))
def ff(row, nm=propname, ids=ids):
- return ids.has_key(str(row.fid))
+ sv = getattr(row, nm)
+ for sr in sv:
+ if ids.has_key(sr.fid):
+ return 1
+ return 0
else:
def ff(row, nm=propname, ids=ids):
- return ids.has_key(str(getattr(row, nm)))
+ return ids.has_key(getattr(row, nm))
ndxview = view.filter(ff)
vws.append(ndxview.unique())
ndxview = vws[0]
for v in vws[1:]:
ndxview = ndxview.union(v)
- view = view.remapwith(ndxview)
+ view = self.getview().remapwith(ndxview)
rslt = []
for row in view:
rslt.append(str(row.id))
view = self.__getview()
self.db.commit()
# ---- end of ping's spec
- def filter(self, search_matches, filterspec, sort, group):
+ def filter(self, search_matches, filterspec, sort=(None,None),
+ group=(None,None)):
# search_matches is None or a set (dict of {nodeid: {propname:[nodeid,...]}})
# filterspec is a dict {propname:value}
- # sort and group are lists of propnames
-
+ # sort and group are (dir, prop) where dir is '+', '-' or None
+ # and prop is a prop name or None
+
where = {'_isdel':0}
mlcriteria = {}
regexes = {}
if sort or group:
sortspec = []
rev = []
- for propname in group + sort:
+ for dir, propname in group, sort:
+ if propname is None: continue
isreversed = 0
- if propname[0] == '-':
- propname = propname[1:]
+ if dir == '-':
isreversed = 1
try:
prop = getattr(v, propname)
self.db.indexer.add_text((self.classname, nodeid, prop),
str(self.get(nodeid, prop)))
+ def export_list(self, propnames, nodeid):
+ ''' Export a node - generate a list of CSV-able data in the order
+ specified by propnames for the given node.
+ '''
+ properties = self.getprops()
+ l = []
+ for prop in propnames:
+ proptype = properties[prop]
+ value = self.get(nodeid, prop)
+ # "marshal" data where needed
+ if value is None:
+ pass
+ elif isinstance(proptype, hyperdb.Date):
+ value = value.get_tuple()
+ elif isinstance(proptype, hyperdb.Interval):
+ value = value.get_tuple()
+ elif isinstance(proptype, hyperdb.Password):
+ value = str(value)
+ l.append(repr(value))
+ return l
+
+ def import_list(self, propnames, proplist):
+ ''' Import a node - all information including "id" is present and
+ should not be sanity checked. Triggers are not triggered. The
+ journal should be initialised using the "creator" and "creation"
+ information.
+
+ Return the nodeid of the node imported.
+ '''
+ if self.db.journaltag is None:
+ raise hyperdb.DatabaseError, 'Database open read-only'
+ properties = self.getprops()
+
+ d = {}
+ view = self.getview(1)
+ for i in range(len(propnames)):
+ value = eval(proplist[i])
+ propname = propnames[i]
+ prop = properties[propname]
+ if propname == 'id':
+ newid = value
+ value = int(value)
+ elif isinstance(prop, hyperdb.Date):
+ value = int(calendar.timegm(value))
+ elif isinstance(prop, hyperdb.Interval):
+ value = str(date.Interval(value))
+ d[propname] = value
+ view.append(d)
+ creator = d.get('creator', None)
+ creation = d.get('creation', None)
+ self.db.addjournal(self.classname, newid, 'create', {}, creator, creation)
+ return newid
+
# --- used by Database
def _commit(self):
""" called post commit of the DB.
if not properties.has_key('files'):
properties['files'] = hyperdb.Multilink("file")
if not properties.has_key('nosy'):
- properties['nosy'] = hyperdb.Multilink("user")
+ # note: journalling is turned off as it really just wastes
+ # space. this behaviour may be overridden in an instance
+ properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
if not properties.has_key('superseder'):
properties['superseder'] = hyperdb.Multilink(classname)
Class.__init__(self, db, classname, **properties)
-CURVERSION = 1
+CURVERSION = 2
class Indexer(indexer.Indexer):
disallows = {'THE':1, 'THIS':1, 'ZZZ':1, 'THAT':1, 'WITH':1}
def __init__(self, path, datadb):
- self.db = metakit.storage(os.path.join(path, 'index.mk4'), 1)
+ self.path = os.path.join(path, 'index.mk4')
+ self.db = metakit.storage(self.path, 1)
self.datadb = datadb
self.reindex = 0
v = self.db.view('version')
v[0].vers = CURVERSION
self.reindex = 1
if self.reindex:
- self.db.getas('ids[tblid:I,nodeid:I,propid:I]')
+ self.db.getas('ids[tblid:I,nodeid:I,propid:I,ignore:I]')
self.db.getas('index[word:S,hits[pos:I]]')
self.db.commit()
self.reindex = 1
return self._getprops(classname).index(propname)
def _getpropname(self, classname, propid):
return self._getprops(classname)[propid]
+
def add_text(self, identifier, text, mime_type='text/plain'):
if mime_type != 'text/plain':
return
raise KeyError, "unknown class %r"%classname
nodeid = int(nodeid)
propid = self._getpropid(classname, property)
- pos = self.db.view('ids').append(tblid=tblid,nodeid=nodeid,propid=propid)
+ ids = self.db.view('ids')
+ oldpos = ids.find(tblid=tblid,nodeid=nodeid,propid=propid,ignore=0)
+ if oldpos > -1:
+ ids[oldpos].ignore = 1
+ self.changed = 1
+ pos = ids.append(tblid=tblid,nodeid=nodeid,propid=propid)
- wordlist = re.findall(r'\b\w{3,25}\b', text)
+ wordlist = re.findall(r'\b\w{2,25}\b', text.upper())
words = {}
for word in wordlist:
- word = word.upper()
if not self.disallows.has_key(word):
words[word] = 1
words = words.keys()
for word in words:
ndx = index.find(word=word)
if ndx < 0:
- ndx = index.append(word=word)
- hits = index[ndx].hits
- if len(hits)==0 or hits.find(pos=pos) < 0:
- hits.append(pos=pos)
- self.changed = 1
+ index.append(word=word)
+ ndx = index.find(word=word)
+ index[ndx].hits.append(pos=pos)
+ self.changed = 1
+
def find(self, wordlist):
hits = None
index = self.db.view('index').ordered(1)
for word in wordlist:
+ word = word.upper()
if not 2 < len(word) < 26:
continue
ndx = index.find(word=word)
tbls = self.datadb.view('tables')
for i in range(len(ids)):
hit = ids[i]
- classname = tbls[hit.tblid].name
- nodeid = str(hit.nodeid)
- property = self._getpropname(classname, hit.propid)
- rslt[i] = (classname, nodeid, property)
+ if not hit.ignore:
+ classname = tbls[hit.tblid].name
+ nodeid = str(hit.nodeid)
+ property = self._getpropname(classname, hit.propid)
+ rslt[i] = (classname, nodeid, property)
return rslt
def save_index(self):
if self.changed:
self.db.commit()
self.changed = 0
+ def rollback(self):
+ if self.changed:
+ self.db.rollback()
+ self.db = metakit.storage(self.path, 1)
+ self.changed = 0