![[tokkee]](http://tokkee.org/images/avatar.png){kind=avatar}
index 1c93af77592aa747a41b08f153f75cd3800c10d1..d78b27f9f8034f176cf7c29718d4c42938faad02 100755 (executable)
-from roundup import hyperdb, date, password, roundupdb
+from roundup import hyperdb, date, password, roundupdb, security
import metakit
+from sessions import Sessions
import re, marshal, os, sys, weakref, time, calendar
-from roundup.indexer import Indexer
+from roundup import indexer
+import locking
-_instances = weakref.WeakValueDictionary()
+_dbs = {}
def Database(config, journaltag=None):
- if _instances.has_key(id(config)):
- db = _instances[id(config)]
- old = db.journaltag
+ db = _dbs.get(config.DATABASE, None)
+ if db is None or db._db is None:
+ db = _Database(config, journaltag)
+ _dbs[config.DATABASE] = db
+ else:
db.journaltag = journaltag
try:
delattr(db, 'curuserid')
except AttributeError:
pass
- return db
- else:
- db = _Database(config, journaltag)
- _instances[id(config)] = db
- return db
+ return db
class _Database(hyperdb.Database):
def __init__(self, config, journaltag=None):
self.config = config
self.journaltag = journaltag
self.classes = {}
- self._classes = []
self.dirty = 0
- self.__RW = 0
+ self.lockfile = None
self._db = self.__open()
- self.indexer = Indexer(self.config.DATABASE)
+ self.indexer = Indexer(self.config.DATABASE, self._db)
+ self.sessions = Sessions(self.config)
+ self.security = security.Security(self)
+
os.umask(0002)
def post_init(self):
if self.indexer.should_reindex():
except KeyError:
x = 0
return x
+ elif classname == 'transactions':
+ return self.dirty
return self.getclass(classname)
def getclass(self, classname):
return self.classes[classname]
# --- exposed methods
def commit(self):
if self.dirty:
- if self.__RW:
- self._db.commit()
- for cl in self.classes.values():
- cl._commit()
- self.indexer.save_index()
- else:
- raise RuntimeError, "metakit is open RO"
+ self._db.commit()
+ for cl in self.classes.values():
+ cl._commit()
+ self.indexer.save_index()
self.dirty = 0
def rollback(self):
if self.dirty:
for cl in self.classes.values():
cl._clear()
def hasnode(self, classname, nodeid):
- return self.getclass(clasname).hasnode(nodeid)
+ return self.getclass(classname).hasnode(nodeid)
def pack(self, pack_before):
pass
def addclass(self, cl):
self.classes[cl.classname] = cl
+ if self.tables.find(name=cl.classname) < 0:
+ self.tables.append(name=cl.classname)
def addjournal(self, tablenm, nodeid, action, params):
tblid = self.tables.find(name=tablenm)
if tblid == -1:
return rslt
def close(self):
- import time
- now = time.time
- start = now()
for cl in self.classes.values():
cl.db = None
- #self._db.rollback()
- #print "pre-close cleanup of DB(%d) took %2.2f secs" % (self.__RW, now()-start)
self._db = None
- #print "close of DB(%d) took %2.2f secs" % (self.__RW, now()-start)
+ locking.release_lock(self.lockfile)
+ del _dbs[self.config.DATABASE]
+ self.lockfile.close()
self.classes = {}
- try:
- del _instances[id(self.config)]
- except KeyError:
- pass
- self.__RW = 0
-
+ self.indexer = None
+
# --- internal
def __open(self):
self.dbnm = db = os.path.join(self.config.DATABASE, 'tracker.mk4')
+ lockfilenm = db[:-3]+'lck'
+ self.lockfile = locking.acquire_lock(lockfilenm)
+ self.lockfile.write(str(os.getpid()))
+ self.lockfile.flush()
self.fastopen = 0
if os.path.exists(db):
dbtm = os.path.getmtime(db)
else:
# can't find schemamod - must be frozen
self.fastopen = 1
- else:
- self.__RW = 1
- if not self.fastopen:
- self.__RW = 1
- db = metakit.storage(db, self.__RW)
+ db = metakit.storage(db, 1)
hist = db.view('history')
tables = db.view('tables')
if not self.fastopen:
self.tables = tables
self.hist = hist
return db
- def isReadOnly(self):
- return self.__RW == 0
- def getWriteAccess(self):
- if self.journaltag is not None and self.__RW == 0:
- #now = time.time
- #start = now()
- self._db = None
- #print "closing the file took %2.2f secs" % (now()-start)
- #start = now()
- self._db = metakit.storage(self.dbnm, 1)
- self.__RW = 1
- self.hist = self._db.view('history')
- self.tables = self._db.view('tables')
- #print "getting RW access took %2.2f secs" % (now()-start)
-
+
_STRINGTYPE = type('')
_LISTTYPE = type([])
_CREATE, _SET, _RETIRE, _LINK, _UNLINK = range(5)
class Class:
privateprops = None
def __init__(self, db, classname, **properties):
- self.db = weakref.proxy(db)
+ #self.db = weakref.proxy(db)
+ self.db = db
self.classname = classname
self.keyname = None
self.ruprops = properties
self.reactors[event].append(detector)
# --- the hyperdb.Class methods
def create(self, **propvalues):
+ self.fireAuditors('create', None, propvalues)
rowdict = {}
rowdict['id'] = newid = self.maxid
self.maxid += 1
if propvalues.has_key('#ISNEW'):
isnew = 1
del propvalues['#ISNEW']
+ if not isnew:
+ self.fireAuditors('set', nodeid, propvalues)
if not propvalues:
- return
+ return propvalues
if propvalues.has_key('id'):
raise KeyError, '"id" is reserved'
if self.db.journaltag is None:
# do stuff based on the prop type
if isinstance(prop, hyperdb.Link):
link_class = prop.classname
+ # must be a string or None
+ if value is not None and not isinstance(value, type('')):
+ raise ValueError, 'property "%s" link value be a string'%(
+ propname)
+ # Roundup sets to "unselected" by passing None
+ if value is None:
+ value = 0
# if it isn't a number, it's a key
- if type(value) != _STRINGTYPE:
- raise ValueError, 'link value must be String'
try:
int(value)
except ValueError:
raise IndexError, 'new property "%s": %s not a %s'%(
key, value, prop.classname)
- if not self.db.getclass(link_class).hasnode(value):
+ if (value is not None and
+ not self.db.getclass(link_class).hasnode(value)):
raise IndexError, '%s has no node %s'%(link_class, value)
setattr(row, key, int(value))
if self.do_journal and prop.do_journal:
# register the unlink with the old linked node
if oldvalue:
- self.db.addjournal(link_class, value, _UNLINK, (self.classname, str(row.id), key))
+ self.db.addjournal(link_class, value, _UNLINK,
+ (self.classname, str(row.id), key))
# register the link with the newly linked node
if value:
- self.db.addjournal(link_class, value, _LINK, (self.classname, str(row.id), key))
+ self.db.addjournal(link_class, value, _LINK,
+ (self.classname, str(row.id), key))
elif isinstance(prop, hyperdb.Multilink):
if type(value) != _LISTTYPE:
for id in adds:
sv.append(fid=int(id))
changes[key] = oldvalue
+ if not rmvd and not adds:
+ del propvalues[key]
elif isinstance(prop, hyperdb.String):
setattr(row, key, str(value))
changes[key] = str(oldvalue)
propvalues[key] = str(value)
+
+ elif value is not None and isinstance(prop, hyperdb.Number):
+ setattr(row, key, int(value))
+ changes[key] = oldvalue
+ propvalues[key] = value
+
+ elif value is not None and isinstance(prop, hyperdb.Boolean):
+ bv = value != 0
+ setattr(row, key, bv)
+ changes[key] = oldvalue
+ propvalues[key] = value
oldnode[key] = oldvalue
# nothing to do?
if not propvalues:
- return
+ return propvalues
if not propvalues.has_key('activity'):
row.activity = int(time.time())
if isnew:
if self.do_journal:
if isnew:
self.db.addjournal(self.classname, nodeid, _CREATE, {})
+ self.fireReactors('create', nodeid, None)
else:
self.db.addjournal(self.classname, nodeid, _SET, changes)
+ self.fireReactors('set', nodeid, oldnode)
+ return propvalues
+
def retire(self, nodeid):
+ self.fireAuditors('retire', nodeid, None)
view = self.getview(1)
ndx = view.find(id=int(nodeid))
if ndx < 0:
row._isdel = 1
if self.do_journal:
self.db.addjournal(self.classname, nodeid, _RETIRE, {})
- iv = self.getindexview(1)
- ndx = iv.find(k=getattr(row, self.keyname),i=row.id)
- if ndx > -1:
- iv.delete(ndx)
+ if self.keyname:
+ iv = self.getindexview(1)
+ ndx = iv.find(k=getattr(row, self.keyname),i=row.id)
+ if ndx > -1:
+ iv.delete(ndx)
self.db.dirty = 1
+ self.fireReactors('retire', nodeid, None)
def history(self, nodeid):
if not self.do_journal:
raise ValueError, 'Journalling is disabled for this class'
if self.db.fastopen and iv.structure():
return
# very first setkey ever
- self.db.getWriteAccess()
self.db.dirty = 1
iv = self.db._db.getas('_%s[k:S,i:I]' % self.classname)
iv = iv.ordered(1)
- #XXX
# print "setkey building index"
for row in self.getview():
iv.append(k=getattr(row, propname), i=row.id)
if ndx > -1:
return str(view[ndx].id)
raise KeyError, keyvalue
+
+ def destroy(self, keyvalue):
+ #TODO clean this up once Richard's said how it should work
+ iv = self.getindexview()
+ if iv:
+ ndx = iv.find(k=keyvalue)
+ if ndx > -1:
+ id = iv[ndx].i
+ iv.delete(ndx)
+ view = self.getview()
+ ndx = view.find(id=id)
+ if ndx > -1:
+ view.delete(ndx)
+
def find(self, **propspec):
"""Get the ids of nodes in this class which link to the given nodes.
if self.ruprops.has_key(key):
raise ValueError, "%s is already a property of %s" % (key, self.classname)
self.ruprops.update(properties)
- self.db.getWriteAccess()
self.db.fastopen = 0
view = self.__getview()
self.db.commit()
regexes[propname] = re.compile(v, re.I)
elif propname == 'id':
where[propname] = int(value)
+ elif isinstance(prop, hyperdb.Boolean):
+ if type(value) is _STRINGTYPE:
+ bv = value.lower() in ('yes', 'true', 'on', '1')
+ else:
+ bv = value
+ where[propname] = bv
+ elif isinstance(prop, hyperdb.Number):
+ where[propname] = int(value)
else:
where[propname] = str(value)
v = self.getview()
try:
prop = getattr(v, propname)
except AttributeError:
+ print "MK has no property %s" % propname
continue
+ propclass = self.ruprops.get(propname, None)
+ if propclass is None:
+ propclass = self.privateprops.get(propname, None)
+ if propclass is None:
+ print "Schema has no property %s" % propname
+ continue
+ if isinstance(propclass, hyperdb.Link):
+ linkclass = self.db.getclass(propclass.classname)
+ lv = linkclass.getview()
+ lv = lv.rename('id', propname)
+ v = v.join(lv, prop, 1)
+ if linkclass.getprops().has_key('order'):
+ propname = 'order'
+ else:
+ propname = linkclass.labelprop()
+ prop = getattr(v, propname)
if isreversed:
rev.append(prop)
sortspec.append(prop)
else:
mkprop = None
if mkprop is None:
- print "%s missing prop %s (%s)" % (self.classname, nm, rutyp.__class__.__name__)
break
if _typmap[rutyp.__class__] != mkprop.type:
- print "%s - prop %s (%s) has wrong mktyp (%s)" % (self.classname, nm, rutyp.__class__.__name__, mkprop.type)
break
else:
return view.ordered(1)
# need to create or restructure the mk view
# id comes first, so MK will order it for us
- self.db.getWriteAccess()
self.db.dirty = 1
s = ["%s[id:I" % self.classname]
for nm, rutyp in self.ruprops.items():
self.db.commit()
return v.ordered(1)
def getview(self, RW=0):
- if RW and self.db.isReadOnly():
- self.db.getWriteAccess()
return self.db._db.view(self.classname).ordered(1)
def getindexview(self, RW=0):
- if RW and self.db.isReadOnly():
- self.db.getWriteAccess()
return self.db._db.view("_%s" % self.classname).ordered(1)
def _fetchML(sv):
hyperdb.Multilink : _fetchML,
hyperdb.Interval : date.Interval,
hyperdb.Password : _fetchPW,
+ hyperdb.Boolean : lambda n: n,
+ hyperdb.Number : lambda n: n,
+ hyperdb.String : str,
}
class FileName(hyperdb.String):
hyperdb.Multilink : 'V',
hyperdb.Interval : 'S',
hyperdb.Password : 'S',
+ hyperdb.Boolean : 'I',
+ hyperdb.Number : 'I',
}
class FileClass(Class):
' like Class but with a content property '
newid = Class.create(self, **propvalues)
if not content:
return newid
- if content.startswith('/tracker/download.php?'):
- self.set(newid, content='http://sourceforge.net'+content)
- return newid
nm = bnm = '%s%s' % (self.classname, newid)
sd = str(int(int(newid) / 1000))
d = os.path.join(self.db.config.DATABASE, 'files', self.classname, sd)
mimetype = propvalues.get('type', self.default_mime_type)
self.db.indexer.add_text((self.classname, newid, 'content'), content, mimetype)
def undo(fnm=nm, action1=os.remove, indexer=self.db.indexer):
- remove(fnm)
+ action1(fnm)
self.rollbackaction(undo)
return newid
def index(self, nodeid):
if not properties.has_key('superseder'):
properties['superseder'] = hyperdb.Multilink(classname)
Class.__init__(self, db, classname, **properties)
+
+CURVERSION = 1
+class Indexer(indexer.Indexer):
+ disallows = {'THE':1, 'THIS':1, 'ZZZ':1, 'THAT':1, 'WITH':1}
+ def __init__(self, path, datadb):
+ self.db = metakit.storage(os.path.join(path, 'index.mk4'), 1)
+ self.datadb = datadb
+ self.reindex = 0
+ v = self.db.view('version')
+ if not v.structure():
+ v = self.db.getas('version[vers:I]')
+ self.db.commit()
+ v.append(vers=CURVERSION)
+ self.reindex = 1
+ elif v[0].vers != CURVERSION:
+ v[0].vers = CURVERSION
+ self.reindex = 1
+ if self.reindex:
+ self.db.getas('ids[tblid:I,nodeid:I,propid:I]')
+ self.db.getas('index[word:S,hits[pos:I]]')
+ self.db.commit()
+ self.reindex = 1
+ self.changed = 0
+ self.propcache = {}
+ def force_reindex(self):
+ v = self.db.view('ids')
+ v[:] = []
+ v = self.db.view('index')
+ v[:] = []
+ self.db.commit()
+ self.reindex = 1
+ def should_reindex(self):
+ return self.reindex
+ def _getprops(self, classname):
+ props = self.propcache.get(classname, None)
+ if props is None:
+ props = self.datadb.view(classname).structure()
+ props = [prop.name for prop in props]
+ self.propcache[classname] = props
+ return props
+ def _getpropid(self, classname, propname):
+ return self._getprops(classname).index(propname)
+ def _getpropname(self, classname, propid):
+ return self._getprops(classname)[propid]
+ def add_text(self, identifier, text, mime_type='text/plain'):
+ if mime_type != 'text/plain':
+ return
+ classname, nodeid, property = identifier
+ tbls = self.datadb.view('tables')
+ tblid = tbls.find(name=classname)
+ if tblid < 0:
+ raise KeyError, "unknown class %r"%classname
+ nodeid = int(nodeid)
+ propid = self._getpropid(classname, property)
+ pos = self.db.view('ids').append(tblid=tblid,nodeid=nodeid,propid=propid)
+
+ wordlist = re.findall(r'\b\w{3,25}\b', text)
+ words = {}
+ for word in wordlist:
+ word = word.upper()
+ if not self.disallows.has_key(word):
+ words[word] = 1
+ words = words.keys()
+
+ index = self.db.view('index').ordered(1)
+ for word in words:
+ ndx = index.find(word=word)
+ if ndx < 0:
+ ndx = index.append(word=word)
+ hits = index[ndx].hits
+ if len(hits)==0 or hits.find(pos=pos) < 0:
+ hits.append(pos=pos)
+ self.changed = 1
+ def find(self, wordlist):
+ hits = None
+ index = self.db.view('index').ordered(1)
+ for word in wordlist:
+ if not 2 < len(word) < 26:
+ continue
+ ndx = index.find(word=word)
+ if ndx < 0:
+ return {}
+ if hits is None:
+ hits = index[ndx].hits
+ else:
+ hits = hits.intersect(index[ndx].hits)
+ if len(hits) == 0:
+ return {}
+ if hits is None:
+ return {}
+ rslt = {}
+ ids = self.db.view('ids').remapwith(hits)
+ tbls = self.datadb.view('tables')
+ for i in range(len(ids)):
+ hit = ids[i]
+ classname = tbls[hit.tblid].name
+ nodeid = str(hit.nodeid)
+ property = self._getpropname(classname, hit.propid)
+ rslt[i] = (classname, nodeid, property)
+ return rslt
+ def save_index(self):
+ if self.changed:
+ self.db.commit()
+ self.changed = 0