![[tokkee]](http://tokkee.org/images/avatar.png){kind=avatar}
index 9d8d53f20badacd88c53a1b1406e7243cb0c9555..d78b27f9f8034f176cf7c29718d4c42938faad02 100755 (executable)
-from roundup import hyperdb, date, password, roundupdb
+from roundup import hyperdb, date, password, roundupdb, security
import metakit
import metakit
+from sessions import Sessions
import re, marshal, os, sys, weakref, time, calendar
import re, marshal, os, sys, weakref, time, calendar
-from roundup.indexer import Indexer
+from roundup import indexer
+import locking
-_instances = weakref.WeakValueDictionary()
+_dbs = {}
def Database(config, journaltag=None):
def Database(config, journaltag=None):
- if _instances.has_key(id(config)):
- db = _instances[id(config)]
- old = db.journaltag
+ db = _dbs.get(config.DATABASE, None)
+ if db is None or db._db is None:
+ db = _Database(config, journaltag)
+ _dbs[config.DATABASE] = db
+ else:
db.journaltag = journaltag
db.journaltag = journaltag
- if hasattr(db, 'curuserid'):
+ try:
delattr(db, 'curuserid')
delattr(db, 'curuserid')
- return db
- else:
- db = _Database(config, journaltag)
- _instances[id(config)] = db
- return db
+ except AttributeError:
+ pass
+ return db
class _Database(hyperdb.Database):
def __init__(self, config, journaltag=None):
self.config = config
self.journaltag = journaltag
self.classes = {}
class _Database(hyperdb.Database):
def __init__(self, config, journaltag=None):
self.config = config
self.journaltag = journaltag
self.classes = {}
- self._classes = []
self.dirty = 0
self.dirty = 0
- self.__RW = 0
+ self.lockfile = None
self._db = self.__open()
self._db = self.__open()
- self.indexer = Indexer(self.config.DATABASE)
+ self.indexer = Indexer(self.config.DATABASE, self._db)
+ self.sessions = Sessions(self.config)
+ self.security = security.Security(self)
+
os.umask(0002)
os.umask(0002)
+ def post_init(self):
+ if self.indexer.should_reindex():
+ self.reindex()
+
+ def reindex(self):
+ for klass in self.classes.values():
+ for nodeid in klass.list():
+ klass.index(nodeid)
+ self.indexer.save_index()
+
# --- defined in ping's spec
def __getattr__(self, classname):
# --- defined in ping's spec
def __getattr__(self, classname):
except KeyError:
x = 0
return x
except KeyError:
x = 0
return x
+ elif classname == 'transactions':
+ return self.dirty
return self.getclass(classname)
def getclass(self, classname):
return self.classes[classname]
return self.getclass(classname)
def getclass(self, classname):
return self.classes[classname]
# --- exposed methods
def commit(self):
if self.dirty:
# --- exposed methods
def commit(self):
if self.dirty:
- if self.__RW:
- self._db.commit()
- for cl in self.classes.values():
- cl._commit()
- else:
- raise RuntimeError, "metakit is open RO"
+ self._db.commit()
+ for cl in self.classes.values():
+ cl._commit()
+ self.indexer.save_index()
self.dirty = 0
def rollback(self):
if self.dirty:
self.dirty = 0
def rollback(self):
if self.dirty:
for cl in self.classes.values():
cl._clear()
def hasnode(self, classname, nodeid):
for cl in self.classes.values():
cl._clear()
def hasnode(self, classname, nodeid):
- return self.getclass(clasname).hasnode(nodeid)
+ return self.getclass(classname).hasnode(nodeid)
def pack(self, pack_before):
pass
def addclass(self, cl):
def pack(self, pack_before):
pass
def addclass(self, cl):
- self.classes[cl.name] = cl
+ self.classes[cl.classname] = cl
+ if self.tables.find(name=cl.classname) < 0:
+ self.tables.append(name=cl.classname)
def addjournal(self, tablenm, nodeid, action, params):
tblid = self.tables.find(name=tablenm)
if tblid == -1:
def addjournal(self, tablenm, nodeid, action, params):
tblid = self.tables.find(name=tablenm)
if tblid == -1:
return rslt
def close(self):
return rslt
def close(self):
- import time
- now = time.time
- start = now()
for cl in self.classes.values():
cl.db = None
for cl in self.classes.values():
cl.db = None
- #self._db.rollback()
- #print "pre-close cleanup of DB(%d) took %2.2f secs" % (self.__RW, now()-start)
self._db = None
self._db = None
- #print "close of DB(%d) took %2.2f secs" % (self.__RW, now()-start)
+ locking.release_lock(self.lockfile)
+ del _dbs[self.config.DATABASE]
+ self.lockfile.close()
self.classes = {}
self.classes = {}
- try:
- del _instances[id(self.config)]
- except KeyError:
- pass
- self.__RW = 0
-
+ self.indexer = None
+
# --- internal
def __open(self):
self.dbnm = db = os.path.join(self.config.DATABASE, 'tracker.mk4')
# --- internal
def __open(self):
self.dbnm = db = os.path.join(self.config.DATABASE, 'tracker.mk4')
+ lockfilenm = db[:-3]+'lck'
+ self.lockfile = locking.acquire_lock(lockfilenm)
+ self.lockfile.write(str(os.getpid()))
+ self.lockfile.flush()
self.fastopen = 0
if os.path.exists(db):
dbtm = os.path.getmtime(db)
self.fastopen = 0
if os.path.exists(db):
dbtm = os.path.getmtime(db)
else:
# can't find schemamod - must be frozen
self.fastopen = 1
else:
# can't find schemamod - must be frozen
self.fastopen = 1
- else:
- self.__RW = 1
- db = metakit.storage(db, self.__RW)
+ db = metakit.storage(db, 1)
hist = db.view('history')
tables = db.view('tables')
if not self.fastopen:
hist = db.view('history')
tables = db.view('tables')
if not self.fastopen:
self.tables = tables
self.hist = hist
return db
self.tables = tables
self.hist = hist
return db
- def isReadOnly(self):
- return self.__RW == 0
- def getWriteAccess(self):
- if self.journaltag is not None and self.__RW == 0:
- now = time.time
- start = now()
- self._db = None
- #print "closing the file took %2.2f secs" % (now()-start)
- start = now()
- self._db = metakit.storage(self.dbnm, 1)
- self.__RW = 1
- self.hist = self._db.view('history')
- self.tables = self._db.view('tables')
- #print "getting RW access took %2.2f secs" % (now()-start)
-
+
_STRINGTYPE = type('')
_LISTTYPE = type([])
_CREATE, _SET, _RETIRE, _LINK, _UNLINK = range(5)
_STRINGTYPE = type('')
_LISTTYPE = type([])
_CREATE, _SET, _RETIRE, _LINK, _UNLINK = range(5)
_ALLOWSETTINGPRIVATEPROPS = 0
_ALLOWSETTINGPRIVATEPROPS = 0
-class Class: # no, I'm not going to subclass the existing!
+class Class:
privateprops = None
def __init__(self, db, classname, **properties):
privateprops = None
def __init__(self, db, classname, **properties):
- self.db = weakref.proxy(db)
- self.name = classname
+ #self.db = weakref.proxy(db)
+ self.db = db
+ self.classname = classname
self.keyname = None
self.ruprops = properties
self.privateprops = { 'id' : hyperdb.String(),
self.keyname = None
self.ruprops = properties
self.privateprops = { 'id' : hyperdb.String(),
self.properties = self.ruprops
self.db.addclass(self)
self.idcache = {}
self.properties = self.ruprops
self.db.addclass(self)
self.idcache = {}
+
+ # default is to journal changes
+ self.do_journal = 1
+
+ def enableJournalling(self):
+ '''Turn journalling on for this class
+ '''
+ self.do_journal = 1
+
+ def disableJournalling(self):
+ '''Turn journalling off for this class
+ '''
+ self.do_journal = 0
# --- the roundup.Class methods
def audit(self, event, detector):
# --- the roundup.Class methods
def audit(self, event, detector):
self.reactors[event].append(detector)
# --- the hyperdb.Class methods
def create(self, **propvalues):
self.reactors[event].append(detector)
# --- the hyperdb.Class methods
def create(self, **propvalues):
+ self.fireAuditors('create', None, propvalues)
rowdict = {}
rowdict['id'] = newid = self.maxid
self.maxid += 1
rowdict = {}
rowdict['id'] = newid = self.maxid
self.maxid += 1
if ndx is None:
ndx = view.find(id=id)
if ndx < 0:
if ndx is None:
ndx = view.find(id=id)
if ndx < 0:
- raise IndexError, "%s has no node %s" % (self.name, nodeid)
+ raise IndexError, "%s has no node %s" % (self.classname, nodeid)
self.idcache[id] = ndx
self.idcache[id] = ndx
- raw = getattr(view[ndx], propname)
+ try:
+ raw = getattr(view[ndx], propname)
+ except AttributeError:
+ raise KeyError, propname
rutyp = self.ruprops.get(propname, None)
if rutyp is None:
rutyp = self.privateprops[propname]
rutyp = self.ruprops.get(propname, None)
if rutyp is None:
rutyp = self.privateprops[propname]
return raw
def set(self, nodeid, **propvalues):
return raw
def set(self, nodeid, **propvalues):
-
isnew = 0
if propvalues.has_key('#ISNEW'):
isnew = 1
del propvalues['#ISNEW']
isnew = 0
if propvalues.has_key('#ISNEW'):
isnew = 1
del propvalues['#ISNEW']
+ if not isnew:
+ self.fireAuditors('set', nodeid, propvalues)
if not propvalues:
if not propvalues:
- return
+ return propvalues
if propvalues.has_key('id'):
raise KeyError, '"id" is reserved'
if self.db.journaltag is None:
if propvalues.has_key('id'):
raise KeyError, '"id" is reserved'
if self.db.journaltag is None:
id = int(nodeid)
ndx = view.find(id=id)
if ndx < 0:
id = int(nodeid)
ndx = view.find(id=id)
if ndx < 0:
- raise IndexError, "%s has no node %s" % (self.name, nodeid)
+ raise IndexError, "%s has no node %s" % (self.classname, nodeid)
row = view[ndx]
if row._isdel:
row = view[ndx]
if row._isdel:
- raise IndexError, "%s has no node %s" % (self.name, nodeid)
+ raise IndexError, "%s has no node %s" % (self.classname, nodeid)
oldnode = self.uncommitted.setdefault(id, {})
changes = {}
oldnode = self.uncommitted.setdefault(id, {})
changes = {}
# do stuff based on the prop type
if isinstance(prop, hyperdb.Link):
link_class = prop.classname
# do stuff based on the prop type
if isinstance(prop, hyperdb.Link):
link_class = prop.classname
+ # must be a string or None
+ if value is not None and not isinstance(value, type('')):
+ raise ValueError, 'property "%s" link value be a string'%(
+ propname)
+ # Roundup sets to "unselected" by passing None
+ if value is None:
+ value = 0
# if it isn't a number, it's a key
# if it isn't a number, it's a key
- if type(value) != _STRINGTYPE:
- raise ValueError, 'link value must be String'
try:
int(value)
except ValueError:
try:
int(value)
except ValueError:
raise IndexError, 'new property "%s": %s not a %s'%(
key, value, prop.classname)
raise IndexError, 'new property "%s": %s not a %s'%(
key, value, prop.classname)
- if not self.db.getclass(link_class).hasnode(value):
+ if (value is not None and
+ not self.db.getclass(link_class).hasnode(value)):
raise IndexError, '%s has no node %s'%(link_class, value)
setattr(row, key, int(value))
changes[key] = oldvalue
raise IndexError, '%s has no node %s'%(link_class, value)
setattr(row, key, int(value))
changes[key] = oldvalue
- if prop.do_journal:
+ if self.do_journal and prop.do_journal:
# register the unlink with the old linked node
if oldvalue:
# register the unlink with the old linked node
if oldvalue:
- self.db.addjournal(link_class, value, _UNLINK, (self.name, str(row.id), key))
+ self.db.addjournal(link_class, value, _UNLINK,
+ (self.classname, str(row.id), key))
# register the link with the newly linked node
if value:
# register the link with the newly linked node
if value:
- self.db.addjournal(link_class, value, _LINK, (self.name, str(row.id), key))
+ self.db.addjournal(link_class, value, _LINK,
+ (self.classname, str(row.id), key))
elif isinstance(prop, hyperdb.Multilink):
if type(value) != _LISTTYPE:
elif isinstance(prop, hyperdb.Multilink):
if type(value) != _LISTTYPE:
if id not in value:
rmvd.append(id)
# register the unlink with the old linked node
if id not in value:
rmvd.append(id)
# register the unlink with the old linked node
- if prop.do_journal:
- self.db.addjournal(link_class, id, _UNLINK, (self.name, str(row.id), key))
+ if self.do_journal and prop.do_journal:
+ self.db.addjournal(link_class, id, _UNLINK, (self.classname, str(row.id), key))
# handle additions
adds = []
# handle additions
adds = []
link_class, id)
adds.append(id)
# register the link with the newly linked node
link_class, id)
adds.append(id)
# register the link with the newly linked node
- if prop.do_journal:
- self.db.addjournal(link_class, id, _LINK, (self.name, str(row.id), key))
+ if self.do_journal and prop.do_journal:
+ self.db.addjournal(link_class, id, _LINK, (self.classname, str(row.id), key))
sv = getattr(row, key)
i = 0
sv = getattr(row, key)
i = 0
for id in adds:
sv.append(fid=int(id))
changes[key] = oldvalue
for id in adds:
sv.append(fid=int(id))
changes[key] = oldvalue
+ if not rmvd and not adds:
+ del propvalues[key]
elif isinstance(prop, hyperdb.String):
elif isinstance(prop, hyperdb.String):
changes[key] = oldvalue
if hasattr(prop, 'isfilename') and prop.isfilename:
propvalues[key] = os.path.basename(value)
changes[key] = oldvalue
if hasattr(prop, 'isfilename') and prop.isfilename:
propvalues[key] = os.path.basename(value)
+ if prop.indexme:
+ self.db.indexer.add_text((self.classname, nodeid, key), value, 'text/plain')
elif isinstance(prop, hyperdb.Password):
if not isinstance(value, password.Password):
elif isinstance(prop, hyperdb.Password):
if not isinstance(value, password.Password):
setattr(row, key, str(value))
changes[key] = str(oldvalue)
propvalues[key] = str(value)
setattr(row, key, str(value))
changes[key] = str(oldvalue)
propvalues[key] = str(value)
+
+ elif value is not None and isinstance(prop, hyperdb.Number):
+ setattr(row, key, int(value))
+ changes[key] = oldvalue
+ propvalues[key] = value
+
+ elif value is not None and isinstance(prop, hyperdb.Boolean):
+ bv = value != 0
+ setattr(row, key, bv)
+ changes[key] = oldvalue
+ propvalues[key] = value
oldnode[key] = oldvalue
# nothing to do?
if not propvalues:
oldnode[key] = oldvalue
# nothing to do?
if not propvalues:
- return
- if not row.activity:
+ return propvalues
+ if not propvalues.has_key('activity'):
row.activity = int(time.time())
if isnew:
if not row.creation:
row.activity = int(time.time())
if isnew:
if not row.creation:
row.creator = self.db.curuserid
self.db.dirty = 1
row.creator = self.db.curuserid
self.db.dirty = 1
- if isnew:
- self.db.addjournal(self.name, nodeid, _CREATE, {})
- else:
- self.db.addjournal(self.name, nodeid, _SET, changes)
+ if self.do_journal:
+ if isnew:
+ self.db.addjournal(self.classname, nodeid, _CREATE, {})
+ self.fireReactors('create', nodeid, None)
+ else:
+ self.db.addjournal(self.classname, nodeid, _SET, changes)
+ self.fireReactors('set', nodeid, oldnode)
+ return propvalues
+
def retire(self, nodeid):
def retire(self, nodeid):
+ self.fireAuditors('retire', nodeid, None)
view = self.getview(1)
ndx = view.find(id=int(nodeid))
if ndx < 0:
view = self.getview(1)
ndx = view.find(id=int(nodeid))
if ndx < 0:
oldvalues = self.uncommitted.setdefault(row.id, {})
oldval = oldvalues['_isdel'] = row._isdel
row._isdel = 1
oldvalues = self.uncommitted.setdefault(row.id, {})
oldval = oldvalues['_isdel'] = row._isdel
row._isdel = 1
- self.db.addjournal(self.name, nodeid, _RETIRE, {})
- iv = self.getindexview(1)
- ndx = iv.find(k=getattr(row, self.keyname),i=row.id)
- if ndx > -1:
- iv.delete(ndx)
+ if self.do_journal:
+ self.db.addjournal(self.classname, nodeid, _RETIRE, {})
+ if self.keyname:
+ iv = self.getindexview(1)
+ ndx = iv.find(k=getattr(row, self.keyname),i=row.id)
+ if ndx > -1:
+ iv.delete(ndx)
self.db.dirty = 1
self.db.dirty = 1
+ self.fireReactors('retire', nodeid, None)
def history(self, nodeid):
def history(self, nodeid):
- return self.db.gethistory(self.name, nodeid)
+ if not self.do_journal:
+ raise ValueError, 'Journalling is disabled for this class'
+ return self.db.gethistory(self.classname, nodeid)
def setkey(self, propname):
if self.keyname:
if propname == self.keyname:
return
def setkey(self, propname):
if self.keyname:
if propname == self.keyname:
return
- raise ValueError, "%s already indexed on %s" % (self.name, self.keyname)
+ raise ValueError, "%s already indexed on %s" % (self.classname, self.keyname)
# first setkey for this run
self.keyname = propname
# first setkey for this run
self.keyname = propname
- iv = self.db._db.view('_%s' % self.name)
- if self.db.fastopen or iv.structure():
+ iv = self.db._db.view('_%s' % self.classname)
+ if self.db.fastopen and iv.structure():
return
# very first setkey ever
return
# very first setkey ever
- iv = self.db._db.getas('_%s[k:S,i:I]' % self.name)
+ self.db.dirty = 1
+ iv = self.db._db.getas('_%s[k:S,i:I]' % self.classname)
iv = iv.ordered(1)
iv = iv.ordered(1)
- #XXX
- print "setkey building index"
+# print "setkey building index"
for row in self.getview():
iv.append(k=getattr(row, propname), i=row.id)
for row in self.getview():
iv.append(k=getattr(row, propname), i=row.id)
+ self.db.commit()
def getkey(self):
return self.keyname
def lookup(self, keyvalue):
def getkey(self):
return self.keyname
def lookup(self, keyvalue):
if ndx > -1:
return str(view[ndx].id)
raise KeyError, keyvalue
if ndx > -1:
return str(view[ndx].id)
raise KeyError, keyvalue
+
+ def destroy(self, keyvalue):
+ #TODO clean this up once Richard's said how it should work
+ iv = self.getindexview()
+ if iv:
+ ndx = iv.find(k=keyvalue)
+ if ndx > -1:
+ id = iv[ndx].i
+ iv.delete(ndx)
+ view = self.getview()
+ ndx = view.find(id=id)
+ if ndx > -1:
+ view.delete(ndx)
+
def find(self, **propspec):
"""Get the ids of nodes in this class which link to the given nodes.
'propspec' consists of keyword args propname={nodeid:1,}
def find(self, **propspec):
"""Get the ids of nodes in this class which link to the given nodes.
'propspec' consists of keyword args propname={nodeid:1,}
- 'propname' must be the name of a property in this class, or a
- KeyError is raised. That property must be a Link or Multilink
- property, or a TypeError is raised.
+ 'propname' must be the name of a property in this class, or a
+ KeyError is raised. That property must be a Link or
+ Multilink property, or a TypeError is raised.
+
Any node in this class whose propname property links to any of the
nodeids will be returned. Used by the full text indexing, which knows
Any node in this class whose propname property links to any of the
nodeids will be returned. Used by the full text indexing, which knows
- that "foo" occurs in msg1, msg3 and file7; so we have hits on these issues:
+ that "foo" occurs in msg1, msg3 and file7; so we have hits on these
+ issues:
+
db.issue.find(messages={'1':1,'3':1}, files={'7':1})
db.issue.find(messages={'1':1,'3':1}, files={'7':1})
+
"""
propspec = propspec.items()
for propname, nodeid in propspec:
# check the prop is OK
prop = self.ruprops[propname]
"""
propspec = propspec.items()
for propname, nodeid in propspec:
# check the prop is OK
prop = self.ruprops[propname]
- if not isinstance(prop, hyperdb.Link) and not isinstance(prop, hyperdb.Multilink):
+ if (not isinstance(prop, hyperdb.Link) and
+ not isinstance(prop, hyperdb.Multilink)):
raise TypeError, "'%s' not a Link/Multilink property"%propname
vws = []
for propname, ids in propspec:
raise TypeError, "'%s' not a Link/Multilink property"%propname
vws = []
for propname, ids in propspec:
+ if type(ids) is _STRINGTYPE:
+ ids = {ids:1}
prop = self.ruprops[propname]
view = self.getview()
if isinstance(prop, hyperdb.Multilink):
prop = self.ruprops[propname]
view = self.getview()
if isinstance(prop, hyperdb.Multilink):
return ids.has_key(str(getattr(row, nm)))
ndxview = view.filter(ff)
vws.append(ndxview.unique())
return ids.has_key(str(getattr(row, nm)))
ndxview = view.filter(ff)
vws.append(ndxview.unique())
+
+ # handle the empty match case
+ if not vws:
+ return []
+
ndxview = vws[0]
for v in vws[1:]:
ndxview = ndxview.union(v)
ndxview = vws[0]
for v in vws[1:]:
ndxview = ndxview.union(v)
def addprop(self, **properties):
for key in properties.keys():
if self.ruprops.has_key(key):
def addprop(self, **properties):
for key in properties.keys():
if self.ruprops.has_key(key):
- raise ValueError, "%s is already a property of %s" % (key, self.name)
+ raise ValueError, "%s is already a property of %s" % (key, self.classname)
self.ruprops.update(properties)
self.ruprops.update(properties)
+ self.db.fastopen = 0
view = self.__getview()
view = self.__getview()
+ self.db.commit()
# ---- end of ping's spec
def filter(self, search_matches, filterspec, sort, group):
# search_matches is None or a set (dict of {nodeid: {propname:[nodeid,...]}})
# ---- end of ping's spec
def filter(self, search_matches, filterspec, sort, group):
# search_matches is None or a set (dict of {nodeid: {propname:[nodeid,...]}})
regexes[propname] = re.compile(v, re.I)
elif propname == 'id':
where[propname] = int(value)
regexes[propname] = re.compile(v, re.I)
elif propname == 'id':
where[propname] = int(value)
+ elif isinstance(prop, hyperdb.Boolean):
+ if type(value) is _STRINGTYPE:
+ bv = value.lower() in ('yes', 'true', 'on', '1')
+ else:
+ bv = value
+ where[propname] = bv
+ elif isinstance(prop, hyperdb.Number):
+ where[propname] = int(value)
else:
where[propname] = str(value)
v = self.getview()
else:
where[propname] = str(value)
v = self.getview()
try:
prop = getattr(v, propname)
except AttributeError:
try:
prop = getattr(v, propname)
except AttributeError:
- # I can't sort on 'activity', cause it's psuedo!!
+ print "MK has no property %s" % propname
continue
continue
+ propclass = self.ruprops.get(propname, None)
+ if propclass is None:
+ propclass = self.privateprops.get(propname, None)
+ if propclass is None:
+ print "Schema has no property %s" % propname
+ continue
+ if isinstance(propclass, hyperdb.Link):
+ linkclass = self.db.getclass(propclass.classname)
+ lv = linkclass.getview()
+ lv = lv.rename('id', propname)
+ v = v.join(lv, prop, 1)
+ if linkclass.getprops().has_key('order'):
+ propname = 'order'
+ else:
+ propname = linkclass.labelprop()
+ prop = getattr(v, propname)
if isreversed:
rev.append(prop)
sortspec.append(prop)
if isreversed:
rev.append(prop)
sortspec.append(prop)
- v = v.sortrev(sortspec, rev)[:] #XXX Aaaabh
+ v = v.sortrev(sortspec, rev)[:] #XXX Metakit bug
#print "filter sort at %s" % time.time()
rslt = []
#print "filter sort at %s" % time.time()
rslt = []
return l
def addjournal(self, nodeid, action, params):
return l
def addjournal(self, nodeid, action, params):
- self.db.addjournal(self.name, nodeid, action, params)
+ self.db.addjournal(self.classname, nodeid, action, params)
+
+ def index(self, nodeid):
+ ''' Add (or refresh) the node to search indexes '''
+ # find all the String properties that have indexme
+ for prop, propclass in self.getprops().items():
+ if isinstance(propclass, hyperdb.String) and propclass.indexme:
+ # index them under (classname, nodeid, property)
+ self.db.indexer.add_text((self.classname, nodeid, prop),
+ str(self.get(nodeid, prop)))
+
# --- used by Database
def _commit(self):
""" called post commit of the DB.
# --- used by Database
def _commit(self):
""" called post commit of the DB.
# --- internal
def __getview(self):
db = self.db._db
# --- internal
def __getview(self):
db = self.db._db
- view = db.view(self.name)
- if self.db.fastopen:
+ view = db.view(self.classname)
+ mkprops = view.structure()
+ if mkprops and self.db.fastopen:
return view.ordered(1)
# is the definition the same?
for nm, rutyp in self.ruprops.items():
return view.ordered(1)
# is the definition the same?
for nm, rutyp in self.ruprops.items():
- mkprop = getattr(view, nm, None)
+ for mkprop in mkprops:
+ if mkprop.name == nm:
+ break
+ else:
+ mkprop = None
if mkprop is None:
if mkprop is None:
- #print "%s missing prop %s (%s)" % (self.name, nm, rutyp.__class__.__name__)
break
if _typmap[rutyp.__class__] != mkprop.type:
break
if _typmap[rutyp.__class__] != mkprop.type:
- #print "%s - prop %s (%s) has wrong mktyp (%s)" % (self.name, nm, rutyp.__class__.__name__, mkprop.type)
break
else:
return view.ordered(1)
# need to create or restructure the mk view
# id comes first, so MK will order it for us
self.db.dirty = 1
break
else:
return view.ordered(1)
# need to create or restructure the mk view
# id comes first, so MK will order it for us
self.db.dirty = 1
- s = ["%s[id:I" % self.name]
+ s = ["%s[id:I" % self.classname]
for nm, rutyp in self.ruprops.items():
mktyp = _typmap[rutyp.__class__]
s.append('%s:%s' % (nm, mktyp))
if mktyp == 'V':
s[-1] += ('[fid:I]')
s.append('_isdel:I,activity:I,creation:I,creator:I]')
for nm, rutyp in self.ruprops.items():
mktyp = _typmap[rutyp.__class__]
s.append('%s:%s' % (nm, mktyp))
if mktyp == 'V':
s[-1] += ('[fid:I]')
s.append('_isdel:I,activity:I,creation:I,creator:I]')
- v = db.getas(','.join(s))
+ v = self.db._db.getas(','.join(s))
+ self.db.commit()
return v.ordered(1)
def getview(self, RW=0):
return v.ordered(1)
def getview(self, RW=0):
- if RW and self.db.isReadOnly():
- self.db.getWriteAccess()
- return self.db._db.view(self.name).ordered(1)
+ return self.db._db.view(self.classname).ordered(1)
def getindexview(self, RW=0):
def getindexview(self, RW=0):
- if RW and self.db.isReadOnly():
- self.db.getWriteAccess()
- return self.db._db.view("_%s" % self.name).ordered(1)
+ return self.db._db.view("_%s" % self.classname).ordered(1)
def _fetchML(sv):
l = []
def _fetchML(sv):
l = []
hyperdb.Multilink : _fetchML,
hyperdb.Interval : date.Interval,
hyperdb.Password : _fetchPW,
hyperdb.Multilink : _fetchML,
hyperdb.Interval : date.Interval,
hyperdb.Password : _fetchPW,
+ hyperdb.Boolean : lambda n: n,
+ hyperdb.Number : lambda n: n,
+ hyperdb.String : str,
}
class FileName(hyperdb.String):
}
class FileName(hyperdb.String):
hyperdb.Multilink : 'V',
hyperdb.Interval : 'S',
hyperdb.Password : 'S',
hyperdb.Multilink : 'V',
hyperdb.Interval : 'S',
hyperdb.Password : 'S',
+ hyperdb.Boolean : 'I',
+ hyperdb.Number : 'I',
}
class FileClass(Class):
' like Class but with a content property '
}
class FileClass(Class):
' like Class but with a content property '
+ default_mime_type = 'text/plain'
def __init__(self, db, classname, **properties):
properties['content'] = FileName()
def __init__(self, db, classname, **properties):
properties['content'] = FileName()
+ if not properties.has_key('type'):
+ properties['type'] = hyperdb.String()
Class.__init__(self, db, classname, **properties)
def get(self, nodeid, propname, default=_marker, cache=1):
x = Class.get(self, nodeid, propname, default, cache)
Class.__init__(self, db, classname, **properties)
def get(self, nodeid, propname, default=_marker, cache=1):
x = Class.get(self, nodeid, propname, default, cache)
newid = Class.create(self, **propvalues)
if not content:
return newid
newid = Class.create(self, **propvalues)
if not content:
return newid
- if content.startswith('/tracker/download.php?'):
- self.set(newid, content='http://sourceforge.net'+content)
- return newid
- nm = bnm = '%s%s' % (self.name, newid)
+ nm = bnm = '%s%s' % (self.classname, newid)
sd = str(int(int(newid) / 1000))
sd = str(int(int(newid) / 1000))
- d = os.path.join(self.db.config.DATABASE, 'files', self.name, sd)
+ d = os.path.join(self.db.config.DATABASE, 'files', self.classname, sd)
if not os.path.exists(d):
os.makedirs(d)
nm = os.path.join(d, nm)
open(nm, 'wb').write(content)
self.set(newid, content = 'file:'+nm)
if not os.path.exists(d):
os.makedirs(d)
nm = os.path.join(d, nm)
open(nm, 'wb').write(content)
self.set(newid, content = 'file:'+nm)
- self.db.indexer.add_files(d, bnm)
- self.db.indexer.save_index()
+ mimetype = propvalues.get('type', self.default_mime_type)
+ self.db.indexer.add_text((self.classname, newid, 'content'), content, mimetype)
def undo(fnm=nm, action1=os.remove, indexer=self.db.indexer):
def undo(fnm=nm, action1=os.remove, indexer=self.db.indexer):
- remove(fnm)
- indexer.purge_entry(fnm, indexer.files, indexer.words)
+ action1(fnm)
self.rollbackaction(undo)
return newid
self.rollbackaction(undo)
return newid
-
-# Yuck - c&p to avoid getting hyperdb.Class
-class IssueClass(Class):
-
+ def index(self, nodeid):
+ Class.index(self, nodeid)
+ mimetype = self.get(nodeid, 'type')
+ if not mimetype:
+ mimetype = self.default_mime_type
+ self.db.indexer.add_text((self.classname, nodeid, 'content'),
+ self.get(nodeid, 'content'), mimetype)
+
+class IssueClass(Class, roundupdb.IssueClass):
# Overridden methods:
# Overridden methods:
-
def __init__(self, db, classname, **properties):
"""The newly-created class automatically includes the "messages",
"files", "nosy", and "superseder" properties. If the 'properties'
dictionary attempts to specify any of these properties or a
"creation" or "activity" property, a ValueError is raised."""
if not properties.has_key('title'):
def __init__(self, db, classname, **properties):
"""The newly-created class automatically includes the "messages",
"files", "nosy", and "superseder" properties. If the 'properties'
dictionary attempts to specify any of these properties or a
"creation" or "activity" property, a ValueError is raised."""
if not properties.has_key('title'):
- properties['title'] = hyperdb.String()
+ properties['title'] = hyperdb.String(indexme='yes')
if not properties.has_key('messages'):
properties['messages'] = hyperdb.Multilink("msg")
if not properties.has_key('files'):
if not properties.has_key('messages'):
properties['messages'] = hyperdb.Multilink("msg")
if not properties.has_key('files'):
if not properties.has_key('superseder'):
properties['superseder'] = hyperdb.Multilink(classname)
Class.__init__(self, db, classname, **properties)
if not properties.has_key('superseder'):
properties['superseder'] = hyperdb.Multilink(classname)
Class.__init__(self, db, classname, **properties)
-
- # New methods:
-
- def addmessage(self, nodeid, summary, text):
- """Add a message to an issue's mail spool.
-
- A new "msg" node is constructed using the current date, the user that
- owns the database connection as the author, and the specified summary
- text.
-
- The "files" and "recipients" fields are left empty.
-
- The given text is saved as the body of the message and the node is
- appended to the "messages" field of the specified issue.
- """
-
- def nosymessage(self, nodeid, msgid, oldvalues):
- """Send a message to the members of an issue's nosy list.
-
- The message is sent only to users on the nosy list who are not
- already on the "recipients" list for the message.
- These users are then added to the message's "recipients" list.
- """
- users = self.db.user
- messages = self.db.msg
-
- # figure the recipient ids
- sendto = []
- r = {}
- recipients = messages.get(msgid, 'recipients')
- for recipid in messages.get(msgid, 'recipients'):
- r[recipid] = 1
-
- # figure the author's id, and indicate they've received the message
- authid = messages.get(msgid, 'author')
-
- # possibly send the message to the author, as long as they aren't
- # anonymous
- if (self.db.config.MESSAGES_TO_AUTHOR == 'yes' and
- users.get(authid, 'username') != 'anonymous'):
- sendto.append(authid)
- r[authid] = 1
-
- # now figure the nosy people who weren't recipients
- nosy = self.get(nodeid, 'nosy')
- for nosyid in nosy:
- # Don't send nosy mail to the anonymous user (that user
- # shouldn't appear in the nosy list, but just in case they
- # do...)
- if users.get(nosyid, 'username') == 'anonymous':
- continue
- # make sure they haven't seen the message already
- if not r.has_key(nosyid):
- # send it to them
- sendto.append(nosyid)
- recipients.append(nosyid)
-
- # generate a change note
- if oldvalues:
- note = self.generateChangeNote(nodeid, oldvalues)
- else:
- note = self.generateCreateNote(nodeid)
-
- # we have new recipients
- if sendto:
- # map userids to addresses
- sendto = [users.get(i, 'address') for i in sendto]
-
- # update the message's recipients list
- messages.set(msgid, recipients=recipients)
-
- # send the message
- self.send_message(nodeid, msgid, note, sendto)
-
- # XXX backwards compatibility - don't remove
- sendmessage = nosymessage
-
- def send_message(self, nodeid, msgid, note, sendto):
- '''Actually send the nominated message from this node to the sendto
- recipients, with the note appended.
- '''
- users = self.db.user
- messages = self.db.msg
- files = self.db.file
-
- # determine the messageid and inreplyto of the message
- inreplyto = messages.get(msgid, 'inreplyto')
- messageid = messages.get(msgid, 'messageid')
-
- # make up a messageid if there isn't one (web edit)
- if not messageid:
- # this is an old message that didn't get a messageid, so
- # create one
- messageid = "<%s.%s.%s%s@%s>"%(time.time(), random.random(),
- self.classname, nodeid, self.db.config.MAIL_DOMAIN)
- messages.set(msgid, messageid=messageid)
-
- # send an email to the people who missed out
- cn = self.classname
- title = self.get(nodeid, 'title') or '%s message copy'%cn
- # figure author information
- authid = messages.get(msgid, 'author')
- authname = users.get(authid, 'realname')
- if not authname:
- authname = users.get(authid, 'username')
- authaddr = users.get(authid, 'address')
- if authaddr:
- authaddr = ' <%s>'%authaddr
- else:
- authaddr = ''
-
- # make the message body
- m = ['']
-
- # put in roundup's signature
- if self.db.config.EMAIL_SIGNATURE_POSITION == 'top':
- m.append(self.email_signature(nodeid, msgid))
-
- # add author information
- if len(self.get(nodeid,'messages')) == 1:
- m.append("New submission from %s%s:"%(authname, authaddr))
- else:
- m.append("%s%s added the comment:"%(authname, authaddr))
- m.append('')
-
- # add the content
- m.append(messages.get(msgid, 'content'))
-
- # add the change note
- if note:
- m.append(note)
-
- # put in roundup's signature
- if self.db.config.EMAIL_SIGNATURE_POSITION == 'bottom':
- m.append(self.email_signature(nodeid, msgid))
-
- # encode the content as quoted-printable
- content = cStringIO.StringIO('\n'.join(m))
- content_encoded = cStringIO.StringIO()
- quopri.encode(content, content_encoded, 0)
- content_encoded = content_encoded.getvalue()
-
- # get the files for this message
- message_files = messages.get(msgid, 'files')
-
- # make sure the To line is always the same (for testing mostly)
- sendto.sort()
-
- # create the message
- message = cStringIO.StringIO()
- writer = MimeWriter.MimeWriter(message)
- writer.addheader('Subject', '[%s%s] %s'%(cn, nodeid, title))
- writer.addheader('To', ', '.join(sendto))
- writer.addheader('From', '%s <%s>'%(authname,
- self.db.config.ISSUE_TRACKER_EMAIL))
- writer.addheader('Reply-To', '%s <%s>'%(self.db.config.INSTANCE_NAME,
- self.db.config.ISSUE_TRACKER_EMAIL))
- writer.addheader('MIME-Version', '1.0')
- if messageid:
- writer.addheader('Message-Id', messageid)
- if inreplyto:
- writer.addheader('In-Reply-To', inreplyto)
-
- # add a uniquely Roundup header to help filtering
- writer.addheader('X-Roundup-Name', self.db.config.INSTANCE_NAME)
-
- # attach files
- if message_files:
- part = writer.startmultipartbody('mixed')
- part = writer.nextpart()
- part.addheader('Content-Transfer-Encoding', 'quoted-printable')
- body = part.startbody('text/plain')
- body.write(content_encoded)
- for fileid in message_files:
- name = files.get(fileid, 'name')
- mime_type = files.get(fileid, 'type')
- content = files.get(fileid, 'content')
- part = writer.nextpart()
- if mime_type == 'text/plain':
- part.addheader('Content-Disposition',
- 'attachment;\n filename="%s"'%name)
- part.addheader('Content-Transfer-Encoding', '7bit')
- body = part.startbody('text/plain')
- body.write(content)
- else:
- # some other type, so encode it
- if not mime_type:
- # this should have been done when the file was saved
- mime_type = mimetypes.guess_type(name)[0]
- if mime_type is None:
- mime_type = 'application/octet-stream'
- part.addheader('Content-Disposition',
- 'attachment;\n filename="%s"'%name)
- part.addheader('Content-Transfer-Encoding', 'base64')
- body = part.startbody(mime_type)
- body.write(base64.encodestring(content))
- writer.lastpart()
- else:
- writer.addheader('Content-Transfer-Encoding', 'quoted-printable')
- body = writer.startbody('text/plain')
- body.write(content_encoded)
-
- # now try to send the message
- if SENDMAILDEBUG:
- open(SENDMAILDEBUG, 'w').write('FROM: %s\nTO: %s\n%s\n'%(
- self.db.config.ADMIN_EMAIL,
- ', '.join(sendto),message.getvalue()))
- else:
- try:
- # send the message as admin so bounces are sent there
- # instead of to roundup
- smtp = smtplib.SMTP(self.db.config.MAILHOST)
- smtp.sendmail(self.db.config.ADMIN_EMAIL, sendto,
- message.getvalue())
- except socket.error, value:
- raise MessageSendError, \
- "Couldn't send confirmation email: mailhost %s"%value
- except smtplib.SMTPException, value:
- raise MessageSendError, \
- "Couldn't send confirmation email: %s"%value
-
- def email_signature(self, nodeid, msgid):
- ''' Add a signature to the e-mail with some useful information
- '''
- web = self.db.config.ISSUE_TRACKER_WEB + 'issue'+ nodeid
- email = '"%s" <%s>'%(self.db.config.INSTANCE_NAME,
- self.db.config.ISSUE_TRACKER_EMAIL)
- line = '_' * max(len(web), len(email))
- return '%s\n%s\n%s\n%s'%(line, email, web, line)
-
- def generateCreateNote(self, nodeid):
- """Generate a create note that lists initial property values
- """
- cn = self.classname
- cl = self.db.classes[cn]
- props = cl.getprops(protected=0)
-
- # list the values
- m = []
- l = props.items()
- l.sort()
- for propname, prop in l:
- value = cl.get(nodeid, propname, None)
- # skip boring entries
- if not value:
+CURVERSION = 1
+
+class Indexer(indexer.Indexer):
+ disallows = {'THE':1, 'THIS':1, 'ZZZ':1, 'THAT':1, 'WITH':1}
+ def __init__(self, path, datadb):
+ self.db = metakit.storage(os.path.join(path, 'index.mk4'), 1)
+ self.datadb = datadb
+ self.reindex = 0
+ v = self.db.view('version')
+ if not v.structure():
+ v = self.db.getas('version[vers:I]')
+ self.db.commit()
+ v.append(vers=CURVERSION)
+ self.reindex = 1
+ elif v[0].vers != CURVERSION:
+ v[0].vers = CURVERSION
+ self.reindex = 1
+ if self.reindex:
+ self.db.getas('ids[tblid:I,nodeid:I,propid:I]')
+ self.db.getas('index[word:S,hits[pos:I]]')
+ self.db.commit()
+ self.reindex = 1
+ self.changed = 0
+ self.propcache = {}
+ def force_reindex(self):
+ v = self.db.view('ids')
+ v[:] = []
+ v = self.db.view('index')
+ v[:] = []
+ self.db.commit()
+ self.reindex = 1
+ def should_reindex(self):
+ return self.reindex
+ def _getprops(self, classname):
+ props = self.propcache.get(classname, None)
+ if props is None:
+ props = self.datadb.view(classname).structure()
+ props = [prop.name for prop in props]
+ self.propcache[classname] = props
+ return props
+ def _getpropid(self, classname, propname):
+ return self._getprops(classname).index(propname)
+ def _getpropname(self, classname, propid):
+ return self._getprops(classname)[propid]
+ def add_text(self, identifier, text, mime_type='text/plain'):
+ if mime_type != 'text/plain':
+ return
+ classname, nodeid, property = identifier
+ tbls = self.datadb.view('tables')
+ tblid = tbls.find(name=classname)
+ if tblid < 0:
+ raise KeyError, "unknown class %r"%classname
+ nodeid = int(nodeid)
+ propid = self._getpropid(classname, property)
+ pos = self.db.view('ids').append(tblid=tblid,nodeid=nodeid,propid=propid)
+
+ wordlist = re.findall(r'\b\w{3,25}\b', text)
+ words = {}
+ for word in wordlist:
+ word = word.upper()
+ if not self.disallows.has_key(word):
+ words[word] = 1
+ words = words.keys()
+
+ index = self.db.view('index').ordered(1)
+ for word in words:
+ ndx = index.find(word=word)
+ if ndx < 0:
+ ndx = index.append(word=word)
+ hits = index[ndx].hits
+ if len(hits)==0 or hits.find(pos=pos) < 0:
+ hits.append(pos=pos)
+ self.changed = 1
+ def find(self, wordlist):
+ hits = None
+ index = self.db.view('index').ordered(1)
+ for word in wordlist:
+ if not 2 < len(word) < 26:
continue
continue
- if isinstance(prop, hyperdb.Link):
- link = self.db.classes[prop.classname]
- if value:
- key = link.labelprop(default_to_id=1)
- if key:
- value = link.get(value, key)
- else:
- value = ''
- elif isinstance(prop, hyperdb.Multilink):
- if value is None: value = []
- l = []
- link = self.db.classes[prop.classname]
- key = link.labelprop(default_to_id=1)
- if key:
- value = [link.get(entry, key) for entry in value]
- value.sort()
- value = ', '.join(value)
- m.append('%s: %s'%(propname, value))
- m.insert(0, '----------')
- m.insert(0, '')
- return '\n'.join(m)
-
- def generateChangeNote(self, nodeid, oldvalues):
- """Generate a change note that lists property changes
- """
- cn = self.classname
- cl = self.db.classes[cn]
- changed = {}
- props = cl.getprops(protected=0)
-
- # determine what changed
- for key in oldvalues.keys():
- if key in ['files','messages']: continue
- new_value = cl.get(nodeid, key)
- # the old value might be non existent
- try:
- old_value = oldvalues[key]
- if type(new_value) is type([]):
- new_value.sort()
- old_value.sort()
- if new_value != old_value:
- changed[key] = old_value
- except:
- changed[key] = new_value
-
- # list the changes
- m = []
- l = changed.items()
- l.sort()
- for propname, oldvalue in l:
- prop = props[propname]
- value = cl.get(nodeid, propname, None)
- if isinstance(prop, hyperdb.Link):
- link = self.db.classes[prop.classname]
- key = link.labelprop(default_to_id=1)
- if key:
- if value:
- value = link.get(value, key)
- else:
- value = ''
- if oldvalue:
- oldvalue = link.get(oldvalue, key)
- else:
- oldvalue = ''
- change = '%s -> %s'%(oldvalue, value)
- elif isinstance(prop, hyperdb.Multilink):
- change = ''
- if value is None: value = []
- if oldvalue is None: oldvalue = []
- l = []
- link = self.db.classes[prop.classname]
- key = link.labelprop(default_to_id=1)
- # check for additions
- for entry in value:
- if entry in oldvalue: continue
- if key:
- l.append(link.get(entry, key))
- else:
- l.append(entry)
- if l:
- change = '+%s'%(', '.join(l))
- l = []
- # check for removals
- for entry in oldvalue:
- if entry in value: continue
- if key:
- l.append(link.get(entry, key))
- else:
- l.append(entry)
- if l:
- change += ' -%s'%(', '.join(l))
+ ndx = index.find(word=word)
+ if ndx < 0:
+ return {}
+ if hits is None:
+ hits = index[ndx].hits
else:
else:
- change = '%s -> %s'%(oldvalue, value)
- m.append('%s: %s'%(propname, change))
- if m:
- m.insert(0, '----------')
- m.insert(0, '')
- return '\n'.join(m)
+ hits = hits.intersect(index[ndx].hits)
+ if len(hits) == 0:
+ return {}
+ if hits is None:
+ return {}
+ rslt = {}
+ ids = self.db.view('ids').remapwith(hits)
+ tbls = self.datadb.view('tables')
+ for i in range(len(ids)):
+ hit = ids[i]
+ classname = tbls[hit.tblid].name
+ nodeid = str(hit.nodeid)
+ property = self._getpropname(classname, hit.propid)
+ rslt[i] = (classname, nodeid, property)
+ return rslt
+ def save_index(self):
+ if self.changed:
+ self.db.commit()
+ self.changed = 0