index 9d8d53f20badacd88c53a1b1406e7243cb0c9555..d78b27f9f8034f176cf7c29718d4c42938faad02 100755 (executable)
-from roundup import hyperdb, date, password, roundupdb
+from roundup import hyperdb, date, password, roundupdb, security
import metakit
+from sessions import Sessions
import re, marshal, os, sys, weakref, time, calendar
-from roundup.indexer import Indexer
+from roundup import indexer
+import locking
-_instances = weakref.WeakValueDictionary()
+_dbs = {}
def Database(config, journaltag=None):
- if _instances.has_key(id(config)):
- db = _instances[id(config)]
- old = db.journaltag
+ db = _dbs.get(config.DATABASE, None)
+ if db is None or db._db is None:
+ db = _Database(config, journaltag)
+ _dbs[config.DATABASE] = db
+ else:
db.journaltag = journaltag
- if hasattr(db, 'curuserid'):
+ try:
delattr(db, 'curuserid')
- return db
- else:
- db = _Database(config, journaltag)
- _instances[id(config)] = db
- return db
+ except AttributeError:
+ pass
+ return db
class _Database(hyperdb.Database):
def __init__(self, config, journaltag=None):
self.config = config
self.journaltag = journaltag
self.classes = {}
- self._classes = []
self.dirty = 0
- self.__RW = 0
+ self.lockfile = None
self._db = self.__open()
- self.indexer = Indexer(self.config.DATABASE)
+ self.indexer = Indexer(self.config.DATABASE, self._db)
+ self.sessions = Sessions(self.config)
+ self.security = security.Security(self)
+
os.umask(0002)
+ def post_init(self):
+ if self.indexer.should_reindex():
+ self.reindex()
+
+ def reindex(self):
+ for klass in self.classes.values():
+ for nodeid in klass.list():
+ klass.index(nodeid)
+ self.indexer.save_index()
+
# --- defined in ping's spec
def __getattr__(self, classname):
except KeyError:
x = 0
return x
+ elif classname == 'transactions':
+ return self.dirty
return self.getclass(classname)
def getclass(self, classname):
return self.classes[classname]
# --- exposed methods
def commit(self):
if self.dirty:
- if self.__RW:
- self._db.commit()
- for cl in self.classes.values():
- cl._commit()
- else:
- raise RuntimeError, "metakit is open RO"
+ self._db.commit()
+ for cl in self.classes.values():
+ cl._commit()
+ self.indexer.save_index()
self.dirty = 0
def rollback(self):
if self.dirty:
for cl in self.classes.values():
cl._clear()
def hasnode(self, classname, nodeid):
- return self.getclass(clasname).hasnode(nodeid)
+ return self.getclass(classname).hasnode(nodeid)
def pack(self, pack_before):
pass
def addclass(self, cl):
- self.classes[cl.name] = cl
+ self.classes[cl.classname] = cl
+ if self.tables.find(name=cl.classname) < 0:
+ self.tables.append(name=cl.classname)
def addjournal(self, tablenm, nodeid, action, params):
tblid = self.tables.find(name=tablenm)
if tblid == -1:
return rslt
def close(self):
- import time
- now = time.time
- start = now()
for cl in self.classes.values():
cl.db = None
- #self._db.rollback()
- #print "pre-close cleanup of DB(%d) took %2.2f secs" % (self.__RW, now()-start)
self._db = None
- #print "close of DB(%d) took %2.2f secs" % (self.__RW, now()-start)
+ locking.release_lock(self.lockfile)
+ del _dbs[self.config.DATABASE]
+ self.lockfile.close()
self.classes = {}
- try:
- del _instances[id(self.config)]
- except KeyError:
- pass
- self.__RW = 0
-
+ self.indexer = None
+
# --- internal
def __open(self):
self.dbnm = db = os.path.join(self.config.DATABASE, 'tracker.mk4')
+ lockfilenm = db[:-3]+'lck'
+ self.lockfile = locking.acquire_lock(lockfilenm)
+ self.lockfile.write(str(os.getpid()))
+ self.lockfile.flush()
self.fastopen = 0
if os.path.exists(db):
dbtm = os.path.getmtime(db)
else:
# can't find schemamod - must be frozen
self.fastopen = 1
- else:
- self.__RW = 1
- db = metakit.storage(db, self.__RW)
+ db = metakit.storage(db, 1)
hist = db.view('history')
tables = db.view('tables')
if not self.fastopen:
self.tables = tables
self.hist = hist
return db
- def isReadOnly(self):
- return self.__RW == 0
- def getWriteAccess(self):
- if self.journaltag is not None and self.__RW == 0:
- now = time.time
- start = now()
- self._db = None
- #print "closing the file took %2.2f secs" % (now()-start)
- start = now()
- self._db = metakit.storage(self.dbnm, 1)
- self.__RW = 1
- self.hist = self._db.view('history')
- self.tables = self._db.view('tables')
- #print "getting RW access took %2.2f secs" % (now()-start)
-
+
_STRINGTYPE = type('')
_LISTTYPE = type([])
_CREATE, _SET, _RETIRE, _LINK, _UNLINK = range(5)
_ALLOWSETTINGPRIVATEPROPS = 0
-class Class: # no, I'm not going to subclass the existing!
+class Class:
privateprops = None
def __init__(self, db, classname, **properties):
- self.db = weakref.proxy(db)
- self.name = classname
+ #self.db = weakref.proxy(db)
+ self.db = db
+ self.classname = classname
self.keyname = None
self.ruprops = properties
self.privateprops = { 'id' : hyperdb.String(),
self.properties = self.ruprops
self.db.addclass(self)
self.idcache = {}
+
+ # default is to journal changes
+ self.do_journal = 1
+
+ def enableJournalling(self):
+ '''Turn journalling on for this class
+ '''
+ self.do_journal = 1
+
+ def disableJournalling(self):
+ '''Turn journalling off for this class
+ '''
+ self.do_journal = 0
# --- the roundup.Class methods
def audit(self, event, detector):
self.reactors[event].append(detector)
# --- the hyperdb.Class methods
def create(self, **propvalues):
+ self.fireAuditors('create', None, propvalues)
rowdict = {}
rowdict['id'] = newid = self.maxid
self.maxid += 1
if ndx is None:
ndx = view.find(id=id)
if ndx < 0:
- raise IndexError, "%s has no node %s" % (self.name, nodeid)
+ raise IndexError, "%s has no node %s" % (self.classname, nodeid)
self.idcache[id] = ndx
- raw = getattr(view[ndx], propname)
+ try:
+ raw = getattr(view[ndx], propname)
+ except AttributeError:
+ raise KeyError, propname
rutyp = self.ruprops.get(propname, None)
if rutyp is None:
rutyp = self.privateprops[propname]
return raw
def set(self, nodeid, **propvalues):
-
isnew = 0
if propvalues.has_key('#ISNEW'):
isnew = 1
del propvalues['#ISNEW']
+ if not isnew:
+ self.fireAuditors('set', nodeid, propvalues)
if not propvalues:
- return
+ return propvalues
if propvalues.has_key('id'):
raise KeyError, '"id" is reserved'
if self.db.journaltag is None:
id = int(nodeid)
ndx = view.find(id=id)
if ndx < 0:
- raise IndexError, "%s has no node %s" % (self.name, nodeid)
+ raise IndexError, "%s has no node %s" % (self.classname, nodeid)
row = view[ndx]
if row._isdel:
- raise IndexError, "%s has no node %s" % (self.name, nodeid)
+ raise IndexError, "%s has no node %s" % (self.classname, nodeid)
oldnode = self.uncommitted.setdefault(id, {})
changes = {}
# do stuff based on the prop type
if isinstance(prop, hyperdb.Link):
link_class = prop.classname
+ # must be a string or None
+ if value is not None and not isinstance(value, type('')):
+ raise ValueError, 'property "%s" link value be a string'%(
+ propname)
+ # Roundup sets to "unselected" by passing None
+ if value is None:
+ value = 0
# if it isn't a number, it's a key
- if type(value) != _STRINGTYPE:
- raise ValueError, 'link value must be String'
try:
int(value)
except ValueError:
raise IndexError, 'new property "%s": %s not a %s'%(
key, value, prop.classname)
- if not self.db.getclass(link_class).hasnode(value):
+ if (value is not None and
+ not self.db.getclass(link_class).hasnode(value)):
raise IndexError, '%s has no node %s'%(link_class, value)
setattr(row, key, int(value))
changes[key] = oldvalue
- if prop.do_journal:
+ if self.do_journal and prop.do_journal:
# register the unlink with the old linked node
if oldvalue:
- self.db.addjournal(link_class, value, _UNLINK, (self.name, str(row.id), key))
+ self.db.addjournal(link_class, value, _UNLINK,
+ (self.classname, str(row.id), key))
# register the link with the newly linked node
if value:
- self.db.addjournal(link_class, value, _LINK, (self.name, str(row.id), key))
+ self.db.addjournal(link_class, value, _LINK,
+ (self.classname, str(row.id), key))
elif isinstance(prop, hyperdb.Multilink):
if type(value) != _LISTTYPE:
if id not in value:
rmvd.append(id)
# register the unlink with the old linked node
- if prop.do_journal:
- self.db.addjournal(link_class, id, _UNLINK, (self.name, str(row.id), key))
+ if self.do_journal and prop.do_journal:
+ self.db.addjournal(link_class, id, _UNLINK, (self.classname, str(row.id), key))
# handle additions
adds = []
link_class, id)
adds.append(id)
# register the link with the newly linked node
- if prop.do_journal:
- self.db.addjournal(link_class, id, _LINK, (self.name, str(row.id), key))
+ if self.do_journal and prop.do_journal:
+ self.db.addjournal(link_class, id, _LINK, (self.classname, str(row.id), key))
sv = getattr(row, key)
i = 0
for id in adds:
sv.append(fid=int(id))
changes[key] = oldvalue
+ if not rmvd and not adds:
+ del propvalues[key]
elif isinstance(prop, hyperdb.String):
changes[key] = oldvalue
if hasattr(prop, 'isfilename') and prop.isfilename:
propvalues[key] = os.path.basename(value)
+ if prop.indexme:
+ self.db.indexer.add_text((self.classname, nodeid, key), value, 'text/plain')
elif isinstance(prop, hyperdb.Password):
if not isinstance(value, password.Password):
setattr(row, key, str(value))
changes[key] = str(oldvalue)
propvalues[key] = str(value)
+
+ elif value is not None and isinstance(prop, hyperdb.Number):
+ setattr(row, key, int(value))
+ changes[key] = oldvalue
+ propvalues[key] = value
+
+ elif value is not None and isinstance(prop, hyperdb.Boolean):
+ bv = value != 0
+ setattr(row, key, bv)
+ changes[key] = oldvalue
+ propvalues[key] = value
oldnode[key] = oldvalue
# nothing to do?
if not propvalues:
- return
- if not row.activity:
+ return propvalues
+ if not propvalues.has_key('activity'):
row.activity = int(time.time())
if isnew:
if not row.creation:
row.creator = self.db.curuserid
self.db.dirty = 1
- if isnew:
- self.db.addjournal(self.name, nodeid, _CREATE, {})
- else:
- self.db.addjournal(self.name, nodeid, _SET, changes)
+ if self.do_journal:
+ if isnew:
+ self.db.addjournal(self.classname, nodeid, _CREATE, {})
+ self.fireReactors('create', nodeid, None)
+ else:
+ self.db.addjournal(self.classname, nodeid, _SET, changes)
+ self.fireReactors('set', nodeid, oldnode)
+ return propvalues
+
def retire(self, nodeid):
+ self.fireAuditors('retire', nodeid, None)
view = self.getview(1)
ndx = view.find(id=int(nodeid))
if ndx < 0:
oldvalues = self.uncommitted.setdefault(row.id, {})
oldval = oldvalues['_isdel'] = row._isdel
row._isdel = 1
- self.db.addjournal(self.name, nodeid, _RETIRE, {})
- iv = self.getindexview(1)
- ndx = iv.find(k=getattr(row, self.keyname),i=row.id)
- if ndx > -1:
- iv.delete(ndx)
+ if self.do_journal:
+ self.db.addjournal(self.classname, nodeid, _RETIRE, {})
+ if self.keyname:
+ iv = self.getindexview(1)
+ ndx = iv.find(k=getattr(row, self.keyname),i=row.id)
+ if ndx > -1:
+ iv.delete(ndx)
self.db.dirty = 1
+ self.fireReactors('retire', nodeid, None)
def history(self, nodeid):
- return self.db.gethistory(self.name, nodeid)
+ if not self.do_journal:
+ raise ValueError, 'Journalling is disabled for this class'
+ return self.db.gethistory(self.classname, nodeid)
def setkey(self, propname):
if self.keyname:
if propname == self.keyname:
return
- raise ValueError, "%s already indexed on %s" % (self.name, self.keyname)
+ raise ValueError, "%s already indexed on %s" % (self.classname, self.keyname)
# first setkey for this run
self.keyname = propname
- iv = self.db._db.view('_%s' % self.name)
- if self.db.fastopen or iv.structure():
+ iv = self.db._db.view('_%s' % self.classname)
+ if self.db.fastopen and iv.structure():
return
# very first setkey ever
- iv = self.db._db.getas('_%s[k:S,i:I]' % self.name)
+ self.db.dirty = 1
+ iv = self.db._db.getas('_%s[k:S,i:I]' % self.classname)
iv = iv.ordered(1)
- #XXX
- print "setkey building index"
+# print "setkey building index"
for row in self.getview():
iv.append(k=getattr(row, propname), i=row.id)
+ self.db.commit()
def getkey(self):
return self.keyname
def lookup(self, keyvalue):
if ndx > -1:
return str(view[ndx].id)
raise KeyError, keyvalue
+
+ def destroy(self, keyvalue):
+ #TODO clean this up once Richard's said how it should work
+ iv = self.getindexview()
+ if iv:
+ ndx = iv.find(k=keyvalue)
+ if ndx > -1:
+ id = iv[ndx].i
+ iv.delete(ndx)
+ view = self.getview()
+ ndx = view.find(id=id)
+ if ndx > -1:
+ view.delete(ndx)
+
def find(self, **propspec):
"""Get the ids of nodes in this class which link to the given nodes.
'propspec' consists of keyword args propname={nodeid:1,}
- 'propname' must be the name of a property in this class, or a
- KeyError is raised. That property must be a Link or Multilink
- property, or a TypeError is raised.
+ 'propname' must be the name of a property in this class, or a
+ KeyError is raised. That property must be a Link or
+ Multilink property, or a TypeError is raised.
+
Any node in this class whose propname property links to any of the
nodeids will be returned. Used by the full text indexing, which knows
- that "foo" occurs in msg1, msg3 and file7; so we have hits on these issues:
+ that "foo" occurs in msg1, msg3 and file7; so we have hits on these
+ issues:
+
db.issue.find(messages={'1':1,'3':1}, files={'7':1})
+
"""
propspec = propspec.items()
for propname, nodeid in propspec:
# check the prop is OK
prop = self.ruprops[propname]
- if not isinstance(prop, hyperdb.Link) and not isinstance(prop, hyperdb.Multilink):
+ if (not isinstance(prop, hyperdb.Link) and
+ not isinstance(prop, hyperdb.Multilink)):
raise TypeError, "'%s' not a Link/Multilink property"%propname
vws = []
for propname, ids in propspec:
+ if type(ids) is _STRINGTYPE:
+ ids = {ids:1}
prop = self.ruprops[propname]
view = self.getview()
if isinstance(prop, hyperdb.Multilink):
return ids.has_key(str(getattr(row, nm)))
ndxview = view.filter(ff)
vws.append(ndxview.unique())
+
+ # handle the empty match case
+ if not vws:
+ return []
+
ndxview = vws[0]
for v in vws[1:]:
ndxview = ndxview.union(v)
def addprop(self, **properties):
for key in properties.keys():
if self.ruprops.has_key(key):
- raise ValueError, "%s is already a property of %s" % (key, self.name)
+ raise ValueError, "%s is already a property of %s" % (key, self.classname)
self.ruprops.update(properties)
+ self.db.fastopen = 0
view = self.__getview()
+ self.db.commit()
# ---- end of ping's spec
def filter(self, search_matches, filterspec, sort, group):
# search_matches is None or a set (dict of {nodeid: {propname:[nodeid,...]}})
regexes[propname] = re.compile(v, re.I)
elif propname == 'id':
where[propname] = int(value)
+ elif isinstance(prop, hyperdb.Boolean):
+ if type(value) is _STRINGTYPE:
+ bv = value.lower() in ('yes', 'true', 'on', '1')
+ else:
+ bv = value
+ where[propname] = bv
+ elif isinstance(prop, hyperdb.Number):
+ where[propname] = int(value)
else:
where[propname] = str(value)
v = self.getview()
try:
prop = getattr(v, propname)
except AttributeError:
- # I can't sort on 'activity', cause it's psuedo!!
+ print "MK has no property %s" % propname
continue
+ propclass = self.ruprops.get(propname, None)
+ if propclass is None:
+ propclass = self.privateprops.get(propname, None)
+ if propclass is None:
+ print "Schema has no property %s" % propname
+ continue
+ if isinstance(propclass, hyperdb.Link):
+ linkclass = self.db.getclass(propclass.classname)
+ lv = linkclass.getview()
+ lv = lv.rename('id', propname)
+ v = v.join(lv, prop, 1)
+ if linkclass.getprops().has_key('order'):
+ propname = 'order'
+ else:
+ propname = linkclass.labelprop()
+ prop = getattr(v, propname)
if isreversed:
rev.append(prop)
sortspec.append(prop)
- v = v.sortrev(sortspec, rev)[:] #XXX Aaaabh
+ v = v.sortrev(sortspec, rev)[:] #XXX Metakit bug
#print "filter sort at %s" % time.time()
rslt = []
return l
def addjournal(self, nodeid, action, params):
- self.db.addjournal(self.name, nodeid, action, params)
+ self.db.addjournal(self.classname, nodeid, action, params)
+
+ def index(self, nodeid):
+ ''' Add (or refresh) the node to search indexes '''
+ # find all the String properties that have indexme
+ for prop, propclass in self.getprops().items():
+ if isinstance(propclass, hyperdb.String) and propclass.indexme:
+ # index them under (classname, nodeid, property)
+ self.db.indexer.add_text((self.classname, nodeid, prop),
+ str(self.get(nodeid, prop)))
+
# --- used by Database
def _commit(self):
""" called post commit of the DB.
# --- internal
def __getview(self):
db = self.db._db
- view = db.view(self.name)
- if self.db.fastopen:
+ view = db.view(self.classname)
+ mkprops = view.structure()
+ if mkprops and self.db.fastopen:
return view.ordered(1)
# is the definition the same?
for nm, rutyp in self.ruprops.items():
- mkprop = getattr(view, nm, None)
+ for mkprop in mkprops:
+ if mkprop.name == nm:
+ break
+ else:
+ mkprop = None
if mkprop is None:
- #print "%s missing prop %s (%s)" % (self.name, nm, rutyp.__class__.__name__)
break
if _typmap[rutyp.__class__] != mkprop.type:
- #print "%s - prop %s (%s) has wrong mktyp (%s)" % (self.name, nm, rutyp.__class__.__name__, mkprop.type)
break
else:
return view.ordered(1)
# need to create or restructure the mk view
# id comes first, so MK will order it for us
self.db.dirty = 1
- s = ["%s[id:I" % self.name]
+ s = ["%s[id:I" % self.classname]
for nm, rutyp in self.ruprops.items():
mktyp = _typmap[rutyp.__class__]
s.append('%s:%s' % (nm, mktyp))
if mktyp == 'V':
s[-1] += ('[fid:I]')
s.append('_isdel:I,activity:I,creation:I,creator:I]')
- v = db.getas(','.join(s))
+ v = self.db._db.getas(','.join(s))
+ self.db.commit()
return v.ordered(1)
def getview(self, RW=0):
- if RW and self.db.isReadOnly():
- self.db.getWriteAccess()
- return self.db._db.view(self.name).ordered(1)
+ return self.db._db.view(self.classname).ordered(1)
def getindexview(self, RW=0):
- if RW and self.db.isReadOnly():
- self.db.getWriteAccess()
- return self.db._db.view("_%s" % self.name).ordered(1)
+ return self.db._db.view("_%s" % self.classname).ordered(1)
def _fetchML(sv):
l = []
hyperdb.Multilink : _fetchML,
hyperdb.Interval : date.Interval,
hyperdb.Password : _fetchPW,
+ hyperdb.Boolean : lambda n: n,
+ hyperdb.Number : lambda n: n,
+ hyperdb.String : str,
}
class FileName(hyperdb.String):
hyperdb.Multilink : 'V',
hyperdb.Interval : 'S',
hyperdb.Password : 'S',
+ hyperdb.Boolean : 'I',
+ hyperdb.Number : 'I',
}
class FileClass(Class):
' like Class but with a content property '
+ default_mime_type = 'text/plain'
def __init__(self, db, classname, **properties):
properties['content'] = FileName()
+ if not properties.has_key('type'):
+ properties['type'] = hyperdb.String()
Class.__init__(self, db, classname, **properties)
def get(self, nodeid, propname, default=_marker, cache=1):
x = Class.get(self, nodeid, propname, default, cache)
newid = Class.create(self, **propvalues)
if not content:
return newid
- if content.startswith('/tracker/download.php?'):
- self.set(newid, content='http://sourceforge.net'+content)
- return newid
- nm = bnm = '%s%s' % (self.name, newid)
+ nm = bnm = '%s%s' % (self.classname, newid)
sd = str(int(int(newid) / 1000))
- d = os.path.join(self.db.config.DATABASE, 'files', self.name, sd)
+ d = os.path.join(self.db.config.DATABASE, 'files', self.classname, sd)
if not os.path.exists(d):
os.makedirs(d)
nm = os.path.join(d, nm)
open(nm, 'wb').write(content)
self.set(newid, content = 'file:'+nm)
- self.db.indexer.add_files(d, bnm)
- self.db.indexer.save_index()
+ mimetype = propvalues.get('type', self.default_mime_type)
+ self.db.indexer.add_text((self.classname, newid, 'content'), content, mimetype)
def undo(fnm=nm, action1=os.remove, indexer=self.db.indexer):
- remove(fnm)
- indexer.purge_entry(fnm, indexer.files, indexer.words)
+ action1(fnm)
self.rollbackaction(undo)
return newid
-
-# Yuck - c&p to avoid getting hyperdb.Class
-class IssueClass(Class):
-
+ def index(self, nodeid):
+ Class.index(self, nodeid)
+ mimetype = self.get(nodeid, 'type')
+ if not mimetype:
+ mimetype = self.default_mime_type
+ self.db.indexer.add_text((self.classname, nodeid, 'content'),
+ self.get(nodeid, 'content'), mimetype)
+
+class IssueClass(Class, roundupdb.IssueClass):
# Overridden methods:
-
def __init__(self, db, classname, **properties):
"""The newly-created class automatically includes the "messages",
"files", "nosy", and "superseder" properties. If the 'properties'
dictionary attempts to specify any of these properties or a
"creation" or "activity" property, a ValueError is raised."""
if not properties.has_key('title'):
- properties['title'] = hyperdb.String()
+ properties['title'] = hyperdb.String(indexme='yes')
if not properties.has_key('messages'):
properties['messages'] = hyperdb.Multilink("msg")
if not properties.has_key('files'):
if not properties.has_key('superseder'):
properties['superseder'] = hyperdb.Multilink(classname)
Class.__init__(self, db, classname, **properties)
-
- # New methods:
-
- def addmessage(self, nodeid, summary, text):
- """Add a message to an issue's mail spool.
-
- A new "msg" node is constructed using the current date, the user that
- owns the database connection as the author, and the specified summary
- text.
-
- The "files" and "recipients" fields are left empty.
-
- The given text is saved as the body of the message and the node is
- appended to the "messages" field of the specified issue.
- """
-
- def nosymessage(self, nodeid, msgid, oldvalues):
- """Send a message to the members of an issue's nosy list.
-
- The message is sent only to users on the nosy list who are not
- already on the "recipients" list for the message.
- These users are then added to the message's "recipients" list.
- """
- users = self.db.user
- messages = self.db.msg
-
- # figure the recipient ids
- sendto = []
- r = {}
- recipients = messages.get(msgid, 'recipients')
- for recipid in messages.get(msgid, 'recipients'):
- r[recipid] = 1
-
- # figure the author's id, and indicate they've received the message
- authid = messages.get(msgid, 'author')
-
- # possibly send the message to the author, as long as they aren't
- # anonymous
- if (self.db.config.MESSAGES_TO_AUTHOR == 'yes' and
- users.get(authid, 'username') != 'anonymous'):
- sendto.append(authid)
- r[authid] = 1
-
- # now figure the nosy people who weren't recipients
- nosy = self.get(nodeid, 'nosy')
- for nosyid in nosy:
- # Don't send nosy mail to the anonymous user (that user
- # shouldn't appear in the nosy list, but just in case they
- # do...)
- if users.get(nosyid, 'username') == 'anonymous':
- continue
- # make sure they haven't seen the message already
- if not r.has_key(nosyid):
- # send it to them
- sendto.append(nosyid)
- recipients.append(nosyid)
-
- # generate a change note
- if oldvalues:
- note = self.generateChangeNote(nodeid, oldvalues)
- else:
- note = self.generateCreateNote(nodeid)
-
- # we have new recipients
- if sendto:
- # map userids to addresses
- sendto = [users.get(i, 'address') for i in sendto]
-
- # update the message's recipients list
- messages.set(msgid, recipients=recipients)
-
- # send the message
- self.send_message(nodeid, msgid, note, sendto)
-
- # XXX backwards compatibility - don't remove
- sendmessage = nosymessage
-
- def send_message(self, nodeid, msgid, note, sendto):
- '''Actually send the nominated message from this node to the sendto
- recipients, with the note appended.
- '''
- users = self.db.user
- messages = self.db.msg
- files = self.db.file
-
- # determine the messageid and inreplyto of the message
- inreplyto = messages.get(msgid, 'inreplyto')
- messageid = messages.get(msgid, 'messageid')
-
- # make up a messageid if there isn't one (web edit)
- if not messageid:
- # this is an old message that didn't get a messageid, so
- # create one
- messageid = "<%s.%s.%s%s@%s>"%(time.time(), random.random(),
- self.classname, nodeid, self.db.config.MAIL_DOMAIN)
- messages.set(msgid, messageid=messageid)
-
- # send an email to the people who missed out
- cn = self.classname
- title = self.get(nodeid, 'title') or '%s message copy'%cn
- # figure author information
- authid = messages.get(msgid, 'author')
- authname = users.get(authid, 'realname')
- if not authname:
- authname = users.get(authid, 'username')
- authaddr = users.get(authid, 'address')
- if authaddr:
- authaddr = ' <%s>'%authaddr
- else:
- authaddr = ''
-
- # make the message body
- m = ['']
-
- # put in roundup's signature
- if self.db.config.EMAIL_SIGNATURE_POSITION == 'top':
- m.append(self.email_signature(nodeid, msgid))
-
- # add author information
- if len(self.get(nodeid,'messages')) == 1:
- m.append("New submission from %s%s:"%(authname, authaddr))
- else:
- m.append("%s%s added the comment:"%(authname, authaddr))
- m.append('')
-
- # add the content
- m.append(messages.get(msgid, 'content'))
-
- # add the change note
- if note:
- m.append(note)
-
- # put in roundup's signature
- if self.db.config.EMAIL_SIGNATURE_POSITION == 'bottom':
- m.append(self.email_signature(nodeid, msgid))
-
- # encode the content as quoted-printable
- content = cStringIO.StringIO('\n'.join(m))
- content_encoded = cStringIO.StringIO()
- quopri.encode(content, content_encoded, 0)
- content_encoded = content_encoded.getvalue()
-
- # get the files for this message
- message_files = messages.get(msgid, 'files')
-
- # make sure the To line is always the same (for testing mostly)
- sendto.sort()
-
- # create the message
- message = cStringIO.StringIO()
- writer = MimeWriter.MimeWriter(message)
- writer.addheader('Subject', '[%s%s] %s'%(cn, nodeid, title))
- writer.addheader('To', ', '.join(sendto))
- writer.addheader('From', '%s <%s>'%(authname,
- self.db.config.ISSUE_TRACKER_EMAIL))
- writer.addheader('Reply-To', '%s <%s>'%(self.db.config.INSTANCE_NAME,
- self.db.config.ISSUE_TRACKER_EMAIL))
- writer.addheader('MIME-Version', '1.0')
- if messageid:
- writer.addheader('Message-Id', messageid)
- if inreplyto:
- writer.addheader('In-Reply-To', inreplyto)
-
- # add a uniquely Roundup header to help filtering
- writer.addheader('X-Roundup-Name', self.db.config.INSTANCE_NAME)
-
- # attach files
- if message_files:
- part = writer.startmultipartbody('mixed')
- part = writer.nextpart()
- part.addheader('Content-Transfer-Encoding', 'quoted-printable')
- body = part.startbody('text/plain')
- body.write(content_encoded)
- for fileid in message_files:
- name = files.get(fileid, 'name')
- mime_type = files.get(fileid, 'type')
- content = files.get(fileid, 'content')
- part = writer.nextpart()
- if mime_type == 'text/plain':
- part.addheader('Content-Disposition',
- 'attachment;\n filename="%s"'%name)
- part.addheader('Content-Transfer-Encoding', '7bit')
- body = part.startbody('text/plain')
- body.write(content)
- else:
- # some other type, so encode it
- if not mime_type:
- # this should have been done when the file was saved
- mime_type = mimetypes.guess_type(name)[0]
- if mime_type is None:
- mime_type = 'application/octet-stream'
- part.addheader('Content-Disposition',
- 'attachment;\n filename="%s"'%name)
- part.addheader('Content-Transfer-Encoding', 'base64')
- body = part.startbody(mime_type)
- body.write(base64.encodestring(content))
- writer.lastpart()
- else:
- writer.addheader('Content-Transfer-Encoding', 'quoted-printable')
- body = writer.startbody('text/plain')
- body.write(content_encoded)
-
- # now try to send the message
- if SENDMAILDEBUG:
- open(SENDMAILDEBUG, 'w').write('FROM: %s\nTO: %s\n%s\n'%(
- self.db.config.ADMIN_EMAIL,
- ', '.join(sendto),message.getvalue()))
- else:
- try:
- # send the message as admin so bounces are sent there
- # instead of to roundup
- smtp = smtplib.SMTP(self.db.config.MAILHOST)
- smtp.sendmail(self.db.config.ADMIN_EMAIL, sendto,
- message.getvalue())
- except socket.error, value:
- raise MessageSendError, \
- "Couldn't send confirmation email: mailhost %s"%value
- except smtplib.SMTPException, value:
- raise MessageSendError, \
- "Couldn't send confirmation email: %s"%value
-
- def email_signature(self, nodeid, msgid):
- ''' Add a signature to the e-mail with some useful information
- '''
- web = self.db.config.ISSUE_TRACKER_WEB + 'issue'+ nodeid
- email = '"%s" <%s>'%(self.db.config.INSTANCE_NAME,
- self.db.config.ISSUE_TRACKER_EMAIL)
- line = '_' * max(len(web), len(email))
- return '%s\n%s\n%s\n%s'%(line, email, web, line)
-
- def generateCreateNote(self, nodeid):
- """Generate a create note that lists initial property values
- """
- cn = self.classname
- cl = self.db.classes[cn]
- props = cl.getprops(protected=0)
-
- # list the values
- m = []
- l = props.items()
- l.sort()
- for propname, prop in l:
- value = cl.get(nodeid, propname, None)
- # skip boring entries
- if not value:
+CURVERSION = 1
+
+class Indexer(indexer.Indexer):
+ disallows = {'THE':1, 'THIS':1, 'ZZZ':1, 'THAT':1, 'WITH':1}
+ def __init__(self, path, datadb):
+ self.db = metakit.storage(os.path.join(path, 'index.mk4'), 1)
+ self.datadb = datadb
+ self.reindex = 0
+ v = self.db.view('version')
+ if not v.structure():
+ v = self.db.getas('version[vers:I]')
+ self.db.commit()
+ v.append(vers=CURVERSION)
+ self.reindex = 1
+ elif v[0].vers != CURVERSION:
+ v[0].vers = CURVERSION
+ self.reindex = 1
+ if self.reindex:
+ self.db.getas('ids[tblid:I,nodeid:I,propid:I]')
+ self.db.getas('index[word:S,hits[pos:I]]')
+ self.db.commit()
+ self.reindex = 1
+ self.changed = 0
+ self.propcache = {}
+ def force_reindex(self):
+ v = self.db.view('ids')
+ v[:] = []
+ v = self.db.view('index')
+ v[:] = []
+ self.db.commit()
+ self.reindex = 1
+ def should_reindex(self):
+ return self.reindex
+ def _getprops(self, classname):
+ props = self.propcache.get(classname, None)
+ if props is None:
+ props = self.datadb.view(classname).structure()
+ props = [prop.name for prop in props]
+ self.propcache[classname] = props
+ return props
+ def _getpropid(self, classname, propname):
+ return self._getprops(classname).index(propname)
+ def _getpropname(self, classname, propid):
+ return self._getprops(classname)[propid]
+ def add_text(self, identifier, text, mime_type='text/plain'):
+ if mime_type != 'text/plain':
+ return
+ classname, nodeid, property = identifier
+ tbls = self.datadb.view('tables')
+ tblid = tbls.find(name=classname)
+ if tblid < 0:
+ raise KeyError, "unknown class %r"%classname
+ nodeid = int(nodeid)
+ propid = self._getpropid(classname, property)
+ pos = self.db.view('ids').append(tblid=tblid,nodeid=nodeid,propid=propid)
+
+ wordlist = re.findall(r'\b\w{3,25}\b', text)
+ words = {}
+ for word in wordlist:
+ word = word.upper()
+ if not self.disallows.has_key(word):
+ words[word] = 1
+ words = words.keys()
+
+ index = self.db.view('index').ordered(1)
+ for word in words:
+ ndx = index.find(word=word)
+ if ndx < 0:
+ ndx = index.append(word=word)
+ hits = index[ndx].hits
+ if len(hits)==0 or hits.find(pos=pos) < 0:
+ hits.append(pos=pos)
+ self.changed = 1
+ def find(self, wordlist):
+ hits = None
+ index = self.db.view('index').ordered(1)
+ for word in wordlist:
+ if not 2 < len(word) < 26:
continue
- if isinstance(prop, hyperdb.Link):
- link = self.db.classes[prop.classname]
- if value:
- key = link.labelprop(default_to_id=1)
- if key:
- value = link.get(value, key)
- else:
- value = ''
- elif isinstance(prop, hyperdb.Multilink):
- if value is None: value = []
- l = []
- link = self.db.classes[prop.classname]
- key = link.labelprop(default_to_id=1)
- if key:
- value = [link.get(entry, key) for entry in value]
- value.sort()
- value = ', '.join(value)
- m.append('%s: %s'%(propname, value))
- m.insert(0, '----------')
- m.insert(0, '')
- return '\n'.join(m)
-
- def generateChangeNote(self, nodeid, oldvalues):
- """Generate a change note that lists property changes
- """
- cn = self.classname
- cl = self.db.classes[cn]
- changed = {}
- props = cl.getprops(protected=0)
-
- # determine what changed
- for key in oldvalues.keys():
- if key in ['files','messages']: continue
- new_value = cl.get(nodeid, key)
- # the old value might be non existent
- try:
- old_value = oldvalues[key]
- if type(new_value) is type([]):
- new_value.sort()
- old_value.sort()
- if new_value != old_value:
- changed[key] = old_value
- except:
- changed[key] = new_value
-
- # list the changes
- m = []
- l = changed.items()
- l.sort()
- for propname, oldvalue in l:
- prop = props[propname]
- value = cl.get(nodeid, propname, None)
- if isinstance(prop, hyperdb.Link):
- link = self.db.classes[prop.classname]
- key = link.labelprop(default_to_id=1)
- if key:
- if value:
- value = link.get(value, key)
- else:
- value = ''
- if oldvalue:
- oldvalue = link.get(oldvalue, key)
- else:
- oldvalue = ''
- change = '%s -> %s'%(oldvalue, value)
- elif isinstance(prop, hyperdb.Multilink):
- change = ''
- if value is None: value = []
- if oldvalue is None: oldvalue = []
- l = []
- link = self.db.classes[prop.classname]
- key = link.labelprop(default_to_id=1)
- # check for additions
- for entry in value:
- if entry in oldvalue: continue
- if key:
- l.append(link.get(entry, key))
- else:
- l.append(entry)
- if l:
- change = '+%s'%(', '.join(l))
- l = []
- # check for removals
- for entry in oldvalue:
- if entry in value: continue
- if key:
- l.append(link.get(entry, key))
- else:
- l.append(entry)
- if l:
- change += ' -%s'%(', '.join(l))
+ ndx = index.find(word=word)
+ if ndx < 0:
+ return {}
+ if hits is None:
+ hits = index[ndx].hits
else:
- change = '%s -> %s'%(oldvalue, value)
- m.append('%s: %s'%(propname, change))
- if m:
- m.insert(0, '----------')
- m.insert(0, '')
- return '\n'.join(m)
+ hits = hits.intersect(index[ndx].hits)
+ if len(hits) == 0:
+ return {}
+ if hits is None:
+ return {}
+ rslt = {}
+ ids = self.db.view('ids').remapwith(hits)
+ tbls = self.datadb.view('tables')
+ for i in range(len(ids)):
+ hit = ids[i]
+ classname = tbls[hit.tblid].name
+ nodeid = str(hit.nodeid)
+ property = self._getpropname(classname, hit.propid)
+ rslt[i] = (classname, nodeid, property)
+ return rslt
+ def save_index(self):
+ if self.changed:
+ self.db.commit()
+ self.changed = 0