962af890995b2e1666231b2da1beecf45eff2927
1 from roundup import hyperdb, date, password, roundupdb
2 import metakit
3 import re, marshal, os, sys, weakref, time, calendar
4 from roundup.roundup_indexer import RoundupIndexer
6 _instances = weakref.WeakValueDictionary()
8 def Database(config, journaltag=None):
9 if _instances.has_key(id(config)):
10 db = _instances[id(config)]
11 old = db.journaltag
12 db.journaltag = journaltag
13 if hasattr(db, 'curuserid'):
14 delattr(db, 'curuserid')
15 return db
16 else:
17 db = _Database(config, journaltag)
18 _instances[id(config)] = db
19 return db
21 class _Database(hyperdb.Database):
22 def __init__(self, config, journaltag=None):
23 self.config = config
24 self.journaltag = journaltag
25 self.classes = {}
26 self._classes = []
27 self.dirty = 0
28 self.__RW = 0
29 self._db = self.__open()
30 self.indexer = RoundupIndexer(self.config.DATABASE)
31 os.umask(0002)
33 # --- defined in ping's spec
34 def __getattr__(self, classname):
35 if classname == 'curuserid':
36 try:
37 self.curuserid = x = int(self.classes['user'].lookup(self.journaltag))
38 except KeyError:
39 x = 0
40 return x
41 return self.getclass(classname)
42 def getclass(self, classname):
43 return self.classes[classname]
44 def getclasses(self):
45 return self.classes.keys()
46 # --- end of ping's spec
47 # --- exposed methods
48 def commit(self):
49 if self.dirty:
50 if self.__RW:
51 self._db.commit()
52 for cl in self.classes.values():
53 cl._commit()
54 else:
55 raise RuntimeError, "metakit is open RO"
56 self.dirty = 0
57 def rollback(self):
58 if self.dirty:
59 for cl in self.classes.values():
60 cl._rollback()
61 self._db.rollback()
62 self.dirty = 0
63 def clear(self):
64 for cl in self.classes.values():
65 cl._clear()
66 def hasnode(self, classname, nodeid):
67 return self.getclass(clasname).hasnode(nodeid)
68 def pack(self, pack_before):
69 pass
70 def addclass(self, cl):
71 self.classes[cl.name] = cl
72 def addjournal(self, tablenm, nodeid, action, params):
73 tblid = self.tables.find(name=tablenm)
74 if tblid == -1:
75 tblid = self.tables.append(name=tablenm)
76 # tableid:I,nodeid:I,date:I,user:I,action:I,params:B
77 self.hist.append(tableid=tblid,
78 nodeid=int(nodeid),
79 date=int(time.time()),
80 action=action,
81 user = self.curuserid,
82 params = marshal.dumps(params))
83 def gethistory(self, tablenm, nodeid):
84 rslt = []
85 tblid = self.tables.find(name=tablenm)
86 if tblid == -1:
87 return rslt
88 q = self.hist.select(tableid=tblid, nodeid=int(nodeid))
89 i = 0
90 userclass = self.getclass('user')
91 for row in q:
92 try:
93 params = marshal.loads(row.params)
94 except ValueError:
95 print "history couldn't unmarshal %r" % row.params
96 params = {}
97 usernm = userclass.get(str(row.user), 'username')
98 dt = date.Date(time.gmtime(row.date))
99 rslt.append((i, dt, usernm, _actionnames[row.action], params))
100 i += 1
101 return rslt
103 def close(self):
104 import time
105 now = time.time
106 start = now()
107 for cl in self.classes.values():
108 cl.db = None
109 #self._db.rollback()
110 #print "pre-close cleanup of DB(%d) took %2.2f secs" % (self.__RW, now()-start)
111 self._db = None
112 #print "close of DB(%d) took %2.2f secs" % (self.__RW, now()-start)
113 self.classes = {}
114 try:
115 del _instances[id(self.config)]
116 except KeyError:
117 pass
118 self.__RW = 0
120 # --- internal
121 def __open(self):
122 self.dbnm = db = os.path.join(self.config.DATABASE, 'tracker.mk4')
123 self.fastopen = 0
124 if os.path.exists(db):
125 dbtm = os.path.getmtime(db)
126 pkgnm = self.config.__name__.split('.')[0]
127 schemamod = sys.modules.get(pkgnm+'.dbinit', None)
128 if schemamod:
129 if os.path.exists(schemamod.__file__):
130 schematm = os.path.getmtime(schemamod.__file__)
131 if schematm < dbtm:
132 # found schema mod - it's older than the db
133 self.fastopen = 1
134 else:
135 # can't find schemamod - must be frozen
136 self.fastopen = 1
137 else:
138 self.__RW = 1
139 db = metakit.storage(db, self.__RW)
140 hist = db.view('history')
141 tables = db.view('tables')
142 if not self.fastopen:
143 if not hist.structure():
144 hist = db.getas('history[tableid:I,nodeid:I,date:I,user:I,action:I,params:B]')
145 if not tables.structure():
146 tables = db.getas('tables[name:S]')
147 self.tables = tables
148 self.hist = hist
149 return db
150 def isReadOnly(self):
151 return self.__RW == 0
152 def getWriteAccess(self):
153 if self.journaltag is not None and self.__RW == 0:
154 now = time.time
155 start = now()
156 self._db = None
157 #print "closing the file took %2.2f secs" % (now()-start)
158 start = now()
159 self._db = metakit.storage(self.dbnm, 1)
160 self.__RW = 1
161 self.hist = self._db.view('history')
162 self.tables = self._db.view('tables')
163 #print "getting RW access took %2.2f secs" % (now()-start)
165 _STRINGTYPE = type('')
166 _LISTTYPE = type([])
167 _CREATE, _SET, _RETIRE, _LINK, _UNLINK = range(5)
169 _actionnames = {
170 _CREATE : 'create',
171 _SET : 'set',
172 _RETIRE : 'retire',
173 _LINK : 'link',
174 _UNLINK : 'unlink',
175 }
177 _marker = []
179 _ALLOWSETTINGPRIVATEPROPS = 0
181 class Class: # no, I'm not going to subclass the existing!
182 privateprops = None
183 def __init__(self, db, classname, **properties):
184 self.db = weakref.proxy(db)
185 self.name = classname
186 self.keyname = None
187 self.ruprops = properties
188 self.privateprops = { 'id' : hyperdb.String(),
189 'activity' : hyperdb.Date(),
190 'creation' : hyperdb.Date(),
191 'creator' : hyperdb.Link('user') }
192 self.auditors = {'create': [], 'set': [], 'retire': []} # event -> list of callables
193 self.reactors = {'create': [], 'set': [], 'retire': []} # ditto
194 view = self.__getview()
195 self.maxid = 1
196 if view:
197 self.maxid = view[-1].id + 1
198 self.uncommitted = {}
199 self.rbactions = []
200 # people reach inside!!
201 self.properties = self.ruprops
202 self.db.addclass(self)
203 self.idcache = {}
205 # --- the roundup.Class methods
206 def audit(self, event, detector):
207 l = self.auditors[event]
208 if detector not in l:
209 self.auditors[event].append(detector)
210 def fireAuditors(self, action, nodeid, newvalues):
211 for audit in self.auditors[action]:
212 audit(self.db, self, nodeid, newvalues)
213 def fireReactors(self, action, nodeid, oldvalues):
214 for react in self.reactors[action]:
215 react(self.db, self, nodeid, oldvalues)
216 def react(self, event, detector):
217 l = self.reactors[event]
218 if detector not in l:
219 self.reactors[event].append(detector)
220 # --- the hyperdb.Class methods
221 def create(self, **propvalues):
222 rowdict = {}
223 rowdict['id'] = newid = self.maxid
224 self.maxid += 1
225 ndx = self.getview(1).append(rowdict)
226 propvalues['#ISNEW'] = 1
227 try:
228 self.set(str(newid), **propvalues)
229 except Exception:
230 self.maxid -= 1
231 raise
232 return str(newid)
234 def get(self, nodeid, propname, default=_marker, cache=1):
235 # default and cache aren't in the spec
236 # cache=0 means "original value"
238 view = self.getview()
239 id = int(nodeid)
240 if cache == 0:
241 oldnode = self.uncommitted.get(id, None)
242 if oldnode and oldnode.has_key(propname):
243 return oldnode[propname]
244 ndx = self.idcache.get(id, None)
245 if ndx is None:
246 ndx = view.find(id=id)
247 if ndx < 0:
248 raise IndexError, "%s has no node %s" % (self.name, nodeid)
249 self.idcache[id] = ndx
250 raw = getattr(view[ndx], propname)
251 rutyp = self.ruprops.get(propname, None)
252 if rutyp is None:
253 rutyp = self.privateprops[propname]
254 converter = _converters.get(rutyp.__class__, None)
255 if converter:
256 raw = converter(raw)
257 return raw
259 def set(self, nodeid, **propvalues):
261 isnew = 0
262 if propvalues.has_key('#ISNEW'):
263 isnew = 1
264 del propvalues['#ISNEW']
265 if not propvalues:
266 return
267 if propvalues.has_key('id'):
268 raise KeyError, '"id" is reserved'
269 if self.db.journaltag is None:
270 raise DatabaseError, 'Database open read-only'
271 view = self.getview(1)
272 # node must exist & not be retired
273 id = int(nodeid)
274 ndx = view.find(id=id)
275 if ndx < 0:
276 raise IndexError, "%s has no node %s" % (self.name, nodeid)
277 row = view[ndx]
278 if row._isdel:
279 raise IndexError, "%s has no node %s" % (self.name, nodeid)
280 oldnode = self.uncommitted.setdefault(id, {})
281 changes = {}
283 for key, value in propvalues.items():
284 # this will raise the KeyError if the property isn't valid
285 # ... we don't use getprops() here because we only care about
286 # the writeable properties.
287 if _ALLOWSETTINGPRIVATEPROPS:
288 prop = self.ruprops.get(key, None)
289 if not prop:
290 prop = self.privateprops[key]
291 else:
292 prop = self.ruprops[key]
293 converter = _converters.get(prop.__class__, lambda v: v)
294 # if the value's the same as the existing value, no sense in
295 # doing anything
296 oldvalue = converter(getattr(row, key))
297 if value == oldvalue:
298 del propvalues[key]
299 continue
301 # check to make sure we're not duplicating an existing key
302 if key == self.keyname:
303 iv = self.getindexview(1)
304 ndx = iv.find(k=value)
305 if ndx == -1:
306 iv.append(k=value, i=row.id)
307 if not isnew:
308 ndx = iv.find(k=oldvalue)
309 if ndx > -1:
310 iv.delete(ndx)
311 else:
312 raise ValueError, 'node with key "%s" exists'%value
314 # do stuff based on the prop type
315 if isinstance(prop, hyperdb.Link):
316 link_class = prop.classname
317 # if it isn't a number, it's a key
318 if type(value) != _STRINGTYPE:
319 raise ValueError, 'link value must be String'
320 try:
321 int(value)
322 except ValueError:
323 try:
324 value = self.db.getclass(link_class).lookup(value)
325 except (TypeError, KeyError):
326 raise IndexError, 'new property "%s": %s not a %s'%(
327 key, value, prop.classname)
329 if not self.db.getclass(link_class).hasnode(value):
330 raise IndexError, '%s has no node %s'%(link_class, value)
332 setattr(row, key, int(value))
333 changes[key] = oldvalue
335 if prop.do_journal:
336 # register the unlink with the old linked node
337 if oldvalue:
338 self.db.addjournal(link_class, value, _UNLINK, (self.name, str(row.id), key))
340 # register the link with the newly linked node
341 if value:
342 self.db.addjournal(link_class, value, _LINK, (self.name, str(row.id), key))
344 elif isinstance(prop, hyperdb.Multilink):
345 if type(value) != _LISTTYPE:
346 raise TypeError, 'new property "%s" not a list of ids'%key
347 link_class = prop.classname
348 l = []
349 for entry in value:
350 if type(entry) != _STRINGTYPE:
351 raise ValueError, 'new property "%s" link value ' \
352 'must be a string'%key
353 # if it isn't a number, it's a key
354 try:
355 int(entry)
356 except ValueError:
357 try:
358 entry = self.db.getclass(link_class).lookup(entry)
359 except (TypeError, KeyError):
360 raise IndexError, 'new property "%s": %s not a %s'%(
361 key, entry, prop.classname)
362 l.append(entry)
363 propvalues[key] = value = l
365 # handle removals
366 rmvd = []
367 for id in oldvalue:
368 if id not in value:
369 rmvd.append(id)
370 # register the unlink with the old linked node
371 if prop.do_journal:
372 self.db.addjournal(link_class, id, _UNLINK, (self.name, str(row.id), key))
374 # handle additions
375 adds = []
376 for id in value:
377 if id not in oldvalue:
378 if not self.db.getclass(link_class).hasnode(id):
379 raise IndexError, '%s has no node %s'%(
380 link_class, id)
381 adds.append(id)
382 # register the link with the newly linked node
383 if prop.do_journal:
384 self.db.addjournal(link_class, id, _LINK, (self.name, str(row.id), key))
386 sv = getattr(row, key)
387 i = 0
388 while i < len(sv):
389 if str(sv[i].fid) in rmvd:
390 sv.delete(i)
391 else:
392 i += 1
393 for id in adds:
394 sv.append(fid=int(id))
395 changes[key] = oldvalue
398 elif isinstance(prop, hyperdb.String):
399 if value is not None and type(value) != _STRINGTYPE:
400 raise TypeError, 'new property "%s" not a string'%key
401 setattr(row, key, value)
402 changes[key] = oldvalue
403 if hasattr(prop, 'isfilename') and prop.isfilename:
404 propvalues[key] = os.path.basename(value)
406 elif isinstance(prop, hyperdb.Password):
407 if not isinstance(value, password.Password):
408 raise TypeError, 'new property "%s" not a Password'% key
409 setattr(row, key, str(value))
410 changes[key] = str(oldvalue)
411 propvalues[key] = str(value)
413 elif value is not None and isinstance(prop, hyperdb.Date):
414 if not isinstance(value, date.Date):
415 raise TypeError, 'new property "%s" not a Date'% key
416 setattr(row, key, int(calendar.timegm(value.get_tuple())))
417 changes[key] = str(oldvalue)
418 propvalues[key] = str(value)
420 elif value is not None and isinstance(prop, hyperdb.Interval):
421 if not isinstance(value, date.Interval):
422 raise TypeError, 'new property "%s" not an Interval'% key
423 setattr(row, key, str(value))
424 changes[key] = str(oldvalue)
425 propvalues[key] = str(value)
427 oldnode[key] = oldvalue
429 # nothing to do?
430 if not propvalues:
431 return
432 if not row.activity:
433 row.activity = int(time.time())
434 if isnew:
435 if not row.creation:
436 row.creation = int(time.time())
437 if not row.creator:
438 row.creator = self.db.curuserid
440 self.db.dirty = 1
441 if isnew:
442 self.db.addjournal(self.name, nodeid, _CREATE, {})
443 else:
444 self.db.addjournal(self.name, nodeid, _SET, changes)
446 def retire(self, nodeid):
447 view = self.getview(1)
448 ndx = view.find(id=int(nodeid))
449 if ndx < 0:
450 raise KeyError, "nodeid %s not found" % nodeid
451 row = view[ndx]
452 oldvalues = self.uncommitted.setdefault(row.id, {})
453 oldval = oldvalues['_isdel'] = row._isdel
454 row._isdel = 1
455 self.db.addjournal(self.name, nodeid, _RETIRE, {})
456 iv = self.getindexview(1)
457 ndx = iv.find(k=getattr(row, self.keyname),i=row.id)
458 if ndx > -1:
459 iv.delete(ndx)
460 self.db.dirty = 1
461 def history(self, nodeid):
462 return self.db.gethistory(self.name, nodeid)
463 def setkey(self, propname):
464 if self.keyname:
465 if propname == self.keyname:
466 return
467 raise ValueError, "%s already indexed on %s" % (self.name, self.keyname)
468 # first setkey for this run
469 self.keyname = propname
470 iv = self.db._db.view('_%s' % self.name)
471 if self.db.fastopen or iv.structure():
472 return
473 # very first setkey ever
474 iv = self.db._db.getas('_%s[k:S,i:I]' % self.name)
475 iv = iv.ordered(1)
476 #XXX
477 print "setkey building index"
478 for row in self.getview():
479 iv.append(k=getattr(row, propname), i=row.id)
480 def getkey(self):
481 return self.keyname
482 def lookup(self, keyvalue):
483 if type(keyvalue) is not _STRINGTYPE:
484 raise TypeError, "%r is not a string" % keyvalue
485 iv = self.getindexview()
486 if iv:
487 ndx = iv.find(k=keyvalue)
488 if ndx > -1:
489 return str(iv[ndx].i)
490 else:
491 view = self.getview()
492 ndx = view.find({self.keyname:keyvalue, '_isdel':0})
493 if ndx > -1:
494 return str(view[ndx].id)
495 raise KeyError, keyvalue
496 def find(self, **propspec):
497 """Get the ids of nodes in this class which link to the given nodes.
499 'propspec' consists of keyword args propname={nodeid:1,}
500 'propname' must be the name of a property in this class, or a
501 KeyError is raised. That property must be a Link or Multilink
502 property, or a TypeError is raised.
503 Any node in this class whose propname property links to any of the
504 nodeids will be returned. Used by the full text indexing, which knows
505 that "foo" occurs in msg1, msg3 and file7; so we have hits on these issues:
506 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
507 """
508 propspec = propspec.items()
509 for propname, nodeid in propspec:
510 # check the prop is OK
511 prop = self.ruprops[propname]
512 if not isinstance(prop, hyperdb.Link) and not isinstance(prop, hyperdb.Multilink):
513 raise TypeError, "'%s' not a Link/Multilink property"%propname
515 vws = []
516 for propname, ids in propspec:
517 prop = self.ruprops[propname]
518 view = self.getview()
519 if isinstance(prop, hyperdb.Multilink):
520 view = view.flatten(getattr(view, propname))
521 def ff(row, nm=propname, ids=ids):
522 return ids.has_key(str(row.fid))
523 else:
524 def ff(row, nm=propname, ids=ids):
525 return ids.has_key(str(getattr(row, nm)))
526 ndxview = view.filter(ff)
527 vws.append(ndxview.unique())
528 ndxview = vws[0]
529 for v in vws[1:]:
530 ndxview = ndxview.union(v)
531 view = view.remapwith(ndxview)
532 rslt = []
533 for row in view:
534 rslt.append(str(row.id))
535 return rslt
538 def list(self):
539 l = []
540 for row in self.getview().select(_isdel=0):
541 l.append(str(row.id))
542 return l
543 def count(self):
544 return len(self.getview())
545 def getprops(self, protected=1):
546 # protected is not in ping's spec
547 allprops = self.ruprops.copy()
548 if protected and self.privateprops is not None:
549 allprops.update(self.privateprops)
550 return allprops
551 def addprop(self, **properties):
552 for key in properties.keys():
553 if self.ruprops.has_key(key):
554 raise ValueError, "%s is already a property of %s" % (key, self.name)
555 self.ruprops.update(properties)
556 view = self.__getview()
557 # ---- end of ping's spec
558 def filter(self, search_matches, filterspec, sort, group):
559 # search_matches is None or a set (dict of {nodeid: {propname:[nodeid,...]}})
560 # filterspec is a dict {propname:value}
561 # sort and group are lists of propnames
563 where = {'_isdel':0}
564 mlcriteria = {}
565 regexes = {}
566 orcriteria = {}
567 for propname, value in filterspec.items():
568 prop = self.ruprops.get(propname, None)
569 if prop is None:
570 prop = self.privateprops[propname]
571 if isinstance(prop, hyperdb.Multilink):
572 if type(value) is not _LISTTYPE:
573 value = [value]
574 # transform keys to ids
575 u = []
576 for item in value:
577 try:
578 item = int(item)
579 except (TypeError, ValueError):
580 item = int(self.db.getclass(prop.classname).lookup(item))
581 if item == -1:
582 item = 0
583 u.append(item)
584 mlcriteria[propname] = u
585 elif isinstance(prop, hyperdb.Link):
586 if type(value) is not _LISTTYPE:
587 value = [value]
588 # transform keys to ids
589 u = []
590 for item in value:
591 try:
592 item = int(item)
593 except (TypeError, ValueError):
594 item = int(self.db.getclass(prop.classname).lookup(item))
595 if item == -1:
596 item = 0
597 u.append(item)
598 if len(u) == 1:
599 where[propname] = u[0]
600 else:
601 orcriteria[propname] = u
602 elif isinstance(prop, hyperdb.String):
603 # simple glob searching
604 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', value)
605 v = v.replace('?', '.')
606 v = v.replace('*', '.*?')
607 regexes[propname] = re.compile(v, re.I)
608 elif propname == 'id':
609 where[propname] = int(value)
610 else:
611 where[propname] = str(value)
612 v = self.getview()
613 #print "filter start at %s" % time.time()
614 if where:
615 v = v.select(where)
616 #print "filter where at %s" % time.time()
618 if mlcriteria:
619 # multilink - if any of the nodeids required by the
620 # filterspec aren't in this node's property, then skip
621 # it
622 def ff(row, ml=mlcriteria):
623 for propname, values in ml.items():
624 sv = getattr(row, propname)
625 for id in values:
626 if sv.find(fid=id) == -1:
627 return 0
628 return 1
629 iv = v.filter(ff)
630 v = v.remapwith(iv)
632 #print "filter mlcrit at %s" % time.time()
634 if orcriteria:
635 def ff(row, crit=orcriteria):
636 for propname, allowed in crit.items():
637 val = getattr(row, propname)
638 if val not in allowed:
639 return 0
640 return 1
642 iv = v.filter(ff)
643 v = v.remapwith(iv)
645 #print "filter orcrit at %s" % time.time()
646 if regexes:
647 def ff(row, r=regexes):
648 for propname, regex in r.items():
649 val = getattr(row, propname)
650 if not regex.search(val):
651 return 0
652 return 1
654 iv = v.filter(ff)
655 v = v.remapwith(iv)
656 #print "filter regexs at %s" % time.time()
658 if sort or group:
659 sortspec = []
660 rev = []
661 for propname in group + sort:
662 isreversed = 0
663 if propname[0] == '-':
664 propname = propname[1:]
665 isreversed = 1
666 try:
667 prop = getattr(v, propname)
668 except AttributeError:
669 # I can't sort on 'activity', cause it's psuedo!!
670 continue
671 if isreversed:
672 rev.append(prop)
673 sortspec.append(prop)
674 v = v.sortrev(sortspec, rev)[:] #XXX Aaaabh
675 #print "filter sort at %s" % time.time()
677 rslt = []
678 for row in v:
679 id = str(row.id)
680 if search_matches is not None:
681 if search_matches.has_key(id):
682 rslt.append(id)
683 else:
684 rslt.append(id)
685 return rslt
687 def hasnode(self, nodeid):
688 return int(nodeid) < self.maxid
690 def labelprop(self, default_to_id=0):
691 ''' Return the property name for a label for the given node.
693 This method attempts to generate a consistent label for the node.
694 It tries the following in order:
695 1. key property
696 2. "name" property
697 3. "title" property
698 4. first property from the sorted property name list
699 '''
700 k = self.getkey()
701 if k:
702 return k
703 props = self.getprops()
704 if props.has_key('name'):
705 return 'name'
706 elif props.has_key('title'):
707 return 'title'
708 if default_to_id:
709 return 'id'
710 props = props.keys()
711 props.sort()
712 return props[0]
713 def stringFind(self, **requirements):
714 """Locate a particular node by matching a set of its String
715 properties in a caseless search.
717 If the property is not a String property, a TypeError is raised.
719 The return is a list of the id of all nodes that match.
720 """
721 for propname in requirements.keys():
722 prop = self.properties[propname]
723 if isinstance(not prop, hyperdb.String):
724 raise TypeError, "'%s' not a String property"%propname
725 requirements[propname] = requirements[propname].lower()
726 requirements['_isdel'] = 0
728 l = []
729 for row in self.getview().select(requirements):
730 l.append(str(row.id))
731 return l
733 def addjournal(self, nodeid, action, params):
734 self.db.addjournal(self.name, nodeid, action, params)
735 # --- used by Database
736 def _commit(self):
737 """ called post commit of the DB.
738 interested subclasses may override """
739 self.uncommitted = {}
740 self.rbactions = []
741 self.idcache = {}
742 def _rollback(self):
743 """ called pre rollback of the DB.
744 interested subclasses may override """
745 for action in self.rbactions:
746 action()
747 self.rbactions = []
748 self.uncommitted = {}
749 self.idcache = {}
750 def _clear(self):
751 view = self.getview(1)
752 if len(view):
753 view[:] = []
754 self.db.dirty = 1
755 iv = self.getindexview(1)
756 if iv:
757 iv[:] = []
758 def rollbackaction(self, action):
759 """ call this to register a callback called on rollback
760 callback is removed on end of transaction """
761 self.rbactions.append(action)
762 # --- internal
763 def __getview(self):
764 db = self.db._db
765 view = db.view(self.name)
766 if self.db.fastopen:
767 return view.ordered(1)
768 # is the definition the same?
769 for nm, rutyp in self.ruprops.items():
770 mkprop = getattr(view, nm, None)
771 if mkprop is None:
772 #print "%s missing prop %s (%s)" % (self.name, nm, rutyp.__class__.__name__)
773 break
774 if _typmap[rutyp.__class__] != mkprop.type:
775 #print "%s - prop %s (%s) has wrong mktyp (%s)" % (self.name, nm, rutyp.__class__.__name__, mkprop.type)
776 break
777 else:
778 return view.ordered(1)
779 # need to create or restructure the mk view
780 # id comes first, so MK will order it for us
781 self.db.dirty = 1
782 s = ["%s[id:I" % self.name]
783 for nm, rutyp in self.ruprops.items():
784 mktyp = _typmap[rutyp.__class__]
785 s.append('%s:%s' % (nm, mktyp))
786 if mktyp == 'V':
787 s[-1] += ('[fid:I]')
788 s.append('_isdel:I,activity:I,creation:I,creator:I]')
789 v = db.getas(','.join(s))
790 return v.ordered(1)
791 def getview(self, RW=0):
792 if RW and self.db.isReadOnly():
793 self.db.getWriteAccess()
794 return self.db._db.view(self.name).ordered(1)
795 def getindexview(self, RW=0):
796 if RW and self.db.isReadOnly():
797 self.db.getWriteAccess()
798 return self.db._db.view("_%s" % self.name).ordered(1)
800 def _fetchML(sv):
801 l = []
802 for row in sv:
803 if row.fid:
804 l.append(str(row.fid))
805 return l
807 def _fetchPW(s):
808 p = password.Password()
809 p.unpack(s)
810 return p
812 def _fetchLink(n):
813 return n and str(n) or None
815 def _fetchDate(n):
816 return date.Date(time.gmtime(n))
818 _converters = {
819 hyperdb.Date : _fetchDate,
820 hyperdb.Link : _fetchLink,
821 hyperdb.Multilink : _fetchML,
822 hyperdb.Interval : date.Interval,
823 hyperdb.Password : _fetchPW,
824 }
826 class FileName(hyperdb.String):
827 isfilename = 1
829 _typmap = {
830 FileName : 'S',
831 hyperdb.String : 'S',
832 hyperdb.Date : 'I',
833 hyperdb.Link : 'I',
834 hyperdb.Multilink : 'V',
835 hyperdb.Interval : 'S',
836 hyperdb.Password : 'S',
837 }
838 class FileClass(Class):
839 ' like Class but with a content property '
840 def __init__(self, db, classname, **properties):
841 properties['content'] = FileName()
842 Class.__init__(self, db, classname, **properties)
843 def get(self, nodeid, propname, default=_marker, cache=1):
844 x = Class.get(self, nodeid, propname, default, cache)
845 if propname == 'content':
846 if x.startswith('file:'):
847 fnm = x[5:]
848 try:
849 x = open(fnm, 'rb').read()
850 except Exception, e:
851 x = repr(e)
852 return x
853 def create(self, **propvalues):
854 content = propvalues['content']
855 del propvalues['content']
856 newid = Class.create(self, **propvalues)
857 if not content:
858 return newid
859 if content.startswith('/tracker/download.php?'):
860 self.set(newid, content='http://sourceforge.net'+content)
861 return newid
862 nm = bnm = '%s%s' % (self.name, newid)
863 sd = str(int(int(newid) / 1000))
864 d = os.path.join(self.db.config.DATABASE, 'files', self.name, sd)
865 if not os.path.exists(d):
866 os.makedirs(d)
867 nm = os.path.join(d, nm)
868 open(nm, 'wb').write(content)
869 self.set(newid, content = 'file:'+nm)
870 self.db.indexer.add_files(d, bnm)
871 self.db.indexer.save_index()
872 def undo(fnm=nm, action1=os.remove, indexer=self.db.indexer):
873 remove(fnm)
874 indexer.purge_entry(fnm, indexer.files, indexer.words)
875 self.rollbackaction(undo)
876 return newid
878 # Yuck - c&p to avoid getting hyperdb.Class
879 class IssueClass(Class):
881 # Overridden methods:
883 def __init__(self, db, classname, **properties):
884 """The newly-created class automatically includes the "messages",
885 "files", "nosy", and "superseder" properties. If the 'properties'
886 dictionary attempts to specify any of these properties or a
887 "creation" or "activity" property, a ValueError is raised."""
888 if not properties.has_key('title'):
889 properties['title'] = hyperdb.String()
890 if not properties.has_key('messages'):
891 properties['messages'] = hyperdb.Multilink("msg")
892 if not properties.has_key('files'):
893 properties['files'] = hyperdb.Multilink("file")
894 if not properties.has_key('nosy'):
895 properties['nosy'] = hyperdb.Multilink("user")
896 if not properties.has_key('superseder'):
897 properties['superseder'] = hyperdb.Multilink(classname)
898 Class.__init__(self, db, classname, **properties)
900 # New methods:
902 def addmessage(self, nodeid, summary, text):
903 """Add a message to an issue's mail spool.
905 A new "msg" node is constructed using the current date, the user that
906 owns the database connection as the author, and the specified summary
907 text.
909 The "files" and "recipients" fields are left empty.
911 The given text is saved as the body of the message and the node is
912 appended to the "messages" field of the specified issue.
913 """
915 def nosymessage(self, nodeid, msgid, oldvalues):
916 """Send a message to the members of an issue's nosy list.
918 The message is sent only to users on the nosy list who are not
919 already on the "recipients" list for the message.
921 These users are then added to the message's "recipients" list.
922 """
923 users = self.db.user
924 messages = self.db.msg
926 # figure the recipient ids
927 sendto = []
928 r = {}
929 recipients = messages.get(msgid, 'recipients')
930 for recipid in messages.get(msgid, 'recipients'):
931 r[recipid] = 1
933 # figure the author's id, and indicate they've received the message
934 authid = messages.get(msgid, 'author')
936 # possibly send the message to the author, as long as they aren't
937 # anonymous
938 if (self.db.config.MESSAGES_TO_AUTHOR == 'yes' and
939 users.get(authid, 'username') != 'anonymous'):
940 sendto.append(authid)
941 r[authid] = 1
943 # now figure the nosy people who weren't recipients
944 nosy = self.get(nodeid, 'nosy')
945 for nosyid in nosy:
946 # Don't send nosy mail to the anonymous user (that user
947 # shouldn't appear in the nosy list, but just in case they
948 # do...)
949 if users.get(nosyid, 'username') == 'anonymous':
950 continue
951 # make sure they haven't seen the message already
952 if not r.has_key(nosyid):
953 # send it to them
954 sendto.append(nosyid)
955 recipients.append(nosyid)
957 # generate a change note
958 if oldvalues:
959 note = self.generateChangeNote(nodeid, oldvalues)
960 else:
961 note = self.generateCreateNote(nodeid)
963 # we have new recipients
964 if sendto:
965 # map userids to addresses
966 sendto = [users.get(i, 'address') for i in sendto]
968 # update the message's recipients list
969 messages.set(msgid, recipients=recipients)
971 # send the message
972 self.send_message(nodeid, msgid, note, sendto)
974 # XXX backwards compatibility - don't remove
975 sendmessage = nosymessage
977 def send_message(self, nodeid, msgid, note, sendto):
978 '''Actually send the nominated message from this node to the sendto
979 recipients, with the note appended.
980 '''
981 users = self.db.user
982 messages = self.db.msg
983 files = self.db.file
985 # determine the messageid and inreplyto of the message
986 inreplyto = messages.get(msgid, 'inreplyto')
987 messageid = messages.get(msgid, 'messageid')
989 # make up a messageid if there isn't one (web edit)
990 if not messageid:
991 # this is an old message that didn't get a messageid, so
992 # create one
993 messageid = "<%s.%s.%s%s@%s>"%(time.time(), random.random(),
994 self.classname, nodeid, self.db.config.MAIL_DOMAIN)
995 messages.set(msgid, messageid=messageid)
997 # send an email to the people who missed out
998 cn = self.classname
999 title = self.get(nodeid, 'title') or '%s message copy'%cn
1000 # figure author information
1001 authid = messages.get(msgid, 'author')
1002 authname = users.get(authid, 'realname')
1003 if not authname:
1004 authname = users.get(authid, 'username')
1005 authaddr = users.get(authid, 'address')
1006 if authaddr:
1007 authaddr = ' <%s>'%authaddr
1008 else:
1009 authaddr = ''
1011 # make the message body
1012 m = ['']
1014 # put in roundup's signature
1015 if self.db.config.EMAIL_SIGNATURE_POSITION == 'top':
1016 m.append(self.email_signature(nodeid, msgid))
1018 # add author information
1019 if len(self.get(nodeid,'messages')) == 1:
1020 m.append("New submission from %s%s:"%(authname, authaddr))
1021 else:
1022 m.append("%s%s added the comment:"%(authname, authaddr))
1023 m.append('')
1025 # add the content
1026 m.append(messages.get(msgid, 'content'))
1028 # add the change note
1029 if note:
1030 m.append(note)
1032 # put in roundup's signature
1033 if self.db.config.EMAIL_SIGNATURE_POSITION == 'bottom':
1034 m.append(self.email_signature(nodeid, msgid))
1036 # encode the content as quoted-printable
1037 content = cStringIO.StringIO('\n'.join(m))
1038 content_encoded = cStringIO.StringIO()
1039 quopri.encode(content, content_encoded, 0)
1040 content_encoded = content_encoded.getvalue()
1042 # get the files for this message
1043 message_files = messages.get(msgid, 'files')
1045 # make sure the To line is always the same (for testing mostly)
1046 sendto.sort()
1048 # create the message
1049 message = cStringIO.StringIO()
1050 writer = MimeWriter.MimeWriter(message)
1051 writer.addheader('Subject', '[%s%s] %s'%(cn, nodeid, title))
1052 writer.addheader('To', ', '.join(sendto))
1053 writer.addheader('From', '%s <%s>'%(authname,
1054 self.db.config.ISSUE_TRACKER_EMAIL))
1055 writer.addheader('Reply-To', '%s <%s>'%(self.db.config.INSTANCE_NAME,
1056 self.db.config.ISSUE_TRACKER_EMAIL))
1057 writer.addheader('MIME-Version', '1.0')
1058 if messageid:
1059 writer.addheader('Message-Id', messageid)
1060 if inreplyto:
1061 writer.addheader('In-Reply-To', inreplyto)
1063 # add a uniquely Roundup header to help filtering
1064 writer.addheader('X-Roundup-Name', self.db.config.INSTANCE_NAME)
1066 # attach files
1067 if message_files:
1068 part = writer.startmultipartbody('mixed')
1069 part = writer.nextpart()
1070 part.addheader('Content-Transfer-Encoding', 'quoted-printable')
1071 body = part.startbody('text/plain')
1072 body.write(content_encoded)
1073 for fileid in message_files:
1074 name = files.get(fileid, 'name')
1075 mime_type = files.get(fileid, 'type')
1076 content = files.get(fileid, 'content')
1077 part = writer.nextpart()
1078 if mime_type == 'text/plain':
1079 part.addheader('Content-Disposition',
1080 'attachment;\n filename="%s"'%name)
1081 part.addheader('Content-Transfer-Encoding', '7bit')
1082 body = part.startbody('text/plain')
1083 body.write(content)
1084 else:
1085 # some other type, so encode it
1086 if not mime_type:
1087 # this should have been done when the file was saved
1088 mime_type = mimetypes.guess_type(name)[0]
1089 if mime_type is None:
1090 mime_type = 'application/octet-stream'
1091 part.addheader('Content-Disposition',
1092 'attachment;\n filename="%s"'%name)
1093 part.addheader('Content-Transfer-Encoding', 'base64')
1094 body = part.startbody(mime_type)
1095 body.write(base64.encodestring(content))
1096 writer.lastpart()
1097 else:
1098 writer.addheader('Content-Transfer-Encoding', 'quoted-printable')
1099 body = writer.startbody('text/plain')
1100 body.write(content_encoded)
1102 # now try to send the message
1103 if SENDMAILDEBUG:
1104 open(SENDMAILDEBUG, 'w').write('FROM: %s\nTO: %s\n%s\n'%(
1105 self.db.config.ADMIN_EMAIL,
1106 ', '.join(sendto),message.getvalue()))
1107 else:
1108 try:
1109 # send the message as admin so bounces are sent there
1110 # instead of to roundup
1111 smtp = smtplib.SMTP(self.db.config.MAILHOST)
1112 smtp.sendmail(self.db.config.ADMIN_EMAIL, sendto,
1113 message.getvalue())
1114 except socket.error, value:
1115 raise MessageSendError, \
1116 "Couldn't send confirmation email: mailhost %s"%value
1117 except smtplib.SMTPException, value:
1118 raise MessageSendError, \
1119 "Couldn't send confirmation email: %s"%value
1121 def email_signature(self, nodeid, msgid):
1122 ''' Add a signature to the e-mail with some useful information
1123 '''
1124 web = self.db.config.ISSUE_TRACKER_WEB + 'issue'+ nodeid
1125 email = '"%s" <%s>'%(self.db.config.INSTANCE_NAME,
1126 self.db.config.ISSUE_TRACKER_EMAIL)
1127 line = '_' * max(len(web), len(email))
1128 return '%s\n%s\n%s\n%s'%(line, email, web, line)
1130 def generateCreateNote(self, nodeid):
1131 """Generate a create note that lists initial property values
1132 """
1133 cn = self.classname
1134 cl = self.db.classes[cn]
1135 props = cl.getprops(protected=0)
1137 # list the values
1138 m = []
1139 l = props.items()
1140 l.sort()
1141 for propname, prop in l:
1142 value = cl.get(nodeid, propname, None)
1143 # skip boring entries
1144 if not value:
1145 continue
1146 if isinstance(prop, hyperdb.Link):
1147 link = self.db.classes[prop.classname]
1148 if value:
1149 key = link.labelprop(default_to_id=1)
1150 if key:
1151 value = link.get(value, key)
1152 else:
1153 value = ''
1154 elif isinstance(prop, hyperdb.Multilink):
1155 if value is None: value = []
1156 l = []
1157 link = self.db.classes[prop.classname]
1158 key = link.labelprop(default_to_id=1)
1159 if key:
1160 value = [link.get(entry, key) for entry in value]
1161 value.sort()
1162 value = ', '.join(value)
1163 m.append('%s: %s'%(propname, value))
1164 m.insert(0, '----------')
1165 m.insert(0, '')
1166 return '\n'.join(m)
1168 def generateChangeNote(self, nodeid, oldvalues):
1169 """Generate a change note that lists property changes
1170 """
1171 cn = self.classname
1172 cl = self.db.classes[cn]
1173 changed = {}
1174 props = cl.getprops(protected=0)
1176 # determine what changed
1177 for key in oldvalues.keys():
1178 if key in ['files','messages']: continue
1179 new_value = cl.get(nodeid, key)
1180 # the old value might be non existent
1181 try:
1182 old_value = oldvalues[key]
1183 if type(new_value) is type([]):
1184 new_value.sort()
1185 old_value.sort()
1186 if new_value != old_value:
1187 changed[key] = old_value
1188 except:
1189 changed[key] = new_value
1191 # list the changes
1192 m = []
1193 l = changed.items()
1194 l.sort()
1195 for propname, oldvalue in l:
1196 prop = props[propname]
1197 value = cl.get(nodeid, propname, None)
1198 if isinstance(prop, hyperdb.Link):
1199 link = self.db.classes[prop.classname]
1200 key = link.labelprop(default_to_id=1)
1201 if key:
1202 if value:
1203 value = link.get(value, key)
1204 else:
1205 value = ''
1206 if oldvalue:
1207 oldvalue = link.get(oldvalue, key)
1208 else:
1209 oldvalue = ''
1210 change = '%s -> %s'%(oldvalue, value)
1211 elif isinstance(prop, hyperdb.Multilink):
1212 change = ''
1213 if value is None: value = []
1214 if oldvalue is None: oldvalue = []
1215 l = []
1216 link = self.db.classes[prop.classname]
1217 key = link.labelprop(default_to_id=1)
1218 # check for additions
1219 for entry in value:
1220 if entry in oldvalue: continue
1221 if key:
1222 l.append(link.get(entry, key))
1223 else:
1224 l.append(entry)
1225 if l:
1226 change = '+%s'%(', '.join(l))
1227 l = []
1228 # check for removals
1229 for entry in oldvalue:
1230 if entry in value: continue
1231 if key:
1232 l.append(link.get(entry, key))
1233 else:
1234 l.append(entry)
1235 if l:
1236 change += ' -%s'%(', '.join(l))
1237 else:
1238 change = '%s -> %s'%(oldvalue, value)
1239 m.append('%s: %s'%(propname, change))
1240 if m:
1241 m.insert(0, '----------')
1242 m.insert(0, '')
1243 return '\n'.join(m)