1 from roundup import hyperdb, date, password, roundupdb
2 import metakit
3 import re, marshal, os, sys, weakref, time, calendar
4 from roundup.indexer import Indexer
6 _instances = weakref.WeakValueDictionary()
8 def Database(config, journaltag=None):
9 if _instances.has_key(id(config)):
10 db = _instances[id(config)]
11 old = db.journaltag
12 db.journaltag = journaltag
13 try:
14 delattr(db, 'curuserid')
15 except AttributeError:
16 pass
17 return db
18 else:
19 db = _Database(config, journaltag)
20 _instances[id(config)] = db
21 return db
23 class _Database(hyperdb.Database):
24 def __init__(self, config, journaltag=None):
25 self.config = config
26 self.journaltag = journaltag
27 self.classes = {}
28 self._classes = []
29 self.dirty = 0
30 self.__RW = 0
31 self._db = self.__open()
32 self.indexer = Indexer(self.config.DATABASE)
33 os.umask(0002)
34 def post_init(self):
35 if self.indexer.should_reindex():
36 self.reindex()
38 def reindex(self):
39 for klass in self.classes.values():
40 for nodeid in klass.list():
41 klass.index(nodeid)
42 self.indexer.save_index()
45 # --- defined in ping's spec
46 def __getattr__(self, classname):
47 if classname == 'curuserid':
48 try:
49 self.curuserid = x = int(self.classes['user'].lookup(self.journaltag))
50 except KeyError:
51 x = 0
52 return x
53 return self.getclass(classname)
54 def getclass(self, classname):
55 return self.classes[classname]
56 def getclasses(self):
57 return self.classes.keys()
58 # --- end of ping's spec
59 # --- exposed methods
60 def commit(self):
61 if self.dirty:
62 if self.__RW:
63 self._db.commit()
64 for cl in self.classes.values():
65 cl._commit()
66 self.indexer.save_index()
67 else:
68 raise RuntimeError, "metakit is open RO"
69 self.dirty = 0
70 def rollback(self):
71 if self.dirty:
72 for cl in self.classes.values():
73 cl._rollback()
74 self._db.rollback()
75 self.dirty = 0
76 def clear(self):
77 for cl in self.classes.values():
78 cl._clear()
79 def hasnode(self, classname, nodeid):
80 return self.getclass(clasname).hasnode(nodeid)
81 def pack(self, pack_before):
82 pass
83 def addclass(self, cl):
84 self.classes[cl.classname] = cl
85 def addjournal(self, tablenm, nodeid, action, params):
86 tblid = self.tables.find(name=tablenm)
87 if tblid == -1:
88 tblid = self.tables.append(name=tablenm)
89 # tableid:I,nodeid:I,date:I,user:I,action:I,params:B
90 self.hist.append(tableid=tblid,
91 nodeid=int(nodeid),
92 date=int(time.time()),
93 action=action,
94 user = self.curuserid,
95 params = marshal.dumps(params))
96 def gethistory(self, tablenm, nodeid):
97 rslt = []
98 tblid = self.tables.find(name=tablenm)
99 if tblid == -1:
100 return rslt
101 q = self.hist.select(tableid=tblid, nodeid=int(nodeid))
102 i = 0
103 userclass = self.getclass('user')
104 for row in q:
105 try:
106 params = marshal.loads(row.params)
107 except ValueError:
108 print "history couldn't unmarshal %r" % row.params
109 params = {}
110 usernm = userclass.get(str(row.user), 'username')
111 dt = date.Date(time.gmtime(row.date))
112 rslt.append((i, dt, usernm, _actionnames[row.action], params))
113 i += 1
114 return rslt
116 def close(self):
117 import time
118 now = time.time
119 start = now()
120 for cl in self.classes.values():
121 cl.db = None
122 #self._db.rollback()
123 #print "pre-close cleanup of DB(%d) took %2.2f secs" % (self.__RW, now()-start)
124 self._db = None
125 #print "close of DB(%d) took %2.2f secs" % (self.__RW, now()-start)
126 self.classes = {}
127 try:
128 del _instances[id(self.config)]
129 except KeyError:
130 pass
131 self.__RW = 0
133 # --- internal
134 def __open(self):
135 self.dbnm = db = os.path.join(self.config.DATABASE, 'tracker.mk4')
136 self.fastopen = 0
137 if os.path.exists(db):
138 dbtm = os.path.getmtime(db)
139 pkgnm = self.config.__name__.split('.')[0]
140 schemamod = sys.modules.get(pkgnm+'.dbinit', None)
141 if schemamod:
142 if os.path.exists(schemamod.__file__):
143 schematm = os.path.getmtime(schemamod.__file__)
144 if schematm < dbtm:
145 # found schema mod - it's older than the db
146 self.fastopen = 1
147 else:
148 # can't find schemamod - must be frozen
149 self.fastopen = 1
150 else:
151 self.__RW = 1
152 if not self.fastopen:
153 self.__RW = 1
154 db = metakit.storage(db, self.__RW)
155 hist = db.view('history')
156 tables = db.view('tables')
157 if not self.fastopen:
158 if not hist.structure():
159 hist = db.getas('history[tableid:I,nodeid:I,date:I,user:I,action:I,params:B]')
160 if not tables.structure():
161 tables = db.getas('tables[name:S]')
162 self.tables = tables
163 self.hist = hist
164 return db
165 def isReadOnly(self):
166 return self.__RW == 0
167 def getWriteAccess(self):
168 if self.journaltag is not None and self.__RW == 0:
169 now = time.time
170 start = now()
171 self._db = None
172 #print "closing the file took %2.2f secs" % (now()-start)
173 start = now()
174 self._db = metakit.storage(self.dbnm, 1)
175 self.__RW = 1
176 self.hist = self._db.view('history')
177 self.tables = self._db.view('tables')
178 #print "getting RW access took %2.2f secs" % (now()-start)
180 _STRINGTYPE = type('')
181 _LISTTYPE = type([])
182 _CREATE, _SET, _RETIRE, _LINK, _UNLINK = range(5)
184 _actionnames = {
185 _CREATE : 'create',
186 _SET : 'set',
187 _RETIRE : 'retire',
188 _LINK : 'link',
189 _UNLINK : 'unlink',
190 }
192 _marker = []
194 _ALLOWSETTINGPRIVATEPROPS = 0
196 class Class: # no, I'm not going to subclass the existing!
197 privateprops = None
198 def __init__(self, db, classname, **properties):
199 self.db = weakref.proxy(db)
200 self.classname = classname
201 self.keyname = None
202 self.ruprops = properties
203 self.privateprops = { 'id' : hyperdb.String(),
204 'activity' : hyperdb.Date(),
205 'creation' : hyperdb.Date(),
206 'creator' : hyperdb.Link('user') }
207 self.auditors = {'create': [], 'set': [], 'retire': []} # event -> list of callables
208 self.reactors = {'create': [], 'set': [], 'retire': []} # ditto
209 view = self.__getview()
210 self.maxid = 1
211 if view:
212 self.maxid = view[-1].id + 1
213 self.uncommitted = {}
214 self.rbactions = []
215 # people reach inside!!
216 self.properties = self.ruprops
217 self.db.addclass(self)
218 self.idcache = {}
220 # default is to journal changes
221 self.do_journal = 1
223 def enableJournalling(self):
224 '''Turn journalling on for this class
225 '''
226 self.do_journal = 1
228 def disableJournalling(self):
229 '''Turn journalling off for this class
230 '''
231 self.do_journal = 0
233 # --- the roundup.Class methods
234 def audit(self, event, detector):
235 l = self.auditors[event]
236 if detector not in l:
237 self.auditors[event].append(detector)
238 def fireAuditors(self, action, nodeid, newvalues):
239 for audit in self.auditors[action]:
240 audit(self.db, self, nodeid, newvalues)
241 def fireReactors(self, action, nodeid, oldvalues):
242 for react in self.reactors[action]:
243 react(self.db, self, nodeid, oldvalues)
244 def react(self, event, detector):
245 l = self.reactors[event]
246 if detector not in l:
247 self.reactors[event].append(detector)
248 # --- the hyperdb.Class methods
249 def create(self, **propvalues):
250 rowdict = {}
251 rowdict['id'] = newid = self.maxid
252 self.maxid += 1
253 ndx = self.getview(1).append(rowdict)
254 propvalues['#ISNEW'] = 1
255 try:
256 self.set(str(newid), **propvalues)
257 except Exception:
258 self.maxid -= 1
259 raise
260 return str(newid)
262 def get(self, nodeid, propname, default=_marker, cache=1):
263 # default and cache aren't in the spec
264 # cache=0 means "original value"
266 view = self.getview()
267 id = int(nodeid)
268 if cache == 0:
269 oldnode = self.uncommitted.get(id, None)
270 if oldnode and oldnode.has_key(propname):
271 return oldnode[propname]
272 ndx = self.idcache.get(id, None)
273 if ndx is None:
274 ndx = view.find(id=id)
275 if ndx < 0:
276 raise IndexError, "%s has no node %s" % (self.classname, nodeid)
277 self.idcache[id] = ndx
278 try:
279 raw = getattr(view[ndx], propname)
280 except AttributeError:
281 raise KeyError, propname
282 rutyp = self.ruprops.get(propname, None)
283 if rutyp is None:
284 rutyp = self.privateprops[propname]
285 converter = _converters.get(rutyp.__class__, None)
286 if converter:
287 raw = converter(raw)
288 return raw
290 def set(self, nodeid, **propvalues):
291 isnew = 0
292 if propvalues.has_key('#ISNEW'):
293 isnew = 1
294 del propvalues['#ISNEW']
295 if not propvalues:
296 return
297 if propvalues.has_key('id'):
298 raise KeyError, '"id" is reserved'
299 if self.db.journaltag is None:
300 raise DatabaseError, 'Database open read-only'
301 view = self.getview(1)
302 # node must exist & not be retired
303 id = int(nodeid)
304 ndx = view.find(id=id)
305 if ndx < 0:
306 raise IndexError, "%s has no node %s" % (self.classname, nodeid)
307 row = view[ndx]
308 if row._isdel:
309 raise IndexError, "%s has no node %s" % (self.classname, nodeid)
310 oldnode = self.uncommitted.setdefault(id, {})
311 changes = {}
313 for key, value in propvalues.items():
314 # this will raise the KeyError if the property isn't valid
315 # ... we don't use getprops() here because we only care about
316 # the writeable properties.
317 if _ALLOWSETTINGPRIVATEPROPS:
318 prop = self.ruprops.get(key, None)
319 if not prop:
320 prop = self.privateprops[key]
321 else:
322 prop = self.ruprops[key]
323 converter = _converters.get(prop.__class__, lambda v: v)
324 # if the value's the same as the existing value, no sense in
325 # doing anything
326 oldvalue = converter(getattr(row, key))
327 if value == oldvalue:
328 del propvalues[key]
329 continue
331 # check to make sure we're not duplicating an existing key
332 if key == self.keyname:
333 iv = self.getindexview(1)
334 ndx = iv.find(k=value)
335 if ndx == -1:
336 iv.append(k=value, i=row.id)
337 if not isnew:
338 ndx = iv.find(k=oldvalue)
339 if ndx > -1:
340 iv.delete(ndx)
341 else:
342 raise ValueError, 'node with key "%s" exists'%value
344 # do stuff based on the prop type
345 if isinstance(prop, hyperdb.Link):
346 link_class = prop.classname
347 # if it isn't a number, it's a key
348 if type(value) != _STRINGTYPE:
349 raise ValueError, 'link value must be String'
350 try:
351 int(value)
352 except ValueError:
353 try:
354 value = self.db.getclass(link_class).lookup(value)
355 except (TypeError, KeyError):
356 raise IndexError, 'new property "%s": %s not a %s'%(
357 key, value, prop.classname)
359 if not self.db.getclass(link_class).hasnode(value):
360 raise IndexError, '%s has no node %s'%(link_class, value)
362 setattr(row, key, int(value))
363 changes[key] = oldvalue
365 if self.do_journal and prop.do_journal:
366 # register the unlink with the old linked node
367 if oldvalue:
368 self.db.addjournal(link_class, value, _UNLINK, (self.classname, str(row.id), key))
370 # register the link with the newly linked node
371 if value:
372 self.db.addjournal(link_class, value, _LINK, (self.classname, str(row.id), key))
374 elif isinstance(prop, hyperdb.Multilink):
375 if type(value) != _LISTTYPE:
376 raise TypeError, 'new property "%s" not a list of ids'%key
377 link_class = prop.classname
378 l = []
379 for entry in value:
380 if type(entry) != _STRINGTYPE:
381 raise ValueError, 'new property "%s" link value ' \
382 'must be a string'%key
383 # if it isn't a number, it's a key
384 try:
385 int(entry)
386 except ValueError:
387 try:
388 entry = self.db.getclass(link_class).lookup(entry)
389 except (TypeError, KeyError):
390 raise IndexError, 'new property "%s": %s not a %s'%(
391 key, entry, prop.classname)
392 l.append(entry)
393 propvalues[key] = value = l
395 # handle removals
396 rmvd = []
397 for id in oldvalue:
398 if id not in value:
399 rmvd.append(id)
400 # register the unlink with the old linked node
401 if self.do_journal and prop.do_journal:
402 self.db.addjournal(link_class, id, _UNLINK, (self.classname, str(row.id), key))
404 # handle additions
405 adds = []
406 for id in value:
407 if id not in oldvalue:
408 if not self.db.getclass(link_class).hasnode(id):
409 raise IndexError, '%s has no node %s'%(
410 link_class, id)
411 adds.append(id)
412 # register the link with the newly linked node
413 if self.do_journal and prop.do_journal:
414 self.db.addjournal(link_class, id, _LINK, (self.classname, str(row.id), key))
416 sv = getattr(row, key)
417 i = 0
418 while i < len(sv):
419 if str(sv[i].fid) in rmvd:
420 sv.delete(i)
421 else:
422 i += 1
423 for id in adds:
424 sv.append(fid=int(id))
425 changes[key] = oldvalue
428 elif isinstance(prop, hyperdb.String):
429 if value is not None and type(value) != _STRINGTYPE:
430 raise TypeError, 'new property "%s" not a string'%key
431 setattr(row, key, value)
432 changes[key] = oldvalue
433 if hasattr(prop, 'isfilename') and prop.isfilename:
434 propvalues[key] = os.path.basename(value)
435 if prop.indexme:
436 self.db.indexer.add_text((self.classname, nodeid, key), value, 'text/plain')
438 elif isinstance(prop, hyperdb.Password):
439 if not isinstance(value, password.Password):
440 raise TypeError, 'new property "%s" not a Password'% key
441 setattr(row, key, str(value))
442 changes[key] = str(oldvalue)
443 propvalues[key] = str(value)
445 elif value is not None and isinstance(prop, hyperdb.Date):
446 if not isinstance(value, date.Date):
447 raise TypeError, 'new property "%s" not a Date'% key
448 setattr(row, key, int(calendar.timegm(value.get_tuple())))
449 changes[key] = str(oldvalue)
450 propvalues[key] = str(value)
452 elif value is not None and isinstance(prop, hyperdb.Interval):
453 if not isinstance(value, date.Interval):
454 raise TypeError, 'new property "%s" not an Interval'% key
455 setattr(row, key, str(value))
456 changes[key] = str(oldvalue)
457 propvalues[key] = str(value)
459 oldnode[key] = oldvalue
461 # nothing to do?
462 if not propvalues:
463 return
464 if not row.activity:
465 row.activity = int(time.time())
466 if isnew:
467 if not row.creation:
468 row.creation = int(time.time())
469 if not row.creator:
470 row.creator = self.db.curuserid
472 self.db.dirty = 1
473 if self.do_journal:
474 if isnew:
475 self.db.addjournal(self.classname, nodeid, _CREATE, {})
476 else:
477 self.db.addjournal(self.classname, nodeid, _SET, changes)
479 def retire(self, nodeid):
480 view = self.getview(1)
481 ndx = view.find(id=int(nodeid))
482 if ndx < 0:
483 raise KeyError, "nodeid %s not found" % nodeid
484 row = view[ndx]
485 oldvalues = self.uncommitted.setdefault(row.id, {})
486 oldval = oldvalues['_isdel'] = row._isdel
487 row._isdel = 1
488 if self.do_journal:
489 self.db.addjournal(self.classname, nodeid, _RETIRE, {})
490 iv = self.getindexview(1)
491 ndx = iv.find(k=getattr(row, self.keyname),i=row.id)
492 if ndx > -1:
493 iv.delete(ndx)
494 self.db.dirty = 1
495 def history(self, nodeid):
496 if not self.do_journal:
497 raise ValueError, 'Journalling is disabled for this class'
498 return self.db.gethistory(self.classname, nodeid)
499 def setkey(self, propname):
500 if self.keyname:
501 if propname == self.keyname:
502 return
503 raise ValueError, "%s already indexed on %s" % (self.classname, self.keyname)
504 # first setkey for this run
505 self.keyname = propname
506 iv = self.db._db.view('_%s' % self.classname)
507 if self.db.fastopen or iv.structure():
508 return
509 # very first setkey ever
510 iv = self.db._db.getas('_%s[k:S,i:I]' % self.classname)
511 iv = iv.ordered(1)
512 #XXX
513 # print "setkey building index"
514 for row in self.getview():
515 iv.append(k=getattr(row, propname), i=row.id)
516 def getkey(self):
517 return self.keyname
518 def lookup(self, keyvalue):
519 if type(keyvalue) is not _STRINGTYPE:
520 raise TypeError, "%r is not a string" % keyvalue
521 iv = self.getindexview()
522 if iv:
523 ndx = iv.find(k=keyvalue)
524 if ndx > -1:
525 return str(iv[ndx].i)
526 else:
527 view = self.getview()
528 ndx = view.find({self.keyname:keyvalue, '_isdel':0})
529 if ndx > -1:
530 return str(view[ndx].id)
531 raise KeyError, keyvalue
532 def find(self, **propspec):
533 """Get the ids of nodes in this class which link to the given nodes.
535 'propspec' consists of keyword args propname={nodeid:1,}
536 'propname' must be the name of a property in this class, or a
537 KeyError is raised. That property must be a Link or
538 Multilink property, or a TypeError is raised.
540 Any node in this class whose propname property links to any of the
541 nodeids will be returned. Used by the full text indexing, which knows
542 that "foo" occurs in msg1, msg3 and file7; so we have hits on these
543 issues:
545 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
547 """
548 propspec = propspec.items()
549 for propname, nodeid in propspec:
550 # check the prop is OK
551 prop = self.ruprops[propname]
552 if (not isinstance(prop, hyperdb.Link) and
553 not isinstance(prop, hyperdb.Multilink)):
554 raise TypeError, "'%s' not a Link/Multilink property"%propname
556 vws = []
557 for propname, ids in propspec:
558 if type(ids) is _STRINGTYPE:
559 ids = {ids:1}
560 prop = self.ruprops[propname]
561 view = self.getview()
562 if isinstance(prop, hyperdb.Multilink):
563 view = view.flatten(getattr(view, propname))
564 def ff(row, nm=propname, ids=ids):
565 return ids.has_key(str(row.fid))
566 else:
567 def ff(row, nm=propname, ids=ids):
568 return ids.has_key(str(getattr(row, nm)))
569 ndxview = view.filter(ff)
570 vws.append(ndxview.unique())
572 # handle the empty match case
573 if not vws:
574 return []
576 ndxview = vws[0]
577 for v in vws[1:]:
578 ndxview = ndxview.union(v)
579 view = view.remapwith(ndxview)
580 rslt = []
581 for row in view:
582 rslt.append(str(row.id))
583 return rslt
586 def list(self):
587 l = []
588 for row in self.getview().select(_isdel=0):
589 l.append(str(row.id))
590 return l
591 def count(self):
592 return len(self.getview())
593 def getprops(self, protected=1):
594 # protected is not in ping's spec
595 allprops = self.ruprops.copy()
596 if protected and self.privateprops is not None:
597 allprops.update(self.privateprops)
598 return allprops
599 def addprop(self, **properties):
600 for key in properties.keys():
601 if self.ruprops.has_key(key):
602 raise ValueError, "%s is already a property of %s" % (key, self.classname)
603 self.ruprops.update(properties)
604 view = self.__getview()
605 # ---- end of ping's spec
606 def filter(self, search_matches, filterspec, sort, group):
607 # search_matches is None or a set (dict of {nodeid: {propname:[nodeid,...]}})
608 # filterspec is a dict {propname:value}
609 # sort and group are lists of propnames
611 where = {'_isdel':0}
612 mlcriteria = {}
613 regexes = {}
614 orcriteria = {}
615 for propname, value in filterspec.items():
616 prop = self.ruprops.get(propname, None)
617 if prop is None:
618 prop = self.privateprops[propname]
619 if isinstance(prop, hyperdb.Multilink):
620 if type(value) is not _LISTTYPE:
621 value = [value]
622 # transform keys to ids
623 u = []
624 for item in value:
625 try:
626 item = int(item)
627 except (TypeError, ValueError):
628 item = int(self.db.getclass(prop.classname).lookup(item))
629 if item == -1:
630 item = 0
631 u.append(item)
632 mlcriteria[propname] = u
633 elif isinstance(prop, hyperdb.Link):
634 if type(value) is not _LISTTYPE:
635 value = [value]
636 # transform keys to ids
637 u = []
638 for item in value:
639 try:
640 item = int(item)
641 except (TypeError, ValueError):
642 item = int(self.db.getclass(prop.classname).lookup(item))
643 if item == -1:
644 item = 0
645 u.append(item)
646 if len(u) == 1:
647 where[propname] = u[0]
648 else:
649 orcriteria[propname] = u
650 elif isinstance(prop, hyperdb.String):
651 # simple glob searching
652 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', value)
653 v = v.replace('?', '.')
654 v = v.replace('*', '.*?')
655 regexes[propname] = re.compile(v, re.I)
656 elif propname == 'id':
657 where[propname] = int(value)
658 else:
659 where[propname] = str(value)
660 v = self.getview()
661 #print "filter start at %s" % time.time()
662 if where:
663 v = v.select(where)
664 #print "filter where at %s" % time.time()
666 if mlcriteria:
667 # multilink - if any of the nodeids required by the
668 # filterspec aren't in this node's property, then skip
669 # it
670 def ff(row, ml=mlcriteria):
671 for propname, values in ml.items():
672 sv = getattr(row, propname)
673 for id in values:
674 if sv.find(fid=id) == -1:
675 return 0
676 return 1
677 iv = v.filter(ff)
678 v = v.remapwith(iv)
680 #print "filter mlcrit at %s" % time.time()
682 if orcriteria:
683 def ff(row, crit=orcriteria):
684 for propname, allowed in crit.items():
685 val = getattr(row, propname)
686 if val not in allowed:
687 return 0
688 return 1
690 iv = v.filter(ff)
691 v = v.remapwith(iv)
693 #print "filter orcrit at %s" % time.time()
694 if regexes:
695 def ff(row, r=regexes):
696 for propname, regex in r.items():
697 val = getattr(row, propname)
698 if not regex.search(val):
699 return 0
700 return 1
702 iv = v.filter(ff)
703 v = v.remapwith(iv)
704 #print "filter regexs at %s" % time.time()
706 if sort or group:
707 sortspec = []
708 rev = []
709 for propname in group + sort:
710 isreversed = 0
711 if propname[0] == '-':
712 propname = propname[1:]
713 isreversed = 1
714 try:
715 prop = getattr(v, propname)
716 except AttributeError:
717 # I can't sort on 'activity', cause it's psuedo!!
718 continue
719 if isreversed:
720 rev.append(prop)
721 sortspec.append(prop)
722 v = v.sortrev(sortspec, rev)[:] #XXX Aaaabh
723 #print "filter sort at %s" % time.time()
725 rslt = []
726 for row in v:
727 id = str(row.id)
728 if search_matches is not None:
729 if search_matches.has_key(id):
730 rslt.append(id)
731 else:
732 rslt.append(id)
733 return rslt
735 def hasnode(self, nodeid):
736 return int(nodeid) < self.maxid
738 def labelprop(self, default_to_id=0):
739 ''' Return the property name for a label for the given node.
741 This method attempts to generate a consistent label for the node.
742 It tries the following in order:
743 1. key property
744 2. "name" property
745 3. "title" property
746 4. first property from the sorted property name list
747 '''
748 k = self.getkey()
749 if k:
750 return k
751 props = self.getprops()
752 if props.has_key('name'):
753 return 'name'
754 elif props.has_key('title'):
755 return 'title'
756 if default_to_id:
757 return 'id'
758 props = props.keys()
759 props.sort()
760 return props[0]
761 def stringFind(self, **requirements):
762 """Locate a particular node by matching a set of its String
763 properties in a caseless search.
765 If the property is not a String property, a TypeError is raised.
767 The return is a list of the id of all nodes that match.
768 """
769 for propname in requirements.keys():
770 prop = self.properties[propname]
771 if isinstance(not prop, hyperdb.String):
772 raise TypeError, "'%s' not a String property"%propname
773 requirements[propname] = requirements[propname].lower()
774 requirements['_isdel'] = 0
776 l = []
777 for row in self.getview().select(requirements):
778 l.append(str(row.id))
779 return l
781 def addjournal(self, nodeid, action, params):
782 self.db.addjournal(self.classname, nodeid, action, params)
784 def index(self, nodeid):
785 ''' Add (or refresh) the node to search indexes '''
786 # find all the String properties that have indexme
787 for prop, propclass in self.getprops().items():
788 if isinstance(propclass, hyperdb.String) and propclass.indexme:
789 # index them under (classname, nodeid, property)
790 self.db.indexer.add_text((self.classname, nodeid, prop),
791 str(self.get(nodeid, prop)))
793 # --- used by Database
794 def _commit(self):
795 """ called post commit of the DB.
796 interested subclasses may override """
797 self.uncommitted = {}
798 self.rbactions = []
799 self.idcache = {}
800 def _rollback(self):
801 """ called pre rollback of the DB.
802 interested subclasses may override """
803 for action in self.rbactions:
804 action()
805 self.rbactions = []
806 self.uncommitted = {}
807 self.idcache = {}
808 def _clear(self):
809 view = self.getview(1)
810 if len(view):
811 view[:] = []
812 self.db.dirty = 1
813 iv = self.getindexview(1)
814 if iv:
815 iv[:] = []
816 def rollbackaction(self, action):
817 """ call this to register a callback called on rollback
818 callback is removed on end of transaction """
819 self.rbactions.append(action)
820 # --- internal
821 def __getview(self):
822 db = self.db._db
823 view = db.view(self.classname)
824 if self.db.fastopen:
825 return view.ordered(1)
826 # is the definition the same?
827 mkprops = view.structure()
828 for nm, rutyp in self.ruprops.items():
829 for mkprop in mkprops:
830 if mkprop.name == nm:
831 break
832 else:
833 mkprop = None
834 if mkprop is None:
835 #print "%s missing prop %s (%s)" % (self.classname, nm, rutyp.__class__.__name__)
836 break
837 if _typmap[rutyp.__class__] != mkprop.type:
838 #print "%s - prop %s (%s) has wrong mktyp (%s)" % (self.classname, nm, rutyp.__class__.__name__, mkprop.type)
839 break
840 else:
841 return view.ordered(1)
842 # need to create or restructure the mk view
843 # id comes first, so MK will order it for us
844 self.db.dirty = 1
845 s = ["%s[id:I" % self.classname]
846 for nm, rutyp in self.ruprops.items():
847 mktyp = _typmap[rutyp.__class__]
848 s.append('%s:%s' % (nm, mktyp))
849 if mktyp == 'V':
850 s[-1] += ('[fid:I]')
851 s.append('_isdel:I,activity:I,creation:I,creator:I]')
852 v = db.getas(','.join(s))
853 return v.ordered(1)
854 def getview(self, RW=0):
855 if RW and self.db.isReadOnly():
856 self.db.getWriteAccess()
857 return self.db._db.view(self.classname).ordered(1)
858 def getindexview(self, RW=0):
859 if RW and self.db.isReadOnly():
860 self.db.getWriteAccess()
861 return self.db._db.view("_%s" % self.classname).ordered(1)
863 def _fetchML(sv):
864 l = []
865 for row in sv:
866 if row.fid:
867 l.append(str(row.fid))
868 return l
870 def _fetchPW(s):
871 p = password.Password()
872 p.unpack(s)
873 return p
875 def _fetchLink(n):
876 return n and str(n) or None
878 def _fetchDate(n):
879 return date.Date(time.gmtime(n))
881 _converters = {
882 hyperdb.Date : _fetchDate,
883 hyperdb.Link : _fetchLink,
884 hyperdb.Multilink : _fetchML,
885 hyperdb.Interval : date.Interval,
886 hyperdb.Password : _fetchPW,
887 }
889 class FileName(hyperdb.String):
890 isfilename = 1
892 _typmap = {
893 FileName : 'S',
894 hyperdb.String : 'S',
895 hyperdb.Date : 'I',
896 hyperdb.Link : 'I',
897 hyperdb.Multilink : 'V',
898 hyperdb.Interval : 'S',
899 hyperdb.Password : 'S',
900 }
901 class FileClass(Class):
902 ' like Class but with a content property '
903 default_mime_type = 'text/plain'
904 def __init__(self, db, classname, **properties):
905 properties['content'] = FileName()
906 if not properties.has_key('type'):
907 properties['type'] = hyperdb.String()
908 Class.__init__(self, db, classname, **properties)
909 def get(self, nodeid, propname, default=_marker, cache=1):
910 x = Class.get(self, nodeid, propname, default, cache)
911 if propname == 'content':
912 if x.startswith('file:'):
913 fnm = x[5:]
914 try:
915 x = open(fnm, 'rb').read()
916 except Exception, e:
917 x = repr(e)
918 return x
919 def create(self, **propvalues):
920 content = propvalues['content']
921 del propvalues['content']
922 newid = Class.create(self, **propvalues)
923 if not content:
924 return newid
925 if content.startswith('/tracker/download.php?'):
926 self.set(newid, content='http://sourceforge.net'+content)
927 return newid
928 nm = bnm = '%s%s' % (self.classname, newid)
929 sd = str(int(int(newid) / 1000))
930 d = os.path.join(self.db.config.DATABASE, 'files', self.classname, sd)
931 if not os.path.exists(d):
932 os.makedirs(d)
933 nm = os.path.join(d, nm)
934 open(nm, 'wb').write(content)
935 self.set(newid, content = 'file:'+nm)
936 mimetype = propvalues.get('type', self.default_mime_type)
937 self.db.indexer.add_text((self.classname, newid, 'content'), content, mimetype)
938 def undo(fnm=nm, action1=os.remove, indexer=self.db.indexer):
939 remove(fnm)
940 self.rollbackaction(undo)
941 return newid
942 def index(self, nodeid):
943 Class.index(self, nodeid)
944 mimetype = self.get(nodeid, 'type')
945 if not mimetype:
946 mimetype = self.default_mime_type
947 self.db.indexer.add_text((self.classname, nodeid, 'content'),
948 self.get(nodeid, 'content'), mimetype)
950 class IssueClass(Class, roundupdb.IssueClass):
951 # Overridden methods:
952 def __init__(self, db, classname, **properties):
953 """The newly-created class automatically includes the "messages",
954 "files", "nosy", and "superseder" properties. If the 'properties'
955 dictionary attempts to specify any of these properties or a
956 "creation" or "activity" property, a ValueError is raised."""
957 if not properties.has_key('title'):
958 properties['title'] = hyperdb.String(indexme='yes')
959 if not properties.has_key('messages'):
960 properties['messages'] = hyperdb.Multilink("msg")
961 if not properties.has_key('files'):
962 properties['files'] = hyperdb.Multilink("file")
963 if not properties.has_key('nosy'):
964 properties['nosy'] = hyperdb.Multilink("user")
965 if not properties.has_key('superseder'):
966 properties['superseder'] = hyperdb.Multilink(classname)
967 Class.__init__(self, db, classname, **properties)