c8e220d8cfdbbafe927eb65854264e25580665ad
1 from roundup import hyperdb, date, password, roundupdb, security
2 import metakit
3 from sessions import Sessions
4 import re, marshal, os, sys, weakref, time, calendar
5 from roundup import indexer
6 import locking
8 _dbs = {}
10 def Database(config, journaltag=None):
11 db = _dbs.get(config.DATABASE, None)
12 if db is None or db._db is None:
13 db = _Database(config, journaltag)
14 _dbs[config.DATABASE] = db
15 else:
16 db.journaltag = journaltag
17 try:
18 delattr(db, 'curuserid')
19 except AttributeError:
20 pass
21 return db
23 class _Database(hyperdb.Database):
24 def __init__(self, config, journaltag=None):
25 self.config = config
26 self.journaltag = journaltag
27 self.classes = {}
28 self.dirty = 0
29 self.lockfile = None
30 self._db = self.__open()
31 self.indexer = Indexer(self.config.DATABASE, self._db)
32 self.sessions = Sessions(self.config)
33 self.security = security.Security(self)
35 os.umask(0002)
37 def post_init(self):
38 if self.indexer.should_reindex():
39 self.reindex()
41 def reindex(self):
42 for klass in self.classes.values():
43 for nodeid in klass.list():
44 klass.index(nodeid)
45 self.indexer.save_index()
48 # --- defined in ping's spec
49 def __getattr__(self, classname):
50 if classname == 'curuserid':
51 if self.journaltag is None:
52 return None
54 try:
55 self.curuserid = x = int(self.classes['user'].lookup(self.journaltag))
56 except KeyError:
57 if self.journaltag == 'admin':
58 self.curuserid = x = 1
59 else:
60 x = 0
61 return x
62 elif classname == 'transactions':
63 return self.dirty
64 return self.getclass(classname)
65 def getclass(self, classname):
66 try:
67 return self.classes[classname]
68 except KeyError:
69 raise KeyError, 'There is no class called "%s"'%classname
70 def getclasses(self):
71 return self.classes.keys()
72 # --- end of ping's spec
73 # --- exposed methods
74 def commit(self):
75 if self.dirty:
76 self._db.commit()
77 for cl in self.classes.values():
78 cl._commit()
79 self.indexer.save_index()
80 self.dirty = 0
81 def rollback(self):
82 if self.dirty:
83 for cl in self.classes.values():
84 cl._rollback()
85 self._db.rollback()
86 self._db = None
87 self._db = metakit.storage(self.dbnm, 1)
88 self.hist = self._db.view('history')
89 self.tables = self._db.view('tables')
90 self.indexer.rollback()
91 self.indexer.datadb = self._db
92 self.dirty = 0
93 def clearCache(self):
94 for cl in self.classes.values():
95 cl._commit()
96 def clear(self):
97 for cl in self.classes.values():
98 cl._clear()
99 def hasnode(self, classname, nodeid):
100 return self.getclass(classname).hasnode(nodeid)
101 def pack(self, pack_before):
102 mindate = int(calendar.timegm(pack_before.get_tuple()))
103 i = 0
104 while i < len(self.hist):
105 if self.hist[i].date < mindate and self.hist[i].action != _CREATE:
106 self.hist.delete(i)
107 else:
108 i = i + 1
109 def addclass(self, cl):
110 self.classes[cl.classname] = cl
111 if self.tables.find(name=cl.classname) < 0:
112 self.tables.append(name=cl.classname)
113 def addjournal(self, tablenm, nodeid, action, params, creator=None,
114 creation=None):
115 tblid = self.tables.find(name=tablenm)
116 if tblid == -1:
117 tblid = self.tables.append(name=tablenm)
118 if creator is None:
119 creator = self.curuserid
120 else:
121 try:
122 creator = int(creator)
123 except TypeError:
124 creator = int(self.getclass('user').lookup(creator))
125 if creation is None:
126 creation = int(time.time())
127 elif isinstance(creation, date.Date):
128 creation = int(calendar.timegm(creation.get_tuple()))
129 # tableid:I,nodeid:I,date:I,user:I,action:I,params:B
130 self.hist.append(tableid=tblid,
131 nodeid=int(nodeid),
132 date=creation,
133 action=action,
134 user = creator,
135 params = marshal.dumps(params))
136 def getjournal(self, tablenm, nodeid):
137 rslt = []
138 tblid = self.tables.find(name=tablenm)
139 if tblid == -1:
140 return rslt
141 q = self.hist.select(tableid=tblid, nodeid=int(nodeid))
142 if len(q) == 0:
143 raise IndexError, "no history for id %s in %s" % (nodeid, tablenm)
144 i = 0
145 #userclass = self.getclass('user')
146 for row in q:
147 try:
148 params = marshal.loads(row.params)
149 except ValueError:
150 print "history couldn't unmarshal %r" % row.params
151 params = {}
152 #usernm = userclass.get(str(row.user), 'username')
153 dt = date.Date(time.gmtime(row.date))
154 #rslt.append((nodeid, dt, usernm, _actionnames[row.action], params))
155 rslt.append((nodeid, dt, str(row.user), _actionnames[row.action], params))
156 return rslt
158 def destroyjournal(self, tablenm, nodeid):
159 nodeid = int(nodeid)
160 tblid = self.tables.find(name=tablenm)
161 if tblid == -1:
162 return
163 i = 0
164 hist = self.hist
165 while i < len(hist):
166 if hist[i].tableid == tblid and hist[i].nodeid == nodeid:
167 hist.delete(i)
168 else:
169 i = i + 1
170 self.dirty = 1
172 def close(self):
173 for cl in self.classes.values():
174 cl.db = None
175 self._db = None
176 if self.lockfile is not None:
177 locking.release_lock(self.lockfile)
178 if _dbs.has_key(self.config.DATABASE):
179 del _dbs[self.config.DATABASE]
180 if self.lockfile is not None:
181 self.lockfile.close()
182 self.lockfile = None
183 self.classes = {}
184 self.indexer = None
186 # --- internal
187 def __open(self):
188 if not os.path.exists(self.config.DATABASE):
189 os.makedirs(self.config.DATABASE)
190 self.dbnm = db = os.path.join(self.config.DATABASE, 'tracker.mk4')
191 lockfilenm = db[:-3]+'lck'
192 self.lockfile = locking.acquire_lock(lockfilenm)
193 self.lockfile.write(str(os.getpid()))
194 self.lockfile.flush()
195 self.fastopen = 0
196 if os.path.exists(db):
197 dbtm = os.path.getmtime(db)
198 pkgnm = self.config.__name__.split('.')[0]
199 schemamod = sys.modules.get(pkgnm+'.dbinit', None)
200 if schemamod:
201 if os.path.exists(schemamod.__file__):
202 schematm = os.path.getmtime(schemamod.__file__)
203 if schematm < dbtm:
204 # found schema mod - it's older than the db
205 self.fastopen = 1
206 else:
207 # can't find schemamod - must be frozen
208 self.fastopen = 1
209 db = metakit.storage(db, 1)
210 hist = db.view('history')
211 tables = db.view('tables')
212 if not self.fastopen:
213 if not hist.structure():
214 hist = db.getas('history[tableid:I,nodeid:I,date:I,user:I,action:I,params:B]')
215 if not tables.structure():
216 tables = db.getas('tables[name:S]')
217 db.commit()
218 self.tables = tables
219 self.hist = hist
220 return db
222 _STRINGTYPE = type('')
223 _LISTTYPE = type([])
224 _CREATE, _SET, _RETIRE, _LINK, _UNLINK = range(5)
226 _actionnames = {
227 _CREATE : 'create',
228 _SET : 'set',
229 _RETIRE : 'retire',
230 _LINK : 'link',
231 _UNLINK : 'unlink',
232 }
234 _marker = []
236 _ALLOWSETTINGPRIVATEPROPS = 0
238 class Class:
239 privateprops = None
240 def __init__(self, db, classname, **properties):
241 #self.db = weakref.proxy(db)
242 self.db = db
243 self.classname = classname
244 self.keyname = None
245 self.ruprops = properties
246 self.privateprops = { 'id' : hyperdb.String(),
247 'activity' : hyperdb.Date(),
248 'creation' : hyperdb.Date(),
249 'creator' : hyperdb.Link('user') }
250 self.auditors = {'create': [], 'set': [], 'retire': []} # event -> list of callables
251 self.reactors = {'create': [], 'set': [], 'retire': []} # ditto
252 view = self.__getview()
253 self.maxid = 1
254 if view:
255 self.maxid = view[-1].id + 1
256 self.uncommitted = {}
257 self.rbactions = []
258 # people reach inside!!
259 self.properties = self.ruprops
260 self.db.addclass(self)
261 self.idcache = {}
263 # default is to journal changes
264 self.do_journal = 1
266 def enableJournalling(self):
267 '''Turn journalling on for this class
268 '''
269 self.do_journal = 1
271 def disableJournalling(self):
272 '''Turn journalling off for this class
273 '''
274 self.do_journal = 0
276 # --- the roundup.Class methods
277 def audit(self, event, detector):
278 l = self.auditors[event]
279 if detector not in l:
280 self.auditors[event].append(detector)
281 def fireAuditors(self, action, nodeid, newvalues):
282 for audit in self.auditors[action]:
283 audit(self.db, self, nodeid, newvalues)
284 def fireReactors(self, action, nodeid, oldvalues):
285 for react in self.reactors[action]:
286 react(self.db, self, nodeid, oldvalues)
287 def react(self, event, detector):
288 l = self.reactors[event]
289 if detector not in l:
290 self.reactors[event].append(detector)
291 # --- the hyperdb.Class methods
292 def create(self, **propvalues):
293 self.fireAuditors('create', None, propvalues)
294 rowdict = {}
295 rowdict['id'] = newid = self.maxid
296 self.maxid += 1
297 ndx = self.getview(1).append(rowdict)
298 propvalues['#ISNEW'] = 1
299 try:
300 self.set(str(newid), **propvalues)
301 except Exception:
302 self.maxid -= 1
303 raise
304 return str(newid)
306 def get(self, nodeid, propname, default=_marker, cache=1):
307 # default and cache aren't in the spec
308 # cache=0 means "original value"
310 view = self.getview()
311 id = int(nodeid)
312 if cache == 0:
313 oldnode = self.uncommitted.get(id, None)
314 if oldnode and oldnode.has_key(propname):
315 return oldnode[propname]
316 ndx = self.idcache.get(id, None)
317 if ndx is None:
318 ndx = view.find(id=id)
319 if ndx < 0:
320 raise IndexError, "%s has no node %s" % (self.classname, nodeid)
321 self.idcache[id] = ndx
322 try:
323 raw = getattr(view[ndx], propname)
324 except AttributeError:
325 raise KeyError, propname
326 rutyp = self.ruprops.get(propname, None)
327 if rutyp is None:
328 rutyp = self.privateprops[propname]
329 converter = _converters.get(rutyp.__class__, None)
330 if converter:
331 raw = converter(raw)
332 return raw
334 def set(self, nodeid, **propvalues):
335 isnew = 0
336 if propvalues.has_key('#ISNEW'):
337 isnew = 1
338 del propvalues['#ISNEW']
339 if not isnew:
340 self.fireAuditors('set', nodeid, propvalues)
341 if not propvalues:
342 return propvalues
343 if propvalues.has_key('id'):
344 raise KeyError, '"id" is reserved'
345 if self.db.journaltag is None:
346 raise hyperdb.DatabaseError, 'Database open read-only'
347 view = self.getview(1)
348 # node must exist & not be retired
349 id = int(nodeid)
350 ndx = view.find(id=id)
351 if ndx < 0:
352 raise IndexError, "%s has no node %s" % (self.classname, nodeid)
353 row = view[ndx]
354 if row._isdel:
355 raise IndexError, "%s has no node %s" % (self.classname, nodeid)
356 oldnode = self.uncommitted.setdefault(id, {})
357 changes = {}
359 for key, value in propvalues.items():
360 # this will raise the KeyError if the property isn't valid
361 # ... we don't use getprops() here because we only care about
362 # the writeable properties.
363 if _ALLOWSETTINGPRIVATEPROPS:
364 prop = self.ruprops.get(key, None)
365 if not prop:
366 prop = self.privateprops[key]
367 else:
368 prop = self.ruprops[key]
369 converter = _converters.get(prop.__class__, lambda v: v)
370 # if the value's the same as the existing value, no sense in
371 # doing anything
372 oldvalue = converter(getattr(row, key))
373 if value == oldvalue:
374 del propvalues[key]
375 continue
377 # check to make sure we're not duplicating an existing key
378 if key == self.keyname:
379 iv = self.getindexview(1)
380 ndx = iv.find(k=value)
381 if ndx == -1:
382 iv.append(k=value, i=row.id)
383 if not isnew:
384 ndx = iv.find(k=oldvalue)
385 if ndx > -1:
386 iv.delete(ndx)
387 else:
388 raise ValueError, 'node with key "%s" exists'%value
390 # do stuff based on the prop type
391 if isinstance(prop, hyperdb.Link):
392 link_class = prop.classname
393 # must be a string or None
394 if value is not None and not isinstance(value, type('')):
395 raise ValueError, 'property "%s" link value be a string'%(
396 key)
397 # Roundup sets to "unselected" by passing None
398 if value is None:
399 value = 0
400 # if it isn't a number, it's a key
401 try:
402 int(value)
403 except ValueError:
404 try:
405 value = self.db.getclass(link_class).lookup(value)
406 except (TypeError, KeyError):
407 raise IndexError, 'new property "%s": %s not a %s'%(
408 key, value, prop.classname)
410 if (value is not None and
411 not self.db.getclass(link_class).hasnode(value)):
412 raise IndexError, '%s has no node %s'%(link_class, value)
414 setattr(row, key, int(value))
415 changes[key] = oldvalue
417 if self.do_journal and prop.do_journal:
418 # register the unlink with the old linked node
419 if oldvalue:
420 self.db.addjournal(link_class, value, _UNLINK,
421 (self.classname, str(row.id), key))
423 # register the link with the newly linked node
424 if value:
425 self.db.addjournal(link_class, value, _LINK,
426 (self.classname, str(row.id), key))
428 elif isinstance(prop, hyperdb.Multilink):
429 if value is not None and type(value) != _LISTTYPE:
430 raise TypeError, 'new property "%s" not a list of ids'%key
431 link_class = prop.classname
432 l = []
433 if value is None:
434 value = []
435 for entry in value:
436 if type(entry) != _STRINGTYPE:
437 raise ValueError, 'new property "%s" link value ' \
438 'must be a string'%key
439 # if it isn't a number, it's a key
440 try:
441 int(entry)
442 except ValueError:
443 try:
444 entry = self.db.getclass(link_class).lookup(entry)
445 except (TypeError, KeyError):
446 raise IndexError, 'new property "%s": %s not a %s'%(
447 key, entry, prop.classname)
448 l.append(entry)
449 propvalues[key] = value = l
451 # handle removals
452 rmvd = []
453 for id in oldvalue:
454 if id not in value:
455 rmvd.append(id)
456 # register the unlink with the old linked node
457 if self.do_journal and prop.do_journal:
458 self.db.addjournal(link_class, id, _UNLINK,
459 (self.classname, str(row.id), key))
461 # handle additions
462 adds = []
463 for id in value:
464 if id not in oldvalue:
465 if not self.db.getclass(link_class).hasnode(id):
466 raise IndexError, '%s has no node %s'%(
467 link_class, id)
468 adds.append(id)
469 # register the link with the newly linked node
470 if self.do_journal and prop.do_journal:
471 self.db.addjournal(link_class, id, _LINK,
472 (self.classname, str(row.id), key))
474 sv = getattr(row, key)
475 i = 0
476 while i < len(sv):
477 if str(sv[i].fid) in rmvd:
478 sv.delete(i)
479 else:
480 i += 1
481 for id in adds:
482 sv.append(fid=int(id))
483 changes[key] = oldvalue
484 if not rmvd and not adds:
485 del propvalues[key]
487 elif isinstance(prop, hyperdb.String):
488 if value is not None and type(value) != _STRINGTYPE:
489 raise TypeError, 'new property "%s" not a string'%key
490 if value is None:
491 value = ''
492 setattr(row, key, value)
493 changes[key] = oldvalue
494 if hasattr(prop, 'isfilename') and prop.isfilename:
495 propvalues[key] = os.path.basename(value)
496 if prop.indexme:
497 self.db.indexer.add_text((self.classname, nodeid, key),
498 value, 'text/plain')
500 elif isinstance(prop, hyperdb.Password):
501 if value is not None and not isinstance(value, password.Password):
502 raise TypeError, 'new property "%s" not a Password'% key
503 if value is None:
504 value = ''
505 setattr(row, key, str(value))
506 changes[key] = str(oldvalue)
507 propvalues[key] = str(value)
509 elif isinstance(prop, hyperdb.Date):
510 if value is not None and not isinstance(value, date.Date):
511 raise TypeError, 'new property "%s" not a Date'% key
512 if value is None:
513 setattr(row, key, 0)
514 else:
515 setattr(row, key, int(calendar.timegm(value.get_tuple())))
516 changes[key] = str(oldvalue)
517 propvalues[key] = str(value)
519 elif isinstance(prop, hyperdb.Interval):
520 if value is not None and not isinstance(value, date.Interval):
521 raise TypeError, 'new property "%s" not an Interval'% key
522 if value is None:
523 setattr(row, key, '')
524 else:
525 setattr(row, key, str(value))
526 changes[key] = str(oldvalue)
527 propvalues[key] = str(value)
529 elif isinstance(prop, hyperdb.Number):
530 if value is None:
531 value = 0
532 try:
533 v = int(value)
534 except ValueError:
535 raise TypeError, "%s (%s) is not numeric" % (key, repr(value))
536 setattr(row, key, v)
537 changes[key] = oldvalue
538 propvalues[key] = value
540 elif isinstance(prop, hyperdb.Boolean):
541 if value is None:
542 bv = 0
543 elif value not in (0,1):
544 raise TypeError, "%s (%s) is not boolean" % (key, repr(value))
545 else:
546 bv = value
547 setattr(row, key, bv)
548 changes[key] = oldvalue
549 propvalues[key] = value
551 oldnode[key] = oldvalue
553 # nothing to do?
554 if not propvalues:
555 return propvalues
556 if not propvalues.has_key('activity'):
557 row.activity = int(time.time())
558 if isnew:
559 if not row.creation:
560 row.creation = int(time.time())
561 if not row.creator:
562 row.creator = self.db.curuserid
564 self.db.dirty = 1
565 if self.do_journal:
566 if isnew:
567 self.db.addjournal(self.classname, nodeid, _CREATE, {})
568 self.fireReactors('create', nodeid, None)
569 else:
570 self.db.addjournal(self.classname, nodeid, _SET, changes)
571 self.fireReactors('set', nodeid, oldnode)
573 return propvalues
575 def retire(self, nodeid):
576 if self.db.journaltag is None:
577 raise hyperdb.DatabaseError, 'Database open read-only'
578 self.fireAuditors('retire', nodeid, None)
579 view = self.getview(1)
580 ndx = view.find(id=int(nodeid))
581 if ndx < 0:
582 raise KeyError, "nodeid %s not found" % nodeid
583 row = view[ndx]
584 oldvalues = self.uncommitted.setdefault(row.id, {})
585 oldval = oldvalues['_isdel'] = row._isdel
586 row._isdel = 1
587 if self.do_journal:
588 self.db.addjournal(self.classname, nodeid, _RETIRE, {})
589 if self.keyname:
590 iv = self.getindexview(1)
591 ndx = iv.find(k=getattr(row, self.keyname),i=row.id)
592 if ndx > -1:
593 iv.delete(ndx)
594 self.db.dirty = 1
595 self.fireReactors('retire', nodeid, None)
596 def history(self, nodeid):
597 if not self.do_journal:
598 raise ValueError, 'Journalling is disabled for this class'
599 return self.db.getjournal(self.classname, nodeid)
600 def setkey(self, propname):
601 if self.keyname:
602 if propname == self.keyname:
603 return
604 raise ValueError, "%s already indexed on %s" % (self.classname, self.keyname)
605 prop = self.properties.get(propname, None)
606 if prop is None:
607 prop = self.privateprops.get(propname, None)
608 if prop is None:
609 raise KeyError, "no property %s" % propname
610 if not isinstance(prop, hyperdb.String):
611 raise TypeError, "%s is not a String" % propname
612 # first setkey for this run
613 self.keyname = propname
614 iv = self.db._db.view('_%s' % self.classname)
615 if self.db.fastopen and iv.structure():
616 return
617 # very first setkey ever
618 self.db.dirty = 1
619 iv = self.db._db.getas('_%s[k:S,i:I]' % self.classname)
620 iv = iv.ordered(1)
621 # print "setkey building index"
622 for row in self.getview():
623 iv.append(k=getattr(row, propname), i=row.id)
624 self.db.commit()
625 def getkey(self):
626 return self.keyname
627 def lookup(self, keyvalue):
628 if type(keyvalue) is not _STRINGTYPE:
629 raise TypeError, "%r is not a string" % keyvalue
630 iv = self.getindexview()
631 if iv:
632 ndx = iv.find(k=keyvalue)
633 if ndx > -1:
634 return str(iv[ndx].i)
635 else:
636 view = self.getview()
637 ndx = view.find({self.keyname:keyvalue, '_isdel':0})
638 if ndx > -1:
639 return str(view[ndx].id)
640 raise KeyError, keyvalue
642 def destroy(self, id):
643 view = self.getview(1)
644 ndx = view.find(id=int(id))
645 if ndx > -1:
646 if self.keyname:
647 keyvalue = getattr(view[ndx], self.keyname)
648 iv = self.getindexview(1)
649 if iv:
650 ivndx = iv.find(k=keyvalue)
651 if ivndx > -1:
652 iv.delete(ivndx)
653 view.delete(ndx)
654 self.db.destroyjournal(self.classname, id)
655 self.db.dirty = 1
657 def find(self, **propspec):
658 """Get the ids of nodes in this class which link to the given nodes.
660 'propspec' consists of keyword args propname={nodeid:1,}
661 'propname' must be the name of a property in this class, or a
662 KeyError is raised. That property must be a Link or
663 Multilink property, or a TypeError is raised.
665 Any node in this class whose propname property links to any of the
666 nodeids will be returned. Used by the full text indexing, which knows
667 that "foo" occurs in msg1, msg3 and file7; so we have hits on these
668 issues:
670 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
672 """
673 propspec = propspec.items()
674 for propname, nodeid in propspec:
675 # check the prop is OK
676 prop = self.ruprops[propname]
677 if (not isinstance(prop, hyperdb.Link) and
678 not isinstance(prop, hyperdb.Multilink)):
679 raise TypeError, "'%s' not a Link/Multilink property"%propname
681 vws = []
682 for propname, ids in propspec:
683 if type(ids) is _STRINGTYPE:
684 ids = {int(ids):1}
685 else:
686 d = {}
687 for id in ids.keys():
688 d[int(id)] = 1
689 ids = d
690 prop = self.ruprops[propname]
691 view = self.getview()
692 if isinstance(prop, hyperdb.Multilink):
693 def ff(row, nm=propname, ids=ids):
694 sv = getattr(row, nm)
695 for sr in sv:
696 if ids.has_key(sr.fid):
697 return 1
698 return 0
699 else:
700 def ff(row, nm=propname, ids=ids):
701 return ids.has_key(getattr(row, nm))
702 ndxview = view.filter(ff)
703 vws.append(ndxview.unique())
705 # handle the empty match case
706 if not vws:
707 return []
709 ndxview = vws[0]
710 for v in vws[1:]:
711 ndxview = ndxview.union(v)
712 view = self.getview().remapwith(ndxview)
713 rslt = []
714 for row in view:
715 rslt.append(str(row.id))
716 return rslt
719 def list(self):
720 l = []
721 for row in self.getview().select(_isdel=0):
722 l.append(str(row.id))
723 return l
724 def count(self):
725 return len(self.getview())
726 def getprops(self, protected=1):
727 # protected is not in ping's spec
728 allprops = self.ruprops.copy()
729 if protected and self.privateprops is not None:
730 allprops.update(self.privateprops)
731 return allprops
732 def addprop(self, **properties):
733 for key in properties.keys():
734 if self.ruprops.has_key(key):
735 raise ValueError, "%s is already a property of %s" % (key, self.classname)
736 self.ruprops.update(properties)
737 self.db.fastopen = 0
738 view = self.__getview()
739 self.db.commit()
740 # ---- end of ping's spec
741 def filter(self, search_matches, filterspec, sort=(None,None),
742 group=(None,None)):
743 # search_matches is None or a set (dict of {nodeid: {propname:[nodeid,...]}})
744 # filterspec is a dict {propname:value}
745 # sort and group are (dir, prop) where dir is '+', '-' or None
746 # and prop is a prop name or None
748 where = {'_isdel':0}
749 mlcriteria = {}
750 regexes = {}
751 orcriteria = {}
752 for propname, value in filterspec.items():
753 prop = self.ruprops.get(propname, None)
754 if prop is None:
755 prop = self.privateprops[propname]
756 if isinstance(prop, hyperdb.Multilink):
757 if type(value) is not _LISTTYPE:
758 value = [value]
759 # transform keys to ids
760 u = []
761 for item in value:
762 try:
763 item = int(item)
764 except (TypeError, ValueError):
765 item = int(self.db.getclass(prop.classname).lookup(item))
766 if item == -1:
767 item = 0
768 u.append(item)
769 mlcriteria[propname] = u
770 elif isinstance(prop, hyperdb.Link):
771 if type(value) is not _LISTTYPE:
772 value = [value]
773 # transform keys to ids
774 u = []
775 for item in value:
776 try:
777 item = int(item)
778 except (TypeError, ValueError):
779 item = int(self.db.getclass(prop.classname).lookup(item))
780 if item == -1:
781 item = 0
782 u.append(item)
783 if len(u) == 1:
784 where[propname] = u[0]
785 else:
786 orcriteria[propname] = u
787 elif isinstance(prop, hyperdb.String):
788 # simple glob searching
789 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', value)
790 v = v.replace('?', '.')
791 v = v.replace('*', '.*?')
792 regexes[propname] = re.compile(v, re.I)
793 elif propname == 'id':
794 where[propname] = int(value)
795 elif isinstance(prop, hyperdb.Boolean):
796 if type(value) is _STRINGTYPE:
797 bv = value.lower() in ('yes', 'true', 'on', '1')
798 else:
799 bv = value
800 where[propname] = bv
801 elif isinstance(prop, hyperdb.Number):
802 where[propname] = int(value)
803 else:
804 where[propname] = str(value)
805 v = self.getview()
806 #print "filter start at %s" % time.time()
807 if where:
808 v = v.select(where)
809 #print "filter where at %s" % time.time()
811 if mlcriteria:
812 # multilink - if any of the nodeids required by the
813 # filterspec aren't in this node's property, then skip
814 # it
815 def ff(row, ml=mlcriteria):
816 for propname, values in ml.items():
817 sv = getattr(row, propname)
818 for id in values:
819 if sv.find(fid=id) == -1:
820 return 0
821 return 1
822 iv = v.filter(ff)
823 v = v.remapwith(iv)
825 #print "filter mlcrit at %s" % time.time()
827 if orcriteria:
828 def ff(row, crit=orcriteria):
829 for propname, allowed in crit.items():
830 val = getattr(row, propname)
831 if val not in allowed:
832 return 0
833 return 1
835 iv = v.filter(ff)
836 v = v.remapwith(iv)
838 #print "filter orcrit at %s" % time.time()
839 if regexes:
840 def ff(row, r=regexes):
841 for propname, regex in r.items():
842 val = getattr(row, propname)
843 if not regex.search(val):
844 return 0
845 return 1
847 iv = v.filter(ff)
848 v = v.remapwith(iv)
849 #print "filter regexs at %s" % time.time()
851 if sort or group:
852 sortspec = []
853 rev = []
854 for dir, propname in group, sort:
855 if propname is None: continue
856 isreversed = 0
857 if dir == '-':
858 isreversed = 1
859 try:
860 prop = getattr(v, propname)
861 except AttributeError:
862 print "MK has no property %s" % propname
863 continue
864 propclass = self.ruprops.get(propname, None)
865 if propclass is None:
866 propclass = self.privateprops.get(propname, None)
867 if propclass is None:
868 print "Schema has no property %s" % propname
869 continue
870 if isinstance(propclass, hyperdb.Link):
871 linkclass = self.db.getclass(propclass.classname)
872 lv = linkclass.getview()
873 lv = lv.rename('id', propname)
874 v = v.join(lv, prop, 1)
875 if linkclass.getprops().has_key('order'):
876 propname = 'order'
877 else:
878 propname = linkclass.labelprop()
879 prop = getattr(v, propname)
880 if isreversed:
881 rev.append(prop)
882 sortspec.append(prop)
883 v = v.sortrev(sortspec, rev)[:] #XXX Metakit bug
884 #print "filter sort at %s" % time.time()
886 rslt = []
887 for row in v:
888 id = str(row.id)
889 if search_matches is not None:
890 if search_matches.has_key(id):
891 rslt.append(id)
892 else:
893 rslt.append(id)
894 return rslt
896 def hasnode(self, nodeid):
897 return int(nodeid) < self.maxid
899 def labelprop(self, default_to_id=0):
900 ''' Return the property name for a label for the given node.
902 This method attempts to generate a consistent label for the node.
903 It tries the following in order:
904 1. key property
905 2. "name" property
906 3. "title" property
907 4. first property from the sorted property name list
908 '''
909 k = self.getkey()
910 if k:
911 return k
912 props = self.getprops()
913 if props.has_key('name'):
914 return 'name'
915 elif props.has_key('title'):
916 return 'title'
917 if default_to_id:
918 return 'id'
919 props = props.keys()
920 props.sort()
921 return props[0]
922 def stringFind(self, **requirements):
923 """Locate a particular node by matching a set of its String
924 properties in a caseless search.
926 If the property is not a String property, a TypeError is raised.
928 The return is a list of the id of all nodes that match.
929 """
930 for propname in requirements.keys():
931 prop = self.properties[propname]
932 if isinstance(not prop, hyperdb.String):
933 raise TypeError, "'%s' not a String property"%propname
934 requirements[propname] = requirements[propname].lower()
935 requirements['_isdel'] = 0
937 l = []
938 for row in self.getview().select(requirements):
939 l.append(str(row.id))
940 return l
942 def addjournal(self, nodeid, action, params):
943 self.db.addjournal(self.classname, nodeid, action, params)
945 def index(self, nodeid):
946 ''' Add (or refresh) the node to search indexes '''
947 # find all the String properties that have indexme
948 for prop, propclass in self.getprops().items():
949 if isinstance(propclass, hyperdb.String) and propclass.indexme:
950 # index them under (classname, nodeid, property)
951 self.db.indexer.add_text((self.classname, nodeid, prop),
952 str(self.get(nodeid, prop)))
954 def export_list(self, propnames, nodeid):
955 ''' Export a node - generate a list of CSV-able data in the order
956 specified by propnames for the given node.
957 '''
958 properties = self.getprops()
959 l = []
960 for prop in propnames:
961 proptype = properties[prop]
962 value = self.get(nodeid, prop)
963 # "marshal" data where needed
964 if value is None:
965 pass
966 elif isinstance(proptype, hyperdb.Date):
967 value = value.get_tuple()
968 elif isinstance(proptype, hyperdb.Interval):
969 value = value.get_tuple()
970 elif isinstance(proptype, hyperdb.Password):
971 value = str(value)
972 l.append(repr(value))
973 return l
975 def import_list(self, propnames, proplist):
976 ''' Import a node - all information including "id" is present and
977 should not be sanity checked. Triggers are not triggered. The
978 journal should be initialised using the "creator" and "creation"
979 information.
981 Return the nodeid of the node imported.
982 '''
983 if self.db.journaltag is None:
984 raise hyperdb.DatabaseError, 'Database open read-only'
985 properties = self.getprops()
987 d = {}
988 view = self.getview(1)
989 for i in range(len(propnames)):
990 value = eval(proplist[i])
991 propname = propnames[i]
992 prop = properties[propname]
993 if propname == 'id':
994 newid = value
995 value = int(value)
996 elif isinstance(prop, hyperdb.Date):
997 value = int(calendar.timegm(value))
998 elif isinstance(prop, hyperdb.Interval):
999 value = str(date.Interval(value))
1000 d[propname] = value
1001 view.append(d)
1002 creator = d.get('creator', None)
1003 creation = d.get('creation', None)
1004 self.db.addjournal(self.classname, newid, 'create', {}, creator, creation)
1005 return newid
1007 # --- used by Database
1008 def _commit(self):
1009 """ called post commit of the DB.
1010 interested subclasses may override """
1011 self.uncommitted = {}
1012 self.rbactions = []
1013 self.idcache = {}
1014 def _rollback(self):
1015 """ called pre rollback of the DB.
1016 interested subclasses may override """
1017 for action in self.rbactions:
1018 action()
1019 self.rbactions = []
1020 self.uncommitted = {}
1021 self.idcache = {}
1022 def _clear(self):
1023 view = self.getview(1)
1024 if len(view):
1025 view[:] = []
1026 self.db.dirty = 1
1027 iv = self.getindexview(1)
1028 if iv:
1029 iv[:] = []
1030 def rollbackaction(self, action):
1031 """ call this to register a callback called on rollback
1032 callback is removed on end of transaction """
1033 self.rbactions.append(action)
1034 # --- internal
1035 def __getview(self):
1036 db = self.db._db
1037 view = db.view(self.classname)
1038 mkprops = view.structure()
1039 if mkprops and self.db.fastopen:
1040 return view.ordered(1)
1041 # is the definition the same?
1042 for nm, rutyp in self.ruprops.items():
1043 for mkprop in mkprops:
1044 if mkprop.name == nm:
1045 break
1046 else:
1047 mkprop = None
1048 if mkprop is None:
1049 break
1050 if _typmap[rutyp.__class__] != mkprop.type:
1051 break
1052 else:
1053 return view.ordered(1)
1054 # need to create or restructure the mk view
1055 # id comes first, so MK will order it for us
1056 self.db.dirty = 1
1057 s = ["%s[id:I" % self.classname]
1058 for nm, rutyp in self.ruprops.items():
1059 mktyp = _typmap[rutyp.__class__]
1060 s.append('%s:%s' % (nm, mktyp))
1061 if mktyp == 'V':
1062 s[-1] += ('[fid:I]')
1063 s.append('_isdel:I,activity:I,creation:I,creator:I]')
1064 v = self.db._db.getas(','.join(s))
1065 self.db.commit()
1066 return v.ordered(1)
1067 def getview(self, RW=0):
1068 return self.db._db.view(self.classname).ordered(1)
1069 def getindexview(self, RW=0):
1070 return self.db._db.view("_%s" % self.classname).ordered(1)
1072 def _fetchML(sv):
1073 l = []
1074 for row in sv:
1075 if row.fid:
1076 l.append(str(row.fid))
1077 return l
1079 def _fetchPW(s):
1080 p = password.Password()
1081 p.unpack(s)
1082 return p
1084 def _fetchLink(n):
1085 return n and str(n) or None
1087 def _fetchDate(n):
1088 return date.Date(time.gmtime(n))
1090 _converters = {
1091 hyperdb.Date : _fetchDate,
1092 hyperdb.Link : _fetchLink,
1093 hyperdb.Multilink : _fetchML,
1094 hyperdb.Interval : date.Interval,
1095 hyperdb.Password : _fetchPW,
1096 hyperdb.Boolean : lambda n: n,
1097 hyperdb.Number : lambda n: n,
1098 hyperdb.String : str,
1099 }
1101 class FileName(hyperdb.String):
1102 isfilename = 1
1104 _typmap = {
1105 FileName : 'S',
1106 hyperdb.String : 'S',
1107 hyperdb.Date : 'I',
1108 hyperdb.Link : 'I',
1109 hyperdb.Multilink : 'V',
1110 hyperdb.Interval : 'S',
1111 hyperdb.Password : 'S',
1112 hyperdb.Boolean : 'I',
1113 hyperdb.Number : 'I',
1114 }
1115 class FileClass(Class):
1116 ' like Class but with a content property '
1117 default_mime_type = 'text/plain'
1118 def __init__(self, db, classname, **properties):
1119 properties['content'] = FileName()
1120 if not properties.has_key('type'):
1121 properties['type'] = hyperdb.String()
1122 Class.__init__(self, db, classname, **properties)
1123 def get(self, nodeid, propname, default=_marker, cache=1):
1124 x = Class.get(self, nodeid, propname, default, cache)
1125 if propname == 'content':
1126 if x.startswith('file:'):
1127 fnm = x[5:]
1128 try:
1129 x = open(fnm, 'rb').read()
1130 except Exception, e:
1131 x = repr(e)
1132 return x
1133 def create(self, **propvalues):
1134 content = propvalues['content']
1135 del propvalues['content']
1136 newid = Class.create(self, **propvalues)
1137 if not content:
1138 return newid
1139 nm = bnm = '%s%s' % (self.classname, newid)
1140 sd = str(int(int(newid) / 1000))
1141 d = os.path.join(self.db.config.DATABASE, 'files', self.classname, sd)
1142 if not os.path.exists(d):
1143 os.makedirs(d)
1144 nm = os.path.join(d, nm)
1145 open(nm, 'wb').write(content)
1146 self.set(newid, content = 'file:'+nm)
1147 mimetype = propvalues.get('type', self.default_mime_type)
1148 self.db.indexer.add_text((self.classname, newid, 'content'), content, mimetype)
1149 def undo(fnm=nm, action1=os.remove, indexer=self.db.indexer):
1150 action1(fnm)
1151 self.rollbackaction(undo)
1152 return newid
1153 def index(self, nodeid):
1154 Class.index(self, nodeid)
1155 mimetype = self.get(nodeid, 'type')
1156 if not mimetype:
1157 mimetype = self.default_mime_type
1158 self.db.indexer.add_text((self.classname, nodeid, 'content'),
1159 self.get(nodeid, 'content'), mimetype)
1161 class IssueClass(Class, roundupdb.IssueClass):
1162 # Overridden methods:
1163 def __init__(self, db, classname, **properties):
1164 """The newly-created class automatically includes the "messages",
1165 "files", "nosy", and "superseder" properties. If the 'properties'
1166 dictionary attempts to specify any of these properties or a
1167 "creation" or "activity" property, a ValueError is raised."""
1168 if not properties.has_key('title'):
1169 properties['title'] = hyperdb.String(indexme='yes')
1170 if not properties.has_key('messages'):
1171 properties['messages'] = hyperdb.Multilink("msg")
1172 if not properties.has_key('files'):
1173 properties['files'] = hyperdb.Multilink("file")
1174 if not properties.has_key('nosy'):
1175 # note: journalling is turned off as it really just wastes
1176 # space. this behaviour may be overridden in an instance
1177 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
1178 if not properties.has_key('superseder'):
1179 properties['superseder'] = hyperdb.Multilink(classname)
1180 Class.__init__(self, db, classname, **properties)
1182 CURVERSION = 2
1184 class Indexer(indexer.Indexer):
1185 disallows = {'THE':1, 'THIS':1, 'ZZZ':1, 'THAT':1, 'WITH':1}
1186 def __init__(self, path, datadb):
1187 self.path = os.path.join(path, 'index.mk4')
1188 self.db = metakit.storage(self.path, 1)
1189 self.datadb = datadb
1190 self.reindex = 0
1191 v = self.db.view('version')
1192 if not v.structure():
1193 v = self.db.getas('version[vers:I]')
1194 self.db.commit()
1195 v.append(vers=CURVERSION)
1196 self.reindex = 1
1197 elif v[0].vers != CURVERSION:
1198 v[0].vers = CURVERSION
1199 self.reindex = 1
1200 if self.reindex:
1201 self.db.getas('ids[tblid:I,nodeid:I,propid:I,ignore:I]')
1202 self.db.getas('index[word:S,hits[pos:I]]')
1203 self.db.commit()
1204 self.reindex = 1
1205 self.changed = 0
1206 self.propcache = {}
1207 def force_reindex(self):
1208 v = self.db.view('ids')
1209 v[:] = []
1210 v = self.db.view('index')
1211 v[:] = []
1212 self.db.commit()
1213 self.reindex = 1
1214 def should_reindex(self):
1215 return self.reindex
1216 def _getprops(self, classname):
1217 props = self.propcache.get(classname, None)
1218 if props is None:
1219 props = self.datadb.view(classname).structure()
1220 props = [prop.name for prop in props]
1221 self.propcache[classname] = props
1222 return props
1223 def _getpropid(self, classname, propname):
1224 return self._getprops(classname).index(propname)
1225 def _getpropname(self, classname, propid):
1226 return self._getprops(classname)[propid]
1228 def add_text(self, identifier, text, mime_type='text/plain'):
1229 if mime_type != 'text/plain':
1230 return
1231 classname, nodeid, property = identifier
1232 tbls = self.datadb.view('tables')
1233 tblid = tbls.find(name=classname)
1234 if tblid < 0:
1235 raise KeyError, "unknown class %r"%classname
1236 nodeid = int(nodeid)
1237 propid = self._getpropid(classname, property)
1238 ids = self.db.view('ids')
1239 oldpos = ids.find(tblid=tblid,nodeid=nodeid,propid=propid,ignore=0)
1240 if oldpos > -1:
1241 ids[oldpos].ignore = 1
1242 self.changed = 1
1243 pos = ids.append(tblid=tblid,nodeid=nodeid,propid=propid)
1245 wordlist = re.findall(r'\b\w{2,25}\b', text.upper())
1246 words = {}
1247 for word in wordlist:
1248 if not self.disallows.has_key(word):
1249 words[word] = 1
1250 words = words.keys()
1252 index = self.db.view('index').ordered(1)
1253 for word in words:
1254 ndx = index.find(word=word)
1255 if ndx < 0:
1256 index.append(word=word)
1257 ndx = index.find(word=word)
1258 index[ndx].hits.append(pos=pos)
1259 self.changed = 1
1261 def find(self, wordlist):
1262 hits = None
1263 index = self.db.view('index').ordered(1)
1264 for word in wordlist:
1265 word = word.upper()
1266 if not 2 < len(word) < 26:
1267 continue
1268 ndx = index.find(word=word)
1269 if ndx < 0:
1270 return {}
1271 if hits is None:
1272 hits = index[ndx].hits
1273 else:
1274 hits = hits.intersect(index[ndx].hits)
1275 if len(hits) == 0:
1276 return {}
1277 if hits is None:
1278 return {}
1279 rslt = {}
1280 ids = self.db.view('ids').remapwith(hits)
1281 tbls = self.datadb.view('tables')
1282 for i in range(len(ids)):
1283 hit = ids[i]
1284 if not hit.ignore:
1285 classname = tbls[hit.tblid].name
1286 nodeid = str(hit.nodeid)
1287 property = self._getpropname(classname, hit.propid)
1288 rslt[i] = (classname, nodeid, property)
1289 return rslt
1290 def save_index(self):
1291 if self.changed:
1292 self.db.commit()
1293 self.changed = 0
1294 def rollback(self):
1295 if self.changed:
1296 self.db.rollback()
1297 self.db = metakit.storage(self.path, 1)
1298 self.changed = 0