eeb93dbb5380be389127bb53d72d7f1476b14dac
1 from roundup import hyperdb, date, password, roundupdb, security
2 import metakit
3 from sessions import Sessions
4 import re, marshal, os, sys, weakref, time, calendar
5 from roundup import indexer
6 import locking
8 _dbs = {}
10 def Database(config, journaltag=None):
11 db = _dbs.get(config.DATABASE, None)
12 if db is None or db._db is None:
13 db = _Database(config, journaltag)
14 _dbs[config.DATABASE] = db
15 else:
16 db.journaltag = journaltag
17 try:
18 delattr(db, 'curuserid')
19 except AttributeError:
20 pass
21 return db
23 class _Database(hyperdb.Database):
24 def __init__(self, config, journaltag=None):
25 self.config = config
26 self.journaltag = journaltag
27 self.classes = {}
28 self.dirty = 0
29 self.lockfile = None
30 self._db = self.__open()
31 self.indexer = Indexer(self.config.DATABASE, self._db)
32 self.sessions = Sessions(self.config)
33 self.security = security.Security(self)
35 os.umask(0002)
37 def post_init(self):
38 if self.indexer.should_reindex():
39 self.reindex()
41 def reindex(self):
42 for klass in self.classes.values():
43 for nodeid in klass.list():
44 klass.index(nodeid)
45 self.indexer.save_index()
48 # --- defined in ping's spec
49 def __getattr__(self, classname):
50 if classname == 'curuserid':
51 if self.journaltag is None:
52 return None
54 try:
55 self.curuserid = x = int(self.classes['user'].lookup(self.journaltag))
56 except KeyError:
57 if self.journaltag == 'admin':
58 self.curuserid = x = 1
59 else:
60 x = 0
61 return x
62 elif classname == 'transactions':
63 return self.dirty
64 return self.getclass(classname)
65 def getclass(self, classname):
66 try:
67 return self.classes[classname]
68 except KeyError:
69 raise KeyError, 'There is no class called "%s"'%classname
70 def getclasses(self):
71 return self.classes.keys()
72 # --- end of ping's spec
73 # --- exposed methods
74 def commit(self):
75 if self.dirty:
76 self._db.commit()
77 for cl in self.classes.values():
78 cl._commit()
79 self.indexer.save_index()
80 self.dirty = 0
81 def rollback(self):
82 if self.dirty:
83 for cl in self.classes.values():
84 cl._rollback()
85 self._db.rollback()
86 self._db = None
87 self._db = metakit.storage(self.dbnm, 1)
88 self.hist = self._db.view('history')
89 self.tables = self._db.view('tables')
90 self.indexer.datadb = self._db
91 self.dirty = 0
92 def clear(self):
93 for cl in self.classes.values():
94 cl._clear()
95 def hasnode(self, classname, nodeid):
96 return self.getclass(classname).hasnode(nodeid)
97 def pack(self, pack_before):
98 mindate = int(calendar.timegm(pack_before.get_tuple()))
99 i = 0
100 while i < len(self.hist):
101 if self.hist[i].date < mindate and self.hist[i].action != _CREATE:
102 self.hist.delete(i)
103 else:
104 i = i + 1
105 def addclass(self, cl):
106 self.classes[cl.classname] = cl
107 if self.tables.find(name=cl.classname) < 0:
108 self.tables.append(name=cl.classname)
109 def addjournal(self, tablenm, nodeid, action, params, creator=None,
110 creation=None):
111 tblid = self.tables.find(name=tablenm)
112 if tblid == -1:
113 tblid = self.tables.append(name=tablenm)
114 if creator is None:
115 creator = self.curuserid
116 else:
117 try:
118 creator = int(creator)
119 except TypeError:
120 creator = int(self.getclass('user').lookup(creator))
121 if creation is None:
122 creation = int(time.time())
123 elif isinstance(creation, date.Date):
124 creation = int(calendar.timegm(creation.get_tuple()))
125 # tableid:I,nodeid:I,date:I,user:I,action:I,params:B
126 self.hist.append(tableid=tblid,
127 nodeid=int(nodeid),
128 date=creation,
129 action=action,
130 user = creator,
131 params = marshal.dumps(params))
132 def getjournal(self, tablenm, nodeid):
133 rslt = []
134 tblid = self.tables.find(name=tablenm)
135 if tblid == -1:
136 return rslt
137 q = self.hist.select(tableid=tblid, nodeid=int(nodeid))
138 if len(q) == 0:
139 raise IndexError, "no history for id %s in %s" % (nodeid, tablenm)
140 i = 0
141 #userclass = self.getclass('user')
142 for row in q:
143 try:
144 params = marshal.loads(row.params)
145 except ValueError:
146 print "history couldn't unmarshal %r" % row.params
147 params = {}
148 #usernm = userclass.get(str(row.user), 'username')
149 dt = date.Date(time.gmtime(row.date))
150 #rslt.append((nodeid, dt, usernm, _actionnames[row.action], params))
151 rslt.append((nodeid, dt, str(row.user), _actionnames[row.action], params))
152 return rslt
154 def destroyjournal(self, tablenm, nodeid):
155 nodeid = int(nodeid)
156 tblid = self.tables.find(name=tablenm)
157 if tblid == -1:
158 return
159 i = 0
160 hist = self.hist
161 while i < len(hist):
162 if hist[i].tableid == tblid and hist[i].nodeid == nodeid:
163 hist.delete(i)
164 else:
165 i = i + 1
166 self.dirty = 1
168 def close(self):
169 for cl in self.classes.values():
170 cl.db = None
171 self._db = None
172 if self.lockfile is not None:
173 locking.release_lock(self.lockfile)
174 if _dbs.has_key(self.config.DATABASE):
175 del _dbs[self.config.DATABASE]
176 if self.lockfile is not None:
177 self.lockfile.close()
178 self.lockfile = None
179 self.classes = {}
180 self.indexer = None
182 # --- internal
183 def __open(self):
184 self.dbnm = db = os.path.join(self.config.DATABASE, 'tracker.mk4')
185 lockfilenm = db[:-3]+'lck'
186 self.lockfile = locking.acquire_lock(lockfilenm)
187 self.lockfile.write(str(os.getpid()))
188 self.lockfile.flush()
189 self.fastopen = 0
190 if os.path.exists(db):
191 dbtm = os.path.getmtime(db)
192 pkgnm = self.config.__name__.split('.')[0]
193 schemamod = sys.modules.get(pkgnm+'.dbinit', None)
194 if schemamod:
195 if os.path.exists(schemamod.__file__):
196 schematm = os.path.getmtime(schemamod.__file__)
197 if schematm < dbtm:
198 # found schema mod - it's older than the db
199 self.fastopen = 1
200 else:
201 # can't find schemamod - must be frozen
202 self.fastopen = 1
203 db = metakit.storage(db, 1)
204 hist = db.view('history')
205 tables = db.view('tables')
206 if not self.fastopen:
207 if not hist.structure():
208 hist = db.getas('history[tableid:I,nodeid:I,date:I,user:I,action:I,params:B]')
209 if not tables.structure():
210 tables = db.getas('tables[name:S]')
211 db.commit()
212 self.tables = tables
213 self.hist = hist
214 return db
216 _STRINGTYPE = type('')
217 _LISTTYPE = type([])
218 _CREATE, _SET, _RETIRE, _LINK, _UNLINK = range(5)
220 _actionnames = {
221 _CREATE : 'create',
222 _SET : 'set',
223 _RETIRE : 'retire',
224 _LINK : 'link',
225 _UNLINK : 'unlink',
226 }
228 _marker = []
230 _ALLOWSETTINGPRIVATEPROPS = 0
232 class Class:
233 privateprops = None
234 def __init__(self, db, classname, **properties):
235 #self.db = weakref.proxy(db)
236 self.db = db
237 self.classname = classname
238 self.keyname = None
239 self.ruprops = properties
240 self.privateprops = { 'id' : hyperdb.String(),
241 'activity' : hyperdb.Date(),
242 'creation' : hyperdb.Date(),
243 'creator' : hyperdb.Link('user') }
244 self.auditors = {'create': [], 'set': [], 'retire': []} # event -> list of callables
245 self.reactors = {'create': [], 'set': [], 'retire': []} # ditto
246 view = self.__getview()
247 self.maxid = 1
248 if view:
249 self.maxid = view[-1].id + 1
250 self.uncommitted = {}
251 self.rbactions = []
252 # people reach inside!!
253 self.properties = self.ruprops
254 self.db.addclass(self)
255 self.idcache = {}
257 # default is to journal changes
258 self.do_journal = 1
260 def enableJournalling(self):
261 '''Turn journalling on for this class
262 '''
263 self.do_journal = 1
265 def disableJournalling(self):
266 '''Turn journalling off for this class
267 '''
268 self.do_journal = 0
270 # --- the roundup.Class methods
271 def audit(self, event, detector):
272 l = self.auditors[event]
273 if detector not in l:
274 self.auditors[event].append(detector)
275 def fireAuditors(self, action, nodeid, newvalues):
276 for audit in self.auditors[action]:
277 audit(self.db, self, nodeid, newvalues)
278 def fireReactors(self, action, nodeid, oldvalues):
279 for react in self.reactors[action]:
280 react(self.db, self, nodeid, oldvalues)
281 def react(self, event, detector):
282 l = self.reactors[event]
283 if detector not in l:
284 self.reactors[event].append(detector)
285 # --- the hyperdb.Class methods
286 def create(self, **propvalues):
287 self.fireAuditors('create', None, propvalues)
288 rowdict = {}
289 rowdict['id'] = newid = self.maxid
290 self.maxid += 1
291 ndx = self.getview(1).append(rowdict)
292 propvalues['#ISNEW'] = 1
293 try:
294 self.set(str(newid), **propvalues)
295 except Exception:
296 self.maxid -= 1
297 raise
298 return str(newid)
300 def get(self, nodeid, propname, default=_marker, cache=1):
301 # default and cache aren't in the spec
302 # cache=0 means "original value"
304 view = self.getview()
305 id = int(nodeid)
306 if cache == 0:
307 oldnode = self.uncommitted.get(id, None)
308 if oldnode and oldnode.has_key(propname):
309 return oldnode[propname]
310 ndx = self.idcache.get(id, None)
311 if ndx is None:
312 ndx = view.find(id=id)
313 if ndx < 0:
314 raise IndexError, "%s has no node %s" % (self.classname, nodeid)
315 self.idcache[id] = ndx
316 try:
317 raw = getattr(view[ndx], propname)
318 except AttributeError:
319 raise KeyError, propname
320 rutyp = self.ruprops.get(propname, None)
321 if rutyp is None:
322 rutyp = self.privateprops[propname]
323 converter = _converters.get(rutyp.__class__, None)
324 if converter:
325 raw = converter(raw)
326 return raw
328 def set(self, nodeid, **propvalues):
329 isnew = 0
330 if propvalues.has_key('#ISNEW'):
331 isnew = 1
332 del propvalues['#ISNEW']
333 if not isnew:
334 self.fireAuditors('set', nodeid, propvalues)
335 if not propvalues:
336 return propvalues
337 if propvalues.has_key('id'):
338 raise KeyError, '"id" is reserved'
339 if self.db.journaltag is None:
340 raise hyperdb.DatabaseError, 'Database open read-only'
341 view = self.getview(1)
342 # node must exist & not be retired
343 id = int(nodeid)
344 ndx = view.find(id=id)
345 if ndx < 0:
346 raise IndexError, "%s has no node %s" % (self.classname, nodeid)
347 row = view[ndx]
348 if row._isdel:
349 raise IndexError, "%s has no node %s" % (self.classname, nodeid)
350 oldnode = self.uncommitted.setdefault(id, {})
351 changes = {}
353 for key, value in propvalues.items():
354 # this will raise the KeyError if the property isn't valid
355 # ... we don't use getprops() here because we only care about
356 # the writeable properties.
357 if _ALLOWSETTINGPRIVATEPROPS:
358 prop = self.ruprops.get(key, None)
359 if not prop:
360 prop = self.privateprops[key]
361 else:
362 prop = self.ruprops[key]
363 converter = _converters.get(prop.__class__, lambda v: v)
364 # if the value's the same as the existing value, no sense in
365 # doing anything
366 oldvalue = converter(getattr(row, key))
367 if value == oldvalue:
368 del propvalues[key]
369 continue
371 # check to make sure we're not duplicating an existing key
372 if key == self.keyname:
373 iv = self.getindexview(1)
374 ndx = iv.find(k=value)
375 if ndx == -1:
376 iv.append(k=value, i=row.id)
377 if not isnew:
378 ndx = iv.find(k=oldvalue)
379 if ndx > -1:
380 iv.delete(ndx)
381 else:
382 raise ValueError, 'node with key "%s" exists'%value
384 # do stuff based on the prop type
385 if isinstance(prop, hyperdb.Link):
386 link_class = prop.classname
387 # must be a string or None
388 if value is not None and not isinstance(value, type('')):
389 raise ValueError, 'property "%s" link value be a string'%(
390 key)
391 # Roundup sets to "unselected" by passing None
392 if value is None:
393 value = 0
394 # if it isn't a number, it's a key
395 try:
396 int(value)
397 except ValueError:
398 try:
399 value = self.db.getclass(link_class).lookup(value)
400 except (TypeError, KeyError):
401 raise IndexError, 'new property "%s": %s not a %s'%(
402 key, value, prop.classname)
404 if (value is not None and
405 not self.db.getclass(link_class).hasnode(value)):
406 raise IndexError, '%s has no node %s'%(link_class, value)
408 setattr(row, key, int(value))
409 changes[key] = oldvalue
411 if self.do_journal and prop.do_journal:
412 # register the unlink with the old linked node
413 if oldvalue:
414 self.db.addjournal(link_class, value, _UNLINK,
415 (self.classname, str(row.id), key))
417 # register the link with the newly linked node
418 if value:
419 self.db.addjournal(link_class, value, _LINK,
420 (self.classname, str(row.id), key))
422 elif isinstance(prop, hyperdb.Multilink):
423 if value is not None and type(value) != _LISTTYPE:
424 raise TypeError, 'new property "%s" not a list of ids'%key
425 link_class = prop.classname
426 l = []
427 if value is None:
428 value = []
429 for entry in value:
430 if type(entry) != _STRINGTYPE:
431 raise ValueError, 'new property "%s" link value ' \
432 'must be a string'%key
433 # if it isn't a number, it's a key
434 try:
435 int(entry)
436 except ValueError:
437 try:
438 entry = self.db.getclass(link_class).lookup(entry)
439 except (TypeError, KeyError):
440 raise IndexError, 'new property "%s": %s not a %s'%(
441 key, entry, prop.classname)
442 l.append(entry)
443 propvalues[key] = value = l
445 # handle removals
446 rmvd = []
447 for id in oldvalue:
448 if id not in value:
449 rmvd.append(id)
450 # register the unlink with the old linked node
451 if self.do_journal and prop.do_journal:
452 self.db.addjournal(link_class, id, _UNLINK,
453 (self.classname, str(row.id), key))
455 # handle additions
456 adds = []
457 for id in value:
458 if id not in oldvalue:
459 if not self.db.getclass(link_class).hasnode(id):
460 raise IndexError, '%s has no node %s'%(
461 link_class, id)
462 adds.append(id)
463 # register the link with the newly linked node
464 if self.do_journal and prop.do_journal:
465 self.db.addjournal(link_class, id, _LINK,
466 (self.classname, str(row.id), key))
468 sv = getattr(row, key)
469 i = 0
470 while i < len(sv):
471 if str(sv[i].fid) in rmvd:
472 sv.delete(i)
473 else:
474 i += 1
475 for id in adds:
476 sv.append(fid=int(id))
477 changes[key] = oldvalue
478 if not rmvd and not adds:
479 del propvalues[key]
481 elif isinstance(prop, hyperdb.String):
482 if value is not None and type(value) != _STRINGTYPE:
483 raise TypeError, 'new property "%s" not a string'%key
484 if value is None:
485 value = ''
486 setattr(row, key, value)
487 changes[key] = oldvalue
488 if hasattr(prop, 'isfilename') and prop.isfilename:
489 propvalues[key] = os.path.basename(value)
490 if prop.indexme and value is not None:
491 self.db.indexer.add_text((self.classname, nodeid, key),
492 value, 'text/plain')
494 elif isinstance(prop, hyperdb.Password):
495 if value is not None and not isinstance(value, password.Password):
496 raise TypeError, 'new property "%s" not a Password'% key
497 if value is None:
498 value = ''
499 setattr(row, key, str(value))
500 changes[key] = str(oldvalue)
501 propvalues[key] = str(value)
503 elif isinstance(prop, hyperdb.Date):
504 if value is not None and not isinstance(value, date.Date):
505 raise TypeError, 'new property "%s" not a Date'% key
506 if value is None:
507 setattr(row, key, 0)
508 else:
509 setattr(row, key, int(calendar.timegm(value.get_tuple())))
510 changes[key] = str(oldvalue)
511 propvalues[key] = str(value)
513 elif isinstance(prop, hyperdb.Interval):
514 if value is not None and not isinstance(value, date.Interval):
515 raise TypeError, 'new property "%s" not an Interval'% key
516 if value is None:
517 setattr(row, key, '')
518 else:
519 setattr(row, key, str(value))
520 changes[key] = str(oldvalue)
521 propvalues[key] = str(value)
523 elif isinstance(prop, hyperdb.Number):
524 if value is None:
525 value = 0
526 try:
527 v = int(value)
528 except ValueError:
529 raise TypeError, "%s (%s) is not numeric" % (key, repr(value))
530 setattr(row, key, v)
531 changes[key] = oldvalue
532 propvalues[key] = value
534 elif isinstance(prop, hyperdb.Boolean):
535 if value is None:
536 bv = 0
537 elif value not in (0,1):
538 raise TypeError, "%s (%s) is not boolean" % (key, repr(value))
539 else:
540 bv = value
541 setattr(row, key, bv)
542 changes[key] = oldvalue
543 propvalues[key] = value
545 oldnode[key] = oldvalue
547 # nothing to do?
548 if not propvalues:
549 return propvalues
550 if not propvalues.has_key('activity'):
551 row.activity = int(time.time())
552 if isnew:
553 if not row.creation:
554 row.creation = int(time.time())
555 if not row.creator:
556 row.creator = self.db.curuserid
558 self.db.dirty = 1
559 if self.do_journal:
560 if isnew:
561 self.db.addjournal(self.classname, nodeid, _CREATE, {})
562 self.fireReactors('create', nodeid, None)
563 else:
564 self.db.addjournal(self.classname, nodeid, _SET, changes)
565 self.fireReactors('set', nodeid, oldnode)
567 return propvalues
569 def retire(self, nodeid):
570 if self.db.journaltag is None:
571 raise hyperdb.DatabaseError, 'Database open read-only'
572 self.fireAuditors('retire', nodeid, None)
573 view = self.getview(1)
574 ndx = view.find(id=int(nodeid))
575 if ndx < 0:
576 raise KeyError, "nodeid %s not found" % nodeid
577 row = view[ndx]
578 oldvalues = self.uncommitted.setdefault(row.id, {})
579 oldval = oldvalues['_isdel'] = row._isdel
580 row._isdel = 1
581 if self.do_journal:
582 self.db.addjournal(self.classname, nodeid, _RETIRE, {})
583 if self.keyname:
584 iv = self.getindexview(1)
585 ndx = iv.find(k=getattr(row, self.keyname),i=row.id)
586 if ndx > -1:
587 iv.delete(ndx)
588 self.db.dirty = 1
589 self.fireReactors('retire', nodeid, None)
590 def history(self, nodeid):
591 if not self.do_journal:
592 raise ValueError, 'Journalling is disabled for this class'
593 return self.db.getjournal(self.classname, nodeid)
594 def setkey(self, propname):
595 if self.keyname:
596 if propname == self.keyname:
597 return
598 raise ValueError, "%s already indexed on %s" % (self.classname, self.keyname)
599 prop = self.properties.get(propname, None)
600 if prop is None:
601 prop = self.privateprops.get(propname, None)
602 if prop is None:
603 raise KeyError, "no property %s" % propname
604 if not isinstance(prop, hyperdb.String):
605 raise TypeError, "%s is not a String" % propname
606 # first setkey for this run
607 self.keyname = propname
608 iv = self.db._db.view('_%s' % self.classname)
609 if self.db.fastopen and iv.structure():
610 return
611 # very first setkey ever
612 self.db.dirty = 1
613 iv = self.db._db.getas('_%s[k:S,i:I]' % self.classname)
614 iv = iv.ordered(1)
615 # print "setkey building index"
616 for row in self.getview():
617 iv.append(k=getattr(row, propname), i=row.id)
618 self.db.commit()
619 def getkey(self):
620 return self.keyname
621 def lookup(self, keyvalue):
622 if type(keyvalue) is not _STRINGTYPE:
623 raise TypeError, "%r is not a string" % keyvalue
624 iv = self.getindexview()
625 if iv:
626 ndx = iv.find(k=keyvalue)
627 if ndx > -1:
628 return str(iv[ndx].i)
629 else:
630 view = self.getview()
631 ndx = view.find({self.keyname:keyvalue, '_isdel':0})
632 if ndx > -1:
633 return str(view[ndx].id)
634 raise KeyError, keyvalue
636 def destroy(self, id):
637 view = self.getview(1)
638 ndx = view.find(id=int(id))
639 if ndx > -1:
640 if self.keyname:
641 keyvalue = getattr(view[ndx], self.keyname)
642 iv = self.getindexview(1)
643 if iv:
644 ivndx = iv.find(k=keyvalue)
645 if ivndx > -1:
646 iv.delete(ivndx)
647 view.delete(ndx)
648 self.db.destroyjournal(self.classname, id)
649 self.db.dirty = 1
651 def find(self, **propspec):
652 """Get the ids of nodes in this class which link to the given nodes.
654 'propspec' consists of keyword args propname={nodeid:1,}
655 'propname' must be the name of a property in this class, or a
656 KeyError is raised. That property must be a Link or
657 Multilink property, or a TypeError is raised.
659 Any node in this class whose propname property links to any of the
660 nodeids will be returned. Used by the full text indexing, which knows
661 that "foo" occurs in msg1, msg3 and file7; so we have hits on these
662 issues:
664 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
666 """
667 propspec = propspec.items()
668 for propname, nodeid in propspec:
669 # check the prop is OK
670 prop = self.ruprops[propname]
671 if (not isinstance(prop, hyperdb.Link) and
672 not isinstance(prop, hyperdb.Multilink)):
673 raise TypeError, "'%s' not a Link/Multilink property"%propname
675 vws = []
676 for propname, ids in propspec:
677 if type(ids) is _STRINGTYPE:
678 ids = {ids:1}
679 prop = self.ruprops[propname]
680 view = self.getview()
681 if isinstance(prop, hyperdb.Multilink):
682 view = view.flatten(getattr(view, propname))
683 def ff(row, nm=propname, ids=ids):
684 return ids.has_key(str(row.fid))
685 else:
686 def ff(row, nm=propname, ids=ids):
687 return ids.has_key(str(getattr(row, nm)))
688 ndxview = view.filter(ff)
689 vws.append(ndxview.unique())
691 # handle the empty match case
692 if not vws:
693 return []
695 ndxview = vws[0]
696 for v in vws[1:]:
697 ndxview = ndxview.union(v)
698 view = view.remapwith(ndxview)
699 rslt = []
700 for row in view:
701 rslt.append(str(row.id))
702 return rslt
705 def list(self):
706 l = []
707 for row in self.getview().select(_isdel=0):
708 l.append(str(row.id))
709 return l
710 def count(self):
711 return len(self.getview())
712 def getprops(self, protected=1):
713 # protected is not in ping's spec
714 allprops = self.ruprops.copy()
715 if protected and self.privateprops is not None:
716 allprops.update(self.privateprops)
717 return allprops
718 def addprop(self, **properties):
719 for key in properties.keys():
720 if self.ruprops.has_key(key):
721 raise ValueError, "%s is already a property of %s" % (key, self.classname)
722 self.ruprops.update(properties)
723 self.db.fastopen = 0
724 view = self.__getview()
725 self.db.commit()
726 # ---- end of ping's spec
727 def filter(self, search_matches, filterspec, sort, group):
728 # search_matches is None or a set (dict of {nodeid: {propname:[nodeid,...]}})
729 # filterspec is a dict {propname:value}
730 # sort and group are lists of propnames
731 # sort and group are (dir, prop) where dir is '+', '-' or None
732 # and prop is a prop name or None
734 where = {'_isdel':0}
735 mlcriteria = {}
736 regexes = {}
737 orcriteria = {}
738 for propname, value in filterspec.items():
739 prop = self.ruprops.get(propname, None)
740 if prop is None:
741 prop = self.privateprops[propname]
742 if isinstance(prop, hyperdb.Multilink):
743 if type(value) is not _LISTTYPE:
744 value = [value]
745 # transform keys to ids
746 u = []
747 for item in value:
748 try:
749 item = int(item)
750 except (TypeError, ValueError):
751 item = int(self.db.getclass(prop.classname).lookup(item))
752 if item == -1:
753 item = 0
754 u.append(item)
755 mlcriteria[propname] = u
756 elif isinstance(prop, hyperdb.Link):
757 if type(value) is not _LISTTYPE:
758 value = [value]
759 # transform keys to ids
760 u = []
761 for item in value:
762 try:
763 item = int(item)
764 except (TypeError, ValueError):
765 item = int(self.db.getclass(prop.classname).lookup(item))
766 if item == -1:
767 item = 0
768 u.append(item)
769 if len(u) == 1:
770 where[propname] = u[0]
771 else:
772 orcriteria[propname] = u
773 elif isinstance(prop, hyperdb.String):
774 # simple glob searching
775 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', value)
776 v = v.replace('?', '.')
777 v = v.replace('*', '.*?')
778 regexes[propname] = re.compile(v, re.I)
779 elif propname == 'id':
780 where[propname] = int(value)
781 elif isinstance(prop, hyperdb.Boolean):
782 if type(value) is _STRINGTYPE:
783 bv = value.lower() in ('yes', 'true', 'on', '1')
784 else:
785 bv = value
786 where[propname] = bv
787 elif isinstance(prop, hyperdb.Number):
788 where[propname] = int(value)
789 else:
790 where[propname] = str(value)
791 v = self.getview()
792 #print "filter start at %s" % time.time()
793 if where:
794 v = v.select(where)
795 #print "filter where at %s" % time.time()
797 if mlcriteria:
798 # multilink - if any of the nodeids required by the
799 # filterspec aren't in this node's property, then skip
800 # it
801 def ff(row, ml=mlcriteria):
802 for propname, values in ml.items():
803 sv = getattr(row, propname)
804 for id in values:
805 if sv.find(fid=id) == -1:
806 return 0
807 return 1
808 iv = v.filter(ff)
809 v = v.remapwith(iv)
811 #print "filter mlcrit at %s" % time.time()
813 if orcriteria:
814 def ff(row, crit=orcriteria):
815 for propname, allowed in crit.items():
816 val = getattr(row, propname)
817 if val not in allowed:
818 return 0
819 return 1
821 iv = v.filter(ff)
822 v = v.remapwith(iv)
824 #print "filter orcrit at %s" % time.time()
825 if regexes:
826 def ff(row, r=regexes):
827 for propname, regex in r.items():
828 val = getattr(row, propname)
829 if not regex.search(val):
830 return 0
831 return 1
833 iv = v.filter(ff)
834 v = v.remapwith(iv)
835 #print "filter regexs at %s" % time.time()
837 if sort or group:
838 sortspec = []
839 rev = []
840 for dir, propname in group, sort:
841 if propname is None: continue
842 isreversed = 0
843 if dir == '-':
844 isreversed = 1
845 try:
846 prop = getattr(v, propname)
847 except AttributeError:
848 print "MK has no property %s" % propname
849 continue
850 propclass = self.ruprops.get(propname, None)
851 if propclass is None:
852 propclass = self.privateprops.get(propname, None)
853 if propclass is None:
854 print "Schema has no property %s" % propname
855 continue
856 if isinstance(propclass, hyperdb.Link):
857 linkclass = self.db.getclass(propclass.classname)
858 lv = linkclass.getview()
859 lv = lv.rename('id', propname)
860 v = v.join(lv, prop, 1)
861 if linkclass.getprops().has_key('order'):
862 propname = 'order'
863 else:
864 propname = linkclass.labelprop()
865 prop = getattr(v, propname)
866 if isreversed:
867 rev.append(prop)
868 sortspec.append(prop)
869 v = v.sortrev(sortspec, rev)[:] #XXX Metakit bug
870 #print "filter sort at %s" % time.time()
872 rslt = []
873 for row in v:
874 id = str(row.id)
875 if search_matches is not None:
876 if search_matches.has_key(id):
877 rslt.append(id)
878 else:
879 rslt.append(id)
880 return rslt
882 def hasnode(self, nodeid):
883 return int(nodeid) < self.maxid
885 def labelprop(self, default_to_id=0):
886 ''' Return the property name for a label for the given node.
888 This method attempts to generate a consistent label for the node.
889 It tries the following in order:
890 1. key property
891 2. "name" property
892 3. "title" property
893 4. first property from the sorted property name list
894 '''
895 k = self.getkey()
896 if k:
897 return k
898 props = self.getprops()
899 if props.has_key('name'):
900 return 'name'
901 elif props.has_key('title'):
902 return 'title'
903 if default_to_id:
904 return 'id'
905 props = props.keys()
906 props.sort()
907 return props[0]
908 def stringFind(self, **requirements):
909 """Locate a particular node by matching a set of its String
910 properties in a caseless search.
912 If the property is not a String property, a TypeError is raised.
914 The return is a list of the id of all nodes that match.
915 """
916 for propname in requirements.keys():
917 prop = self.properties[propname]
918 if isinstance(not prop, hyperdb.String):
919 raise TypeError, "'%s' not a String property"%propname
920 requirements[propname] = requirements[propname].lower()
921 requirements['_isdel'] = 0
923 l = []
924 for row in self.getview().select(requirements):
925 l.append(str(row.id))
926 return l
928 def addjournal(self, nodeid, action, params):
929 self.db.addjournal(self.classname, nodeid, action, params)
931 def index(self, nodeid):
932 ''' Add (or refresh) the node to search indexes '''
933 # find all the String properties that have indexme
934 for prop, propclass in self.getprops().items():
935 if isinstance(propclass, hyperdb.String) and propclass.indexme:
936 # index them under (classname, nodeid, property)
937 self.db.indexer.add_text((self.classname, nodeid, prop),
938 str(self.get(nodeid, prop)))
940 def export_list(self, propnames, nodeid):
941 ''' Export a node - generate a list of CSV-able data in the order
942 specified by propnames for the given node.
943 '''
944 properties = self.getprops()
945 l = []
946 for prop in propnames:
947 proptype = properties[prop]
948 value = self.get(nodeid, prop)
949 # "marshal" data where needed
950 if value is None:
951 pass
952 elif isinstance(proptype, hyperdb.Date):
953 value = value.get_tuple()
954 elif isinstance(proptype, hyperdb.Interval):
955 value = value.get_tuple()
956 elif isinstance(proptype, hyperdb.Password):
957 value = str(value)
958 l.append(repr(value))
959 return l
961 def import_list(self, propnames, proplist):
962 ''' Import a node - all information including "id" is present and
963 should not be sanity checked. Triggers are not triggered. The
964 journal should be initialised using the "creator" and "creation"
965 information.
967 Return the nodeid of the node imported.
968 '''
969 if self.db.journaltag is None:
970 raise hyperdb.DatabaseError, 'Database open read-only'
971 properties = self.getprops()
973 d = {}
974 view = self.getview(1)
975 for i in range(len(propnames)):
976 value = eval(proplist[i])
977 propname = propnames[i]
978 prop = properties[propname]
979 if propname == 'id':
980 newid = value
981 value = int(value)
982 elif isinstance(prop, hyperdb.Date):
983 value = int(calendar.timegm(value))
984 elif isinstance(prop, hyperdb.Interval):
985 value = str(date.Interval(value))
986 d[propname] = value
987 view.append(d)
988 creator = d.get('creator', None)
989 creation = d.get('creation', None)
990 self.db.addjournal(self.classname, newid, 'create', {}, creator, creation)
991 return newid
993 # --- used by Database
994 def _commit(self):
995 """ called post commit of the DB.
996 interested subclasses may override """
997 self.uncommitted = {}
998 self.rbactions = []
999 self.idcache = {}
1000 def _rollback(self):
1001 """ called pre rollback of the DB.
1002 interested subclasses may override """
1003 for action in self.rbactions:
1004 action()
1005 self.rbactions = []
1006 self.uncommitted = {}
1007 self.idcache = {}
1008 def _clear(self):
1009 view = self.getview(1)
1010 if len(view):
1011 view[:] = []
1012 self.db.dirty = 1
1013 iv = self.getindexview(1)
1014 if iv:
1015 iv[:] = []
1016 def rollbackaction(self, action):
1017 """ call this to register a callback called on rollback
1018 callback is removed on end of transaction """
1019 self.rbactions.append(action)
1020 # --- internal
1021 def __getview(self):
1022 db = self.db._db
1023 view = db.view(self.classname)
1024 mkprops = view.structure()
1025 if mkprops and self.db.fastopen:
1026 return view.ordered(1)
1027 # is the definition the same?
1028 for nm, rutyp in self.ruprops.items():
1029 for mkprop in mkprops:
1030 if mkprop.name == nm:
1031 break
1032 else:
1033 mkprop = None
1034 if mkprop is None:
1035 break
1036 if _typmap[rutyp.__class__] != mkprop.type:
1037 break
1038 else:
1039 return view.ordered(1)
1040 # need to create or restructure the mk view
1041 # id comes first, so MK will order it for us
1042 self.db.dirty = 1
1043 s = ["%s[id:I" % self.classname]
1044 for nm, rutyp in self.ruprops.items():
1045 mktyp = _typmap[rutyp.__class__]
1046 s.append('%s:%s' % (nm, mktyp))
1047 if mktyp == 'V':
1048 s[-1] += ('[fid:I]')
1049 s.append('_isdel:I,activity:I,creation:I,creator:I]')
1050 v = self.db._db.getas(','.join(s))
1051 self.db.commit()
1052 return v.ordered(1)
1053 def getview(self, RW=0):
1054 return self.db._db.view(self.classname).ordered(1)
1055 def getindexview(self, RW=0):
1056 return self.db._db.view("_%s" % self.classname).ordered(1)
1058 def _fetchML(sv):
1059 l = []
1060 for row in sv:
1061 if row.fid:
1062 l.append(str(row.fid))
1063 return l
1065 def _fetchPW(s):
1066 p = password.Password()
1067 p.unpack(s)
1068 return p
1070 def _fetchLink(n):
1071 return n and str(n) or None
1073 def _fetchDate(n):
1074 return date.Date(time.gmtime(n))
1076 _converters = {
1077 hyperdb.Date : _fetchDate,
1078 hyperdb.Link : _fetchLink,
1079 hyperdb.Multilink : _fetchML,
1080 hyperdb.Interval : date.Interval,
1081 hyperdb.Password : _fetchPW,
1082 hyperdb.Boolean : lambda n: n,
1083 hyperdb.Number : lambda n: n,
1084 hyperdb.String : str,
1085 }
1087 class FileName(hyperdb.String):
1088 isfilename = 1
1090 _typmap = {
1091 FileName : 'S',
1092 hyperdb.String : 'S',
1093 hyperdb.Date : 'I',
1094 hyperdb.Link : 'I',
1095 hyperdb.Multilink : 'V',
1096 hyperdb.Interval : 'S',
1097 hyperdb.Password : 'S',
1098 hyperdb.Boolean : 'I',
1099 hyperdb.Number : 'I',
1100 }
1101 class FileClass(Class):
1102 ' like Class but with a content property '
1103 default_mime_type = 'text/plain'
1104 def __init__(self, db, classname, **properties):
1105 properties['content'] = FileName()
1106 if not properties.has_key('type'):
1107 properties['type'] = hyperdb.String()
1108 Class.__init__(self, db, classname, **properties)
1109 def get(self, nodeid, propname, default=_marker, cache=1):
1110 x = Class.get(self, nodeid, propname, default, cache)
1111 if propname == 'content':
1112 if x.startswith('file:'):
1113 fnm = x[5:]
1114 try:
1115 x = open(fnm, 'rb').read()
1116 except Exception, e:
1117 x = repr(e)
1118 return x
1119 def create(self, **propvalues):
1120 content = propvalues['content']
1121 del propvalues['content']
1122 newid = Class.create(self, **propvalues)
1123 if not content:
1124 return newid
1125 nm = bnm = '%s%s' % (self.classname, newid)
1126 sd = str(int(int(newid) / 1000))
1127 d = os.path.join(self.db.config.DATABASE, 'files', self.classname, sd)
1128 if not os.path.exists(d):
1129 os.makedirs(d)
1130 nm = os.path.join(d, nm)
1131 open(nm, 'wb').write(content)
1132 self.set(newid, content = 'file:'+nm)
1133 mimetype = propvalues.get('type', self.default_mime_type)
1134 self.db.indexer.add_text((self.classname, newid, 'content'), content, mimetype)
1135 def undo(fnm=nm, action1=os.remove, indexer=self.db.indexer):
1136 action1(fnm)
1137 self.rollbackaction(undo)
1138 return newid
1139 def index(self, nodeid):
1140 Class.index(self, nodeid)
1141 mimetype = self.get(nodeid, 'type')
1142 if not mimetype:
1143 mimetype = self.default_mime_type
1144 self.db.indexer.add_text((self.classname, nodeid, 'content'),
1145 self.get(nodeid, 'content'), mimetype)
1147 class IssueClass(Class, roundupdb.IssueClass):
1148 # Overridden methods:
1149 def __init__(self, db, classname, **properties):
1150 """The newly-created class automatically includes the "messages",
1151 "files", "nosy", and "superseder" properties. If the 'properties'
1152 dictionary attempts to specify any of these properties or a
1153 "creation" or "activity" property, a ValueError is raised."""
1154 if not properties.has_key('title'):
1155 properties['title'] = hyperdb.String(indexme='yes')
1156 if not properties.has_key('messages'):
1157 properties['messages'] = hyperdb.Multilink("msg")
1158 if not properties.has_key('files'):
1159 properties['files'] = hyperdb.Multilink("file")
1160 if not properties.has_key('nosy'):
1161 # note: journalling is turned off as it really just wastes
1162 # space. this behaviour may be overridden in an instance
1163 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
1164 if not properties.has_key('superseder'):
1165 properties['superseder'] = hyperdb.Multilink(classname)
1166 Class.__init__(self, db, classname, **properties)
1168 CURVERSION = 1
1170 class Indexer(indexer.Indexer):
1171 disallows = {'THE':1, 'THIS':1, 'ZZZ':1, 'THAT':1, 'WITH':1}
1172 def __init__(self, path, datadb):
1173 self.db = metakit.storage(os.path.join(path, 'index.mk4'), 1)
1174 self.datadb = datadb
1175 self.reindex = 0
1176 v = self.db.view('version')
1177 if not v.structure():
1178 v = self.db.getas('version[vers:I]')
1179 self.db.commit()
1180 v.append(vers=CURVERSION)
1181 self.reindex = 1
1182 elif v[0].vers != CURVERSION:
1183 v[0].vers = CURVERSION
1184 self.reindex = 1
1185 if self.reindex:
1186 self.db.getas('ids[tblid:I,nodeid:I,propid:I]')
1187 self.db.getas('index[word:S,hits[pos:I]]')
1188 self.db.commit()
1189 self.reindex = 1
1190 self.changed = 0
1191 self.propcache = {}
1192 def force_reindex(self):
1193 v = self.db.view('ids')
1194 v[:] = []
1195 v = self.db.view('index')
1196 v[:] = []
1197 self.db.commit()
1198 self.reindex = 1
1199 def should_reindex(self):
1200 return self.reindex
1201 def _getprops(self, classname):
1202 props = self.propcache.get(classname, None)
1203 if props is None:
1204 props = self.datadb.view(classname).structure()
1205 props = [prop.name for prop in props]
1206 self.propcache[classname] = props
1207 return props
1208 def _getpropid(self, classname, propname):
1209 return self._getprops(classname).index(propname)
1210 def _getpropname(self, classname, propid):
1211 return self._getprops(classname)[propid]
1213 def add_text(self, identifier, text, mime_type='text/plain'):
1214 if mime_type != 'text/plain':
1215 return
1216 classname, nodeid, property = identifier
1217 tbls = self.datadb.view('tables')
1218 tblid = tbls.find(name=classname)
1219 if tblid < 0:
1220 raise KeyError, "unknown class %r"%classname
1221 nodeid = int(nodeid)
1222 propid = self._getpropid(classname, property)
1223 pos = self.db.view('ids').append(tblid=tblid,nodeid=nodeid,propid=propid)
1225 wordlist = re.findall(r'\b\w{3,25}\b', text)
1226 words = {}
1227 for word in wordlist:
1228 word = word.upper()
1229 if not self.disallows.has_key(word):
1230 words[word] = 1
1231 words = words.keys()
1233 index = self.db.view('index').ordered(1)
1234 for word in words:
1235 ndx = index.find(word=word)
1236 if ndx < 0:
1237 ndx = index.append(word=word)
1238 hits = index[ndx].hits
1239 if len(hits)==0 or hits.find(pos=pos) < 0:
1240 hits.append(pos=pos)
1241 self.changed = 1
1243 def find(self, wordlist):
1244 hits = None
1245 index = self.db.view('index').ordered(1)
1246 for word in wordlist:
1247 if not 2 < len(word) < 26:
1248 continue
1249 ndx = index.find(word=word)
1250 if ndx < 0:
1251 return {}
1252 if hits is None:
1253 hits = index[ndx].hits
1254 else:
1255 hits = hits.intersect(index[ndx].hits)
1256 if len(hits) == 0:
1257 return {}
1258 if hits is None:
1259 return {}
1260 rslt = {}
1261 ids = self.db.view('ids').remapwith(hits)
1262 tbls = self.datadb.view('tables')
1263 for i in range(len(ids)):
1264 hit = ids[i]
1265 classname = tbls[hit.tblid].name
1266 nodeid = str(hit.nodeid)
1267 property = self._getpropname(classname, hit.propid)
1268 rslt[i] = (classname, nodeid, property)
1269 return rslt
1270 def save_index(self):
1271 if self.changed:
1272 self.db.commit()
1273 self.changed = 0