1 from roundup import hyperdb, date, password, roundupdb, security
2 import metakit
3 from sessions import Sessions
4 import re, marshal, os, sys, weakref, time, calendar
5 from roundup import indexer
6 import locking
8 _dbs = {}
10 def Database(config, journaltag=None):
11 db = _dbs.get(config.DATABASE, None)
12 if db is None or db._db is None:
13 db = _Database(config, journaltag)
14 _dbs[config.DATABASE] = db
15 else:
16 db.journaltag = journaltag
17 try:
18 delattr(db, 'curuserid')
19 except AttributeError:
20 pass
21 return db
23 class _Database(hyperdb.Database):
24 def __init__(self, config, journaltag=None):
25 self.config = config
26 self.journaltag = journaltag
27 self.classes = {}
28 self.dirty = 0
29 self.lockfile = None
30 self._db = self.__open()
31 self.indexer = Indexer(self.config.DATABASE, self._db)
32 self.sessions = Sessions(self.config)
33 self.security = security.Security(self)
35 os.umask(0002)
37 def post_init(self):
38 if self.indexer.should_reindex():
39 self.reindex()
41 def reindex(self):
42 for klass in self.classes.values():
43 for nodeid in klass.list():
44 klass.index(nodeid)
45 self.indexer.save_index()
48 # --- defined in ping's spec
49 def __getattr__(self, classname):
50 if classname == 'curuserid':
51 if self.journaltag is None:
52 return None
54 try:
55 self.curuserid = x = int(self.classes['user'].lookup(self.journaltag))
56 except KeyError:
57 if self.journaltag == 'admin':
58 self.curuserid = x = 1
59 else:
60 x = 0
61 return x
62 elif classname == 'transactions':
63 return self.dirty
64 return self.getclass(classname)
65 def getclass(self, classname):
66 try:
67 return self.classes[classname]
68 except KeyError:
69 raise KeyError, 'There is no class called "%s"'%classname
70 def getclasses(self):
71 return self.classes.keys()
72 # --- end of ping's spec
73 # --- exposed methods
74 def commit(self):
75 if self.dirty:
76 self._db.commit()
77 for cl in self.classes.values():
78 cl._commit()
79 self.indexer.save_index()
80 self.dirty = 0
81 def rollback(self):
82 if self.dirty:
83 for cl in self.classes.values():
84 cl._rollback()
85 self._db.rollback()
86 self.dirty = 0
87 def clear(self):
88 for cl in self.classes.values():
89 cl._clear()
90 def hasnode(self, classname, nodeid):
91 return self.getclass(classname).hasnode(nodeid)
92 def pack(self, pack_before):
93 pass
94 def addclass(self, cl):
95 self.classes[cl.classname] = cl
96 if self.tables.find(name=cl.classname) < 0:
97 self.tables.append(name=cl.classname)
98 def addjournal(self, tablenm, nodeid, action, params):
99 tblid = self.tables.find(name=tablenm)
100 if tblid == -1:
101 tblid = self.tables.append(name=tablenm)
102 # tableid:I,nodeid:I,date:I,user:I,action:I,params:B
103 self.hist.append(tableid=tblid,
104 nodeid=int(nodeid),
105 date=int(time.time()),
106 action=action,
107 user = self.curuserid,
108 params = marshal.dumps(params))
109 def gethistory(self, tablenm, nodeid):
110 rslt = []
111 tblid = self.tables.find(name=tablenm)
112 if tblid == -1:
113 return rslt
114 q = self.hist.select(tableid=tblid, nodeid=int(nodeid))
115 i = 0
116 userclass = self.getclass('user')
117 for row in q:
118 try:
119 params = marshal.loads(row.params)
120 except ValueError:
121 print "history couldn't unmarshal %r" % row.params
122 params = {}
123 usernm = userclass.get(str(row.user), 'username')
124 dt = date.Date(time.gmtime(row.date))
125 rslt.append((i, dt, usernm, _actionnames[row.action], params))
126 i += 1
127 return rslt
129 def close(self):
130 for cl in self.classes.values():
131 cl.db = None
132 self._db = None
133 if self.lockfile is not None:
134 locking.release_lock(self.lockfile)
135 if _dbs.has_key(self.config.DATABASE):
136 del _dbs[self.config.DATABASE]
137 if self.lockfile is not None:
138 self.lockfile.close()
139 self.lockfile = None
140 self.classes = {}
141 self.indexer = None
143 # --- internal
144 def __open(self):
145 self.dbnm = db = os.path.join(self.config.DATABASE, 'tracker.mk4')
146 lockfilenm = db[:-3]+'lck'
147 self.lockfile = locking.acquire_lock(lockfilenm)
148 self.lockfile.write(str(os.getpid()))
149 self.lockfile.flush()
150 self.fastopen = 0
151 if os.path.exists(db):
152 dbtm = os.path.getmtime(db)
153 pkgnm = self.config.__name__.split('.')[0]
154 schemamod = sys.modules.get(pkgnm+'.dbinit', None)
155 if schemamod:
156 if os.path.exists(schemamod.__file__):
157 schematm = os.path.getmtime(schemamod.__file__)
158 if schematm < dbtm:
159 # found schema mod - it's older than the db
160 self.fastopen = 1
161 else:
162 # can't find schemamod - must be frozen
163 self.fastopen = 1
164 db = metakit.storage(db, 1)
165 hist = db.view('history')
166 tables = db.view('tables')
167 if not self.fastopen:
168 if not hist.structure():
169 hist = db.getas('history[tableid:I,nodeid:I,date:I,user:I,action:I,params:B]')
170 if not tables.structure():
171 tables = db.getas('tables[name:S]')
172 self.tables = tables
173 self.hist = hist
174 return db
176 _STRINGTYPE = type('')
177 _LISTTYPE = type([])
178 _CREATE, _SET, _RETIRE, _LINK, _UNLINK = range(5)
180 _actionnames = {
181 _CREATE : 'create',
182 _SET : 'set',
183 _RETIRE : 'retire',
184 _LINK : 'link',
185 _UNLINK : 'unlink',
186 }
188 _marker = []
190 _ALLOWSETTINGPRIVATEPROPS = 0
192 class Class:
193 privateprops = None
194 def __init__(self, db, classname, **properties):
195 #self.db = weakref.proxy(db)
196 self.db = db
197 self.classname = classname
198 self.keyname = None
199 self.ruprops = properties
200 self.privateprops = { 'id' : hyperdb.String(),
201 'activity' : hyperdb.Date(),
202 'creation' : hyperdb.Date(),
203 'creator' : hyperdb.Link('user') }
204 self.auditors = {'create': [], 'set': [], 'retire': []} # event -> list of callables
205 self.reactors = {'create': [], 'set': [], 'retire': []} # ditto
206 view = self.__getview()
207 self.maxid = 1
208 if view:
209 self.maxid = view[-1].id + 1
210 self.uncommitted = {}
211 self.rbactions = []
212 # people reach inside!!
213 self.properties = self.ruprops
214 self.db.addclass(self)
215 self.idcache = {}
217 # default is to journal changes
218 self.do_journal = 1
220 def enableJournalling(self):
221 '''Turn journalling on for this class
222 '''
223 self.do_journal = 1
225 def disableJournalling(self):
226 '''Turn journalling off for this class
227 '''
228 self.do_journal = 0
230 # --- the roundup.Class methods
231 def audit(self, event, detector):
232 l = self.auditors[event]
233 if detector not in l:
234 self.auditors[event].append(detector)
235 def fireAuditors(self, action, nodeid, newvalues):
236 for audit in self.auditors[action]:
237 audit(self.db, self, nodeid, newvalues)
238 def fireReactors(self, action, nodeid, oldvalues):
239 for react in self.reactors[action]:
240 react(self.db, self, nodeid, oldvalues)
241 def react(self, event, detector):
242 l = self.reactors[event]
243 if detector not in l:
244 self.reactors[event].append(detector)
245 # --- the hyperdb.Class methods
246 def create(self, **propvalues):
247 self.fireAuditors('create', None, propvalues)
248 rowdict = {}
249 rowdict['id'] = newid = self.maxid
250 self.maxid += 1
251 ndx = self.getview(1).append(rowdict)
252 propvalues['#ISNEW'] = 1
253 try:
254 self.set(str(newid), **propvalues)
255 except Exception:
256 self.maxid -= 1
257 raise
258 return str(newid)
260 def get(self, nodeid, propname, default=_marker, cache=1):
261 # default and cache aren't in the spec
262 # cache=0 means "original value"
264 view = self.getview()
265 id = int(nodeid)
266 if cache == 0:
267 oldnode = self.uncommitted.get(id, None)
268 if oldnode and oldnode.has_key(propname):
269 return oldnode[propname]
270 ndx = self.idcache.get(id, None)
271 if ndx is None:
272 ndx = view.find(id=id)
273 if ndx < 0:
274 raise IndexError, "%s has no node %s" % (self.classname, nodeid)
275 self.idcache[id] = ndx
276 try:
277 raw = getattr(view[ndx], propname)
278 except AttributeError:
279 raise KeyError, propname
280 rutyp = self.ruprops.get(propname, None)
281 if rutyp is None:
282 rutyp = self.privateprops[propname]
283 converter = _converters.get(rutyp.__class__, None)
284 if converter:
285 raw = converter(raw)
286 return raw
288 def set(self, nodeid, **propvalues):
289 isnew = 0
290 if propvalues.has_key('#ISNEW'):
291 isnew = 1
292 del propvalues['#ISNEW']
293 if not isnew:
294 self.fireAuditors('set', nodeid, propvalues)
295 if not propvalues:
296 return propvalues
297 if propvalues.has_key('id'):
298 raise KeyError, '"id" is reserved'
299 if self.db.journaltag is None:
300 raise hyperdb.DatabaseError, 'Database open read-only'
301 view = self.getview(1)
302 # node must exist & not be retired
303 id = int(nodeid)
304 ndx = view.find(id=id)
305 if ndx < 0:
306 raise IndexError, "%s has no node %s" % (self.classname, nodeid)
307 row = view[ndx]
308 if row._isdel:
309 raise IndexError, "%s has no node %s" % (self.classname, nodeid)
310 oldnode = self.uncommitted.setdefault(id, {})
311 changes = {}
313 for key, value in propvalues.items():
314 # this will raise the KeyError if the property isn't valid
315 # ... we don't use getprops() here because we only care about
316 # the writeable properties.
317 if _ALLOWSETTINGPRIVATEPROPS:
318 prop = self.ruprops.get(key, None)
319 if not prop:
320 prop = self.privateprops[key]
321 else:
322 prop = self.ruprops[key]
323 converter = _converters.get(prop.__class__, lambda v: v)
324 # if the value's the same as the existing value, no sense in
325 # doing anything
326 oldvalue = converter(getattr(row, key))
327 if value == oldvalue:
328 del propvalues[key]
329 continue
331 # check to make sure we're not duplicating an existing key
332 if key == self.keyname:
333 iv = self.getindexview(1)
334 ndx = iv.find(k=value)
335 if ndx == -1:
336 iv.append(k=value, i=row.id)
337 if not isnew:
338 ndx = iv.find(k=oldvalue)
339 if ndx > -1:
340 iv.delete(ndx)
341 else:
342 raise ValueError, 'node with key "%s" exists'%value
344 # do stuff based on the prop type
345 if isinstance(prop, hyperdb.Link):
346 link_class = prop.classname
347 # must be a string or None
348 if value is not None and not isinstance(value, type('')):
349 raise ValueError, 'property "%s" link value be a string'%(
350 key)
351 # Roundup sets to "unselected" by passing None
352 if value is None:
353 value = 0
354 # if it isn't a number, it's a key
355 try:
356 int(value)
357 except ValueError:
358 try:
359 value = self.db.getclass(link_class).lookup(value)
360 except (TypeError, KeyError):
361 raise IndexError, 'new property "%s": %s not a %s'%(
362 key, value, prop.classname)
364 if (value is not None and
365 not self.db.getclass(link_class).hasnode(value)):
366 raise IndexError, '%s has no node %s'%(link_class, value)
368 setattr(row, key, int(value))
369 changes[key] = oldvalue
371 if self.do_journal and prop.do_journal:
372 # register the unlink with the old linked node
373 if oldvalue:
374 self.db.addjournal(link_class, value, _UNLINK,
375 (self.classname, str(row.id), key))
377 # register the link with the newly linked node
378 if value:
379 self.db.addjournal(link_class, value, _LINK,
380 (self.classname, str(row.id), key))
382 elif isinstance(prop, hyperdb.Multilink):
383 if type(value) != _LISTTYPE:
384 raise TypeError, 'new property "%s" not a list of ids'%key
385 link_class = prop.classname
386 l = []
387 for entry in value:
388 if type(entry) != _STRINGTYPE:
389 raise ValueError, 'new property "%s" link value ' \
390 'must be a string'%key
391 # if it isn't a number, it's a key
392 try:
393 int(entry)
394 except ValueError:
395 try:
396 entry = self.db.getclass(link_class).lookup(entry)
397 except (TypeError, KeyError):
398 raise IndexError, 'new property "%s": %s not a %s'%(
399 key, entry, prop.classname)
400 l.append(entry)
401 propvalues[key] = value = l
403 # handle removals
404 rmvd = []
405 for id in oldvalue:
406 if id not in value:
407 rmvd.append(id)
408 # register the unlink with the old linked node
409 if self.do_journal and prop.do_journal:
410 self.db.addjournal(link_class, id, _UNLINK,
411 (self.classname, str(row.id), key))
413 # handle additions
414 adds = []
415 for id in value:
416 if id not in oldvalue:
417 if not self.db.getclass(link_class).hasnode(id):
418 raise IndexError, '%s has no node %s'%(
419 link_class, id)
420 adds.append(id)
421 # register the link with the newly linked node
422 if self.do_journal and prop.do_journal:
423 self.db.addjournal(link_class, id, _LINK,
424 (self.classname, str(row.id), key))
426 sv = getattr(row, key)
427 i = 0
428 while i < len(sv):
429 if str(sv[i].fid) in rmvd:
430 sv.delete(i)
431 else:
432 i += 1
433 for id in adds:
434 sv.append(fid=int(id))
435 changes[key] = oldvalue
436 if not rmvd and not adds:
437 del propvalues[key]
439 elif isinstance(prop, hyperdb.String):
440 if value is not None and type(value) != _STRINGTYPE:
441 raise TypeError, 'new property "%s" not a string'%key
442 setattr(row, key, value)
443 changes[key] = oldvalue
444 if hasattr(prop, 'isfilename') and prop.isfilename:
445 propvalues[key] = os.path.basename(value)
446 if prop.indexme and value is not None:
447 self.db.indexer.add_text((self.classname, nodeid, key),
448 value, 'text/plain')
450 elif isinstance(prop, hyperdb.Password):
451 if not isinstance(value, password.Password):
452 raise TypeError, 'new property "%s" not a Password'% key
453 setattr(row, key, str(value))
454 changes[key] = str(oldvalue)
455 propvalues[key] = str(value)
457 elif value is not None and isinstance(prop, hyperdb.Date):
458 if not isinstance(value, date.Date):
459 raise TypeError, 'new property "%s" not a Date'% key
460 setattr(row, key, int(calendar.timegm(value.get_tuple())))
461 changes[key] = str(oldvalue)
462 propvalues[key] = str(value)
464 elif value is not None and isinstance(prop, hyperdb.Interval):
465 if not isinstance(value, date.Interval):
466 raise TypeError, 'new property "%s" not an Interval'% key
467 setattr(row, key, str(value))
468 changes[key] = str(oldvalue)
469 propvalues[key] = str(value)
471 elif value is not None and isinstance(prop, hyperdb.Number):
472 setattr(row, key, int(value))
473 changes[key] = oldvalue
474 propvalues[key] = value
476 elif value is not None and isinstance(prop, hyperdb.Boolean):
477 bv = value != 0
478 setattr(row, key, bv)
479 changes[key] = oldvalue
480 propvalues[key] = value
482 oldnode[key] = oldvalue
484 # nothing to do?
485 if not propvalues:
486 return propvalues
487 if not propvalues.has_key('activity'):
488 row.activity = int(time.time())
489 if isnew:
490 if not row.creation:
491 row.creation = int(time.time())
492 if not row.creator:
493 row.creator = self.db.curuserid
495 self.db.dirty = 1
496 if self.do_journal:
497 if isnew:
498 self.db.addjournal(self.classname, nodeid, _CREATE, {})
499 self.fireReactors('create', nodeid, None)
500 else:
501 self.db.addjournal(self.classname, nodeid, _SET, changes)
502 self.fireReactors('set', nodeid, oldnode)
504 return propvalues
506 def retire(self, nodeid):
507 self.fireAuditors('retire', nodeid, None)
508 view = self.getview(1)
509 ndx = view.find(id=int(nodeid))
510 if ndx < 0:
511 raise KeyError, "nodeid %s not found" % nodeid
512 row = view[ndx]
513 oldvalues = self.uncommitted.setdefault(row.id, {})
514 oldval = oldvalues['_isdel'] = row._isdel
515 row._isdel = 1
516 if self.do_journal:
517 self.db.addjournal(self.classname, nodeid, _RETIRE, {})
518 if self.keyname:
519 iv = self.getindexview(1)
520 ndx = iv.find(k=getattr(row, self.keyname),i=row.id)
521 if ndx > -1:
522 iv.delete(ndx)
523 self.db.dirty = 1
524 self.fireReactors('retire', nodeid, None)
525 def history(self, nodeid):
526 if not self.do_journal:
527 raise ValueError, 'Journalling is disabled for this class'
528 return self.db.gethistory(self.classname, nodeid)
529 def setkey(self, propname):
530 if self.keyname:
531 if propname == self.keyname:
532 return
533 raise ValueError, "%s already indexed on %s" % (self.classname, self.keyname)
534 # first setkey for this run
535 self.keyname = propname
536 iv = self.db._db.view('_%s' % self.classname)
537 if self.db.fastopen and iv.structure():
538 return
539 # very first setkey ever
540 self.db.dirty = 1
541 iv = self.db._db.getas('_%s[k:S,i:I]' % self.classname)
542 iv = iv.ordered(1)
543 # print "setkey building index"
544 for row in self.getview():
545 iv.append(k=getattr(row, propname), i=row.id)
546 self.db.commit()
547 def getkey(self):
548 return self.keyname
549 def lookup(self, keyvalue):
550 if type(keyvalue) is not _STRINGTYPE:
551 raise TypeError, "%r is not a string" % keyvalue
552 iv = self.getindexview()
553 if iv:
554 ndx = iv.find(k=keyvalue)
555 if ndx > -1:
556 return str(iv[ndx].i)
557 else:
558 view = self.getview()
559 ndx = view.find({self.keyname:keyvalue, '_isdel':0})
560 if ndx > -1:
561 return str(view[ndx].id)
562 raise KeyError, keyvalue
564 def destroy(self, keyvalue):
565 #TODO clean this up once Richard's said how it should work
566 iv = self.getindexview()
567 if iv:
568 ndx = iv.find(k=keyvalue)
569 if ndx > -1:
570 id = iv[ndx].i
571 iv.delete(ndx)
572 view = self.getview()
573 ndx = view.find(id=id)
574 if ndx > -1:
575 view.delete(ndx)
577 def find(self, **propspec):
578 """Get the ids of nodes in this class which link to the given nodes.
580 'propspec' consists of keyword args propname={nodeid:1,}
581 'propname' must be the name of a property in this class, or a
582 KeyError is raised. That property must be a Link or
583 Multilink property, or a TypeError is raised.
585 Any node in this class whose propname property links to any of the
586 nodeids will be returned. Used by the full text indexing, which knows
587 that "foo" occurs in msg1, msg3 and file7; so we have hits on these
588 issues:
590 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
592 """
593 propspec = propspec.items()
594 for propname, nodeid in propspec:
595 # check the prop is OK
596 prop = self.ruprops[propname]
597 if (not isinstance(prop, hyperdb.Link) and
598 not isinstance(prop, hyperdb.Multilink)):
599 raise TypeError, "'%s' not a Link/Multilink property"%propname
601 vws = []
602 for propname, ids in propspec:
603 if type(ids) is _STRINGTYPE:
604 ids = {ids:1}
605 prop = self.ruprops[propname]
606 view = self.getview()
607 if isinstance(prop, hyperdb.Multilink):
608 view = view.flatten(getattr(view, propname))
609 def ff(row, nm=propname, ids=ids):
610 return ids.has_key(str(row.fid))
611 else:
612 def ff(row, nm=propname, ids=ids):
613 return ids.has_key(str(getattr(row, nm)))
614 ndxview = view.filter(ff)
615 vws.append(ndxview.unique())
617 # handle the empty match case
618 if not vws:
619 return []
621 ndxview = vws[0]
622 for v in vws[1:]:
623 ndxview = ndxview.union(v)
624 view = view.remapwith(ndxview)
625 rslt = []
626 for row in view:
627 rslt.append(str(row.id))
628 return rslt
631 def list(self):
632 l = []
633 for row in self.getview().select(_isdel=0):
634 l.append(str(row.id))
635 return l
636 def count(self):
637 return len(self.getview())
638 def getprops(self, protected=1):
639 # protected is not in ping's spec
640 allprops = self.ruprops.copy()
641 if protected and self.privateprops is not None:
642 allprops.update(self.privateprops)
643 return allprops
644 def addprop(self, **properties):
645 for key in properties.keys():
646 if self.ruprops.has_key(key):
647 raise ValueError, "%s is already a property of %s" % (key, self.classname)
648 self.ruprops.update(properties)
649 self.db.fastopen = 0
650 view = self.__getview()
651 self.db.commit()
652 # ---- end of ping's spec
653 def filter(self, search_matches, filterspec, sort, group):
654 # search_matches is None or a set (dict of {nodeid: {propname:[nodeid,...]}})
655 # filterspec is a dict {propname:value}
656 # sort and group are lists of propnames
657 # sort and group are (dir, prop) where dir is '+', '-' or None
658 # and prop is a prop name or None
660 where = {'_isdel':0}
661 mlcriteria = {}
662 regexes = {}
663 orcriteria = {}
664 for propname, value in filterspec.items():
665 prop = self.ruprops.get(propname, None)
666 if prop is None:
667 prop = self.privateprops[propname]
668 if isinstance(prop, hyperdb.Multilink):
669 if type(value) is not _LISTTYPE:
670 value = [value]
671 # transform keys to ids
672 u = []
673 for item in value:
674 try:
675 item = int(item)
676 except (TypeError, ValueError):
677 item = int(self.db.getclass(prop.classname).lookup(item))
678 if item == -1:
679 item = 0
680 u.append(item)
681 mlcriteria[propname] = u
682 elif isinstance(prop, hyperdb.Link):
683 if type(value) is not _LISTTYPE:
684 value = [value]
685 # transform keys to ids
686 u = []
687 for item in value:
688 try:
689 item = int(item)
690 except (TypeError, ValueError):
691 item = int(self.db.getclass(prop.classname).lookup(item))
692 if item == -1:
693 item = 0
694 u.append(item)
695 if len(u) == 1:
696 where[propname] = u[0]
697 else:
698 orcriteria[propname] = u
699 elif isinstance(prop, hyperdb.String):
700 # simple glob searching
701 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', value)
702 v = v.replace('?', '.')
703 v = v.replace('*', '.*?')
704 regexes[propname] = re.compile(v, re.I)
705 elif propname == 'id':
706 where[propname] = int(value)
707 elif isinstance(prop, hyperdb.Boolean):
708 if type(value) is _STRINGTYPE:
709 bv = value.lower() in ('yes', 'true', 'on', '1')
710 else:
711 bv = value
712 where[propname] = bv
713 elif isinstance(prop, hyperdb.Number):
714 where[propname] = int(value)
715 else:
716 where[propname] = str(value)
717 v = self.getview()
718 #print "filter start at %s" % time.time()
719 if where:
720 v = v.select(where)
721 #print "filter where at %s" % time.time()
723 if mlcriteria:
724 # multilink - if any of the nodeids required by the
725 # filterspec aren't in this node's property, then skip
726 # it
727 def ff(row, ml=mlcriteria):
728 for propname, values in ml.items():
729 sv = getattr(row, propname)
730 for id in values:
731 if sv.find(fid=id) == -1:
732 return 0
733 return 1
734 iv = v.filter(ff)
735 v = v.remapwith(iv)
737 #print "filter mlcrit at %s" % time.time()
739 if orcriteria:
740 def ff(row, crit=orcriteria):
741 for propname, allowed in crit.items():
742 val = getattr(row, propname)
743 if val not in allowed:
744 return 0
745 return 1
747 iv = v.filter(ff)
748 v = v.remapwith(iv)
750 #print "filter orcrit at %s" % time.time()
751 if regexes:
752 def ff(row, r=regexes):
753 for propname, regex in r.items():
754 val = getattr(row, propname)
755 if not regex.search(val):
756 return 0
757 return 1
759 iv = v.filter(ff)
760 v = v.remapwith(iv)
761 #print "filter regexs at %s" % time.time()
763 if sort or group:
764 sortspec = []
765 rev = []
766 for dir, propname in group, sort:
767 if propname is None: continue
768 isreversed = 0
769 if dir == '-':
770 isreversed = 1
771 try:
772 prop = getattr(v, propname)
773 except AttributeError:
774 print "MK has no property %s" % propname
775 continue
776 propclass = self.ruprops.get(propname, None)
777 if propclass is None:
778 propclass = self.privateprops.get(propname, None)
779 if propclass is None:
780 print "Schema has no property %s" % propname
781 continue
782 if isinstance(propclass, hyperdb.Link):
783 linkclass = self.db.getclass(propclass.classname)
784 lv = linkclass.getview()
785 lv = lv.rename('id', propname)
786 v = v.join(lv, prop, 1)
787 if linkclass.getprops().has_key('order'):
788 propname = 'order'
789 else:
790 propname = linkclass.labelprop()
791 prop = getattr(v, propname)
792 if isreversed:
793 rev.append(prop)
794 sortspec.append(prop)
795 v = v.sortrev(sortspec, rev)[:] #XXX Metakit bug
796 #print "filter sort at %s" % time.time()
798 rslt = []
799 for row in v:
800 id = str(row.id)
801 if search_matches is not None:
802 if search_matches.has_key(id):
803 rslt.append(id)
804 else:
805 rslt.append(id)
806 return rslt
808 def hasnode(self, nodeid):
809 return int(nodeid) < self.maxid
811 def labelprop(self, default_to_id=0):
812 ''' Return the property name for a label for the given node.
814 This method attempts to generate a consistent label for the node.
815 It tries the following in order:
816 1. key property
817 2. "name" property
818 3. "title" property
819 4. first property from the sorted property name list
820 '''
821 k = self.getkey()
822 if k:
823 return k
824 props = self.getprops()
825 if props.has_key('name'):
826 return 'name'
827 elif props.has_key('title'):
828 return 'title'
829 if default_to_id:
830 return 'id'
831 props = props.keys()
832 props.sort()
833 return props[0]
834 def stringFind(self, **requirements):
835 """Locate a particular node by matching a set of its String
836 properties in a caseless search.
838 If the property is not a String property, a TypeError is raised.
840 The return is a list of the id of all nodes that match.
841 """
842 for propname in requirements.keys():
843 prop = self.properties[propname]
844 if isinstance(not prop, hyperdb.String):
845 raise TypeError, "'%s' not a String property"%propname
846 requirements[propname] = requirements[propname].lower()
847 requirements['_isdel'] = 0
849 l = []
850 for row in self.getview().select(requirements):
851 l.append(str(row.id))
852 return l
854 def addjournal(self, nodeid, action, params):
855 self.db.addjournal(self.classname, nodeid, action, params)
857 def index(self, nodeid):
858 ''' Add (or refresh) the node to search indexes '''
859 # find all the String properties that have indexme
860 for prop, propclass in self.getprops().items():
861 if isinstance(propclass, hyperdb.String) and propclass.indexme:
862 # index them under (classname, nodeid, property)
863 self.db.indexer.add_text((self.classname, nodeid, prop),
864 str(self.get(nodeid, prop)))
866 # --- used by Database
867 def _commit(self):
868 """ called post commit of the DB.
869 interested subclasses may override """
870 self.uncommitted = {}
871 self.rbactions = []
872 self.idcache = {}
873 def _rollback(self):
874 """ called pre rollback of the DB.
875 interested subclasses may override """
876 for action in self.rbactions:
877 action()
878 self.rbactions = []
879 self.uncommitted = {}
880 self.idcache = {}
881 def _clear(self):
882 view = self.getview(1)
883 if len(view):
884 view[:] = []
885 self.db.dirty = 1
886 iv = self.getindexview(1)
887 if iv:
888 iv[:] = []
889 def rollbackaction(self, action):
890 """ call this to register a callback called on rollback
891 callback is removed on end of transaction """
892 self.rbactions.append(action)
893 # --- internal
894 def __getview(self):
895 db = self.db._db
896 view = db.view(self.classname)
897 mkprops = view.structure()
898 if mkprops and self.db.fastopen:
899 return view.ordered(1)
900 # is the definition the same?
901 for nm, rutyp in self.ruprops.items():
902 for mkprop in mkprops:
903 if mkprop.name == nm:
904 break
905 else:
906 mkprop = None
907 if mkprop is None:
908 break
909 if _typmap[rutyp.__class__] != mkprop.type:
910 break
911 else:
912 return view.ordered(1)
913 # need to create or restructure the mk view
914 # id comes first, so MK will order it for us
915 self.db.dirty = 1
916 s = ["%s[id:I" % self.classname]
917 for nm, rutyp in self.ruprops.items():
918 mktyp = _typmap[rutyp.__class__]
919 s.append('%s:%s' % (nm, mktyp))
920 if mktyp == 'V':
921 s[-1] += ('[fid:I]')
922 s.append('_isdel:I,activity:I,creation:I,creator:I]')
923 v = self.db._db.getas(','.join(s))
924 self.db.commit()
925 return v.ordered(1)
926 def getview(self, RW=0):
927 return self.db._db.view(self.classname).ordered(1)
928 def getindexview(self, RW=0):
929 return self.db._db.view("_%s" % self.classname).ordered(1)
931 def _fetchML(sv):
932 l = []
933 for row in sv:
934 if row.fid:
935 l.append(str(row.fid))
936 return l
938 def _fetchPW(s):
939 p = password.Password()
940 p.unpack(s)
941 return p
943 def _fetchLink(n):
944 return n and str(n) or None
946 def _fetchDate(n):
947 return date.Date(time.gmtime(n))
949 _converters = {
950 hyperdb.Date : _fetchDate,
951 hyperdb.Link : _fetchLink,
952 hyperdb.Multilink : _fetchML,
953 hyperdb.Interval : date.Interval,
954 hyperdb.Password : _fetchPW,
955 hyperdb.Boolean : lambda n: n,
956 hyperdb.Number : lambda n: n,
957 hyperdb.String : str,
958 }
960 class FileName(hyperdb.String):
961 isfilename = 1
963 _typmap = {
964 FileName : 'S',
965 hyperdb.String : 'S',
966 hyperdb.Date : 'I',
967 hyperdb.Link : 'I',
968 hyperdb.Multilink : 'V',
969 hyperdb.Interval : 'S',
970 hyperdb.Password : 'S',
971 hyperdb.Boolean : 'I',
972 hyperdb.Number : 'I',
973 }
974 class FileClass(Class):
975 ' like Class but with a content property '
976 default_mime_type = 'text/plain'
977 def __init__(self, db, classname, **properties):
978 properties['content'] = FileName()
979 if not properties.has_key('type'):
980 properties['type'] = hyperdb.String()
981 Class.__init__(self, db, classname, **properties)
982 def get(self, nodeid, propname, default=_marker, cache=1):
983 x = Class.get(self, nodeid, propname, default, cache)
984 if propname == 'content':
985 if x.startswith('file:'):
986 fnm = x[5:]
987 try:
988 x = open(fnm, 'rb').read()
989 except Exception, e:
990 x = repr(e)
991 return x
992 def create(self, **propvalues):
993 content = propvalues['content']
994 del propvalues['content']
995 newid = Class.create(self, **propvalues)
996 if not content:
997 return newid
998 nm = bnm = '%s%s' % (self.classname, newid)
999 sd = str(int(int(newid) / 1000))
1000 d = os.path.join(self.db.config.DATABASE, 'files', self.classname, sd)
1001 if not os.path.exists(d):
1002 os.makedirs(d)
1003 nm = os.path.join(d, nm)
1004 open(nm, 'wb').write(content)
1005 self.set(newid, content = 'file:'+nm)
1006 mimetype = propvalues.get('type', self.default_mime_type)
1007 self.db.indexer.add_text((self.classname, newid, 'content'), content, mimetype)
1008 def undo(fnm=nm, action1=os.remove, indexer=self.db.indexer):
1009 action1(fnm)
1010 self.rollbackaction(undo)
1011 return newid
1012 def index(self, nodeid):
1013 Class.index(self, nodeid)
1014 mimetype = self.get(nodeid, 'type')
1015 if not mimetype:
1016 mimetype = self.default_mime_type
1017 self.db.indexer.add_text((self.classname, nodeid, 'content'),
1018 self.get(nodeid, 'content'), mimetype)
1020 class IssueClass(Class, roundupdb.IssueClass):
1021 # Overridden methods:
1022 def __init__(self, db, classname, **properties):
1023 """The newly-created class automatically includes the "messages",
1024 "files", "nosy", and "superseder" properties. If the 'properties'
1025 dictionary attempts to specify any of these properties or a
1026 "creation" or "activity" property, a ValueError is raised."""
1027 if not properties.has_key('title'):
1028 properties['title'] = hyperdb.String(indexme='yes')
1029 if not properties.has_key('messages'):
1030 properties['messages'] = hyperdb.Multilink("msg")
1031 if not properties.has_key('files'):
1032 properties['files'] = hyperdb.Multilink("file")
1033 if not properties.has_key('nosy'):
1034 # note: journalling is turned off as it really just wastes
1035 # space. this behaviour may be overridden in an instance
1036 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
1037 if not properties.has_key('superseder'):
1038 properties['superseder'] = hyperdb.Multilink(classname)
1039 Class.__init__(self, db, classname, **properties)
1041 CURVERSION = 1
1043 class Indexer(indexer.Indexer):
1044 disallows = {'THE':1, 'THIS':1, 'ZZZ':1, 'THAT':1, 'WITH':1}
1045 def __init__(self, path, datadb):
1046 self.db = metakit.storage(os.path.join(path, 'index.mk4'), 1)
1047 self.datadb = datadb
1048 self.reindex = 0
1049 v = self.db.view('version')
1050 if not v.structure():
1051 v = self.db.getas('version[vers:I]')
1052 self.db.commit()
1053 v.append(vers=CURVERSION)
1054 self.reindex = 1
1055 elif v[0].vers != CURVERSION:
1056 v[0].vers = CURVERSION
1057 self.reindex = 1
1058 if self.reindex:
1059 self.db.getas('ids[tblid:I,nodeid:I,propid:I]')
1060 self.db.getas('index[word:S,hits[pos:I]]')
1061 self.db.commit()
1062 self.reindex = 1
1063 self.changed = 0
1064 self.propcache = {}
1065 def force_reindex(self):
1066 v = self.db.view('ids')
1067 v[:] = []
1068 v = self.db.view('index')
1069 v[:] = []
1070 self.db.commit()
1071 self.reindex = 1
1072 def should_reindex(self):
1073 return self.reindex
1074 def _getprops(self, classname):
1075 props = self.propcache.get(classname, None)
1076 if props is None:
1077 props = self.datadb.view(classname).structure()
1078 props = [prop.name for prop in props]
1079 self.propcache[classname] = props
1080 return props
1081 def _getpropid(self, classname, propname):
1082 return self._getprops(classname).index(propname)
1083 def _getpropname(self, classname, propid):
1084 return self._getprops(classname)[propid]
1086 def add_text(self, identifier, text, mime_type='text/plain'):
1087 if mime_type != 'text/plain':
1088 return
1089 classname, nodeid, property = identifier
1090 tbls = self.datadb.view('tables')
1091 tblid = tbls.find(name=classname)
1092 if tblid < 0:
1093 raise KeyError, "unknown class %r"%classname
1094 nodeid = int(nodeid)
1095 propid = self._getpropid(classname, property)
1096 pos = self.db.view('ids').append(tblid=tblid,nodeid=nodeid,propid=propid)
1098 wordlist = re.findall(r'\b\w{3,25}\b', text)
1099 words = {}
1100 for word in wordlist:
1101 word = word.upper()
1102 if not self.disallows.has_key(word):
1103 words[word] = 1
1104 words = words.keys()
1106 index = self.db.view('index').ordered(1)
1107 for word in words:
1108 ndx = index.find(word=word)
1109 if ndx < 0:
1110 ndx = index.append(word=word)
1111 hits = index[ndx].hits
1112 if len(hits)==0 or hits.find(pos=pos) < 0:
1113 hits.append(pos=pos)
1114 self.changed = 1
1116 def find(self, wordlist):
1117 hits = None
1118 index = self.db.view('index').ordered(1)
1119 for word in wordlist:
1120 if not 2 < len(word) < 26:
1121 continue
1122 ndx = index.find(word=word)
1123 if ndx < 0:
1124 return {}
1125 if hits is None:
1126 hits = index[ndx].hits
1127 else:
1128 hits = hits.intersect(index[ndx].hits)
1129 if len(hits) == 0:
1130 return {}
1131 if hits is None:
1132 return {}
1133 rslt = {}
1134 ids = self.db.view('ids').remapwith(hits)
1135 tbls = self.datadb.view('tables')
1136 for i in range(len(ids)):
1137 hit = ids[i]
1138 classname = tbls[hit.tblid].name
1139 nodeid = str(hit.nodeid)
1140 property = self._getpropname(classname, hit.propid)
1141 rslt[i] = (classname, nodeid, property)
1142 return rslt
1143 def save_index(self):
1144 if self.changed:
1145 self.db.commit()
1146 self.changed = 0