1 from roundup import hyperdb, date, password, roundupdb, security
2 import metakit
3 from sessions import Sessions
4 import re, marshal, os, sys, weakref, time, calendar
5 from roundup import indexer
6 import locking
8 _dbs = {}
10 def Database(config, journaltag=None):
11 db = _dbs.get(config.DATABASE, None)
12 if db is None or db._db is None:
13 db = _Database(config, journaltag)
14 _dbs[config.DATABASE] = db
15 else:
16 db.journaltag = journaltag
17 try:
18 delattr(db, 'curuserid')
19 except AttributeError:
20 pass
21 return db
23 class _Database(hyperdb.Database):
24 def __init__(self, config, journaltag=None):
25 self.config = config
26 self.journaltag = journaltag
27 self.classes = {}
28 self.dirty = 0
29 self.lockfile = None
30 self._db = self.__open()
31 self.indexer = Indexer(self.config.DATABASE, self._db)
32 self.sessions = Sessions(self.config)
33 self.security = security.Security(self)
35 os.umask(0002)
37 def post_init(self):
38 if self.indexer.should_reindex():
39 self.reindex()
41 def reindex(self):
42 for klass in self.classes.values():
43 for nodeid in klass.list():
44 klass.index(nodeid)
45 self.indexer.save_index()
48 # --- defined in ping's spec
49 def __getattr__(self, classname):
50 if classname == 'curuserid':
51 try:
52 self.curuserid = x = int(self.classes['user'].lookup(self.journaltag))
53 except KeyError:
54 x = 0
55 return x
56 elif classname == 'transactions':
57 return self.dirty
58 return self.getclass(classname)
59 def getclass(self, classname):
60 try:
61 return self.classes[classname]
62 except KeyError:
63 raise KeyError, 'There is no class called "%s"'%classname
64 def getclasses(self):
65 return self.classes.keys()
66 # --- end of ping's spec
67 # --- exposed methods
68 def commit(self):
69 if self.dirty:
70 self._db.commit()
71 for cl in self.classes.values():
72 cl._commit()
73 self.indexer.save_index()
74 self.dirty = 0
75 def rollback(self):
76 if self.dirty:
77 for cl in self.classes.values():
78 cl._rollback()
79 self._db.rollback()
80 self.dirty = 0
81 def clear(self):
82 for cl in self.classes.values():
83 cl._clear()
84 def hasnode(self, classname, nodeid):
85 return self.getclass(classname).hasnode(nodeid)
86 def pack(self, pack_before):
87 pass
88 def addclass(self, cl):
89 self.classes[cl.classname] = cl
90 if self.tables.find(name=cl.classname) < 0:
91 self.tables.append(name=cl.classname)
92 def addjournal(self, tablenm, nodeid, action, params):
93 tblid = self.tables.find(name=tablenm)
94 if tblid == -1:
95 tblid = self.tables.append(name=tablenm)
96 # tableid:I,nodeid:I,date:I,user:I,action:I,params:B
97 self.hist.append(tableid=tblid,
98 nodeid=int(nodeid),
99 date=int(time.time()),
100 action=action,
101 user = self.curuserid,
102 params = marshal.dumps(params))
103 def gethistory(self, tablenm, nodeid):
104 rslt = []
105 tblid = self.tables.find(name=tablenm)
106 if tblid == -1:
107 return rslt
108 q = self.hist.select(tableid=tblid, nodeid=int(nodeid))
109 i = 0
110 userclass = self.getclass('user')
111 for row in q:
112 try:
113 params = marshal.loads(row.params)
114 except ValueError:
115 print "history couldn't unmarshal %r" % row.params
116 params = {}
117 usernm = userclass.get(str(row.user), 'username')
118 dt = date.Date(time.gmtime(row.date))
119 rslt.append((i, dt, usernm, _actionnames[row.action], params))
120 i += 1
121 return rslt
123 def close(self):
124 for cl in self.classes.values():
125 cl.db = None
126 self._db = None
127 if self.lockfile is not None:
128 locking.release_lock(self.lockfile)
129 if _dbs.has_key(self.config.DATABASE):
130 del _dbs[self.config.DATABASE]
131 if self.lockfile is not None:
132 self.lockfile.close()
133 self.lockfile = None
134 self.classes = {}
135 self.indexer = None
137 # --- internal
138 def __open(self):
139 self.dbnm = db = os.path.join(self.config.DATABASE, 'tracker.mk4')
140 lockfilenm = db[:-3]+'lck'
141 self.lockfile = locking.acquire_lock(lockfilenm)
142 self.lockfile.write(str(os.getpid()))
143 self.lockfile.flush()
144 self.fastopen = 0
145 if os.path.exists(db):
146 dbtm = os.path.getmtime(db)
147 pkgnm = self.config.__name__.split('.')[0]
148 schemamod = sys.modules.get(pkgnm+'.dbinit', None)
149 if schemamod:
150 if os.path.exists(schemamod.__file__):
151 schematm = os.path.getmtime(schemamod.__file__)
152 if schematm < dbtm:
153 # found schema mod - it's older than the db
154 self.fastopen = 1
155 else:
156 # can't find schemamod - must be frozen
157 self.fastopen = 1
158 db = metakit.storage(db, 1)
159 hist = db.view('history')
160 tables = db.view('tables')
161 if not self.fastopen:
162 if not hist.structure():
163 hist = db.getas('history[tableid:I,nodeid:I,date:I,user:I,action:I,params:B]')
164 if not tables.structure():
165 tables = db.getas('tables[name:S]')
166 self.tables = tables
167 self.hist = hist
168 return db
170 _STRINGTYPE = type('')
171 _LISTTYPE = type([])
172 _CREATE, _SET, _RETIRE, _LINK, _UNLINK = range(5)
174 _actionnames = {
175 _CREATE : 'create',
176 _SET : 'set',
177 _RETIRE : 'retire',
178 _LINK : 'link',
179 _UNLINK : 'unlink',
180 }
182 _marker = []
184 _ALLOWSETTINGPRIVATEPROPS = 0
186 class Class:
187 privateprops = None
188 def __init__(self, db, classname, **properties):
189 #self.db = weakref.proxy(db)
190 self.db = db
191 self.classname = classname
192 self.keyname = None
193 self.ruprops = properties
194 self.privateprops = { 'id' : hyperdb.String(),
195 'activity' : hyperdb.Date(),
196 'creation' : hyperdb.Date(),
197 'creator' : hyperdb.Link('user') }
198 self.auditors = {'create': [], 'set': [], 'retire': []} # event -> list of callables
199 self.reactors = {'create': [], 'set': [], 'retire': []} # ditto
200 view = self.__getview()
201 self.maxid = 1
202 if view:
203 self.maxid = view[-1].id + 1
204 self.uncommitted = {}
205 self.rbactions = []
206 # people reach inside!!
207 self.properties = self.ruprops
208 self.db.addclass(self)
209 self.idcache = {}
211 # default is to journal changes
212 self.do_journal = 1
214 def enableJournalling(self):
215 '''Turn journalling on for this class
216 '''
217 self.do_journal = 1
219 def disableJournalling(self):
220 '''Turn journalling off for this class
221 '''
222 self.do_journal = 0
224 # --- the roundup.Class methods
225 def audit(self, event, detector):
226 l = self.auditors[event]
227 if detector not in l:
228 self.auditors[event].append(detector)
229 def fireAuditors(self, action, nodeid, newvalues):
230 for audit in self.auditors[action]:
231 audit(self.db, self, nodeid, newvalues)
232 def fireReactors(self, action, nodeid, oldvalues):
233 for react in self.reactors[action]:
234 react(self.db, self, nodeid, oldvalues)
235 def react(self, event, detector):
236 l = self.reactors[event]
237 if detector not in l:
238 self.reactors[event].append(detector)
239 # --- the hyperdb.Class methods
240 def create(self, **propvalues):
241 self.fireAuditors('create', None, propvalues)
242 rowdict = {}
243 rowdict['id'] = newid = self.maxid
244 self.maxid += 1
245 ndx = self.getview(1).append(rowdict)
246 propvalues['#ISNEW'] = 1
247 try:
248 self.set(str(newid), **propvalues)
249 except Exception:
250 self.maxid -= 1
251 raise
252 return str(newid)
254 def get(self, nodeid, propname, default=_marker, cache=1):
255 # default and cache aren't in the spec
256 # cache=0 means "original value"
258 view = self.getview()
259 id = int(nodeid)
260 if cache == 0:
261 oldnode = self.uncommitted.get(id, None)
262 if oldnode and oldnode.has_key(propname):
263 return oldnode[propname]
264 ndx = self.idcache.get(id, None)
265 if ndx is None:
266 ndx = view.find(id=id)
267 if ndx < 0:
268 raise IndexError, "%s has no node %s" % (self.classname, nodeid)
269 self.idcache[id] = ndx
270 try:
271 raw = getattr(view[ndx], propname)
272 except AttributeError:
273 raise KeyError, propname
274 rutyp = self.ruprops.get(propname, None)
275 if rutyp is None:
276 rutyp = self.privateprops[propname]
277 converter = _converters.get(rutyp.__class__, None)
278 if converter:
279 raw = converter(raw)
280 return raw
282 def set(self, nodeid, **propvalues):
283 isnew = 0
284 if propvalues.has_key('#ISNEW'):
285 isnew = 1
286 del propvalues['#ISNEW']
287 if not isnew:
288 self.fireAuditors('set', nodeid, propvalues)
289 if not propvalues:
290 return propvalues
291 if propvalues.has_key('id'):
292 raise KeyError, '"id" is reserved'
293 if self.db.journaltag is None:
294 raise DatabaseError, 'Database open read-only'
295 view = self.getview(1)
296 # node must exist & not be retired
297 id = int(nodeid)
298 ndx = view.find(id=id)
299 if ndx < 0:
300 raise IndexError, "%s has no node %s" % (self.classname, nodeid)
301 row = view[ndx]
302 if row._isdel:
303 raise IndexError, "%s has no node %s" % (self.classname, nodeid)
304 oldnode = self.uncommitted.setdefault(id, {})
305 changes = {}
307 for key, value in propvalues.items():
308 # this will raise the KeyError if the property isn't valid
309 # ... we don't use getprops() here because we only care about
310 # the writeable properties.
311 if _ALLOWSETTINGPRIVATEPROPS:
312 prop = self.ruprops.get(key, None)
313 if not prop:
314 prop = self.privateprops[key]
315 else:
316 prop = self.ruprops[key]
317 converter = _converters.get(prop.__class__, lambda v: v)
318 # if the value's the same as the existing value, no sense in
319 # doing anything
320 oldvalue = converter(getattr(row, key))
321 if value == oldvalue:
322 del propvalues[key]
323 continue
325 # check to make sure we're not duplicating an existing key
326 if key == self.keyname:
327 iv = self.getindexview(1)
328 ndx = iv.find(k=value)
329 if ndx == -1:
330 iv.append(k=value, i=row.id)
331 if not isnew:
332 ndx = iv.find(k=oldvalue)
333 if ndx > -1:
334 iv.delete(ndx)
335 else:
336 raise ValueError, 'node with key "%s" exists'%value
338 # do stuff based on the prop type
339 if isinstance(prop, hyperdb.Link):
340 link_class = prop.classname
341 # must be a string or None
342 if value is not None and not isinstance(value, type('')):
343 raise ValueError, 'property "%s" link value be a string'%(
344 propname)
345 # Roundup sets to "unselected" by passing None
346 if value is None:
347 value = 0
348 # if it isn't a number, it's a key
349 try:
350 int(value)
351 except ValueError:
352 try:
353 value = self.db.getclass(link_class).lookup(value)
354 except (TypeError, KeyError):
355 raise IndexError, 'new property "%s": %s not a %s'%(
356 key, value, prop.classname)
358 if (value is not None and
359 not self.db.getclass(link_class).hasnode(value)):
360 raise IndexError, '%s has no node %s'%(link_class, value)
362 setattr(row, key, int(value))
363 changes[key] = oldvalue
365 if self.do_journal and prop.do_journal:
366 # register the unlink with the old linked node
367 if oldvalue:
368 self.db.addjournal(link_class, value, _UNLINK,
369 (self.classname, str(row.id), key))
371 # register the link with the newly linked node
372 if value:
373 self.db.addjournal(link_class, value, _LINK,
374 (self.classname, str(row.id), key))
376 elif isinstance(prop, hyperdb.Multilink):
377 if type(value) != _LISTTYPE:
378 raise TypeError, 'new property "%s" not a list of ids'%key
379 link_class = prop.classname
380 l = []
381 for entry in value:
382 if type(entry) != _STRINGTYPE:
383 raise ValueError, 'new property "%s" link value ' \
384 'must be a string'%key
385 # if it isn't a number, it's a key
386 try:
387 int(entry)
388 except ValueError:
389 try:
390 entry = self.db.getclass(link_class).lookup(entry)
391 except (TypeError, KeyError):
392 raise IndexError, 'new property "%s": %s not a %s'%(
393 key, entry, prop.classname)
394 l.append(entry)
395 propvalues[key] = value = l
397 # handle removals
398 rmvd = []
399 for id in oldvalue:
400 if id not in value:
401 rmvd.append(id)
402 # register the unlink with the old linked node
403 if self.do_journal and prop.do_journal:
404 self.db.addjournal(link_class, id, _UNLINK,
405 (self.classname, str(row.id), key))
407 # handle additions
408 adds = []
409 for id in value:
410 if id not in oldvalue:
411 if not self.db.getclass(link_class).hasnode(id):
412 raise IndexError, '%s has no node %s'%(
413 link_class, id)
414 adds.append(id)
415 # register the link with the newly linked node
416 if self.do_journal and prop.do_journal:
417 self.db.addjournal(link_class, id, _LINK,
418 (self.classname, str(row.id), key))
420 sv = getattr(row, key)
421 i = 0
422 while i < len(sv):
423 if str(sv[i].fid) in rmvd:
424 sv.delete(i)
425 else:
426 i += 1
427 for id in adds:
428 sv.append(fid=int(id))
429 changes[key] = oldvalue
430 if not rmvd and not adds:
431 del propvalues[key]
433 elif isinstance(prop, hyperdb.String):
434 if value is not None and type(value) != _STRINGTYPE:
435 raise TypeError, 'new property "%s" not a string'%key
436 setattr(row, key, value)
437 changes[key] = oldvalue
438 if hasattr(prop, 'isfilename') and prop.isfilename:
439 propvalues[key] = os.path.basename(value)
440 if prop.indexme and value is not None:
441 self.db.indexer.add_text((self.classname, nodeid, key),
442 value, 'text/plain')
444 elif isinstance(prop, hyperdb.Password):
445 if not isinstance(value, password.Password):
446 raise TypeError, 'new property "%s" not a Password'% key
447 setattr(row, key, str(value))
448 changes[key] = str(oldvalue)
449 propvalues[key] = str(value)
451 elif value is not None and isinstance(prop, hyperdb.Date):
452 if not isinstance(value, date.Date):
453 raise TypeError, 'new property "%s" not a Date'% key
454 setattr(row, key, int(calendar.timegm(value.get_tuple())))
455 changes[key] = str(oldvalue)
456 propvalues[key] = str(value)
458 elif value is not None and isinstance(prop, hyperdb.Interval):
459 if not isinstance(value, date.Interval):
460 raise TypeError, 'new property "%s" not an Interval'% key
461 setattr(row, key, str(value))
462 changes[key] = str(oldvalue)
463 propvalues[key] = str(value)
465 elif value is not None and isinstance(prop, hyperdb.Number):
466 setattr(row, key, int(value))
467 changes[key] = oldvalue
468 propvalues[key] = value
470 elif value is not None and isinstance(prop, hyperdb.Boolean):
471 bv = value != 0
472 setattr(row, key, bv)
473 changes[key] = oldvalue
474 propvalues[key] = value
476 oldnode[key] = oldvalue
478 # nothing to do?
479 if not propvalues:
480 return propvalues
481 if not propvalues.has_key('activity'):
482 row.activity = int(time.time())
483 if isnew:
484 if not row.creation:
485 row.creation = int(time.time())
486 if not row.creator:
487 row.creator = self.db.curuserid
489 self.db.dirty = 1
490 if self.do_journal:
491 if isnew:
492 self.db.addjournal(self.classname, nodeid, _CREATE, {})
493 self.fireReactors('create', nodeid, None)
494 else:
495 self.db.addjournal(self.classname, nodeid, _SET, changes)
496 self.fireReactors('set', nodeid, oldnode)
498 return propvalues
500 def retire(self, nodeid):
501 self.fireAuditors('retire', nodeid, None)
502 view = self.getview(1)
503 ndx = view.find(id=int(nodeid))
504 if ndx < 0:
505 raise KeyError, "nodeid %s not found" % nodeid
506 row = view[ndx]
507 oldvalues = self.uncommitted.setdefault(row.id, {})
508 oldval = oldvalues['_isdel'] = row._isdel
509 row._isdel = 1
510 if self.do_journal:
511 self.db.addjournal(self.classname, nodeid, _RETIRE, {})
512 if self.keyname:
513 iv = self.getindexview(1)
514 ndx = iv.find(k=getattr(row, self.keyname),i=row.id)
515 if ndx > -1:
516 iv.delete(ndx)
517 self.db.dirty = 1
518 self.fireReactors('retire', nodeid, None)
519 def history(self, nodeid):
520 if not self.do_journal:
521 raise ValueError, 'Journalling is disabled for this class'
522 return self.db.gethistory(self.classname, nodeid)
523 def setkey(self, propname):
524 if self.keyname:
525 if propname == self.keyname:
526 return
527 raise ValueError, "%s already indexed on %s" % (self.classname, self.keyname)
528 # first setkey for this run
529 self.keyname = propname
530 iv = self.db._db.view('_%s' % self.classname)
531 if self.db.fastopen and iv.structure():
532 return
533 # very first setkey ever
534 self.db.dirty = 1
535 iv = self.db._db.getas('_%s[k:S,i:I]' % self.classname)
536 iv = iv.ordered(1)
537 # print "setkey building index"
538 for row in self.getview():
539 iv.append(k=getattr(row, propname), i=row.id)
540 self.db.commit()
541 def getkey(self):
542 return self.keyname
543 def lookup(self, keyvalue):
544 if type(keyvalue) is not _STRINGTYPE:
545 raise TypeError, "%r is not a string" % keyvalue
546 iv = self.getindexview()
547 if iv:
548 ndx = iv.find(k=keyvalue)
549 if ndx > -1:
550 return str(iv[ndx].i)
551 else:
552 view = self.getview()
553 ndx = view.find({self.keyname:keyvalue, '_isdel':0})
554 if ndx > -1:
555 return str(view[ndx].id)
556 raise KeyError, keyvalue
558 def destroy(self, keyvalue):
559 #TODO clean this up once Richard's said how it should work
560 iv = self.getindexview()
561 if iv:
562 ndx = iv.find(k=keyvalue)
563 if ndx > -1:
564 id = iv[ndx].i
565 iv.delete(ndx)
566 view = self.getview()
567 ndx = view.find(id=id)
568 if ndx > -1:
569 view.delete(ndx)
571 def find(self, **propspec):
572 """Get the ids of nodes in this class which link to the given nodes.
574 'propspec' consists of keyword args propname={nodeid:1,}
575 'propname' must be the name of a property in this class, or a
576 KeyError is raised. That property must be a Link or
577 Multilink property, or a TypeError is raised.
579 Any node in this class whose propname property links to any of the
580 nodeids will be returned. Used by the full text indexing, which knows
581 that "foo" occurs in msg1, msg3 and file7; so we have hits on these
582 issues:
584 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
586 """
587 propspec = propspec.items()
588 for propname, nodeid in propspec:
589 # check the prop is OK
590 prop = self.ruprops[propname]
591 if (not isinstance(prop, hyperdb.Link) and
592 not isinstance(prop, hyperdb.Multilink)):
593 raise TypeError, "'%s' not a Link/Multilink property"%propname
595 vws = []
596 for propname, ids in propspec:
597 if type(ids) is _STRINGTYPE:
598 ids = {ids:1}
599 prop = self.ruprops[propname]
600 view = self.getview()
601 if isinstance(prop, hyperdb.Multilink):
602 view = view.flatten(getattr(view, propname))
603 def ff(row, nm=propname, ids=ids):
604 return ids.has_key(str(row.fid))
605 else:
606 def ff(row, nm=propname, ids=ids):
607 return ids.has_key(str(getattr(row, nm)))
608 ndxview = view.filter(ff)
609 vws.append(ndxview.unique())
611 # handle the empty match case
612 if not vws:
613 return []
615 ndxview = vws[0]
616 for v in vws[1:]:
617 ndxview = ndxview.union(v)
618 view = view.remapwith(ndxview)
619 rslt = []
620 for row in view:
621 rslt.append(str(row.id))
622 return rslt
625 def list(self):
626 l = []
627 for row in self.getview().select(_isdel=0):
628 l.append(str(row.id))
629 return l
630 def count(self):
631 return len(self.getview())
632 def getprops(self, protected=1):
633 # protected is not in ping's spec
634 allprops = self.ruprops.copy()
635 if protected and self.privateprops is not None:
636 allprops.update(self.privateprops)
637 return allprops
638 def addprop(self, **properties):
639 for key in properties.keys():
640 if self.ruprops.has_key(key):
641 raise ValueError, "%s is already a property of %s" % (key, self.classname)
642 self.ruprops.update(properties)
643 self.db.fastopen = 0
644 view = self.__getview()
645 self.db.commit()
646 # ---- end of ping's spec
647 def filter(self, search_matches, filterspec, sort, group):
648 # search_matches is None or a set (dict of {nodeid: {propname:[nodeid,...]}})
649 # filterspec is a dict {propname:value}
650 # sort and group are lists of propnames
651 # sort and group are (dir, prop) where dir is '+', '-' or None
652 # and prop is a prop name or None
654 where = {'_isdel':0}
655 mlcriteria = {}
656 regexes = {}
657 orcriteria = {}
658 for propname, value in filterspec.items():
659 prop = self.ruprops.get(propname, None)
660 if prop is None:
661 prop = self.privateprops[propname]
662 if isinstance(prop, hyperdb.Multilink):
663 if type(value) is not _LISTTYPE:
664 value = [value]
665 # transform keys to ids
666 u = []
667 for item in value:
668 try:
669 item = int(item)
670 except (TypeError, ValueError):
671 item = int(self.db.getclass(prop.classname).lookup(item))
672 if item == -1:
673 item = 0
674 u.append(item)
675 mlcriteria[propname] = u
676 elif isinstance(prop, hyperdb.Link):
677 if type(value) is not _LISTTYPE:
678 value = [value]
679 # transform keys to ids
680 u = []
681 for item in value:
682 try:
683 item = int(item)
684 except (TypeError, ValueError):
685 item = int(self.db.getclass(prop.classname).lookup(item))
686 if item == -1:
687 item = 0
688 u.append(item)
689 if len(u) == 1:
690 where[propname] = u[0]
691 else:
692 orcriteria[propname] = u
693 elif isinstance(prop, hyperdb.String):
694 # simple glob searching
695 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', value)
696 v = v.replace('?', '.')
697 v = v.replace('*', '.*?')
698 regexes[propname] = re.compile(v, re.I)
699 elif propname == 'id':
700 where[propname] = int(value)
701 elif isinstance(prop, hyperdb.Boolean):
702 if type(value) is _STRINGTYPE:
703 bv = value.lower() in ('yes', 'true', 'on', '1')
704 else:
705 bv = value
706 where[propname] = bv
707 elif isinstance(prop, hyperdb.Number):
708 where[propname] = int(value)
709 else:
710 where[propname] = str(value)
711 v = self.getview()
712 #print "filter start at %s" % time.time()
713 if where:
714 v = v.select(where)
715 #print "filter where at %s" % time.time()
717 if mlcriteria:
718 # multilink - if any of the nodeids required by the
719 # filterspec aren't in this node's property, then skip
720 # it
721 def ff(row, ml=mlcriteria):
722 for propname, values in ml.items():
723 sv = getattr(row, propname)
724 for id in values:
725 if sv.find(fid=id) == -1:
726 return 0
727 return 1
728 iv = v.filter(ff)
729 v = v.remapwith(iv)
731 #print "filter mlcrit at %s" % time.time()
733 if orcriteria:
734 def ff(row, crit=orcriteria):
735 for propname, allowed in crit.items():
736 val = getattr(row, propname)
737 if val not in allowed:
738 return 0
739 return 1
741 iv = v.filter(ff)
742 v = v.remapwith(iv)
744 #print "filter orcrit at %s" % time.time()
745 if regexes:
746 def ff(row, r=regexes):
747 for propname, regex in r.items():
748 val = getattr(row, propname)
749 if not regex.search(val):
750 return 0
751 return 1
753 iv = v.filter(ff)
754 v = v.remapwith(iv)
755 #print "filter regexs at %s" % time.time()
757 if sort or group:
758 sortspec = []
759 rev = []
760 for dir, propname in group, sort:
761 if propname is None: continue
762 isreversed = 0
763 if dir == '-':
764 isreversed = 1
765 try:
766 prop = getattr(v, propname)
767 except AttributeError:
768 print "MK has no property %s" % propname
769 continue
770 propclass = self.ruprops.get(propname, None)
771 if propclass is None:
772 propclass = self.privateprops.get(propname, None)
773 if propclass is None:
774 print "Schema has no property %s" % propname
775 continue
776 if isinstance(propclass, hyperdb.Link):
777 linkclass = self.db.getclass(propclass.classname)
778 lv = linkclass.getview()
779 lv = lv.rename('id', propname)
780 v = v.join(lv, prop, 1)
781 if linkclass.getprops().has_key('order'):
782 propname = 'order'
783 else:
784 propname = linkclass.labelprop()
785 prop = getattr(v, propname)
786 if isreversed:
787 rev.append(prop)
788 sortspec.append(prop)
789 v = v.sortrev(sortspec, rev)[:] #XXX Metakit bug
790 #print "filter sort at %s" % time.time()
792 rslt = []
793 for row in v:
794 id = str(row.id)
795 if search_matches is not None:
796 if search_matches.has_key(id):
797 rslt.append(id)
798 else:
799 rslt.append(id)
800 return rslt
802 def hasnode(self, nodeid):
803 return int(nodeid) < self.maxid
805 def labelprop(self, default_to_id=0):
806 ''' Return the property name for a label for the given node.
808 This method attempts to generate a consistent label for the node.
809 It tries the following in order:
810 1. key property
811 2. "name" property
812 3. "title" property
813 4. first property from the sorted property name list
814 '''
815 k = self.getkey()
816 if k:
817 return k
818 props = self.getprops()
819 if props.has_key('name'):
820 return 'name'
821 elif props.has_key('title'):
822 return 'title'
823 if default_to_id:
824 return 'id'
825 props = props.keys()
826 props.sort()
827 return props[0]
828 def stringFind(self, **requirements):
829 """Locate a particular node by matching a set of its String
830 properties in a caseless search.
832 If the property is not a String property, a TypeError is raised.
834 The return is a list of the id of all nodes that match.
835 """
836 for propname in requirements.keys():
837 prop = self.properties[propname]
838 if isinstance(not prop, hyperdb.String):
839 raise TypeError, "'%s' not a String property"%propname
840 requirements[propname] = requirements[propname].lower()
841 requirements['_isdel'] = 0
843 l = []
844 for row in self.getview().select(requirements):
845 l.append(str(row.id))
846 return l
848 def addjournal(self, nodeid, action, params):
849 self.db.addjournal(self.classname, nodeid, action, params)
851 def index(self, nodeid):
852 ''' Add (or refresh) the node to search indexes '''
853 # find all the String properties that have indexme
854 for prop, propclass in self.getprops().items():
855 if isinstance(propclass, hyperdb.String) and propclass.indexme:
856 # index them under (classname, nodeid, property)
857 self.db.indexer.add_text((self.classname, nodeid, prop),
858 str(self.get(nodeid, prop)))
860 # --- used by Database
861 def _commit(self):
862 """ called post commit of the DB.
863 interested subclasses may override """
864 self.uncommitted = {}
865 self.rbactions = []
866 self.idcache = {}
867 def _rollback(self):
868 """ called pre rollback of the DB.
869 interested subclasses may override """
870 for action in self.rbactions:
871 action()
872 self.rbactions = []
873 self.uncommitted = {}
874 self.idcache = {}
875 def _clear(self):
876 view = self.getview(1)
877 if len(view):
878 view[:] = []
879 self.db.dirty = 1
880 iv = self.getindexview(1)
881 if iv:
882 iv[:] = []
883 def rollbackaction(self, action):
884 """ call this to register a callback called on rollback
885 callback is removed on end of transaction """
886 self.rbactions.append(action)
887 # --- internal
888 def __getview(self):
889 db = self.db._db
890 view = db.view(self.classname)
891 mkprops = view.structure()
892 if mkprops and self.db.fastopen:
893 return view.ordered(1)
894 # is the definition the same?
895 for nm, rutyp in self.ruprops.items():
896 for mkprop in mkprops:
897 if mkprop.name == nm:
898 break
899 else:
900 mkprop = None
901 if mkprop is None:
902 break
903 if _typmap[rutyp.__class__] != mkprop.type:
904 break
905 else:
906 return view.ordered(1)
907 # need to create or restructure the mk view
908 # id comes first, so MK will order it for us
909 self.db.dirty = 1
910 s = ["%s[id:I" % self.classname]
911 for nm, rutyp in self.ruprops.items():
912 mktyp = _typmap[rutyp.__class__]
913 s.append('%s:%s' % (nm, mktyp))
914 if mktyp == 'V':
915 s[-1] += ('[fid:I]')
916 s.append('_isdel:I,activity:I,creation:I,creator:I]')
917 v = self.db._db.getas(','.join(s))
918 self.db.commit()
919 return v.ordered(1)
920 def getview(self, RW=0):
921 return self.db._db.view(self.classname).ordered(1)
922 def getindexview(self, RW=0):
923 return self.db._db.view("_%s" % self.classname).ordered(1)
925 def _fetchML(sv):
926 l = []
927 for row in sv:
928 if row.fid:
929 l.append(str(row.fid))
930 return l
932 def _fetchPW(s):
933 p = password.Password()
934 p.unpack(s)
935 return p
937 def _fetchLink(n):
938 return n and str(n) or None
940 def _fetchDate(n):
941 return date.Date(time.gmtime(n))
943 _converters = {
944 hyperdb.Date : _fetchDate,
945 hyperdb.Link : _fetchLink,
946 hyperdb.Multilink : _fetchML,
947 hyperdb.Interval : date.Interval,
948 hyperdb.Password : _fetchPW,
949 hyperdb.Boolean : lambda n: n,
950 hyperdb.Number : lambda n: n,
951 hyperdb.String : str,
952 }
954 class FileName(hyperdb.String):
955 isfilename = 1
957 _typmap = {
958 FileName : 'S',
959 hyperdb.String : 'S',
960 hyperdb.Date : 'I',
961 hyperdb.Link : 'I',
962 hyperdb.Multilink : 'V',
963 hyperdb.Interval : 'S',
964 hyperdb.Password : 'S',
965 hyperdb.Boolean : 'I',
966 hyperdb.Number : 'I',
967 }
968 class FileClass(Class):
969 ' like Class but with a content property '
970 default_mime_type = 'text/plain'
971 def __init__(self, db, classname, **properties):
972 properties['content'] = FileName()
973 if not properties.has_key('type'):
974 properties['type'] = hyperdb.String()
975 Class.__init__(self, db, classname, **properties)
976 def get(self, nodeid, propname, default=_marker, cache=1):
977 x = Class.get(self, nodeid, propname, default, cache)
978 if propname == 'content':
979 if x.startswith('file:'):
980 fnm = x[5:]
981 try:
982 x = open(fnm, 'rb').read()
983 except Exception, e:
984 x = repr(e)
985 return x
986 def create(self, **propvalues):
987 content = propvalues['content']
988 del propvalues['content']
989 newid = Class.create(self, **propvalues)
990 if not content:
991 return newid
992 nm = bnm = '%s%s' % (self.classname, newid)
993 sd = str(int(int(newid) / 1000))
994 d = os.path.join(self.db.config.DATABASE, 'files', self.classname, sd)
995 if not os.path.exists(d):
996 os.makedirs(d)
997 nm = os.path.join(d, nm)
998 open(nm, 'wb').write(content)
999 self.set(newid, content = 'file:'+nm)
1000 mimetype = propvalues.get('type', self.default_mime_type)
1001 self.db.indexer.add_text((self.classname, newid, 'content'), content, mimetype)
1002 def undo(fnm=nm, action1=os.remove, indexer=self.db.indexer):
1003 action1(fnm)
1004 self.rollbackaction(undo)
1005 return newid
1006 def index(self, nodeid):
1007 Class.index(self, nodeid)
1008 mimetype = self.get(nodeid, 'type')
1009 if not mimetype:
1010 mimetype = self.default_mime_type
1011 self.db.indexer.add_text((self.classname, nodeid, 'content'),
1012 self.get(nodeid, 'content'), mimetype)
1014 class IssueClass(Class, roundupdb.IssueClass):
1015 # Overridden methods:
1016 def __init__(self, db, classname, **properties):
1017 """The newly-created class automatically includes the "messages",
1018 "files", "nosy", and "superseder" properties. If the 'properties'
1019 dictionary attempts to specify any of these properties or a
1020 "creation" or "activity" property, a ValueError is raised."""
1021 if not properties.has_key('title'):
1022 properties['title'] = hyperdb.String(indexme='yes')
1023 if not properties.has_key('messages'):
1024 properties['messages'] = hyperdb.Multilink("msg")
1025 if not properties.has_key('files'):
1026 properties['files'] = hyperdb.Multilink("file")
1027 if not properties.has_key('nosy'):
1028 # note: journalling is turned off as it really just wastes
1029 # space. this behaviour may be overridden in an instance
1030 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
1031 if not properties.has_key('superseder'):
1032 properties['superseder'] = hyperdb.Multilink(classname)
1033 Class.__init__(self, db, classname, **properties)
1035 CURVERSION = 1
1037 class Indexer(indexer.Indexer):
1038 disallows = {'THE':1, 'THIS':1, 'ZZZ':1, 'THAT':1, 'WITH':1}
1039 def __init__(self, path, datadb):
1040 self.db = metakit.storage(os.path.join(path, 'index.mk4'), 1)
1041 self.datadb = datadb
1042 self.reindex = 0
1043 v = self.db.view('version')
1044 if not v.structure():
1045 v = self.db.getas('version[vers:I]')
1046 self.db.commit()
1047 v.append(vers=CURVERSION)
1048 self.reindex = 1
1049 elif v[0].vers != CURVERSION:
1050 v[0].vers = CURVERSION
1051 self.reindex = 1
1052 if self.reindex:
1053 self.db.getas('ids[tblid:I,nodeid:I,propid:I]')
1054 self.db.getas('index[word:S,hits[pos:I]]')
1055 self.db.commit()
1056 self.reindex = 1
1057 self.changed = 0
1058 self.propcache = {}
1059 def force_reindex(self):
1060 v = self.db.view('ids')
1061 v[:] = []
1062 v = self.db.view('index')
1063 v[:] = []
1064 self.db.commit()
1065 self.reindex = 1
1066 def should_reindex(self):
1067 return self.reindex
1068 def _getprops(self, classname):
1069 props = self.propcache.get(classname, None)
1070 if props is None:
1071 props = self.datadb.view(classname).structure()
1072 props = [prop.name for prop in props]
1073 self.propcache[classname] = props
1074 return props
1075 def _getpropid(self, classname, propname):
1076 return self._getprops(classname).index(propname)
1077 def _getpropname(self, classname, propid):
1078 return self._getprops(classname)[propid]
1080 def add_text(self, identifier, text, mime_type='text/plain'):
1081 if mime_type != 'text/plain':
1082 return
1083 classname, nodeid, property = identifier
1084 tbls = self.datadb.view('tables')
1085 tblid = tbls.find(name=classname)
1086 if tblid < 0:
1087 raise KeyError, "unknown class %r"%classname
1088 nodeid = int(nodeid)
1089 propid = self._getpropid(classname, property)
1090 pos = self.db.view('ids').append(tblid=tblid,nodeid=nodeid,propid=propid)
1092 wordlist = re.findall(r'\b\w{3,25}\b', text)
1093 words = {}
1094 for word in wordlist:
1095 word = word.upper()
1096 if not self.disallows.has_key(word):
1097 words[word] = 1
1098 words = words.keys()
1100 index = self.db.view('index').ordered(1)
1101 for word in words:
1102 ndx = index.find(word=word)
1103 if ndx < 0:
1104 ndx = index.append(word=word)
1105 hits = index[ndx].hits
1106 if len(hits)==0 or hits.find(pos=pos) < 0:
1107 hits.append(pos=pos)
1108 self.changed = 1
1110 def find(self, wordlist):
1111 hits = None
1112 index = self.db.view('index').ordered(1)
1113 for word in wordlist:
1114 if not 2 < len(word) < 26:
1115 continue
1116 ndx = index.find(word=word)
1117 if ndx < 0:
1118 return {}
1119 if hits is None:
1120 hits = index[ndx].hits
1121 else:
1122 hits = hits.intersect(index[ndx].hits)
1123 if len(hits) == 0:
1124 return {}
1125 if hits is None:
1126 return {}
1127 rslt = {}
1128 ids = self.db.view('ids').remapwith(hits)
1129 tbls = self.datadb.view('tables')
1130 for i in range(len(ids)):
1131 hit = ids[i]
1132 classname = tbls[hit.tblid].name
1133 nodeid = str(hit.nodeid)
1134 property = self._getpropname(classname, hit.propid)
1135 rslt[i] = (classname, nodeid, property)
1136 return rslt
1137 def save_index(self):
1138 if self.changed:
1139 self.db.commit()
1140 self.changed = 0