d78b27f9f8034f176cf7c29718d4c42938faad02
1 from roundup import hyperdb, date, password, roundupdb, security
2 import metakit
3 from sessions import Sessions
4 import re, marshal, os, sys, weakref, time, calendar
5 from roundup import indexer
6 import locking
8 _dbs = {}
10 def Database(config, journaltag=None):
11 db = _dbs.get(config.DATABASE, None)
12 if db is None or db._db is None:
13 db = _Database(config, journaltag)
14 _dbs[config.DATABASE] = db
15 else:
16 db.journaltag = journaltag
17 try:
18 delattr(db, 'curuserid')
19 except AttributeError:
20 pass
21 return db
23 class _Database(hyperdb.Database):
24 def __init__(self, config, journaltag=None):
25 self.config = config
26 self.journaltag = journaltag
27 self.classes = {}
28 self.dirty = 0
29 self.lockfile = None
30 self._db = self.__open()
31 self.indexer = Indexer(self.config.DATABASE, self._db)
32 self.sessions = Sessions(self.config)
33 self.security = security.Security(self)
35 os.umask(0002)
36 def post_init(self):
37 if self.indexer.should_reindex():
38 self.reindex()
40 def reindex(self):
41 for klass in self.classes.values():
42 for nodeid in klass.list():
43 klass.index(nodeid)
44 self.indexer.save_index()
47 # --- defined in ping's spec
48 def __getattr__(self, classname):
49 if classname == 'curuserid':
50 try:
51 self.curuserid = x = int(self.classes['user'].lookup(self.journaltag))
52 except KeyError:
53 x = 0
54 return x
55 elif classname == 'transactions':
56 return self.dirty
57 return self.getclass(classname)
58 def getclass(self, classname):
59 return self.classes[classname]
60 def getclasses(self):
61 return self.classes.keys()
62 # --- end of ping's spec
63 # --- exposed methods
64 def commit(self):
65 if self.dirty:
66 self._db.commit()
67 for cl in self.classes.values():
68 cl._commit()
69 self.indexer.save_index()
70 self.dirty = 0
71 def rollback(self):
72 if self.dirty:
73 for cl in self.classes.values():
74 cl._rollback()
75 self._db.rollback()
76 self.dirty = 0
77 def clear(self):
78 for cl in self.classes.values():
79 cl._clear()
80 def hasnode(self, classname, nodeid):
81 return self.getclass(classname).hasnode(nodeid)
82 def pack(self, pack_before):
83 pass
84 def addclass(self, cl):
85 self.classes[cl.classname] = cl
86 if self.tables.find(name=cl.classname) < 0:
87 self.tables.append(name=cl.classname)
88 def addjournal(self, tablenm, nodeid, action, params):
89 tblid = self.tables.find(name=tablenm)
90 if tblid == -1:
91 tblid = self.tables.append(name=tablenm)
92 # tableid:I,nodeid:I,date:I,user:I,action:I,params:B
93 self.hist.append(tableid=tblid,
94 nodeid=int(nodeid),
95 date=int(time.time()),
96 action=action,
97 user = self.curuserid,
98 params = marshal.dumps(params))
99 def gethistory(self, tablenm, nodeid):
100 rslt = []
101 tblid = self.tables.find(name=tablenm)
102 if tblid == -1:
103 return rslt
104 q = self.hist.select(tableid=tblid, nodeid=int(nodeid))
105 i = 0
106 userclass = self.getclass('user')
107 for row in q:
108 try:
109 params = marshal.loads(row.params)
110 except ValueError:
111 print "history couldn't unmarshal %r" % row.params
112 params = {}
113 usernm = userclass.get(str(row.user), 'username')
114 dt = date.Date(time.gmtime(row.date))
115 rslt.append((i, dt, usernm, _actionnames[row.action], params))
116 i += 1
117 return rslt
119 def close(self):
120 for cl in self.classes.values():
121 cl.db = None
122 self._db = None
123 locking.release_lock(self.lockfile)
124 del _dbs[self.config.DATABASE]
125 self.lockfile.close()
126 self.classes = {}
127 self.indexer = None
129 # --- internal
130 def __open(self):
131 self.dbnm = db = os.path.join(self.config.DATABASE, 'tracker.mk4')
132 lockfilenm = db[:-3]+'lck'
133 self.lockfile = locking.acquire_lock(lockfilenm)
134 self.lockfile.write(str(os.getpid()))
135 self.lockfile.flush()
136 self.fastopen = 0
137 if os.path.exists(db):
138 dbtm = os.path.getmtime(db)
139 pkgnm = self.config.__name__.split('.')[0]
140 schemamod = sys.modules.get(pkgnm+'.dbinit', None)
141 if schemamod:
142 if os.path.exists(schemamod.__file__):
143 schematm = os.path.getmtime(schemamod.__file__)
144 if schematm < dbtm:
145 # found schema mod - it's older than the db
146 self.fastopen = 1
147 else:
148 # can't find schemamod - must be frozen
149 self.fastopen = 1
150 db = metakit.storage(db, 1)
151 hist = db.view('history')
152 tables = db.view('tables')
153 if not self.fastopen:
154 if not hist.structure():
155 hist = db.getas('history[tableid:I,nodeid:I,date:I,user:I,action:I,params:B]')
156 if not tables.structure():
157 tables = db.getas('tables[name:S]')
158 self.tables = tables
159 self.hist = hist
160 return db
162 _STRINGTYPE = type('')
163 _LISTTYPE = type([])
164 _CREATE, _SET, _RETIRE, _LINK, _UNLINK = range(5)
166 _actionnames = {
167 _CREATE : 'create',
168 _SET : 'set',
169 _RETIRE : 'retire',
170 _LINK : 'link',
171 _UNLINK : 'unlink',
172 }
174 _marker = []
176 _ALLOWSETTINGPRIVATEPROPS = 0
178 class Class:
179 privateprops = None
180 def __init__(self, db, classname, **properties):
181 #self.db = weakref.proxy(db)
182 self.db = db
183 self.classname = classname
184 self.keyname = None
185 self.ruprops = properties
186 self.privateprops = { 'id' : hyperdb.String(),
187 'activity' : hyperdb.Date(),
188 'creation' : hyperdb.Date(),
189 'creator' : hyperdb.Link('user') }
190 self.auditors = {'create': [], 'set': [], 'retire': []} # event -> list of callables
191 self.reactors = {'create': [], 'set': [], 'retire': []} # ditto
192 view = self.__getview()
193 self.maxid = 1
194 if view:
195 self.maxid = view[-1].id + 1
196 self.uncommitted = {}
197 self.rbactions = []
198 # people reach inside!!
199 self.properties = self.ruprops
200 self.db.addclass(self)
201 self.idcache = {}
203 # default is to journal changes
204 self.do_journal = 1
206 def enableJournalling(self):
207 '''Turn journalling on for this class
208 '''
209 self.do_journal = 1
211 def disableJournalling(self):
212 '''Turn journalling off for this class
213 '''
214 self.do_journal = 0
216 # --- the roundup.Class methods
217 def audit(self, event, detector):
218 l = self.auditors[event]
219 if detector not in l:
220 self.auditors[event].append(detector)
221 def fireAuditors(self, action, nodeid, newvalues):
222 for audit in self.auditors[action]:
223 audit(self.db, self, nodeid, newvalues)
224 def fireReactors(self, action, nodeid, oldvalues):
225 for react in self.reactors[action]:
226 react(self.db, self, nodeid, oldvalues)
227 def react(self, event, detector):
228 l = self.reactors[event]
229 if detector not in l:
230 self.reactors[event].append(detector)
231 # --- the hyperdb.Class methods
232 def create(self, **propvalues):
233 self.fireAuditors('create', None, propvalues)
234 rowdict = {}
235 rowdict['id'] = newid = self.maxid
236 self.maxid += 1
237 ndx = self.getview(1).append(rowdict)
238 propvalues['#ISNEW'] = 1
239 try:
240 self.set(str(newid), **propvalues)
241 except Exception:
242 self.maxid -= 1
243 raise
244 return str(newid)
246 def get(self, nodeid, propname, default=_marker, cache=1):
247 # default and cache aren't in the spec
248 # cache=0 means "original value"
250 view = self.getview()
251 id = int(nodeid)
252 if cache == 0:
253 oldnode = self.uncommitted.get(id, None)
254 if oldnode and oldnode.has_key(propname):
255 return oldnode[propname]
256 ndx = self.idcache.get(id, None)
257 if ndx is None:
258 ndx = view.find(id=id)
259 if ndx < 0:
260 raise IndexError, "%s has no node %s" % (self.classname, nodeid)
261 self.idcache[id] = ndx
262 try:
263 raw = getattr(view[ndx], propname)
264 except AttributeError:
265 raise KeyError, propname
266 rutyp = self.ruprops.get(propname, None)
267 if rutyp is None:
268 rutyp = self.privateprops[propname]
269 converter = _converters.get(rutyp.__class__, None)
270 if converter:
271 raw = converter(raw)
272 return raw
274 def set(self, nodeid, **propvalues):
275 isnew = 0
276 if propvalues.has_key('#ISNEW'):
277 isnew = 1
278 del propvalues['#ISNEW']
279 if not isnew:
280 self.fireAuditors('set', nodeid, propvalues)
281 if not propvalues:
282 return propvalues
283 if propvalues.has_key('id'):
284 raise KeyError, '"id" is reserved'
285 if self.db.journaltag is None:
286 raise DatabaseError, 'Database open read-only'
287 view = self.getview(1)
288 # node must exist & not be retired
289 id = int(nodeid)
290 ndx = view.find(id=id)
291 if ndx < 0:
292 raise IndexError, "%s has no node %s" % (self.classname, nodeid)
293 row = view[ndx]
294 if row._isdel:
295 raise IndexError, "%s has no node %s" % (self.classname, nodeid)
296 oldnode = self.uncommitted.setdefault(id, {})
297 changes = {}
299 for key, value in propvalues.items():
300 # this will raise the KeyError if the property isn't valid
301 # ... we don't use getprops() here because we only care about
302 # the writeable properties.
303 if _ALLOWSETTINGPRIVATEPROPS:
304 prop = self.ruprops.get(key, None)
305 if not prop:
306 prop = self.privateprops[key]
307 else:
308 prop = self.ruprops[key]
309 converter = _converters.get(prop.__class__, lambda v: v)
310 # if the value's the same as the existing value, no sense in
311 # doing anything
312 oldvalue = converter(getattr(row, key))
313 if value == oldvalue:
314 del propvalues[key]
315 continue
317 # check to make sure we're not duplicating an existing key
318 if key == self.keyname:
319 iv = self.getindexview(1)
320 ndx = iv.find(k=value)
321 if ndx == -1:
322 iv.append(k=value, i=row.id)
323 if not isnew:
324 ndx = iv.find(k=oldvalue)
325 if ndx > -1:
326 iv.delete(ndx)
327 else:
328 raise ValueError, 'node with key "%s" exists'%value
330 # do stuff based on the prop type
331 if isinstance(prop, hyperdb.Link):
332 link_class = prop.classname
333 # must be a string or None
334 if value is not None and not isinstance(value, type('')):
335 raise ValueError, 'property "%s" link value be a string'%(
336 propname)
337 # Roundup sets to "unselected" by passing None
338 if value is None:
339 value = 0
340 # if it isn't a number, it's a key
341 try:
342 int(value)
343 except ValueError:
344 try:
345 value = self.db.getclass(link_class).lookup(value)
346 except (TypeError, KeyError):
347 raise IndexError, 'new property "%s": %s not a %s'%(
348 key, value, prop.classname)
350 if (value is not None and
351 not self.db.getclass(link_class).hasnode(value)):
352 raise IndexError, '%s has no node %s'%(link_class, value)
354 setattr(row, key, int(value))
355 changes[key] = oldvalue
357 if self.do_journal and prop.do_journal:
358 # register the unlink with the old linked node
359 if oldvalue:
360 self.db.addjournal(link_class, value, _UNLINK,
361 (self.classname, str(row.id), key))
363 # register the link with the newly linked node
364 if value:
365 self.db.addjournal(link_class, value, _LINK,
366 (self.classname, str(row.id), key))
368 elif isinstance(prop, hyperdb.Multilink):
369 if type(value) != _LISTTYPE:
370 raise TypeError, 'new property "%s" not a list of ids'%key
371 link_class = prop.classname
372 l = []
373 for entry in value:
374 if type(entry) != _STRINGTYPE:
375 raise ValueError, 'new property "%s" link value ' \
376 'must be a string'%key
377 # if it isn't a number, it's a key
378 try:
379 int(entry)
380 except ValueError:
381 try:
382 entry = self.db.getclass(link_class).lookup(entry)
383 except (TypeError, KeyError):
384 raise IndexError, 'new property "%s": %s not a %s'%(
385 key, entry, prop.classname)
386 l.append(entry)
387 propvalues[key] = value = l
389 # handle removals
390 rmvd = []
391 for id in oldvalue:
392 if id not in value:
393 rmvd.append(id)
394 # register the unlink with the old linked node
395 if self.do_journal and prop.do_journal:
396 self.db.addjournal(link_class, id, _UNLINK, (self.classname, str(row.id), key))
398 # handle additions
399 adds = []
400 for id in value:
401 if id not in oldvalue:
402 if not self.db.getclass(link_class).hasnode(id):
403 raise IndexError, '%s has no node %s'%(
404 link_class, id)
405 adds.append(id)
406 # register the link with the newly linked node
407 if self.do_journal and prop.do_journal:
408 self.db.addjournal(link_class, id, _LINK, (self.classname, str(row.id), key))
410 sv = getattr(row, key)
411 i = 0
412 while i < len(sv):
413 if str(sv[i].fid) in rmvd:
414 sv.delete(i)
415 else:
416 i += 1
417 for id in adds:
418 sv.append(fid=int(id))
419 changes[key] = oldvalue
420 if not rmvd and not adds:
421 del propvalues[key]
424 elif isinstance(prop, hyperdb.String):
425 if value is not None and type(value) != _STRINGTYPE:
426 raise TypeError, 'new property "%s" not a string'%key
427 setattr(row, key, value)
428 changes[key] = oldvalue
429 if hasattr(prop, 'isfilename') and prop.isfilename:
430 propvalues[key] = os.path.basename(value)
431 if prop.indexme:
432 self.db.indexer.add_text((self.classname, nodeid, key), value, 'text/plain')
434 elif isinstance(prop, hyperdb.Password):
435 if not isinstance(value, password.Password):
436 raise TypeError, 'new property "%s" not a Password'% key
437 setattr(row, key, str(value))
438 changes[key] = str(oldvalue)
439 propvalues[key] = str(value)
441 elif value is not None and isinstance(prop, hyperdb.Date):
442 if not isinstance(value, date.Date):
443 raise TypeError, 'new property "%s" not a Date'% key
444 setattr(row, key, int(calendar.timegm(value.get_tuple())))
445 changes[key] = str(oldvalue)
446 propvalues[key] = str(value)
448 elif value is not None and isinstance(prop, hyperdb.Interval):
449 if not isinstance(value, date.Interval):
450 raise TypeError, 'new property "%s" not an Interval'% key
451 setattr(row, key, str(value))
452 changes[key] = str(oldvalue)
453 propvalues[key] = str(value)
455 elif value is not None and isinstance(prop, hyperdb.Number):
456 setattr(row, key, int(value))
457 changes[key] = oldvalue
458 propvalues[key] = value
460 elif value is not None and isinstance(prop, hyperdb.Boolean):
461 bv = value != 0
462 setattr(row, key, bv)
463 changes[key] = oldvalue
464 propvalues[key] = value
466 oldnode[key] = oldvalue
468 # nothing to do?
469 if not propvalues:
470 return propvalues
471 if not propvalues.has_key('activity'):
472 row.activity = int(time.time())
473 if isnew:
474 if not row.creation:
475 row.creation = int(time.time())
476 if not row.creator:
477 row.creator = self.db.curuserid
479 self.db.dirty = 1
480 if self.do_journal:
481 if isnew:
482 self.db.addjournal(self.classname, nodeid, _CREATE, {})
483 self.fireReactors('create', nodeid, None)
484 else:
485 self.db.addjournal(self.classname, nodeid, _SET, changes)
486 self.fireReactors('set', nodeid, oldnode)
488 return propvalues
490 def retire(self, nodeid):
491 self.fireAuditors('retire', nodeid, None)
492 view = self.getview(1)
493 ndx = view.find(id=int(nodeid))
494 if ndx < 0:
495 raise KeyError, "nodeid %s not found" % nodeid
496 row = view[ndx]
497 oldvalues = self.uncommitted.setdefault(row.id, {})
498 oldval = oldvalues['_isdel'] = row._isdel
499 row._isdel = 1
500 if self.do_journal:
501 self.db.addjournal(self.classname, nodeid, _RETIRE, {})
502 if self.keyname:
503 iv = self.getindexview(1)
504 ndx = iv.find(k=getattr(row, self.keyname),i=row.id)
505 if ndx > -1:
506 iv.delete(ndx)
507 self.db.dirty = 1
508 self.fireReactors('retire', nodeid, None)
509 def history(self, nodeid):
510 if not self.do_journal:
511 raise ValueError, 'Journalling is disabled for this class'
512 return self.db.gethistory(self.classname, nodeid)
513 def setkey(self, propname):
514 if self.keyname:
515 if propname == self.keyname:
516 return
517 raise ValueError, "%s already indexed on %s" % (self.classname, self.keyname)
518 # first setkey for this run
519 self.keyname = propname
520 iv = self.db._db.view('_%s' % self.classname)
521 if self.db.fastopen and iv.structure():
522 return
523 # very first setkey ever
524 self.db.dirty = 1
525 iv = self.db._db.getas('_%s[k:S,i:I]' % self.classname)
526 iv = iv.ordered(1)
527 # print "setkey building index"
528 for row in self.getview():
529 iv.append(k=getattr(row, propname), i=row.id)
530 self.db.commit()
531 def getkey(self):
532 return self.keyname
533 def lookup(self, keyvalue):
534 if type(keyvalue) is not _STRINGTYPE:
535 raise TypeError, "%r is not a string" % keyvalue
536 iv = self.getindexview()
537 if iv:
538 ndx = iv.find(k=keyvalue)
539 if ndx > -1:
540 return str(iv[ndx].i)
541 else:
542 view = self.getview()
543 ndx = view.find({self.keyname:keyvalue, '_isdel':0})
544 if ndx > -1:
545 return str(view[ndx].id)
546 raise KeyError, keyvalue
548 def destroy(self, keyvalue):
549 #TODO clean this up once Richard's said how it should work
550 iv = self.getindexview()
551 if iv:
552 ndx = iv.find(k=keyvalue)
553 if ndx > -1:
554 id = iv[ndx].i
555 iv.delete(ndx)
556 view = self.getview()
557 ndx = view.find(id=id)
558 if ndx > -1:
559 view.delete(ndx)
561 def find(self, **propspec):
562 """Get the ids of nodes in this class which link to the given nodes.
564 'propspec' consists of keyword args propname={nodeid:1,}
565 'propname' must be the name of a property in this class, or a
566 KeyError is raised. That property must be a Link or
567 Multilink property, or a TypeError is raised.
569 Any node in this class whose propname property links to any of the
570 nodeids will be returned. Used by the full text indexing, which knows
571 that "foo" occurs in msg1, msg3 and file7; so we have hits on these
572 issues:
574 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
576 """
577 propspec = propspec.items()
578 for propname, nodeid in propspec:
579 # check the prop is OK
580 prop = self.ruprops[propname]
581 if (not isinstance(prop, hyperdb.Link) and
582 not isinstance(prop, hyperdb.Multilink)):
583 raise TypeError, "'%s' not a Link/Multilink property"%propname
585 vws = []
586 for propname, ids in propspec:
587 if type(ids) is _STRINGTYPE:
588 ids = {ids:1}
589 prop = self.ruprops[propname]
590 view = self.getview()
591 if isinstance(prop, hyperdb.Multilink):
592 view = view.flatten(getattr(view, propname))
593 def ff(row, nm=propname, ids=ids):
594 return ids.has_key(str(row.fid))
595 else:
596 def ff(row, nm=propname, ids=ids):
597 return ids.has_key(str(getattr(row, nm)))
598 ndxview = view.filter(ff)
599 vws.append(ndxview.unique())
601 # handle the empty match case
602 if not vws:
603 return []
605 ndxview = vws[0]
606 for v in vws[1:]:
607 ndxview = ndxview.union(v)
608 view = view.remapwith(ndxview)
609 rslt = []
610 for row in view:
611 rslt.append(str(row.id))
612 return rslt
615 def list(self):
616 l = []
617 for row in self.getview().select(_isdel=0):
618 l.append(str(row.id))
619 return l
620 def count(self):
621 return len(self.getview())
622 def getprops(self, protected=1):
623 # protected is not in ping's spec
624 allprops = self.ruprops.copy()
625 if protected and self.privateprops is not None:
626 allprops.update(self.privateprops)
627 return allprops
628 def addprop(self, **properties):
629 for key in properties.keys():
630 if self.ruprops.has_key(key):
631 raise ValueError, "%s is already a property of %s" % (key, self.classname)
632 self.ruprops.update(properties)
633 self.db.fastopen = 0
634 view = self.__getview()
635 self.db.commit()
636 # ---- end of ping's spec
637 def filter(self, search_matches, filterspec, sort, group):
638 # search_matches is None or a set (dict of {nodeid: {propname:[nodeid,...]}})
639 # filterspec is a dict {propname:value}
640 # sort and group are lists of propnames
642 where = {'_isdel':0}
643 mlcriteria = {}
644 regexes = {}
645 orcriteria = {}
646 for propname, value in filterspec.items():
647 prop = self.ruprops.get(propname, None)
648 if prop is None:
649 prop = self.privateprops[propname]
650 if isinstance(prop, hyperdb.Multilink):
651 if type(value) is not _LISTTYPE:
652 value = [value]
653 # transform keys to ids
654 u = []
655 for item in value:
656 try:
657 item = int(item)
658 except (TypeError, ValueError):
659 item = int(self.db.getclass(prop.classname).lookup(item))
660 if item == -1:
661 item = 0
662 u.append(item)
663 mlcriteria[propname] = u
664 elif isinstance(prop, hyperdb.Link):
665 if type(value) is not _LISTTYPE:
666 value = [value]
667 # transform keys to ids
668 u = []
669 for item in value:
670 try:
671 item = int(item)
672 except (TypeError, ValueError):
673 item = int(self.db.getclass(prop.classname).lookup(item))
674 if item == -1:
675 item = 0
676 u.append(item)
677 if len(u) == 1:
678 where[propname] = u[0]
679 else:
680 orcriteria[propname] = u
681 elif isinstance(prop, hyperdb.String):
682 # simple glob searching
683 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', value)
684 v = v.replace('?', '.')
685 v = v.replace('*', '.*?')
686 regexes[propname] = re.compile(v, re.I)
687 elif propname == 'id':
688 where[propname] = int(value)
689 elif isinstance(prop, hyperdb.Boolean):
690 if type(value) is _STRINGTYPE:
691 bv = value.lower() in ('yes', 'true', 'on', '1')
692 else:
693 bv = value
694 where[propname] = bv
695 elif isinstance(prop, hyperdb.Number):
696 where[propname] = int(value)
697 else:
698 where[propname] = str(value)
699 v = self.getview()
700 #print "filter start at %s" % time.time()
701 if where:
702 v = v.select(where)
703 #print "filter where at %s" % time.time()
705 if mlcriteria:
706 # multilink - if any of the nodeids required by the
707 # filterspec aren't in this node's property, then skip
708 # it
709 def ff(row, ml=mlcriteria):
710 for propname, values in ml.items():
711 sv = getattr(row, propname)
712 for id in values:
713 if sv.find(fid=id) == -1:
714 return 0
715 return 1
716 iv = v.filter(ff)
717 v = v.remapwith(iv)
719 #print "filter mlcrit at %s" % time.time()
721 if orcriteria:
722 def ff(row, crit=orcriteria):
723 for propname, allowed in crit.items():
724 val = getattr(row, propname)
725 if val not in allowed:
726 return 0
727 return 1
729 iv = v.filter(ff)
730 v = v.remapwith(iv)
732 #print "filter orcrit at %s" % time.time()
733 if regexes:
734 def ff(row, r=regexes):
735 for propname, regex in r.items():
736 val = getattr(row, propname)
737 if not regex.search(val):
738 return 0
739 return 1
741 iv = v.filter(ff)
742 v = v.remapwith(iv)
743 #print "filter regexs at %s" % time.time()
745 if sort or group:
746 sortspec = []
747 rev = []
748 for propname in group + sort:
749 isreversed = 0
750 if propname[0] == '-':
751 propname = propname[1:]
752 isreversed = 1
753 try:
754 prop = getattr(v, propname)
755 except AttributeError:
756 print "MK has no property %s" % propname
757 continue
758 propclass = self.ruprops.get(propname, None)
759 if propclass is None:
760 propclass = self.privateprops.get(propname, None)
761 if propclass is None:
762 print "Schema has no property %s" % propname
763 continue
764 if isinstance(propclass, hyperdb.Link):
765 linkclass = self.db.getclass(propclass.classname)
766 lv = linkclass.getview()
767 lv = lv.rename('id', propname)
768 v = v.join(lv, prop, 1)
769 if linkclass.getprops().has_key('order'):
770 propname = 'order'
771 else:
772 propname = linkclass.labelprop()
773 prop = getattr(v, propname)
774 if isreversed:
775 rev.append(prop)
776 sortspec.append(prop)
777 v = v.sortrev(sortspec, rev)[:] #XXX Metakit bug
778 #print "filter sort at %s" % time.time()
780 rslt = []
781 for row in v:
782 id = str(row.id)
783 if search_matches is not None:
784 if search_matches.has_key(id):
785 rslt.append(id)
786 else:
787 rslt.append(id)
788 return rslt
790 def hasnode(self, nodeid):
791 return int(nodeid) < self.maxid
793 def labelprop(self, default_to_id=0):
794 ''' Return the property name for a label for the given node.
796 This method attempts to generate a consistent label for the node.
797 It tries the following in order:
798 1. key property
799 2. "name" property
800 3. "title" property
801 4. first property from the sorted property name list
802 '''
803 k = self.getkey()
804 if k:
805 return k
806 props = self.getprops()
807 if props.has_key('name'):
808 return 'name'
809 elif props.has_key('title'):
810 return 'title'
811 if default_to_id:
812 return 'id'
813 props = props.keys()
814 props.sort()
815 return props[0]
816 def stringFind(self, **requirements):
817 """Locate a particular node by matching a set of its String
818 properties in a caseless search.
820 If the property is not a String property, a TypeError is raised.
822 The return is a list of the id of all nodes that match.
823 """
824 for propname in requirements.keys():
825 prop = self.properties[propname]
826 if isinstance(not prop, hyperdb.String):
827 raise TypeError, "'%s' not a String property"%propname
828 requirements[propname] = requirements[propname].lower()
829 requirements['_isdel'] = 0
831 l = []
832 for row in self.getview().select(requirements):
833 l.append(str(row.id))
834 return l
836 def addjournal(self, nodeid, action, params):
837 self.db.addjournal(self.classname, nodeid, action, params)
839 def index(self, nodeid):
840 ''' Add (or refresh) the node to search indexes '''
841 # find all the String properties that have indexme
842 for prop, propclass in self.getprops().items():
843 if isinstance(propclass, hyperdb.String) and propclass.indexme:
844 # index them under (classname, nodeid, property)
845 self.db.indexer.add_text((self.classname, nodeid, prop),
846 str(self.get(nodeid, prop)))
848 # --- used by Database
849 def _commit(self):
850 """ called post commit of the DB.
851 interested subclasses may override """
852 self.uncommitted = {}
853 self.rbactions = []
854 self.idcache = {}
855 def _rollback(self):
856 """ called pre rollback of the DB.
857 interested subclasses may override """
858 for action in self.rbactions:
859 action()
860 self.rbactions = []
861 self.uncommitted = {}
862 self.idcache = {}
863 def _clear(self):
864 view = self.getview(1)
865 if len(view):
866 view[:] = []
867 self.db.dirty = 1
868 iv = self.getindexview(1)
869 if iv:
870 iv[:] = []
871 def rollbackaction(self, action):
872 """ call this to register a callback called on rollback
873 callback is removed on end of transaction """
874 self.rbactions.append(action)
875 # --- internal
876 def __getview(self):
877 db = self.db._db
878 view = db.view(self.classname)
879 mkprops = view.structure()
880 if mkprops and self.db.fastopen:
881 return view.ordered(1)
882 # is the definition the same?
883 for nm, rutyp in self.ruprops.items():
884 for mkprop in mkprops:
885 if mkprop.name == nm:
886 break
887 else:
888 mkprop = None
889 if mkprop is None:
890 break
891 if _typmap[rutyp.__class__] != mkprop.type:
892 break
893 else:
894 return view.ordered(1)
895 # need to create or restructure the mk view
896 # id comes first, so MK will order it for us
897 self.db.dirty = 1
898 s = ["%s[id:I" % self.classname]
899 for nm, rutyp in self.ruprops.items():
900 mktyp = _typmap[rutyp.__class__]
901 s.append('%s:%s' % (nm, mktyp))
902 if mktyp == 'V':
903 s[-1] += ('[fid:I]')
904 s.append('_isdel:I,activity:I,creation:I,creator:I]')
905 v = self.db._db.getas(','.join(s))
906 self.db.commit()
907 return v.ordered(1)
908 def getview(self, RW=0):
909 return self.db._db.view(self.classname).ordered(1)
910 def getindexview(self, RW=0):
911 return self.db._db.view("_%s" % self.classname).ordered(1)
913 def _fetchML(sv):
914 l = []
915 for row in sv:
916 if row.fid:
917 l.append(str(row.fid))
918 return l
920 def _fetchPW(s):
921 p = password.Password()
922 p.unpack(s)
923 return p
925 def _fetchLink(n):
926 return n and str(n) or None
928 def _fetchDate(n):
929 return date.Date(time.gmtime(n))
931 _converters = {
932 hyperdb.Date : _fetchDate,
933 hyperdb.Link : _fetchLink,
934 hyperdb.Multilink : _fetchML,
935 hyperdb.Interval : date.Interval,
936 hyperdb.Password : _fetchPW,
937 hyperdb.Boolean : lambda n: n,
938 hyperdb.Number : lambda n: n,
939 hyperdb.String : str,
940 }
942 class FileName(hyperdb.String):
943 isfilename = 1
945 _typmap = {
946 FileName : 'S',
947 hyperdb.String : 'S',
948 hyperdb.Date : 'I',
949 hyperdb.Link : 'I',
950 hyperdb.Multilink : 'V',
951 hyperdb.Interval : 'S',
952 hyperdb.Password : 'S',
953 hyperdb.Boolean : 'I',
954 hyperdb.Number : 'I',
955 }
956 class FileClass(Class):
957 ' like Class but with a content property '
958 default_mime_type = 'text/plain'
959 def __init__(self, db, classname, **properties):
960 properties['content'] = FileName()
961 if not properties.has_key('type'):
962 properties['type'] = hyperdb.String()
963 Class.__init__(self, db, classname, **properties)
964 def get(self, nodeid, propname, default=_marker, cache=1):
965 x = Class.get(self, nodeid, propname, default, cache)
966 if propname == 'content':
967 if x.startswith('file:'):
968 fnm = x[5:]
969 try:
970 x = open(fnm, 'rb').read()
971 except Exception, e:
972 x = repr(e)
973 return x
974 def create(self, **propvalues):
975 content = propvalues['content']
976 del propvalues['content']
977 newid = Class.create(self, **propvalues)
978 if not content:
979 return newid
980 nm = bnm = '%s%s' % (self.classname, newid)
981 sd = str(int(int(newid) / 1000))
982 d = os.path.join(self.db.config.DATABASE, 'files', self.classname, sd)
983 if not os.path.exists(d):
984 os.makedirs(d)
985 nm = os.path.join(d, nm)
986 open(nm, 'wb').write(content)
987 self.set(newid, content = 'file:'+nm)
988 mimetype = propvalues.get('type', self.default_mime_type)
989 self.db.indexer.add_text((self.classname, newid, 'content'), content, mimetype)
990 def undo(fnm=nm, action1=os.remove, indexer=self.db.indexer):
991 action1(fnm)
992 self.rollbackaction(undo)
993 return newid
994 def index(self, nodeid):
995 Class.index(self, nodeid)
996 mimetype = self.get(nodeid, 'type')
997 if not mimetype:
998 mimetype = self.default_mime_type
999 self.db.indexer.add_text((self.classname, nodeid, 'content'),
1000 self.get(nodeid, 'content'), mimetype)
1002 class IssueClass(Class, roundupdb.IssueClass):
1003 # Overridden methods:
1004 def __init__(self, db, classname, **properties):
1005 """The newly-created class automatically includes the "messages",
1006 "files", "nosy", and "superseder" properties. If the 'properties'
1007 dictionary attempts to specify any of these properties or a
1008 "creation" or "activity" property, a ValueError is raised."""
1009 if not properties.has_key('title'):
1010 properties['title'] = hyperdb.String(indexme='yes')
1011 if not properties.has_key('messages'):
1012 properties['messages'] = hyperdb.Multilink("msg")
1013 if not properties.has_key('files'):
1014 properties['files'] = hyperdb.Multilink("file")
1015 if not properties.has_key('nosy'):
1016 properties['nosy'] = hyperdb.Multilink("user")
1017 if not properties.has_key('superseder'):
1018 properties['superseder'] = hyperdb.Multilink(classname)
1019 Class.__init__(self, db, classname, **properties)
1021 CURVERSION = 1
1023 class Indexer(indexer.Indexer):
1024 disallows = {'THE':1, 'THIS':1, 'ZZZ':1, 'THAT':1, 'WITH':1}
1025 def __init__(self, path, datadb):
1026 self.db = metakit.storage(os.path.join(path, 'index.mk4'), 1)
1027 self.datadb = datadb
1028 self.reindex = 0
1029 v = self.db.view('version')
1030 if not v.structure():
1031 v = self.db.getas('version[vers:I]')
1032 self.db.commit()
1033 v.append(vers=CURVERSION)
1034 self.reindex = 1
1035 elif v[0].vers != CURVERSION:
1036 v[0].vers = CURVERSION
1037 self.reindex = 1
1038 if self.reindex:
1039 self.db.getas('ids[tblid:I,nodeid:I,propid:I]')
1040 self.db.getas('index[word:S,hits[pos:I]]')
1041 self.db.commit()
1042 self.reindex = 1
1043 self.changed = 0
1044 self.propcache = {}
1045 def force_reindex(self):
1046 v = self.db.view('ids')
1047 v[:] = []
1048 v = self.db.view('index')
1049 v[:] = []
1050 self.db.commit()
1051 self.reindex = 1
1052 def should_reindex(self):
1053 return self.reindex
1054 def _getprops(self, classname):
1055 props = self.propcache.get(classname, None)
1056 if props is None:
1057 props = self.datadb.view(classname).structure()
1058 props = [prop.name for prop in props]
1059 self.propcache[classname] = props
1060 return props
1061 def _getpropid(self, classname, propname):
1062 return self._getprops(classname).index(propname)
1063 def _getpropname(self, classname, propid):
1064 return self._getprops(classname)[propid]
1065 def add_text(self, identifier, text, mime_type='text/plain'):
1066 if mime_type != 'text/plain':
1067 return
1068 classname, nodeid, property = identifier
1069 tbls = self.datadb.view('tables')
1070 tblid = tbls.find(name=classname)
1071 if tblid < 0:
1072 raise KeyError, "unknown class %r"%classname
1073 nodeid = int(nodeid)
1074 propid = self._getpropid(classname, property)
1075 pos = self.db.view('ids').append(tblid=tblid,nodeid=nodeid,propid=propid)
1077 wordlist = re.findall(r'\b\w{3,25}\b', text)
1078 words = {}
1079 for word in wordlist:
1080 word = word.upper()
1081 if not self.disallows.has_key(word):
1082 words[word] = 1
1083 words = words.keys()
1085 index = self.db.view('index').ordered(1)
1086 for word in words:
1087 ndx = index.find(word=word)
1088 if ndx < 0:
1089 ndx = index.append(word=word)
1090 hits = index[ndx].hits
1091 if len(hits)==0 or hits.find(pos=pos) < 0:
1092 hits.append(pos=pos)
1093 self.changed = 1
1094 def find(self, wordlist):
1095 hits = None
1096 index = self.db.view('index').ordered(1)
1097 for word in wordlist:
1098 if not 2 < len(word) < 26:
1099 continue
1100 ndx = index.find(word=word)
1101 if ndx < 0:
1102 return {}
1103 if hits is None:
1104 hits = index[ndx].hits
1105 else:
1106 hits = hits.intersect(index[ndx].hits)
1107 if len(hits) == 0:
1108 return {}
1109 if hits is None:
1110 return {}
1111 rslt = {}
1112 ids = self.db.view('ids').remapwith(hits)
1113 tbls = self.datadb.view('tables')
1114 for i in range(len(ids)):
1115 hit = ids[i]
1116 classname = tbls[hit.tblid].name
1117 nodeid = str(hit.nodeid)
1118 property = self._getpropname(classname, hit.propid)
1119 rslt[i] = (classname, nodeid, property)
1120 return rslt
1121 def save_index(self):
1122 if self.changed:
1123 self.db.commit()
1124 self.changed = 0