1 # $Id: back_metakit.py,v 1.66 2004-03-21 23:39:08 richard Exp $
2 '''Metakit backend for Roundup, originally by Gordon McMillan.
4 Known Current Bugs:
6 - You can't change a class' key properly. This shouldn't be too hard to fix.
7 - Some unit tests are overridden.
9 Notes by Richard:
11 This backend has some behaviour specific to metakit:
13 - there's no concept of an explicit "unset" in metakit, so all types
14 have some "unset" value:
16 ========= ===== ======================================================
17 Type Value Action when fetching from mk
18 ========= ===== ======================================================
19 Strings '' convert to None
20 Date 0 (seconds since 1970-01-01.00:00:00) convert to None
21 Interval '' convert to None
22 Number 0 ambiguious :( - do nothing (see BACKWARDS_COMPATIBLE)
23 Boolean 0 ambiguious :( - do nothing (see BACKWARDS_COMPATABILE)
24 Link 0 convert to None
25 Multilink [] actually, mk can handle this one ;)
26 Password '' convert to None
27 ========= ===== ======================================================
29 The get/set routines handle these values accordingly by converting
30 to/from None where they can. The Number/Boolean types are not able
31 to handle an "unset" at all, so they default the "unset" to 0.
32 - Metakit relies in reference counting to close the database, there is
33 no explicit close call. This can cause issues if a metakit
34 database is referenced multiple times, one might not actually be
35 closing the db.
36 - probably a bunch of stuff that I'm not aware of yet because I haven't
37 fully read through the source. One of these days....
38 '''
39 __docformat__ = 'restructuredtext'
40 # Enable this flag to break backwards compatibility (i.e. can't read old
41 # databases) but comply with more roundup features, like adding NULL support.
42 BACKWARDS_COMPATIBLE = True
44 from roundup import hyperdb, date, password, roundupdb, security
45 import metakit
46 from sessions_dbm import Sessions, OneTimeKeys
47 import re, marshal, os, sys, time, calendar
48 from indexer_dbm import Indexer
49 import locking
50 from roundup.date import Range
52 # view modes for opening
53 # XXX FIXME BPK -> these don't do anything, they are ignored
54 # should we just get rid of them for simplicities sake?
55 READ = 0
56 READWRITE = 1
58 # general metakit error
59 class MKBackendError(Exception):
60 pass
62 _dbs = {}
64 def Database(config, journaltag=None):
65 ''' Only have a single instance of the Database class for each instance
66 '''
67 db = _dbs.get(config.DATABASE, None)
68 if db is None or db._db is None:
69 db = _Database(config, journaltag)
70 _dbs[config.DATABASE] = db
71 else:
72 db.journaltag = journaltag
73 return db
75 class _Database(hyperdb.Database, roundupdb.Database):
76 def __init__(self, config, journaltag=None):
77 self.config = config
78 self.journaltag = journaltag
79 self.classes = {}
80 self.dirty = 0
81 self.lockfile = None
82 self._db = self.__open()
83 self.indexer = Indexer(self.config.DATABASE, self._db)
84 self.security = security.Security(self)
86 os.umask(0002)
88 def post_init(self):
89 if self.indexer.should_reindex():
90 self.reindex()
92 def refresh_database(self):
93 # XXX handle refresh
94 self.reindex()
96 def reindex(self):
97 for klass in self.classes.values():
98 for nodeid in klass.list():
99 klass.index(nodeid)
100 self.indexer.save_index()
102 def getSessionManager(self):
103 return Sessions(self)
105 def getOTKManager(self):
106 return OneTimeKeys(self)
108 # --- defined in ping's spec
109 def __getattr__(self, classname):
110 if classname == 'transactions':
111 return self.dirty
112 # fall back on the classes
113 try:
114 return self.getclass(classname)
115 except KeyError, msg:
116 # KeyError's not appropriate here
117 raise AttributeError, str(msg)
118 def getclass(self, classname):
119 try:
120 return self.classes[classname]
121 except KeyError:
122 raise KeyError, 'There is no class called "%s"'%classname
123 def getclasses(self):
124 return self.classes.keys()
125 # --- end of ping's spec
127 # --- exposed methods
128 def commit(self):
129 '''commit all changes to the database'''
130 if self.dirty:
131 self._db.commit()
132 for cl in self.classes.values():
133 cl._commit()
134 self.indexer.save_index()
135 self.dirty = 0
136 def rollback(self):
137 '''roll back all changes since the last commit'''
138 if self.dirty:
139 for cl in self.classes.values():
140 cl._rollback()
141 self._db.rollback()
142 self._db = None
143 self._db = metakit.storage(self.dbnm, 1)
144 self.hist = self._db.view('history')
145 self.tables = self._db.view('tables')
146 self.indexer.rollback()
147 self.indexer.datadb = self._db
148 self.dirty = 0
149 def clearCache(self):
150 '''clear the internal cache by committing all pending database changes'''
151 for cl in self.classes.values():
152 cl._commit()
153 def clear(self):
154 '''clear the internal cache but don't commit any changes'''
155 for cl in self.classes.values():
156 cl._clear()
157 def hasnode(self, classname, nodeid):
158 '''does a particular class contain a nodeid?'''
159 return self.getclass(classname).hasnode(nodeid)
160 def pack(self, pack_before):
161 ''' Delete all journal entries except "create" before 'pack_before'.
162 '''
163 mindate = int(calendar.timegm(pack_before.get_tuple()))
164 i = 0
165 while i < len(self.hist):
166 if self.hist[i].date < mindate and self.hist[i].action != _CREATE:
167 self.hist.delete(i)
168 else:
169 i = i + 1
170 def addclass(self, cl):
171 ''' Add a Class to the hyperdatabase.
172 '''
173 self.classes[cl.classname] = cl
174 if self.tables.find(name=cl.classname) < 0:
175 self.tables.append(name=cl.classname)
177 # add default Edit and View permissions
178 self.security.addPermission(name="Edit", klass=cl.classname,
179 description="User is allowed to edit "+cl.classname)
180 self.security.addPermission(name="View", klass=cl.classname,
181 description="User is allowed to access "+cl.classname)
183 def addjournal(self, tablenm, nodeid, action, params, creator=None,
184 creation=None):
185 ''' Journal the Action
186 'action' may be:
188 'create' or 'set' -- 'params' is a dictionary of property values
189 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
190 'retire' -- 'params' is None
191 '''
192 tblid = self.tables.find(name=tablenm)
193 if tblid == -1:
194 tblid = self.tables.append(name=tablenm)
195 if creator is None:
196 creator = int(self.getuid())
197 else:
198 try:
199 creator = int(creator)
200 except TypeError:
201 creator = int(self.getclass('user').lookup(creator))
202 if creation is None:
203 creation = int(time.time())
204 elif isinstance(creation, date.Date):
205 creation = int(calendar.timegm(creation.get_tuple()))
206 # tableid:I,nodeid:I,date:I,user:I,action:I,params:B
207 self.hist.append(tableid=tblid,
208 nodeid=int(nodeid),
209 date=creation,
210 action=action,
211 user = creator,
212 params = marshal.dumps(params))
213 def getjournal(self, tablenm, nodeid):
214 ''' get the journal for id
215 '''
216 rslt = []
217 tblid = self.tables.find(name=tablenm)
218 if tblid == -1:
219 return rslt
220 q = self.hist.select(tableid=tblid, nodeid=int(nodeid))
221 if len(q) == 0:
222 raise IndexError, "no history for id %s in %s" % (nodeid, tablenm)
223 i = 0
224 #userclass = self.getclass('user')
225 for row in q:
226 try:
227 params = marshal.loads(row.params)
228 except ValueError:
229 print "history couldn't unmarshal %r" % row.params
230 params = {}
231 #usernm = userclass.get(str(row.user), 'username')
232 dt = date.Date(time.gmtime(row.date))
233 #rslt.append((nodeid, dt, usernm, _actionnames[row.action], params))
234 rslt.append((nodeid, dt, str(row.user), _actionnames[row.action],
235 params))
236 return rslt
238 def destroyjournal(self, tablenm, nodeid):
239 nodeid = int(nodeid)
240 tblid = self.tables.find(name=tablenm)
241 if tblid == -1:
242 return
243 i = 0
244 hist = self.hist
245 while i < len(hist):
246 if hist[i].tableid == tblid and hist[i].nodeid == nodeid:
247 hist.delete(i)
248 else:
249 i = i + 1
250 self.dirty = 1
252 def close(self):
253 ''' Close off the connection.
254 '''
255 # de-reference count the metakit databases,
256 # as this is the only way they will be closed
257 for cl in self.classes.values():
258 cl.db = None
259 self._db = None
260 if self.lockfile is not None:
261 locking.release_lock(self.lockfile)
262 if _dbs.has_key(self.config.DATABASE):
263 del _dbs[self.config.DATABASE]
264 if self.lockfile is not None:
265 self.lockfile.close()
266 self.lockfile = None
267 self.classes = {}
269 # force the indexer to close
270 self.indexer.close()
271 self.indexer = None
273 # --- internal
274 def __open(self):
275 ''' Open the metakit database
276 '''
277 # make the database dir if it doesn't exist
278 if not os.path.exists(self.config.DATABASE):
279 os.makedirs(self.config.DATABASE)
281 # figure the file names
282 self.dbnm = db = os.path.join(self.config.DATABASE, 'tracker.mk4')
283 lockfilenm = db[:-3]+'lck'
285 # get the database lock
286 self.lockfile = locking.acquire_lock(lockfilenm)
287 self.lockfile.write(str(os.getpid()))
288 self.lockfile.flush()
290 # see if the schema has changed since last db access
291 self.fastopen = 0
292 if os.path.exists(db):
293 dbtm = os.path.getmtime(db)
294 pkgnm = self.config.__name__.split('.')[0]
295 schemamod = sys.modules.get(pkgnm+'.dbinit', None)
296 if schemamod:
297 if os.path.exists(schemamod.__file__):
298 schematm = os.path.getmtime(schemamod.__file__)
299 if schematm < dbtm:
300 # found schema mod - it's older than the db
301 self.fastopen = 1
302 else:
303 # can't find schemamod - must be frozen
304 self.fastopen = 1
306 # open the db
307 db = metakit.storage(db, 1)
308 hist = db.view('history')
309 tables = db.view('tables')
310 if not self.fastopen:
311 # create the database if it's brand new
312 if not hist.structure():
313 hist = db.getas('history[tableid:I,nodeid:I,date:I,user:I,action:I,params:B]')
314 if not tables.structure():
315 tables = db.getas('tables[name:S]')
316 db.commit()
318 # we now have an open, initialised database
319 self.tables = tables
320 self.hist = hist
321 return db
323 def setid(self, classname, maxid):
324 ''' No-op in metakit
325 '''
326 pass
328 _STRINGTYPE = type('')
329 _LISTTYPE = type([])
330 _CREATE, _SET, _RETIRE, _LINK, _UNLINK, _RESTORE = range(6)
332 _actionnames = {
333 _CREATE : 'create',
334 _SET : 'set',
335 _RETIRE : 'retire',
336 _RESTORE : 'restore',
337 _LINK : 'link',
338 _UNLINK : 'unlink',
339 }
341 _marker = []
343 _ALLOWSETTINGPRIVATEPROPS = 0
345 class Class(hyperdb.Class):
346 ''' The handle to a particular class of nodes in a hyperdatabase.
348 All methods except __repr__ and getnode must be implemented by a
349 concrete backend Class of which this is one.
350 '''
352 privateprops = None
353 def __init__(self, db, classname, **properties):
354 if (properties.has_key('creation') or properties.has_key('activity')
355 or properties.has_key('creator') or properties.has_key('actor')):
356 raise ValueError, '"creation", "activity" and "creator" are '\
357 'reserved'
358 if hasattr(db, classname):
359 raise ValueError, "Class %s already exists"%classname
361 self.db = db
362 self.classname = classname
363 self.key = None
364 self.ruprops = properties
365 self.privateprops = { 'id' : hyperdb.String(),
366 'activity' : hyperdb.Date(),
367 'actor' : hyperdb.Link('user'),
368 'creation' : hyperdb.Date(),
369 'creator' : hyperdb.Link('user') }
371 # event -> list of callables
372 self.auditors = {'create': [], 'set': [], 'retire': [], 'restore': []}
373 self.reactors = {'create': [], 'set': [], 'retire': [], 'restore': []}
375 view = self.__getview()
376 self.maxid = 1
377 if view:
378 self.maxid = view[-1].id + 1
379 self.uncommitted = {}
380 self.rbactions = []
382 # people reach inside!!
383 self.properties = self.ruprops
384 self.db.addclass(self)
385 self.idcache = {}
387 # default is to journal changes
388 self.do_journal = 1
390 def enableJournalling(self):
391 '''Turn journalling on for this class
392 '''
393 self.do_journal = 1
395 def disableJournalling(self):
396 '''Turn journalling off for this class
397 '''
398 self.do_journal = 0
400 #
401 # Detector/reactor interface
402 #
403 def audit(self, event, detector):
404 '''Register a detector
405 '''
406 l = self.auditors[event]
407 if detector not in l:
408 self.auditors[event].append(detector)
410 def fireAuditors(self, action, nodeid, newvalues):
411 '''Fire all registered auditors.
412 '''
413 for audit in self.auditors[action]:
414 audit(self.db, self, nodeid, newvalues)
416 def react(self, event, detector):
417 '''Register a reactor
418 '''
419 l = self.reactors[event]
420 if detector not in l:
421 self.reactors[event].append(detector)
423 def fireReactors(self, action, nodeid, oldvalues):
424 '''Fire all registered reactors.
425 '''
426 for react in self.reactors[action]:
427 react(self.db, self, nodeid, oldvalues)
429 # --- the hyperdb.Class methods
430 def create(self, **propvalues):
431 ''' Create a new node of this class and return its id.
433 The keyword arguments in 'propvalues' map property names to values.
435 The values of arguments must be acceptable for the types of their
436 corresponding properties or a TypeError is raised.
438 If this class has a key property, it must be present and its value
439 must not collide with other key strings or a ValueError is raised.
441 Any other properties on this class that are missing from the
442 'propvalues' dictionary are set to None.
444 If an id in a link or multilink property does not refer to a valid
445 node, an IndexError is raised.
446 '''
447 if not propvalues:
448 raise ValueError, "Need something to create!"
449 self.fireAuditors('create', None, propvalues)
450 newid = self.create_inner(**propvalues)
451 self.fireReactors('create', newid, None)
452 return newid
454 def create_inner(self, **propvalues):
455 ''' Called by create, in-between the audit and react calls.
456 '''
457 rowdict = {}
458 rowdict['id'] = newid = self.maxid
459 self.maxid += 1
460 ndx = self.getview(READWRITE).append(rowdict)
461 propvalues['#ISNEW'] = 1
462 try:
463 self.set(str(newid), **propvalues)
464 except Exception:
465 self.maxid -= 1
466 raise
467 return str(newid)
469 def get(self, nodeid, propname, default=_marker, cache=1):
470 '''Get the value of a property on an existing node of this class.
472 'nodeid' must be the id of an existing node of this class or an
473 IndexError is raised. 'propname' must be the name of a property
474 of this class or a KeyError is raised.
476 'cache' exists for backwards compatibility, and is not used.
477 '''
478 view = self.getview()
479 id = int(nodeid)
480 if cache == 0:
481 oldnode = self.uncommitted.get(id, None)
482 if oldnode and oldnode.has_key(propname):
483 raw = oldnode[propname]
484 converter = _converters.get(rutyp.__class__, None)
485 if converter:
486 return converter(raw)
487 return raw
488 ndx = self.idcache.get(id, None)
490 if ndx is None:
491 ndx = view.find(id=id)
492 if ndx < 0:
493 raise IndexError, "%s has no node %s" % (self.classname, nodeid)
494 self.idcache[id] = ndx
495 try:
496 raw = getattr(view[ndx], propname)
497 except AttributeError:
498 raise KeyError, propname
499 rutyp = self.ruprops.get(propname, None)
501 if rutyp is None:
502 rutyp = self.privateprops[propname]
504 converter = _converters.get(rutyp.__class__, None)
505 if converter:
506 raw = converter(raw)
507 return raw
509 def set(self, nodeid, **propvalues):
510 '''Modify a property on an existing node of this class.
512 'nodeid' must be the id of an existing node of this class or an
513 IndexError is raised.
515 Each key in 'propvalues' must be the name of a property of this
516 class or a KeyError is raised.
518 All values in 'propvalues' must be acceptable types for their
519 corresponding properties or a TypeError is raised.
521 If the value of the key property is set, it must not collide with
522 other key strings or a ValueError is raised.
524 If the value of a Link or Multilink property contains an invalid
525 node id, a ValueError is raised.
526 '''
527 self.fireAuditors('set', nodeid, propvalues)
528 propvalues, oldnode = self.set_inner(nodeid, **propvalues)
529 self.fireReactors('set', nodeid, oldnode)
531 def set_inner(self, nodeid, **propvalues):
532 '''Called outside of auditors'''
533 isnew = 0
534 if propvalues.has_key('#ISNEW'):
535 isnew = 1
536 del propvalues['#ISNEW']
538 if propvalues.has_key('id'):
539 raise KeyError, '"id" is reserved'
540 if self.db.journaltag is None:
541 raise hyperdb.DatabaseError, 'Database open read-only'
542 view = self.getview(READWRITE)
544 # node must exist & not be retired
545 id = int(nodeid)
546 ndx = view.find(id=id)
547 if ndx < 0:
548 raise IndexError, "%s has no node %s" % (self.classname, nodeid)
549 row = view[ndx]
550 if row._isdel:
551 raise IndexError, "%s has no node %s" % (self.classname, nodeid)
552 oldnode = self.uncommitted.setdefault(id, {})
553 changes = {}
555 for key, value in propvalues.items():
556 # this will raise the KeyError if the property isn't valid
557 # ... we don't use getprops() here because we only care about
558 # the writeable properties.
559 if _ALLOWSETTINGPRIVATEPROPS:
560 prop = self.ruprops.get(key, None)
561 if not prop:
562 prop = self.privateprops[key]
563 else:
564 prop = self.ruprops[key]
565 converter = _converters.get(prop.__class__, lambda v: v)
566 # if the value's the same as the existing value, no sense in
567 # doing anything
568 oldvalue = converter(getattr(row, key))
569 if value == oldvalue:
570 del propvalues[key]
571 continue
573 # check to make sure we're not duplicating an existing key
574 if key == self.key:
575 iv = self.getindexview(READWRITE)
576 ndx = iv.find(k=value)
577 if ndx == -1:
578 iv.append(k=value, i=row.id)
579 if not isnew:
580 ndx = iv.find(k=oldvalue)
581 if ndx > -1:
582 iv.delete(ndx)
583 else:
584 raise ValueError, 'node with key "%s" exists'%value
586 # do stuff based on the prop type
587 if isinstance(prop, hyperdb.Link):
588 link_class = prop.classname
589 # must be a string or None
590 if value is not None and not isinstance(value, type('')):
591 raise ValueError, 'property "%s" link value be a string'%(
592 key)
593 # Roundup sets to "unselected" by passing None
594 if value is None:
595 value = 0
596 # if it isn't a number, it's a key
597 try:
598 int(value)
599 except ValueError:
600 try:
601 value = self.db.getclass(link_class).lookup(value)
602 except (TypeError, KeyError):
603 raise IndexError, 'new property "%s": %s not a %s'%(
604 key, value, prop.classname)
606 if (value is not None and
607 not self.db.getclass(link_class).hasnode(value)):
608 raise IndexError, '%s has no node %s'%(link_class, value)
610 setattr(row, key, int(value))
611 changes[key] = oldvalue
613 if self.do_journal and prop.do_journal:
614 # register the unlink with the old linked node
615 if oldvalue:
616 self.db.addjournal(link_class, oldvalue, _UNLINK,
617 (self.classname, str(row.id), key))
619 # register the link with the newly linked node
620 if value:
621 self.db.addjournal(link_class, value, _LINK,
622 (self.classname, str(row.id), key))
624 elif isinstance(prop, hyperdb.Multilink):
625 if value is not None and type(value) != _LISTTYPE:
626 raise TypeError, 'new property "%s" not a list of ids'%key
627 link_class = prop.classname
628 l = []
629 if value is None:
630 value = []
631 for entry in value:
632 if type(entry) != _STRINGTYPE:
633 raise ValueError, 'new property "%s" link value ' \
634 'must be a string'%key
635 # if it isn't a number, it's a key
636 try:
637 int(entry)
638 except ValueError:
639 try:
640 entry = self.db.getclass(link_class).lookup(entry)
641 except (TypeError, KeyError):
642 raise IndexError, 'new property "%s": %s not a %s'%(
643 key, entry, prop.classname)
644 l.append(entry)
645 propvalues[key] = value = l
647 # handle removals
648 rmvd = []
649 for id in oldvalue:
650 if id not in value:
651 rmvd.append(id)
652 # register the unlink with the old linked node
653 if self.do_journal and prop.do_journal:
654 self.db.addjournal(link_class, id, _UNLINK,
655 (self.classname, str(row.id), key))
657 # handle additions
658 adds = []
659 for id in value:
660 if id not in oldvalue:
661 if not self.db.getclass(link_class).hasnode(id):
662 raise IndexError, '%s has no node %s'%(
663 link_class, id)
664 adds.append(id)
665 # register the link with the newly linked node
666 if self.do_journal and prop.do_journal:
667 self.db.addjournal(link_class, id, _LINK,
668 (self.classname, str(row.id), key))
670 # perform the modifications on the actual property value
671 sv = getattr(row, key)
672 i = 0
673 while i < len(sv):
674 if str(sv[i].fid) in rmvd:
675 sv.delete(i)
676 else:
677 i += 1
678 for id in adds:
679 sv.append(fid=int(id))
681 # figure the journal entry
682 l = []
683 if adds:
684 l.append(('+', adds))
685 if rmvd:
686 l.append(('-', rmvd))
687 if l:
688 changes[key] = tuple(l)
689 #changes[key] = oldvalue
691 if not rmvd and not adds:
692 del propvalues[key]
694 elif isinstance(prop, hyperdb.String):
695 if value is not None and type(value) != _STRINGTYPE:
696 raise TypeError, 'new property "%s" not a string'%key
697 if value is None:
698 value = ''
699 setattr(row, key, value)
700 changes[key] = oldvalue
701 if hasattr(prop, 'isfilename') and prop.isfilename:
702 propvalues[key] = os.path.basename(value)
703 if prop.indexme:
704 self.db.indexer.add_text((self.classname, nodeid, key),
705 value, 'text/plain')
707 elif isinstance(prop, hyperdb.Password):
708 if value is not None and not isinstance(value, password.Password):
709 raise TypeError, 'new property "%s" not a Password'% key
710 if value is None:
711 value = ''
712 setattr(row, key, str(value))
713 changes[key] = str(oldvalue)
714 propvalues[key] = str(value)
716 elif isinstance(prop, hyperdb.Date):
717 if value is not None and not isinstance(value, date.Date):
718 raise TypeError, 'new property "%s" not a Date'% key
719 if value is None:
720 setattr(row, key, 0)
721 else:
722 setattr(row, key, int(calendar.timegm(value.get_tuple())))
723 changes[key] = str(oldvalue)
724 propvalues[key] = str(value)
726 elif isinstance(prop, hyperdb.Interval):
727 if value is not None and not isinstance(value, date.Interval):
728 raise TypeError, 'new property "%s" not an Interval'% key
729 if value is None:
730 setattr(row, key, '')
731 else:
732 # kedder: we should store interval values serialized
733 setattr(row, key, value.serialise())
734 changes[key] = str(oldvalue)
735 propvalues[key] = str(value)
737 elif isinstance(prop, hyperdb.Number):
738 if value is None:
739 v = 0
740 else:
741 try:
742 v = int(value)
743 except ValueError:
744 raise TypeError, "%s (%s) is not numeric"%(key, repr(value))
745 if not BACKWARDS_COMPATIBLE:
746 if v >=0:
747 v = v + 1
748 setattr(row, key, v)
749 changes[key] = oldvalue
750 propvalues[key] = value
752 elif isinstance(prop, hyperdb.Boolean):
753 if value is None:
754 bv = 0
755 elif value not in (0,1):
756 raise TypeError, "%s (%s) is not boolean"%(key, repr(value))
757 else:
758 bv = value
759 if not BACKWARDS_COMPATIBLE:
760 bv += 1
761 setattr(row, key, bv)
762 changes[key] = oldvalue
763 propvalues[key] = value
765 oldnode[key] = oldvalue
767 # nothing to do?
768 if not propvalues:
769 return propvalues, oldnode
770 if not propvalues.has_key('activity'):
771 row.activity = int(time.time())
772 if not propvalues.has_key('actor'):
773 row.actor = int(self.db.getuid())
774 if isnew:
775 if not row.creation:
776 row.creation = int(time.time())
777 if not row.creator:
778 row.creator = int(self.db.getuid())
780 self.db.dirty = 1
782 if self.do_journal:
783 if isnew:
784 self.db.addjournal(self.classname, nodeid, _CREATE, changes)
785 else:
786 self.db.addjournal(self.classname, nodeid, _SET, changes)
788 return propvalues, oldnode
790 def retire(self, nodeid):
791 '''Retire a node.
793 The properties on the node remain available from the get() method,
794 and the node's id is never reused.
796 Retired nodes are not returned by the find(), list(), or lookup()
797 methods, and other nodes may reuse the values of their key properties.
798 '''
799 if self.db.journaltag is None:
800 raise hyperdb.DatabaseError, 'Database open read-only'
801 self.fireAuditors('retire', nodeid, None)
802 view = self.getview(READWRITE)
803 ndx = view.find(id=int(nodeid))
804 if ndx < 0:
805 raise KeyError, "nodeid %s not found" % nodeid
807 row = view[ndx]
808 oldvalues = self.uncommitted.setdefault(row.id, {})
809 oldval = oldvalues['_isdel'] = row._isdel
810 row._isdel = 1
812 if self.do_journal:
813 self.db.addjournal(self.classname, nodeid, _RETIRE, {})
814 if self.key:
815 iv = self.getindexview(READWRITE)
816 ndx = iv.find(k=getattr(row, self.key))
817 # find is broken with multiple attribute lookups
818 # on ordered views
819 #ndx = iv.find(k=getattr(row, self.key),i=row.id)
820 if ndx > -1 and iv[ndx].i == row.id:
821 iv.delete(ndx)
823 self.db.dirty = 1
824 self.fireReactors('retire', nodeid, None)
826 def restore(self, nodeid):
827 '''Restore a retired node.
829 Make node available for all operations like it was before retirement.
830 '''
831 if self.db.journaltag is None:
832 raise hyperdb.DatabaseError, 'Database open read-only'
834 # check if key property was overrided
835 key = self.getkey()
836 keyvalue = self.get(nodeid, key)
838 try:
839 id = self.lookup(keyvalue)
840 except KeyError:
841 pass
842 else:
843 raise KeyError, "Key property (%s) of retired node clashes with \
844 existing one (%s)" % (key, keyvalue)
845 # Now we can safely restore node
846 self.fireAuditors('restore', nodeid, None)
847 view = self.getview(READWRITE)
848 ndx = view.find(id=int(nodeid))
849 if ndx < 0:
850 raise KeyError, "nodeid %s not found" % nodeid
852 row = view[ndx]
853 oldvalues = self.uncommitted.setdefault(row.id, {})
854 oldval = oldvalues['_isdel'] = row._isdel
855 row._isdel = 0
857 if self.do_journal:
858 self.db.addjournal(self.classname, nodeid, _RESTORE, {})
859 if self.key:
860 iv = self.getindexview(READWRITE)
861 ndx = iv.find(k=getattr(row, self.key),i=row.id)
862 if ndx > -1:
863 iv.delete(ndx)
864 self.db.dirty = 1
865 self.fireReactors('restore', nodeid, None)
867 def is_retired(self, nodeid):
868 '''Return true if the node is retired
869 '''
870 view = self.getview(READWRITE)
871 # node must exist & not be retired
872 id = int(nodeid)
873 ndx = view.find(id=id)
874 if ndx < 0:
875 raise IndexError, "%s has no node %s" % (self.classname, nodeid)
876 row = view[ndx]
877 return row._isdel
879 def history(self, nodeid):
880 '''Retrieve the journal of edits on a particular node.
882 'nodeid' must be the id of an existing node of this class or an
883 IndexError is raised.
885 The returned list contains tuples of the form
887 (nodeid, date, tag, action, params)
889 'date' is a Timestamp object specifying the time of the change and
890 'tag' is the journaltag specified when the database was opened.
891 '''
892 if not self.do_journal:
893 raise ValueError, 'Journalling is disabled for this class'
894 return self.db.getjournal(self.classname, nodeid)
896 def setkey(self, propname):
897 '''Select a String property of this class to be the key property.
899 'propname' must be the name of a String property of this class or
900 None, or a TypeError is raised. The values of the key property on
901 all existing nodes must be unique or a ValueError is raised.
902 '''
903 if self.key:
904 if propname == self.key:
905 return
906 else:
907 # drop the old key table
908 tablename = "_%s.%s"%(self.classname, self.key)
909 self.db._db.getas(tablename)
911 #raise ValueError, "%s already indexed on %s"%(self.classname,
912 # self.key)
914 prop = self.properties.get(propname, None)
915 if prop is None:
916 prop = self.privateprops.get(propname, None)
917 if prop is None:
918 raise KeyError, "no property %s" % propname
919 if not isinstance(prop, hyperdb.String):
920 raise TypeError, "%s is not a String" % propname
922 # the way he index on properties is by creating a
923 # table named _%(classname)s.%(key)s, if this table
924 # exists then everything is okay. If this table
925 # doesn't exist, then generate a new table on the
926 # key value.
928 # first setkey for this run or key has been changed
929 self.key = propname
930 tablename = "_%s.%s"%(self.classname, self.key)
932 iv = self.db._db.view(tablename)
933 if self.db.fastopen and iv.structure():
934 return
936 # very first setkey ever or the key has changed
937 self.db.dirty = 1
938 iv = self.db._db.getas('_%s[k:S,i:I]' % tablename)
939 iv = iv.ordered(1)
940 for row in self.getview():
941 iv.append(k=getattr(row, propname), i=row.id)
942 self.db.commit()
944 def getkey(self):
945 '''Return the name of the key property for this class or None.'''
946 return self.key
948 def lookup(self, keyvalue):
949 '''Locate a particular node by its key property and return its id.
951 If this class has no key property, a TypeError is raised. If the
952 keyvalue matches one of the values for the key property among
953 the nodes in this class, the matching node's id is returned;
954 otherwise a KeyError is raised.
955 '''
956 if not self.key:
957 raise TypeError, 'No key property set for class %s'%self.classname
959 if type(keyvalue) is not _STRINGTYPE:
960 raise TypeError, '%r is not a string'%keyvalue
962 # XXX FIX ME -> this is a bit convoluted
963 # First we search the index view to get the id
964 # which is a quicker look up.
965 # Then we lookup the row with id=id
966 # if the _isdel property of the row is 0, return the
967 # string version of the id. (Why string version???)
968 #
969 # Otherwise, just lookup the non-indexed key
970 # in the non-index table and check the _isdel property
971 iv = self.getindexview()
972 if iv:
973 # look up the index view for the id,
974 # then instead of looking up the keyvalue, lookup the
975 # quicker id
976 ndx = iv.find(k=keyvalue)
977 if ndx > -1:
978 view = self.getview()
979 ndx = view.find(id=iv[ndx].i)
980 if ndx > -1:
981 row = view[ndx]
982 if not row._isdel:
983 return str(row.id)
984 else:
985 # perform the slower query
986 view = self.getview()
987 ndx = view.find({self.key:keyvalue})
988 if ndx > -1:
989 row = view[ndx]
990 if not row._isdel:
991 return str(row.id)
993 raise KeyError, keyvalue
995 def destroy(self, id):
996 '''Destroy a node.
998 WARNING: this method should never be used except in extremely rare
999 situations where there could never be links to the node being
1000 deleted
1002 WARNING: use retire() instead
1004 WARNING: the properties of this node will not be available ever again
1006 WARNING: really, use retire() instead
1008 Well, I think that's enough warnings. This method exists mostly to
1009 support the session storage of the cgi interface.
1011 The node is completely removed from the hyperdb, including all journal
1012 entries. It will no longer be available, and will generally break code
1013 if there are any references to the node.
1014 '''
1015 view = self.getview(READWRITE)
1016 ndx = view.find(id=int(id))
1017 if ndx > -1:
1018 if self.key:
1019 keyvalue = getattr(view[ndx], self.key)
1020 iv = self.getindexview(READWRITE)
1021 if iv:
1022 ivndx = iv.find(k=keyvalue)
1023 if ivndx > -1:
1024 iv.delete(ivndx)
1025 view.delete(ndx)
1026 self.db.destroyjournal(self.classname, id)
1027 self.db.dirty = 1
1029 def find(self, **propspec):
1030 '''Get the ids of nodes in this class which link to the given nodes.
1032 'propspec'
1033 consists of keyword args propname={nodeid:1,}
1034 'propname'
1035 must be the name of a property in this class, or a
1036 KeyError is raised. That property must be a Link or
1037 Multilink property, or a TypeError is raised.
1039 Any node in this class whose propname property links to any of the
1040 nodeids will be returned. Used by the full text indexing, which knows
1041 that "foo" occurs in msg1, msg3 and file7; so we have hits on these
1042 issues::
1044 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1045 '''
1046 propspec = propspec.items()
1047 for propname, nodeid in propspec:
1048 # check the prop is OK
1049 prop = self.ruprops[propname]
1050 if (not isinstance(prop, hyperdb.Link) and
1051 not isinstance(prop, hyperdb.Multilink)):
1052 raise TypeError, "'%s' not a Link/Multilink property"%propname
1054 vws = []
1055 for propname, ids in propspec:
1056 if type(ids) is _STRINGTYPE:
1057 ids = {int(ids):1}
1058 elif ids is None:
1059 ids = {0:1}
1060 else:
1061 d = {}
1062 for id in ids.keys():
1063 if id is None:
1064 d[0] = 1
1065 else:
1066 d[int(id)] = 1
1067 ids = d
1068 prop = self.ruprops[propname]
1069 view = self.getview()
1070 if isinstance(prop, hyperdb.Multilink):
1071 def ff(row, nm=propname, ids=ids):
1072 if not row._isdel:
1073 sv = getattr(row, nm)
1074 for sr in sv:
1075 if ids.has_key(sr.fid):
1076 return 1
1077 return 0
1078 else:
1079 def ff(row, nm=propname, ids=ids):
1080 return not row._isdel and ids.has_key(getattr(row, nm))
1081 ndxview = view.filter(ff)
1082 vws.append(ndxview.unique())
1084 # handle the empty match case
1085 if not vws:
1086 return []
1088 ndxview = vws[0]
1089 for v in vws[1:]:
1090 ndxview = ndxview.union(v)
1091 view = self.getview().remapwith(ndxview)
1092 rslt = []
1093 for row in view:
1094 rslt.append(str(row.id))
1095 return rslt
1098 def list(self):
1099 ''' Return a list of the ids of the active nodes in this class.
1100 '''
1101 l = []
1102 for row in self.getview().select(_isdel=0):
1103 l.append(str(row.id))
1104 return l
1106 def getnodeids(self):
1107 ''' Retrieve all the ids of the nodes for a particular Class.
1109 Set retired=None to get all nodes. Otherwise it'll get all the
1110 retired or non-retired nodes, depending on the flag.
1111 '''
1112 l = []
1113 for row in self.getview():
1114 l.append(str(row.id))
1115 return l
1117 def count(self):
1118 return len(self.getview())
1120 def getprops(self, protected=1):
1121 # protected is not in ping's spec
1122 allprops = self.ruprops.copy()
1123 if protected and self.privateprops is not None:
1124 allprops.update(self.privateprops)
1125 return allprops
1127 def addprop(self, **properties):
1128 for key in properties.keys():
1129 if self.ruprops.has_key(key):
1130 raise ValueError, "%s is already a property of %s"%(key,
1131 self.classname)
1132 self.ruprops.update(properties)
1133 # Class structure has changed
1134 self.db.fastopen = 0
1135 view = self.__getview()
1136 self.db.commit()
1137 # ---- end of ping's spec
1139 def filter(self, search_matches, filterspec, sort=(None,None),
1140 group=(None,None)):
1141 '''Return a list of the ids of the active nodes in this class that
1142 match the 'filter' spec, sorted by the group spec and then the
1143 sort spec
1145 "filterspec" is {propname: value(s)}
1147 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1148 and prop is a prop name or None
1150 "search_matches" is {nodeid: marker}
1152 The filter must match all properties specificed - but if the
1153 property value to match is a list, any one of the values in the
1154 list may match for that property to match.
1155 '''
1156 # search_matches is None or a set (dict of {nodeid: {propname:[nodeid,...]}})
1157 # filterspec is a dict {propname:value}
1158 # sort and group are (dir, prop) where dir is '+', '-' or None
1159 # and prop is a prop name or None
1161 timezone = self.db.getUserTimezone()
1163 where = {'_isdel':0}
1164 wherehigh = {}
1165 mlcriteria = {}
1166 regexes = {}
1167 orcriteria = {}
1168 for propname, value in filterspec.items():
1169 prop = self.ruprops.get(propname, None)
1170 if prop is None:
1171 prop = self.privateprops[propname]
1172 if isinstance(prop, hyperdb.Multilink):
1173 if value in ('-1', ['-1']):
1174 value = []
1175 elif type(value) is not _LISTTYPE:
1176 value = [value]
1177 # transform keys to ids
1178 u = []
1179 for item in value:
1180 try:
1181 item = int(item)
1182 except (TypeError, ValueError):
1183 item = int(self.db.getclass(prop.classname).lookup(item))
1184 if item == -1:
1185 item = 0
1186 u.append(item)
1187 mlcriteria[propname] = u
1188 elif isinstance(prop, hyperdb.Link):
1189 if type(value) is not _LISTTYPE:
1190 value = [value]
1191 # transform keys to ids
1192 u = []
1193 for item in value:
1194 try:
1195 item = int(item)
1196 except (TypeError, ValueError):
1197 item = int(self.db.getclass(prop.classname).lookup(item))
1198 if item == -1:
1199 item = 0
1200 u.append(item)
1201 if len(u) == 1:
1202 where[propname] = u[0]
1203 else:
1204 orcriteria[propname] = u
1205 elif isinstance(prop, hyperdb.String):
1206 if type(value) is not type([]):
1207 value = [value]
1208 m = []
1209 for v in value:
1210 # simple glob searching
1211 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1212 v = v.replace('?', '.')
1213 v = v.replace('*', '.*?')
1214 m.append(v)
1215 regexes[propname] = re.compile('(%s)'%('|'.join(m)), re.I)
1216 elif propname == 'id':
1217 where[propname] = int(value)
1218 elif isinstance(prop, hyperdb.Boolean):
1219 if type(value) is _STRINGTYPE:
1220 bv = value.lower() in ('yes', 'true', 'on', '1')
1221 else:
1222 bv = value
1223 where[propname] = bv
1224 elif isinstance(prop, hyperdb.Date):
1225 try:
1226 # Try to filter on range of dates
1227 date_rng = Range(value, date.Date, offset=timezone)
1228 if date_rng.from_value:
1229 t = date_rng.from_value.get_tuple()
1230 where[propname] = int(calendar.timegm(t))
1231 else:
1232 # use minimum possible value to exclude items without
1233 # 'prop' property
1234 where[propname] = 0
1235 if date_rng.to_value:
1236 t = date_rng.to_value.get_tuple()
1237 wherehigh[propname] = int(calendar.timegm(t))
1238 else:
1239 wherehigh[propname] = None
1240 except ValueError:
1241 # If range creation fails - ignore that search parameter
1242 pass
1243 elif isinstance(prop, hyperdb.Interval):
1244 try:
1245 # Try to filter on range of intervals
1246 date_rng = Range(value, date.Interval)
1247 if date_rng.from_value:
1248 #t = date_rng.from_value.get_tuple()
1249 where[propname] = date_rng.from_value.serialise()
1250 else:
1251 # use minimum possible value to exclude items without
1252 # 'prop' property
1253 where[propname] = '-99999999999999'
1254 if date_rng.to_value:
1255 #t = date_rng.to_value.get_tuple()
1256 wherehigh[propname] = date_rng.to_value.serialise()
1257 else:
1258 wherehigh[propname] = None
1259 except ValueError:
1260 # If range creation fails - ignore that search parameter
1261 pass
1262 elif isinstance(prop, hyperdb.Number):
1263 where[propname] = int(value)
1264 else:
1265 where[propname] = str(value)
1266 v = self.getview()
1267 #print "filter start at %s" % time.time()
1268 if where:
1269 where_higherbound = where.copy()
1270 where_higherbound.update(wherehigh)
1271 v = v.select(where, where_higherbound)
1272 #print "filter where at %s" % time.time()
1274 if mlcriteria:
1275 # multilink - if any of the nodeids required by the
1276 # filterspec aren't in this node's property, then skip it
1277 def ff(row, ml=mlcriteria):
1278 for propname, values in ml.items():
1279 sv = getattr(row, propname)
1280 if not values and sv:
1281 return 0
1282 for id in values:
1283 if sv.find(fid=id) == -1:
1284 return 0
1285 return 1
1286 iv = v.filter(ff)
1287 v = v.remapwith(iv)
1289 #print "filter mlcrit at %s" % time.time()
1291 if orcriteria:
1292 def ff(row, crit=orcriteria):
1293 for propname, allowed in crit.items():
1294 val = getattr(row, propname)
1295 if val not in allowed:
1296 return 0
1297 return 1
1299 iv = v.filter(ff)
1300 v = v.remapwith(iv)
1302 #print "filter orcrit at %s" % time.time()
1303 if regexes:
1304 def ff(row, r=regexes):
1305 for propname, regex in r.items():
1306 val = str(getattr(row, propname))
1307 if not regex.search(val):
1308 return 0
1309 return 1
1311 iv = v.filter(ff)
1312 v = v.remapwith(iv)
1313 #print "filter regexs at %s" % time.time()
1315 if sort or group:
1316 sortspec = []
1317 rev = []
1318 for dir, propname in group, sort:
1319 if propname is None: continue
1320 isreversed = 0
1321 if dir == '-':
1322 isreversed = 1
1323 try:
1324 prop = getattr(v, propname)
1325 except AttributeError:
1326 print "MK has no property %s" % propname
1327 continue
1328 propclass = self.ruprops.get(propname, None)
1329 if propclass is None:
1330 propclass = self.privateprops.get(propname, None)
1331 if propclass is None:
1332 print "Schema has no property %s" % propname
1333 continue
1334 if isinstance(propclass, hyperdb.Link):
1335 linkclass = self.db.getclass(propclass.classname)
1336 lv = linkclass.getview()
1337 lv = lv.rename('id', propname)
1338 v = v.join(lv, prop, 1)
1339 if linkclass.getprops().has_key('order'):
1340 propname = 'order'
1341 else:
1342 propname = linkclass.labelprop()
1343 prop = getattr(v, propname)
1344 if isreversed:
1345 rev.append(prop)
1346 sortspec.append(prop)
1347 v = v.sortrev(sortspec, rev)[:] #XXX Metakit bug
1348 #print "filter sort at %s" % time.time()
1350 rslt = []
1351 for row in v:
1352 id = str(row.id)
1353 if search_matches is not None:
1354 if search_matches.has_key(id):
1355 rslt.append(id)
1356 else:
1357 rslt.append(id)
1358 return rslt
1360 def hasnode(self, nodeid):
1361 '''Determine if the given nodeid actually exists
1362 '''
1363 return int(nodeid) < self.maxid
1365 def labelprop(self, default_to_id=0):
1366 '''Return the property name for a label for the given node.
1368 This method attempts to generate a consistent label for the node.
1369 It tries the following in order:
1371 1. key property
1372 2. "name" property
1373 3. "title" property
1374 4. first property from the sorted property name list
1375 '''
1376 k = self.getkey()
1377 if k:
1378 return k
1379 props = self.getprops()
1380 if props.has_key('name'):
1381 return 'name'
1382 elif props.has_key('title'):
1383 return 'title'
1384 if default_to_id:
1385 return 'id'
1386 props = props.keys()
1387 props.sort()
1388 return props[0]
1390 def stringFind(self, **requirements):
1391 '''Locate a particular node by matching a set of its String
1392 properties in a caseless search.
1394 If the property is not a String property, a TypeError is raised.
1396 The return is a list of the id of all nodes that match.
1397 '''
1398 for propname in requirements.keys():
1399 prop = self.properties[propname]
1400 if isinstance(not prop, hyperdb.String):
1401 raise TypeError, "'%s' not a String property"%propname
1402 requirements[propname] = requirements[propname].lower()
1403 requirements['_isdel'] = 0
1405 l = []
1406 for row in self.getview().select(requirements):
1407 l.append(str(row.id))
1408 return l
1410 def addjournal(self, nodeid, action, params):
1411 '''Add a journal to the given nodeid,
1412 'action' may be:
1414 'create' or 'set' -- 'params' is a dictionary of property values
1415 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
1416 'retire' -- 'params' is None
1417 '''
1418 self.db.addjournal(self.classname, nodeid, action, params)
1420 def index(self, nodeid):
1421 ''' Add (or refresh) the node to search indexes '''
1422 # find all the String properties that have indexme
1423 for prop, propclass in self.getprops().items():
1424 if isinstance(propclass, hyperdb.String) and propclass.indexme:
1425 # index them under (classname, nodeid, property)
1426 self.db.indexer.add_text((self.classname, nodeid, prop),
1427 str(self.get(nodeid, prop)))
1429 def export_list(self, propnames, nodeid):
1430 ''' Export a node - generate a list of CSV-able data in the order
1431 specified by propnames for the given node.
1432 '''
1433 properties = self.getprops()
1434 l = []
1435 for prop in propnames:
1436 proptype = properties[prop]
1437 value = self.get(nodeid, prop)
1438 # "marshal" data where needed
1439 if value is None:
1440 pass
1441 elif isinstance(proptype, hyperdb.Date):
1442 value = value.get_tuple()
1443 elif isinstance(proptype, hyperdb.Interval):
1444 value = value.get_tuple()
1445 elif isinstance(proptype, hyperdb.Password):
1446 value = str(value)
1447 l.append(repr(value))
1449 # append retired flag
1450 l.append(repr(self.is_retired(nodeid)))
1452 return l
1454 def import_list(self, propnames, proplist):
1455 ''' Import a node - all information including "id" is present and
1456 should not be sanity checked. Triggers are not triggered. The
1457 journal should be initialised using the "creator" and "creation"
1458 information.
1460 Return the nodeid of the node imported.
1461 '''
1462 if self.db.journaltag is None:
1463 raise hyperdb.DatabaseError, 'Database open read-only'
1464 properties = self.getprops()
1466 d = {}
1467 view = self.getview(READWRITE)
1468 for i in range(len(propnames)):
1469 value = eval(proplist[i])
1470 if not value:
1471 continue
1473 propname = propnames[i]
1474 if propname == 'id':
1475 newid = value = int(value)
1476 elif propname == 'is retired':
1477 # is the item retired?
1478 if int(value):
1479 d['_isdel'] = 1
1480 continue
1481 elif value is None:
1482 d[propname] = None
1483 continue
1485 prop = properties[propname]
1486 if isinstance(prop, hyperdb.Date):
1487 value = int(calendar.timegm(value))
1488 elif isinstance(prop, hyperdb.Interval):
1489 value = date.Interval(value).serialise()
1490 elif isinstance(prop, hyperdb.Number):
1491 value = int(value)
1492 elif isinstance(prop, hyperdb.Boolean):
1493 value = int(value)
1494 elif isinstance(prop, hyperdb.Link) and value:
1495 value = int(value)
1496 elif isinstance(prop, hyperdb.Multilink):
1497 # we handle multilinks separately
1498 continue
1499 d[propname] = value
1501 # possibly make a new node
1502 if not d.has_key('id'):
1503 d['id'] = newid = self.maxid
1504 self.maxid += 1
1506 # save off the node
1507 view.append(d)
1509 # fix up multilinks
1510 ndx = view.find(id=newid)
1511 row = view[ndx]
1512 for i in range(len(propnames)):
1513 value = eval(proplist[i])
1514 propname = propnames[i]
1515 if propname == 'is retired':
1516 continue
1517 prop = properties[propname]
1518 if not isinstance(prop, hyperdb.Multilink):
1519 continue
1520 sv = getattr(row, propname)
1521 for entry in value:
1522 sv.append((int(entry),))
1524 self.db.dirty = 1
1525 creator = d.get('creator', 0)
1526 creation = d.get('creation', 0)
1527 self.db.addjournal(self.classname, str(newid), _CREATE, {}, creator,
1528 creation)
1529 return newid
1531 # --- used by Database
1532 def _commit(self):
1533 ''' called post commit of the DB.
1534 interested subclasses may override '''
1535 self.uncommitted = {}
1536 self.rbactions = []
1537 self.idcache = {}
1538 def _rollback(self):
1539 ''' called pre rollback of the DB.
1540 interested subclasses may override '''
1541 for action in self.rbactions:
1542 action()
1543 self.rbactions = []
1544 self.uncommitted = {}
1545 self.idcache = {}
1546 def _clear(self):
1547 view = self.getview(READWRITE)
1548 if len(view):
1549 view[:] = []
1550 self.db.dirty = 1
1551 iv = self.getindexview(READWRITE)
1552 if iv:
1553 iv[:] = []
1554 def rollbackaction(self, action):
1555 ''' call this to register a callback called on rollback
1556 callback is removed on end of transaction '''
1557 self.rbactions.append(action)
1558 # --- internal
1559 def __getview(self):
1560 ''' Find the interface for a specific Class in the hyperdb.
1562 This method checks to see whether the schema has changed and
1563 re-works the underlying metakit structure if it has.
1564 '''
1565 db = self.db._db
1566 view = db.view(self.classname)
1567 mkprops = view.structure()
1569 # if we have structure in the database, and the structure hasn't
1570 # changed
1571 # note on view.ordered ->
1572 # return a metakit view ordered on the id column
1573 # id is always the first column. This speeds up
1574 # look-ups on the id column.
1576 if mkprops and self.db.fastopen:
1577 return view.ordered(1)
1579 # is the definition the same?
1580 for nm, rutyp in self.ruprops.items():
1581 for mkprop in mkprops:
1582 if mkprop.name == nm:
1583 break
1584 else:
1585 mkprop = None
1586 if mkprop is None:
1587 break
1588 if _typmap[rutyp.__class__] != mkprop.type:
1589 break
1590 else:
1591 # make sure we have the 'actor' property too
1592 for mkprop in mkprops:
1593 if mkprop.name == 'actor':
1594 return view.ordered(1)
1596 # The schema has changed. We need to create or restructure the mk view
1597 # id comes first, so we can use view.ordered(1) so that
1598 # MK will order it for us to allow binary-search quick lookups on
1599 # the id column
1600 self.db.dirty = 1
1601 s = ["%s[id:I" % self.classname]
1603 # these columns will always be added, we can't trample them :)
1604 _columns = {"id":"I", "_isdel":"I", "activity":"I", "actor": "I",
1605 "creation":"I", "creator":"I"}
1607 for nm, rutyp in self.ruprops.items():
1608 mktyp = _typmap[rutyp.__class__].upper()
1609 if nm in _columns and _columns[nm] != mktyp:
1610 # oops, two columns with the same name and different properties
1611 raise MKBackendError("column %s for table %sis defined with multiple types"%(nm, self.classname))
1612 _columns[nm] = mktyp
1613 s.append('%s:%s' % (nm, mktyp))
1614 if mktyp == 'V':
1615 s[-1] += ('[fid:I]')
1617 # XXX FIX ME -> in some tests, creation:I becomes creation:S is this
1618 # okay? Does this need to be supported?
1619 s.append('_isdel:I,activity:I,actor:I,creation:I,creator:I]')
1620 view = self.db._db.getas(','.join(s))
1621 self.db.commit()
1622 return view.ordered(1)
1623 def getview(self, RW=0):
1624 # XXX FIX ME -> The RW flag doesn't do anything.
1625 return self.db._db.view(self.classname).ordered(1)
1626 def getindexview(self, RW=0):
1627 # XXX FIX ME -> The RW flag doesn't do anything.
1628 tablename = "_%s.%s"%(self.classname, self.key)
1629 return self.db._db.view("_%s" % tablename).ordered(1)
1631 def _fetchML(sv):
1632 l = []
1633 for row in sv:
1634 if row.fid:
1635 l.append(str(row.fid))
1636 return l
1638 def _fetchPW(s):
1639 ''' Convert to a password.Password unless the password is '' which is
1640 our sentinel for "unset".
1641 '''
1642 if s == '':
1643 return None
1644 p = password.Password()
1645 p.unpack(s)
1646 return p
1648 def _fetchLink(n):
1649 ''' Return None if the link is 0 - otherwise strify it.
1650 '''
1651 return n and str(n) or None
1653 def _fetchDate(n):
1654 ''' Convert the timestamp to a date.Date instance - unless it's 0 which
1655 is our sentinel for "unset".
1656 '''
1657 if n == 0:
1658 return None
1659 return date.Date(time.gmtime(n))
1661 def _fetchInterval(n):
1662 ''' Convert to a date.Interval unless the interval is '' which is our
1663 sentinel for "unset".
1664 '''
1665 if n == '':
1666 return None
1667 return date.Interval(n)
1669 # Converters for boolean and numbers to properly
1670 # return None values.
1671 # These are in conjunction with the setters above
1672 # look for hyperdb.Boolean and hyperdb.Number
1673 if BACKWARDS_COMPATIBLE:
1674 def getBoolean(bool): return bool
1675 def getNumber(number): return number
1676 else:
1677 def getBoolean(bool):
1678 if not bool: res = None
1679 else: res = bool - 1
1680 return res
1682 def getNumber(number):
1683 if number == 0: res = None
1684 elif number < 0: res = number
1685 else: res = number - 1
1686 return res
1688 _converters = {
1689 hyperdb.Date : _fetchDate,
1690 hyperdb.Link : _fetchLink,
1691 hyperdb.Multilink : _fetchML,
1692 hyperdb.Interval : _fetchInterval,
1693 hyperdb.Password : _fetchPW,
1694 hyperdb.Boolean : getBoolean,
1695 hyperdb.Number : getNumber,
1696 hyperdb.String : lambda s: s and str(s) or None,
1697 }
1699 class FileName(hyperdb.String):
1700 isfilename = 1
1702 _typmap = {
1703 FileName : 'S',
1704 hyperdb.String : 'S',
1705 hyperdb.Date : 'I',
1706 hyperdb.Link : 'I',
1707 hyperdb.Multilink : 'V',
1708 hyperdb.Interval : 'S',
1709 hyperdb.Password : 'S',
1710 hyperdb.Boolean : 'I',
1711 hyperdb.Number : 'I',
1712 }
1713 class FileClass(Class, hyperdb.FileClass):
1714 ''' like Class but with a content property
1715 '''
1716 default_mime_type = 'text/plain'
1717 def __init__(self, db, classname, **properties):
1718 properties['content'] = FileName()
1719 if not properties.has_key('type'):
1720 properties['type'] = hyperdb.String()
1721 Class.__init__(self, db, classname, **properties)
1723 def gen_filename(self, nodeid):
1724 nm = '%s%s' % (self.classname, nodeid)
1725 sd = str(int(int(nodeid) / 1000))
1726 d = os.path.join(self.db.config.DATABASE, 'files', self.classname, sd)
1727 if not os.path.exists(d):
1728 os.makedirs(d)
1729 return os.path.join(d, nm)
1731 def get(self, nodeid, propname, default=_marker, cache=1):
1732 if propname == 'content':
1733 poss_msg = 'Possibly an access right configuration problem.'
1734 fnm = self.gen_filename(nodeid)
1735 try:
1736 f = open(fnm, 'rb')
1737 except IOError, (strerror):
1738 # XXX by catching this we donot see an error in the log.
1739 return 'ERROR reading file: %s%s\n%s\n%s'%(
1740 self.classname, nodeid, poss_msg, strerror)
1741 x = f.read()
1742 f.close()
1743 else:
1744 x = Class.get(self, nodeid, propname, default)
1745 return x
1747 def create(self, **propvalues):
1748 if not propvalues:
1749 raise ValueError, "Need something to create!"
1750 self.fireAuditors('create', None, propvalues)
1752 content = propvalues['content']
1753 del propvalues['content']
1755 newid = Class.create_inner(self, **propvalues)
1756 if not content:
1757 return newid
1759 # figure a filename
1760 nm = self.gen_filename(newid)
1761 open(nm, 'wb').write(content)
1763 mimetype = propvalues.get('type', self.default_mime_type)
1764 self.db.indexer.add_text((self.classname, newid, 'content'), content,
1765 mimetype)
1766 def undo(fnm=nm):
1767 os.remove(fnm)
1768 self.rollbackaction(undo)
1769 return newid
1771 def index(self, nodeid):
1772 Class.index(self, nodeid)
1773 mimetype = self.get(nodeid, 'type')
1774 if not mimetype:
1775 mimetype = self.default_mime_type
1776 self.db.indexer.add_text((self.classname, nodeid, 'content'),
1777 self.get(nodeid, 'content'), mimetype)
1779 class IssueClass(Class, roundupdb.IssueClass):
1780 ''' The newly-created class automatically includes the "messages",
1781 "files", "nosy", and "superseder" properties. If the 'properties'
1782 dictionary attempts to specify any of these properties or a
1783 "creation" or "activity" property, a ValueError is raised.
1784 '''
1785 def __init__(self, db, classname, **properties):
1786 if not properties.has_key('title'):
1787 properties['title'] = hyperdb.String(indexme='yes')
1788 if not properties.has_key('messages'):
1789 properties['messages'] = hyperdb.Multilink("msg")
1790 if not properties.has_key('files'):
1791 properties['files'] = hyperdb.Multilink("file")
1792 if not properties.has_key('nosy'):
1793 # note: journalling is turned off as it really just wastes
1794 # space. this behaviour may be overridden in an instance
1795 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
1796 if not properties.has_key('superseder'):
1797 properties['superseder'] = hyperdb.Multilink(classname)
1798 Class.__init__(self, db, classname, **properties)
1800 CURVERSION = 2
1802 class Indexer(Indexer):
1803 disallows = {'THE':1, 'THIS':1, 'ZZZ':1, 'THAT':1, 'WITH':1}
1804 def __init__(self, path, datadb):
1805 self.path = os.path.join(path, 'index.mk4')
1806 self.db = metakit.storage(self.path, 1)
1807 self.datadb = datadb
1808 self.reindex = 0
1809 v = self.db.view('version')
1810 if not v.structure():
1811 v = self.db.getas('version[vers:I]')
1812 self.db.commit()
1813 v.append(vers=CURVERSION)
1814 self.reindex = 1
1815 elif v[0].vers != CURVERSION:
1816 v[0].vers = CURVERSION
1817 self.reindex = 1
1818 if self.reindex:
1819 self.db.getas('ids[tblid:I,nodeid:I,propid:I,ignore:I]')
1820 self.db.getas('index[word:S,hits[pos:I]]')
1821 self.db.commit()
1822 self.reindex = 1
1823 self.changed = 0
1824 self.propcache = {}
1826 def close(self):
1827 '''close the indexing database'''
1828 del self.db
1829 self.db = None
1831 def force_reindex(self):
1832 '''Force a reindexing of the database. This essentially
1833 empties the tables ids and index and sets a flag so
1834 that the databases are reindexed'''
1835 v = self.db.view('ids')
1836 v[:] = []
1837 v = self.db.view('index')
1838 v[:] = []
1839 self.db.commit()
1840 self.reindex = 1
1842 def should_reindex(self):
1843 '''returns True if the indexes need to be rebuilt'''
1844 return self.reindex
1846 def _getprops(self, classname):
1847 props = self.propcache.get(classname, None)
1848 if props is None:
1849 props = self.datadb.view(classname).structure()
1850 props = [prop.name for prop in props]
1851 self.propcache[classname] = props
1852 return props
1854 def _getpropid(self, classname, propname):
1855 return self._getprops(classname).index(propname)
1857 def _getpropname(self, classname, propid):
1858 return self._getprops(classname)[propid]
1860 def add_text(self, identifier, text, mime_type='text/plain'):
1861 if mime_type != 'text/plain':
1862 return
1863 classname, nodeid, property = identifier
1864 tbls = self.datadb.view('tables')
1865 tblid = tbls.find(name=classname)
1866 if tblid < 0:
1867 raise KeyError, "unknown class %r"%classname
1868 nodeid = int(nodeid)
1869 propid = self._getpropid(classname, property)
1870 ids = self.db.view('ids')
1871 oldpos = ids.find(tblid=tblid,nodeid=nodeid,propid=propid,ignore=0)
1872 if oldpos > -1:
1873 ids[oldpos].ignore = 1
1874 self.changed = 1
1875 pos = ids.append(tblid=tblid,nodeid=nodeid,propid=propid)
1877 wordlist = re.findall(r'\b\w{2,25}\b', text.upper())
1878 words = {}
1879 for word in wordlist:
1880 if not self.disallows.has_key(word):
1881 words[word] = 1
1882 words = words.keys()
1884 index = self.db.view('index').ordered(1)
1885 for word in words:
1886 ndx = index.find(word=word)
1887 if ndx < 0:
1888 index.append(word=word)
1889 ndx = index.find(word=word)
1890 index[ndx].hits.append(pos=pos)
1891 self.changed = 1
1893 def find(self, wordlist):
1894 '''look up all the words in the wordlist.
1895 If none are found return an empty dictionary
1896 * more rules here
1897 '''
1898 hits = None
1899 index = self.db.view('index').ordered(1)
1900 for word in wordlist:
1901 word = word.upper()
1902 if not 2 < len(word) < 26:
1903 continue
1904 ndx = index.find(word=word)
1905 if ndx < 0:
1906 return {}
1907 if hits is None:
1908 hits = index[ndx].hits
1909 else:
1910 hits = hits.intersect(index[ndx].hits)
1911 if len(hits) == 0:
1912 return {}
1913 if hits is None:
1914 return {}
1915 rslt = {}
1916 ids = self.db.view('ids').remapwith(hits)
1917 tbls = self.datadb.view('tables')
1918 for i in range(len(ids)):
1919 hit = ids[i]
1920 if not hit.ignore:
1921 classname = tbls[hit.tblid].name
1922 nodeid = str(hit.nodeid)
1923 property = self._getpropname(classname, hit.propid)
1924 rslt[i] = (classname, nodeid, property)
1925 return rslt
1927 def save_index(self):
1928 if self.changed:
1929 self.db.commit()
1930 self.changed = 0
1932 def rollback(self):
1933 if self.changed:
1934 self.db.rollback()
1935 self.db = metakit.storage(self.path, 1)
1936 self.changed = 0