1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: back_anydbm.py,v 1.47 2002-07-14 23:18:20 richard Exp $
19 '''
20 This module defines a backend that saves the hyperdatabase in a database
21 chosen by anydbm. It is guaranteed to always be available in python
22 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
23 serious bugs, and is not available)
24 '''
26 import whichdb, anydbm, os, marshal, re, weakref, string, copy
27 from roundup import hyperdb, date, password, roundupdb
28 from blobfiles import FileStorage
29 from roundup.indexer import Indexer
30 from locking import acquire_lock, release_lock
31 from roundup.hyperdb import String, Password, Date, Interval, Link, \
32 Multilink, DatabaseError
34 #
35 # Now the database
36 #
37 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
38 """A database for storing records containing flexible data types.
40 Transaction stuff TODO:
41 . check the timestamp of the class file and nuke the cache if it's
42 modified. Do some sort of conflict checking on the dirty stuff.
43 . perhaps detect write collisions (related to above)?
45 """
46 def __init__(self, config, journaltag=None):
47 """Open a hyperdatabase given a specifier to some storage.
49 The 'storagelocator' is obtained from config.DATABASE.
50 The meaning of 'storagelocator' depends on the particular
51 implementation of the hyperdatabase. It could be a file name,
52 a directory path, a socket descriptor for a connection to a
53 database over the network, etc.
55 The 'journaltag' is a token that will be attached to the journal
56 entries for any edits done on the database. If 'journaltag' is
57 None, the database is opened in read-only mode: the Class.create(),
58 Class.set(), and Class.retire() methods are disabled.
59 """
60 self.config, self.journaltag = config, journaltag
61 self.dir = config.DATABASE
62 self.classes = {}
63 self.cache = {} # cache of nodes loaded or created
64 self.dirtynodes = {} # keep track of the dirty nodes by class
65 self.newnodes = {} # keep track of the new nodes by class
66 self.transactions = []
67 self.indexer = Indexer(self.dir)
68 # ensure files are group readable and writable
69 os.umask(0002)
71 def post_init(self):
72 """Called once the schema initialisation has finished."""
73 # reindex the db if necessary
74 if self.indexer.should_reindex():
75 self.reindex()
77 def reindex(self):
78 for klass in self.classes.values():
79 for nodeid in klass.list():
80 klass.index(nodeid)
81 self.indexer.save_index()
83 def __repr__(self):
84 return '<back_anydbm instance at %x>'%id(self)
86 #
87 # Classes
88 #
89 def __getattr__(self, classname):
90 """A convenient way of calling self.getclass(classname)."""
91 if self.classes.has_key(classname):
92 if __debug__:
93 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
94 return self.classes[classname]
95 raise AttributeError, classname
97 def addclass(self, cl):
98 if __debug__:
99 print >>hyperdb.DEBUG, 'addclass', (self, cl)
100 cn = cl.classname
101 if self.classes.has_key(cn):
102 raise ValueError, cn
103 self.classes[cn] = cl
105 def getclasses(self):
106 """Return a list of the names of all existing classes."""
107 if __debug__:
108 print >>hyperdb.DEBUG, 'getclasses', (self,)
109 l = self.classes.keys()
110 l.sort()
111 return l
113 def getclass(self, classname):
114 """Get the Class object representing a particular class.
116 If 'classname' is not a valid class name, a KeyError is raised.
117 """
118 if __debug__:
119 print >>hyperdb.DEBUG, 'getclass', (self, classname)
120 return self.classes[classname]
122 #
123 # Class DBs
124 #
125 def clear(self):
126 '''Delete all database contents
127 '''
128 if __debug__:
129 print >>hyperdb.DEBUG, 'clear', (self,)
130 for cn in self.classes.keys():
131 for dummy in 'nodes', 'journals':
132 path = os.path.join(self.dir, 'journals.%s'%cn)
133 if os.path.exists(path):
134 os.remove(path)
135 elif os.path.exists(path+'.db'): # dbm appends .db
136 os.remove(path+'.db')
138 def getclassdb(self, classname, mode='r'):
139 ''' grab a connection to the class db that will be used for
140 multiple actions
141 '''
142 if __debug__:
143 print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode)
144 return self._opendb('nodes.%s'%classname, mode)
146 def determine_db_type(self, path):
147 ''' determine which DB wrote the class file
148 '''
149 db_type = ''
150 if os.path.exists(path):
151 db_type = whichdb.whichdb(path)
152 if not db_type:
153 raise hyperdb.DatabaseError, "Couldn't identify database type"
154 elif os.path.exists(path+'.db'):
155 # if the path ends in '.db', it's a dbm database, whether
156 # anydbm says it's dbhash or not!
157 db_type = 'dbm'
158 return db_type
160 def _opendb(self, name, mode):
161 '''Low-level database opener that gets around anydbm/dbm
162 eccentricities.
163 '''
164 if __debug__:
165 print >>hyperdb.DEBUG, '_opendb', (self, name, mode)
167 # figure the class db type
168 path = os.path.join(os.getcwd(), self.dir, name)
169 db_type = self.determine_db_type(path)
171 # new database? let anydbm pick the best dbm
172 if not db_type:
173 if __debug__:
174 print >>hyperdb.DEBUG, "_opendb anydbm.open(%r, 'n')"%path
175 return anydbm.open(path, 'n')
177 # open the database with the correct module
178 try:
179 dbm = __import__(db_type)
180 except ImportError:
181 raise hyperdb.DatabaseError, \
182 "Couldn't open database - the required module '%s'"\
183 " is not available"%db_type
184 if __debug__:
185 print >>hyperdb.DEBUG, "_opendb %r.open(%r, %r)"%(db_type, path,
186 mode)
187 return dbm.open(path, mode)
189 def _lockdb(self, name):
190 ''' Lock a database file
191 '''
192 path = os.path.join(os.getcwd(), self.dir, '%s.lock'%name)
193 return acquire_lock(path)
195 #
196 # Node IDs
197 #
198 def newid(self, classname):
199 ''' Generate a new id for the given class
200 '''
201 # open the ids DB - create if if doesn't exist
202 lock = self._lockdb('_ids')
203 db = self._opendb('_ids', 'c')
204 if db.has_key(classname):
205 newid = db[classname] = str(int(db[classname]) + 1)
206 else:
207 # the count() bit is transitional - older dbs won't start at 1
208 newid = str(self.getclass(classname).count()+1)
209 db[classname] = newid
210 db.close()
211 release_lock(lock)
212 return newid
214 #
215 # Nodes
216 #
217 def addnode(self, classname, nodeid, node):
218 ''' add the specified node to its class's db
219 '''
220 if __debug__:
221 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
222 self.newnodes.setdefault(classname, {})[nodeid] = 1
223 self.cache.setdefault(classname, {})[nodeid] = node
224 self.savenode(classname, nodeid, node)
226 def setnode(self, classname, nodeid, node):
227 ''' change the specified node
228 '''
229 if __debug__:
230 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
231 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
233 # can't set without having already loaded the node
234 self.cache[classname][nodeid] = node
235 self.savenode(classname, nodeid, node)
237 def savenode(self, classname, nodeid, node):
238 ''' perform the saving of data specified by the set/addnode
239 '''
240 if __debug__:
241 print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node)
242 self.transactions.append((self._doSaveNode, (classname, nodeid, node)))
244 def getnode(self, classname, nodeid, db=None, cache=1):
245 ''' get a node from the database
246 '''
247 if __debug__:
248 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid, db)
249 if cache:
250 # try the cache
251 cache_dict = self.cache.setdefault(classname, {})
252 if cache_dict.has_key(nodeid):
253 if __debug__:
254 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
255 nodeid)
256 return cache_dict[nodeid]
258 if __debug__:
259 print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid)
261 # get from the database and save in the cache
262 if db is None:
263 db = self.getclassdb(classname)
264 if not db.has_key(nodeid):
265 raise IndexError, "no such %s %s"%(classname, nodeid)
267 # decode
268 res = marshal.loads(db[nodeid])
270 # reverse the serialisation
271 res = self.unserialise(classname, res)
273 # store off in the cache dict
274 if cache:
275 cache_dict[nodeid] = res
277 return res
279 def serialise(self, classname, node):
280 '''Copy the node contents, converting non-marshallable data into
281 marshallable data.
282 '''
283 if __debug__:
284 print >>hyperdb.DEBUG, 'serialise', classname, node
285 properties = self.getclass(classname).getprops()
286 d = {}
287 for k, v in node.items():
288 # if the property doesn't exist, or is the "retired" flag then
289 # it won't be in the properties dict
290 if not properties.has_key(k):
291 d[k] = v
292 continue
294 # get the property spec
295 prop = properties[k]
297 if isinstance(prop, Password):
298 d[k] = str(v)
299 elif isinstance(prop, Date) and v is not None:
300 d[k] = v.get_tuple()
301 elif isinstance(prop, Interval) and v is not None:
302 d[k] = v.get_tuple()
303 else:
304 d[k] = v
305 return d
307 def unserialise(self, classname, node):
308 '''Decode the marshalled node data
309 '''
310 if __debug__:
311 print >>hyperdb.DEBUG, 'unserialise', classname, node
312 properties = self.getclass(classname).getprops()
313 d = {}
314 for k, v in node.items():
315 # if the property doesn't exist, or is the "retired" flag then
316 # it won't be in the properties dict
317 if not properties.has_key(k):
318 d[k] = v
319 continue
321 # get the property spec
322 prop = properties[k]
324 if isinstance(prop, Date) and v is not None:
325 d[k] = date.Date(v)
326 elif isinstance(prop, Interval) and v is not None:
327 d[k] = date.Interval(v)
328 elif isinstance(prop, Password):
329 p = password.Password()
330 p.unpack(v)
331 d[k] = p
332 else:
333 d[k] = v
334 return d
336 def hasnode(self, classname, nodeid, db=None):
337 ''' determine if the database has a given node
338 '''
339 if __debug__:
340 print >>hyperdb.DEBUG, 'hasnode', (self, classname, nodeid, db)
342 # try the cache
343 cache = self.cache.setdefault(classname, {})
344 if cache.has_key(nodeid):
345 if __debug__:
346 print >>hyperdb.TRACE, 'has %s %s cached'%(classname, nodeid)
347 return 1
348 if __debug__:
349 print >>hyperdb.TRACE, 'has %s %s'%(classname, nodeid)
351 # not in the cache - check the database
352 if db is None:
353 db = self.getclassdb(classname)
354 res = db.has_key(nodeid)
355 return res
357 def countnodes(self, classname, db=None):
358 if __debug__:
359 print >>hyperdb.DEBUG, 'countnodes', (self, classname, db)
360 # include the new nodes not saved to the DB yet
361 count = len(self.newnodes.get(classname, {}))
363 # and count those in the DB
364 if db is None:
365 db = self.getclassdb(classname)
366 count = count + len(db.keys())
367 return count
369 def getnodeids(self, classname, db=None):
370 if __debug__:
371 print >>hyperdb.DEBUG, 'getnodeids', (self, classname, db)
372 # start off with the new nodes
373 res = self.newnodes.get(classname, {}).keys()
375 if db is None:
376 db = self.getclassdb(classname)
377 res = res + db.keys()
378 return res
381 #
382 # Files - special node properties
383 # inherited from FileStorage
385 #
386 # Journal
387 #
388 def addjournal(self, classname, nodeid, action, params):
389 ''' Journal the Action
390 'action' may be:
392 'create' or 'set' -- 'params' is a dictionary of property values
393 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
394 'retire' -- 'params' is None
395 '''
396 if __debug__:
397 print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid,
398 action, params)
399 self.transactions.append((self._doSaveJournal, (classname, nodeid,
400 action, params)))
402 def getjournal(self, classname, nodeid):
403 ''' get the journal for id
404 '''
405 if __debug__:
406 print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid)
407 # attempt to open the journal - in some rare cases, the journal may
408 # not exist
409 try:
410 db = self._opendb('journals.%s'%classname, 'r')
411 except anydbm.error, error:
412 if str(error) == "need 'c' or 'n' flag to open new db": return []
413 elif error.args[0] != 2: raise
414 return []
415 try:
416 journal = marshal.loads(db[nodeid])
417 except KeyError:
418 db.close()
419 raise KeyError, 'no such %s %s'%(classname, nodeid)
420 db.close()
421 res = []
422 for entry in journal:
423 (nodeid, date_stamp, user, action, params) = entry
424 date_obj = date.Date(date_stamp)
425 res.append((nodeid, date_obj, user, action, params))
426 return res
428 def pack(self, pack_before):
429 ''' delete all journal entries before 'pack_before' '''
430 if __debug__:
431 print >>hyperdb.DEBUG, 'packjournal', (self, pack_before)
433 pack_before = pack_before.get_tuple()
435 classes = self.getclasses()
437 # figure the class db type
439 for classname in classes:
440 db_name = 'journals.%s'%classname
441 path = os.path.join(os.getcwd(), self.dir, classname)
442 db_type = self.determine_db_type(path)
443 db = self._opendb(db_name, 'w')
445 for key in db.keys():
446 journal = marshal.loads(db[key])
447 l = []
448 last_set_entry = None
449 for entry in journal:
450 (nodeid, date_stamp, self.journaltag, action,
451 params) = entry
452 if date_stamp > pack_before or action == 'create':
453 l.append(entry)
454 elif action == 'set':
455 # grab the last set entry to keep information on
456 # activity
457 last_set_entry = entry
458 if last_set_entry:
459 date_stamp = last_set_entry[1]
460 # if the last set entry was made after the pack date
461 # then it is already in the list
462 if date_stamp < pack_before:
463 l.append(last_set_entry)
464 db[key] = marshal.dumps(l)
465 if db_type == 'gdbm':
466 db.reorganize()
467 db.close()
470 #
471 # Basic transaction support
472 #
473 def commit(self):
474 ''' Commit the current transactions.
475 '''
476 if __debug__:
477 print >>hyperdb.DEBUG, 'commit', (self,)
478 # TODO: lock the DB
480 # keep a handle to all the database files opened
481 self.databases = {}
483 # now, do all the transactions
484 reindex = {}
485 for method, args in self.transactions:
486 reindex[method(*args)] = 1
488 # now close all the database files
489 for db in self.databases.values():
490 db.close()
491 del self.databases
492 # TODO: unlock the DB
494 # reindex the nodes that request it
495 for classname, nodeid in filter(None, reindex.keys()):
496 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
497 self.getclass(classname).index(nodeid)
499 # save the indexer state
500 self.indexer.save_index()
502 # all transactions committed, back to normal
503 self.cache = {}
504 self.dirtynodes = {}
505 self.newnodes = {}
506 self.transactions = []
508 def _doSaveNode(self, classname, nodeid, node):
509 if __debug__:
510 print >>hyperdb.DEBUG, '_doSaveNode', (self, classname, nodeid,
511 node)
513 # get the database handle
514 db_name = 'nodes.%s'%classname
515 if self.databases.has_key(db_name):
516 db = self.databases[db_name]
517 else:
518 db = self.databases[db_name] = self.getclassdb(classname, 'c')
520 # now save the marshalled data
521 db[nodeid] = marshal.dumps(self.serialise(classname, node))
523 # return the classname, nodeid so we reindex this content
524 return (classname, nodeid)
526 def _doSaveJournal(self, classname, nodeid, action, params):
527 # serialise first
528 if action in ('set', 'create'):
529 params = self.serialise(classname, params)
531 # create the journal entry
532 entry = (nodeid, date.Date().get_tuple(), self.journaltag, action,
533 params)
535 if __debug__:
536 print >>hyperdb.DEBUG, '_doSaveJournal', entry
538 # get the database handle
539 db_name = 'journals.%s'%classname
540 if self.databases.has_key(db_name):
541 db = self.databases[db_name]
542 else:
543 db = self.databases[db_name] = self._opendb(db_name, 'c')
545 # now insert the journal entry
546 if db.has_key(nodeid):
547 # append to existing
548 s = db[nodeid]
549 l = marshal.loads(s)
550 l.append(entry)
551 else:
552 l = [entry]
554 db[nodeid] = marshal.dumps(l)
556 def rollback(self):
557 ''' Reverse all actions from the current transaction.
558 '''
559 if __debug__:
560 print >>hyperdb.DEBUG, 'rollback', (self, )
561 for method, args in self.transactions:
562 # delete temporary files
563 if method == self._doStoreFile:
564 self._rollbackStoreFile(*args)
565 self.cache = {}
566 self.dirtynodes = {}
567 self.newnodes = {}
568 self.transactions = []
570 _marker = []
571 class Class(hyperdb.Class):
572 """The handle to a particular class of nodes in a hyperdatabase."""
574 def __init__(self, db, classname, **properties):
575 """Create a new class with a given name and property specification.
577 'classname' must not collide with the name of an existing class,
578 or a ValueError is raised. The keyword arguments in 'properties'
579 must map names to property objects, or a TypeError is raised.
580 """
581 if (properties.has_key('creation') or properties.has_key('activity')
582 or properties.has_key('creator')):
583 raise ValueError, '"creation", "activity" and "creator" are '\
584 'reserved'
586 self.classname = classname
587 self.properties = properties
588 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
589 self.key = ''
591 # should we journal changes (default yes)
592 self.do_journal = 1
594 # do the db-related init stuff
595 db.addclass(self)
597 self.auditors = {'create': [], 'set': [], 'retire': []}
598 self.reactors = {'create': [], 'set': [], 'retire': []}
600 def enableJournalling(self):
601 '''Turn journalling on for this class
602 '''
603 self.do_journal = 1
605 def disableJournalling(self):
606 '''Turn journalling off for this class
607 '''
608 self.do_journal = 0
610 # Editing nodes:
612 def create(self, **propvalues):
613 """Create a new node of this class and return its id.
615 The keyword arguments in 'propvalues' map property names to values.
617 The values of arguments must be acceptable for the types of their
618 corresponding properties or a TypeError is raised.
620 If this class has a key property, it must be present and its value
621 must not collide with other key strings or a ValueError is raised.
623 Any other properties on this class that are missing from the
624 'propvalues' dictionary are set to None.
626 If an id in a link or multilink property does not refer to a valid
627 node, an IndexError is raised.
629 These operations trigger detectors and can be vetoed. Attempts
630 to modify the "creation" or "activity" properties cause a KeyError.
631 """
632 if propvalues.has_key('id'):
633 raise KeyError, '"id" is reserved'
635 if self.db.journaltag is None:
636 raise DatabaseError, 'Database open read-only'
638 if propvalues.has_key('creation') or propvalues.has_key('activity'):
639 raise KeyError, '"creation" and "activity" are reserved'
641 self.fireAuditors('create', None, propvalues)
643 # new node's id
644 newid = self.db.newid(self.classname)
646 # validate propvalues
647 num_re = re.compile('^\d+$')
648 for key, value in propvalues.items():
649 if key == self.key:
650 try:
651 self.lookup(value)
652 except KeyError:
653 pass
654 else:
655 raise ValueError, 'node with key "%s" exists'%value
657 # try to handle this property
658 try:
659 prop = self.properties[key]
660 except KeyError:
661 raise KeyError, '"%s" has no property "%s"'%(self.classname,
662 key)
664 if isinstance(prop, Link):
665 if type(value) != type(''):
666 raise ValueError, 'link value must be String'
667 link_class = self.properties[key].classname
668 # if it isn't a number, it's a key
669 if not num_re.match(value):
670 try:
671 value = self.db.classes[link_class].lookup(value)
672 except (TypeError, KeyError):
673 raise IndexError, 'new property "%s": %s not a %s'%(
674 key, value, link_class)
675 elif not self.db.hasnode(link_class, value):
676 raise IndexError, '%s has no node %s'%(link_class, value)
678 # save off the value
679 propvalues[key] = value
681 # register the link with the newly linked node
682 if self.do_journal and self.properties[key].do_journal:
683 self.db.addjournal(link_class, value, 'link',
684 (self.classname, newid, key))
686 elif isinstance(prop, Multilink):
687 if type(value) != type([]):
688 raise TypeError, 'new property "%s" not a list of ids'%key
690 # clean up and validate the list of links
691 link_class = self.properties[key].classname
692 l = []
693 for entry in value:
694 if type(entry) != type(''):
695 raise ValueError, '"%s" link value (%s) must be '\
696 'String'%(key, value)
697 # if it isn't a number, it's a key
698 if not num_re.match(entry):
699 try:
700 entry = self.db.classes[link_class].lookup(entry)
701 except (TypeError, KeyError):
702 raise IndexError, 'new property "%s": %s not a %s'%(
703 key, entry, self.properties[key].classname)
704 l.append(entry)
705 value = l
706 propvalues[key] = value
708 # handle additions
709 for id in value:
710 if not self.db.hasnode(link_class, id):
711 raise IndexError, '%s has no node %s'%(link_class, id)
712 # register the link with the newly linked node
713 if self.do_journal and self.properties[key].do_journal:
714 self.db.addjournal(link_class, id, 'link',
715 (self.classname, newid, key))
717 elif isinstance(prop, String):
718 if type(value) != type(''):
719 raise TypeError, 'new property "%s" not a string'%key
721 elif isinstance(prop, Password):
722 if not isinstance(value, password.Password):
723 raise TypeError, 'new property "%s" not a Password'%key
725 elif isinstance(prop, Date):
726 if value is not None and not isinstance(value, date.Date):
727 raise TypeError, 'new property "%s" not a Date'%key
729 elif isinstance(prop, Interval):
730 if value is not None and not isinstance(value, date.Interval):
731 raise TypeError, 'new property "%s" not an Interval'%key
733 # make sure there's data where there needs to be
734 for key, prop in self.properties.items():
735 if propvalues.has_key(key):
736 continue
737 if key == self.key:
738 raise ValueError, 'key property "%s" is required'%key
739 if isinstance(prop, Multilink):
740 propvalues[key] = []
741 else:
742 propvalues[key] = None
744 # done
745 self.db.addnode(self.classname, newid, propvalues)
746 if self.do_journal:
747 self.db.addjournal(self.classname, newid, 'create', propvalues)
749 self.fireReactors('create', newid, None)
751 return newid
753 def get(self, nodeid, propname, default=_marker, cache=1):
754 """Get the value of a property on an existing node of this class.
756 'nodeid' must be the id of an existing node of this class or an
757 IndexError is raised. 'propname' must be the name of a property
758 of this class or a KeyError is raised.
760 'cache' indicates whether the transaction cache should be queried
761 for the node. If the node has been modified and you need to
762 determine what its values prior to modification are, you need to
763 set cache=0.
765 Attempts to get the "creation" or "activity" properties should
766 do the right thing.
767 """
768 if propname == 'id':
769 return nodeid
771 if propname == 'creation':
772 if not self.do_journal:
773 raise ValueError, 'Journalling is disabled for this class'
774 journal = self.db.getjournal(self.classname, nodeid)
775 if journal:
776 return self.db.getjournal(self.classname, nodeid)[0][1]
777 else:
778 # on the strange chance that there's no journal
779 return date.Date()
780 if propname == 'activity':
781 if not self.do_journal:
782 raise ValueError, 'Journalling is disabled for this class'
783 journal = self.db.getjournal(self.classname, nodeid)
784 if journal:
785 return self.db.getjournal(self.classname, nodeid)[-1][1]
786 else:
787 # on the strange chance that there's no journal
788 return date.Date()
789 if propname == 'creator':
790 if not self.do_journal:
791 raise ValueError, 'Journalling is disabled for this class'
792 journal = self.db.getjournal(self.classname, nodeid)
793 if journal:
794 name = self.db.getjournal(self.classname, nodeid)[0][2]
795 else:
796 return None
797 return self.db.user.lookup(name)
799 # get the property (raises KeyErorr if invalid)
800 prop = self.properties[propname]
802 # get the node's dict
803 d = self.db.getnode(self.classname, nodeid, cache=cache)
805 if not d.has_key(propname):
806 if default is _marker:
807 if isinstance(prop, Multilink):
808 return []
809 else:
810 return None
811 else:
812 return default
814 return d[propname]
816 # XXX not in spec
817 def getnode(self, nodeid, cache=1):
818 ''' Return a convenience wrapper for the node.
820 'nodeid' must be the id of an existing node of this class or an
821 IndexError is raised.
823 'cache' indicates whether the transaction cache should be queried
824 for the node. If the node has been modified and you need to
825 determine what its values prior to modification are, you need to
826 set cache=0.
827 '''
828 return Node(self, nodeid, cache=cache)
830 def set(self, nodeid, **propvalues):
831 """Modify a property on an existing node of this class.
833 'nodeid' must be the id of an existing node of this class or an
834 IndexError is raised.
836 Each key in 'propvalues' must be the name of a property of this
837 class or a KeyError is raised.
839 All values in 'propvalues' must be acceptable types for their
840 corresponding properties or a TypeError is raised.
842 If the value of the key property is set, it must not collide with
843 other key strings or a ValueError is raised.
845 If the value of a Link or Multilink property contains an invalid
846 node id, a ValueError is raised.
848 These operations trigger detectors and can be vetoed. Attempts
849 to modify the "creation" or "activity" properties cause a KeyError.
850 """
851 if not propvalues:
852 return
854 if propvalues.has_key('creation') or propvalues.has_key('activity'):
855 raise KeyError, '"creation" and "activity" are reserved'
857 if propvalues.has_key('id'):
858 raise KeyError, '"id" is reserved'
860 if self.db.journaltag is None:
861 raise DatabaseError, 'Database open read-only'
863 self.fireAuditors('set', nodeid, propvalues)
864 # Take a copy of the node dict so that the subsequent set
865 # operation doesn't modify the oldvalues structure.
866 try:
867 # try not using the cache initially
868 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid,
869 cache=0))
870 except IndexError:
871 # this will be needed if somone does a create() and set()
872 # with no intervening commit()
873 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
875 node = self.db.getnode(self.classname, nodeid)
876 if node.has_key(self.db.RETIRED_FLAG):
877 raise IndexError
878 num_re = re.compile('^\d+$')
880 # if the journal value is to be different, store it in here
881 journalvalues = {}
883 for propname, value in propvalues.items():
884 # check to make sure we're not duplicating an existing key
885 if propname == self.key and node[propname] != value:
886 try:
887 self.lookup(value)
888 except KeyError:
889 pass
890 else:
891 raise ValueError, 'node with key "%s" exists'%value
893 # this will raise the KeyError if the property isn't valid
894 # ... we don't use getprops() here because we only care about
895 # the writeable properties.
896 prop = self.properties[propname]
898 # if the value's the same as the existing value, no sense in
899 # doing anything
900 if node.has_key(propname) and value == node[propname]:
901 del propvalues[propname]
902 continue
904 # do stuff based on the prop type
905 if isinstance(prop, Link):
906 link_class = self.properties[propname].classname
907 # if it isn't a number, it's a key
908 if type(value) != type(''):
909 raise ValueError, 'link value must be String'
910 if not num_re.match(value):
911 try:
912 value = self.db.classes[link_class].lookup(value)
913 except (TypeError, KeyError):
914 raise IndexError, 'new property "%s": %s not a %s'%(
915 propname, value, self.properties[propname].classname)
917 if not self.db.hasnode(link_class, value):
918 raise IndexError, '%s has no node %s'%(link_class, value)
920 if self.do_journal and self.properties[propname].do_journal:
921 # register the unlink with the old linked node
922 if node[propname] is not None:
923 self.db.addjournal(link_class, node[propname], 'unlink',
924 (self.classname, nodeid, propname))
926 # register the link with the newly linked node
927 if value is not None:
928 self.db.addjournal(link_class, value, 'link',
929 (self.classname, nodeid, propname))
931 elif isinstance(prop, Multilink):
932 if type(value) != type([]):
933 raise TypeError, 'new property "%s" not a list of'\
934 ' ids'%propname
935 link_class = self.properties[propname].classname
936 l = []
937 for entry in value:
938 # if it isn't a number, it's a key
939 if type(entry) != type(''):
940 raise ValueError, 'new property "%s" link value ' \
941 'must be a string'%propname
942 if not num_re.match(entry):
943 try:
944 entry = self.db.classes[link_class].lookup(entry)
945 except (TypeError, KeyError):
946 raise IndexError, 'new property "%s": %s not a %s'%(
947 propname, entry,
948 self.properties[propname].classname)
949 l.append(entry)
950 value = l
951 propvalues[propname] = value
953 # figure the journal entry for this property
954 add = []
955 remove = []
957 # handle removals
958 if node.has_key(propname):
959 l = node[propname]
960 else:
961 l = []
962 for id in l[:]:
963 if id in value:
964 continue
965 # register the unlink with the old linked node
966 if self.do_journal and self.properties[propname].do_journal:
967 self.db.addjournal(link_class, id, 'unlink',
968 (self.classname, nodeid, propname))
969 l.remove(id)
970 remove.append(id)
972 # handle additions
973 for id in value:
974 if not self.db.hasnode(link_class, id):
975 raise IndexError, '%s has no node %s'%(link_class, id)
976 if id in l:
977 continue
978 # register the link with the newly linked node
979 if self.do_journal and self.properties[propname].do_journal:
980 self.db.addjournal(link_class, id, 'link',
981 (self.classname, nodeid, propname))
982 l.append(id)
983 add.append(id)
985 # figure the journal entry
986 l = []
987 if add:
988 l.append(('add', add))
989 if remove:
990 l.append(('remove', remove))
991 if l:
992 journalvalues[propname] = tuple(l)
994 elif isinstance(prop, String):
995 if value is not None and type(value) != type(''):
996 raise TypeError, 'new property "%s" not a string'%propname
998 elif isinstance(prop, Password):
999 if not isinstance(value, password.Password):
1000 raise TypeError, 'new property "%s" not a Password'%propname
1001 propvalues[propname] = value
1003 elif value is not None and isinstance(prop, Date):
1004 if not isinstance(value, date.Date):
1005 raise TypeError, 'new property "%s" not a Date'% propname
1006 propvalues[propname] = value
1008 elif value is not None and isinstance(prop, Interval):
1009 if not isinstance(value, date.Interval):
1010 raise TypeError, 'new property "%s" not an '\
1011 'Interval'%propname
1012 propvalues[propname] = value
1014 node[propname] = value
1016 # nothing to do?
1017 if not propvalues:
1018 return
1020 # do the set, and journal it
1021 self.db.setnode(self.classname, nodeid, node)
1023 if self.do_journal:
1024 propvalues.update(journalvalues)
1025 self.db.addjournal(self.classname, nodeid, 'set', propvalues)
1027 self.fireReactors('set', nodeid, oldvalues)
1029 def retire(self, nodeid):
1030 """Retire a node.
1032 The properties on the node remain available from the get() method,
1033 and the node's id is never reused.
1035 Retired nodes are not returned by the find(), list(), or lookup()
1036 methods, and other nodes may reuse the values of their key properties.
1038 These operations trigger detectors and can be vetoed. Attempts
1039 to modify the "creation" or "activity" properties cause a KeyError.
1040 """
1041 if self.db.journaltag is None:
1042 raise DatabaseError, 'Database open read-only'
1044 self.fireAuditors('retire', nodeid, None)
1046 node = self.db.getnode(self.classname, nodeid)
1047 node[self.db.RETIRED_FLAG] = 1
1048 self.db.setnode(self.classname, nodeid, node)
1049 if self.do_journal:
1050 self.db.addjournal(self.classname, nodeid, 'retired', None)
1052 self.fireReactors('retire', nodeid, None)
1054 def history(self, nodeid):
1055 """Retrieve the journal of edits on a particular node.
1057 'nodeid' must be the id of an existing node of this class or an
1058 IndexError is raised.
1060 The returned list contains tuples of the form
1062 (date, tag, action, params)
1064 'date' is a Timestamp object specifying the time of the change and
1065 'tag' is the journaltag specified when the database was opened.
1066 """
1067 if not self.do_journal:
1068 raise ValueError, 'Journalling is disabled for this class'
1069 return self.db.getjournal(self.classname, nodeid)
1071 # Locating nodes:
1072 def hasnode(self, nodeid):
1073 '''Determine if the given nodeid actually exists
1074 '''
1075 return self.db.hasnode(self.classname, nodeid)
1077 def setkey(self, propname):
1078 """Select a String property of this class to be the key property.
1080 'propname' must be the name of a String property of this class or
1081 None, or a TypeError is raised. The values of the key property on
1082 all existing nodes must be unique or a ValueError is raised. If the
1083 property doesn't exist, KeyError is raised.
1084 """
1085 prop = self.getprops()[propname]
1086 if not isinstance(prop, String):
1087 raise TypeError, 'key properties must be String'
1088 self.key = propname
1090 def getkey(self):
1091 """Return the name of the key property for this class or None."""
1092 return self.key
1094 def labelprop(self, default_to_id=0):
1095 ''' Return the property name for a label for the given node.
1097 This method attempts to generate a consistent label for the node.
1098 It tries the following in order:
1099 1. key property
1100 2. "name" property
1101 3. "title" property
1102 4. first property from the sorted property name list
1103 '''
1104 k = self.getkey()
1105 if k:
1106 return k
1107 props = self.getprops()
1108 if props.has_key('name'):
1109 return 'name'
1110 elif props.has_key('title'):
1111 return 'title'
1112 if default_to_id:
1113 return 'id'
1114 props = props.keys()
1115 props.sort()
1116 return props[0]
1118 # TODO: set up a separate index db file for this? profile?
1119 def lookup(self, keyvalue):
1120 """Locate a particular node by its key property and return its id.
1122 If this class has no key property, a TypeError is raised. If the
1123 'keyvalue' matches one of the values for the key property among
1124 the nodes in this class, the matching node's id is returned;
1125 otherwise a KeyError is raised.
1126 """
1127 cldb = self.db.getclassdb(self.classname)
1128 try:
1129 for nodeid in self.db.getnodeids(self.classname, cldb):
1130 node = self.db.getnode(self.classname, nodeid, cldb)
1131 if node.has_key(self.db.RETIRED_FLAG):
1132 continue
1133 if node[self.key] == keyvalue:
1134 cldb.close()
1135 return nodeid
1136 finally:
1137 cldb.close()
1138 raise KeyError, keyvalue
1140 # XXX: change from spec - allows multiple props to match
1141 def find(self, **propspec):
1142 """Get the ids of nodes in this class which link to the given nodes.
1144 'propspec' consists of keyword args propname={nodeid:1,}
1145 'propname' must be the name of a property in this class, or a
1146 KeyError is raised. That property must be a Link or Multilink
1147 property, or a TypeError is raised.
1149 Any node in this class whose 'propname' property links to any of the
1150 nodeids will be returned. Used by the full text indexing, which knows
1151 that "foo" occurs in msg1, msg3 and file7, so we have hits on these issues:
1152 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1153 """
1154 propspec = propspec.items()
1155 for propname, nodeids in propspec:
1156 # check the prop is OK
1157 prop = self.properties[propname]
1158 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1159 raise TypeError, "'%s' not a Link/Multilink property"%propname
1160 #XXX edit is expensive and of questionable use
1161 #for nodeid in nodeids:
1162 # if not self.db.hasnode(prop.classname, nodeid):
1163 # raise ValueError, '%s has no node %s'%(prop.classname, nodeid)
1165 # ok, now do the find
1166 cldb = self.db.getclassdb(self.classname)
1167 l = []
1168 try:
1169 for id in self.db.getnodeids(self.classname, db=cldb):
1170 node = self.db.getnode(self.classname, id, db=cldb)
1171 if node.has_key(self.db.RETIRED_FLAG):
1172 continue
1173 for propname, nodeids in propspec:
1174 # can't test if the node doesn't have this property
1175 if not node.has_key(propname):
1176 continue
1177 if type(nodeids) is type(''):
1178 nodeids = {nodeids:1}
1179 prop = self.properties[propname]
1180 value = node[propname]
1181 if isinstance(prop, Link) and nodeids.has_key(value):
1182 l.append(id)
1183 break
1184 elif isinstance(prop, Multilink):
1185 hit = 0
1186 for v in value:
1187 if nodeids.has_key(v):
1188 l.append(id)
1189 hit = 1
1190 break
1191 if hit:
1192 break
1193 finally:
1194 cldb.close()
1195 return l
1197 def stringFind(self, **requirements):
1198 """Locate a particular node by matching a set of its String
1199 properties in a caseless search.
1201 If the property is not a String property, a TypeError is raised.
1203 The return is a list of the id of all nodes that match.
1204 """
1205 for propname in requirements.keys():
1206 prop = self.properties[propname]
1207 if isinstance(not prop, String):
1208 raise TypeError, "'%s' not a String property"%propname
1209 requirements[propname] = requirements[propname].lower()
1210 l = []
1211 cldb = self.db.getclassdb(self.classname)
1212 try:
1213 for nodeid in self.db.getnodeids(self.classname, cldb):
1214 node = self.db.getnode(self.classname, nodeid, cldb)
1215 if node.has_key(self.db.RETIRED_FLAG):
1216 continue
1217 for key, value in requirements.items():
1218 if node[key] and node[key].lower() != value:
1219 break
1220 else:
1221 l.append(nodeid)
1222 finally:
1223 cldb.close()
1224 return l
1226 def list(self):
1227 """Return a list of the ids of the active nodes in this class."""
1228 l = []
1229 cn = self.classname
1230 cldb = self.db.getclassdb(cn)
1231 try:
1232 for nodeid in self.db.getnodeids(cn, cldb):
1233 node = self.db.getnode(cn, nodeid, cldb)
1234 if node.has_key(self.db.RETIRED_FLAG):
1235 continue
1236 l.append(nodeid)
1237 finally:
1238 cldb.close()
1239 l.sort()
1240 return l
1242 # XXX not in spec
1243 def filter(self, search_matches, filterspec, sort, group,
1244 num_re = re.compile('^\d+$')):
1245 ''' Return a list of the ids of the active nodes in this class that
1246 match the 'filter' spec, sorted by the group spec and then the
1247 sort spec
1248 '''
1249 cn = self.classname
1251 # optimise filterspec
1252 l = []
1253 props = self.getprops()
1254 for k, v in filterspec.items():
1255 propclass = props[k]
1256 if isinstance(propclass, Link):
1257 if type(v) is not type([]):
1258 v = [v]
1259 # replace key values with node ids
1260 u = []
1261 link_class = self.db.classes[propclass.classname]
1262 for entry in v:
1263 if entry == '-1': entry = None
1264 elif not num_re.match(entry):
1265 try:
1266 entry = link_class.lookup(entry)
1267 except (TypeError,KeyError):
1268 raise ValueError, 'property "%s": %s not a %s'%(
1269 k, entry, self.properties[k].classname)
1270 u.append(entry)
1272 l.append((0, k, u))
1273 elif isinstance(propclass, Multilink):
1274 if type(v) is not type([]):
1275 v = [v]
1276 # replace key values with node ids
1277 u = []
1278 link_class = self.db.classes[propclass.classname]
1279 for entry in v:
1280 if not num_re.match(entry):
1281 try:
1282 entry = link_class.lookup(entry)
1283 except (TypeError,KeyError):
1284 raise ValueError, 'new property "%s": %s not a %s'%(
1285 k, entry, self.properties[k].classname)
1286 u.append(entry)
1287 l.append((1, k, u))
1288 elif isinstance(propclass, String):
1289 # simple glob searching
1290 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1291 v = v.replace('?', '.')
1292 v = v.replace('*', '.*?')
1293 l.append((2, k, re.compile(v, re.I)))
1294 else:
1295 l.append((6, k, v))
1296 filterspec = l
1298 # now, find all the nodes that are active and pass filtering
1299 l = []
1300 cldb = self.db.getclassdb(cn)
1301 try:
1302 for nodeid in self.db.getnodeids(cn, cldb):
1303 node = self.db.getnode(cn, nodeid, cldb)
1304 if node.has_key(self.db.RETIRED_FLAG):
1305 continue
1306 # apply filter
1307 for t, k, v in filterspec:
1308 # this node doesn't have this property, so reject it
1309 if not node.has_key(k): break
1311 if t == 0 and node[k] not in v:
1312 # link - if this node'd property doesn't appear in the
1313 # filterspec's nodeid list, skip it
1314 break
1315 elif t == 1:
1316 # multilink - if any of the nodeids required by the
1317 # filterspec aren't in this node's property, then skip
1318 # it
1319 for value in v:
1320 if value not in node[k]:
1321 break
1322 else:
1323 continue
1324 break
1325 elif t == 2 and (node[k] is None or not v.search(node[k])):
1326 # RE search
1327 break
1328 elif t == 6 and node[k] != v:
1329 # straight value comparison for the other types
1330 break
1331 else:
1332 l.append((nodeid, node))
1333 finally:
1334 cldb.close()
1335 l.sort()
1337 # filter based on full text search
1338 if search_matches is not None:
1339 k = []
1340 l_debug = []
1341 for v in l:
1342 l_debug.append(v[0])
1343 if search_matches.has_key(v[0]):
1344 k.append(v)
1345 l = k
1347 # optimise sort
1348 m = []
1349 for entry in sort:
1350 if entry[0] != '-':
1351 m.append(('+', entry))
1352 else:
1353 m.append((entry[0], entry[1:]))
1354 sort = m
1356 # optimise group
1357 m = []
1358 for entry in group:
1359 if entry[0] != '-':
1360 m.append(('+', entry))
1361 else:
1362 m.append((entry[0], entry[1:]))
1363 group = m
1364 # now, sort the result
1365 def sortfun(a, b, sort=sort, group=group, properties=self.getprops(),
1366 db = self.db, cl=self):
1367 a_id, an = a
1368 b_id, bn = b
1369 # sort by group and then sort
1370 for list in group, sort:
1371 for dir, prop in list:
1372 # sorting is class-specific
1373 propclass = properties[prop]
1375 # handle the properties that might be "faked"
1376 # also, handle possible missing properties
1377 try:
1378 if not an.has_key(prop):
1379 an[prop] = cl.get(a_id, prop)
1380 av = an[prop]
1381 except KeyError:
1382 # the node doesn't have a value for this property
1383 if isinstance(propclass, Multilink): av = []
1384 else: av = ''
1385 try:
1386 if not bn.has_key(prop):
1387 bn[prop] = cl.get(b_id, prop)
1388 bv = bn[prop]
1389 except KeyError:
1390 # the node doesn't have a value for this property
1391 if isinstance(propclass, Multilink): bv = []
1392 else: bv = ''
1394 # String and Date values are sorted in the natural way
1395 if isinstance(propclass, String):
1396 # clean up the strings
1397 if av and av[0] in string.uppercase:
1398 av = an[prop] = av.lower()
1399 if bv and bv[0] in string.uppercase:
1400 bv = bn[prop] = bv.lower()
1401 if (isinstance(propclass, String) or
1402 isinstance(propclass, Date)):
1403 # it might be a string that's really an integer
1404 try:
1405 av = int(av)
1406 bv = int(bv)
1407 except:
1408 pass
1409 if dir == '+':
1410 r = cmp(av, bv)
1411 if r != 0: return r
1412 elif dir == '-':
1413 r = cmp(bv, av)
1414 if r != 0: return r
1416 # Link properties are sorted according to the value of
1417 # the "order" property on the linked nodes if it is
1418 # present; or otherwise on the key string of the linked
1419 # nodes; or finally on the node ids.
1420 elif isinstance(propclass, Link):
1421 link = db.classes[propclass.classname]
1422 if av is None and bv is not None: return -1
1423 if av is not None and bv is None: return 1
1424 if av is None and bv is None: continue
1425 if link.getprops().has_key('order'):
1426 if dir == '+':
1427 r = cmp(link.get(av, 'order'),
1428 link.get(bv, 'order'))
1429 if r != 0: return r
1430 elif dir == '-':
1431 r = cmp(link.get(bv, 'order'),
1432 link.get(av, 'order'))
1433 if r != 0: return r
1434 elif link.getkey():
1435 key = link.getkey()
1436 if dir == '+':
1437 r = cmp(link.get(av, key), link.get(bv, key))
1438 if r != 0: return r
1439 elif dir == '-':
1440 r = cmp(link.get(bv, key), link.get(av, key))
1441 if r != 0: return r
1442 else:
1443 if dir == '+':
1444 r = cmp(av, bv)
1445 if r != 0: return r
1446 elif dir == '-':
1447 r = cmp(bv, av)
1448 if r != 0: return r
1450 # Multilink properties are sorted according to how many
1451 # links are present.
1452 elif isinstance(propclass, Multilink):
1453 if dir == '+':
1454 r = cmp(len(av), len(bv))
1455 if r != 0: return r
1456 elif dir == '-':
1457 r = cmp(len(bv), len(av))
1458 if r != 0: return r
1459 # end for dir, prop in list:
1460 # end for list in sort, group:
1461 # if all else fails, compare the ids
1462 return cmp(a[0], b[0])
1464 l.sort(sortfun)
1465 return [i[0] for i in l]
1467 def count(self):
1468 """Get the number of nodes in this class.
1470 If the returned integer is 'numnodes', the ids of all the nodes
1471 in this class run from 1 to numnodes, and numnodes+1 will be the
1472 id of the next node to be created in this class.
1473 """
1474 return self.db.countnodes(self.classname)
1476 # Manipulating properties:
1478 def getprops(self, protected=1):
1479 """Return a dictionary mapping property names to property objects.
1480 If the "protected" flag is true, we include protected properties -
1481 those which may not be modified.
1483 In addition to the actual properties on the node, these
1484 methods provide the "creation" and "activity" properties. If the
1485 "protected" flag is true, we include protected properties - those
1486 which may not be modified.
1487 """
1488 d = self.properties.copy()
1489 if protected:
1490 d['id'] = String()
1491 d['creation'] = hyperdb.Date()
1492 d['activity'] = hyperdb.Date()
1493 d['creator'] = hyperdb.Link("user")
1494 return d
1496 def addprop(self, **properties):
1497 """Add properties to this class.
1499 The keyword arguments in 'properties' must map names to property
1500 objects, or a TypeError is raised. None of the keys in 'properties'
1501 may collide with the names of existing properties, or a ValueError
1502 is raised before any properties have been added.
1503 """
1504 for key in properties.keys():
1505 if self.properties.has_key(key):
1506 raise ValueError, key
1507 self.properties.update(properties)
1509 def index(self, nodeid):
1510 '''Add (or refresh) the node to search indexes
1511 '''
1512 # find all the String properties that have indexme
1513 for prop, propclass in self.getprops().items():
1514 if isinstance(propclass, String) and propclass.indexme:
1515 # and index them under (classname, nodeid, property)
1516 self.db.indexer.add_text((self.classname, nodeid, prop),
1517 str(self.get(nodeid, prop)))
1519 #
1520 # Detector interface
1521 #
1522 def audit(self, event, detector):
1523 """Register a detector
1524 """
1525 l = self.auditors[event]
1526 if detector not in l:
1527 self.auditors[event].append(detector)
1529 def fireAuditors(self, action, nodeid, newvalues):
1530 """Fire all registered auditors.
1531 """
1532 for audit in self.auditors[action]:
1533 audit(self.db, self, nodeid, newvalues)
1535 def react(self, event, detector):
1536 """Register a detector
1537 """
1538 l = self.reactors[event]
1539 if detector not in l:
1540 self.reactors[event].append(detector)
1542 def fireReactors(self, action, nodeid, oldvalues):
1543 """Fire all registered reactors.
1544 """
1545 for react in self.reactors[action]:
1546 react(self.db, self, nodeid, oldvalues)
1548 class FileClass(Class):
1549 '''This class defines a large chunk of data. To support this, it has a
1550 mandatory String property "content" which is typically saved off
1551 externally to the hyperdb.
1553 The default MIME type of this data is defined by the
1554 "default_mime_type" class attribute, which may be overridden by each
1555 node if the class defines a "type" String property.
1556 '''
1557 default_mime_type = 'text/plain'
1559 def create(self, **propvalues):
1560 ''' snaffle the file propvalue and store in a file
1561 '''
1562 content = propvalues['content']
1563 del propvalues['content']
1564 newid = Class.create(self, **propvalues)
1565 self.db.storefile(self.classname, newid, None, content)
1566 return newid
1568 def get(self, nodeid, propname, default=_marker, cache=1):
1569 ''' trap the content propname and get it from the file
1570 '''
1572 poss_msg = 'Possibly a access right configuration problem.'
1573 if propname == 'content':
1574 try:
1575 return self.db.getfile(self.classname, nodeid, None)
1576 except IOError, (strerror):
1577 # BUG: by catching this we donot see an error in the log.
1578 return 'ERROR reading file: %s%s\n%s\n%s'%(
1579 self.classname, nodeid, poss_msg, strerror)
1580 if default is not _marker:
1581 return Class.get(self, nodeid, propname, default, cache=cache)
1582 else:
1583 return Class.get(self, nodeid, propname, cache=cache)
1585 def getprops(self, protected=1):
1586 ''' In addition to the actual properties on the node, these methods
1587 provide the "content" property. If the "protected" flag is true,
1588 we include protected properties - those which may not be
1589 modified.
1590 '''
1591 d = Class.getprops(self, protected=protected).copy()
1592 if protected:
1593 d['content'] = hyperdb.String()
1594 return d
1596 def index(self, nodeid):
1597 ''' Index the node in the search index.
1599 We want to index the content in addition to the normal String
1600 property indexing.
1601 '''
1602 # perform normal indexing
1603 Class.index(self, nodeid)
1605 # get the content to index
1606 content = self.get(nodeid, 'content')
1608 # figure the mime type
1609 if self.properties.has_key('type'):
1610 mime_type = self.get(nodeid, 'type')
1611 else:
1612 mime_type = self.default_mime_type
1614 # and index!
1615 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
1616 mime_type)
1618 # XXX deviation from spec - was called ItemClass
1619 class IssueClass(Class, roundupdb.IssueClass):
1620 # Overridden methods:
1621 def __init__(self, db, classname, **properties):
1622 """The newly-created class automatically includes the "messages",
1623 "files", "nosy", and "superseder" properties. If the 'properties'
1624 dictionary attempts to specify any of these properties or a
1625 "creation" or "activity" property, a ValueError is raised.
1626 """
1627 if not properties.has_key('title'):
1628 properties['title'] = hyperdb.String(indexme='yes')
1629 if not properties.has_key('messages'):
1630 properties['messages'] = hyperdb.Multilink("msg")
1631 if not properties.has_key('files'):
1632 properties['files'] = hyperdb.Multilink("file")
1633 if not properties.has_key('nosy'):
1634 properties['nosy'] = hyperdb.Multilink("user")
1635 if not properties.has_key('superseder'):
1636 properties['superseder'] = hyperdb.Multilink(classname)
1637 Class.__init__(self, db, classname, **properties)
1639 #
1640 #$Log: not supported by cvs2svn $
1641 #Revision 1.46 2002/07/14 06:06:34 richard
1642 #Did some old TODOs
1643 #
1644 #Revision 1.45 2002/07/14 04:03:14 richard
1645 #Implemented a switch to disable journalling for a Class. CGI session
1646 #database now uses it.
1647 #
1648 #Revision 1.44 2002/07/14 02:05:53 richard
1649 #. all storage-specific code (ie. backend) is now implemented by the backends
1650 #
1651 #Revision 1.43 2002/07/10 06:30:30 richard
1652 #...except of course it's nice to use valid Python syntax
1653 #
1654 #Revision 1.42 2002/07/10 06:21:38 richard
1655 #Be extra safe
1656 #
1657 #Revision 1.41 2002/07/10 00:21:45 richard
1658 #explicit database closing
1659 #
1660 #Revision 1.40 2002/07/09 04:19:09 richard
1661 #Added reindex command to roundup-admin.
1662 #Fixed reindex on first access.
1663 #Also fixed reindexing of entries that change.
1664 #
1665 #Revision 1.39 2002/07/09 03:02:52 richard
1666 #More indexer work:
1667 #- all String properties may now be indexed too. Currently there's a bit of
1668 # "issue" specific code in the actual searching which needs to be
1669 # addressed. In a nutshell:
1670 # + pass 'indexme="yes"' as a String() property initialisation arg, eg:
1671 # file = FileClass(db, "file", name=String(), type=String(),
1672 # comment=String(indexme="yes"))
1673 # + the comment will then be indexed and be searchable, with the results
1674 # related back to the issue that the file is linked to
1675 #- as a result of this work, the FileClass has a default MIME type that may
1676 # be overridden in a subclass, or by the use of a "type" property as is
1677 # done in the default templates.
1678 #- the regeneration of the indexes (if necessary) is done once the schema is
1679 # set up in the dbinit.
1680 #
1681 #Revision 1.38 2002/07/08 06:58:15 richard
1682 #cleaned up the indexer code:
1683 # - it splits more words out (much simpler, faster splitter)
1684 # - removed code we'll never use (roundup.roundup_indexer has the full
1685 # implementation, and replaces roundup.indexer)
1686 # - only index text/plain and rfc822/message (ideas for other text formats to
1687 # index are welcome)
1688 # - added simple unit test for indexer. Needs more tests for regression.
1689 #
1690 #Revision 1.37 2002/06/20 23:52:35 richard
1691 #More informative error message
1692 #
1693 #Revision 1.36 2002/06/19 03:07:19 richard
1694 #Moved the file storage commit into blobfiles where it belongs.
1695 #
1696 #Revision 1.35 2002/05/25 07:16:24 rochecompaan
1697 #Merged search_indexing-branch with HEAD
1698 #
1699 #Revision 1.34 2002/05/15 06:21:21 richard
1700 # . node caching now works, and gives a small boost in performance
1701 #
1702 #As a part of this, I cleaned up the DEBUG output and implemented TRACE
1703 #output (HYPERDBTRACE='file to trace to') with checkpoints at the start of
1704 #CGI requests. Run roundup with python -O to skip all the DEBUG/TRACE stuff
1705 #(using if __debug__ which is compiled out with -O)
1706 #
1707 #Revision 1.33 2002/04/24 10:38:26 rochecompaan
1708 #All database files are now created group readable and writable.
1709 #
1710 #Revision 1.32 2002/04/15 23:25:15 richard
1711 #. node ids are now generated from a lockable store - no more race conditions
1712 #
1713 #We're using the portalocker code by Jonathan Feinberg that was contributed
1714 #to the ASPN Python cookbook. This gives us locking across Unix and Windows.
1715 #
1716 #Revision 1.31 2002/04/03 05:54:31 richard
1717 #Fixed serialisation problem by moving the serialisation step out of the
1718 #hyperdb.Class (get, set) into the hyperdb.Database.
1719 #
1720 #Also fixed htmltemplate after the showid changes I made yesterday.
1721 #
1722 #Unit tests for all of the above written.
1723 #
1724 #Revision 1.30.2.1 2002/04/03 11:55:57 rochecompaan
1725 # . Added feature #526730 - search for messages capability
1726 #
1727 #Revision 1.30 2002/02/27 03:40:59 richard
1728 #Ran it through pychecker, made fixes
1729 #
1730 #Revision 1.29 2002/02/25 14:34:31 grubert
1731 # . use blobfiles in back_anydbm which is used in back_bsddb.
1732 # change test_db as dirlist does not work for subdirectories.
1733 # ATTENTION: blobfiles now creates subdirectories for files.
1734 #
1735 #Revision 1.28 2002/02/16 09:14:17 richard
1736 # . #514854 ] History: "User" is always ticket creator
1737 #
1738 #Revision 1.27 2002/01/22 07:21:13 richard
1739 #. fixed back_bsddb so it passed the journal tests
1740 #
1741 #... it didn't seem happy using the back_anydbm _open method, which is odd.
1742 #Yet another occurrance of whichdb not being able to recognise older bsddb
1743 #databases. Yadda yadda. Made the HYPERDBDEBUG stuff more sane in the
1744 #process.
1745 #
1746 #Revision 1.26 2002/01/22 05:18:38 rochecompaan
1747 #last_set_entry was referenced before assignment
1748 #
1749 #Revision 1.25 2002/01/22 05:06:08 rochecompaan
1750 #We need to keep the last 'set' entry in the journal to preserve
1751 #information on 'activity' for nodes.
1752 #
1753 #Revision 1.24 2002/01/21 16:33:20 rochecompaan
1754 #You can now use the roundup-admin tool to pack the database
1755 #
1756 #Revision 1.23 2002/01/18 04:32:04 richard
1757 #Rollback was breaking because a message hadn't actually been written to the file. Needs
1758 #more investigation.
1759 #
1760 #Revision 1.22 2002/01/14 02:20:15 richard
1761 # . changed all config accesses so they access either the instance or the
1762 # config attriubute on the db. This means that all config is obtained from
1763 # instance_config instead of the mish-mash of classes. This will make
1764 # switching to a ConfigParser setup easier too, I hope.
1765 #
1766 #At a minimum, this makes migration a _little_ easier (a lot easier in the
1767 #0.5.0 switch, I hope!)
1768 #
1769 #Revision 1.21 2002/01/02 02:31:38 richard
1770 #Sorry for the huge checkin message - I was only intending to implement #496356
1771 #but I found a number of places where things had been broken by transactions:
1772 # . modified ROUNDUPDBSENDMAILDEBUG to be SENDMAILDEBUG and hold a filename
1773 # for _all_ roundup-generated smtp messages to be sent to.
1774 # . the transaction cache had broken the roundupdb.Class set() reactors
1775 # . newly-created author users in the mailgw weren't being committed to the db
1776 #
1777 #Stuff that made it into CHANGES.txt (ie. the stuff I was actually working
1778 #on when I found that stuff :):
1779 # . #496356 ] Use threading in messages
1780 # . detectors were being registered multiple times
1781 # . added tests for mailgw
1782 # . much better attaching of erroneous messages in the mail gateway
1783 #
1784 #Revision 1.20 2001/12/18 15:30:34 rochecompaan
1785 #Fixed bugs:
1786 # . Fixed file creation and retrieval in same transaction in anydbm
1787 # backend
1788 # . Cgi interface now renders new issue after issue creation
1789 # . Could not set issue status to resolved through cgi interface
1790 # . Mail gateway was changing status back to 'chatting' if status was
1791 # omitted as an argument
1792 #
1793 #Revision 1.19 2001/12/17 03:52:48 richard
1794 #Implemented file store rollback. As a bonus, the hyperdb is now capable of
1795 #storing more than one file per node - if a property name is supplied,
1796 #the file is called designator.property.
1797 #I decided not to migrate the existing files stored over to the new naming
1798 #scheme - the FileClass just doesn't specify the property name.
1799 #
1800 #Revision 1.18 2001/12/16 10:53:38 richard
1801 #take a copy of the node dict so that the subsequent set
1802 #operation doesn't modify the oldvalues structure
1803 #
1804 #Revision 1.17 2001/12/14 23:42:57 richard
1805 #yuck, a gdbm instance tests false :(
1806 #I've left the debugging code in - it should be removed one day if we're ever
1807 #_really_ anal about performace :)
1808 #
1809 #Revision 1.16 2001/12/12 03:23:14 richard
1810 #Cor blimey this anydbm/whichdb stuff is yecchy. Turns out that whichdb
1811 #incorrectly identifies a dbm file as a dbhash file on my system. This has
1812 #been submitted to the python bug tracker as issue #491888:
1813 #https://sourceforge.net/tracker/index.php?func=detail&aid=491888&group_id=5470&atid=105470
1814 #
1815 #Revision 1.15 2001/12/12 02:30:51 richard
1816 #I fixed the problems with people whose anydbm was using the dbm module at the
1817 #backend. It turns out the dbm module modifies the file name to append ".db"
1818 #and my check to determine if we're opening an existing or new db just
1819 #tested os.path.exists() on the filename. Well, no longer! We now perform a
1820 #much better check _and_ cope with the anydbm implementation module changing
1821 #too!
1822 #I also fixed the backends __init__ so only ImportError is squashed.
1823 #
1824 #Revision 1.14 2001/12/10 22:20:01 richard
1825 #Enabled transaction support in the bsddb backend. It uses the anydbm code
1826 #where possible, only replacing methods where the db is opened (it uses the
1827 #btree opener specifically.)
1828 #Also cleaned up some change note generation.
1829 #Made the backends package work with pydoc too.
1830 #
1831 #Revision 1.13 2001/12/02 05:06:16 richard
1832 #. We now use weakrefs in the Classes to keep the database reference, so
1833 # the close() method on the database is no longer needed.
1834 # I bumped the minimum python requirement up to 2.1 accordingly.
1835 #. #487480 ] roundup-server
1836 #. #487476 ] INSTALL.txt
1837 #
1838 #I also cleaned up the change message / post-edit stuff in the cgi client.
1839 #There's now a clearly marked "TODO: append the change note" where I believe
1840 #the change note should be added there. The "changes" list will obviously
1841 #have to be modified to be a dict of the changes, or somesuch.
1842 #
1843 #More testing needed.
1844 #
1845 #Revision 1.12 2001/12/01 07:17:50 richard
1846 #. We now have basic transaction support! Information is only written to
1847 # the database when the commit() method is called. Only the anydbm
1848 # backend is modified in this way - neither of the bsddb backends have been.
1849 # The mail, admin and cgi interfaces all use commit (except the admin tool
1850 # doesn't have a commit command, so interactive users can't commit...)
1851 #. Fixed login/registration forwarding the user to the right page (or not,
1852 # on a failure)
1853 #
1854 #Revision 1.11 2001/11/21 02:34:18 richard
1855 #Added a target version field to the extended issue schema
1856 #
1857 #Revision 1.10 2001/10/09 23:58:10 richard
1858 #Moved the data stringification up into the hyperdb.Class class' get, set
1859 #and create methods. This means that the data is also stringified for the
1860 #journal call, and removes duplication of code from the backends. The
1861 #backend code now only sees strings.
1862 #
1863 #Revision 1.9 2001/10/09 07:25:59 richard
1864 #Added the Password property type. See "pydoc roundup.password" for
1865 #implementation details. Have updated some of the documentation too.
1866 #
1867 #Revision 1.8 2001/09/29 13:27:00 richard
1868 #CGI interfaces now spit up a top-level index of all the instances they can
1869 #serve.
1870 #
1871 #Revision 1.7 2001/08/12 06:32:36 richard
1872 #using isinstance(blah, Foo) now instead of isFooType
1873 #
1874 #Revision 1.6 2001/08/07 00:24:42 richard
1875 #stupid typo
1876 #
1877 #Revision 1.5 2001/08/07 00:15:51 richard
1878 #Added the copyright/license notice to (nearly) all files at request of
1879 #Bizar Software.
1880 #
1881 #Revision 1.4 2001/07/30 01:41:36 richard
1882 #Makes schema changes mucho easier.
1883 #
1884 #Revision 1.3 2001/07/25 01:23:07 richard
1885 #Added the Roundup spec to the new documentation directory.
1886 #
1887 #Revision 1.2 2001/07/23 08:20:44 richard
1888 #Moved over to using marshal in the bsddb and anydbm backends.
1889 #roundup-admin now has a "freshen" command that'll load/save all nodes (not
1890 # retired - mod hyperdb.Class.list() so it lists retired nodes)
1891 #
1892 #