1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: back_anydbm.py,v 1.51 2002-07-18 23:07:08 richard Exp $
19 '''
20 This module defines a backend that saves the hyperdatabase in a database
21 chosen by anydbm. It is guaranteed to always be available in python
22 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
23 serious bugs, and is not available)
24 '''
26 import whichdb, anydbm, os, marshal, re, weakref, string, copy
27 from roundup import hyperdb, date, password, roundupdb
28 from blobfiles import FileStorage
29 from roundup.indexer import Indexer
30 from locking import acquire_lock, release_lock
31 from roundup.hyperdb import String, Password, Date, Interval, Link, \
32 Multilink, DatabaseError, Boolean, Number
34 #
35 # Now the database
36 #
37 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
38 """A database for storing records containing flexible data types.
40 Transaction stuff TODO:
41 . check the timestamp of the class file and nuke the cache if it's
42 modified. Do some sort of conflict checking on the dirty stuff.
43 . perhaps detect write collisions (related to above)?
45 """
46 def __init__(self, config, journaltag=None):
47 """Open a hyperdatabase given a specifier to some storage.
49 The 'storagelocator' is obtained from config.DATABASE.
50 The meaning of 'storagelocator' depends on the particular
51 implementation of the hyperdatabase. It could be a file name,
52 a directory path, a socket descriptor for a connection to a
53 database over the network, etc.
55 The 'journaltag' is a token that will be attached to the journal
56 entries for any edits done on the database. If 'journaltag' is
57 None, the database is opened in read-only mode: the Class.create(),
58 Class.set(), and Class.retire() methods are disabled.
59 """
60 self.config, self.journaltag = config, journaltag
61 self.dir = config.DATABASE
62 self.classes = {}
63 self.cache = {} # cache of nodes loaded or created
64 self.dirtynodes = {} # keep track of the dirty nodes by class
65 self.newnodes = {} # keep track of the new nodes by class
66 self.transactions = []
67 self.indexer = Indexer(self.dir)
68 # ensure files are group readable and writable
69 os.umask(0002)
71 def post_init(self):
72 """Called once the schema initialisation has finished."""
73 # reindex the db if necessary
74 if self.indexer.should_reindex():
75 self.reindex()
77 def reindex(self):
78 for klass in self.classes.values():
79 for nodeid in klass.list():
80 klass.index(nodeid)
81 self.indexer.save_index()
83 def __repr__(self):
84 return '<back_anydbm instance at %x>'%id(self)
86 #
87 # Classes
88 #
89 def __getattr__(self, classname):
90 """A convenient way of calling self.getclass(classname)."""
91 if self.classes.has_key(classname):
92 if __debug__:
93 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
94 return self.classes[classname]
95 raise AttributeError, classname
97 def addclass(self, cl):
98 if __debug__:
99 print >>hyperdb.DEBUG, 'addclass', (self, cl)
100 cn = cl.classname
101 if self.classes.has_key(cn):
102 raise ValueError, cn
103 self.classes[cn] = cl
105 def getclasses(self):
106 """Return a list of the names of all existing classes."""
107 if __debug__:
108 print >>hyperdb.DEBUG, 'getclasses', (self,)
109 l = self.classes.keys()
110 l.sort()
111 return l
113 def getclass(self, classname):
114 """Get the Class object representing a particular class.
116 If 'classname' is not a valid class name, a KeyError is raised.
117 """
118 if __debug__:
119 print >>hyperdb.DEBUG, 'getclass', (self, classname)
120 return self.classes[classname]
122 #
123 # Class DBs
124 #
125 def clear(self):
126 '''Delete all database contents
127 '''
128 if __debug__:
129 print >>hyperdb.DEBUG, 'clear', (self,)
130 for cn in self.classes.keys():
131 for dummy in 'nodes', 'journals':
132 path = os.path.join(self.dir, 'journals.%s'%cn)
133 if os.path.exists(path):
134 os.remove(path)
135 elif os.path.exists(path+'.db'): # dbm appends .db
136 os.remove(path+'.db')
138 def getclassdb(self, classname, mode='r'):
139 ''' grab a connection to the class db that will be used for
140 multiple actions
141 '''
142 if __debug__:
143 print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode)
144 return self._opendb('nodes.%s'%classname, mode)
146 def determine_db_type(self, path):
147 ''' determine which DB wrote the class file
148 '''
149 db_type = ''
150 if os.path.exists(path):
151 db_type = whichdb.whichdb(path)
152 if not db_type:
153 raise hyperdb.DatabaseError, "Couldn't identify database type"
154 elif os.path.exists(path+'.db'):
155 # if the path ends in '.db', it's a dbm database, whether
156 # anydbm says it's dbhash or not!
157 db_type = 'dbm'
158 return db_type
160 def _opendb(self, name, mode):
161 '''Low-level database opener that gets around anydbm/dbm
162 eccentricities.
163 '''
164 if __debug__:
165 print >>hyperdb.DEBUG, '_opendb', (self, name, mode)
167 # figure the class db type
168 path = os.path.join(os.getcwd(), self.dir, name)
169 db_type = self.determine_db_type(path)
171 # new database? let anydbm pick the best dbm
172 if not db_type:
173 if __debug__:
174 print >>hyperdb.DEBUG, "_opendb anydbm.open(%r, 'n')"%path
175 return anydbm.open(path, 'n')
177 # open the database with the correct module
178 try:
179 dbm = __import__(db_type)
180 except ImportError:
181 raise hyperdb.DatabaseError, \
182 "Couldn't open database - the required module '%s'"\
183 " is not available"%db_type
184 if __debug__:
185 print >>hyperdb.DEBUG, "_opendb %r.open(%r, %r)"%(db_type, path,
186 mode)
187 return dbm.open(path, mode)
189 def _lockdb(self, name):
190 ''' Lock a database file
191 '''
192 path = os.path.join(os.getcwd(), self.dir, '%s.lock'%name)
193 return acquire_lock(path)
195 #
196 # Node IDs
197 #
198 def newid(self, classname):
199 ''' Generate a new id for the given class
200 '''
201 # open the ids DB - create if if doesn't exist
202 lock = self._lockdb('_ids')
203 db = self._opendb('_ids', 'c')
204 if db.has_key(classname):
205 newid = db[classname] = str(int(db[classname]) + 1)
206 else:
207 # the count() bit is transitional - older dbs won't start at 1
208 newid = str(self.getclass(classname).count()+1)
209 db[classname] = newid
210 db.close()
211 release_lock(lock)
212 return newid
214 #
215 # Nodes
216 #
217 def addnode(self, classname, nodeid, node):
218 ''' add the specified node to its class's db
219 '''
220 if __debug__:
221 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
222 self.newnodes.setdefault(classname, {})[nodeid] = 1
223 self.cache.setdefault(classname, {})[nodeid] = node
224 self.savenode(classname, nodeid, node)
226 def setnode(self, classname, nodeid, node):
227 ''' change the specified node
228 '''
229 if __debug__:
230 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
231 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
233 # can't set without having already loaded the node
234 self.cache[classname][nodeid] = node
235 self.savenode(classname, nodeid, node)
237 def savenode(self, classname, nodeid, node):
238 ''' perform the saving of data specified by the set/addnode
239 '''
240 if __debug__:
241 print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node)
242 self.transactions.append((self._doSaveNode, (classname, nodeid, node)))
244 def getnode(self, classname, nodeid, db=None, cache=1):
245 ''' get a node from the database
246 '''
247 if __debug__:
248 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid, db)
249 if cache:
250 # try the cache
251 cache_dict = self.cache.setdefault(classname, {})
252 if cache_dict.has_key(nodeid):
253 if __debug__:
254 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
255 nodeid)
256 return cache_dict[nodeid]
258 if __debug__:
259 print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid)
261 # get from the database and save in the cache
262 if db is None:
263 db = self.getclassdb(classname)
264 if not db.has_key(nodeid):
265 raise IndexError, "no such %s %s"%(classname, nodeid)
267 # decode
268 res = marshal.loads(db[nodeid])
270 # reverse the serialisation
271 res = self.unserialise(classname, res)
273 # store off in the cache dict
274 if cache:
275 cache_dict[nodeid] = res
277 return res
279 def serialise(self, classname, node):
280 '''Copy the node contents, converting non-marshallable data into
281 marshallable data.
282 '''
283 if __debug__:
284 print >>hyperdb.DEBUG, 'serialise', classname, node
285 properties = self.getclass(classname).getprops()
286 d = {}
287 for k, v in node.items():
288 # if the property doesn't exist, or is the "retired" flag then
289 # it won't be in the properties dict
290 if not properties.has_key(k):
291 d[k] = v
292 continue
294 # get the property spec
295 prop = properties[k]
297 if isinstance(prop, Password):
298 d[k] = str(v)
299 elif isinstance(prop, Date) and v is not None:
300 d[k] = v.get_tuple()
301 elif isinstance(prop, Interval) and v is not None:
302 d[k] = v.get_tuple()
303 else:
304 d[k] = v
305 return d
307 def unserialise(self, classname, node):
308 '''Decode the marshalled node data
309 '''
310 if __debug__:
311 print >>hyperdb.DEBUG, 'unserialise', classname, node
312 properties = self.getclass(classname).getprops()
313 d = {}
314 for k, v in node.items():
315 # if the property doesn't exist, or is the "retired" flag then
316 # it won't be in the properties dict
317 if not properties.has_key(k):
318 d[k] = v
319 continue
321 # get the property spec
322 prop = properties[k]
324 if isinstance(prop, Date) and v is not None:
325 d[k] = date.Date(v)
326 elif isinstance(prop, Interval) and v is not None:
327 d[k] = date.Interval(v)
328 elif isinstance(prop, Password):
329 p = password.Password()
330 p.unpack(v)
331 d[k] = p
332 else:
333 d[k] = v
334 return d
336 def hasnode(self, classname, nodeid, db=None):
337 ''' determine if the database has a given node
338 '''
339 if __debug__:
340 print >>hyperdb.DEBUG, 'hasnode', (self, classname, nodeid, db)
342 # try the cache
343 cache = self.cache.setdefault(classname, {})
344 if cache.has_key(nodeid):
345 if __debug__:
346 print >>hyperdb.TRACE, 'has %s %s cached'%(classname, nodeid)
347 return 1
348 if __debug__:
349 print >>hyperdb.TRACE, 'has %s %s'%(classname, nodeid)
351 # not in the cache - check the database
352 if db is None:
353 db = self.getclassdb(classname)
354 res = db.has_key(nodeid)
355 return res
357 def countnodes(self, classname, db=None):
358 if __debug__:
359 print >>hyperdb.DEBUG, 'countnodes', (self, classname, db)
360 # include the new nodes not saved to the DB yet
361 count = len(self.newnodes.get(classname, {}))
363 # and count those in the DB
364 if db is None:
365 db = self.getclassdb(classname)
366 count = count + len(db.keys())
367 return count
369 def getnodeids(self, classname, db=None):
370 if __debug__:
371 print >>hyperdb.DEBUG, 'getnodeids', (self, classname, db)
372 # start off with the new nodes
373 res = self.newnodes.get(classname, {}).keys()
375 if db is None:
376 db = self.getclassdb(classname)
377 res = res + db.keys()
378 return res
381 #
382 # Files - special node properties
383 # inherited from FileStorage
385 #
386 # Journal
387 #
388 def addjournal(self, classname, nodeid, action, params):
389 ''' Journal the Action
390 'action' may be:
392 'create' or 'set' -- 'params' is a dictionary of property values
393 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
394 'retire' -- 'params' is None
395 '''
396 if __debug__:
397 print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid,
398 action, params)
399 self.transactions.append((self._doSaveJournal, (classname, nodeid,
400 action, params)))
402 def getjournal(self, classname, nodeid):
403 ''' get the journal for id
404 '''
405 if __debug__:
406 print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid)
407 # attempt to open the journal - in some rare cases, the journal may
408 # not exist
409 try:
410 db = self._opendb('journals.%s'%classname, 'r')
411 except anydbm.error, error:
412 if str(error) == "need 'c' or 'n' flag to open new db": return []
413 elif error.args[0] != 2: raise
414 return []
415 try:
416 journal = marshal.loads(db[nodeid])
417 except KeyError:
418 db.close()
419 raise KeyError, 'no such %s %s'%(classname, nodeid)
420 db.close()
421 res = []
422 for entry in journal:
423 (nodeid, date_stamp, user, action, params) = entry
424 date_obj = date.Date(date_stamp)
425 res.append((nodeid, date_obj, user, action, params))
426 return res
428 def pack(self, pack_before):
429 ''' delete all journal entries before 'pack_before' '''
430 if __debug__:
431 print >>hyperdb.DEBUG, 'packjournal', (self, pack_before)
433 pack_before = pack_before.get_tuple()
435 classes = self.getclasses()
437 # figure the class db type
439 for classname in classes:
440 db_name = 'journals.%s'%classname
441 path = os.path.join(os.getcwd(), self.dir, classname)
442 db_type = self.determine_db_type(path)
443 db = self._opendb(db_name, 'w')
445 for key in db.keys():
446 journal = marshal.loads(db[key])
447 l = []
448 last_set_entry = None
449 for entry in journal:
450 (nodeid, date_stamp, self.journaltag, action,
451 params) = entry
452 if date_stamp > pack_before or action == 'create':
453 l.append(entry)
454 elif action == 'set':
455 # grab the last set entry to keep information on
456 # activity
457 last_set_entry = entry
458 if last_set_entry:
459 date_stamp = last_set_entry[1]
460 # if the last set entry was made after the pack date
461 # then it is already in the list
462 if date_stamp < pack_before:
463 l.append(last_set_entry)
464 db[key] = marshal.dumps(l)
465 if db_type == 'gdbm':
466 db.reorganize()
467 db.close()
470 #
471 # Basic transaction support
472 #
473 def commit(self):
474 ''' Commit the current transactions.
475 '''
476 if __debug__:
477 print >>hyperdb.DEBUG, 'commit', (self,)
478 # TODO: lock the DB
480 # keep a handle to all the database files opened
481 self.databases = {}
483 # now, do all the transactions
484 reindex = {}
485 for method, args in self.transactions:
486 reindex[method(*args)] = 1
488 # now close all the database files
489 for db in self.databases.values():
490 db.close()
491 del self.databases
492 # TODO: unlock the DB
494 # reindex the nodes that request it
495 for classname, nodeid in filter(None, reindex.keys()):
496 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
497 self.getclass(classname).index(nodeid)
499 # save the indexer state
500 self.indexer.save_index()
502 # all transactions committed, back to normal
503 self.cache = {}
504 self.dirtynodes = {}
505 self.newnodes = {}
506 self.transactions = []
508 def _doSaveNode(self, classname, nodeid, node):
509 if __debug__:
510 print >>hyperdb.DEBUG, '_doSaveNode', (self, classname, nodeid,
511 node)
513 # get the database handle
514 db_name = 'nodes.%s'%classname
515 if self.databases.has_key(db_name):
516 db = self.databases[db_name]
517 else:
518 db = self.databases[db_name] = self.getclassdb(classname, 'c')
520 # now save the marshalled data
521 db[nodeid] = marshal.dumps(self.serialise(classname, node))
523 # return the classname, nodeid so we reindex this content
524 return (classname, nodeid)
526 def _doSaveJournal(self, classname, nodeid, action, params):
527 # serialise first
528 if action in ('set', 'create'):
529 params = self.serialise(classname, params)
531 # create the journal entry
532 entry = (nodeid, date.Date().get_tuple(), self.journaltag, action,
533 params)
535 if __debug__:
536 print >>hyperdb.DEBUG, '_doSaveJournal', entry
538 # get the database handle
539 db_name = 'journals.%s'%classname
540 if self.databases.has_key(db_name):
541 db = self.databases[db_name]
542 else:
543 db = self.databases[db_name] = self._opendb(db_name, 'c')
545 # now insert the journal entry
546 if db.has_key(nodeid):
547 # append to existing
548 s = db[nodeid]
549 l = marshal.loads(s)
550 l.append(entry)
551 else:
552 l = [entry]
554 db[nodeid] = marshal.dumps(l)
556 def rollback(self):
557 ''' Reverse all actions from the current transaction.
558 '''
559 if __debug__:
560 print >>hyperdb.DEBUG, 'rollback', (self, )
561 for method, args in self.transactions:
562 # delete temporary files
563 if method == self._doStoreFile:
564 self._rollbackStoreFile(*args)
565 self.cache = {}
566 self.dirtynodes = {}
567 self.newnodes = {}
568 self.transactions = []
570 _marker = []
571 class Class(hyperdb.Class):
572 """The handle to a particular class of nodes in a hyperdatabase."""
574 def __init__(self, db, classname, **properties):
575 """Create a new class with a given name and property specification.
577 'classname' must not collide with the name of an existing class,
578 or a ValueError is raised. The keyword arguments in 'properties'
579 must map names to property objects, or a TypeError is raised.
580 """
581 if (properties.has_key('creation') or properties.has_key('activity')
582 or properties.has_key('creator')):
583 raise ValueError, '"creation", "activity" and "creator" are '\
584 'reserved'
586 self.classname = classname
587 self.properties = properties
588 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
589 self.key = ''
591 # should we journal changes (default yes)
592 self.do_journal = 1
594 # do the db-related init stuff
595 db.addclass(self)
597 self.auditors = {'create': [], 'set': [], 'retire': []}
598 self.reactors = {'create': [], 'set': [], 'retire': []}
600 def enableJournalling(self):
601 '''Turn journalling on for this class
602 '''
603 self.do_journal = 1
605 def disableJournalling(self):
606 '''Turn journalling off for this class
607 '''
608 self.do_journal = 0
610 # Editing nodes:
612 def create(self, **propvalues):
613 """Create a new node of this class and return its id.
615 The keyword arguments in 'propvalues' map property names to values.
617 The values of arguments must be acceptable for the types of their
618 corresponding properties or a TypeError is raised.
620 If this class has a key property, it must be present and its value
621 must not collide with other key strings or a ValueError is raised.
623 Any other properties on this class that are missing from the
624 'propvalues' dictionary are set to None.
626 If an id in a link or multilink property does not refer to a valid
627 node, an IndexError is raised.
629 These operations trigger detectors and can be vetoed. Attempts
630 to modify the "creation" or "activity" properties cause a KeyError.
631 """
632 if propvalues.has_key('id'):
633 raise KeyError, '"id" is reserved'
635 if self.db.journaltag is None:
636 raise DatabaseError, 'Database open read-only'
638 if propvalues.has_key('creation') or propvalues.has_key('activity'):
639 raise KeyError, '"creation" and "activity" are reserved'
641 self.fireAuditors('create', None, propvalues)
643 # new node's id
644 newid = self.db.newid(self.classname)
646 # validate propvalues
647 num_re = re.compile('^\d+$')
648 for key, value in propvalues.items():
649 if key == self.key:
650 try:
651 self.lookup(value)
652 except KeyError:
653 pass
654 else:
655 raise ValueError, 'node with key "%s" exists'%value
657 # try to handle this property
658 try:
659 prop = self.properties[key]
660 except KeyError:
661 raise KeyError, '"%s" has no property "%s"'%(self.classname,
662 key)
664 if isinstance(prop, Link):
665 if type(value) != type(''):
666 raise ValueError, 'link value must be String'
667 link_class = self.properties[key].classname
668 # if it isn't a number, it's a key
669 if not num_re.match(value):
670 try:
671 value = self.db.classes[link_class].lookup(value)
672 except (TypeError, KeyError):
673 raise IndexError, 'new property "%s": %s not a %s'%(
674 key, value, link_class)
675 elif not self.db.hasnode(link_class, value):
676 raise IndexError, '%s has no node %s'%(link_class, value)
678 # save off the value
679 propvalues[key] = value
681 # register the link with the newly linked node
682 if self.do_journal and self.properties[key].do_journal:
683 self.db.addjournal(link_class, value, 'link',
684 (self.classname, newid, key))
686 elif isinstance(prop, Multilink):
687 if type(value) != type([]):
688 raise TypeError, 'new property "%s" not a list of ids'%key
690 # clean up and validate the list of links
691 link_class = self.properties[key].classname
692 l = []
693 for entry in value:
694 if type(entry) != type(''):
695 raise ValueError, '"%s" link value (%s) must be '\
696 'String'%(key, value)
697 # if it isn't a number, it's a key
698 if not num_re.match(entry):
699 try:
700 entry = self.db.classes[link_class].lookup(entry)
701 except (TypeError, KeyError):
702 raise IndexError, 'new property "%s": %s not a %s'%(
703 key, entry, self.properties[key].classname)
704 l.append(entry)
705 value = l
706 propvalues[key] = value
708 # handle additions
709 for id in value:
710 if not self.db.hasnode(link_class, id):
711 raise IndexError, '%s has no node %s'%(link_class, id)
712 # register the link with the newly linked node
713 if self.do_journal and self.properties[key].do_journal:
714 self.db.addjournal(link_class, id, 'link',
715 (self.classname, newid, key))
717 elif isinstance(prop, String):
718 if type(value) != type(''):
719 raise TypeError, 'new property "%s" not a string'%key
721 elif isinstance(prop, Password):
722 if not isinstance(value, password.Password):
723 raise TypeError, 'new property "%s" not a Password'%key
725 elif isinstance(prop, Date):
726 if value is not None and not isinstance(value, date.Date):
727 raise TypeError, 'new property "%s" not a Date'%key
729 elif isinstance(prop, Interval):
730 if value is not None and not isinstance(value, date.Interval):
731 raise TypeError, 'new property "%s" not an Interval'%key
733 elif value is not None and isinstance(prop, Number):
734 try:
735 float(value)
736 except ValueError:
737 raise TypeError, 'new property "%s" not numeric'%key
739 elif value is not None and isinstance(prop, Boolean):
740 try:
741 int(value)
742 except ValueError:
743 raise TypeError, 'new property "%s" not boolean'%key
745 # make sure there's data where there needs to be
746 for key, prop in self.properties.items():
747 if propvalues.has_key(key):
748 continue
749 if key == self.key:
750 raise ValueError, 'key property "%s" is required'%key
751 if isinstance(prop, Multilink):
752 propvalues[key] = []
753 else:
754 propvalues[key] = None
756 # done
757 self.db.addnode(self.classname, newid, propvalues)
758 if self.do_journal:
759 self.db.addjournal(self.classname, newid, 'create', propvalues)
761 self.fireReactors('create', newid, None)
763 return newid
765 def get(self, nodeid, propname, default=_marker, cache=1):
766 """Get the value of a property on an existing node of this class.
768 'nodeid' must be the id of an existing node of this class or an
769 IndexError is raised. 'propname' must be the name of a property
770 of this class or a KeyError is raised.
772 'cache' indicates whether the transaction cache should be queried
773 for the node. If the node has been modified and you need to
774 determine what its values prior to modification are, you need to
775 set cache=0.
777 Attempts to get the "creation" or "activity" properties should
778 do the right thing.
779 """
780 if propname == 'id':
781 return nodeid
783 if propname == 'creation':
784 if not self.do_journal:
785 raise ValueError, 'Journalling is disabled for this class'
786 journal = self.db.getjournal(self.classname, nodeid)
787 if journal:
788 return self.db.getjournal(self.classname, nodeid)[0][1]
789 else:
790 # on the strange chance that there's no journal
791 return date.Date()
792 if propname == 'activity':
793 if not self.do_journal:
794 raise ValueError, 'Journalling is disabled for this class'
795 journal = self.db.getjournal(self.classname, nodeid)
796 if journal:
797 return self.db.getjournal(self.classname, nodeid)[-1][1]
798 else:
799 # on the strange chance that there's no journal
800 return date.Date()
801 if propname == 'creator':
802 if not self.do_journal:
803 raise ValueError, 'Journalling is disabled for this class'
804 journal = self.db.getjournal(self.classname, nodeid)
805 if journal:
806 name = self.db.getjournal(self.classname, nodeid)[0][2]
807 else:
808 return None
809 return self.db.user.lookup(name)
811 # get the property (raises KeyErorr if invalid)
812 prop = self.properties[propname]
814 # get the node's dict
815 d = self.db.getnode(self.classname, nodeid, cache=cache)
817 if not d.has_key(propname):
818 if default is _marker:
819 if isinstance(prop, Multilink):
820 return []
821 else:
822 return None
823 else:
824 return default
826 return d[propname]
828 # XXX not in spec
829 def getnode(self, nodeid, cache=1):
830 ''' Return a convenience wrapper for the node.
832 'nodeid' must be the id of an existing node of this class or an
833 IndexError is raised.
835 'cache' indicates whether the transaction cache should be queried
836 for the node. If the node has been modified and you need to
837 determine what its values prior to modification are, you need to
838 set cache=0.
839 '''
840 return Node(self, nodeid, cache=cache)
842 def set(self, nodeid, **propvalues):
843 """Modify a property on an existing node of this class.
845 'nodeid' must be the id of an existing node of this class or an
846 IndexError is raised.
848 Each key in 'propvalues' must be the name of a property of this
849 class or a KeyError is raised.
851 All values in 'propvalues' must be acceptable types for their
852 corresponding properties or a TypeError is raised.
854 If the value of the key property is set, it must not collide with
855 other key strings or a ValueError is raised.
857 If the value of a Link or Multilink property contains an invalid
858 node id, a ValueError is raised.
860 These operations trigger detectors and can be vetoed. Attempts
861 to modify the "creation" or "activity" properties cause a KeyError.
862 """
863 if not propvalues:
864 return
866 if propvalues.has_key('creation') or propvalues.has_key('activity'):
867 raise KeyError, '"creation" and "activity" are reserved'
869 if propvalues.has_key('id'):
870 raise KeyError, '"id" is reserved'
872 if self.db.journaltag is None:
873 raise DatabaseError, 'Database open read-only'
875 self.fireAuditors('set', nodeid, propvalues)
876 # Take a copy of the node dict so that the subsequent set
877 # operation doesn't modify the oldvalues structure.
878 try:
879 # try not using the cache initially
880 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid,
881 cache=0))
882 except IndexError:
883 # this will be needed if somone does a create() and set()
884 # with no intervening commit()
885 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
887 node = self.db.getnode(self.classname, nodeid)
888 if node.has_key(self.db.RETIRED_FLAG):
889 raise IndexError
890 num_re = re.compile('^\d+$')
892 # if the journal value is to be different, store it in here
893 journalvalues = {}
895 for propname, value in propvalues.items():
896 # check to make sure we're not duplicating an existing key
897 if propname == self.key and node[propname] != value:
898 try:
899 self.lookup(value)
900 except KeyError:
901 pass
902 else:
903 raise ValueError, 'node with key "%s" exists'%value
905 # this will raise the KeyError if the property isn't valid
906 # ... we don't use getprops() here because we only care about
907 # the writeable properties.
908 prop = self.properties[propname]
910 # if the value's the same as the existing value, no sense in
911 # doing anything
912 if node.has_key(propname) and value == node[propname]:
913 del propvalues[propname]
914 continue
916 # do stuff based on the prop type
917 if isinstance(prop, Link):
918 link_class = self.properties[propname].classname
919 # if it isn't a number, it's a key
920 if type(value) != type(''):
921 raise ValueError, 'link value must be String'
922 if not num_re.match(value):
923 try:
924 value = self.db.classes[link_class].lookup(value)
925 except (TypeError, KeyError):
926 raise IndexError, 'new property "%s": %s not a %s'%(
927 propname, value, self.properties[propname].classname)
929 if not self.db.hasnode(link_class, value):
930 raise IndexError, '%s has no node %s'%(link_class, value)
932 if self.do_journal and self.properties[propname].do_journal:
933 # register the unlink with the old linked node
934 if node[propname] is not None:
935 self.db.addjournal(link_class, node[propname], 'unlink',
936 (self.classname, nodeid, propname))
938 # register the link with the newly linked node
939 if value is not None:
940 self.db.addjournal(link_class, value, 'link',
941 (self.classname, nodeid, propname))
943 elif isinstance(prop, Multilink):
944 if type(value) != type([]):
945 raise TypeError, 'new property "%s" not a list of'\
946 ' ids'%propname
947 link_class = self.properties[propname].classname
948 l = []
949 for entry in value:
950 # if it isn't a number, it's a key
951 if type(entry) != type(''):
952 raise ValueError, 'new property "%s" link value ' \
953 'must be a string'%propname
954 if not num_re.match(entry):
955 try:
956 entry = self.db.classes[link_class].lookup(entry)
957 except (TypeError, KeyError):
958 raise IndexError, 'new property "%s": %s not a %s'%(
959 propname, entry,
960 self.properties[propname].classname)
961 l.append(entry)
962 value = l
963 propvalues[propname] = value
965 # figure the journal entry for this property
966 add = []
967 remove = []
969 # handle removals
970 if node.has_key(propname):
971 l = node[propname]
972 else:
973 l = []
974 for id in l[:]:
975 if id in value:
976 continue
977 # register the unlink with the old linked node
978 if self.do_journal and self.properties[propname].do_journal:
979 self.db.addjournal(link_class, id, 'unlink',
980 (self.classname, nodeid, propname))
981 l.remove(id)
982 remove.append(id)
984 # handle additions
985 for id in value:
986 if not self.db.hasnode(link_class, id):
987 raise IndexError, '%s has no node %s'%(link_class, id)
988 if id in l:
989 continue
990 # register the link with the newly linked node
991 if self.do_journal and self.properties[propname].do_journal:
992 self.db.addjournal(link_class, id, 'link',
993 (self.classname, nodeid, propname))
994 l.append(id)
995 add.append(id)
997 # figure the journal entry
998 l = []
999 if add:
1000 l.append(('add', add))
1001 if remove:
1002 l.append(('remove', remove))
1003 if l:
1004 journalvalues[propname] = tuple(l)
1006 elif isinstance(prop, String):
1007 if value is not None and type(value) != type(''):
1008 raise TypeError, 'new property "%s" not a string'%propname
1010 elif isinstance(prop, Password):
1011 if not isinstance(value, password.Password):
1012 raise TypeError, 'new property "%s" not a Password'%propname
1013 propvalues[propname] = value
1015 elif value is not None and isinstance(prop, Date):
1016 if not isinstance(value, date.Date):
1017 raise TypeError, 'new property "%s" not a Date'% propname
1018 propvalues[propname] = value
1020 elif value is not None and isinstance(prop, Interval):
1021 if not isinstance(value, date.Interval):
1022 raise TypeError, 'new property "%s" not an '\
1023 'Interval'%propname
1024 propvalues[propname] = value
1026 elif value is not None and isinstance(prop, Number):
1027 try:
1028 float(value)
1029 except ValueError:
1030 raise TypeError, 'new property "%s" not numeric'%propname
1032 elif value is not None and isinstance(prop, Boolean):
1033 try:
1034 int(value)
1035 except ValueError:
1036 raise TypeError, 'new property "%s" not boolean'%propname
1038 node[propname] = value
1040 # nothing to do?
1041 if not propvalues:
1042 return
1044 # do the set, and journal it
1045 self.db.setnode(self.classname, nodeid, node)
1047 if self.do_journal:
1048 propvalues.update(journalvalues)
1049 self.db.addjournal(self.classname, nodeid, 'set', propvalues)
1051 self.fireReactors('set', nodeid, oldvalues)
1053 def retire(self, nodeid):
1054 """Retire a node.
1056 The properties on the node remain available from the get() method,
1057 and the node's id is never reused.
1059 Retired nodes are not returned by the find(), list(), or lookup()
1060 methods, and other nodes may reuse the values of their key properties.
1062 These operations trigger detectors and can be vetoed. Attempts
1063 to modify the "creation" or "activity" properties cause a KeyError.
1064 """
1065 if self.db.journaltag is None:
1066 raise DatabaseError, 'Database open read-only'
1068 self.fireAuditors('retire', nodeid, None)
1070 node = self.db.getnode(self.classname, nodeid)
1071 node[self.db.RETIRED_FLAG] = 1
1072 self.db.setnode(self.classname, nodeid, node)
1073 if self.do_journal:
1074 self.db.addjournal(self.classname, nodeid, 'retired', None)
1076 self.fireReactors('retire', nodeid, None)
1078 def history(self, nodeid):
1079 """Retrieve the journal of edits on a particular node.
1081 'nodeid' must be the id of an existing node of this class or an
1082 IndexError is raised.
1084 The returned list contains tuples of the form
1086 (date, tag, action, params)
1088 'date' is a Timestamp object specifying the time of the change and
1089 'tag' is the journaltag specified when the database was opened.
1090 """
1091 if not self.do_journal:
1092 raise ValueError, 'Journalling is disabled for this class'
1093 return self.db.getjournal(self.classname, nodeid)
1095 # Locating nodes:
1096 def hasnode(self, nodeid):
1097 '''Determine if the given nodeid actually exists
1098 '''
1099 return self.db.hasnode(self.classname, nodeid)
1101 def setkey(self, propname):
1102 """Select a String property of this class to be the key property.
1104 'propname' must be the name of a String property of this class or
1105 None, or a TypeError is raised. The values of the key property on
1106 all existing nodes must be unique or a ValueError is raised. If the
1107 property doesn't exist, KeyError is raised.
1108 """
1109 prop = self.getprops()[propname]
1110 if not isinstance(prop, String):
1111 raise TypeError, 'key properties must be String'
1112 self.key = propname
1114 def getkey(self):
1115 """Return the name of the key property for this class or None."""
1116 return self.key
1118 def labelprop(self, default_to_id=0):
1119 ''' Return the property name for a label for the given node.
1121 This method attempts to generate a consistent label for the node.
1122 It tries the following in order:
1123 1. key property
1124 2. "name" property
1125 3. "title" property
1126 4. first property from the sorted property name list
1127 '''
1128 k = self.getkey()
1129 if k:
1130 return k
1131 props = self.getprops()
1132 if props.has_key('name'):
1133 return 'name'
1134 elif props.has_key('title'):
1135 return 'title'
1136 if default_to_id:
1137 return 'id'
1138 props = props.keys()
1139 props.sort()
1140 return props[0]
1142 # TODO: set up a separate index db file for this? profile?
1143 def lookup(self, keyvalue):
1144 """Locate a particular node by its key property and return its id.
1146 If this class has no key property, a TypeError is raised. If the
1147 'keyvalue' matches one of the values for the key property among
1148 the nodes in this class, the matching node's id is returned;
1149 otherwise a KeyError is raised.
1150 """
1151 cldb = self.db.getclassdb(self.classname)
1152 try:
1153 for nodeid in self.db.getnodeids(self.classname, cldb):
1154 node = self.db.getnode(self.classname, nodeid, cldb)
1155 if node.has_key(self.db.RETIRED_FLAG):
1156 continue
1157 if node[self.key] == keyvalue:
1158 cldb.close()
1159 return nodeid
1160 finally:
1161 cldb.close()
1162 raise KeyError, keyvalue
1164 # XXX: change from spec - allows multiple props to match
1165 def find(self, **propspec):
1166 """Get the ids of nodes in this class which link to the given nodes.
1168 'propspec' consists of keyword args propname={nodeid:1,}
1169 'propname' must be the name of a property in this class, or a
1170 KeyError is raised. That property must be a Link or Multilink
1171 property, or a TypeError is raised.
1173 Any node in this class whose 'propname' property links to any of the
1174 nodeids will be returned. Used by the full text indexing, which knows
1175 that "foo" occurs in msg1, msg3 and file7, so we have hits on these issues:
1176 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1177 """
1178 propspec = propspec.items()
1179 for propname, nodeids in propspec:
1180 # check the prop is OK
1181 prop = self.properties[propname]
1182 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1183 raise TypeError, "'%s' not a Link/Multilink property"%propname
1184 #XXX edit is expensive and of questionable use
1185 #for nodeid in nodeids:
1186 # if not self.db.hasnode(prop.classname, nodeid):
1187 # raise ValueError, '%s has no node %s'%(prop.classname, nodeid)
1189 # ok, now do the find
1190 cldb = self.db.getclassdb(self.classname)
1191 l = []
1192 try:
1193 for id in self.db.getnodeids(self.classname, db=cldb):
1194 node = self.db.getnode(self.classname, id, db=cldb)
1195 if node.has_key(self.db.RETIRED_FLAG):
1196 continue
1197 for propname, nodeids in propspec:
1198 # can't test if the node doesn't have this property
1199 if not node.has_key(propname):
1200 continue
1201 if type(nodeids) is type(''):
1202 nodeids = {nodeids:1}
1203 prop = self.properties[propname]
1204 value = node[propname]
1205 if isinstance(prop, Link) and nodeids.has_key(value):
1206 l.append(id)
1207 break
1208 elif isinstance(prop, Multilink):
1209 hit = 0
1210 for v in value:
1211 if nodeids.has_key(v):
1212 l.append(id)
1213 hit = 1
1214 break
1215 if hit:
1216 break
1217 finally:
1218 cldb.close()
1219 return l
1221 def stringFind(self, **requirements):
1222 """Locate a particular node by matching a set of its String
1223 properties in a caseless search.
1225 If the property is not a String property, a TypeError is raised.
1227 The return is a list of the id of all nodes that match.
1228 """
1229 for propname in requirements.keys():
1230 prop = self.properties[propname]
1231 if isinstance(not prop, String):
1232 raise TypeError, "'%s' not a String property"%propname
1233 requirements[propname] = requirements[propname].lower()
1234 l = []
1235 cldb = self.db.getclassdb(self.classname)
1236 try:
1237 for nodeid in self.db.getnodeids(self.classname, cldb):
1238 node = self.db.getnode(self.classname, nodeid, cldb)
1239 if node.has_key(self.db.RETIRED_FLAG):
1240 continue
1241 for key, value in requirements.items():
1242 if node[key] and node[key].lower() != value:
1243 break
1244 else:
1245 l.append(nodeid)
1246 finally:
1247 cldb.close()
1248 return l
1250 def list(self):
1251 """Return a list of the ids of the active nodes in this class."""
1252 l = []
1253 cn = self.classname
1254 cldb = self.db.getclassdb(cn)
1255 try:
1256 for nodeid in self.db.getnodeids(cn, cldb):
1257 node = self.db.getnode(cn, nodeid, cldb)
1258 if node.has_key(self.db.RETIRED_FLAG):
1259 continue
1260 l.append(nodeid)
1261 finally:
1262 cldb.close()
1263 l.sort()
1264 return l
1266 # XXX not in spec
1267 def filter(self, search_matches, filterspec, sort, group,
1268 num_re = re.compile('^\d+$')):
1269 ''' Return a list of the ids of the active nodes in this class that
1270 match the 'filter' spec, sorted by the group spec and then the
1271 sort spec
1272 '''
1273 cn = self.classname
1275 # optimise filterspec
1276 l = []
1277 props = self.getprops()
1278 for k, v in filterspec.items():
1279 propclass = props[k]
1280 if isinstance(propclass, Link):
1281 if type(v) is not type([]):
1282 v = [v]
1283 # replace key values with node ids
1284 u = []
1285 link_class = self.db.classes[propclass.classname]
1286 for entry in v:
1287 if entry == '-1': entry = None
1288 elif not num_re.match(entry):
1289 try:
1290 entry = link_class.lookup(entry)
1291 except (TypeError,KeyError):
1292 raise ValueError, 'property "%s": %s not a %s'%(
1293 k, entry, self.properties[k].classname)
1294 u.append(entry)
1296 l.append((0, k, u))
1297 elif isinstance(propclass, Multilink):
1298 if type(v) is not type([]):
1299 v = [v]
1300 # replace key values with node ids
1301 u = []
1302 link_class = self.db.classes[propclass.classname]
1303 for entry in v:
1304 if not num_re.match(entry):
1305 try:
1306 entry = link_class.lookup(entry)
1307 except (TypeError,KeyError):
1308 raise ValueError, 'new property "%s": %s not a %s'%(
1309 k, entry, self.properties[k].classname)
1310 u.append(entry)
1311 l.append((1, k, u))
1312 elif isinstance(propclass, String):
1313 # simple glob searching
1314 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1315 v = v.replace('?', '.')
1316 v = v.replace('*', '.*?')
1317 l.append((2, k, re.compile(v, re.I)))
1318 elif isinstance(propclass, Boolean):
1319 if type(v) is type(''):
1320 bv = v.lower() in ('yes', 'true', 'on', '1')
1321 else:
1322 bv = v
1323 l.append((6, k, bv))
1324 elif isinstance(propclass, Number):
1325 l.append((6, k, int(v)))
1326 else:
1327 l.append((6, k, v))
1328 filterspec = l
1330 # now, find all the nodes that are active and pass filtering
1331 l = []
1332 cldb = self.db.getclassdb(cn)
1333 try:
1334 for nodeid in self.db.getnodeids(cn, cldb):
1335 node = self.db.getnode(cn, nodeid, cldb)
1336 if node.has_key(self.db.RETIRED_FLAG):
1337 continue
1338 # apply filter
1339 for t, k, v in filterspec:
1340 # this node doesn't have this property, so reject it
1341 if not node.has_key(k): break
1343 if t == 0 and node[k] not in v:
1344 # link - if this node'd property doesn't appear in the
1345 # filterspec's nodeid list, skip it
1346 break
1347 elif t == 1:
1348 # multilink - if any of the nodeids required by the
1349 # filterspec aren't in this node's property, then skip
1350 # it
1351 for value in v:
1352 if value not in node[k]:
1353 break
1354 else:
1355 continue
1356 break
1357 elif t == 2 and (node[k] is None or not v.search(node[k])):
1358 # RE search
1359 break
1360 elif t == 6 and node[k] != v:
1361 # straight value comparison for the other types
1362 break
1363 else:
1364 l.append((nodeid, node))
1365 finally:
1366 cldb.close()
1367 l.sort()
1369 # filter based on full text search
1370 if search_matches is not None:
1371 k = []
1372 l_debug = []
1373 for v in l:
1374 l_debug.append(v[0])
1375 if search_matches.has_key(v[0]):
1376 k.append(v)
1377 l = k
1379 # optimise sort
1380 m = []
1381 for entry in sort:
1382 if entry[0] != '-':
1383 m.append(('+', entry))
1384 else:
1385 m.append((entry[0], entry[1:]))
1386 sort = m
1388 # optimise group
1389 m = []
1390 for entry in group:
1391 if entry[0] != '-':
1392 m.append(('+', entry))
1393 else:
1394 m.append((entry[0], entry[1:]))
1395 group = m
1396 # now, sort the result
1397 def sortfun(a, b, sort=sort, group=group, properties=self.getprops(),
1398 db = self.db, cl=self):
1399 a_id, an = a
1400 b_id, bn = b
1401 # sort by group and then sort
1402 for list in group, sort:
1403 for dir, prop in list:
1404 # sorting is class-specific
1405 propclass = properties[prop]
1407 # handle the properties that might be "faked"
1408 # also, handle possible missing properties
1409 try:
1410 if not an.has_key(prop):
1411 an[prop] = cl.get(a_id, prop)
1412 av = an[prop]
1413 except KeyError:
1414 # the node doesn't have a value for this property
1415 if isinstance(propclass, Multilink): av = []
1416 else: av = ''
1417 try:
1418 if not bn.has_key(prop):
1419 bn[prop] = cl.get(b_id, prop)
1420 bv = bn[prop]
1421 except KeyError:
1422 # the node doesn't have a value for this property
1423 if isinstance(propclass, Multilink): bv = []
1424 else: bv = ''
1426 # String and Date values are sorted in the natural way
1427 if isinstance(propclass, String):
1428 # clean up the strings
1429 if av and av[0] in string.uppercase:
1430 av = an[prop] = av.lower()
1431 if bv and bv[0] in string.uppercase:
1432 bv = bn[prop] = bv.lower()
1433 if (isinstance(propclass, String) or
1434 isinstance(propclass, Date)):
1435 # it might be a string that's really an integer
1436 try:
1437 av = int(av)
1438 bv = int(bv)
1439 except:
1440 pass
1441 if dir == '+':
1442 r = cmp(av, bv)
1443 if r != 0: return r
1444 elif dir == '-':
1445 r = cmp(bv, av)
1446 if r != 0: return r
1448 # Link properties are sorted according to the value of
1449 # the "order" property on the linked nodes if it is
1450 # present; or otherwise on the key string of the linked
1451 # nodes; or finally on the node ids.
1452 elif isinstance(propclass, Link):
1453 link = db.classes[propclass.classname]
1454 if av is None and bv is not None: return -1
1455 if av is not None and bv is None: return 1
1456 if av is None and bv is None: continue
1457 if link.getprops().has_key('order'):
1458 if dir == '+':
1459 r = cmp(link.get(av, 'order'),
1460 link.get(bv, 'order'))
1461 if r != 0: return r
1462 elif dir == '-':
1463 r = cmp(link.get(bv, 'order'),
1464 link.get(av, 'order'))
1465 if r != 0: return r
1466 elif link.getkey():
1467 key = link.getkey()
1468 if dir == '+':
1469 r = cmp(link.get(av, key), link.get(bv, key))
1470 if r != 0: return r
1471 elif dir == '-':
1472 r = cmp(link.get(bv, key), link.get(av, key))
1473 if r != 0: return r
1474 else:
1475 if dir == '+':
1476 r = cmp(av, bv)
1477 if r != 0: return r
1478 elif dir == '-':
1479 r = cmp(bv, av)
1480 if r != 0: return r
1482 # Multilink properties are sorted according to how many
1483 # links are present.
1484 elif isinstance(propclass, Multilink):
1485 if dir == '+':
1486 r = cmp(len(av), len(bv))
1487 if r != 0: return r
1488 elif dir == '-':
1489 r = cmp(len(bv), len(av))
1490 if r != 0: return r
1491 elif isinstance(propclass, Number) or isinstance(propclass, Boolean):
1492 if dir == '+':
1493 r = cmp(av, bv)
1494 elif dir == '-':
1495 r = cmp(bv, av)
1497 # end for dir, prop in list:
1498 # end for list in sort, group:
1499 # if all else fails, compare the ids
1500 return cmp(a[0], b[0])
1502 l.sort(sortfun)
1503 return [i[0] for i in l]
1505 def count(self):
1506 """Get the number of nodes in this class.
1508 If the returned integer is 'numnodes', the ids of all the nodes
1509 in this class run from 1 to numnodes, and numnodes+1 will be the
1510 id of the next node to be created in this class.
1511 """
1512 return self.db.countnodes(self.classname)
1514 # Manipulating properties:
1516 def getprops(self, protected=1):
1517 """Return a dictionary mapping property names to property objects.
1518 If the "protected" flag is true, we include protected properties -
1519 those which may not be modified.
1521 In addition to the actual properties on the node, these
1522 methods provide the "creation" and "activity" properties. If the
1523 "protected" flag is true, we include protected properties - those
1524 which may not be modified.
1525 """
1526 d = self.properties.copy()
1527 if protected:
1528 d['id'] = String()
1529 d['creation'] = hyperdb.Date()
1530 d['activity'] = hyperdb.Date()
1531 d['creator'] = hyperdb.Link("user")
1532 return d
1534 def addprop(self, **properties):
1535 """Add properties to this class.
1537 The keyword arguments in 'properties' must map names to property
1538 objects, or a TypeError is raised. None of the keys in 'properties'
1539 may collide with the names of existing properties, or a ValueError
1540 is raised before any properties have been added.
1541 """
1542 for key in properties.keys():
1543 if self.properties.has_key(key):
1544 raise ValueError, key
1545 self.properties.update(properties)
1547 def index(self, nodeid):
1548 '''Add (or refresh) the node to search indexes
1549 '''
1550 # find all the String properties that have indexme
1551 for prop, propclass in self.getprops().items():
1552 if isinstance(propclass, String) and propclass.indexme:
1553 # and index them under (classname, nodeid, property)
1554 self.db.indexer.add_text((self.classname, nodeid, prop),
1555 str(self.get(nodeid, prop)))
1557 #
1558 # Detector interface
1559 #
1560 def audit(self, event, detector):
1561 """Register a detector
1562 """
1563 l = self.auditors[event]
1564 if detector not in l:
1565 self.auditors[event].append(detector)
1567 def fireAuditors(self, action, nodeid, newvalues):
1568 """Fire all registered auditors.
1569 """
1570 for audit in self.auditors[action]:
1571 audit(self.db, self, nodeid, newvalues)
1573 def react(self, event, detector):
1574 """Register a detector
1575 """
1576 l = self.reactors[event]
1577 if detector not in l:
1578 self.reactors[event].append(detector)
1580 def fireReactors(self, action, nodeid, oldvalues):
1581 """Fire all registered reactors.
1582 """
1583 for react in self.reactors[action]:
1584 react(self.db, self, nodeid, oldvalues)
1586 class FileClass(Class):
1587 '''This class defines a large chunk of data. To support this, it has a
1588 mandatory String property "content" which is typically saved off
1589 externally to the hyperdb.
1591 The default MIME type of this data is defined by the
1592 "default_mime_type" class attribute, which may be overridden by each
1593 node if the class defines a "type" String property.
1594 '''
1595 default_mime_type = 'text/plain'
1597 def create(self, **propvalues):
1598 ''' snaffle the file propvalue and store in a file
1599 '''
1600 content = propvalues['content']
1601 del propvalues['content']
1602 newid = Class.create(self, **propvalues)
1603 self.db.storefile(self.classname, newid, None, content)
1604 return newid
1606 def get(self, nodeid, propname, default=_marker, cache=1):
1607 ''' trap the content propname and get it from the file
1608 '''
1610 poss_msg = 'Possibly a access right configuration problem.'
1611 if propname == 'content':
1612 try:
1613 return self.db.getfile(self.classname, nodeid, None)
1614 except IOError, (strerror):
1615 # BUG: by catching this we donot see an error in the log.
1616 return 'ERROR reading file: %s%s\n%s\n%s'%(
1617 self.classname, nodeid, poss_msg, strerror)
1618 if default is not _marker:
1619 return Class.get(self, nodeid, propname, default, cache=cache)
1620 else:
1621 return Class.get(self, nodeid, propname, cache=cache)
1623 def getprops(self, protected=1):
1624 ''' In addition to the actual properties on the node, these methods
1625 provide the "content" property. If the "protected" flag is true,
1626 we include protected properties - those which may not be
1627 modified.
1628 '''
1629 d = Class.getprops(self, protected=protected).copy()
1630 if protected:
1631 d['content'] = hyperdb.String()
1632 return d
1634 def index(self, nodeid):
1635 ''' Index the node in the search index.
1637 We want to index the content in addition to the normal String
1638 property indexing.
1639 '''
1640 # perform normal indexing
1641 Class.index(self, nodeid)
1643 # get the content to index
1644 content = self.get(nodeid, 'content')
1646 # figure the mime type
1647 if self.properties.has_key('type'):
1648 mime_type = self.get(nodeid, 'type')
1649 else:
1650 mime_type = self.default_mime_type
1652 # and index!
1653 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
1654 mime_type)
1656 # XXX deviation from spec - was called ItemClass
1657 class IssueClass(Class, roundupdb.IssueClass):
1658 # Overridden methods:
1659 def __init__(self, db, classname, **properties):
1660 """The newly-created class automatically includes the "messages",
1661 "files", "nosy", and "superseder" properties. If the 'properties'
1662 dictionary attempts to specify any of these properties or a
1663 "creation" or "activity" property, a ValueError is raised.
1664 """
1665 if not properties.has_key('title'):
1666 properties['title'] = hyperdb.String(indexme='yes')
1667 if not properties.has_key('messages'):
1668 properties['messages'] = hyperdb.Multilink("msg")
1669 if not properties.has_key('files'):
1670 properties['files'] = hyperdb.Multilink("file")
1671 if not properties.has_key('nosy'):
1672 properties['nosy'] = hyperdb.Multilink("user")
1673 if not properties.has_key('superseder'):
1674 properties['superseder'] = hyperdb.Multilink(classname)
1675 Class.__init__(self, db, classname, **properties)
1677 #
1678 #$Log: not supported by cvs2svn $
1679 #Revision 1.50 2002/07/18 11:50:58 richard
1680 #added tests for number type too
1681 #
1682 #Revision 1.49 2002/07/18 11:41:10 richard
1683 #added tests for boolean type, and fixes to anydbm backend
1684 #
1685 #Revision 1.48 2002/07/18 11:17:31 gmcm
1686 #Add Number and Boolean types to hyperdb.
1687 #Add conversion cases to web, mail & admin interfaces.
1688 #Add storage/serialization cases to back_anydbm & back_metakit.
1689 #
1690 #Revision 1.47 2002/07/14 23:18:20 richard
1691 #. fixed the journal bloat from multilink changes - we just log the add or
1692 # remove operations, not the whole list
1693 #
1694 #Revision 1.46 2002/07/14 06:06:34 richard
1695 #Did some old TODOs
1696 #
1697 #Revision 1.45 2002/07/14 04:03:14 richard
1698 #Implemented a switch to disable journalling for a Class. CGI session
1699 #database now uses it.
1700 #
1701 #Revision 1.44 2002/07/14 02:05:53 richard
1702 #. all storage-specific code (ie. backend) is now implemented by the backends
1703 #
1704 #Revision 1.43 2002/07/10 06:30:30 richard
1705 #...except of course it's nice to use valid Python syntax
1706 #
1707 #Revision 1.42 2002/07/10 06:21:38 richard
1708 #Be extra safe
1709 #
1710 #Revision 1.41 2002/07/10 00:21:45 richard
1711 #explicit database closing
1712 #
1713 #Revision 1.40 2002/07/09 04:19:09 richard
1714 #Added reindex command to roundup-admin.
1715 #Fixed reindex on first access.
1716 #Also fixed reindexing of entries that change.
1717 #
1718 #Revision 1.39 2002/07/09 03:02:52 richard
1719 #More indexer work:
1720 #- all String properties may now be indexed too. Currently there's a bit of
1721 # "issue" specific code in the actual searching which needs to be
1722 # addressed. In a nutshell:
1723 # + pass 'indexme="yes"' as a String() property initialisation arg, eg:
1724 # file = FileClass(db, "file", name=String(), type=String(),
1725 # comment=String(indexme="yes"))
1726 # + the comment will then be indexed and be searchable, with the results
1727 # related back to the issue that the file is linked to
1728 #- as a result of this work, the FileClass has a default MIME type that may
1729 # be overridden in a subclass, or by the use of a "type" property as is
1730 # done in the default templates.
1731 #- the regeneration of the indexes (if necessary) is done once the schema is
1732 # set up in the dbinit.
1733 #
1734 #Revision 1.38 2002/07/08 06:58:15 richard
1735 #cleaned up the indexer code:
1736 # - it splits more words out (much simpler, faster splitter)
1737 # - removed code we'll never use (roundup.roundup_indexer has the full
1738 # implementation, and replaces roundup.indexer)
1739 # - only index text/plain and rfc822/message (ideas for other text formats to
1740 # index are welcome)
1741 # - added simple unit test for indexer. Needs more tests for regression.
1742 #
1743 #Revision 1.37 2002/06/20 23:52:35 richard
1744 #More informative error message
1745 #
1746 #Revision 1.36 2002/06/19 03:07:19 richard
1747 #Moved the file storage commit into blobfiles where it belongs.
1748 #
1749 #Revision 1.35 2002/05/25 07:16:24 rochecompaan
1750 #Merged search_indexing-branch with HEAD
1751 #
1752 #Revision 1.34 2002/05/15 06:21:21 richard
1753 # . node caching now works, and gives a small boost in performance
1754 #
1755 #As a part of this, I cleaned up the DEBUG output and implemented TRACE
1756 #output (HYPERDBTRACE='file to trace to') with checkpoints at the start of
1757 #CGI requests. Run roundup with python -O to skip all the DEBUG/TRACE stuff
1758 #(using if __debug__ which is compiled out with -O)
1759 #
1760 #Revision 1.33 2002/04/24 10:38:26 rochecompaan
1761 #All database files are now created group readable and writable.
1762 #
1763 #Revision 1.32 2002/04/15 23:25:15 richard
1764 #. node ids are now generated from a lockable store - no more race conditions
1765 #
1766 #We're using the portalocker code by Jonathan Feinberg that was contributed
1767 #to the ASPN Python cookbook. This gives us locking across Unix and Windows.
1768 #
1769 #Revision 1.31 2002/04/03 05:54:31 richard
1770 #Fixed serialisation problem by moving the serialisation step out of the
1771 #hyperdb.Class (get, set) into the hyperdb.Database.
1772 #
1773 #Also fixed htmltemplate after the showid changes I made yesterday.
1774 #
1775 #Unit tests for all of the above written.
1776 #
1777 #Revision 1.30.2.1 2002/04/03 11:55:57 rochecompaan
1778 # . Added feature #526730 - search for messages capability
1779 #
1780 #Revision 1.30 2002/02/27 03:40:59 richard
1781 #Ran it through pychecker, made fixes
1782 #
1783 #Revision 1.29 2002/02/25 14:34:31 grubert
1784 # . use blobfiles in back_anydbm which is used in back_bsddb.
1785 # change test_db as dirlist does not work for subdirectories.
1786 # ATTENTION: blobfiles now creates subdirectories for files.
1787 #
1788 #Revision 1.28 2002/02/16 09:14:17 richard
1789 # . #514854 ] History: "User" is always ticket creator
1790 #
1791 #Revision 1.27 2002/01/22 07:21:13 richard
1792 #. fixed back_bsddb so it passed the journal tests
1793 #
1794 #... it didn't seem happy using the back_anydbm _open method, which is odd.
1795 #Yet another occurrance of whichdb not being able to recognise older bsddb
1796 #databases. Yadda yadda. Made the HYPERDBDEBUG stuff more sane in the
1797 #process.
1798 #
1799 #Revision 1.26 2002/01/22 05:18:38 rochecompaan
1800 #last_set_entry was referenced before assignment
1801 #
1802 #Revision 1.25 2002/01/22 05:06:08 rochecompaan
1803 #We need to keep the last 'set' entry in the journal to preserve
1804 #information on 'activity' for nodes.
1805 #
1806 #Revision 1.24 2002/01/21 16:33:20 rochecompaan
1807 #You can now use the roundup-admin tool to pack the database
1808 #
1809 #Revision 1.23 2002/01/18 04:32:04 richard
1810 #Rollback was breaking because a message hadn't actually been written to the file. Needs
1811 #more investigation.
1812 #
1813 #Revision 1.22 2002/01/14 02:20:15 richard
1814 # . changed all config accesses so they access either the instance or the
1815 # config attriubute on the db. This means that all config is obtained from
1816 # instance_config instead of the mish-mash of classes. This will make
1817 # switching to a ConfigParser setup easier too, I hope.
1818 #
1819 #At a minimum, this makes migration a _little_ easier (a lot easier in the
1820 #0.5.0 switch, I hope!)
1821 #
1822 #Revision 1.21 2002/01/02 02:31:38 richard
1823 #Sorry for the huge checkin message - I was only intending to implement #496356
1824 #but I found a number of places where things had been broken by transactions:
1825 # . modified ROUNDUPDBSENDMAILDEBUG to be SENDMAILDEBUG and hold a filename
1826 # for _all_ roundup-generated smtp messages to be sent to.
1827 # . the transaction cache had broken the roundupdb.Class set() reactors
1828 # . newly-created author users in the mailgw weren't being committed to the db
1829 #
1830 #Stuff that made it into CHANGES.txt (ie. the stuff I was actually working
1831 #on when I found that stuff :):
1832 # . #496356 ] Use threading in messages
1833 # . detectors were being registered multiple times
1834 # . added tests for mailgw
1835 # . much better attaching of erroneous messages in the mail gateway
1836 #
1837 #Revision 1.20 2001/12/18 15:30:34 rochecompaan
1838 #Fixed bugs:
1839 # . Fixed file creation and retrieval in same transaction in anydbm
1840 # backend
1841 # . Cgi interface now renders new issue after issue creation
1842 # . Could not set issue status to resolved through cgi interface
1843 # . Mail gateway was changing status back to 'chatting' if status was
1844 # omitted as an argument
1845 #
1846 #Revision 1.19 2001/12/17 03:52:48 richard
1847 #Implemented file store rollback. As a bonus, the hyperdb is now capable of
1848 #storing more than one file per node - if a property name is supplied,
1849 #the file is called designator.property.
1850 #I decided not to migrate the existing files stored over to the new naming
1851 #scheme - the FileClass just doesn't specify the property name.
1852 #
1853 #Revision 1.18 2001/12/16 10:53:38 richard
1854 #take a copy of the node dict so that the subsequent set
1855 #operation doesn't modify the oldvalues structure
1856 #
1857 #Revision 1.17 2001/12/14 23:42:57 richard
1858 #yuck, a gdbm instance tests false :(
1859 #I've left the debugging code in - it should be removed one day if we're ever
1860 #_really_ anal about performace :)
1861 #
1862 #Revision 1.16 2001/12/12 03:23:14 richard
1863 #Cor blimey this anydbm/whichdb stuff is yecchy. Turns out that whichdb
1864 #incorrectly identifies a dbm file as a dbhash file on my system. This has
1865 #been submitted to the python bug tracker as issue #491888:
1866 #https://sourceforge.net/tracker/index.php?func=detail&aid=491888&group_id=5470&atid=105470
1867 #
1868 #Revision 1.15 2001/12/12 02:30:51 richard
1869 #I fixed the problems with people whose anydbm was using the dbm module at the
1870 #backend. It turns out the dbm module modifies the file name to append ".db"
1871 #and my check to determine if we're opening an existing or new db just
1872 #tested os.path.exists() on the filename. Well, no longer! We now perform a
1873 #much better check _and_ cope with the anydbm implementation module changing
1874 #too!
1875 #I also fixed the backends __init__ so only ImportError is squashed.
1876 #
1877 #Revision 1.14 2001/12/10 22:20:01 richard
1878 #Enabled transaction support in the bsddb backend. It uses the anydbm code
1879 #where possible, only replacing methods where the db is opened (it uses the
1880 #btree opener specifically.)
1881 #Also cleaned up some change note generation.
1882 #Made the backends package work with pydoc too.
1883 #
1884 #Revision 1.13 2001/12/02 05:06:16 richard
1885 #. We now use weakrefs in the Classes to keep the database reference, so
1886 # the close() method on the database is no longer needed.
1887 # I bumped the minimum python requirement up to 2.1 accordingly.
1888 #. #487480 ] roundup-server
1889 #. #487476 ] INSTALL.txt
1890 #
1891 #I also cleaned up the change message / post-edit stuff in the cgi client.
1892 #There's now a clearly marked "TODO: append the change note" where I believe
1893 #the change note should be added there. The "changes" list will obviously
1894 #have to be modified to be a dict of the changes, or somesuch.
1895 #
1896 #More testing needed.
1897 #
1898 #Revision 1.12 2001/12/01 07:17:50 richard
1899 #. We now have basic transaction support! Information is only written to
1900 # the database when the commit() method is called. Only the anydbm
1901 # backend is modified in this way - neither of the bsddb backends have been.
1902 # The mail, admin and cgi interfaces all use commit (except the admin tool
1903 # doesn't have a commit command, so interactive users can't commit...)
1904 #. Fixed login/registration forwarding the user to the right page (or not,
1905 # on a failure)
1906 #
1907 #Revision 1.11 2001/11/21 02:34:18 richard
1908 #Added a target version field to the extended issue schema
1909 #
1910 #Revision 1.10 2001/10/09 23:58:10 richard
1911 #Moved the data stringification up into the hyperdb.Class class' get, set
1912 #and create methods. This means that the data is also stringified for the
1913 #journal call, and removes duplication of code from the backends. The
1914 #backend code now only sees strings.
1915 #
1916 #Revision 1.9 2001/10/09 07:25:59 richard
1917 #Added the Password property type. See "pydoc roundup.password" for
1918 #implementation details. Have updated some of the documentation too.
1919 #
1920 #Revision 1.8 2001/09/29 13:27:00 richard
1921 #CGI interfaces now spit up a top-level index of all the instances they can
1922 #serve.
1923 #
1924 #Revision 1.7 2001/08/12 06:32:36 richard
1925 #using isinstance(blah, Foo) now instead of isFooType
1926 #
1927 #Revision 1.6 2001/08/07 00:24:42 richard
1928 #stupid typo
1929 #
1930 #Revision 1.5 2001/08/07 00:15:51 richard
1931 #Added the copyright/license notice to (nearly) all files at request of
1932 #Bizar Software.
1933 #
1934 #Revision 1.4 2001/07/30 01:41:36 richard
1935 #Makes schema changes mucho easier.
1936 #
1937 #Revision 1.3 2001/07/25 01:23:07 richard
1938 #Added the Roundup spec to the new documentation directory.
1939 #
1940 #Revision 1.2 2001/07/23 08:20:44 richard
1941 #Moved over to using marshal in the bsddb and anydbm backends.
1942 #roundup-admin now has a "freshen" command that'll load/save all nodes (not
1943 # retired - mod hyperdb.Class.list() so it lists retired nodes)
1944 #
1945 #