1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: back_anydbm.py,v 1.57 2002-07-31 23:57:36 richard Exp $
19 '''
20 This module defines a backend that saves the hyperdatabase in a database
21 chosen by anydbm. It is guaranteed to always be available in python
22 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
23 serious bugs, and is not available)
24 '''
26 import whichdb, anydbm, os, marshal, re, weakref, string, copy
27 from roundup import hyperdb, date, password, roundupdb, security
28 from blobfiles import FileStorage
29 from sessions import Sessions
30 from roundup.indexer import Indexer
31 from locking import acquire_lock, release_lock
32 from roundup.hyperdb import String, Password, Date, Interval, Link, \
33 Multilink, DatabaseError, Boolean, Number
35 #
36 # Now the database
37 #
38 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
39 """A database for storing records containing flexible data types.
41 Transaction stuff TODO:
42 . check the timestamp of the class file and nuke the cache if it's
43 modified. Do some sort of conflict checking on the dirty stuff.
44 . perhaps detect write collisions (related to above)?
46 """
47 def __init__(self, config, journaltag=None):
48 """Open a hyperdatabase given a specifier to some storage.
50 The 'storagelocator' is obtained from config.DATABASE.
51 The meaning of 'storagelocator' depends on the particular
52 implementation of the hyperdatabase. It could be a file name,
53 a directory path, a socket descriptor for a connection to a
54 database over the network, etc.
56 The 'journaltag' is a token that will be attached to the journal
57 entries for any edits done on the database. If 'journaltag' is
58 None, the database is opened in read-only mode: the Class.create(),
59 Class.set(), and Class.retire() methods are disabled.
60 """
61 self.config, self.journaltag = config, journaltag
62 self.dir = config.DATABASE
63 self.classes = {}
64 self.cache = {} # cache of nodes loaded or created
65 self.dirtynodes = {} # keep track of the dirty nodes by class
66 self.newnodes = {} # keep track of the new nodes by class
67 self.destroyednodes = {}# keep track of the destroyed nodes by class
68 self.transactions = []
69 self.indexer = Indexer(self.dir)
70 self.sessions = Sessions(self.config)
71 self.security = security.Security(self)
72 # ensure files are group readable and writable
73 os.umask(0002)
75 def post_init(self):
76 """Called once the schema initialisation has finished."""
77 # reindex the db if necessary
78 if self.indexer.should_reindex():
79 self.reindex()
81 def reindex(self):
82 for klass in self.classes.values():
83 for nodeid in klass.list():
84 klass.index(nodeid)
85 self.indexer.save_index()
87 def __repr__(self):
88 return '<back_anydbm instance at %x>'%id(self)
90 #
91 # Classes
92 #
93 def __getattr__(self, classname):
94 """A convenient way of calling self.getclass(classname)."""
95 if self.classes.has_key(classname):
96 if __debug__:
97 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
98 return self.classes[classname]
99 raise AttributeError, classname
101 def addclass(self, cl):
102 if __debug__:
103 print >>hyperdb.DEBUG, 'addclass', (self, cl)
104 cn = cl.classname
105 if self.classes.has_key(cn):
106 raise ValueError, cn
107 self.classes[cn] = cl
109 def getclasses(self):
110 """Return a list of the names of all existing classes."""
111 if __debug__:
112 print >>hyperdb.DEBUG, 'getclasses', (self,)
113 l = self.classes.keys()
114 l.sort()
115 return l
117 def getclass(self, classname):
118 """Get the Class object representing a particular class.
120 If 'classname' is not a valid class name, a KeyError is raised.
121 """
122 if __debug__:
123 print >>hyperdb.DEBUG, 'getclass', (self, classname)
124 return self.classes[classname]
126 #
127 # Class DBs
128 #
129 def clear(self):
130 '''Delete all database contents
131 '''
132 if __debug__:
133 print >>hyperdb.DEBUG, 'clear', (self,)
134 for cn in self.classes.keys():
135 for dummy in 'nodes', 'journals':
136 path = os.path.join(self.dir, 'journals.%s'%cn)
137 if os.path.exists(path):
138 os.remove(path)
139 elif os.path.exists(path+'.db'): # dbm appends .db
140 os.remove(path+'.db')
142 def getclassdb(self, classname, mode='r'):
143 ''' grab a connection to the class db that will be used for
144 multiple actions
145 '''
146 if __debug__:
147 print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode)
148 return self.opendb('nodes.%s'%classname, mode)
150 def determine_db_type(self, path):
151 ''' determine which DB wrote the class file
152 '''
153 db_type = ''
154 if os.path.exists(path):
155 db_type = whichdb.whichdb(path)
156 if not db_type:
157 raise hyperdb.DatabaseError, "Couldn't identify database type"
158 elif os.path.exists(path+'.db'):
159 # if the path ends in '.db', it's a dbm database, whether
160 # anydbm says it's dbhash or not!
161 db_type = 'dbm'
162 return db_type
164 def opendb(self, name, mode):
165 '''Low-level database opener that gets around anydbm/dbm
166 eccentricities.
167 '''
168 if __debug__:
169 print >>hyperdb.DEBUG, 'opendb', (self, name, mode)
171 # figure the class db type
172 path = os.path.join(os.getcwd(), self.dir, name)
173 db_type = self.determine_db_type(path)
175 # new database? let anydbm pick the best dbm
176 if not db_type:
177 if __debug__:
178 print >>hyperdb.DEBUG, "opendb anydbm.open(%r, 'n')"%path
179 return anydbm.open(path, 'n')
181 # open the database with the correct module
182 try:
183 dbm = __import__(db_type)
184 except ImportError:
185 raise hyperdb.DatabaseError, \
186 "Couldn't open database - the required module '%s'"\
187 " is not available"%db_type
188 if __debug__:
189 print >>hyperdb.DEBUG, "opendb %r.open(%r, %r)"%(db_type, path,
190 mode)
191 return dbm.open(path, mode)
193 def lockdb(self, name):
194 ''' Lock a database file
195 '''
196 path = os.path.join(os.getcwd(), self.dir, '%s.lock'%name)
197 return acquire_lock(path)
199 #
200 # Node IDs
201 #
202 def newid(self, classname):
203 ''' Generate a new id for the given class
204 '''
205 # open the ids DB - create if if doesn't exist
206 lock = self.lockdb('_ids')
207 db = self.opendb('_ids', 'c')
208 if db.has_key(classname):
209 newid = db[classname] = str(int(db[classname]) + 1)
210 else:
211 # the count() bit is transitional - older dbs won't start at 1
212 newid = str(self.getclass(classname).count()+1)
213 db[classname] = newid
214 db.close()
215 release_lock(lock)
216 return newid
218 #
219 # Nodes
220 #
221 def addnode(self, classname, nodeid, node):
222 ''' add the specified node to its class's db
223 '''
224 if __debug__:
225 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
226 self.newnodes.setdefault(classname, {})[nodeid] = 1
227 self.cache.setdefault(classname, {})[nodeid] = node
228 self.savenode(classname, nodeid, node)
230 def setnode(self, classname, nodeid, node):
231 ''' change the specified node
232 '''
233 if __debug__:
234 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
235 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
237 # can't set without having already loaded the node
238 self.cache[classname][nodeid] = node
239 self.savenode(classname, nodeid, node)
241 def savenode(self, classname, nodeid, node):
242 ''' perform the saving of data specified by the set/addnode
243 '''
244 if __debug__:
245 print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node)
246 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
248 def getnode(self, classname, nodeid, db=None, cache=1):
249 ''' get a node from the database
250 '''
251 if __debug__:
252 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid, db)
253 if cache:
254 # try the cache
255 cache_dict = self.cache.setdefault(classname, {})
256 if cache_dict.has_key(nodeid):
257 if __debug__:
258 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
259 nodeid)
260 return cache_dict[nodeid]
262 if __debug__:
263 print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid)
265 # get from the database and save in the cache
266 if db is None:
267 db = self.getclassdb(classname)
268 if not db.has_key(nodeid):
269 raise IndexError, "no such %s %s"%(classname, nodeid)
271 # check the uncommitted, destroyed nodes
272 if (self.destroyednodes.has_key(classname) and
273 self.destroyednodes[classname].has_key(nodeid)):
274 raise IndexError, "no such %s %s"%(classname, nodeid)
276 # decode
277 res = marshal.loads(db[nodeid])
279 # reverse the serialisation
280 res = self.unserialise(classname, res)
282 # store off in the cache dict
283 if cache:
284 cache_dict[nodeid] = res
286 return res
288 def destroynode(self, classname, nodeid):
289 '''Remove a node from the database. Called exclusively by the
290 destroy() method on Class.
291 '''
292 if __debug__:
293 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
295 # remove from cache and newnodes if it's there
296 if (self.cache.has_key(classname) and
297 self.cache[classname].has_key(nodeid)):
298 del self.cache[classname][nodeid]
299 if (self.newnodes.has_key(classname) and
300 self.newnodes[classname].has_key(nodeid)):
301 del self.newnodes[classname][nodeid]
303 # see if there's any obvious commit actions that we should get rid of
304 for entry in self.transactions[:]:
305 if entry[1][:2] == (classname, nodeid):
306 self.transactions.remove(entry)
308 # add to the destroyednodes map
309 self.destroyednodes.setdefault(classname, {})[nodeid] = 1
311 # add the destroy commit action
312 self.transactions.append((self.doDestroyNode, (classname, nodeid)))
314 def serialise(self, classname, node):
315 '''Copy the node contents, converting non-marshallable data into
316 marshallable data.
317 '''
318 if __debug__:
319 print >>hyperdb.DEBUG, 'serialise', classname, node
320 properties = self.getclass(classname).getprops()
321 d = {}
322 for k, v in node.items():
323 # if the property doesn't exist, or is the "retired" flag then
324 # it won't be in the properties dict
325 if not properties.has_key(k):
326 d[k] = v
327 continue
329 # get the property spec
330 prop = properties[k]
332 if isinstance(prop, Password):
333 d[k] = str(v)
334 elif isinstance(prop, Date) and v is not None:
335 d[k] = v.get_tuple()
336 elif isinstance(prop, Interval) and v is not None:
337 d[k] = v.get_tuple()
338 else:
339 d[k] = v
340 return d
342 def unserialise(self, classname, node):
343 '''Decode the marshalled node data
344 '''
345 if __debug__:
346 print >>hyperdb.DEBUG, 'unserialise', classname, node
347 properties = self.getclass(classname).getprops()
348 d = {}
349 for k, v in node.items():
350 # if the property doesn't exist, or is the "retired" flag then
351 # it won't be in the properties dict
352 if not properties.has_key(k):
353 d[k] = v
354 continue
356 # get the property spec
357 prop = properties[k]
359 if isinstance(prop, Date) and v is not None:
360 d[k] = date.Date(v)
361 elif isinstance(prop, Interval) and v is not None:
362 d[k] = date.Interval(v)
363 elif isinstance(prop, Password):
364 p = password.Password()
365 p.unpack(v)
366 d[k] = p
367 else:
368 d[k] = v
369 return d
371 def hasnode(self, classname, nodeid, db=None):
372 ''' determine if the database has a given node
373 '''
374 if __debug__:
375 print >>hyperdb.DEBUG, 'hasnode', (self, classname, nodeid, db)
377 # try the cache
378 cache = self.cache.setdefault(classname, {})
379 if cache.has_key(nodeid):
380 if __debug__:
381 print >>hyperdb.TRACE, 'has %s %s cached'%(classname, nodeid)
382 return 1
383 if __debug__:
384 print >>hyperdb.TRACE, 'has %s %s'%(classname, nodeid)
386 # not in the cache - check the database
387 if db is None:
388 db = self.getclassdb(classname)
389 res = db.has_key(nodeid)
390 return res
392 def countnodes(self, classname, db=None):
393 if __debug__:
394 print >>hyperdb.DEBUG, 'countnodes', (self, classname, db)
396 count = 0
398 # include the uncommitted nodes
399 if self.newnodes.has_key(classname):
400 count += len(self.newnodes[classname])
401 if self.destroyednodes.has_key(classname):
402 count -= len(self.destroyednodes[classname])
404 # and count those in the DB
405 if db is None:
406 db = self.getclassdb(classname)
407 count = count + len(db.keys())
408 return count
410 def getnodeids(self, classname, db=None):
411 if __debug__:
412 print >>hyperdb.DEBUG, 'getnodeids', (self, classname, db)
414 res = []
416 # start off with the new nodes
417 if self.newnodes.has_key(classname):
418 res += self.newnodes[classname].keys()
420 if db is None:
421 db = self.getclassdb(classname)
422 res = res + db.keys()
424 # remove the uncommitted, destroyed nodes
425 if self.destroyednodes.has_key(classname):
426 for nodeid in self.destroyednodes[classname].keys():
427 if db.has_key(nodeid):
428 res.remove(nodeid)
430 return res
433 #
434 # Files - special node properties
435 # inherited from FileStorage
437 #
438 # Journal
439 #
440 def addjournal(self, classname, nodeid, action, params):
441 ''' Journal the Action
442 'action' may be:
444 'create' or 'set' -- 'params' is a dictionary of property values
445 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
446 'retire' -- 'params' is None
447 '''
448 if __debug__:
449 print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid,
450 action, params)
451 self.transactions.append((self.doSaveJournal, (classname, nodeid,
452 action, params)))
454 def getjournal(self, classname, nodeid):
455 ''' get the journal for id
457 Raise IndexError if the node doesn't exist (as per history()'s
458 API)
459 '''
460 if __debug__:
461 print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid)
462 # attempt to open the journal - in some rare cases, the journal may
463 # not exist
464 try:
465 db = self.opendb('journals.%s'%classname, 'r')
466 except anydbm.error, error:
467 if str(error) == "need 'c' or 'n' flag to open new db":
468 raise IndexError, 'no such %s %s'%(classname, nodeid)
469 elif error.args[0] != 2:
470 raise
471 raise IndexError, 'no such %s %s'%(classname, nodeid)
472 try:
473 journal = marshal.loads(db[nodeid])
474 except KeyError:
475 db.close()
476 raise IndexError, 'no such %s %s'%(classname, nodeid)
477 db.close()
478 res = []
479 for nodeid, date_stamp, user, action, params in journal:
480 res.append((nodeid, date.Date(date_stamp), user, action, params))
481 return res
483 def pack(self, pack_before):
484 ''' delete all journal entries before 'pack_before' '''
485 if __debug__:
486 print >>hyperdb.DEBUG, 'packjournal', (self, pack_before)
488 pack_before = pack_before.get_tuple()
490 classes = self.getclasses()
492 # figure the class db type
494 for classname in classes:
495 db_name = 'journals.%s'%classname
496 path = os.path.join(os.getcwd(), self.dir, classname)
497 db_type = self.determine_db_type(path)
498 db = self.opendb(db_name, 'w')
500 for key in db.keys():
501 journal = marshal.loads(db[key])
502 l = []
503 last_set_entry = None
504 for entry in journal:
505 (nodeid, date_stamp, self.journaltag, action,
506 params) = entry
507 if date_stamp > pack_before or action == 'create':
508 l.append(entry)
509 elif action == 'set':
510 # grab the last set entry to keep information on
511 # activity
512 last_set_entry = entry
513 if last_set_entry:
514 date_stamp = last_set_entry[1]
515 # if the last set entry was made after the pack date
516 # then it is already in the list
517 if date_stamp < pack_before:
518 l.append(last_set_entry)
519 db[key] = marshal.dumps(l)
520 if db_type == 'gdbm':
521 db.reorganize()
522 db.close()
525 #
526 # Basic transaction support
527 #
528 def commit(self):
529 ''' Commit the current transactions.
530 '''
531 if __debug__:
532 print >>hyperdb.DEBUG, 'commit', (self,)
533 # TODO: lock the DB
535 # keep a handle to all the database files opened
536 self.databases = {}
538 # now, do all the transactions
539 reindex = {}
540 for method, args in self.transactions:
541 reindex[method(*args)] = 1
543 # now close all the database files
544 for db in self.databases.values():
545 db.close()
546 del self.databases
547 # TODO: unlock the DB
549 # reindex the nodes that request it
550 for classname, nodeid in filter(None, reindex.keys()):
551 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
552 self.getclass(classname).index(nodeid)
554 # save the indexer state
555 self.indexer.save_index()
557 # all transactions committed, back to normal
558 self.cache = {}
559 self.dirtynodes = {}
560 self.newnodes = {}
561 self.destroyednodes = {}
562 self.transactions = []
564 def getCachedClassDB(self, classname):
565 ''' get the class db, looking in our cache of databases for commit
566 '''
567 # get the database handle
568 db_name = 'nodes.%s'%classname
569 if not self.databases.has_key(db_name):
570 self.databases[db_name] = self.getclassdb(classname, 'c')
571 return self.databases[db_name]
573 def doSaveNode(self, classname, nodeid, node):
574 if __debug__:
575 print >>hyperdb.DEBUG, 'doSaveNode', (self, classname, nodeid,
576 node)
578 db = self.getCachedClassDB(classname)
580 # now save the marshalled data
581 db[nodeid] = marshal.dumps(self.serialise(classname, node))
583 # return the classname, nodeid so we reindex this content
584 return (classname, nodeid)
586 def getCachedJournalDB(self, classname):
587 ''' get the journal db, looking in our cache of databases for commit
588 '''
589 # get the database handle
590 db_name = 'journals.%s'%classname
591 if not self.databases.has_key(db_name):
592 self.databases[db_name] = self.opendb(db_name, 'c')
593 return self.databases[db_name]
595 def doSaveJournal(self, classname, nodeid, action, params):
596 # serialise first
597 if action in ('set', 'create'):
598 params = self.serialise(classname, params)
600 # create the journal entry
601 entry = (nodeid, date.Date().get_tuple(), self.journaltag, action,
602 params)
604 if __debug__:
605 print >>hyperdb.DEBUG, 'doSaveJournal', entry
607 db = self.getCachedJournalDB(classname)
609 # now insert the journal entry
610 if db.has_key(nodeid):
611 # append to existing
612 s = db[nodeid]
613 l = marshal.loads(s)
614 l.append(entry)
615 else:
616 l = [entry]
618 db[nodeid] = marshal.dumps(l)
620 def doDestroyNode(self, classname, nodeid):
621 if __debug__:
622 print >>hyperdb.DEBUG, 'doDestroyNode', (self, classname, nodeid)
624 # delete from the class database
625 db = self.getCachedClassDB(classname)
626 if db.has_key(nodeid):
627 del db[nodeid]
629 # delete from the database
630 db = self.getCachedJournalDB(classname)
631 if db.has_key(nodeid):
632 del db[nodeid]
634 # return the classname, nodeid so we reindex this content
635 return (classname, nodeid)
637 def rollback(self):
638 ''' Reverse all actions from the current transaction.
639 '''
640 if __debug__:
641 print >>hyperdb.DEBUG, 'rollback', (self, )
642 for method, args in self.transactions:
643 # delete temporary files
644 if method == self.doStoreFile:
645 self.rollbackStoreFile(*args)
646 self.cache = {}
647 self.dirtynodes = {}
648 self.newnodes = {}
649 self.destroyednodes = {}
650 self.transactions = []
652 _marker = []
653 class Class(hyperdb.Class):
654 """The handle to a particular class of nodes in a hyperdatabase."""
656 def __init__(self, db, classname, **properties):
657 """Create a new class with a given name and property specification.
659 'classname' must not collide with the name of an existing class,
660 or a ValueError is raised. The keyword arguments in 'properties'
661 must map names to property objects, or a TypeError is raised.
662 """
663 if (properties.has_key('creation') or properties.has_key('activity')
664 or properties.has_key('creator')):
665 raise ValueError, '"creation", "activity" and "creator" are '\
666 'reserved'
668 self.classname = classname
669 self.properties = properties
670 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
671 self.key = ''
673 # should we journal changes (default yes)
674 self.do_journal = 1
676 # do the db-related init stuff
677 db.addclass(self)
679 self.auditors = {'create': [], 'set': [], 'retire': []}
680 self.reactors = {'create': [], 'set': [], 'retire': []}
682 def enableJournalling(self):
683 '''Turn journalling on for this class
684 '''
685 self.do_journal = 1
687 def disableJournalling(self):
688 '''Turn journalling off for this class
689 '''
690 self.do_journal = 0
692 # Editing nodes:
694 def create(self, **propvalues):
695 """Create a new node of this class and return its id.
697 The keyword arguments in 'propvalues' map property names to values.
699 The values of arguments must be acceptable for the types of their
700 corresponding properties or a TypeError is raised.
702 If this class has a key property, it must be present and its value
703 must not collide with other key strings or a ValueError is raised.
705 Any other properties on this class that are missing from the
706 'propvalues' dictionary are set to None.
708 If an id in a link or multilink property does not refer to a valid
709 node, an IndexError is raised.
711 These operations trigger detectors and can be vetoed. Attempts
712 to modify the "creation" or "activity" properties cause a KeyError.
713 """
714 if propvalues.has_key('id'):
715 raise KeyError, '"id" is reserved'
717 if self.db.journaltag is None:
718 raise DatabaseError, 'Database open read-only'
720 if propvalues.has_key('creation') or propvalues.has_key('activity'):
721 raise KeyError, '"creation" and "activity" are reserved'
723 self.fireAuditors('create', None, propvalues)
725 # new node's id
726 newid = self.db.newid(self.classname)
728 # validate propvalues
729 num_re = re.compile('^\d+$')
730 for key, value in propvalues.items():
731 if key == self.key:
732 try:
733 self.lookup(value)
734 except KeyError:
735 pass
736 else:
737 raise ValueError, 'node with key "%s" exists'%value
739 # try to handle this property
740 try:
741 prop = self.properties[key]
742 except KeyError:
743 raise KeyError, '"%s" has no property "%s"'%(self.classname,
744 key)
746 if isinstance(prop, Link):
747 if type(value) != type(''):
748 raise ValueError, 'link value must be String'
749 link_class = self.properties[key].classname
750 # if it isn't a number, it's a key
751 if not num_re.match(value):
752 try:
753 value = self.db.classes[link_class].lookup(value)
754 except (TypeError, KeyError):
755 raise IndexError, 'new property "%s": %s not a %s'%(
756 key, value, link_class)
757 elif not self.db.getclass(link_class).hasnode(value):
758 raise IndexError, '%s has no node %s'%(link_class, value)
760 # save off the value
761 propvalues[key] = value
763 # register the link with the newly linked node
764 if self.do_journal and self.properties[key].do_journal:
765 self.db.addjournal(link_class, value, 'link',
766 (self.classname, newid, key))
768 elif isinstance(prop, Multilink):
769 if type(value) != type([]):
770 raise TypeError, 'new property "%s" not a list of ids'%key
772 # clean up and validate the list of links
773 link_class = self.properties[key].classname
774 l = []
775 for entry in value:
776 if type(entry) != type(''):
777 raise ValueError, '"%s" link value (%s) must be '\
778 'String'%(key, value)
779 # if it isn't a number, it's a key
780 if not num_re.match(entry):
781 try:
782 entry = self.db.classes[link_class].lookup(entry)
783 except (TypeError, KeyError):
784 raise IndexError, 'new property "%s": %s not a %s'%(
785 key, entry, self.properties[key].classname)
786 l.append(entry)
787 value = l
788 propvalues[key] = value
790 # handle additions
791 for nodeid in value:
792 if not self.db.getclass(link_class).hasnode(nodeid):
793 raise IndexError, '%s has no node %s'%(link_class,
794 nodeid)
795 # register the link with the newly linked node
796 if self.do_journal and self.properties[key].do_journal:
797 self.db.addjournal(link_class, nodeid, 'link',
798 (self.classname, newid, key))
800 elif isinstance(prop, String):
801 if type(value) != type(''):
802 raise TypeError, 'new property "%s" not a string'%key
804 elif isinstance(prop, Password):
805 if not isinstance(value, password.Password):
806 raise TypeError, 'new property "%s" not a Password'%key
808 elif isinstance(prop, Date):
809 if value is not None and not isinstance(value, date.Date):
810 raise TypeError, 'new property "%s" not a Date'%key
812 elif isinstance(prop, Interval):
813 if value is not None and not isinstance(value, date.Interval):
814 raise TypeError, 'new property "%s" not an Interval'%key
816 elif value is not None and isinstance(prop, Number):
817 try:
818 float(value)
819 except ValueError:
820 raise TypeError, 'new property "%s" not numeric'%key
822 elif value is not None and isinstance(prop, Boolean):
823 try:
824 int(value)
825 except ValueError:
826 raise TypeError, 'new property "%s" not boolean'%key
828 # make sure there's data where there needs to be
829 for key, prop in self.properties.items():
830 if propvalues.has_key(key):
831 continue
832 if key == self.key:
833 raise ValueError, 'key property "%s" is required'%key
834 if isinstance(prop, Multilink):
835 propvalues[key] = []
836 else:
837 propvalues[key] = None
839 # done
840 self.db.addnode(self.classname, newid, propvalues)
841 if self.do_journal:
842 self.db.addjournal(self.classname, newid, 'create', propvalues)
844 self.fireReactors('create', newid, None)
846 return newid
848 def get(self, nodeid, propname, default=_marker, cache=1):
849 """Get the value of a property on an existing node of this class.
851 'nodeid' must be the id of an existing node of this class or an
852 IndexError is raised. 'propname' must be the name of a property
853 of this class or a KeyError is raised.
855 'cache' indicates whether the transaction cache should be queried
856 for the node. If the node has been modified and you need to
857 determine what its values prior to modification are, you need to
858 set cache=0.
860 Attempts to get the "creation" or "activity" properties should
861 do the right thing.
862 """
863 if propname == 'id':
864 return nodeid
866 if propname == 'creation':
867 if not self.do_journal:
868 raise ValueError, 'Journalling is disabled for this class'
869 journal = self.db.getjournal(self.classname, nodeid)
870 if journal:
871 return self.db.getjournal(self.classname, nodeid)[0][1]
872 else:
873 # on the strange chance that there's no journal
874 return date.Date()
875 if propname == 'activity':
876 if not self.do_journal:
877 raise ValueError, 'Journalling is disabled for this class'
878 journal = self.db.getjournal(self.classname, nodeid)
879 if journal:
880 return self.db.getjournal(self.classname, nodeid)[-1][1]
881 else:
882 # on the strange chance that there's no journal
883 return date.Date()
884 if propname == 'creator':
885 if not self.do_journal:
886 raise ValueError, 'Journalling is disabled for this class'
887 journal = self.db.getjournal(self.classname, nodeid)
888 if journal:
889 name = self.db.getjournal(self.classname, nodeid)[0][2]
890 else:
891 return None
892 return self.db.user.lookup(name)
894 # get the property (raises KeyErorr if invalid)
895 prop = self.properties[propname]
897 # get the node's dict
898 d = self.db.getnode(self.classname, nodeid, cache=cache)
900 if not d.has_key(propname):
901 if default is _marker:
902 if isinstance(prop, Multilink):
903 return []
904 else:
905 return None
906 else:
907 return default
909 return d[propname]
911 # XXX not in spec
912 def getnode(self, nodeid, cache=1):
913 ''' Return a convenience wrapper for the node.
915 'nodeid' must be the id of an existing node of this class or an
916 IndexError is raised.
918 'cache' indicates whether the transaction cache should be queried
919 for the node. If the node has been modified and you need to
920 determine what its values prior to modification are, you need to
921 set cache=0.
922 '''
923 return Node(self, nodeid, cache=cache)
925 def set(self, nodeid, **propvalues):
926 """Modify a property on an existing node of this class.
928 'nodeid' must be the id of an existing node of this class or an
929 IndexError is raised.
931 Each key in 'propvalues' must be the name of a property of this
932 class or a KeyError is raised.
934 All values in 'propvalues' must be acceptable types for their
935 corresponding properties or a TypeError is raised.
937 If the value of the key property is set, it must not collide with
938 other key strings or a ValueError is raised.
940 If the value of a Link or Multilink property contains an invalid
941 node id, a ValueError is raised.
943 These operations trigger detectors and can be vetoed. Attempts
944 to modify the "creation" or "activity" properties cause a KeyError.
945 """
946 if not propvalues:
947 return
949 if propvalues.has_key('creation') or propvalues.has_key('activity'):
950 raise KeyError, '"creation" and "activity" are reserved'
952 if propvalues.has_key('id'):
953 raise KeyError, '"id" is reserved'
955 if self.db.journaltag is None:
956 raise DatabaseError, 'Database open read-only'
958 self.fireAuditors('set', nodeid, propvalues)
959 # Take a copy of the node dict so that the subsequent set
960 # operation doesn't modify the oldvalues structure.
961 try:
962 # try not using the cache initially
963 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid,
964 cache=0))
965 except IndexError:
966 # this will be needed if somone does a create() and set()
967 # with no intervening commit()
968 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
970 node = self.db.getnode(self.classname, nodeid)
971 if node.has_key(self.db.RETIRED_FLAG):
972 raise IndexError
973 num_re = re.compile('^\d+$')
975 # if the journal value is to be different, store it in here
976 journalvalues = {}
978 for propname, value in propvalues.items():
979 # check to make sure we're not duplicating an existing key
980 if propname == self.key and node[propname] != value:
981 try:
982 self.lookup(value)
983 except KeyError:
984 pass
985 else:
986 raise ValueError, 'node with key "%s" exists'%value
988 # this will raise the KeyError if the property isn't valid
989 # ... we don't use getprops() here because we only care about
990 # the writeable properties.
991 prop = self.properties[propname]
993 # if the value's the same as the existing value, no sense in
994 # doing anything
995 if node.has_key(propname) and value == node[propname]:
996 del propvalues[propname]
997 continue
999 # do stuff based on the prop type
1000 if isinstance(prop, Link):
1001 link_class = prop.classname
1002 # if it isn't a number, it's a key
1003 if value is not None and not isinstance(value, type('')):
1004 raise ValueError, 'property "%s" link value be a string'%(
1005 propname)
1006 if isinstance(value, type('')) and not num_re.match(value):
1007 try:
1008 value = self.db.classes[link_class].lookup(value)
1009 except (TypeError, KeyError):
1010 raise IndexError, 'new property "%s": %s not a %s'%(
1011 propname, value, prop.classname)
1013 if (value is not None and
1014 not self.db.getclass(link_class).hasnode(value)):
1015 raise IndexError, '%s has no node %s'%(link_class, value)
1017 if self.do_journal and prop.do_journal:
1018 # register the unlink with the old linked node
1019 if node[propname] is not None:
1020 self.db.addjournal(link_class, node[propname], 'unlink',
1021 (self.classname, nodeid, propname))
1023 # register the link with the newly linked node
1024 if value is not None:
1025 self.db.addjournal(link_class, value, 'link',
1026 (self.classname, nodeid, propname))
1028 elif isinstance(prop, Multilink):
1029 if type(value) != type([]):
1030 raise TypeError, 'new property "%s" not a list of'\
1031 ' ids'%propname
1032 link_class = self.properties[propname].classname
1033 l = []
1034 for entry in value:
1035 # if it isn't a number, it's a key
1036 if type(entry) != type(''):
1037 raise ValueError, 'new property "%s" link value ' \
1038 'must be a string'%propname
1039 if not num_re.match(entry):
1040 try:
1041 entry = self.db.classes[link_class].lookup(entry)
1042 except (TypeError, KeyError):
1043 raise IndexError, 'new property "%s": %s not a %s'%(
1044 propname, entry,
1045 self.properties[propname].classname)
1046 l.append(entry)
1047 value = l
1048 propvalues[propname] = value
1050 # figure the journal entry for this property
1051 add = []
1052 remove = []
1054 # handle removals
1055 if node.has_key(propname):
1056 l = node[propname]
1057 else:
1058 l = []
1059 for id in l[:]:
1060 if id in value:
1061 continue
1062 # register the unlink with the old linked node
1063 if self.do_journal and self.properties[propname].do_journal:
1064 self.db.addjournal(link_class, id, 'unlink',
1065 (self.classname, nodeid, propname))
1066 l.remove(id)
1067 remove.append(id)
1069 # handle additions
1070 for id in value:
1071 if not self.db.getclass(link_class).hasnode(id):
1072 raise IndexError, '%s has no node %s'%(link_class, id)
1073 if id in l:
1074 continue
1075 # register the link with the newly linked node
1076 if self.do_journal and self.properties[propname].do_journal:
1077 self.db.addjournal(link_class, id, 'link',
1078 (self.classname, nodeid, propname))
1079 l.append(id)
1080 add.append(id)
1082 # figure the journal entry
1083 l = []
1084 if add:
1085 l.append(('add', add))
1086 if remove:
1087 l.append(('remove', remove))
1088 if l:
1089 journalvalues[propname] = tuple(l)
1091 elif isinstance(prop, String):
1092 if value is not None and type(value) != type(''):
1093 raise TypeError, 'new property "%s" not a string'%propname
1095 elif isinstance(prop, Password):
1096 if not isinstance(value, password.Password):
1097 raise TypeError, 'new property "%s" not a Password'%propname
1098 propvalues[propname] = value
1100 elif value is not None and isinstance(prop, Date):
1101 if not isinstance(value, date.Date):
1102 raise TypeError, 'new property "%s" not a Date'% propname
1103 propvalues[propname] = value
1105 elif value is not None and isinstance(prop, Interval):
1106 if not isinstance(value, date.Interval):
1107 raise TypeError, 'new property "%s" not an '\
1108 'Interval'%propname
1109 propvalues[propname] = value
1111 elif value is not None and isinstance(prop, Number):
1112 try:
1113 float(value)
1114 except ValueError:
1115 raise TypeError, 'new property "%s" not numeric'%propname
1117 elif value is not None and isinstance(prop, Boolean):
1118 try:
1119 int(value)
1120 except ValueError:
1121 raise TypeError, 'new property "%s" not boolean'%propname
1123 node[propname] = value
1125 # nothing to do?
1126 if not propvalues:
1127 return
1129 # do the set, and journal it
1130 self.db.setnode(self.classname, nodeid, node)
1132 if self.do_journal:
1133 propvalues.update(journalvalues)
1134 self.db.addjournal(self.classname, nodeid, 'set', propvalues)
1136 self.fireReactors('set', nodeid, oldvalues)
1138 def retire(self, nodeid):
1139 """Retire a node.
1141 The properties on the node remain available from the get() method,
1142 and the node's id is never reused.
1144 Retired nodes are not returned by the find(), list(), or lookup()
1145 methods, and other nodes may reuse the values of their key properties.
1147 These operations trigger detectors and can be vetoed. Attempts
1148 to modify the "creation" or "activity" properties cause a KeyError.
1149 """
1150 if self.db.journaltag is None:
1151 raise DatabaseError, 'Database open read-only'
1153 self.fireAuditors('retire', nodeid, None)
1155 node = self.db.getnode(self.classname, nodeid)
1156 node[self.db.RETIRED_FLAG] = 1
1157 self.db.setnode(self.classname, nodeid, node)
1158 if self.do_journal:
1159 self.db.addjournal(self.classname, nodeid, 'retired', None)
1161 self.fireReactors('retire', nodeid, None)
1163 def destroy(self, nodeid):
1164 """Destroy a node.
1166 WARNING: this method should never be used except in extremely rare
1167 situations where there could never be links to the node being
1168 deleted
1169 WARNING: use retire() instead
1170 WARNING: the properties of this node will not be available ever again
1171 WARNING: really, use retire() instead
1173 Well, I think that's enough warnings. This method exists mostly to
1174 support the session storage of the cgi interface.
1175 """
1176 if self.db.journaltag is None:
1177 raise DatabaseError, 'Database open read-only'
1178 self.db.destroynode(self.classname, nodeid)
1180 def history(self, nodeid):
1181 """Retrieve the journal of edits on a particular node.
1183 'nodeid' must be the id of an existing node of this class or an
1184 IndexError is raised.
1186 The returned list contains tuples of the form
1188 (date, tag, action, params)
1190 'date' is a Timestamp object specifying the time of the change and
1191 'tag' is the journaltag specified when the database was opened.
1192 """
1193 if not self.do_journal:
1194 raise ValueError, 'Journalling is disabled for this class'
1195 return self.db.getjournal(self.classname, nodeid)
1197 # Locating nodes:
1198 def hasnode(self, nodeid):
1199 '''Determine if the given nodeid actually exists
1200 '''
1201 return self.db.hasnode(self.classname, nodeid)
1203 def setkey(self, propname):
1204 """Select a String property of this class to be the key property.
1206 'propname' must be the name of a String property of this class or
1207 None, or a TypeError is raised. The values of the key property on
1208 all existing nodes must be unique or a ValueError is raised. If the
1209 property doesn't exist, KeyError is raised.
1210 """
1211 prop = self.getprops()[propname]
1212 if not isinstance(prop, String):
1213 raise TypeError, 'key properties must be String'
1214 self.key = propname
1216 def getkey(self):
1217 """Return the name of the key property for this class or None."""
1218 return self.key
1220 def labelprop(self, default_to_id=0):
1221 ''' Return the property name for a label for the given node.
1223 This method attempts to generate a consistent label for the node.
1224 It tries the following in order:
1225 1. key property
1226 2. "name" property
1227 3. "title" property
1228 4. first property from the sorted property name list
1229 '''
1230 k = self.getkey()
1231 if k:
1232 return k
1233 props = self.getprops()
1234 if props.has_key('name'):
1235 return 'name'
1236 elif props.has_key('title'):
1237 return 'title'
1238 if default_to_id:
1239 return 'id'
1240 props = props.keys()
1241 props.sort()
1242 return props[0]
1244 # TODO: set up a separate index db file for this? profile?
1245 def lookup(self, keyvalue):
1246 """Locate a particular node by its key property and return its id.
1248 If this class has no key property, a TypeError is raised. If the
1249 'keyvalue' matches one of the values for the key property among
1250 the nodes in this class, the matching node's id is returned;
1251 otherwise a KeyError is raised.
1252 """
1253 cldb = self.db.getclassdb(self.classname)
1254 try:
1255 for nodeid in self.db.getnodeids(self.classname, cldb):
1256 node = self.db.getnode(self.classname, nodeid, cldb)
1257 if node.has_key(self.db.RETIRED_FLAG):
1258 continue
1259 if node[self.key] == keyvalue:
1260 cldb.close()
1261 return nodeid
1262 finally:
1263 cldb.close()
1264 raise KeyError, keyvalue
1266 # XXX: change from spec - allows multiple props to match
1267 def find(self, **propspec):
1268 """Get the ids of nodes in this class which link to the given nodes.
1270 'propspec' consists of keyword args propname={nodeid:1,}
1271 'propname' must be the name of a property in this class, or a
1272 KeyError is raised. That property must be a Link or Multilink
1273 property, or a TypeError is raised.
1275 Any node in this class whose 'propname' property links to any of the
1276 nodeids will be returned. Used by the full text indexing, which knows
1277 that "foo" occurs in msg1, msg3 and file7, so we have hits on these issues:
1278 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1279 """
1280 propspec = propspec.items()
1281 for propname, nodeids in propspec:
1282 # check the prop is OK
1283 prop = self.properties[propname]
1284 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1285 raise TypeError, "'%s' not a Link/Multilink property"%propname
1287 # ok, now do the find
1288 cldb = self.db.getclassdb(self.classname)
1289 l = []
1290 try:
1291 for id in self.db.getnodeids(self.classname, db=cldb):
1292 node = self.db.getnode(self.classname, id, db=cldb)
1293 if node.has_key(self.db.RETIRED_FLAG):
1294 continue
1295 for propname, nodeids in propspec:
1296 # can't test if the node doesn't have this property
1297 if not node.has_key(propname):
1298 continue
1299 if type(nodeids) is type(''):
1300 nodeids = {nodeids:1}
1301 prop = self.properties[propname]
1302 value = node[propname]
1303 if isinstance(prop, Link) and nodeids.has_key(value):
1304 l.append(id)
1305 break
1306 elif isinstance(prop, Multilink):
1307 hit = 0
1308 for v in value:
1309 if nodeids.has_key(v):
1310 l.append(id)
1311 hit = 1
1312 break
1313 if hit:
1314 break
1315 finally:
1316 cldb.close()
1317 return l
1319 def stringFind(self, **requirements):
1320 """Locate a particular node by matching a set of its String
1321 properties in a caseless search.
1323 If the property is not a String property, a TypeError is raised.
1325 The return is a list of the id of all nodes that match.
1326 """
1327 for propname in requirements.keys():
1328 prop = self.properties[propname]
1329 if isinstance(not prop, String):
1330 raise TypeError, "'%s' not a String property"%propname
1331 requirements[propname] = requirements[propname].lower()
1332 l = []
1333 cldb = self.db.getclassdb(self.classname)
1334 try:
1335 for nodeid in self.db.getnodeids(self.classname, cldb):
1336 node = self.db.getnode(self.classname, nodeid, cldb)
1337 if node.has_key(self.db.RETIRED_FLAG):
1338 continue
1339 for key, value in requirements.items():
1340 if node[key] is None or node[key].lower() != value:
1341 break
1342 else:
1343 l.append(nodeid)
1344 finally:
1345 cldb.close()
1346 return l
1348 def list(self):
1349 """Return a list of the ids of the active nodes in this class."""
1350 l = []
1351 cn = self.classname
1352 cldb = self.db.getclassdb(cn)
1353 try:
1354 for nodeid in self.db.getnodeids(cn, cldb):
1355 node = self.db.getnode(cn, nodeid, cldb)
1356 if node.has_key(self.db.RETIRED_FLAG):
1357 continue
1358 l.append(nodeid)
1359 finally:
1360 cldb.close()
1361 l.sort()
1362 return l
1364 def filter(self, search_matches, filterspec, sort, group,
1365 num_re = re.compile('^\d+$')):
1366 ''' Return a list of the ids of the active nodes in this class that
1367 match the 'filter' spec, sorted by the group spec and then the
1368 sort spec.
1370 "filterspec" is {propname: value(s)}
1371 "sort" is ['+propname', '-propname', 'propname', ...]
1372 "group is ['+propname', '-propname', 'propname', ...]
1373 '''
1374 cn = self.classname
1376 # optimise filterspec
1377 l = []
1378 props = self.getprops()
1379 LINK = 0
1380 MULTILINK = 1
1381 STRING = 2
1382 OTHER = 6
1383 for k, v in filterspec.items():
1384 propclass = props[k]
1385 if isinstance(propclass, Link):
1386 if type(v) is not type([]):
1387 v = [v]
1388 # replace key values with node ids
1389 u = []
1390 link_class = self.db.classes[propclass.classname]
1391 for entry in v:
1392 if entry == '-1': entry = None
1393 elif not num_re.match(entry):
1394 try:
1395 entry = link_class.lookup(entry)
1396 except (TypeError,KeyError):
1397 raise ValueError, 'property "%s": %s not a %s'%(
1398 k, entry, self.properties[k].classname)
1399 u.append(entry)
1401 l.append((LINK, k, u))
1402 elif isinstance(propclass, Multilink):
1403 if type(v) is not type([]):
1404 v = [v]
1405 # replace key values with node ids
1406 u = []
1407 link_class = self.db.classes[propclass.classname]
1408 for entry in v:
1409 if not num_re.match(entry):
1410 try:
1411 entry = link_class.lookup(entry)
1412 except (TypeError,KeyError):
1413 raise ValueError, 'new property "%s": %s not a %s'%(
1414 k, entry, self.properties[k].classname)
1415 u.append(entry)
1416 l.append((MULTILINK, k, u))
1417 elif isinstance(propclass, String):
1418 # simple glob searching
1419 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1420 v = v.replace('?', '.')
1421 v = v.replace('*', '.*?')
1422 l.append((STRING, k, re.compile(v, re.I)))
1423 elif isinstance(propclass, Boolean):
1424 if type(v) is type(''):
1425 bv = v.lower() in ('yes', 'true', 'on', '1')
1426 else:
1427 bv = v
1428 l.append((OTHER, k, bv))
1429 elif isinstance(propclass, Number):
1430 l.append((OTHER, k, int(v)))
1431 else:
1432 l.append((OTHER, k, v))
1433 filterspec = l
1435 # now, find all the nodes that are active and pass filtering
1436 l = []
1437 cldb = self.db.getclassdb(cn)
1438 try:
1439 # TODO: only full-scan once (use items())
1440 for nodeid in self.db.getnodeids(cn, cldb):
1441 node = self.db.getnode(cn, nodeid, cldb)
1442 if node.has_key(self.db.RETIRED_FLAG):
1443 continue
1444 # apply filter
1445 for t, k, v in filterspec:
1446 # make sure the node has the property
1447 if not node.has_key(k):
1448 # this node doesn't have this property, so reject it
1449 break
1451 # now apply the property filter
1452 if t == LINK:
1453 # link - if this node's property doesn't appear in the
1454 # filterspec's nodeid list, skip it
1455 if node[k] not in v:
1456 break
1457 elif t == MULTILINK:
1458 # multilink - if any of the nodeids required by the
1459 # filterspec aren't in this node's property, then skip
1460 # it
1461 have = node[k]
1462 for want in v:
1463 if want not in have:
1464 break
1465 else:
1466 continue
1467 break
1468 elif t == STRING:
1469 # RE search
1470 if node[k] is None or not v.search(node[k]):
1471 break
1472 elif t == OTHER:
1473 # straight value comparison for the other types
1474 if node[k] != v:
1475 break
1476 else:
1477 l.append((nodeid, node))
1478 finally:
1479 cldb.close()
1480 l.sort()
1482 # filter based on full text search
1483 if search_matches is not None:
1484 k = []
1485 for v in l:
1486 if search_matches.has_key(v[0]):
1487 k.append(v)
1488 l = k
1490 # optimise sort
1491 m = []
1492 for entry in sort:
1493 if entry[0] != '-':
1494 m.append(('+', entry))
1495 else:
1496 m.append((entry[0], entry[1:]))
1497 sort = m
1499 # optimise group
1500 m = []
1501 for entry in group:
1502 if entry[0] != '-':
1503 m.append(('+', entry))
1504 else:
1505 m.append((entry[0], entry[1:]))
1506 group = m
1507 # now, sort the result
1508 def sortfun(a, b, sort=sort, group=group, properties=self.getprops(),
1509 db = self.db, cl=self):
1510 a_id, an = a
1511 b_id, bn = b
1512 # sort by group and then sort
1513 for list in group, sort:
1514 for dir, prop in list:
1515 # sorting is class-specific
1516 propclass = properties[prop]
1518 # handle the properties that might be "faked"
1519 # also, handle possible missing properties
1520 try:
1521 if not an.has_key(prop):
1522 an[prop] = cl.get(a_id, prop)
1523 av = an[prop]
1524 except KeyError:
1525 # the node doesn't have a value for this property
1526 if isinstance(propclass, Multilink): av = []
1527 else: av = ''
1528 try:
1529 if not bn.has_key(prop):
1530 bn[prop] = cl.get(b_id, prop)
1531 bv = bn[prop]
1532 except KeyError:
1533 # the node doesn't have a value for this property
1534 if isinstance(propclass, Multilink): bv = []
1535 else: bv = ''
1537 # String and Date values are sorted in the natural way
1538 if isinstance(propclass, String):
1539 # clean up the strings
1540 if av and av[0] in string.uppercase:
1541 av = an[prop] = av.lower()
1542 if bv and bv[0] in string.uppercase:
1543 bv = bn[prop] = bv.lower()
1544 if (isinstance(propclass, String) or
1545 isinstance(propclass, Date)):
1546 # it might be a string that's really an integer
1547 try:
1548 av = int(av)
1549 bv = int(bv)
1550 except:
1551 pass
1552 if dir == '+':
1553 r = cmp(av, bv)
1554 if r != 0: return r
1555 elif dir == '-':
1556 r = cmp(bv, av)
1557 if r != 0: return r
1559 # Link properties are sorted according to the value of
1560 # the "order" property on the linked nodes if it is
1561 # present; or otherwise on the key string of the linked
1562 # nodes; or finally on the node ids.
1563 elif isinstance(propclass, Link):
1564 link = db.classes[propclass.classname]
1565 if av is None and bv is not None: return -1
1566 if av is not None and bv is None: return 1
1567 if av is None and bv is None: continue
1568 if link.getprops().has_key('order'):
1569 if dir == '+':
1570 r = cmp(link.get(av, 'order'),
1571 link.get(bv, 'order'))
1572 if r != 0: return r
1573 elif dir == '-':
1574 r = cmp(link.get(bv, 'order'),
1575 link.get(av, 'order'))
1576 if r != 0: return r
1577 elif link.getkey():
1578 key = link.getkey()
1579 if dir == '+':
1580 r = cmp(link.get(av, key), link.get(bv, key))
1581 if r != 0: return r
1582 elif dir == '-':
1583 r = cmp(link.get(bv, key), link.get(av, key))
1584 if r != 0: return r
1585 else:
1586 if dir == '+':
1587 r = cmp(av, bv)
1588 if r != 0: return r
1589 elif dir == '-':
1590 r = cmp(bv, av)
1591 if r != 0: return r
1593 # Multilink properties are sorted according to how many
1594 # links are present.
1595 elif isinstance(propclass, Multilink):
1596 if dir == '+':
1597 r = cmp(len(av), len(bv))
1598 if r != 0: return r
1599 elif dir == '-':
1600 r = cmp(len(bv), len(av))
1601 if r != 0: return r
1602 elif isinstance(propclass, Number) or isinstance(propclass, Boolean):
1603 if dir == '+':
1604 r = cmp(av, bv)
1605 elif dir == '-':
1606 r = cmp(bv, av)
1608 # end for dir, prop in list:
1609 # end for list in sort, group:
1610 # if all else fails, compare the ids
1611 return cmp(a[0], b[0])
1613 l.sort(sortfun)
1614 return [i[0] for i in l]
1616 def count(self):
1617 """Get the number of nodes in this class.
1619 If the returned integer is 'numnodes', the ids of all the nodes
1620 in this class run from 1 to numnodes, and numnodes+1 will be the
1621 id of the next node to be created in this class.
1622 """
1623 return self.db.countnodes(self.classname)
1625 # Manipulating properties:
1627 def getprops(self, protected=1):
1628 """Return a dictionary mapping property names to property objects.
1629 If the "protected" flag is true, we include protected properties -
1630 those which may not be modified.
1632 In addition to the actual properties on the node, these
1633 methods provide the "creation" and "activity" properties. If the
1634 "protected" flag is true, we include protected properties - those
1635 which may not be modified.
1636 """
1637 d = self.properties.copy()
1638 if protected:
1639 d['id'] = String()
1640 d['creation'] = hyperdb.Date()
1641 d['activity'] = hyperdb.Date()
1642 d['creator'] = hyperdb.Link("user")
1643 return d
1645 def addprop(self, **properties):
1646 """Add properties to this class.
1648 The keyword arguments in 'properties' must map names to property
1649 objects, or a TypeError is raised. None of the keys in 'properties'
1650 may collide with the names of existing properties, or a ValueError
1651 is raised before any properties have been added.
1652 """
1653 for key in properties.keys():
1654 if self.properties.has_key(key):
1655 raise ValueError, key
1656 self.properties.update(properties)
1658 def index(self, nodeid):
1659 '''Add (or refresh) the node to search indexes
1660 '''
1661 # find all the String properties that have indexme
1662 for prop, propclass in self.getprops().items():
1663 if isinstance(propclass, String) and propclass.indexme:
1664 try:
1665 value = str(self.get(nodeid, prop))
1666 except IndexError:
1667 # node no longer exists - entry should be removed
1668 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1669 else:
1670 # and index them under (classname, nodeid, property)
1671 self.db.indexer.add_text((self.classname, nodeid, prop),
1672 value)
1674 #
1675 # Detector interface
1676 #
1677 def audit(self, event, detector):
1678 """Register a detector
1679 """
1680 l = self.auditors[event]
1681 if detector not in l:
1682 self.auditors[event].append(detector)
1684 def fireAuditors(self, action, nodeid, newvalues):
1685 """Fire all registered auditors.
1686 """
1687 for audit in self.auditors[action]:
1688 audit(self.db, self, nodeid, newvalues)
1690 def react(self, event, detector):
1691 """Register a detector
1692 """
1693 l = self.reactors[event]
1694 if detector not in l:
1695 self.reactors[event].append(detector)
1697 def fireReactors(self, action, nodeid, oldvalues):
1698 """Fire all registered reactors.
1699 """
1700 for react in self.reactors[action]:
1701 react(self.db, self, nodeid, oldvalues)
1703 class FileClass(Class):
1704 '''This class defines a large chunk of data. To support this, it has a
1705 mandatory String property "content" which is typically saved off
1706 externally to the hyperdb.
1708 The default MIME type of this data is defined by the
1709 "default_mime_type" class attribute, which may be overridden by each
1710 node if the class defines a "type" String property.
1711 '''
1712 default_mime_type = 'text/plain'
1714 def create(self, **propvalues):
1715 ''' snaffle the file propvalue and store in a file
1716 '''
1717 content = propvalues['content']
1718 del propvalues['content']
1719 newid = Class.create(self, **propvalues)
1720 self.db.storefile(self.classname, newid, None, content)
1721 return newid
1723 def get(self, nodeid, propname, default=_marker, cache=1):
1724 ''' trap the content propname and get it from the file
1725 '''
1727 poss_msg = 'Possibly a access right configuration problem.'
1728 if propname == 'content':
1729 try:
1730 return self.db.getfile(self.classname, nodeid, None)
1731 except IOError, (strerror):
1732 # BUG: by catching this we donot see an error in the log.
1733 return 'ERROR reading file: %s%s\n%s\n%s'%(
1734 self.classname, nodeid, poss_msg, strerror)
1735 if default is not _marker:
1736 return Class.get(self, nodeid, propname, default, cache=cache)
1737 else:
1738 return Class.get(self, nodeid, propname, cache=cache)
1740 def getprops(self, protected=1):
1741 ''' In addition to the actual properties on the node, these methods
1742 provide the "content" property. If the "protected" flag is true,
1743 we include protected properties - those which may not be
1744 modified.
1745 '''
1746 d = Class.getprops(self, protected=protected).copy()
1747 if protected:
1748 d['content'] = hyperdb.String()
1749 return d
1751 def index(self, nodeid):
1752 ''' Index the node in the search index.
1754 We want to index the content in addition to the normal String
1755 property indexing.
1756 '''
1757 # perform normal indexing
1758 Class.index(self, nodeid)
1760 # get the content to index
1761 content = self.get(nodeid, 'content')
1763 # figure the mime type
1764 if self.properties.has_key('type'):
1765 mime_type = self.get(nodeid, 'type')
1766 else:
1767 mime_type = self.default_mime_type
1769 # and index!
1770 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
1771 mime_type)
1773 # XXX deviation from spec - was called ItemClass
1774 class IssueClass(Class, roundupdb.IssueClass):
1775 # Overridden methods:
1776 def __init__(self, db, classname, **properties):
1777 """The newly-created class automatically includes the "messages",
1778 "files", "nosy", and "superseder" properties. If the 'properties'
1779 dictionary attempts to specify any of these properties or a
1780 "creation" or "activity" property, a ValueError is raised.
1781 """
1782 if not properties.has_key('title'):
1783 properties['title'] = hyperdb.String(indexme='yes')
1784 if not properties.has_key('messages'):
1785 properties['messages'] = hyperdb.Multilink("msg")
1786 if not properties.has_key('files'):
1787 properties['files'] = hyperdb.Multilink("file")
1788 if not properties.has_key('nosy'):
1789 properties['nosy'] = hyperdb.Multilink("user")
1790 if not properties.has_key('superseder'):
1791 properties['superseder'] = hyperdb.Multilink(classname)
1792 Class.__init__(self, db, classname, **properties)
1794 #
1795 #$Log: not supported by cvs2svn $
1796 #Revision 1.56 2002/07/31 22:04:33 richard
1797 #cleanup
1798 #
1799 #Revision 1.55 2002/07/30 08:22:38 richard
1800 #Session storage in the hyperdb was horribly, horribly inefficient. We use
1801 #a simple anydbm wrapper now - which could be overridden by the metakit
1802 #backend or RDB backend if necessary.
1803 #Much, much better.
1804 #
1805 #Revision 1.54 2002/07/26 08:26:59 richard
1806 #Very close now. The cgi and mailgw now use the new security API. The two
1807 #templates have been migrated to that setup. Lots of unit tests. Still some
1808 #issue in the web form for editing Roles assigned to users.
1809 #
1810 #Revision 1.53 2002/07/25 07:14:06 richard
1811 #Bugger it. Here's the current shape of the new security implementation.
1812 #Still to do:
1813 # . call the security funcs from cgi and mailgw
1814 # . change shipped templates to include correct initialisation and remove
1815 # the old config vars
1816 #... that seems like a lot. The bulk of the work has been done though. Honest :)
1817 #
1818 #Revision 1.52 2002/07/19 03:36:34 richard
1819 #Implemented the destroy() method needed by the session database (and possibly
1820 #others). At the same time, I removed the leading underscores from the hyperdb
1821 #methods that Really Didn't Need Them.
1822 #The journal also raises IndexError now for all situations where there is a
1823 #request for the journal of a node that doesn't have one. It used to return
1824 #[] in _some_ situations, but not all. This _may_ break code, but the tests
1825 #pass...
1826 #
1827 #Revision 1.51 2002/07/18 23:07:08 richard
1828 #Unit tests and a few fixes.
1829 #
1830 #Revision 1.50 2002/07/18 11:50:58 richard
1831 #added tests for number type too
1832 #
1833 #Revision 1.49 2002/07/18 11:41:10 richard
1834 #added tests for boolean type, and fixes to anydbm backend
1835 #
1836 #Revision 1.48 2002/07/18 11:17:31 gmcm
1837 #Add Number and Boolean types to hyperdb.
1838 #Add conversion cases to web, mail & admin interfaces.
1839 #Add storage/serialization cases to back_anydbm & back_metakit.
1840 #
1841 #Revision 1.47 2002/07/14 23:18:20 richard
1842 #. fixed the journal bloat from multilink changes - we just log the add or
1843 # remove operations, not the whole list
1844 #
1845 #Revision 1.46 2002/07/14 06:06:34 richard
1846 #Did some old TODOs
1847 #
1848 #Revision 1.45 2002/07/14 04:03:14 richard
1849 #Implemented a switch to disable journalling for a Class. CGI session
1850 #database now uses it.
1851 #
1852 #Revision 1.44 2002/07/14 02:05:53 richard
1853 #. all storage-specific code (ie. backend) is now implemented by the backends
1854 #
1855 #Revision 1.43 2002/07/10 06:30:30 richard
1856 #...except of course it's nice to use valid Python syntax
1857 #
1858 #Revision 1.42 2002/07/10 06:21:38 richard
1859 #Be extra safe
1860 #
1861 #Revision 1.41 2002/07/10 00:21:45 richard
1862 #explicit database closing
1863 #
1864 #Revision 1.40 2002/07/09 04:19:09 richard
1865 #Added reindex command to roundup-admin.
1866 #Fixed reindex on first access.
1867 #Also fixed reindexing of entries that change.
1868 #
1869 #Revision 1.39 2002/07/09 03:02:52 richard
1870 #More indexer work:
1871 #- all String properties may now be indexed too. Currently there's a bit of
1872 # "issue" specific code in the actual searching which needs to be
1873 # addressed. In a nutshell:
1874 # + pass 'indexme="yes"' as a String() property initialisation arg, eg:
1875 # file = FileClass(db, "file", name=String(), type=String(),
1876 # comment=String(indexme="yes"))
1877 # + the comment will then be indexed and be searchable, with the results
1878 # related back to the issue that the file is linked to
1879 #- as a result of this work, the FileClass has a default MIME type that may
1880 # be overridden in a subclass, or by the use of a "type" property as is
1881 # done in the default templates.
1882 #- the regeneration of the indexes (if necessary) is done once the schema is
1883 # set up in the dbinit.
1884 #
1885 #Revision 1.38 2002/07/08 06:58:15 richard
1886 #cleaned up the indexer code:
1887 # - it splits more words out (much simpler, faster splitter)
1888 # - removed code we'll never use (roundup.roundup_indexer has the full
1889 # implementation, and replaces roundup.indexer)
1890 # - only index text/plain and rfc822/message (ideas for other text formats to
1891 # index are welcome)
1892 # - added simple unit test for indexer. Needs more tests for regression.
1893 #
1894 #Revision 1.37 2002/06/20 23:52:35 richard
1895 #More informative error message
1896 #
1897 #Revision 1.36 2002/06/19 03:07:19 richard
1898 #Moved the file storage commit into blobfiles where it belongs.
1899 #
1900 #Revision 1.35 2002/05/25 07:16:24 rochecompaan
1901 #Merged search_indexing-branch with HEAD
1902 #
1903 #Revision 1.34 2002/05/15 06:21:21 richard
1904 # . node caching now works, and gives a small boost in performance
1905 #
1906 #As a part of this, I cleaned up the DEBUG output and implemented TRACE
1907 #output (HYPERDBTRACE='file to trace to') with checkpoints at the start of
1908 #CGI requests. Run roundup with python -O to skip all the DEBUG/TRACE stuff
1909 #(using if __debug__ which is compiled out with -O)
1910 #
1911 #Revision 1.33 2002/04/24 10:38:26 rochecompaan
1912 #All database files are now created group readable and writable.
1913 #
1914 #Revision 1.32 2002/04/15 23:25:15 richard
1915 #. node ids are now generated from a lockable store - no more race conditions
1916 #
1917 #We're using the portalocker code by Jonathan Feinberg that was contributed
1918 #to the ASPN Python cookbook. This gives us locking across Unix and Windows.
1919 #
1920 #Revision 1.31 2002/04/03 05:54:31 richard
1921 #Fixed serialisation problem by moving the serialisation step out of the
1922 #hyperdb.Class (get, set) into the hyperdb.Database.
1923 #
1924 #Also fixed htmltemplate after the showid changes I made yesterday.
1925 #
1926 #Unit tests for all of the above written.
1927 #
1928 #Revision 1.30.2.1 2002/04/03 11:55:57 rochecompaan
1929 # . Added feature #526730 - search for messages capability
1930 #
1931 #Revision 1.30 2002/02/27 03:40:59 richard
1932 #Ran it through pychecker, made fixes
1933 #
1934 #Revision 1.29 2002/02/25 14:34:31 grubert
1935 # . use blobfiles in back_anydbm which is used in back_bsddb.
1936 # change test_db as dirlist does not work for subdirectories.
1937 # ATTENTION: blobfiles now creates subdirectories for files.
1938 #
1939 #Revision 1.28 2002/02/16 09:14:17 richard
1940 # . #514854 ] History: "User" is always ticket creator
1941 #
1942 #Revision 1.27 2002/01/22 07:21:13 richard
1943 #. fixed back_bsddb so it passed the journal tests
1944 #
1945 #... it didn't seem happy using the back_anydbm _open method, which is odd.
1946 #Yet another occurrance of whichdb not being able to recognise older bsddb
1947 #databases. Yadda yadda. Made the HYPERDBDEBUG stuff more sane in the
1948 #process.
1949 #
1950 #Revision 1.26 2002/01/22 05:18:38 rochecompaan
1951 #last_set_entry was referenced before assignment
1952 #
1953 #Revision 1.25 2002/01/22 05:06:08 rochecompaan
1954 #We need to keep the last 'set' entry in the journal to preserve
1955 #information on 'activity' for nodes.
1956 #
1957 #Revision 1.24 2002/01/21 16:33:20 rochecompaan
1958 #You can now use the roundup-admin tool to pack the database
1959 #
1960 #Revision 1.23 2002/01/18 04:32:04 richard
1961 #Rollback was breaking because a message hadn't actually been written to the file. Needs
1962 #more investigation.
1963 #
1964 #Revision 1.22 2002/01/14 02:20:15 richard
1965 # . changed all config accesses so they access either the instance or the
1966 # config attriubute on the db. This means that all config is obtained from
1967 # instance_config instead of the mish-mash of classes. This will make
1968 # switching to a ConfigParser setup easier too, I hope.
1969 #
1970 #At a minimum, this makes migration a _little_ easier (a lot easier in the
1971 #0.5.0 switch, I hope!)
1972 #
1973 #Revision 1.21 2002/01/02 02:31:38 richard
1974 #Sorry for the huge checkin message - I was only intending to implement #496356
1975 #but I found a number of places where things had been broken by transactions:
1976 # . modified ROUNDUPDBSENDMAILDEBUG to be SENDMAILDEBUG and hold a filename
1977 # for _all_ roundup-generated smtp messages to be sent to.
1978 # . the transaction cache had broken the roundupdb.Class set() reactors
1979 # . newly-created author users in the mailgw weren't being committed to the db
1980 #
1981 #Stuff that made it into CHANGES.txt (ie. the stuff I was actually working
1982 #on when I found that stuff :):
1983 # . #496356 ] Use threading in messages
1984 # . detectors were being registered multiple times
1985 # . added tests for mailgw
1986 # . much better attaching of erroneous messages in the mail gateway
1987 #
1988 #Revision 1.20 2001/12/18 15:30:34 rochecompaan
1989 #Fixed bugs:
1990 # . Fixed file creation and retrieval in same transaction in anydbm
1991 # backend
1992 # . Cgi interface now renders new issue after issue creation
1993 # . Could not set issue status to resolved through cgi interface
1994 # . Mail gateway was changing status back to 'chatting' if status was
1995 # omitted as an argument
1996 #
1997 #Revision 1.19 2001/12/17 03:52:48 richard
1998 #Implemented file store rollback. As a bonus, the hyperdb is now capable of
1999 #storing more than one file per node - if a property name is supplied,
2000 #the file is called designator.property.
2001 #I decided not to migrate the existing files stored over to the new naming
2002 #scheme - the FileClass just doesn't specify the property name.
2003 #
2004 #Revision 1.18 2001/12/16 10:53:38 richard
2005 #take a copy of the node dict so that the subsequent set
2006 #operation doesn't modify the oldvalues structure
2007 #
2008 #Revision 1.17 2001/12/14 23:42:57 richard
2009 #yuck, a gdbm instance tests false :(
2010 #I've left the debugging code in - it should be removed one day if we're ever
2011 #_really_ anal about performace :)
2012 #
2013 #Revision 1.16 2001/12/12 03:23:14 richard
2014 #Cor blimey this anydbm/whichdb stuff is yecchy. Turns out that whichdb
2015 #incorrectly identifies a dbm file as a dbhash file on my system. This has
2016 #been submitted to the python bug tracker as issue #491888:
2017 #https://sourceforge.net/tracker/index.php?func=detail&aid=491888&group_id=5470&atid=105470
2018 #
2019 #Revision 1.15 2001/12/12 02:30:51 richard
2020 #I fixed the problems with people whose anydbm was using the dbm module at the
2021 #backend. It turns out the dbm module modifies the file name to append ".db"
2022 #and my check to determine if we're opening an existing or new db just
2023 #tested os.path.exists() on the filename. Well, no longer! We now perform a
2024 #much better check _and_ cope with the anydbm implementation module changing
2025 #too!
2026 #I also fixed the backends __init__ so only ImportError is squashed.
2027 #
2028 #Revision 1.14 2001/12/10 22:20:01 richard
2029 #Enabled transaction support in the bsddb backend. It uses the anydbm code
2030 #where possible, only replacing methods where the db is opened (it uses the
2031 #btree opener specifically.)
2032 #Also cleaned up some change note generation.
2033 #Made the backends package work with pydoc too.
2034 #
2035 #Revision 1.13 2001/12/02 05:06:16 richard
2036 #. We now use weakrefs in the Classes to keep the database reference, so
2037 # the close() method on the database is no longer needed.
2038 # I bumped the minimum python requirement up to 2.1 accordingly.
2039 #. #487480 ] roundup-server
2040 #. #487476 ] INSTALL.txt
2041 #
2042 #I also cleaned up the change message / post-edit stuff in the cgi client.
2043 #There's now a clearly marked "TODO: append the change note" where I believe
2044 #the change note should be added there. The "changes" list will obviously
2045 #have to be modified to be a dict of the changes, or somesuch.
2046 #
2047 #More testing needed.
2048 #
2049 #Revision 1.12 2001/12/01 07:17:50 richard
2050 #. We now have basic transaction support! Information is only written to
2051 # the database when the commit() method is called. Only the anydbm
2052 # backend is modified in this way - neither of the bsddb backends have been.
2053 # The mail, admin and cgi interfaces all use commit (except the admin tool
2054 # doesn't have a commit command, so interactive users can't commit...)
2055 #. Fixed login/registration forwarding the user to the right page (or not,
2056 # on a failure)
2057 #
2058 #Revision 1.11 2001/11/21 02:34:18 richard
2059 #Added a target version field to the extended issue schema
2060 #
2061 #Revision 1.10 2001/10/09 23:58:10 richard
2062 #Moved the data stringification up into the hyperdb.Class class' get, set
2063 #and create methods. This means that the data is also stringified for the
2064 #journal call, and removes duplication of code from the backends. The
2065 #backend code now only sees strings.
2066 #
2067 #Revision 1.9 2001/10/09 07:25:59 richard
2068 #Added the Password property type. See "pydoc roundup.password" for
2069 #implementation details. Have updated some of the documentation too.
2070 #
2071 #Revision 1.8 2001/09/29 13:27:00 richard
2072 #CGI interfaces now spit up a top-level index of all the instances they can
2073 #serve.
2074 #
2075 #Revision 1.7 2001/08/12 06:32:36 richard
2076 #using isinstance(blah, Foo) now instead of isFooType
2077 #
2078 #Revision 1.6 2001/08/07 00:24:42 richard
2079 #stupid typo
2080 #
2081 #Revision 1.5 2001/08/07 00:15:51 richard
2082 #Added the copyright/license notice to (nearly) all files at request of
2083 #Bizar Software.
2084 #
2085 #Revision 1.4 2001/07/30 01:41:36 richard
2086 #Makes schema changes mucho easier.
2087 #
2088 #Revision 1.3 2001/07/25 01:23:07 richard
2089 #Added the Roundup spec to the new documentation directory.
2090 #
2091 #Revision 1.2 2001/07/23 08:20:44 richard
2092 #Moved over to using marshal in the bsddb and anydbm backends.
2093 #roundup-admin now has a "freshen" command that'll load/save all nodes (not
2094 # retired - mod hyperdb.Class.list() so it lists retired nodes)
2095 #
2096 #