fa67c05eeae6a67167aeba734605019096811e59
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: back_anydbm.py,v 1.56 2002-07-31 22:04:33 richard Exp $
19 '''
20 This module defines a backend that saves the hyperdatabase in a database
21 chosen by anydbm. It is guaranteed to always be available in python
22 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
23 serious bugs, and is not available)
24 '''
26 import whichdb, anydbm, os, marshal, re, weakref, string, copy
27 from roundup import hyperdb, date, password, roundupdb, security
28 from blobfiles import FileStorage
29 from sessions import Sessions
30 from roundup.indexer import Indexer
31 from locking import acquire_lock, release_lock
32 from roundup.hyperdb import String, Password, Date, Interval, Link, \
33 Multilink, DatabaseError, Boolean, Number
35 #
36 # Now the database
37 #
38 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
39 """A database for storing records containing flexible data types.
41 Transaction stuff TODO:
42 . check the timestamp of the class file and nuke the cache if it's
43 modified. Do some sort of conflict checking on the dirty stuff.
44 . perhaps detect write collisions (related to above)?
46 """
47 def __init__(self, config, journaltag=None):
48 """Open a hyperdatabase given a specifier to some storage.
50 The 'storagelocator' is obtained from config.DATABASE.
51 The meaning of 'storagelocator' depends on the particular
52 implementation of the hyperdatabase. It could be a file name,
53 a directory path, a socket descriptor for a connection to a
54 database over the network, etc.
56 The 'journaltag' is a token that will be attached to the journal
57 entries for any edits done on the database. If 'journaltag' is
58 None, the database is opened in read-only mode: the Class.create(),
59 Class.set(), and Class.retire() methods are disabled.
60 """
61 self.config, self.journaltag = config, journaltag
62 self.dir = config.DATABASE
63 self.classes = {}
64 self.cache = {} # cache of nodes loaded or created
65 self.dirtynodes = {} # keep track of the dirty nodes by class
66 self.newnodes = {} # keep track of the new nodes by class
67 self.destroyednodes = {}# keep track of the destroyed nodes by class
68 self.transactions = []
69 self.indexer = Indexer(self.dir)
70 self.sessions = Sessions(self.config)
71 self.security = security.Security(self)
72 # ensure files are group readable and writable
73 os.umask(0002)
75 def post_init(self):
76 """Called once the schema initialisation has finished."""
77 # reindex the db if necessary
78 if self.indexer.should_reindex():
79 self.reindex()
81 def reindex(self):
82 for klass in self.classes.values():
83 for nodeid in klass.list():
84 klass.index(nodeid)
85 self.indexer.save_index()
87 def __repr__(self):
88 return '<back_anydbm instance at %x>'%id(self)
90 #
91 # Classes
92 #
93 def __getattr__(self, classname):
94 """A convenient way of calling self.getclass(classname)."""
95 if self.classes.has_key(classname):
96 if __debug__:
97 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
98 return self.classes[classname]
99 raise AttributeError, classname
101 def addclass(self, cl):
102 if __debug__:
103 print >>hyperdb.DEBUG, 'addclass', (self, cl)
104 cn = cl.classname
105 if self.classes.has_key(cn):
106 raise ValueError, cn
107 self.classes[cn] = cl
109 def getclasses(self):
110 """Return a list of the names of all existing classes."""
111 if __debug__:
112 print >>hyperdb.DEBUG, 'getclasses', (self,)
113 l = self.classes.keys()
114 l.sort()
115 return l
117 def getclass(self, classname):
118 """Get the Class object representing a particular class.
120 If 'classname' is not a valid class name, a KeyError is raised.
121 """
122 if __debug__:
123 print >>hyperdb.DEBUG, 'getclass', (self, classname)
124 return self.classes[classname]
126 #
127 # Class DBs
128 #
129 def clear(self):
130 '''Delete all database contents
131 '''
132 if __debug__:
133 print >>hyperdb.DEBUG, 'clear', (self,)
134 for cn in self.classes.keys():
135 for dummy in 'nodes', 'journals':
136 path = os.path.join(self.dir, 'journals.%s'%cn)
137 if os.path.exists(path):
138 os.remove(path)
139 elif os.path.exists(path+'.db'): # dbm appends .db
140 os.remove(path+'.db')
142 def getclassdb(self, classname, mode='r'):
143 ''' grab a connection to the class db that will be used for
144 multiple actions
145 '''
146 if __debug__:
147 print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode)
148 return self.opendb('nodes.%s'%classname, mode)
150 def determine_db_type(self, path):
151 ''' determine which DB wrote the class file
152 '''
153 db_type = ''
154 if os.path.exists(path):
155 db_type = whichdb.whichdb(path)
156 if not db_type:
157 raise hyperdb.DatabaseError, "Couldn't identify database type"
158 elif os.path.exists(path+'.db'):
159 # if the path ends in '.db', it's a dbm database, whether
160 # anydbm says it's dbhash or not!
161 db_type = 'dbm'
162 return db_type
164 def opendb(self, name, mode):
165 '''Low-level database opener that gets around anydbm/dbm
166 eccentricities.
167 '''
168 if __debug__:
169 print >>hyperdb.DEBUG, 'opendb', (self, name, mode)
171 # figure the class db type
172 path = os.path.join(os.getcwd(), self.dir, name)
173 db_type = self.determine_db_type(path)
175 # new database? let anydbm pick the best dbm
176 if not db_type:
177 if __debug__:
178 print >>hyperdb.DEBUG, "opendb anydbm.open(%r, 'n')"%path
179 return anydbm.open(path, 'n')
181 # open the database with the correct module
182 try:
183 dbm = __import__(db_type)
184 except ImportError:
185 raise hyperdb.DatabaseError, \
186 "Couldn't open database - the required module '%s'"\
187 " is not available"%db_type
188 if __debug__:
189 print >>hyperdb.DEBUG, "opendb %r.open(%r, %r)"%(db_type, path,
190 mode)
191 return dbm.open(path, mode)
193 def lockdb(self, name):
194 ''' Lock a database file
195 '''
196 path = os.path.join(os.getcwd(), self.dir, '%s.lock'%name)
197 return acquire_lock(path)
199 #
200 # Node IDs
201 #
202 def newid(self, classname):
203 ''' Generate a new id for the given class
204 '''
205 # open the ids DB - create if if doesn't exist
206 lock = self.lockdb('_ids')
207 db = self.opendb('_ids', 'c')
208 if db.has_key(classname):
209 newid = db[classname] = str(int(db[classname]) + 1)
210 else:
211 # the count() bit is transitional - older dbs won't start at 1
212 newid = str(self.getclass(classname).count()+1)
213 db[classname] = newid
214 db.close()
215 release_lock(lock)
216 return newid
218 #
219 # Nodes
220 #
221 def addnode(self, classname, nodeid, node):
222 ''' add the specified node to its class's db
223 '''
224 if __debug__:
225 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
226 self.newnodes.setdefault(classname, {})[nodeid] = 1
227 self.cache.setdefault(classname, {})[nodeid] = node
228 self.savenode(classname, nodeid, node)
230 def setnode(self, classname, nodeid, node):
231 ''' change the specified node
232 '''
233 if __debug__:
234 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
235 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
237 # can't set without having already loaded the node
238 self.cache[classname][nodeid] = node
239 self.savenode(classname, nodeid, node)
241 def savenode(self, classname, nodeid, node):
242 ''' perform the saving of data specified by the set/addnode
243 '''
244 if __debug__:
245 print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node)
246 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
248 def getnode(self, classname, nodeid, db=None, cache=1):
249 ''' get a node from the database
250 '''
251 if __debug__:
252 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid, db)
253 if cache:
254 # try the cache
255 cache_dict = self.cache.setdefault(classname, {})
256 if cache_dict.has_key(nodeid):
257 if __debug__:
258 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
259 nodeid)
260 return cache_dict[nodeid]
262 if __debug__:
263 print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid)
265 # get from the database and save in the cache
266 if db is None:
267 db = self.getclassdb(classname)
268 if not db.has_key(nodeid):
269 raise IndexError, "no such %s %s"%(classname, nodeid)
271 # check the uncommitted, destroyed nodes
272 if (self.destroyednodes.has_key(classname) and
273 self.destroyednodes[classname].has_key(nodeid)):
274 raise IndexError, "no such %s %s"%(classname, nodeid)
276 # decode
277 res = marshal.loads(db[nodeid])
279 # reverse the serialisation
280 res = self.unserialise(classname, res)
282 # store off in the cache dict
283 if cache:
284 cache_dict[nodeid] = res
286 return res
288 def destroynode(self, classname, nodeid):
289 '''Remove a node from the database. Called exclusively by the
290 destroy() method on Class.
291 '''
292 if __debug__:
293 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
295 # remove from cache and newnodes if it's there
296 if (self.cache.has_key(classname) and
297 self.cache[classname].has_key(nodeid)):
298 del self.cache[classname][nodeid]
299 if (self.newnodes.has_key(classname) and
300 self.newnodes[classname].has_key(nodeid)):
301 del self.newnodes[classname][nodeid]
303 # see if there's any obvious commit actions that we should get rid of
304 for entry in self.transactions[:]:
305 if entry[1][:2] == (classname, nodeid):
306 self.transactions.remove(entry)
308 # add to the destroyednodes map
309 self.destroyednodes.setdefault(classname, {})[nodeid] = 1
311 # add the destroy commit action
312 self.transactions.append((self.doDestroyNode, (classname, nodeid)))
314 def serialise(self, classname, node):
315 '''Copy the node contents, converting non-marshallable data into
316 marshallable data.
317 '''
318 if __debug__:
319 print >>hyperdb.DEBUG, 'serialise', classname, node
320 properties = self.getclass(classname).getprops()
321 d = {}
322 for k, v in node.items():
323 # if the property doesn't exist, or is the "retired" flag then
324 # it won't be in the properties dict
325 if not properties.has_key(k):
326 d[k] = v
327 continue
329 # get the property spec
330 prop = properties[k]
332 if isinstance(prop, Password):
333 d[k] = str(v)
334 elif isinstance(prop, Date) and v is not None:
335 d[k] = v.get_tuple()
336 elif isinstance(prop, Interval) and v is not None:
337 d[k] = v.get_tuple()
338 else:
339 d[k] = v
340 return d
342 def unserialise(self, classname, node):
343 '''Decode the marshalled node data
344 '''
345 if __debug__:
346 print >>hyperdb.DEBUG, 'unserialise', classname, node
347 properties = self.getclass(classname).getprops()
348 d = {}
349 for k, v in node.items():
350 # if the property doesn't exist, or is the "retired" flag then
351 # it won't be in the properties dict
352 if not properties.has_key(k):
353 d[k] = v
354 continue
356 # get the property spec
357 prop = properties[k]
359 if isinstance(prop, Date) and v is not None:
360 d[k] = date.Date(v)
361 elif isinstance(prop, Interval) and v is not None:
362 d[k] = date.Interval(v)
363 elif isinstance(prop, Password):
364 p = password.Password()
365 p.unpack(v)
366 d[k] = p
367 else:
368 d[k] = v
369 return d
371 def hasnode(self, classname, nodeid, db=None):
372 ''' determine if the database has a given node
373 '''
374 if __debug__:
375 print >>hyperdb.DEBUG, 'hasnode', (self, classname, nodeid, db)
377 # try the cache
378 cache = self.cache.setdefault(classname, {})
379 if cache.has_key(nodeid):
380 if __debug__:
381 print >>hyperdb.TRACE, 'has %s %s cached'%(classname, nodeid)
382 return 1
383 if __debug__:
384 print >>hyperdb.TRACE, 'has %s %s'%(classname, nodeid)
386 # not in the cache - check the database
387 if db is None:
388 db = self.getclassdb(classname)
389 res = db.has_key(nodeid)
390 return res
392 def countnodes(self, classname, db=None):
393 if __debug__:
394 print >>hyperdb.DEBUG, 'countnodes', (self, classname, db)
396 count = 0
398 # include the uncommitted nodes
399 if self.newnodes.has_key(classname):
400 count += len(self.newnodes[classname])
401 if self.destroyednodes.has_key(classname):
402 count -= len(self.destroyednodes[classname])
404 # and count those in the DB
405 if db is None:
406 db = self.getclassdb(classname)
407 count = count + len(db.keys())
408 return count
410 def getnodeids(self, classname, db=None):
411 if __debug__:
412 print >>hyperdb.DEBUG, 'getnodeids', (self, classname, db)
414 res = []
416 # start off with the new nodes
417 if self.newnodes.has_key(classname):
418 res += self.newnodes[classname].keys()
420 if db is None:
421 db = self.getclassdb(classname)
422 res = res + db.keys()
424 # remove the uncommitted, destroyed nodes
425 if self.destroyednodes.has_key(classname):
426 for nodeid in self.destroyednodes[classname].keys():
427 if db.has_key(nodeid):
428 res.remove(nodeid)
430 return res
433 #
434 # Files - special node properties
435 # inherited from FileStorage
437 #
438 # Journal
439 #
440 def addjournal(self, classname, nodeid, action, params):
441 ''' Journal the Action
442 'action' may be:
444 'create' or 'set' -- 'params' is a dictionary of property values
445 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
446 'retire' -- 'params' is None
447 '''
448 if __debug__:
449 print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid,
450 action, params)
451 self.transactions.append((self.doSaveJournal, (classname, nodeid,
452 action, params)))
454 def getjournal(self, classname, nodeid):
455 ''' get the journal for id
457 Raise IndexError if the node doesn't exist (as per history()'s
458 API)
459 '''
460 if __debug__:
461 print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid)
462 # attempt to open the journal - in some rare cases, the journal may
463 # not exist
464 try:
465 db = self.opendb('journals.%s'%classname, 'r')
466 except anydbm.error, error:
467 if str(error) == "need 'c' or 'n' flag to open new db":
468 raise IndexError, 'no such %s %s'%(classname, nodeid)
469 elif error.args[0] != 2:
470 raise
471 raise IndexError, 'no such %s %s'%(classname, nodeid)
472 try:
473 journal = marshal.loads(db[nodeid])
474 except KeyError:
475 db.close()
476 raise IndexError, 'no such %s %s'%(classname, nodeid)
477 db.close()
478 res = []
479 for nodeid, date_stamp, user, action, params in journal:
480 res.append((nodeid, date.Date(date_stamp), user, action, params))
481 return res
483 def pack(self, pack_before):
484 ''' delete all journal entries before 'pack_before' '''
485 if __debug__:
486 print >>hyperdb.DEBUG, 'packjournal', (self, pack_before)
488 pack_before = pack_before.get_tuple()
490 classes = self.getclasses()
492 # figure the class db type
494 for classname in classes:
495 db_name = 'journals.%s'%classname
496 path = os.path.join(os.getcwd(), self.dir, classname)
497 db_type = self.determine_db_type(path)
498 db = self.opendb(db_name, 'w')
500 for key in db.keys():
501 journal = marshal.loads(db[key])
502 l = []
503 last_set_entry = None
504 for entry in journal:
505 (nodeid, date_stamp, self.journaltag, action,
506 params) = entry
507 if date_stamp > pack_before or action == 'create':
508 l.append(entry)
509 elif action == 'set':
510 # grab the last set entry to keep information on
511 # activity
512 last_set_entry = entry
513 if last_set_entry:
514 date_stamp = last_set_entry[1]
515 # if the last set entry was made after the pack date
516 # then it is already in the list
517 if date_stamp < pack_before:
518 l.append(last_set_entry)
519 db[key] = marshal.dumps(l)
520 if db_type == 'gdbm':
521 db.reorganize()
522 db.close()
525 #
526 # Basic transaction support
527 #
528 def commit(self):
529 ''' Commit the current transactions.
530 '''
531 if __debug__:
532 print >>hyperdb.DEBUG, 'commit', (self,)
533 # TODO: lock the DB
535 # keep a handle to all the database files opened
536 self.databases = {}
538 # now, do all the transactions
539 reindex = {}
540 for method, args in self.transactions:
541 reindex[method(*args)] = 1
543 # now close all the database files
544 for db in self.databases.values():
545 db.close()
546 del self.databases
547 # TODO: unlock the DB
549 # reindex the nodes that request it
550 for classname, nodeid in filter(None, reindex.keys()):
551 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
552 self.getclass(classname).index(nodeid)
554 # save the indexer state
555 self.indexer.save_index()
557 # all transactions committed, back to normal
558 self.cache = {}
559 self.dirtynodes = {}
560 self.newnodes = {}
561 self.destroyednodes = {}
562 self.transactions = []
564 def getCachedClassDB(self, classname):
565 ''' get the class db, looking in our cache of databases for commit
566 '''
567 # get the database handle
568 db_name = 'nodes.%s'%classname
569 if not self.databases.has_key(db_name):
570 self.databases[db_name] = self.getclassdb(classname, 'c')
571 return self.databases[db_name]
573 def doSaveNode(self, classname, nodeid, node):
574 if __debug__:
575 print >>hyperdb.DEBUG, 'doSaveNode', (self, classname, nodeid,
576 node)
578 db = self.getCachedClassDB(classname)
580 # now save the marshalled data
581 db[nodeid] = marshal.dumps(self.serialise(classname, node))
583 # return the classname, nodeid so we reindex this content
584 return (classname, nodeid)
586 def getCachedJournalDB(self, classname):
587 ''' get the journal db, looking in our cache of databases for commit
588 '''
589 # get the database handle
590 db_name = 'journals.%s'%classname
591 if not self.databases.has_key(db_name):
592 self.databases[db_name] = self.opendb(db_name, 'c')
593 return self.databases[db_name]
595 def doSaveJournal(self, classname, nodeid, action, params):
596 # serialise first
597 if action in ('set', 'create'):
598 params = self.serialise(classname, params)
600 # create the journal entry
601 entry = (nodeid, date.Date().get_tuple(), self.journaltag, action,
602 params)
604 if __debug__:
605 print >>hyperdb.DEBUG, 'doSaveJournal', entry
607 db = self.getCachedJournalDB(classname)
609 # now insert the journal entry
610 if db.has_key(nodeid):
611 # append to existing
612 s = db[nodeid]
613 l = marshal.loads(s)
614 l.append(entry)
615 else:
616 l = [entry]
618 db[nodeid] = marshal.dumps(l)
620 def doDestroyNode(self, classname, nodeid):
621 if __debug__:
622 print >>hyperdb.DEBUG, 'doDestroyNode', (self, classname, nodeid)
624 # delete from the class database
625 db = self.getCachedClassDB(classname)
626 if db.has_key(nodeid):
627 del db[nodeid]
629 # delete from the database
630 db = self.getCachedJournalDB(classname)
631 if db.has_key(nodeid):
632 del db[nodeid]
634 # return the classname, nodeid so we reindex this content
635 return (classname, nodeid)
637 def rollback(self):
638 ''' Reverse all actions from the current transaction.
639 '''
640 if __debug__:
641 print >>hyperdb.DEBUG, 'rollback', (self, )
642 for method, args in self.transactions:
643 # delete temporary files
644 if method == self.doStoreFile:
645 self.rollbackStoreFile(*args)
646 self.cache = {}
647 self.dirtynodes = {}
648 self.newnodes = {}
649 self.destroyednodes = {}
650 self.transactions = []
652 _marker = []
653 class Class(hyperdb.Class):
654 """The handle to a particular class of nodes in a hyperdatabase."""
656 def __init__(self, db, classname, **properties):
657 """Create a new class with a given name and property specification.
659 'classname' must not collide with the name of an existing class,
660 or a ValueError is raised. The keyword arguments in 'properties'
661 must map names to property objects, or a TypeError is raised.
662 """
663 if (properties.has_key('creation') or properties.has_key('activity')
664 or properties.has_key('creator')):
665 raise ValueError, '"creation", "activity" and "creator" are '\
666 'reserved'
668 self.classname = classname
669 self.properties = properties
670 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
671 self.key = ''
673 # should we journal changes (default yes)
674 self.do_journal = 1
676 # do the db-related init stuff
677 db.addclass(self)
679 self.auditors = {'create': [], 'set': [], 'retire': []}
680 self.reactors = {'create': [], 'set': [], 'retire': []}
682 def enableJournalling(self):
683 '''Turn journalling on for this class
684 '''
685 self.do_journal = 1
687 def disableJournalling(self):
688 '''Turn journalling off for this class
689 '''
690 self.do_journal = 0
692 # Editing nodes:
694 def create(self, **propvalues):
695 """Create a new node of this class and return its id.
697 The keyword arguments in 'propvalues' map property names to values.
699 The values of arguments must be acceptable for the types of their
700 corresponding properties or a TypeError is raised.
702 If this class has a key property, it must be present and its value
703 must not collide with other key strings or a ValueError is raised.
705 Any other properties on this class that are missing from the
706 'propvalues' dictionary are set to None.
708 If an id in a link or multilink property does not refer to a valid
709 node, an IndexError is raised.
711 These operations trigger detectors and can be vetoed. Attempts
712 to modify the "creation" or "activity" properties cause a KeyError.
713 """
714 if propvalues.has_key('id'):
715 raise KeyError, '"id" is reserved'
717 if self.db.journaltag is None:
718 raise DatabaseError, 'Database open read-only'
720 if propvalues.has_key('creation') or propvalues.has_key('activity'):
721 raise KeyError, '"creation" and "activity" are reserved'
723 self.fireAuditors('create', None, propvalues)
725 # new node's id
726 newid = self.db.newid(self.classname)
728 # validate propvalues
729 num_re = re.compile('^\d+$')
730 for key, value in propvalues.items():
731 if key == self.key:
732 try:
733 self.lookup(value)
734 except KeyError:
735 pass
736 else:
737 raise ValueError, 'node with key "%s" exists'%value
739 # try to handle this property
740 try:
741 prop = self.properties[key]
742 except KeyError:
743 raise KeyError, '"%s" has no property "%s"'%(self.classname,
744 key)
746 if isinstance(prop, Link):
747 if type(value) != type(''):
748 raise ValueError, 'link value must be String'
749 link_class = self.properties[key].classname
750 # if it isn't a number, it's a key
751 if not num_re.match(value):
752 try:
753 value = self.db.classes[link_class].lookup(value)
754 except (TypeError, KeyError):
755 raise IndexError, 'new property "%s": %s not a %s'%(
756 key, value, link_class)
757 elif not self.db.getclass(link_class).hasnode(value):
758 raise IndexError, '%s has no node %s'%(link_class, value)
760 # save off the value
761 propvalues[key] = value
763 # register the link with the newly linked node
764 if self.do_journal and self.properties[key].do_journal:
765 self.db.addjournal(link_class, value, 'link',
766 (self.classname, newid, key))
768 elif isinstance(prop, Multilink):
769 if type(value) != type([]):
770 raise TypeError, 'new property "%s" not a list of ids'%key
772 # clean up and validate the list of links
773 link_class = self.properties[key].classname
774 l = []
775 for entry in value:
776 if type(entry) != type(''):
777 raise ValueError, '"%s" link value (%s) must be '\
778 'String'%(key, value)
779 # if it isn't a number, it's a key
780 if not num_re.match(entry):
781 try:
782 entry = self.db.classes[link_class].lookup(entry)
783 except (TypeError, KeyError):
784 raise IndexError, 'new property "%s": %s not a %s'%(
785 key, entry, self.properties[key].classname)
786 l.append(entry)
787 value = l
788 propvalues[key] = value
790 # handle additions
791 for nodeid in value:
792 if not self.db.getclass(link_class).hasnode(nodeid):
793 raise IndexError, '%s has no node %s'%(link_class,
794 nodeid)
795 # register the link with the newly linked node
796 if self.do_journal and self.properties[key].do_journal:
797 self.db.addjournal(link_class, nodeid, 'link',
798 (self.classname, newid, key))
800 elif isinstance(prop, String):
801 if type(value) != type(''):
802 raise TypeError, 'new property "%s" not a string'%key
804 elif isinstance(prop, Password):
805 if not isinstance(value, password.Password):
806 raise TypeError, 'new property "%s" not a Password'%key
808 elif isinstance(prop, Date):
809 if value is not None and not isinstance(value, date.Date):
810 raise TypeError, 'new property "%s" not a Date'%key
812 elif isinstance(prop, Interval):
813 if value is not None and not isinstance(value, date.Interval):
814 raise TypeError, 'new property "%s" not an Interval'%key
816 elif value is not None and isinstance(prop, Number):
817 try:
818 float(value)
819 except ValueError:
820 raise TypeError, 'new property "%s" not numeric'%key
822 elif value is not None and isinstance(prop, Boolean):
823 try:
824 int(value)
825 except ValueError:
826 raise TypeError, 'new property "%s" not boolean'%key
828 # make sure there's data where there needs to be
829 for key, prop in self.properties.items():
830 if propvalues.has_key(key):
831 continue
832 if key == self.key:
833 raise ValueError, 'key property "%s" is required'%key
834 if isinstance(prop, Multilink):
835 propvalues[key] = []
836 else:
837 propvalues[key] = None
839 # done
840 self.db.addnode(self.classname, newid, propvalues)
841 if self.do_journal:
842 self.db.addjournal(self.classname, newid, 'create', propvalues)
844 self.fireReactors('create', newid, None)
846 return newid
848 def get(self, nodeid, propname, default=_marker, cache=1):
849 """Get the value of a property on an existing node of this class.
851 'nodeid' must be the id of an existing node of this class or an
852 IndexError is raised. 'propname' must be the name of a property
853 of this class or a KeyError is raised.
855 'cache' indicates whether the transaction cache should be queried
856 for the node. If the node has been modified and you need to
857 determine what its values prior to modification are, you need to
858 set cache=0.
860 Attempts to get the "creation" or "activity" properties should
861 do the right thing.
862 """
863 if propname == 'id':
864 return nodeid
866 if propname == 'creation':
867 if not self.do_journal:
868 raise ValueError, 'Journalling is disabled for this class'
869 journal = self.db.getjournal(self.classname, nodeid)
870 if journal:
871 return self.db.getjournal(self.classname, nodeid)[0][1]
872 else:
873 # on the strange chance that there's no journal
874 return date.Date()
875 if propname == 'activity':
876 if not self.do_journal:
877 raise ValueError, 'Journalling is disabled for this class'
878 journal = self.db.getjournal(self.classname, nodeid)
879 if journal:
880 return self.db.getjournal(self.classname, nodeid)[-1][1]
881 else:
882 # on the strange chance that there's no journal
883 return date.Date()
884 if propname == 'creator':
885 if not self.do_journal:
886 raise ValueError, 'Journalling is disabled for this class'
887 journal = self.db.getjournal(self.classname, nodeid)
888 if journal:
889 name = self.db.getjournal(self.classname, nodeid)[0][2]
890 else:
891 return None
892 return self.db.user.lookup(name)
894 # get the property (raises KeyErorr if invalid)
895 prop = self.properties[propname]
897 # get the node's dict
898 d = self.db.getnode(self.classname, nodeid, cache=cache)
900 if not d.has_key(propname):
901 if default is _marker:
902 if isinstance(prop, Multilink):
903 return []
904 else:
905 return None
906 else:
907 return default
909 return d[propname]
911 # XXX not in spec
912 def getnode(self, nodeid, cache=1):
913 ''' Return a convenience wrapper for the node.
915 'nodeid' must be the id of an existing node of this class or an
916 IndexError is raised.
918 'cache' indicates whether the transaction cache should be queried
919 for the node. If the node has been modified and you need to
920 determine what its values prior to modification are, you need to
921 set cache=0.
922 '''
923 return Node(self, nodeid, cache=cache)
925 def set(self, nodeid, **propvalues):
926 """Modify a property on an existing node of this class.
928 'nodeid' must be the id of an existing node of this class or an
929 IndexError is raised.
931 Each key in 'propvalues' must be the name of a property of this
932 class or a KeyError is raised.
934 All values in 'propvalues' must be acceptable types for their
935 corresponding properties or a TypeError is raised.
937 If the value of the key property is set, it must not collide with
938 other key strings or a ValueError is raised.
940 If the value of a Link or Multilink property contains an invalid
941 node id, a ValueError is raised.
943 These operations trigger detectors and can be vetoed. Attempts
944 to modify the "creation" or "activity" properties cause a KeyError.
945 """
946 if not propvalues:
947 return
949 if propvalues.has_key('creation') or propvalues.has_key('activity'):
950 raise KeyError, '"creation" and "activity" are reserved'
952 if propvalues.has_key('id'):
953 raise KeyError, '"id" is reserved'
955 if self.db.journaltag is None:
956 raise DatabaseError, 'Database open read-only'
958 self.fireAuditors('set', nodeid, propvalues)
959 # Take a copy of the node dict so that the subsequent set
960 # operation doesn't modify the oldvalues structure.
961 try:
962 # try not using the cache initially
963 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid,
964 cache=0))
965 except IndexError:
966 # this will be needed if somone does a create() and set()
967 # with no intervening commit()
968 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
970 node = self.db.getnode(self.classname, nodeid)
971 if node.has_key(self.db.RETIRED_FLAG):
972 raise IndexError
973 num_re = re.compile('^\d+$')
975 # if the journal value is to be different, store it in here
976 journalvalues = {}
978 for propname, value in propvalues.items():
979 # check to make sure we're not duplicating an existing key
980 if propname == self.key and node[propname] != value:
981 try:
982 self.lookup(value)
983 except KeyError:
984 pass
985 else:
986 raise ValueError, 'node with key "%s" exists'%value
988 # this will raise the KeyError if the property isn't valid
989 # ... we don't use getprops() here because we only care about
990 # the writeable properties.
991 prop = self.properties[propname]
993 # if the value's the same as the existing value, no sense in
994 # doing anything
995 if node.has_key(propname) and value == node[propname]:
996 del propvalues[propname]
997 continue
999 # do stuff based on the prop type
1000 if isinstance(prop, Link):
1001 link_class = self.properties[propname].classname
1002 # if it isn't a number, it's a key
1003 if type(value) != type(''):
1004 raise ValueError, 'link value must be String'
1005 if not num_re.match(value):
1006 try:
1007 value = self.db.classes[link_class].lookup(value)
1008 except (TypeError, KeyError):
1009 raise IndexError, 'new property "%s": %s not a %s'%(
1010 propname, value, self.properties[propname].classname)
1012 if not self.db.getclass(link_class).hasnode(value):
1013 raise IndexError, '%s has no node %s'%(link_class, value)
1015 if self.do_journal and self.properties[propname].do_journal:
1016 # register the unlink with the old linked node
1017 if node[propname] is not None:
1018 self.db.addjournal(link_class, node[propname], 'unlink',
1019 (self.classname, nodeid, propname))
1021 # register the link with the newly linked node
1022 if value is not None:
1023 self.db.addjournal(link_class, value, 'link',
1024 (self.classname, nodeid, propname))
1026 elif isinstance(prop, Multilink):
1027 if type(value) != type([]):
1028 raise TypeError, 'new property "%s" not a list of'\
1029 ' ids'%propname
1030 link_class = self.properties[propname].classname
1031 l = []
1032 for entry in value:
1033 # if it isn't a number, it's a key
1034 if type(entry) != type(''):
1035 raise ValueError, 'new property "%s" link value ' \
1036 'must be a string'%propname
1037 if not num_re.match(entry):
1038 try:
1039 entry = self.db.classes[link_class].lookup(entry)
1040 except (TypeError, KeyError):
1041 raise IndexError, 'new property "%s": %s not a %s'%(
1042 propname, entry,
1043 self.properties[propname].classname)
1044 l.append(entry)
1045 value = l
1046 propvalues[propname] = value
1048 # figure the journal entry for this property
1049 add = []
1050 remove = []
1052 # handle removals
1053 if node.has_key(propname):
1054 l = node[propname]
1055 else:
1056 l = []
1057 for id in l[:]:
1058 if id in value:
1059 continue
1060 # register the unlink with the old linked node
1061 if self.do_journal and self.properties[propname].do_journal:
1062 self.db.addjournal(link_class, id, 'unlink',
1063 (self.classname, nodeid, propname))
1064 l.remove(id)
1065 remove.append(id)
1067 # handle additions
1068 for id in value:
1069 if not self.db.getclass(link_class).hasnode(id):
1070 raise IndexError, '%s has no node %s'%(link_class, id)
1071 if id in l:
1072 continue
1073 # register the link with the newly linked node
1074 if self.do_journal and self.properties[propname].do_journal:
1075 self.db.addjournal(link_class, id, 'link',
1076 (self.classname, nodeid, propname))
1077 l.append(id)
1078 add.append(id)
1080 # figure the journal entry
1081 l = []
1082 if add:
1083 l.append(('add', add))
1084 if remove:
1085 l.append(('remove', remove))
1086 if l:
1087 journalvalues[propname] = tuple(l)
1089 elif isinstance(prop, String):
1090 if value is not None and type(value) != type(''):
1091 raise TypeError, 'new property "%s" not a string'%propname
1093 elif isinstance(prop, Password):
1094 if not isinstance(value, password.Password):
1095 raise TypeError, 'new property "%s" not a Password'%propname
1096 propvalues[propname] = value
1098 elif value is not None and isinstance(prop, Date):
1099 if not isinstance(value, date.Date):
1100 raise TypeError, 'new property "%s" not a Date'% propname
1101 propvalues[propname] = value
1103 elif value is not None and isinstance(prop, Interval):
1104 if not isinstance(value, date.Interval):
1105 raise TypeError, 'new property "%s" not an '\
1106 'Interval'%propname
1107 propvalues[propname] = value
1109 elif value is not None and isinstance(prop, Number):
1110 try:
1111 float(value)
1112 except ValueError:
1113 raise TypeError, 'new property "%s" not numeric'%propname
1115 elif value is not None and isinstance(prop, Boolean):
1116 try:
1117 int(value)
1118 except ValueError:
1119 raise TypeError, 'new property "%s" not boolean'%propname
1121 node[propname] = value
1123 # nothing to do?
1124 if not propvalues:
1125 return
1127 # do the set, and journal it
1128 self.db.setnode(self.classname, nodeid, node)
1130 if self.do_journal:
1131 propvalues.update(journalvalues)
1132 self.db.addjournal(self.classname, nodeid, 'set', propvalues)
1134 self.fireReactors('set', nodeid, oldvalues)
1136 def retire(self, nodeid):
1137 """Retire a node.
1139 The properties on the node remain available from the get() method,
1140 and the node's id is never reused.
1142 Retired nodes are not returned by the find(), list(), or lookup()
1143 methods, and other nodes may reuse the values of their key properties.
1145 These operations trigger detectors and can be vetoed. Attempts
1146 to modify the "creation" or "activity" properties cause a KeyError.
1147 """
1148 if self.db.journaltag is None:
1149 raise DatabaseError, 'Database open read-only'
1151 self.fireAuditors('retire', nodeid, None)
1153 node = self.db.getnode(self.classname, nodeid)
1154 node[self.db.RETIRED_FLAG] = 1
1155 self.db.setnode(self.classname, nodeid, node)
1156 if self.do_journal:
1157 self.db.addjournal(self.classname, nodeid, 'retired', None)
1159 self.fireReactors('retire', nodeid, None)
1161 def destroy(self, nodeid):
1162 """Destroy a node.
1164 WARNING: this method should never be used except in extremely rare
1165 situations where there could never be links to the node being
1166 deleted
1167 WARNING: use retire() instead
1168 WARNING: the properties of this node will not be available ever again
1169 WARNING: really, use retire() instead
1171 Well, I think that's enough warnings. This method exists mostly to
1172 support the session storage of the cgi interface.
1173 """
1174 if self.db.journaltag is None:
1175 raise DatabaseError, 'Database open read-only'
1176 self.db.destroynode(self.classname, nodeid)
1178 def history(self, nodeid):
1179 """Retrieve the journal of edits on a particular node.
1181 'nodeid' must be the id of an existing node of this class or an
1182 IndexError is raised.
1184 The returned list contains tuples of the form
1186 (date, tag, action, params)
1188 'date' is a Timestamp object specifying the time of the change and
1189 'tag' is the journaltag specified when the database was opened.
1190 """
1191 if not self.do_journal:
1192 raise ValueError, 'Journalling is disabled for this class'
1193 return self.db.getjournal(self.classname, nodeid)
1195 # Locating nodes:
1196 def hasnode(self, nodeid):
1197 '''Determine if the given nodeid actually exists
1198 '''
1199 return self.db.hasnode(self.classname, nodeid)
1201 def setkey(self, propname):
1202 """Select a String property of this class to be the key property.
1204 'propname' must be the name of a String property of this class or
1205 None, or a TypeError is raised. The values of the key property on
1206 all existing nodes must be unique or a ValueError is raised. If the
1207 property doesn't exist, KeyError is raised.
1208 """
1209 prop = self.getprops()[propname]
1210 if not isinstance(prop, String):
1211 raise TypeError, 'key properties must be String'
1212 self.key = propname
1214 def getkey(self):
1215 """Return the name of the key property for this class or None."""
1216 return self.key
1218 def labelprop(self, default_to_id=0):
1219 ''' Return the property name for a label for the given node.
1221 This method attempts to generate a consistent label for the node.
1222 It tries the following in order:
1223 1. key property
1224 2. "name" property
1225 3. "title" property
1226 4. first property from the sorted property name list
1227 '''
1228 k = self.getkey()
1229 if k:
1230 return k
1231 props = self.getprops()
1232 if props.has_key('name'):
1233 return 'name'
1234 elif props.has_key('title'):
1235 return 'title'
1236 if default_to_id:
1237 return 'id'
1238 props = props.keys()
1239 props.sort()
1240 return props[0]
1242 # TODO: set up a separate index db file for this? profile?
1243 def lookup(self, keyvalue):
1244 """Locate a particular node by its key property and return its id.
1246 If this class has no key property, a TypeError is raised. If the
1247 'keyvalue' matches one of the values for the key property among
1248 the nodes in this class, the matching node's id is returned;
1249 otherwise a KeyError is raised.
1250 """
1251 cldb = self.db.getclassdb(self.classname)
1252 try:
1253 for nodeid in self.db.getnodeids(self.classname, cldb):
1254 node = self.db.getnode(self.classname, nodeid, cldb)
1255 if node.has_key(self.db.RETIRED_FLAG):
1256 continue
1257 if node[self.key] == keyvalue:
1258 cldb.close()
1259 return nodeid
1260 finally:
1261 cldb.close()
1262 raise KeyError, keyvalue
1264 # XXX: change from spec - allows multiple props to match
1265 def find(self, **propspec):
1266 """Get the ids of nodes in this class which link to the given nodes.
1268 'propspec' consists of keyword args propname={nodeid:1,}
1269 'propname' must be the name of a property in this class, or a
1270 KeyError is raised. That property must be a Link or Multilink
1271 property, or a TypeError is raised.
1273 Any node in this class whose 'propname' property links to any of the
1274 nodeids will be returned. Used by the full text indexing, which knows
1275 that "foo" occurs in msg1, msg3 and file7, so we have hits on these issues:
1276 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1277 """
1278 propspec = propspec.items()
1279 for propname, nodeids in propspec:
1280 # check the prop is OK
1281 prop = self.properties[propname]
1282 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1283 raise TypeError, "'%s' not a Link/Multilink property"%propname
1285 # ok, now do the find
1286 cldb = self.db.getclassdb(self.classname)
1287 l = []
1288 try:
1289 for id in self.db.getnodeids(self.classname, db=cldb):
1290 node = self.db.getnode(self.classname, id, db=cldb)
1291 if node.has_key(self.db.RETIRED_FLAG):
1292 continue
1293 for propname, nodeids in propspec:
1294 # can't test if the node doesn't have this property
1295 if not node.has_key(propname):
1296 continue
1297 if type(nodeids) is type(''):
1298 nodeids = {nodeids:1}
1299 prop = self.properties[propname]
1300 value = node[propname]
1301 if isinstance(prop, Link) and nodeids.has_key(value):
1302 l.append(id)
1303 break
1304 elif isinstance(prop, Multilink):
1305 hit = 0
1306 for v in value:
1307 if nodeids.has_key(v):
1308 l.append(id)
1309 hit = 1
1310 break
1311 if hit:
1312 break
1313 finally:
1314 cldb.close()
1315 return l
1317 def stringFind(self, **requirements):
1318 """Locate a particular node by matching a set of its String
1319 properties in a caseless search.
1321 If the property is not a String property, a TypeError is raised.
1323 The return is a list of the id of all nodes that match.
1324 """
1325 for propname in requirements.keys():
1326 prop = self.properties[propname]
1327 if isinstance(not prop, String):
1328 raise TypeError, "'%s' not a String property"%propname
1329 requirements[propname] = requirements[propname].lower()
1330 l = []
1331 cldb = self.db.getclassdb(self.classname)
1332 try:
1333 for nodeid in self.db.getnodeids(self.classname, cldb):
1334 node = self.db.getnode(self.classname, nodeid, cldb)
1335 if node.has_key(self.db.RETIRED_FLAG):
1336 continue
1337 for key, value in requirements.items():
1338 if node[key] is None or node[key].lower() != value:
1339 break
1340 else:
1341 l.append(nodeid)
1342 finally:
1343 cldb.close()
1344 return l
1346 def list(self):
1347 """Return a list of the ids of the active nodes in this class."""
1348 l = []
1349 cn = self.classname
1350 cldb = self.db.getclassdb(cn)
1351 try:
1352 for nodeid in self.db.getnodeids(cn, cldb):
1353 node = self.db.getnode(cn, nodeid, cldb)
1354 if node.has_key(self.db.RETIRED_FLAG):
1355 continue
1356 l.append(nodeid)
1357 finally:
1358 cldb.close()
1359 l.sort()
1360 return l
1362 def filter(self, search_matches, filterspec, sort, group,
1363 num_re = re.compile('^\d+$')):
1364 ''' Return a list of the ids of the active nodes in this class that
1365 match the 'filter' spec, sorted by the group spec and then the
1366 sort spec.
1368 "filterspec" is {propname: value(s)}
1369 "sort" is ['+propname', '-propname', 'propname', ...]
1370 "group is ['+propname', '-propname', 'propname', ...]
1371 '''
1372 cn = self.classname
1374 # optimise filterspec
1375 l = []
1376 props = self.getprops()
1377 LINK = 0
1378 MULTILINK = 1
1379 STRING = 2
1380 OTHER = 6
1381 for k, v in filterspec.items():
1382 propclass = props[k]
1383 if isinstance(propclass, Link):
1384 if type(v) is not type([]):
1385 v = [v]
1386 # replace key values with node ids
1387 u = []
1388 link_class = self.db.classes[propclass.classname]
1389 for entry in v:
1390 if entry == '-1': entry = None
1391 elif not num_re.match(entry):
1392 try:
1393 entry = link_class.lookup(entry)
1394 except (TypeError,KeyError):
1395 raise ValueError, 'property "%s": %s not a %s'%(
1396 k, entry, self.properties[k].classname)
1397 u.append(entry)
1399 l.append((LINK, k, u))
1400 elif isinstance(propclass, Multilink):
1401 if type(v) is not type([]):
1402 v = [v]
1403 # replace key values with node ids
1404 u = []
1405 link_class = self.db.classes[propclass.classname]
1406 for entry in v:
1407 if not num_re.match(entry):
1408 try:
1409 entry = link_class.lookup(entry)
1410 except (TypeError,KeyError):
1411 raise ValueError, 'new property "%s": %s not a %s'%(
1412 k, entry, self.properties[k].classname)
1413 u.append(entry)
1414 l.append((MULTILINK, k, u))
1415 elif isinstance(propclass, String):
1416 # simple glob searching
1417 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1418 v = v.replace('?', '.')
1419 v = v.replace('*', '.*?')
1420 l.append((STRING, k, re.compile(v, re.I)))
1421 elif isinstance(propclass, Boolean):
1422 if type(v) is type(''):
1423 bv = v.lower() in ('yes', 'true', 'on', '1')
1424 else:
1425 bv = v
1426 l.append((OTHER, k, bv))
1427 elif isinstance(propclass, Number):
1428 l.append((OTHER, k, int(v)))
1429 else:
1430 l.append((OTHER, k, v))
1431 filterspec = l
1433 # now, find all the nodes that are active and pass filtering
1434 l = []
1435 cldb = self.db.getclassdb(cn)
1436 try:
1437 # TODO: only full-scan once (use items())
1438 for nodeid in self.db.getnodeids(cn, cldb):
1439 node = self.db.getnode(cn, nodeid, cldb)
1440 if node.has_key(self.db.RETIRED_FLAG):
1441 continue
1442 # apply filter
1443 for t, k, v in filterspec:
1444 # make sure the node has the property
1445 if not node.has_key(k):
1446 # this node doesn't have this property, so reject it
1447 break
1449 # now apply the property filter
1450 if t == LINK:
1451 # link - if this node's property doesn't appear in the
1452 # filterspec's nodeid list, skip it
1453 if node[k] not in v:
1454 break
1455 elif t == MULTILINK:
1456 # multilink - if any of the nodeids required by the
1457 # filterspec aren't in this node's property, then skip
1458 # it
1459 have = node[k]
1460 for want in v:
1461 if want not in have:
1462 break
1463 else:
1464 continue
1465 break
1466 elif t == STRING:
1467 # RE search
1468 if node[k] is None or not v.search(node[k]):
1469 break
1470 elif t == OTHER:
1471 # straight value comparison for the other types
1472 if node[k] != v:
1473 break
1474 else:
1475 l.append((nodeid, node))
1476 finally:
1477 cldb.close()
1478 l.sort()
1480 # filter based on full text search
1481 if search_matches is not None:
1482 k = []
1483 for v in l:
1484 if search_matches.has_key(v[0]):
1485 k.append(v)
1486 l = k
1488 # optimise sort
1489 m = []
1490 for entry in sort:
1491 if entry[0] != '-':
1492 m.append(('+', entry))
1493 else:
1494 m.append((entry[0], entry[1:]))
1495 sort = m
1497 # optimise group
1498 m = []
1499 for entry in group:
1500 if entry[0] != '-':
1501 m.append(('+', entry))
1502 else:
1503 m.append((entry[0], entry[1:]))
1504 group = m
1505 # now, sort the result
1506 def sortfun(a, b, sort=sort, group=group, properties=self.getprops(),
1507 db = self.db, cl=self):
1508 a_id, an = a
1509 b_id, bn = b
1510 # sort by group and then sort
1511 for list in group, sort:
1512 for dir, prop in list:
1513 # sorting is class-specific
1514 propclass = properties[prop]
1516 # handle the properties that might be "faked"
1517 # also, handle possible missing properties
1518 try:
1519 if not an.has_key(prop):
1520 an[prop] = cl.get(a_id, prop)
1521 av = an[prop]
1522 except KeyError:
1523 # the node doesn't have a value for this property
1524 if isinstance(propclass, Multilink): av = []
1525 else: av = ''
1526 try:
1527 if not bn.has_key(prop):
1528 bn[prop] = cl.get(b_id, prop)
1529 bv = bn[prop]
1530 except KeyError:
1531 # the node doesn't have a value for this property
1532 if isinstance(propclass, Multilink): bv = []
1533 else: bv = ''
1535 # String and Date values are sorted in the natural way
1536 if isinstance(propclass, String):
1537 # clean up the strings
1538 if av and av[0] in string.uppercase:
1539 av = an[prop] = av.lower()
1540 if bv and bv[0] in string.uppercase:
1541 bv = bn[prop] = bv.lower()
1542 if (isinstance(propclass, String) or
1543 isinstance(propclass, Date)):
1544 # it might be a string that's really an integer
1545 try:
1546 av = int(av)
1547 bv = int(bv)
1548 except:
1549 pass
1550 if dir == '+':
1551 r = cmp(av, bv)
1552 if r != 0: return r
1553 elif dir == '-':
1554 r = cmp(bv, av)
1555 if r != 0: return r
1557 # Link properties are sorted according to the value of
1558 # the "order" property on the linked nodes if it is
1559 # present; or otherwise on the key string of the linked
1560 # nodes; or finally on the node ids.
1561 elif isinstance(propclass, Link):
1562 link = db.classes[propclass.classname]
1563 if av is None and bv is not None: return -1
1564 if av is not None and bv is None: return 1
1565 if av is None and bv is None: continue
1566 if link.getprops().has_key('order'):
1567 if dir == '+':
1568 r = cmp(link.get(av, 'order'),
1569 link.get(bv, 'order'))
1570 if r != 0: return r
1571 elif dir == '-':
1572 r = cmp(link.get(bv, 'order'),
1573 link.get(av, 'order'))
1574 if r != 0: return r
1575 elif link.getkey():
1576 key = link.getkey()
1577 if dir == '+':
1578 r = cmp(link.get(av, key), link.get(bv, key))
1579 if r != 0: return r
1580 elif dir == '-':
1581 r = cmp(link.get(bv, key), link.get(av, key))
1582 if r != 0: return r
1583 else:
1584 if dir == '+':
1585 r = cmp(av, bv)
1586 if r != 0: return r
1587 elif dir == '-':
1588 r = cmp(bv, av)
1589 if r != 0: return r
1591 # Multilink properties are sorted according to how many
1592 # links are present.
1593 elif isinstance(propclass, Multilink):
1594 if dir == '+':
1595 r = cmp(len(av), len(bv))
1596 if r != 0: return r
1597 elif dir == '-':
1598 r = cmp(len(bv), len(av))
1599 if r != 0: return r
1600 elif isinstance(propclass, Number) or isinstance(propclass, Boolean):
1601 if dir == '+':
1602 r = cmp(av, bv)
1603 elif dir == '-':
1604 r = cmp(bv, av)
1606 # end for dir, prop in list:
1607 # end for list in sort, group:
1608 # if all else fails, compare the ids
1609 return cmp(a[0], b[0])
1611 l.sort(sortfun)
1612 return [i[0] for i in l]
1614 def count(self):
1615 """Get the number of nodes in this class.
1617 If the returned integer is 'numnodes', the ids of all the nodes
1618 in this class run from 1 to numnodes, and numnodes+1 will be the
1619 id of the next node to be created in this class.
1620 """
1621 return self.db.countnodes(self.classname)
1623 # Manipulating properties:
1625 def getprops(self, protected=1):
1626 """Return a dictionary mapping property names to property objects.
1627 If the "protected" flag is true, we include protected properties -
1628 those which may not be modified.
1630 In addition to the actual properties on the node, these
1631 methods provide the "creation" and "activity" properties. If the
1632 "protected" flag is true, we include protected properties - those
1633 which may not be modified.
1634 """
1635 d = self.properties.copy()
1636 if protected:
1637 d['id'] = String()
1638 d['creation'] = hyperdb.Date()
1639 d['activity'] = hyperdb.Date()
1640 d['creator'] = hyperdb.Link("user")
1641 return d
1643 def addprop(self, **properties):
1644 """Add properties to this class.
1646 The keyword arguments in 'properties' must map names to property
1647 objects, or a TypeError is raised. None of the keys in 'properties'
1648 may collide with the names of existing properties, or a ValueError
1649 is raised before any properties have been added.
1650 """
1651 for key in properties.keys():
1652 if self.properties.has_key(key):
1653 raise ValueError, key
1654 self.properties.update(properties)
1656 def index(self, nodeid):
1657 '''Add (or refresh) the node to search indexes
1658 '''
1659 # find all the String properties that have indexme
1660 for prop, propclass in self.getprops().items():
1661 if isinstance(propclass, String) and propclass.indexme:
1662 try:
1663 value = str(self.get(nodeid, prop))
1664 except IndexError:
1665 # node no longer exists - entry should be removed
1666 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1667 else:
1668 # and index them under (classname, nodeid, property)
1669 self.db.indexer.add_text((self.classname, nodeid, prop),
1670 value)
1672 #
1673 # Detector interface
1674 #
1675 def audit(self, event, detector):
1676 """Register a detector
1677 """
1678 l = self.auditors[event]
1679 if detector not in l:
1680 self.auditors[event].append(detector)
1682 def fireAuditors(self, action, nodeid, newvalues):
1683 """Fire all registered auditors.
1684 """
1685 for audit in self.auditors[action]:
1686 audit(self.db, self, nodeid, newvalues)
1688 def react(self, event, detector):
1689 """Register a detector
1690 """
1691 l = self.reactors[event]
1692 if detector not in l:
1693 self.reactors[event].append(detector)
1695 def fireReactors(self, action, nodeid, oldvalues):
1696 """Fire all registered reactors.
1697 """
1698 for react in self.reactors[action]:
1699 react(self.db, self, nodeid, oldvalues)
1701 class FileClass(Class):
1702 '''This class defines a large chunk of data. To support this, it has a
1703 mandatory String property "content" which is typically saved off
1704 externally to the hyperdb.
1706 The default MIME type of this data is defined by the
1707 "default_mime_type" class attribute, which may be overridden by each
1708 node if the class defines a "type" String property.
1709 '''
1710 default_mime_type = 'text/plain'
1712 def create(self, **propvalues):
1713 ''' snaffle the file propvalue and store in a file
1714 '''
1715 content = propvalues['content']
1716 del propvalues['content']
1717 newid = Class.create(self, **propvalues)
1718 self.db.storefile(self.classname, newid, None, content)
1719 return newid
1721 def get(self, nodeid, propname, default=_marker, cache=1):
1722 ''' trap the content propname and get it from the file
1723 '''
1725 poss_msg = 'Possibly a access right configuration problem.'
1726 if propname == 'content':
1727 try:
1728 return self.db.getfile(self.classname, nodeid, None)
1729 except IOError, (strerror):
1730 # BUG: by catching this we donot see an error in the log.
1731 return 'ERROR reading file: %s%s\n%s\n%s'%(
1732 self.classname, nodeid, poss_msg, strerror)
1733 if default is not _marker:
1734 return Class.get(self, nodeid, propname, default, cache=cache)
1735 else:
1736 return Class.get(self, nodeid, propname, cache=cache)
1738 def getprops(self, protected=1):
1739 ''' In addition to the actual properties on the node, these methods
1740 provide the "content" property. If the "protected" flag is true,
1741 we include protected properties - those which may not be
1742 modified.
1743 '''
1744 d = Class.getprops(self, protected=protected).copy()
1745 if protected:
1746 d['content'] = hyperdb.String()
1747 return d
1749 def index(self, nodeid):
1750 ''' Index the node in the search index.
1752 We want to index the content in addition to the normal String
1753 property indexing.
1754 '''
1755 # perform normal indexing
1756 Class.index(self, nodeid)
1758 # get the content to index
1759 content = self.get(nodeid, 'content')
1761 # figure the mime type
1762 if self.properties.has_key('type'):
1763 mime_type = self.get(nodeid, 'type')
1764 else:
1765 mime_type = self.default_mime_type
1767 # and index!
1768 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
1769 mime_type)
1771 # XXX deviation from spec - was called ItemClass
1772 class IssueClass(Class, roundupdb.IssueClass):
1773 # Overridden methods:
1774 def __init__(self, db, classname, **properties):
1775 """The newly-created class automatically includes the "messages",
1776 "files", "nosy", and "superseder" properties. If the 'properties'
1777 dictionary attempts to specify any of these properties or a
1778 "creation" or "activity" property, a ValueError is raised.
1779 """
1780 if not properties.has_key('title'):
1781 properties['title'] = hyperdb.String(indexme='yes')
1782 if not properties.has_key('messages'):
1783 properties['messages'] = hyperdb.Multilink("msg")
1784 if not properties.has_key('files'):
1785 properties['files'] = hyperdb.Multilink("file")
1786 if not properties.has_key('nosy'):
1787 properties['nosy'] = hyperdb.Multilink("user")
1788 if not properties.has_key('superseder'):
1789 properties['superseder'] = hyperdb.Multilink(classname)
1790 Class.__init__(self, db, classname, **properties)
1792 #
1793 #$Log: not supported by cvs2svn $
1794 #Revision 1.55 2002/07/30 08:22:38 richard
1795 #Session storage in the hyperdb was horribly, horribly inefficient. We use
1796 #a simple anydbm wrapper now - which could be overridden by the metakit
1797 #backend or RDB backend if necessary.
1798 #Much, much better.
1799 #
1800 #Revision 1.54 2002/07/26 08:26:59 richard
1801 #Very close now. The cgi and mailgw now use the new security API. The two
1802 #templates have been migrated to that setup. Lots of unit tests. Still some
1803 #issue in the web form for editing Roles assigned to users.
1804 #
1805 #Revision 1.53 2002/07/25 07:14:06 richard
1806 #Bugger it. Here's the current shape of the new security implementation.
1807 #Still to do:
1808 # . call the security funcs from cgi and mailgw
1809 # . change shipped templates to include correct initialisation and remove
1810 # the old config vars
1811 #... that seems like a lot. The bulk of the work has been done though. Honest :)
1812 #
1813 #Revision 1.52 2002/07/19 03:36:34 richard
1814 #Implemented the destroy() method needed by the session database (and possibly
1815 #others). At the same time, I removed the leading underscores from the hyperdb
1816 #methods that Really Didn't Need Them.
1817 #The journal also raises IndexError now for all situations where there is a
1818 #request for the journal of a node that doesn't have one. It used to return
1819 #[] in _some_ situations, but not all. This _may_ break code, but the tests
1820 #pass...
1821 #
1822 #Revision 1.51 2002/07/18 23:07:08 richard
1823 #Unit tests and a few fixes.
1824 #
1825 #Revision 1.50 2002/07/18 11:50:58 richard
1826 #added tests for number type too
1827 #
1828 #Revision 1.49 2002/07/18 11:41:10 richard
1829 #added tests for boolean type, and fixes to anydbm backend
1830 #
1831 #Revision 1.48 2002/07/18 11:17:31 gmcm
1832 #Add Number and Boolean types to hyperdb.
1833 #Add conversion cases to web, mail & admin interfaces.
1834 #Add storage/serialization cases to back_anydbm & back_metakit.
1835 #
1836 #Revision 1.47 2002/07/14 23:18:20 richard
1837 #. fixed the journal bloat from multilink changes - we just log the add or
1838 # remove operations, not the whole list
1839 #
1840 #Revision 1.46 2002/07/14 06:06:34 richard
1841 #Did some old TODOs
1842 #
1843 #Revision 1.45 2002/07/14 04:03:14 richard
1844 #Implemented a switch to disable journalling for a Class. CGI session
1845 #database now uses it.
1846 #
1847 #Revision 1.44 2002/07/14 02:05:53 richard
1848 #. all storage-specific code (ie. backend) is now implemented by the backends
1849 #
1850 #Revision 1.43 2002/07/10 06:30:30 richard
1851 #...except of course it's nice to use valid Python syntax
1852 #
1853 #Revision 1.42 2002/07/10 06:21:38 richard
1854 #Be extra safe
1855 #
1856 #Revision 1.41 2002/07/10 00:21:45 richard
1857 #explicit database closing
1858 #
1859 #Revision 1.40 2002/07/09 04:19:09 richard
1860 #Added reindex command to roundup-admin.
1861 #Fixed reindex on first access.
1862 #Also fixed reindexing of entries that change.
1863 #
1864 #Revision 1.39 2002/07/09 03:02:52 richard
1865 #More indexer work:
1866 #- all String properties may now be indexed too. Currently there's a bit of
1867 # "issue" specific code in the actual searching which needs to be
1868 # addressed. In a nutshell:
1869 # + pass 'indexme="yes"' as a String() property initialisation arg, eg:
1870 # file = FileClass(db, "file", name=String(), type=String(),
1871 # comment=String(indexme="yes"))
1872 # + the comment will then be indexed and be searchable, with the results
1873 # related back to the issue that the file is linked to
1874 #- as a result of this work, the FileClass has a default MIME type that may
1875 # be overridden in a subclass, or by the use of a "type" property as is
1876 # done in the default templates.
1877 #- the regeneration of the indexes (if necessary) is done once the schema is
1878 # set up in the dbinit.
1879 #
1880 #Revision 1.38 2002/07/08 06:58:15 richard
1881 #cleaned up the indexer code:
1882 # - it splits more words out (much simpler, faster splitter)
1883 # - removed code we'll never use (roundup.roundup_indexer has the full
1884 # implementation, and replaces roundup.indexer)
1885 # - only index text/plain and rfc822/message (ideas for other text formats to
1886 # index are welcome)
1887 # - added simple unit test for indexer. Needs more tests for regression.
1888 #
1889 #Revision 1.37 2002/06/20 23:52:35 richard
1890 #More informative error message
1891 #
1892 #Revision 1.36 2002/06/19 03:07:19 richard
1893 #Moved the file storage commit into blobfiles where it belongs.
1894 #
1895 #Revision 1.35 2002/05/25 07:16:24 rochecompaan
1896 #Merged search_indexing-branch with HEAD
1897 #
1898 #Revision 1.34 2002/05/15 06:21:21 richard
1899 # . node caching now works, and gives a small boost in performance
1900 #
1901 #As a part of this, I cleaned up the DEBUG output and implemented TRACE
1902 #output (HYPERDBTRACE='file to trace to') with checkpoints at the start of
1903 #CGI requests. Run roundup with python -O to skip all the DEBUG/TRACE stuff
1904 #(using if __debug__ which is compiled out with -O)
1905 #
1906 #Revision 1.33 2002/04/24 10:38:26 rochecompaan
1907 #All database files are now created group readable and writable.
1908 #
1909 #Revision 1.32 2002/04/15 23:25:15 richard
1910 #. node ids are now generated from a lockable store - no more race conditions
1911 #
1912 #We're using the portalocker code by Jonathan Feinberg that was contributed
1913 #to the ASPN Python cookbook. This gives us locking across Unix and Windows.
1914 #
1915 #Revision 1.31 2002/04/03 05:54:31 richard
1916 #Fixed serialisation problem by moving the serialisation step out of the
1917 #hyperdb.Class (get, set) into the hyperdb.Database.
1918 #
1919 #Also fixed htmltemplate after the showid changes I made yesterday.
1920 #
1921 #Unit tests for all of the above written.
1922 #
1923 #Revision 1.30.2.1 2002/04/03 11:55:57 rochecompaan
1924 # . Added feature #526730 - search for messages capability
1925 #
1926 #Revision 1.30 2002/02/27 03:40:59 richard
1927 #Ran it through pychecker, made fixes
1928 #
1929 #Revision 1.29 2002/02/25 14:34:31 grubert
1930 # . use blobfiles in back_anydbm which is used in back_bsddb.
1931 # change test_db as dirlist does not work for subdirectories.
1932 # ATTENTION: blobfiles now creates subdirectories for files.
1933 #
1934 #Revision 1.28 2002/02/16 09:14:17 richard
1935 # . #514854 ] History: "User" is always ticket creator
1936 #
1937 #Revision 1.27 2002/01/22 07:21:13 richard
1938 #. fixed back_bsddb so it passed the journal tests
1939 #
1940 #... it didn't seem happy using the back_anydbm _open method, which is odd.
1941 #Yet another occurrance of whichdb not being able to recognise older bsddb
1942 #databases. Yadda yadda. Made the HYPERDBDEBUG stuff more sane in the
1943 #process.
1944 #
1945 #Revision 1.26 2002/01/22 05:18:38 rochecompaan
1946 #last_set_entry was referenced before assignment
1947 #
1948 #Revision 1.25 2002/01/22 05:06:08 rochecompaan
1949 #We need to keep the last 'set' entry in the journal to preserve
1950 #information on 'activity' for nodes.
1951 #
1952 #Revision 1.24 2002/01/21 16:33:20 rochecompaan
1953 #You can now use the roundup-admin tool to pack the database
1954 #
1955 #Revision 1.23 2002/01/18 04:32:04 richard
1956 #Rollback was breaking because a message hadn't actually been written to the file. Needs
1957 #more investigation.
1958 #
1959 #Revision 1.22 2002/01/14 02:20:15 richard
1960 # . changed all config accesses so they access either the instance or the
1961 # config attriubute on the db. This means that all config is obtained from
1962 # instance_config instead of the mish-mash of classes. This will make
1963 # switching to a ConfigParser setup easier too, I hope.
1964 #
1965 #At a minimum, this makes migration a _little_ easier (a lot easier in the
1966 #0.5.0 switch, I hope!)
1967 #
1968 #Revision 1.21 2002/01/02 02:31:38 richard
1969 #Sorry for the huge checkin message - I was only intending to implement #496356
1970 #but I found a number of places where things had been broken by transactions:
1971 # . modified ROUNDUPDBSENDMAILDEBUG to be SENDMAILDEBUG and hold a filename
1972 # for _all_ roundup-generated smtp messages to be sent to.
1973 # . the transaction cache had broken the roundupdb.Class set() reactors
1974 # . newly-created author users in the mailgw weren't being committed to the db
1975 #
1976 #Stuff that made it into CHANGES.txt (ie. the stuff I was actually working
1977 #on when I found that stuff :):
1978 # . #496356 ] Use threading in messages
1979 # . detectors were being registered multiple times
1980 # . added tests for mailgw
1981 # . much better attaching of erroneous messages in the mail gateway
1982 #
1983 #Revision 1.20 2001/12/18 15:30:34 rochecompaan
1984 #Fixed bugs:
1985 # . Fixed file creation and retrieval in same transaction in anydbm
1986 # backend
1987 # . Cgi interface now renders new issue after issue creation
1988 # . Could not set issue status to resolved through cgi interface
1989 # . Mail gateway was changing status back to 'chatting' if status was
1990 # omitted as an argument
1991 #
1992 #Revision 1.19 2001/12/17 03:52:48 richard
1993 #Implemented file store rollback. As a bonus, the hyperdb is now capable of
1994 #storing more than one file per node - if a property name is supplied,
1995 #the file is called designator.property.
1996 #I decided not to migrate the existing files stored over to the new naming
1997 #scheme - the FileClass just doesn't specify the property name.
1998 #
1999 #Revision 1.18 2001/12/16 10:53:38 richard
2000 #take a copy of the node dict so that the subsequent set
2001 #operation doesn't modify the oldvalues structure
2002 #
2003 #Revision 1.17 2001/12/14 23:42:57 richard
2004 #yuck, a gdbm instance tests false :(
2005 #I've left the debugging code in - it should be removed one day if we're ever
2006 #_really_ anal about performace :)
2007 #
2008 #Revision 1.16 2001/12/12 03:23:14 richard
2009 #Cor blimey this anydbm/whichdb stuff is yecchy. Turns out that whichdb
2010 #incorrectly identifies a dbm file as a dbhash file on my system. This has
2011 #been submitted to the python bug tracker as issue #491888:
2012 #https://sourceforge.net/tracker/index.php?func=detail&aid=491888&group_id=5470&atid=105470
2013 #
2014 #Revision 1.15 2001/12/12 02:30:51 richard
2015 #I fixed the problems with people whose anydbm was using the dbm module at the
2016 #backend. It turns out the dbm module modifies the file name to append ".db"
2017 #and my check to determine if we're opening an existing or new db just
2018 #tested os.path.exists() on the filename. Well, no longer! We now perform a
2019 #much better check _and_ cope with the anydbm implementation module changing
2020 #too!
2021 #I also fixed the backends __init__ so only ImportError is squashed.
2022 #
2023 #Revision 1.14 2001/12/10 22:20:01 richard
2024 #Enabled transaction support in the bsddb backend. It uses the anydbm code
2025 #where possible, only replacing methods where the db is opened (it uses the
2026 #btree opener specifically.)
2027 #Also cleaned up some change note generation.
2028 #Made the backends package work with pydoc too.
2029 #
2030 #Revision 1.13 2001/12/02 05:06:16 richard
2031 #. We now use weakrefs in the Classes to keep the database reference, so
2032 # the close() method on the database is no longer needed.
2033 # I bumped the minimum python requirement up to 2.1 accordingly.
2034 #. #487480 ] roundup-server
2035 #. #487476 ] INSTALL.txt
2036 #
2037 #I also cleaned up the change message / post-edit stuff in the cgi client.
2038 #There's now a clearly marked "TODO: append the change note" where I believe
2039 #the change note should be added there. The "changes" list will obviously
2040 #have to be modified to be a dict of the changes, or somesuch.
2041 #
2042 #More testing needed.
2043 #
2044 #Revision 1.12 2001/12/01 07:17:50 richard
2045 #. We now have basic transaction support! Information is only written to
2046 # the database when the commit() method is called. Only the anydbm
2047 # backend is modified in this way - neither of the bsddb backends have been.
2048 # The mail, admin and cgi interfaces all use commit (except the admin tool
2049 # doesn't have a commit command, so interactive users can't commit...)
2050 #. Fixed login/registration forwarding the user to the right page (or not,
2051 # on a failure)
2052 #
2053 #Revision 1.11 2001/11/21 02:34:18 richard
2054 #Added a target version field to the extended issue schema
2055 #
2056 #Revision 1.10 2001/10/09 23:58:10 richard
2057 #Moved the data stringification up into the hyperdb.Class class' get, set
2058 #and create methods. This means that the data is also stringified for the
2059 #journal call, and removes duplication of code from the backends. The
2060 #backend code now only sees strings.
2061 #
2062 #Revision 1.9 2001/10/09 07:25:59 richard
2063 #Added the Password property type. See "pydoc roundup.password" for
2064 #implementation details. Have updated some of the documentation too.
2065 #
2066 #Revision 1.8 2001/09/29 13:27:00 richard
2067 #CGI interfaces now spit up a top-level index of all the instances they can
2068 #serve.
2069 #
2070 #Revision 1.7 2001/08/12 06:32:36 richard
2071 #using isinstance(blah, Foo) now instead of isFooType
2072 #
2073 #Revision 1.6 2001/08/07 00:24:42 richard
2074 #stupid typo
2075 #
2076 #Revision 1.5 2001/08/07 00:15:51 richard
2077 #Added the copyright/license notice to (nearly) all files at request of
2078 #Bizar Software.
2079 #
2080 #Revision 1.4 2001/07/30 01:41:36 richard
2081 #Makes schema changes mucho easier.
2082 #
2083 #Revision 1.3 2001/07/25 01:23:07 richard
2084 #Added the Roundup spec to the new documentation directory.
2085 #
2086 #Revision 1.2 2001/07/23 08:20:44 richard
2087 #Moved over to using marshal in the bsddb and anydbm backends.
2088 #roundup-admin now has a "freshen" command that'll load/save all nodes (not
2089 # retired - mod hyperdb.Class.list() so it lists retired nodes)
2090 #
2091 #