![[tokkee]](http://tokkee.org/images/avatar.png){kind=avatar}
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: back_anydbm.py,v 1.59 2002-08-16 04:28:13 richard Exp $
19 '''
20 This module defines a backend that saves the hyperdatabase in a database
21 chosen by anydbm. It is guaranteed to always be available in python
22 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
23 serious bugs, and is not available)
24 '''
26 import whichdb, anydbm, os, marshal, re, weakref, string, copy
27 from roundup import hyperdb, date, password, roundupdb, security
28 from blobfiles import FileStorage
29 from sessions import Sessions
30 from roundup.indexer import Indexer
31 from locking import acquire_lock, release_lock
32 from roundup.hyperdb import String, Password, Date, Interval, Link, \
33 Multilink, DatabaseError, Boolean, Number
35 #
36 # Now the database
37 #
38 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
39 """A database for storing records containing flexible data types.
41 Transaction stuff TODO:
42 . check the timestamp of the class file and nuke the cache if it's
43 modified. Do some sort of conflict checking on the dirty stuff.
44 . perhaps detect write collisions (related to above)?
46 """
47 def __init__(self, config, journaltag=None):
48 """Open a hyperdatabase given a specifier to some storage.
50 The 'storagelocator' is obtained from config.DATABASE.
51 The meaning of 'storagelocator' depends on the particular
52 implementation of the hyperdatabase. It could be a file name,
53 a directory path, a socket descriptor for a connection to a
54 database over the network, etc.
56 The 'journaltag' is a token that will be attached to the journal
57 entries for any edits done on the database. If 'journaltag' is
58 None, the database is opened in read-only mode: the Class.create(),
59 Class.set(), and Class.retire() methods are disabled.
60 """
61 self.config, self.journaltag = config, journaltag
62 self.dir = config.DATABASE
63 self.classes = {}
64 self.cache = {} # cache of nodes loaded or created
65 self.dirtynodes = {} # keep track of the dirty nodes by class
66 self.newnodes = {} # keep track of the new nodes by class
67 self.destroyednodes = {}# keep track of the destroyed nodes by class
68 self.transactions = []
69 self.indexer = Indexer(self.dir)
70 self.sessions = Sessions(self.config)
71 self.security = security.Security(self)
72 # ensure files are group readable and writable
73 os.umask(0002)
75 def post_init(self):
76 """Called once the schema initialisation has finished."""
77 # reindex the db if necessary
78 if self.indexer.should_reindex():
79 self.reindex()
81 def reindex(self):
82 for klass in self.classes.values():
83 for nodeid in klass.list():
84 klass.index(nodeid)
85 self.indexer.save_index()
87 def __repr__(self):
88 return '<back_anydbm instance at %x>'%id(self)
90 #
91 # Classes
92 #
93 def __getattr__(self, classname):
94 """A convenient way of calling self.getclass(classname)."""
95 if self.classes.has_key(classname):
96 if __debug__:
97 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
98 return self.classes[classname]
99 raise AttributeError, classname
101 def addclass(self, cl):
102 if __debug__:
103 print >>hyperdb.DEBUG, 'addclass', (self, cl)
104 cn = cl.classname
105 if self.classes.has_key(cn):
106 raise ValueError, cn
107 self.classes[cn] = cl
109 def getclasses(self):
110 """Return a list of the names of all existing classes."""
111 if __debug__:
112 print >>hyperdb.DEBUG, 'getclasses', (self,)
113 l = self.classes.keys()
114 l.sort()
115 return l
117 def getclass(self, classname):
118 """Get the Class object representing a particular class.
120 If 'classname' is not a valid class name, a KeyError is raised.
121 """
122 if __debug__:
123 print >>hyperdb.DEBUG, 'getclass', (self, classname)
124 return self.classes[classname]
126 #
127 # Class DBs
128 #
129 def clear(self):
130 '''Delete all database contents
131 '''
132 if __debug__:
133 print >>hyperdb.DEBUG, 'clear', (self,)
134 for cn in self.classes.keys():
135 for dummy in 'nodes', 'journals':
136 path = os.path.join(self.dir, 'journals.%s'%cn)
137 if os.path.exists(path):
138 os.remove(path)
139 elif os.path.exists(path+'.db'): # dbm appends .db
140 os.remove(path+'.db')
142 def getclassdb(self, classname, mode='r'):
143 ''' grab a connection to the class db that will be used for
144 multiple actions
145 '''
146 if __debug__:
147 print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode)
148 return self.opendb('nodes.%s'%classname, mode)
150 def determine_db_type(self, path):
151 ''' determine which DB wrote the class file
152 '''
153 db_type = ''
154 if os.path.exists(path):
155 db_type = whichdb.whichdb(path)
156 if not db_type:
157 raise hyperdb.DatabaseError, "Couldn't identify database type"
158 elif os.path.exists(path+'.db'):
159 # if the path ends in '.db', it's a dbm database, whether
160 # anydbm says it's dbhash or not!
161 db_type = 'dbm'
162 return db_type
164 def opendb(self, name, mode):
165 '''Low-level database opener that gets around anydbm/dbm
166 eccentricities.
167 '''
168 if __debug__:
169 print >>hyperdb.DEBUG, 'opendb', (self, name, mode)
171 # figure the class db type
172 path = os.path.join(os.getcwd(), self.dir, name)
173 db_type = self.determine_db_type(path)
175 # new database? let anydbm pick the best dbm
176 if not db_type:
177 if __debug__:
178 print >>hyperdb.DEBUG, "opendb anydbm.open(%r, 'n')"%path
179 return anydbm.open(path, 'n')
181 # open the database with the correct module
182 try:
183 dbm = __import__(db_type)
184 except ImportError:
185 raise hyperdb.DatabaseError, \
186 "Couldn't open database - the required module '%s'"\
187 " is not available"%db_type
188 if __debug__:
189 print >>hyperdb.DEBUG, "opendb %r.open(%r, %r)"%(db_type, path,
190 mode)
191 return dbm.open(path, mode)
193 def lockdb(self, name):
194 ''' Lock a database file
195 '''
196 path = os.path.join(os.getcwd(), self.dir, '%s.lock'%name)
197 return acquire_lock(path)
199 #
200 # Node IDs
201 #
202 def newid(self, classname):
203 ''' Generate a new id for the given class
204 '''
205 # open the ids DB - create if if doesn't exist
206 lock = self.lockdb('_ids')
207 db = self.opendb('_ids', 'c')
208 if db.has_key(classname):
209 newid = db[classname] = str(int(db[classname]) + 1)
210 else:
211 # the count() bit is transitional - older dbs won't start at 1
212 newid = str(self.getclass(classname).count()+1)
213 db[classname] = newid
214 db.close()
215 release_lock(lock)
216 return newid
218 #
219 # Nodes
220 #
221 def addnode(self, classname, nodeid, node):
222 ''' add the specified node to its class's db
223 '''
224 if __debug__:
225 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
226 self.newnodes.setdefault(classname, {})[nodeid] = 1
227 self.cache.setdefault(classname, {})[nodeid] = node
228 self.savenode(classname, nodeid, node)
230 def setnode(self, classname, nodeid, node):
231 ''' change the specified node
232 '''
233 if __debug__:
234 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
235 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
237 # can't set without having already loaded the node
238 self.cache[classname][nodeid] = node
239 self.savenode(classname, nodeid, node)
241 def savenode(self, classname, nodeid, node):
242 ''' perform the saving of data specified by the set/addnode
243 '''
244 if __debug__:
245 print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node)
246 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
248 def getnode(self, classname, nodeid, db=None, cache=1):
249 ''' get a node from the database
250 '''
251 if __debug__:
252 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid, db)
253 if cache:
254 # try the cache
255 cache_dict = self.cache.setdefault(classname, {})
256 if cache_dict.has_key(nodeid):
257 if __debug__:
258 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
259 nodeid)
260 return cache_dict[nodeid]
262 if __debug__:
263 print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid)
265 # get from the database and save in the cache
266 if db is None:
267 db = self.getclassdb(classname)
268 if not db.has_key(nodeid):
269 raise IndexError, "no such %s %s"%(classname, nodeid)
271 # check the uncommitted, destroyed nodes
272 if (self.destroyednodes.has_key(classname) and
273 self.destroyednodes[classname].has_key(nodeid)):
274 raise IndexError, "no such %s %s"%(classname, nodeid)
276 # decode
277 res = marshal.loads(db[nodeid])
279 # reverse the serialisation
280 res = self.unserialise(classname, res)
282 # store off in the cache dict
283 if cache:
284 cache_dict[nodeid] = res
286 return res
288 def destroynode(self, classname, nodeid):
289 '''Remove a node from the database. Called exclusively by the
290 destroy() method on Class.
291 '''
292 if __debug__:
293 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
295 # remove from cache and newnodes if it's there
296 if (self.cache.has_key(classname) and
297 self.cache[classname].has_key(nodeid)):
298 del self.cache[classname][nodeid]
299 if (self.newnodes.has_key(classname) and
300 self.newnodes[classname].has_key(nodeid)):
301 del self.newnodes[classname][nodeid]
303 # see if there's any obvious commit actions that we should get rid of
304 for entry in self.transactions[:]:
305 if entry[1][:2] == (classname, nodeid):
306 self.transactions.remove(entry)
308 # add to the destroyednodes map
309 self.destroyednodes.setdefault(classname, {})[nodeid] = 1
311 # add the destroy commit action
312 self.transactions.append((self.doDestroyNode, (classname, nodeid)))
314 def serialise(self, classname, node):
315 '''Copy the node contents, converting non-marshallable data into
316 marshallable data.
317 '''
318 if __debug__:
319 print >>hyperdb.DEBUG, 'serialise', classname, node
320 properties = self.getclass(classname).getprops()
321 d = {}
322 for k, v in node.items():
323 # if the property doesn't exist, or is the "retired" flag then
324 # it won't be in the properties dict
325 if not properties.has_key(k):
326 d[k] = v
327 continue
329 # get the property spec
330 prop = properties[k]
332 if isinstance(prop, Password):
333 d[k] = str(v)
334 elif isinstance(prop, Date) and v is not None:
335 d[k] = v.get_tuple()
336 elif isinstance(prop, Interval) and v is not None:
337 d[k] = v.get_tuple()
338 else:
339 d[k] = v
340 return d
342 def unserialise(self, classname, node):
343 '''Decode the marshalled node data
344 '''
345 if __debug__:
346 print >>hyperdb.DEBUG, 'unserialise', classname, node
347 properties = self.getclass(classname).getprops()
348 d = {}
349 for k, v in node.items():
350 # if the property doesn't exist, or is the "retired" flag then
351 # it won't be in the properties dict
352 if not properties.has_key(k):
353 d[k] = v
354 continue
356 # get the property spec
357 prop = properties[k]
359 if isinstance(prop, Date) and v is not None:
360 d[k] = date.Date(v)
361 elif isinstance(prop, Interval) and v is not None:
362 d[k] = date.Interval(v)
363 elif isinstance(prop, Password):
364 p = password.Password()
365 p.unpack(v)
366 d[k] = p
367 else:
368 d[k] = v
369 return d
371 def hasnode(self, classname, nodeid, db=None):
372 ''' determine if the database has a given node
373 '''
374 if __debug__:
375 print >>hyperdb.DEBUG, 'hasnode', (self, classname, nodeid, db)
377 # try the cache
378 cache = self.cache.setdefault(classname, {})
379 if cache.has_key(nodeid):
380 if __debug__:
381 print >>hyperdb.TRACE, 'has %s %s cached'%(classname, nodeid)
382 return 1
383 if __debug__:
384 print >>hyperdb.TRACE, 'has %s %s'%(classname, nodeid)
386 # not in the cache - check the database
387 if db is None:
388 db = self.getclassdb(classname)
389 res = db.has_key(nodeid)
390 return res
392 def countnodes(self, classname, db=None):
393 if __debug__:
394 print >>hyperdb.DEBUG, 'countnodes', (self, classname, db)
396 count = 0
398 # include the uncommitted nodes
399 if self.newnodes.has_key(classname):
400 count += len(self.newnodes[classname])
401 if self.destroyednodes.has_key(classname):
402 count -= len(self.destroyednodes[classname])
404 # and count those in the DB
405 if db is None:
406 db = self.getclassdb(classname)
407 count = count + len(db.keys())
408 return count
410 def getnodeids(self, classname, db=None):
411 if __debug__:
412 print >>hyperdb.DEBUG, 'getnodeids', (self, classname, db)
414 res = []
416 # start off with the new nodes
417 if self.newnodes.has_key(classname):
418 res += self.newnodes[classname].keys()
420 if db is None:
421 db = self.getclassdb(classname)
422 res = res + db.keys()
424 # remove the uncommitted, destroyed nodes
425 if self.destroyednodes.has_key(classname):
426 for nodeid in self.destroyednodes[classname].keys():
427 if db.has_key(nodeid):
428 res.remove(nodeid)
430 return res
433 #
434 # Files - special node properties
435 # inherited from FileStorage
437 #
438 # Journal
439 #
440 def addjournal(self, classname, nodeid, action, params):
441 ''' Journal the Action
442 'action' may be:
444 'create' or 'set' -- 'params' is a dictionary of property values
445 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
446 'retire' -- 'params' is None
447 '''
448 if __debug__:
449 print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid,
450 action, params)
451 self.transactions.append((self.doSaveJournal, (classname, nodeid,
452 action, params)))
454 def getjournal(self, classname, nodeid):
455 ''' get the journal for id
457 Raise IndexError if the node doesn't exist (as per history()'s
458 API)
459 '''
460 if __debug__:
461 print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid)
462 # attempt to open the journal - in some rare cases, the journal may
463 # not exist
464 try:
465 db = self.opendb('journals.%s'%classname, 'r')
466 except anydbm.error, error:
467 if str(error) == "need 'c' or 'n' flag to open new db":
468 raise IndexError, 'no such %s %s'%(classname, nodeid)
469 elif error.args[0] != 2:
470 raise
471 raise IndexError, 'no such %s %s'%(classname, nodeid)
472 try:
473 journal = marshal.loads(db[nodeid])
474 except KeyError:
475 db.close()
476 raise IndexError, 'no such %s %s'%(classname, nodeid)
477 db.close()
478 res = []
479 for nodeid, date_stamp, user, action, params in journal:
480 res.append((nodeid, date.Date(date_stamp), user, action, params))
481 return res
483 def pack(self, pack_before):
484 ''' delete all journal entries before 'pack_before' '''
485 if __debug__:
486 print >>hyperdb.DEBUG, 'packjournal', (self, pack_before)
488 pack_before = pack_before.get_tuple()
490 classes = self.getclasses()
492 # figure the class db type
494 for classname in classes:
495 db_name = 'journals.%s'%classname
496 path = os.path.join(os.getcwd(), self.dir, classname)
497 db_type = self.determine_db_type(path)
498 db = self.opendb(db_name, 'w')
500 for key in db.keys():
501 journal = marshal.loads(db[key])
502 l = []
503 last_set_entry = None
504 for entry in journal:
505 (nodeid, date_stamp, self.journaltag, action,
506 params) = entry
507 if date_stamp > pack_before or action == 'create':
508 l.append(entry)
509 elif action == 'set':
510 # grab the last set entry to keep information on
511 # activity
512 last_set_entry = entry
513 if last_set_entry:
514 date_stamp = last_set_entry[1]
515 # if the last set entry was made after the pack date
516 # then it is already in the list
517 if date_stamp < pack_before:
518 l.append(last_set_entry)
519 db[key] = marshal.dumps(l)
520 if db_type == 'gdbm':
521 db.reorganize()
522 db.close()
525 #
526 # Basic transaction support
527 #
528 def commit(self):
529 ''' Commit the current transactions.
530 '''
531 if __debug__:
532 print >>hyperdb.DEBUG, 'commit', (self,)
533 # TODO: lock the DB
535 # keep a handle to all the database files opened
536 self.databases = {}
538 # now, do all the transactions
539 reindex = {}
540 for method, args in self.transactions:
541 reindex[method(*args)] = 1
543 # now close all the database files
544 for db in self.databases.values():
545 db.close()
546 del self.databases
547 # TODO: unlock the DB
549 # reindex the nodes that request it
550 for classname, nodeid in filter(None, reindex.keys()):
551 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
552 self.getclass(classname).index(nodeid)
554 # save the indexer state
555 self.indexer.save_index()
557 # all transactions committed, back to normal
558 self.cache = {}
559 self.dirtynodes = {}
560 self.newnodes = {}
561 self.destroyednodes = {}
562 self.transactions = []
564 def getCachedClassDB(self, classname):
565 ''' get the class db, looking in our cache of databases for commit
566 '''
567 # get the database handle
568 db_name = 'nodes.%s'%classname
569 if not self.databases.has_key(db_name):
570 self.databases[db_name] = self.getclassdb(classname, 'c')
571 return self.databases[db_name]
573 def doSaveNode(self, classname, nodeid, node):
574 if __debug__:
575 print >>hyperdb.DEBUG, 'doSaveNode', (self, classname, nodeid,
576 node)
578 db = self.getCachedClassDB(classname)
580 # now save the marshalled data
581 db[nodeid] = marshal.dumps(self.serialise(classname, node))
583 # return the classname, nodeid so we reindex this content
584 return (classname, nodeid)
586 def getCachedJournalDB(self, classname):
587 ''' get the journal db, looking in our cache of databases for commit
588 '''
589 # get the database handle
590 db_name = 'journals.%s'%classname
591 if not self.databases.has_key(db_name):
592 self.databases[db_name] = self.opendb(db_name, 'c')
593 return self.databases[db_name]
595 def doSaveJournal(self, classname, nodeid, action, params):
596 # serialise first
597 if action in ('set', 'create'):
598 params = self.serialise(classname, params)
600 # create the journal entry
601 entry = (nodeid, date.Date().get_tuple(), self.journaltag, action,
602 params)
604 if __debug__:
605 print >>hyperdb.DEBUG, 'doSaveJournal', entry
607 db = self.getCachedJournalDB(classname)
609 # now insert the journal entry
610 if db.has_key(nodeid):
611 # append to existing
612 s = db[nodeid]
613 l = marshal.loads(s)
614 l.append(entry)
615 else:
616 l = [entry]
618 db[nodeid] = marshal.dumps(l)
620 def doDestroyNode(self, classname, nodeid):
621 if __debug__:
622 print >>hyperdb.DEBUG, 'doDestroyNode', (self, classname, nodeid)
624 # delete from the class database
625 db = self.getCachedClassDB(classname)
626 if db.has_key(nodeid):
627 del db[nodeid]
629 # delete from the database
630 db = self.getCachedJournalDB(classname)
631 if db.has_key(nodeid):
632 del db[nodeid]
634 # return the classname, nodeid so we reindex this content
635 return (classname, nodeid)
637 def rollback(self):
638 ''' Reverse all actions from the current transaction.
639 '''
640 if __debug__:
641 print >>hyperdb.DEBUG, 'rollback', (self, )
642 for method, args in self.transactions:
643 # delete temporary files
644 if method == self.doStoreFile:
645 self.rollbackStoreFile(*args)
646 self.cache = {}
647 self.dirtynodes = {}
648 self.newnodes = {}
649 self.destroyednodes = {}
650 self.transactions = []
652 _marker = []
653 class Class(hyperdb.Class):
654 """The handle to a particular class of nodes in a hyperdatabase."""
656 def __init__(self, db, classname, **properties):
657 """Create a new class with a given name and property specification.
659 'classname' must not collide with the name of an existing class,
660 or a ValueError is raised. The keyword arguments in 'properties'
661 must map names to property objects, or a TypeError is raised.
662 """
663 if (properties.has_key('creation') or properties.has_key('activity')
664 or properties.has_key('creator')):
665 raise ValueError, '"creation", "activity" and "creator" are '\
666 'reserved'
668 self.classname = classname
669 self.properties = properties
670 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
671 self.key = ''
673 # should we journal changes (default yes)
674 self.do_journal = 1
676 # do the db-related init stuff
677 db.addclass(self)
679 self.auditors = {'create': [], 'set': [], 'retire': []}
680 self.reactors = {'create': [], 'set': [], 'retire': []}
682 def enableJournalling(self):
683 '''Turn journalling on for this class
684 '''
685 self.do_journal = 1
687 def disableJournalling(self):
688 '''Turn journalling off for this class
689 '''
690 self.do_journal = 0
692 # Editing nodes:
694 def create(self, **propvalues):
695 """Create a new node of this class and return its id.
697 The keyword arguments in 'propvalues' map property names to values.
699 The values of arguments must be acceptable for the types of their
700 corresponding properties or a TypeError is raised.
702 If this class has a key property, it must be present and its value
703 must not collide with other key strings or a ValueError is raised.
705 Any other properties on this class that are missing from the
706 'propvalues' dictionary are set to None.
708 If an id in a link or multilink property does not refer to a valid
709 node, an IndexError is raised.
711 These operations trigger detectors and can be vetoed. Attempts
712 to modify the "creation" or "activity" properties cause a KeyError.
713 """
714 if propvalues.has_key('id'):
715 raise KeyError, '"id" is reserved'
717 if self.db.journaltag is None:
718 raise DatabaseError, 'Database open read-only'
720 if propvalues.has_key('creation') or propvalues.has_key('activity'):
721 raise KeyError, '"creation" and "activity" are reserved'
723 self.fireAuditors('create', None, propvalues)
725 # new node's id
726 newid = self.db.newid(self.classname)
728 # validate propvalues
729 num_re = re.compile('^\d+$')
730 for key, value in propvalues.items():
731 if key == self.key:
732 try:
733 self.lookup(value)
734 except KeyError:
735 pass
736 else:
737 raise ValueError, 'node with key "%s" exists'%value
739 # try to handle this property
740 try:
741 prop = self.properties[key]
742 except KeyError:
743 raise KeyError, '"%s" has no property "%s"'%(self.classname,
744 key)
746 if isinstance(prop, Link):
747 if type(value) != type(''):
748 raise ValueError, 'link value must be String'
749 link_class = self.properties[key].classname
750 # if it isn't a number, it's a key
751 if not num_re.match(value):
752 try:
753 value = self.db.classes[link_class].lookup(value)
754 except (TypeError, KeyError):
755 raise IndexError, 'new property "%s": %s not a %s'%(
756 key, value, link_class)
757 elif not self.db.getclass(link_class).hasnode(value):
758 raise IndexError, '%s has no node %s'%(link_class, value)
760 # save off the value
761 propvalues[key] = value
763 # register the link with the newly linked node
764 if self.do_journal and self.properties[key].do_journal:
765 self.db.addjournal(link_class, value, 'link',
766 (self.classname, newid, key))
768 elif isinstance(prop, Multilink):
769 if type(value) != type([]):
770 raise TypeError, 'new property "%s" not a list of ids'%key
772 # clean up and validate the list of links
773 link_class = self.properties[key].classname
774 l = []
775 for entry in value:
776 if type(entry) != type(''):
777 raise ValueError, '"%s" link value (%s) must be '\
778 'String'%(key, value)
779 # if it isn't a number, it's a key
780 if not num_re.match(entry):
781 try:
782 entry = self.db.classes[link_class].lookup(entry)
783 except (TypeError, KeyError):
784 raise IndexError, 'new property "%s": %s not a %s'%(
785 key, entry, self.properties[key].classname)
786 l.append(entry)
787 value = l
788 propvalues[key] = value
790 # handle additions
791 for nodeid in value:
792 if not self.db.getclass(link_class).hasnode(nodeid):
793 raise IndexError, '%s has no node %s'%(link_class,
794 nodeid)
795 # register the link with the newly linked node
796 if self.do_journal and self.properties[key].do_journal:
797 self.db.addjournal(link_class, nodeid, 'link',
798 (self.classname, newid, key))
800 elif isinstance(prop, String):
801 if type(value) != type(''):
802 raise TypeError, 'new property "%s" not a string'%key
804 elif isinstance(prop, Password):
805 if not isinstance(value, password.Password):
806 raise TypeError, 'new property "%s" not a Password'%key
808 elif isinstance(prop, Date):
809 if value is not None and not isinstance(value, date.Date):
810 raise TypeError, 'new property "%s" not a Date'%key
812 elif isinstance(prop, Interval):
813 if value is not None and not isinstance(value, date.Interval):
814 raise TypeError, 'new property "%s" not an Interval'%key
816 elif value is not None and isinstance(prop, Number):
817 try:
818 float(value)
819 except ValueError:
820 raise TypeError, 'new property "%s" not numeric'%key
822 elif value is not None and isinstance(prop, Boolean):
823 try:
824 int(value)
825 except ValueError:
826 raise TypeError, 'new property "%s" not boolean'%key
828 # make sure there's data where there needs to be
829 for key, prop in self.properties.items():
830 if propvalues.has_key(key):
831 continue
832 if key == self.key:
833 raise ValueError, 'key property "%s" is required'%key
834 if isinstance(prop, Multilink):
835 propvalues[key] = []
836 else:
837 propvalues[key] = None
839 # done
840 self.db.addnode(self.classname, newid, propvalues)
841 if self.do_journal:
842 self.db.addjournal(self.classname, newid, 'create', propvalues)
844 self.fireReactors('create', newid, None)
846 return newid
848 def get(self, nodeid, propname, default=_marker, cache=1):
849 """Get the value of a property on an existing node of this class.
851 'nodeid' must be the id of an existing node of this class or an
852 IndexError is raised. 'propname' must be the name of a property
853 of this class or a KeyError is raised.
855 'cache' indicates whether the transaction cache should be queried
856 for the node. If the node has been modified and you need to
857 determine what its values prior to modification are, you need to
858 set cache=0.
860 Attempts to get the "creation" or "activity" properties should
861 do the right thing.
862 """
863 if propname == 'id':
864 return nodeid
866 if propname == 'creation':
867 if not self.do_journal:
868 raise ValueError, 'Journalling is disabled for this class'
869 journal = self.db.getjournal(self.classname, nodeid)
870 if journal:
871 return self.db.getjournal(self.classname, nodeid)[0][1]
872 else:
873 # on the strange chance that there's no journal
874 return date.Date()
875 if propname == 'activity':
876 if not self.do_journal:
877 raise ValueError, 'Journalling is disabled for this class'
878 journal = self.db.getjournal(self.classname, nodeid)
879 if journal:
880 return self.db.getjournal(self.classname, nodeid)[-1][1]
881 else:
882 # on the strange chance that there's no journal
883 return date.Date()
884 if propname == 'creator':
885 if not self.do_journal:
886 raise ValueError, 'Journalling is disabled for this class'
887 journal = self.db.getjournal(self.classname, nodeid)
888 if journal:
889 name = self.db.getjournal(self.classname, nodeid)[0][2]
890 else:
891 return None
892 return self.db.user.lookup(name)
894 # get the property (raises KeyErorr if invalid)
895 prop = self.properties[propname]
897 # get the node's dict
898 d = self.db.getnode(self.classname, nodeid, cache=cache)
900 if not d.has_key(propname):
901 if default is _marker:
902 if isinstance(prop, Multilink):
903 return []
904 else:
905 return None
906 else:
907 return default
909 return d[propname]
911 # XXX not in spec
912 def getnode(self, nodeid, cache=1):
913 ''' Return a convenience wrapper for the node.
915 'nodeid' must be the id of an existing node of this class or an
916 IndexError is raised.
918 'cache' indicates whether the transaction cache should be queried
919 for the node. If the node has been modified and you need to
920 determine what its values prior to modification are, you need to
921 set cache=0.
922 '''
923 return Node(self, nodeid, cache=cache)
925 def set(self, nodeid, **propvalues):
926 """Modify a property on an existing node of this class.
928 'nodeid' must be the id of an existing node of this class or an
929 IndexError is raised.
931 Each key in 'propvalues' must be the name of a property of this
932 class or a KeyError is raised.
934 All values in 'propvalues' must be acceptable types for their
935 corresponding properties or a TypeError is raised.
937 If the value of the key property is set, it must not collide with
938 other key strings or a ValueError is raised.
940 If the value of a Link or Multilink property contains an invalid
941 node id, a ValueError is raised.
943 These operations trigger detectors and can be vetoed. Attempts
944 to modify the "creation" or "activity" properties cause a KeyError.
945 """
946 if not propvalues:
947 return propvalues
949 if propvalues.has_key('creation') or propvalues.has_key('activity'):
950 raise KeyError, '"creation" and "activity" are reserved'
952 if propvalues.has_key('id'):
953 raise KeyError, '"id" is reserved'
955 if self.db.journaltag is None:
956 raise DatabaseError, 'Database open read-only'
958 self.fireAuditors('set', nodeid, propvalues)
959 # Take a copy of the node dict so that the subsequent set
960 # operation doesn't modify the oldvalues structure.
961 try:
962 # try not using the cache initially
963 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid,
964 cache=0))
965 except IndexError:
966 # this will be needed if somone does a create() and set()
967 # with no intervening commit()
968 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
970 node = self.db.getnode(self.classname, nodeid)
971 if node.has_key(self.db.RETIRED_FLAG):
972 raise IndexError
973 num_re = re.compile('^\d+$')
975 # if the journal value is to be different, store it in here
976 journalvalues = {}
978 for propname, value in propvalues.items():
979 # check to make sure we're not duplicating an existing key
980 if propname == self.key and node[propname] != value:
981 try:
982 self.lookup(value)
983 except KeyError:
984 pass
985 else:
986 raise ValueError, 'node with key "%s" exists'%value
988 # this will raise the KeyError if the property isn't valid
989 # ... we don't use getprops() here because we only care about
990 # the writeable properties.
991 prop = self.properties[propname]
993 # if the value's the same as the existing value, no sense in
994 # doing anything
995 if node.has_key(propname) and value == node[propname]:
996 del propvalues[propname]
997 continue
999 # do stuff based on the prop type
1000 if isinstance(prop, Link):
1001 link_class = prop.classname
1002 # if it isn't a number, it's a key
1003 if value is not None and not isinstance(value, type('')):
1004 raise ValueError, 'property "%s" link value be a string'%(
1005 propname)
1006 if isinstance(value, type('')) and not num_re.match(value):
1007 try:
1008 value = self.db.classes[link_class].lookup(value)
1009 except (TypeError, KeyError):
1010 raise IndexError, 'new property "%s": %s not a %s'%(
1011 propname, value, prop.classname)
1013 if (value is not None and
1014 not self.db.getclass(link_class).hasnode(value)):
1015 raise IndexError, '%s has no node %s'%(link_class, value)
1017 if self.do_journal and prop.do_journal:
1018 # register the unlink with the old linked node
1019 if node[propname] is not None:
1020 self.db.addjournal(link_class, node[propname], 'unlink',
1021 (self.classname, nodeid, propname))
1023 # register the link with the newly linked node
1024 if value is not None:
1025 self.db.addjournal(link_class, value, 'link',
1026 (self.classname, nodeid, propname))
1028 elif isinstance(prop, Multilink):
1029 if type(value) != type([]):
1030 raise TypeError, 'new property "%s" not a list of'\
1031 ' ids'%propname
1032 link_class = self.properties[propname].classname
1033 l = []
1034 for entry in value:
1035 # if it isn't a number, it's a key
1036 if type(entry) != type(''):
1037 raise ValueError, 'new property "%s" link value ' \
1038 'must be a string'%propname
1039 if not num_re.match(entry):
1040 try:
1041 entry = self.db.classes[link_class].lookup(entry)
1042 except (TypeError, KeyError):
1043 raise IndexError, 'new property "%s": %s not a %s'%(
1044 propname, entry,
1045 self.properties[propname].classname)
1046 l.append(entry)
1047 value = l
1048 propvalues[propname] = value
1050 # figure the journal entry for this property
1051 add = []
1052 remove = []
1054 # handle removals
1055 if node.has_key(propname):
1056 l = node[propname]
1057 else:
1058 l = []
1059 for id in l[:]:
1060 if id in value:
1061 continue
1062 # register the unlink with the old linked node
1063 if self.do_journal and self.properties[propname].do_journal:
1064 self.db.addjournal(link_class, id, 'unlink',
1065 (self.classname, nodeid, propname))
1066 l.remove(id)
1067 remove.append(id)
1069 # handle additions
1070 for id in value:
1071 if not self.db.getclass(link_class).hasnode(id):
1072 raise IndexError, '%s has no node %s'%(link_class, id)
1073 if id in l:
1074 continue
1075 # register the link with the newly linked node
1076 if self.do_journal and self.properties[propname].do_journal:
1077 self.db.addjournal(link_class, id, 'link',
1078 (self.classname, nodeid, propname))
1079 l.append(id)
1080 add.append(id)
1082 # figure the journal entry
1083 l = []
1084 if add:
1085 l.append(('add', add))
1086 if remove:
1087 l.append(('remove', remove))
1088 if l:
1089 journalvalues[propname] = tuple(l)
1091 elif isinstance(prop, String):
1092 if value is not None and type(value) != type(''):
1093 raise TypeError, 'new property "%s" not a string'%propname
1095 elif isinstance(prop, Password):
1096 if not isinstance(value, password.Password):
1097 raise TypeError, 'new property "%s" not a Password'%propname
1098 propvalues[propname] = value
1100 elif value is not None and isinstance(prop, Date):
1101 if not isinstance(value, date.Date):
1102 raise TypeError, 'new property "%s" not a Date'% propname
1103 propvalues[propname] = value
1105 elif value is not None and isinstance(prop, Interval):
1106 if not isinstance(value, date.Interval):
1107 raise TypeError, 'new property "%s" not an '\
1108 'Interval'%propname
1109 propvalues[propname] = value
1111 elif value is not None and isinstance(prop, Number):
1112 try:
1113 float(value)
1114 except ValueError:
1115 raise TypeError, 'new property "%s" not numeric'%propname
1117 elif value is not None and isinstance(prop, Boolean):
1118 try:
1119 int(value)
1120 except ValueError:
1121 raise TypeError, 'new property "%s" not boolean'%propname
1123 node[propname] = value
1125 # nothing to do?
1126 if not propvalues:
1127 return propvalues
1129 # do the set, and journal it
1130 self.db.setnode(self.classname, nodeid, node)
1132 if self.do_journal:
1133 propvalues.update(journalvalues)
1134 self.db.addjournal(self.classname, nodeid, 'set', propvalues)
1136 self.fireReactors('set', nodeid, oldvalues)
1138 return propvalues
1140 def retire(self, nodeid):
1141 """Retire a node.
1143 The properties on the node remain available from the get() method,
1144 and the node's id is never reused.
1146 Retired nodes are not returned by the find(), list(), or lookup()
1147 methods, and other nodes may reuse the values of their key properties.
1149 These operations trigger detectors and can be vetoed. Attempts
1150 to modify the "creation" or "activity" properties cause a KeyError.
1151 """
1152 if self.db.journaltag is None:
1153 raise DatabaseError, 'Database open read-only'
1155 self.fireAuditors('retire', nodeid, None)
1157 node = self.db.getnode(self.classname, nodeid)
1158 node[self.db.RETIRED_FLAG] = 1
1159 self.db.setnode(self.classname, nodeid, node)
1160 if self.do_journal:
1161 self.db.addjournal(self.classname, nodeid, 'retired', None)
1163 self.fireReactors('retire', nodeid, None)
1165 def is_retired(self, nodeid):
1166 '''Return true if the node is retired.
1167 '''
1168 node = self.db.getnode(cn, nodeid, cldb)
1169 if node.has_key(self.db.RETIRED_FLAG):
1170 return 1
1171 return 0
1173 def destroy(self, nodeid):
1174 """Destroy a node.
1176 WARNING: this method should never be used except in extremely rare
1177 situations where there could never be links to the node being
1178 deleted
1179 WARNING: use retire() instead
1180 WARNING: the properties of this node will not be available ever again
1181 WARNING: really, use retire() instead
1183 Well, I think that's enough warnings. This method exists mostly to
1184 support the session storage of the cgi interface.
1185 """
1186 if self.db.journaltag is None:
1187 raise DatabaseError, 'Database open read-only'
1188 self.db.destroynode(self.classname, nodeid)
1190 def history(self, nodeid):
1191 """Retrieve the journal of edits on a particular node.
1193 'nodeid' must be the id of an existing node of this class or an
1194 IndexError is raised.
1196 The returned list contains tuples of the form
1198 (date, tag, action, params)
1200 'date' is a Timestamp object specifying the time of the change and
1201 'tag' is the journaltag specified when the database was opened.
1202 """
1203 if not self.do_journal:
1204 raise ValueError, 'Journalling is disabled for this class'
1205 return self.db.getjournal(self.classname, nodeid)
1207 # Locating nodes:
1208 def hasnode(self, nodeid):
1209 '''Determine if the given nodeid actually exists
1210 '''
1211 return self.db.hasnode(self.classname, nodeid)
1213 def setkey(self, propname):
1214 """Select a String property of this class to be the key property.
1216 'propname' must be the name of a String property of this class or
1217 None, or a TypeError is raised. The values of the key property on
1218 all existing nodes must be unique or a ValueError is raised. If the
1219 property doesn't exist, KeyError is raised.
1220 """
1221 prop = self.getprops()[propname]
1222 if not isinstance(prop, String):
1223 raise TypeError, 'key properties must be String'
1224 self.key = propname
1226 def getkey(self):
1227 """Return the name of the key property for this class or None."""
1228 return self.key
1230 def labelprop(self, default_to_id=0):
1231 ''' Return the property name for a label for the given node.
1233 This method attempts to generate a consistent label for the node.
1234 It tries the following in order:
1235 1. key property
1236 2. "name" property
1237 3. "title" property
1238 4. first property from the sorted property name list
1239 '''
1240 k = self.getkey()
1241 if k:
1242 return k
1243 props = self.getprops()
1244 if props.has_key('name'):
1245 return 'name'
1246 elif props.has_key('title'):
1247 return 'title'
1248 if default_to_id:
1249 return 'id'
1250 props = props.keys()
1251 props.sort()
1252 return props[0]
1254 # TODO: set up a separate index db file for this? profile?
1255 def lookup(self, keyvalue):
1256 """Locate a particular node by its key property and return its id.
1258 If this class has no key property, a TypeError is raised. If the
1259 'keyvalue' matches one of the values for the key property among
1260 the nodes in this class, the matching node's id is returned;
1261 otherwise a KeyError is raised.
1262 """
1263 cldb = self.db.getclassdb(self.classname)
1264 try:
1265 for nodeid in self.db.getnodeids(self.classname, cldb):
1266 node = self.db.getnode(self.classname, nodeid, cldb)
1267 if node.has_key(self.db.RETIRED_FLAG):
1268 continue
1269 if node[self.key] == keyvalue:
1270 cldb.close()
1271 return nodeid
1272 finally:
1273 cldb.close()
1274 raise KeyError, keyvalue
1276 # XXX: change from spec - allows multiple props to match
1277 def find(self, **propspec):
1278 """Get the ids of nodes in this class which link to the given nodes.
1280 'propspec' consists of keyword args propname={nodeid:1,}
1281 'propname' must be the name of a property in this class, or a
1282 KeyError is raised. That property must be a Link or Multilink
1283 property, or a TypeError is raised.
1285 Any node in this class whose 'propname' property links to any of the
1286 nodeids will be returned. Used by the full text indexing, which knows
1287 that "foo" occurs in msg1, msg3 and file7, so we have hits on these issues:
1288 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1289 """
1290 propspec = propspec.items()
1291 for propname, nodeids in propspec:
1292 # check the prop is OK
1293 prop = self.properties[propname]
1294 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1295 raise TypeError, "'%s' not a Link/Multilink property"%propname
1297 # ok, now do the find
1298 cldb = self.db.getclassdb(self.classname)
1299 l = []
1300 try:
1301 for id in self.db.getnodeids(self.classname, db=cldb):
1302 node = self.db.getnode(self.classname, id, db=cldb)
1303 if node.has_key(self.db.RETIRED_FLAG):
1304 continue
1305 for propname, nodeids in propspec:
1306 # can't test if the node doesn't have this property
1307 if not node.has_key(propname):
1308 continue
1309 if type(nodeids) is type(''):
1310 nodeids = {nodeids:1}
1311 prop = self.properties[propname]
1312 value = node[propname]
1313 if isinstance(prop, Link) and nodeids.has_key(value):
1314 l.append(id)
1315 break
1316 elif isinstance(prop, Multilink):
1317 hit = 0
1318 for v in value:
1319 if nodeids.has_key(v):
1320 l.append(id)
1321 hit = 1
1322 break
1323 if hit:
1324 break
1325 finally:
1326 cldb.close()
1327 return l
1329 def stringFind(self, **requirements):
1330 """Locate a particular node by matching a set of its String
1331 properties in a caseless search.
1333 If the property is not a String property, a TypeError is raised.
1335 The return is a list of the id of all nodes that match.
1336 """
1337 for propname in requirements.keys():
1338 prop = self.properties[propname]
1339 if isinstance(not prop, String):
1340 raise TypeError, "'%s' not a String property"%propname
1341 requirements[propname] = requirements[propname].lower()
1342 l = []
1343 cldb = self.db.getclassdb(self.classname)
1344 try:
1345 for nodeid in self.db.getnodeids(self.classname, cldb):
1346 node = self.db.getnode(self.classname, nodeid, cldb)
1347 if node.has_key(self.db.RETIRED_FLAG):
1348 continue
1349 for key, value in requirements.items():
1350 if node[key] is None or node[key].lower() != value:
1351 break
1352 else:
1353 l.append(nodeid)
1354 finally:
1355 cldb.close()
1356 return l
1358 def list(self):
1359 """Return a list of the ids of the active nodes in this class."""
1360 l = []
1361 cn = self.classname
1362 cldb = self.db.getclassdb(cn)
1363 try:
1364 for nodeid in self.db.getnodeids(cn, cldb):
1365 node = self.db.getnode(cn, nodeid, cldb)
1366 if node.has_key(self.db.RETIRED_FLAG):
1367 continue
1368 l.append(nodeid)
1369 finally:
1370 cldb.close()
1371 l.sort()
1372 return l
1374 def filter(self, search_matches, filterspec, sort, group,
1375 num_re = re.compile('^\d+$')):
1376 ''' Return a list of the ids of the active nodes in this class that
1377 match the 'filter' spec, sorted by the group spec and then the
1378 sort spec.
1380 "filterspec" is {propname: value(s)}
1381 "sort" is ['+propname', '-propname', 'propname', ...]
1382 "group is ['+propname', '-propname', 'propname', ...]
1383 '''
1384 cn = self.classname
1386 # optimise filterspec
1387 l = []
1388 props = self.getprops()
1389 LINK = 0
1390 MULTILINK = 1
1391 STRING = 2
1392 OTHER = 6
1393 for k, v in filterspec.items():
1394 propclass = props[k]
1395 if isinstance(propclass, Link):
1396 if type(v) is not type([]):
1397 v = [v]
1398 # replace key values with node ids
1399 u = []
1400 link_class = self.db.classes[propclass.classname]
1401 for entry in v:
1402 if entry == '-1': entry = None
1403 elif not num_re.match(entry):
1404 try:
1405 entry = link_class.lookup(entry)
1406 except (TypeError,KeyError):
1407 raise ValueError, 'property "%s": %s not a %s'%(
1408 k, entry, self.properties[k].classname)
1409 u.append(entry)
1411 l.append((LINK, k, u))
1412 elif isinstance(propclass, Multilink):
1413 if type(v) is not type([]):
1414 v = [v]
1415 # replace key values with node ids
1416 u = []
1417 link_class = self.db.classes[propclass.classname]
1418 for entry in v:
1419 if not num_re.match(entry):
1420 try:
1421 entry = link_class.lookup(entry)
1422 except (TypeError,KeyError):
1423 raise ValueError, 'new property "%s": %s not a %s'%(
1424 k, entry, self.properties[k].classname)
1425 u.append(entry)
1426 l.append((MULTILINK, k, u))
1427 elif isinstance(propclass, String):
1428 # simple glob searching
1429 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1430 v = v.replace('?', '.')
1431 v = v.replace('*', '.*?')
1432 l.append((STRING, k, re.compile(v, re.I)))
1433 elif isinstance(propclass, Boolean):
1434 if type(v) is type(''):
1435 bv = v.lower() in ('yes', 'true', 'on', '1')
1436 else:
1437 bv = v
1438 l.append((OTHER, k, bv))
1439 elif isinstance(propclass, Number):
1440 l.append((OTHER, k, int(v)))
1441 else:
1442 l.append((OTHER, k, v))
1443 filterspec = l
1445 # now, find all the nodes that are active and pass filtering
1446 l = []
1447 cldb = self.db.getclassdb(cn)
1448 try:
1449 # TODO: only full-scan once (use items())
1450 for nodeid in self.db.getnodeids(cn, cldb):
1451 node = self.db.getnode(cn, nodeid, cldb)
1452 if node.has_key(self.db.RETIRED_FLAG):
1453 continue
1454 # apply filter
1455 for t, k, v in filterspec:
1456 # make sure the node has the property
1457 if not node.has_key(k):
1458 # this node doesn't have this property, so reject it
1459 break
1461 # now apply the property filter
1462 if t == LINK:
1463 # link - if this node's property doesn't appear in the
1464 # filterspec's nodeid list, skip it
1465 if node[k] not in v:
1466 break
1467 elif t == MULTILINK:
1468 # multilink - if any of the nodeids required by the
1469 # filterspec aren't in this node's property, then skip
1470 # it
1471 have = node[k]
1472 for want in v:
1473 if want not in have:
1474 break
1475 else:
1476 continue
1477 break
1478 elif t == STRING:
1479 # RE search
1480 if node[k] is None or not v.search(node[k]):
1481 break
1482 elif t == OTHER:
1483 # straight value comparison for the other types
1484 if node[k] != v:
1485 break
1486 else:
1487 l.append((nodeid, node))
1488 finally:
1489 cldb.close()
1490 l.sort()
1492 # filter based on full text search
1493 if search_matches is not None:
1494 k = []
1495 for v in l:
1496 if search_matches.has_key(v[0]):
1497 k.append(v)
1498 l = k
1500 # optimise sort
1501 m = []
1502 for entry in sort:
1503 if entry[0] != '-':
1504 m.append(('+', entry))
1505 else:
1506 m.append((entry[0], entry[1:]))
1507 sort = m
1509 # optimise group
1510 m = []
1511 for entry in group:
1512 if entry[0] != '-':
1513 m.append(('+', entry))
1514 else:
1515 m.append((entry[0], entry[1:]))
1516 group = m
1517 # now, sort the result
1518 def sortfun(a, b, sort=sort, group=group, properties=self.getprops(),
1519 db = self.db, cl=self):
1520 a_id, an = a
1521 b_id, bn = b
1522 # sort by group and then sort
1523 for list in group, sort:
1524 for dir, prop in list:
1525 # sorting is class-specific
1526 propclass = properties[prop]
1528 # handle the properties that might be "faked"
1529 # also, handle possible missing properties
1530 try:
1531 if not an.has_key(prop):
1532 an[prop] = cl.get(a_id, prop)
1533 av = an[prop]
1534 except KeyError:
1535 # the node doesn't have a value for this property
1536 if isinstance(propclass, Multilink): av = []
1537 else: av = ''
1538 try:
1539 if not bn.has_key(prop):
1540 bn[prop] = cl.get(b_id, prop)
1541 bv = bn[prop]
1542 except KeyError:
1543 # the node doesn't have a value for this property
1544 if isinstance(propclass, Multilink): bv = []
1545 else: bv = ''
1547 # String and Date values are sorted in the natural way
1548 if isinstance(propclass, String):
1549 # clean up the strings
1550 if av and av[0] in string.uppercase:
1551 av = an[prop] = av.lower()
1552 if bv and bv[0] in string.uppercase:
1553 bv = bn[prop] = bv.lower()
1554 if (isinstance(propclass, String) or
1555 isinstance(propclass, Date)):
1556 # it might be a string that's really an integer
1557 try:
1558 av = int(av)
1559 bv = int(bv)
1560 except:
1561 pass
1562 if dir == '+':
1563 r = cmp(av, bv)
1564 if r != 0: return r
1565 elif dir == '-':
1566 r = cmp(bv, av)
1567 if r != 0: return r
1569 # Link properties are sorted according to the value of
1570 # the "order" property on the linked nodes if it is
1571 # present; or otherwise on the key string of the linked
1572 # nodes; or finally on the node ids.
1573 elif isinstance(propclass, Link):
1574 link = db.classes[propclass.classname]
1575 if av is None and bv is not None: return -1
1576 if av is not None and bv is None: return 1
1577 if av is None and bv is None: continue
1578 if link.getprops().has_key('order'):
1579 if dir == '+':
1580 r = cmp(link.get(av, 'order'),
1581 link.get(bv, 'order'))
1582 if r != 0: return r
1583 elif dir == '-':
1584 r = cmp(link.get(bv, 'order'),
1585 link.get(av, 'order'))
1586 if r != 0: return r
1587 elif link.getkey():
1588 key = link.getkey()
1589 if dir == '+':
1590 r = cmp(link.get(av, key), link.get(bv, key))
1591 if r != 0: return r
1592 elif dir == '-':
1593 r = cmp(link.get(bv, key), link.get(av, key))
1594 if r != 0: return r
1595 else:
1596 if dir == '+':
1597 r = cmp(av, bv)
1598 if r != 0: return r
1599 elif dir == '-':
1600 r = cmp(bv, av)
1601 if r != 0: return r
1603 # Multilink properties are sorted according to how many
1604 # links are present.
1605 elif isinstance(propclass, Multilink):
1606 if dir == '+':
1607 r = cmp(len(av), len(bv))
1608 if r != 0: return r
1609 elif dir == '-':
1610 r = cmp(len(bv), len(av))
1611 if r != 0: return r
1612 elif isinstance(propclass, Number) or isinstance(propclass, Boolean):
1613 if dir == '+':
1614 r = cmp(av, bv)
1615 elif dir == '-':
1616 r = cmp(bv, av)
1618 # end for dir, prop in list:
1619 # end for list in sort, group:
1620 # if all else fails, compare the ids
1621 return cmp(a[0], b[0])
1623 l.sort(sortfun)
1624 return [i[0] for i in l]
1626 def count(self):
1627 """Get the number of nodes in this class.
1629 If the returned integer is 'numnodes', the ids of all the nodes
1630 in this class run from 1 to numnodes, and numnodes+1 will be the
1631 id of the next node to be created in this class.
1632 """
1633 return self.db.countnodes(self.classname)
1635 # Manipulating properties:
1637 def getprops(self, protected=1):
1638 """Return a dictionary mapping property names to property objects.
1639 If the "protected" flag is true, we include protected properties -
1640 those which may not be modified.
1642 In addition to the actual properties on the node, these
1643 methods provide the "creation" and "activity" properties. If the
1644 "protected" flag is true, we include protected properties - those
1645 which may not be modified.
1646 """
1647 d = self.properties.copy()
1648 if protected:
1649 d['id'] = String()
1650 d['creation'] = hyperdb.Date()
1651 d['activity'] = hyperdb.Date()
1652 d['creator'] = hyperdb.Link("user")
1653 return d
1655 def addprop(self, **properties):
1656 """Add properties to this class.
1658 The keyword arguments in 'properties' must map names to property
1659 objects, or a TypeError is raised. None of the keys in 'properties'
1660 may collide with the names of existing properties, or a ValueError
1661 is raised before any properties have been added.
1662 """
1663 for key in properties.keys():
1664 if self.properties.has_key(key):
1665 raise ValueError, key
1666 self.properties.update(properties)
1668 def index(self, nodeid):
1669 '''Add (or refresh) the node to search indexes
1670 '''
1671 # find all the String properties that have indexme
1672 for prop, propclass in self.getprops().items():
1673 if isinstance(propclass, String) and propclass.indexme:
1674 try:
1675 value = str(self.get(nodeid, prop))
1676 except IndexError:
1677 # node no longer exists - entry should be removed
1678 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1679 else:
1680 # and index them under (classname, nodeid, property)
1681 self.db.indexer.add_text((self.classname, nodeid, prop),
1682 value)
1684 #
1685 # Detector interface
1686 #
1687 def audit(self, event, detector):
1688 """Register a detector
1689 """
1690 l = self.auditors[event]
1691 if detector not in l:
1692 self.auditors[event].append(detector)
1694 def fireAuditors(self, action, nodeid, newvalues):
1695 """Fire all registered auditors.
1696 """
1697 for audit in self.auditors[action]:
1698 audit(self.db, self, nodeid, newvalues)
1700 def react(self, event, detector):
1701 """Register a detector
1702 """
1703 l = self.reactors[event]
1704 if detector not in l:
1705 self.reactors[event].append(detector)
1707 def fireReactors(self, action, nodeid, oldvalues):
1708 """Fire all registered reactors.
1709 """
1710 for react in self.reactors[action]:
1711 react(self.db, self, nodeid, oldvalues)
1713 class FileClass(Class):
1714 '''This class defines a large chunk of data. To support this, it has a
1715 mandatory String property "content" which is typically saved off
1716 externally to the hyperdb.
1718 The default MIME type of this data is defined by the
1719 "default_mime_type" class attribute, which may be overridden by each
1720 node if the class defines a "type" String property.
1721 '''
1722 default_mime_type = 'text/plain'
1724 def create(self, **propvalues):
1725 ''' snaffle the file propvalue and store in a file
1726 '''
1727 content = propvalues['content']
1728 del propvalues['content']
1729 newid = Class.create(self, **propvalues)
1730 self.db.storefile(self.classname, newid, None, content)
1731 return newid
1733 def get(self, nodeid, propname, default=_marker, cache=1):
1734 ''' trap the content propname and get it from the file
1735 '''
1737 poss_msg = 'Possibly a access right configuration problem.'
1738 if propname == 'content':
1739 try:
1740 return self.db.getfile(self.classname, nodeid, None)
1741 except IOError, (strerror):
1742 # BUG: by catching this we donot see an error in the log.
1743 return 'ERROR reading file: %s%s\n%s\n%s'%(
1744 self.classname, nodeid, poss_msg, strerror)
1745 if default is not _marker:
1746 return Class.get(self, nodeid, propname, default, cache=cache)
1747 else:
1748 return Class.get(self, nodeid, propname, cache=cache)
1750 def getprops(self, protected=1):
1751 ''' In addition to the actual properties on the node, these methods
1752 provide the "content" property. If the "protected" flag is true,
1753 we include protected properties - those which may not be
1754 modified.
1755 '''
1756 d = Class.getprops(self, protected=protected).copy()
1757 if protected:
1758 d['content'] = hyperdb.String()
1759 return d
1761 def index(self, nodeid):
1762 ''' Index the node in the search index.
1764 We want to index the content in addition to the normal String
1765 property indexing.
1766 '''
1767 # perform normal indexing
1768 Class.index(self, nodeid)
1770 # get the content to index
1771 content = self.get(nodeid, 'content')
1773 # figure the mime type
1774 if self.properties.has_key('type'):
1775 mime_type = self.get(nodeid, 'type')
1776 else:
1777 mime_type = self.default_mime_type
1779 # and index!
1780 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
1781 mime_type)
1783 # XXX deviation from spec - was called ItemClass
1784 class IssueClass(Class, roundupdb.IssueClass):
1785 # Overridden methods:
1786 def __init__(self, db, classname, **properties):
1787 """The newly-created class automatically includes the "messages",
1788 "files", "nosy", and "superseder" properties. If the 'properties'
1789 dictionary attempts to specify any of these properties or a
1790 "creation" or "activity" property, a ValueError is raised.
1791 """
1792 if not properties.has_key('title'):
1793 properties['title'] = hyperdb.String(indexme='yes')
1794 if not properties.has_key('messages'):
1795 properties['messages'] = hyperdb.Multilink("msg")
1796 if not properties.has_key('files'):
1797 properties['files'] = hyperdb.Multilink("file")
1798 if not properties.has_key('nosy'):
1799 properties['nosy'] = hyperdb.Multilink("user")
1800 if not properties.has_key('superseder'):
1801 properties['superseder'] = hyperdb.Multilink(classname)
1802 Class.__init__(self, db, classname, **properties)
1804 #
1805 #$Log: not supported by cvs2svn $
1806 #Revision 1.58 2002/08/01 15:06:24 gmcm
1807 #Use same regex to split search terms as used to index text.
1808 #Fix to back_metakit for not changing journaltag on reopen.
1809 #Fix htmltemplate's do_link so [No <whatever>] strings are href'd.
1810 #Fix bogus "nosy edited ok" msg - the **d syntax does NOT share d between caller and callee.
1811 #
1812 #Revision 1.57 2002/07/31 23:57:36 richard
1813 # . web forms may now unset Link values (like assignedto)
1814 #
1815 #Revision 1.56 2002/07/31 22:04:33 richard
1816 #cleanup
1817 #
1818 #Revision 1.55 2002/07/30 08:22:38 richard
1819 #Session storage in the hyperdb was horribly, horribly inefficient. We use
1820 #a simple anydbm wrapper now - which could be overridden by the metakit
1821 #backend or RDB backend if necessary.
1822 #Much, much better.
1823 #
1824 #Revision 1.54 2002/07/26 08:26:59 richard
1825 #Very close now. The cgi and mailgw now use the new security API. The two
1826 #templates have been migrated to that setup. Lots of unit tests. Still some
1827 #issue in the web form for editing Roles assigned to users.
1828 #
1829 #Revision 1.53 2002/07/25 07:14:06 richard
1830 #Bugger it. Here's the current shape of the new security implementation.
1831 #Still to do:
1832 # . call the security funcs from cgi and mailgw
1833 # . change shipped templates to include correct initialisation and remove
1834 # the old config vars
1835 #... that seems like a lot. The bulk of the work has been done though. Honest :)
1836 #
1837 #Revision 1.52 2002/07/19 03:36:34 richard
1838 #Implemented the destroy() method needed by the session database (and possibly
1839 #others). At the same time, I removed the leading underscores from the hyperdb
1840 #methods that Really Didn't Need Them.
1841 #The journal also raises IndexError now for all situations where there is a
1842 #request for the journal of a node that doesn't have one. It used to return
1843 #[] in _some_ situations, but not all. This _may_ break code, but the tests
1844 #pass...
1845 #
1846 #Revision 1.51 2002/07/18 23:07:08 richard
1847 #Unit tests and a few fixes.
1848 #
1849 #Revision 1.50 2002/07/18 11:50:58 richard
1850 #added tests for number type too
1851 #
1852 #Revision 1.49 2002/07/18 11:41:10 richard
1853 #added tests for boolean type, and fixes to anydbm backend
1854 #
1855 #Revision 1.48 2002/07/18 11:17:31 gmcm
1856 #Add Number and Boolean types to hyperdb.
1857 #Add conversion cases to web, mail & admin interfaces.
1858 #Add storage/serialization cases to back_anydbm & back_metakit.
1859 #
1860 #Revision 1.47 2002/07/14 23:18:20 richard
1861 #. fixed the journal bloat from multilink changes - we just log the add or
1862 # remove operations, not the whole list
1863 #
1864 #Revision 1.46 2002/07/14 06:06:34 richard
1865 #Did some old TODOs
1866 #
1867 #Revision 1.45 2002/07/14 04:03:14 richard
1868 #Implemented a switch to disable journalling for a Class. CGI session
1869 #database now uses it.
1870 #
1871 #Revision 1.44 2002/07/14 02:05:53 richard
1872 #. all storage-specific code (ie. backend) is now implemented by the backends
1873 #
1874 #Revision 1.43 2002/07/10 06:30:30 richard
1875 #...except of course it's nice to use valid Python syntax
1876 #
1877 #Revision 1.42 2002/07/10 06:21:38 richard
1878 #Be extra safe
1879 #
1880 #Revision 1.41 2002/07/10 00:21:45 richard
1881 #explicit database closing
1882 #
1883 #Revision 1.40 2002/07/09 04:19:09 richard
1884 #Added reindex command to roundup-admin.
1885 #Fixed reindex on first access.
1886 #Also fixed reindexing of entries that change.
1887 #
1888 #Revision 1.39 2002/07/09 03:02:52 richard
1889 #More indexer work:
1890 #- all String properties may now be indexed too. Currently there's a bit of
1891 # "issue" specific code in the actual searching which needs to be
1892 # addressed. In a nutshell:
1893 # + pass 'indexme="yes"' as a String() property initialisation arg, eg:
1894 # file = FileClass(db, "file", name=String(), type=String(),
1895 # comment=String(indexme="yes"))
1896 # + the comment will then be indexed and be searchable, with the results
1897 # related back to the issue that the file is linked to
1898 #- as a result of this work, the FileClass has a default MIME type that may
1899 # be overridden in a subclass, or by the use of a "type" property as is
1900 # done in the default templates.
1901 #- the regeneration of the indexes (if necessary) is done once the schema is
1902 # set up in the dbinit.
1903 #
1904 #Revision 1.38 2002/07/08 06:58:15 richard
1905 #cleaned up the indexer code:
1906 # - it splits more words out (much simpler, faster splitter)
1907 # - removed code we'll never use (roundup.roundup_indexer has the full
1908 # implementation, and replaces roundup.indexer)
1909 # - only index text/plain and rfc822/message (ideas for other text formats to
1910 # index are welcome)
1911 # - added simple unit test for indexer. Needs more tests for regression.
1912 #
1913 #Revision 1.37 2002/06/20 23:52:35 richard
1914 #More informative error message
1915 #
1916 #Revision 1.36 2002/06/19 03:07:19 richard
1917 #Moved the file storage commit into blobfiles where it belongs.
1918 #
1919 #Revision 1.35 2002/05/25 07:16:24 rochecompaan
1920 #Merged search_indexing-branch with HEAD
1921 #
1922 #Revision 1.34 2002/05/15 06:21:21 richard
1923 # . node caching now works, and gives a small boost in performance
1924 #
1925 #As a part of this, I cleaned up the DEBUG output and implemented TRACE
1926 #output (HYPERDBTRACE='file to trace to') with checkpoints at the start of
1927 #CGI requests. Run roundup with python -O to skip all the DEBUG/TRACE stuff
1928 #(using if __debug__ which is compiled out with -O)
1929 #
1930 #Revision 1.33 2002/04/24 10:38:26 rochecompaan
1931 #All database files are now created group readable and writable.
1932 #
1933 #Revision 1.32 2002/04/15 23:25:15 richard
1934 #. node ids are now generated from a lockable store - no more race conditions
1935 #
1936 #We're using the portalocker code by Jonathan Feinberg that was contributed
1937 #to the ASPN Python cookbook. This gives us locking across Unix and Windows.
1938 #
1939 #Revision 1.31 2002/04/03 05:54:31 richard
1940 #Fixed serialisation problem by moving the serialisation step out of the
1941 #hyperdb.Class (get, set) into the hyperdb.Database.
1942 #
1943 #Also fixed htmltemplate after the showid changes I made yesterday.
1944 #
1945 #Unit tests for all of the above written.
1946 #
1947 #Revision 1.30.2.1 2002/04/03 11:55:57 rochecompaan
1948 # . Added feature #526730 - search for messages capability
1949 #
1950 #Revision 1.30 2002/02/27 03:40:59 richard
1951 #Ran it through pychecker, made fixes
1952 #
1953 #Revision 1.29 2002/02/25 14:34:31 grubert
1954 # . use blobfiles in back_anydbm which is used in back_bsddb.
1955 # change test_db as dirlist does not work for subdirectories.
1956 # ATTENTION: blobfiles now creates subdirectories for files.
1957 #
1958 #Revision 1.28 2002/02/16 09:14:17 richard
1959 # . #514854 ] History: "User" is always ticket creator
1960 #
1961 #Revision 1.27 2002/01/22 07:21:13 richard
1962 #. fixed back_bsddb so it passed the journal tests
1963 #
1964 #... it didn't seem happy using the back_anydbm _open method, which is odd.
1965 #Yet another occurrance of whichdb not being able to recognise older bsddb
1966 #databases. Yadda yadda. Made the HYPERDBDEBUG stuff more sane in the
1967 #process.
1968 #
1969 #Revision 1.26 2002/01/22 05:18:38 rochecompaan
1970 #last_set_entry was referenced before assignment
1971 #
1972 #Revision 1.25 2002/01/22 05:06:08 rochecompaan
1973 #We need to keep the last 'set' entry in the journal to preserve
1974 #information on 'activity' for nodes.
1975 #
1976 #Revision 1.24 2002/01/21 16:33:20 rochecompaan
1977 #You can now use the roundup-admin tool to pack the database
1978 #
1979 #Revision 1.23 2002/01/18 04:32:04 richard
1980 #Rollback was breaking because a message hadn't actually been written to the file. Needs
1981 #more investigation.
1982 #
1983 #Revision 1.22 2002/01/14 02:20:15 richard
1984 # . changed all config accesses so they access either the instance or the
1985 # config attriubute on the db. This means that all config is obtained from
1986 # instance_config instead of the mish-mash of classes. This will make
1987 # switching to a ConfigParser setup easier too, I hope.
1988 #
1989 #At a minimum, this makes migration a _little_ easier (a lot easier in the
1990 #0.5.0 switch, I hope!)
1991 #
1992 #Revision 1.21 2002/01/02 02:31:38 richard
1993 #Sorry for the huge checkin message - I was only intending to implement #496356
1994 #but I found a number of places where things had been broken by transactions:
1995 # . modified ROUNDUPDBSENDMAILDEBUG to be SENDMAILDEBUG and hold a filename
1996 # for _all_ roundup-generated smtp messages to be sent to.
1997 # . the transaction cache had broken the roundupdb.Class set() reactors
1998 # . newly-created author users in the mailgw weren't being committed to the db
1999 #
2000 #Stuff that made it into CHANGES.txt (ie. the stuff I was actually working
2001 #on when I found that stuff :):
2002 # . #496356 ] Use threading in messages
2003 # . detectors were being registered multiple times
2004 # . added tests for mailgw
2005 # . much better attaching of erroneous messages in the mail gateway
2006 #
2007 #Revision 1.20 2001/12/18 15:30:34 rochecompaan
2008 #Fixed bugs:
2009 # . Fixed file creation and retrieval in same transaction in anydbm
2010 # backend
2011 # . Cgi interface now renders new issue after issue creation
2012 # . Could not set issue status to resolved through cgi interface
2013 # . Mail gateway was changing status back to 'chatting' if status was
2014 # omitted as an argument
2015 #
2016 #Revision 1.19 2001/12/17 03:52:48 richard
2017 #Implemented file store rollback. As a bonus, the hyperdb is now capable of
2018 #storing more than one file per node - if a property name is supplied,
2019 #the file is called designator.property.
2020 #I decided not to migrate the existing files stored over to the new naming
2021 #scheme - the FileClass just doesn't specify the property name.
2022 #
2023 #Revision 1.18 2001/12/16 10:53:38 richard
2024 #take a copy of the node dict so that the subsequent set
2025 #operation doesn't modify the oldvalues structure
2026 #
2027 #Revision 1.17 2001/12/14 23:42:57 richard
2028 #yuck, a gdbm instance tests false :(
2029 #I've left the debugging code in - it should be removed one day if we're ever
2030 #_really_ anal about performace :)
2031 #
2032 #Revision 1.16 2001/12/12 03:23:14 richard
2033 #Cor blimey this anydbm/whichdb stuff is yecchy. Turns out that whichdb
2034 #incorrectly identifies a dbm file as a dbhash file on my system. This has
2035 #been submitted to the python bug tracker as issue #491888:
2036 #https://sourceforge.net/tracker/index.php?func=detail&aid=491888&group_id=5470&atid=105470
2037 #
2038 #Revision 1.15 2001/12/12 02:30:51 richard
2039 #I fixed the problems with people whose anydbm was using the dbm module at the
2040 #backend. It turns out the dbm module modifies the file name to append ".db"
2041 #and my check to determine if we're opening an existing or new db just
2042 #tested os.path.exists() on the filename. Well, no longer! We now perform a
2043 #much better check _and_ cope with the anydbm implementation module changing
2044 #too!
2045 #I also fixed the backends __init__ so only ImportError is squashed.
2046 #
2047 #Revision 1.14 2001/12/10 22:20:01 richard
2048 #Enabled transaction support in the bsddb backend. It uses the anydbm code
2049 #where possible, only replacing methods where the db is opened (it uses the
2050 #btree opener specifically.)
2051 #Also cleaned up some change note generation.
2052 #Made the backends package work with pydoc too.
2053 #
2054 #Revision 1.13 2001/12/02 05:06:16 richard
2055 #. We now use weakrefs in the Classes to keep the database reference, so
2056 # the close() method on the database is no longer needed.
2057 # I bumped the minimum python requirement up to 2.1 accordingly.
2058 #. #487480 ] roundup-server
2059 #. #487476 ] INSTALL.txt
2060 #
2061 #I also cleaned up the change message / post-edit stuff in the cgi client.
2062 #There's now a clearly marked "TODO: append the change note" where I believe
2063 #the change note should be added there. The "changes" list will obviously
2064 #have to be modified to be a dict of the changes, or somesuch.
2065 #
2066 #More testing needed.
2067 #
2068 #Revision 1.12 2001/12/01 07:17:50 richard
2069 #. We now have basic transaction support! Information is only written to
2070 # the database when the commit() method is called. Only the anydbm
2071 # backend is modified in this way - neither of the bsddb backends have been.
2072 # The mail, admin and cgi interfaces all use commit (except the admin tool
2073 # doesn't have a commit command, so interactive users can't commit...)
2074 #. Fixed login/registration forwarding the user to the right page (or not,
2075 # on a failure)
2076 #
2077 #Revision 1.11 2001/11/21 02:34:18 richard
2078 #Added a target version field to the extended issue schema
2079 #
2080 #Revision 1.10 2001/10/09 23:58:10 richard
2081 #Moved the data stringification up into the hyperdb.Class class' get, set
2082 #and create methods. This means that the data is also stringified for the
2083 #journal call, and removes duplication of code from the backends. The
2084 #backend code now only sees strings.
2085 #
2086 #Revision 1.9 2001/10/09 07:25:59 richard
2087 #Added the Password property type. See "pydoc roundup.password" for
2088 #implementation details. Have updated some of the documentation too.
2089 #
2090 #Revision 1.8 2001/09/29 13:27:00 richard
2091 #CGI interfaces now spit up a top-level index of all the instances they can
2092 #serve.
2093 #
2094 #Revision 1.7 2001/08/12 06:32:36 richard
2095 #using isinstance(blah, Foo) now instead of isFooType
2096 #
2097 #Revision 1.6 2001/08/07 00:24:42 richard
2098 #stupid typo
2099 #
2100 #Revision 1.5 2001/08/07 00:15:51 richard
2101 #Added the copyright/license notice to (nearly) all files at request of
2102 #Bizar Software.
2103 #
2104 #Revision 1.4 2001/07/30 01:41:36 richard
2105 #Makes schema changes mucho easier.
2106 #
2107 #Revision 1.3 2001/07/25 01:23:07 richard
2108 #Added the Roundup spec to the new documentation directory.
2109 #
2110 #Revision 1.2 2001/07/23 08:20:44 richard
2111 #Moved over to using marshal in the bsddb and anydbm backends.
2112 #roundup-admin now has a "freshen" command that'll load/save all nodes (not
2113 # retired - mod hyperdb.Class.list() so it lists retired nodes)
2114 #
2115 #