![[tokkee]](http://tokkee.org/images/avatar.png){kind=avatar}
0de77023d9e3a38db5eb38ddc315630cabddcb3b
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: back_anydbm.py,v 1.58 2002-08-01 15:06:24 gmcm Exp $
19 '''
20 This module defines a backend that saves the hyperdatabase in a database
21 chosen by anydbm. It is guaranteed to always be available in python
22 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
23 serious bugs, and is not available)
24 '''
26 import whichdb, anydbm, os, marshal, re, weakref, string, copy
27 from roundup import hyperdb, date, password, roundupdb, security
28 from blobfiles import FileStorage
29 from sessions import Sessions
30 from roundup.indexer import Indexer
31 from locking import acquire_lock, release_lock
32 from roundup.hyperdb import String, Password, Date, Interval, Link, \
33 Multilink, DatabaseError, Boolean, Number
35 #
36 # Now the database
37 #
38 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
39 """A database for storing records containing flexible data types.
41 Transaction stuff TODO:
42 . check the timestamp of the class file and nuke the cache if it's
43 modified. Do some sort of conflict checking on the dirty stuff.
44 . perhaps detect write collisions (related to above)?
46 """
47 def __init__(self, config, journaltag=None):
48 """Open a hyperdatabase given a specifier to some storage.
50 The 'storagelocator' is obtained from config.DATABASE.
51 The meaning of 'storagelocator' depends on the particular
52 implementation of the hyperdatabase. It could be a file name,
53 a directory path, a socket descriptor for a connection to a
54 database over the network, etc.
56 The 'journaltag' is a token that will be attached to the journal
57 entries for any edits done on the database. If 'journaltag' is
58 None, the database is opened in read-only mode: the Class.create(),
59 Class.set(), and Class.retire() methods are disabled.
60 """
61 self.config, self.journaltag = config, journaltag
62 self.dir = config.DATABASE
63 self.classes = {}
64 self.cache = {} # cache of nodes loaded or created
65 self.dirtynodes = {} # keep track of the dirty nodes by class
66 self.newnodes = {} # keep track of the new nodes by class
67 self.destroyednodes = {}# keep track of the destroyed nodes by class
68 self.transactions = []
69 self.indexer = Indexer(self.dir)
70 self.sessions = Sessions(self.config)
71 self.security = security.Security(self)
72 # ensure files are group readable and writable
73 os.umask(0002)
75 def post_init(self):
76 """Called once the schema initialisation has finished."""
77 # reindex the db if necessary
78 if self.indexer.should_reindex():
79 self.reindex()
81 def reindex(self):
82 for klass in self.classes.values():
83 for nodeid in klass.list():
84 klass.index(nodeid)
85 self.indexer.save_index()
87 def __repr__(self):
88 return '<back_anydbm instance at %x>'%id(self)
90 #
91 # Classes
92 #
93 def __getattr__(self, classname):
94 """A convenient way of calling self.getclass(classname)."""
95 if self.classes.has_key(classname):
96 if __debug__:
97 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
98 return self.classes[classname]
99 raise AttributeError, classname
101 def addclass(self, cl):
102 if __debug__:
103 print >>hyperdb.DEBUG, 'addclass', (self, cl)
104 cn = cl.classname
105 if self.classes.has_key(cn):
106 raise ValueError, cn
107 self.classes[cn] = cl
109 def getclasses(self):
110 """Return a list of the names of all existing classes."""
111 if __debug__:
112 print >>hyperdb.DEBUG, 'getclasses', (self,)
113 l = self.classes.keys()
114 l.sort()
115 return l
117 def getclass(self, classname):
118 """Get the Class object representing a particular class.
120 If 'classname' is not a valid class name, a KeyError is raised.
121 """
122 if __debug__:
123 print >>hyperdb.DEBUG, 'getclass', (self, classname)
124 return self.classes[classname]
126 #
127 # Class DBs
128 #
129 def clear(self):
130 '''Delete all database contents
131 '''
132 if __debug__:
133 print >>hyperdb.DEBUG, 'clear', (self,)
134 for cn in self.classes.keys():
135 for dummy in 'nodes', 'journals':
136 path = os.path.join(self.dir, 'journals.%s'%cn)
137 if os.path.exists(path):
138 os.remove(path)
139 elif os.path.exists(path+'.db'): # dbm appends .db
140 os.remove(path+'.db')
142 def getclassdb(self, classname, mode='r'):
143 ''' grab a connection to the class db that will be used for
144 multiple actions
145 '''
146 if __debug__:
147 print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode)
148 return self.opendb('nodes.%s'%classname, mode)
150 def determine_db_type(self, path):
151 ''' determine which DB wrote the class file
152 '''
153 db_type = ''
154 if os.path.exists(path):
155 db_type = whichdb.whichdb(path)
156 if not db_type:
157 raise hyperdb.DatabaseError, "Couldn't identify database type"
158 elif os.path.exists(path+'.db'):
159 # if the path ends in '.db', it's a dbm database, whether
160 # anydbm says it's dbhash or not!
161 db_type = 'dbm'
162 return db_type
164 def opendb(self, name, mode):
165 '''Low-level database opener that gets around anydbm/dbm
166 eccentricities.
167 '''
168 if __debug__:
169 print >>hyperdb.DEBUG, 'opendb', (self, name, mode)
171 # figure the class db type
172 path = os.path.join(os.getcwd(), self.dir, name)
173 db_type = self.determine_db_type(path)
175 # new database? let anydbm pick the best dbm
176 if not db_type:
177 if __debug__:
178 print >>hyperdb.DEBUG, "opendb anydbm.open(%r, 'n')"%path
179 return anydbm.open(path, 'n')
181 # open the database with the correct module
182 try:
183 dbm = __import__(db_type)
184 except ImportError:
185 raise hyperdb.DatabaseError, \
186 "Couldn't open database - the required module '%s'"\
187 " is not available"%db_type
188 if __debug__:
189 print >>hyperdb.DEBUG, "opendb %r.open(%r, %r)"%(db_type, path,
190 mode)
191 return dbm.open(path, mode)
193 def lockdb(self, name):
194 ''' Lock a database file
195 '''
196 path = os.path.join(os.getcwd(), self.dir, '%s.lock'%name)
197 return acquire_lock(path)
199 #
200 # Node IDs
201 #
202 def newid(self, classname):
203 ''' Generate a new id for the given class
204 '''
205 # open the ids DB - create if if doesn't exist
206 lock = self.lockdb('_ids')
207 db = self.opendb('_ids', 'c')
208 if db.has_key(classname):
209 newid = db[classname] = str(int(db[classname]) + 1)
210 else:
211 # the count() bit is transitional - older dbs won't start at 1
212 newid = str(self.getclass(classname).count()+1)
213 db[classname] = newid
214 db.close()
215 release_lock(lock)
216 return newid
218 #
219 # Nodes
220 #
221 def addnode(self, classname, nodeid, node):
222 ''' add the specified node to its class's db
223 '''
224 if __debug__:
225 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
226 self.newnodes.setdefault(classname, {})[nodeid] = 1
227 self.cache.setdefault(classname, {})[nodeid] = node
228 self.savenode(classname, nodeid, node)
230 def setnode(self, classname, nodeid, node):
231 ''' change the specified node
232 '''
233 if __debug__:
234 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
235 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
237 # can't set without having already loaded the node
238 self.cache[classname][nodeid] = node
239 self.savenode(classname, nodeid, node)
241 def savenode(self, classname, nodeid, node):
242 ''' perform the saving of data specified by the set/addnode
243 '''
244 if __debug__:
245 print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node)
246 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
248 def getnode(self, classname, nodeid, db=None, cache=1):
249 ''' get a node from the database
250 '''
251 if __debug__:
252 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid, db)
253 if cache:
254 # try the cache
255 cache_dict = self.cache.setdefault(classname, {})
256 if cache_dict.has_key(nodeid):
257 if __debug__:
258 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
259 nodeid)
260 return cache_dict[nodeid]
262 if __debug__:
263 print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid)
265 # get from the database and save in the cache
266 if db is None:
267 db = self.getclassdb(classname)
268 if not db.has_key(nodeid):
269 raise IndexError, "no such %s %s"%(classname, nodeid)
271 # check the uncommitted, destroyed nodes
272 if (self.destroyednodes.has_key(classname) and
273 self.destroyednodes[classname].has_key(nodeid)):
274 raise IndexError, "no such %s %s"%(classname, nodeid)
276 # decode
277 res = marshal.loads(db[nodeid])
279 # reverse the serialisation
280 res = self.unserialise(classname, res)
282 # store off in the cache dict
283 if cache:
284 cache_dict[nodeid] = res
286 return res
288 def destroynode(self, classname, nodeid):
289 '''Remove a node from the database. Called exclusively by the
290 destroy() method on Class.
291 '''
292 if __debug__:
293 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
295 # remove from cache and newnodes if it's there
296 if (self.cache.has_key(classname) and
297 self.cache[classname].has_key(nodeid)):
298 del self.cache[classname][nodeid]
299 if (self.newnodes.has_key(classname) and
300 self.newnodes[classname].has_key(nodeid)):
301 del self.newnodes[classname][nodeid]
303 # see if there's any obvious commit actions that we should get rid of
304 for entry in self.transactions[:]:
305 if entry[1][:2] == (classname, nodeid):
306 self.transactions.remove(entry)
308 # add to the destroyednodes map
309 self.destroyednodes.setdefault(classname, {})[nodeid] = 1
311 # add the destroy commit action
312 self.transactions.append((self.doDestroyNode, (classname, nodeid)))
314 def serialise(self, classname, node):
315 '''Copy the node contents, converting non-marshallable data into
316 marshallable data.
317 '''
318 if __debug__:
319 print >>hyperdb.DEBUG, 'serialise', classname, node
320 properties = self.getclass(classname).getprops()
321 d = {}
322 for k, v in node.items():
323 # if the property doesn't exist, or is the "retired" flag then
324 # it won't be in the properties dict
325 if not properties.has_key(k):
326 d[k] = v
327 continue
329 # get the property spec
330 prop = properties[k]
332 if isinstance(prop, Password):
333 d[k] = str(v)
334 elif isinstance(prop, Date) and v is not None:
335 d[k] = v.get_tuple()
336 elif isinstance(prop, Interval) and v is not None:
337 d[k] = v.get_tuple()
338 else:
339 d[k] = v
340 return d
342 def unserialise(self, classname, node):
343 '''Decode the marshalled node data
344 '''
345 if __debug__:
346 print >>hyperdb.DEBUG, 'unserialise', classname, node
347 properties = self.getclass(classname).getprops()
348 d = {}
349 for k, v in node.items():
350 # if the property doesn't exist, or is the "retired" flag then
351 # it won't be in the properties dict
352 if not properties.has_key(k):
353 d[k] = v
354 continue
356 # get the property spec
357 prop = properties[k]
359 if isinstance(prop, Date) and v is not None:
360 d[k] = date.Date(v)
361 elif isinstance(prop, Interval) and v is not None:
362 d[k] = date.Interval(v)
363 elif isinstance(prop, Password):
364 p = password.Password()
365 p.unpack(v)
366 d[k] = p
367 else:
368 d[k] = v
369 return d
371 def hasnode(self, classname, nodeid, db=None):
372 ''' determine if the database has a given node
373 '''
374 if __debug__:
375 print >>hyperdb.DEBUG, 'hasnode', (self, classname, nodeid, db)
377 # try the cache
378 cache = self.cache.setdefault(classname, {})
379 if cache.has_key(nodeid):
380 if __debug__:
381 print >>hyperdb.TRACE, 'has %s %s cached'%(classname, nodeid)
382 return 1
383 if __debug__:
384 print >>hyperdb.TRACE, 'has %s %s'%(classname, nodeid)
386 # not in the cache - check the database
387 if db is None:
388 db = self.getclassdb(classname)
389 res = db.has_key(nodeid)
390 return res
392 def countnodes(self, classname, db=None):
393 if __debug__:
394 print >>hyperdb.DEBUG, 'countnodes', (self, classname, db)
396 count = 0
398 # include the uncommitted nodes
399 if self.newnodes.has_key(classname):
400 count += len(self.newnodes[classname])
401 if self.destroyednodes.has_key(classname):
402 count -= len(self.destroyednodes[classname])
404 # and count those in the DB
405 if db is None:
406 db = self.getclassdb(classname)
407 count = count + len(db.keys())
408 return count
410 def getnodeids(self, classname, db=None):
411 if __debug__:
412 print >>hyperdb.DEBUG, 'getnodeids', (self, classname, db)
414 res = []
416 # start off with the new nodes
417 if self.newnodes.has_key(classname):
418 res += self.newnodes[classname].keys()
420 if db is None:
421 db = self.getclassdb(classname)
422 res = res + db.keys()
424 # remove the uncommitted, destroyed nodes
425 if self.destroyednodes.has_key(classname):
426 for nodeid in self.destroyednodes[classname].keys():
427 if db.has_key(nodeid):
428 res.remove(nodeid)
430 return res
433 #
434 # Files - special node properties
435 # inherited from FileStorage
437 #
438 # Journal
439 #
440 def addjournal(self, classname, nodeid, action, params):
441 ''' Journal the Action
442 'action' may be:
444 'create' or 'set' -- 'params' is a dictionary of property values
445 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
446 'retire' -- 'params' is None
447 '''
448 if __debug__:
449 print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid,
450 action, params)
451 self.transactions.append((self.doSaveJournal, (classname, nodeid,
452 action, params)))
454 def getjournal(self, classname, nodeid):
455 ''' get the journal for id
457 Raise IndexError if the node doesn't exist (as per history()'s
458 API)
459 '''
460 if __debug__:
461 print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid)
462 # attempt to open the journal - in some rare cases, the journal may
463 # not exist
464 try:
465 db = self.opendb('journals.%s'%classname, 'r')
466 except anydbm.error, error:
467 if str(error) == "need 'c' or 'n' flag to open new db":
468 raise IndexError, 'no such %s %s'%(classname, nodeid)
469 elif error.args[0] != 2:
470 raise
471 raise IndexError, 'no such %s %s'%(classname, nodeid)
472 try:
473 journal = marshal.loads(db[nodeid])
474 except KeyError:
475 db.close()
476 raise IndexError, 'no such %s %s'%(classname, nodeid)
477 db.close()
478 res = []
479 for nodeid, date_stamp, user, action, params in journal:
480 res.append((nodeid, date.Date(date_stamp), user, action, params))
481 return res
483 def pack(self, pack_before):
484 ''' delete all journal entries before 'pack_before' '''
485 if __debug__:
486 print >>hyperdb.DEBUG, 'packjournal', (self, pack_before)
488 pack_before = pack_before.get_tuple()
490 classes = self.getclasses()
492 # figure the class db type
494 for classname in classes:
495 db_name = 'journals.%s'%classname
496 path = os.path.join(os.getcwd(), self.dir, classname)
497 db_type = self.determine_db_type(path)
498 db = self.opendb(db_name, 'w')
500 for key in db.keys():
501 journal = marshal.loads(db[key])
502 l = []
503 last_set_entry = None
504 for entry in journal:
505 (nodeid, date_stamp, self.journaltag, action,
506 params) = entry
507 if date_stamp > pack_before or action == 'create':
508 l.append(entry)
509 elif action == 'set':
510 # grab the last set entry to keep information on
511 # activity
512 last_set_entry = entry
513 if last_set_entry:
514 date_stamp = last_set_entry[1]
515 # if the last set entry was made after the pack date
516 # then it is already in the list
517 if date_stamp < pack_before:
518 l.append(last_set_entry)
519 db[key] = marshal.dumps(l)
520 if db_type == 'gdbm':
521 db.reorganize()
522 db.close()
525 #
526 # Basic transaction support
527 #
528 def commit(self):
529 ''' Commit the current transactions.
530 '''
531 if __debug__:
532 print >>hyperdb.DEBUG, 'commit', (self,)
533 # TODO: lock the DB
535 # keep a handle to all the database files opened
536 self.databases = {}
538 # now, do all the transactions
539 reindex = {}
540 for method, args in self.transactions:
541 reindex[method(*args)] = 1
543 # now close all the database files
544 for db in self.databases.values():
545 db.close()
546 del self.databases
547 # TODO: unlock the DB
549 # reindex the nodes that request it
550 for classname, nodeid in filter(None, reindex.keys()):
551 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
552 self.getclass(classname).index(nodeid)
554 # save the indexer state
555 self.indexer.save_index()
557 # all transactions committed, back to normal
558 self.cache = {}
559 self.dirtynodes = {}
560 self.newnodes = {}
561 self.destroyednodes = {}
562 self.transactions = []
564 def getCachedClassDB(self, classname):
565 ''' get the class db, looking in our cache of databases for commit
566 '''
567 # get the database handle
568 db_name = 'nodes.%s'%classname
569 if not self.databases.has_key(db_name):
570 self.databases[db_name] = self.getclassdb(classname, 'c')
571 return self.databases[db_name]
573 def doSaveNode(self, classname, nodeid, node):
574 if __debug__:
575 print >>hyperdb.DEBUG, 'doSaveNode', (self, classname, nodeid,
576 node)
578 db = self.getCachedClassDB(classname)
580 # now save the marshalled data
581 db[nodeid] = marshal.dumps(self.serialise(classname, node))
583 # return the classname, nodeid so we reindex this content
584 return (classname, nodeid)
586 def getCachedJournalDB(self, classname):
587 ''' get the journal db, looking in our cache of databases for commit
588 '''
589 # get the database handle
590 db_name = 'journals.%s'%classname
591 if not self.databases.has_key(db_name):
592 self.databases[db_name] = self.opendb(db_name, 'c')
593 return self.databases[db_name]
595 def doSaveJournal(self, classname, nodeid, action, params):
596 # serialise first
597 if action in ('set', 'create'):
598 params = self.serialise(classname, params)
600 # create the journal entry
601 entry = (nodeid, date.Date().get_tuple(), self.journaltag, action,
602 params)
604 if __debug__:
605 print >>hyperdb.DEBUG, 'doSaveJournal', entry
607 db = self.getCachedJournalDB(classname)
609 # now insert the journal entry
610 if db.has_key(nodeid):
611 # append to existing
612 s = db[nodeid]
613 l = marshal.loads(s)
614 l.append(entry)
615 else:
616 l = [entry]
618 db[nodeid] = marshal.dumps(l)
620 def doDestroyNode(self, classname, nodeid):
621 if __debug__:
622 print >>hyperdb.DEBUG, 'doDestroyNode', (self, classname, nodeid)
624 # delete from the class database
625 db = self.getCachedClassDB(classname)
626 if db.has_key(nodeid):
627 del db[nodeid]
629 # delete from the database
630 db = self.getCachedJournalDB(classname)
631 if db.has_key(nodeid):
632 del db[nodeid]
634 # return the classname, nodeid so we reindex this content
635 return (classname, nodeid)
637 def rollback(self):
638 ''' Reverse all actions from the current transaction.
639 '''
640 if __debug__:
641 print >>hyperdb.DEBUG, 'rollback', (self, )
642 for method, args in self.transactions:
643 # delete temporary files
644 if method == self.doStoreFile:
645 self.rollbackStoreFile(*args)
646 self.cache = {}
647 self.dirtynodes = {}
648 self.newnodes = {}
649 self.destroyednodes = {}
650 self.transactions = []
652 _marker = []
653 class Class(hyperdb.Class):
654 """The handle to a particular class of nodes in a hyperdatabase."""
656 def __init__(self, db, classname, **properties):
657 """Create a new class with a given name and property specification.
659 'classname' must not collide with the name of an existing class,
660 or a ValueError is raised. The keyword arguments in 'properties'
661 must map names to property objects, or a TypeError is raised.
662 """
663 if (properties.has_key('creation') or properties.has_key('activity')
664 or properties.has_key('creator')):
665 raise ValueError, '"creation", "activity" and "creator" are '\
666 'reserved'
668 self.classname = classname
669 self.properties = properties
670 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
671 self.key = ''
673 # should we journal changes (default yes)
674 self.do_journal = 1
676 # do the db-related init stuff
677 db.addclass(self)
679 self.auditors = {'create': [], 'set': [], 'retire': []}
680 self.reactors = {'create': [], 'set': [], 'retire': []}
682 def enableJournalling(self):
683 '''Turn journalling on for this class
684 '''
685 self.do_journal = 1
687 def disableJournalling(self):
688 '''Turn journalling off for this class
689 '''
690 self.do_journal = 0
692 # Editing nodes:
694 def create(self, **propvalues):
695 """Create a new node of this class and return its id.
697 The keyword arguments in 'propvalues' map property names to values.
699 The values of arguments must be acceptable for the types of their
700 corresponding properties or a TypeError is raised.
702 If this class has a key property, it must be present and its value
703 must not collide with other key strings or a ValueError is raised.
705 Any other properties on this class that are missing from the
706 'propvalues' dictionary are set to None.
708 If an id in a link or multilink property does not refer to a valid
709 node, an IndexError is raised.
711 These operations trigger detectors and can be vetoed. Attempts
712 to modify the "creation" or "activity" properties cause a KeyError.
713 """
714 if propvalues.has_key('id'):
715 raise KeyError, '"id" is reserved'
717 if self.db.journaltag is None:
718 raise DatabaseError, 'Database open read-only'
720 if propvalues.has_key('creation') or propvalues.has_key('activity'):
721 raise KeyError, '"creation" and "activity" are reserved'
723 self.fireAuditors('create', None, propvalues)
725 # new node's id
726 newid = self.db.newid(self.classname)
728 # validate propvalues
729 num_re = re.compile('^\d+$')
730 for key, value in propvalues.items():
731 if key == self.key:
732 try:
733 self.lookup(value)
734 except KeyError:
735 pass
736 else:
737 raise ValueError, 'node with key "%s" exists'%value
739 # try to handle this property
740 try:
741 prop = self.properties[key]
742 except KeyError:
743 raise KeyError, '"%s" has no property "%s"'%(self.classname,
744 key)
746 if isinstance(prop, Link):
747 if type(value) != type(''):
748 raise ValueError, 'link value must be String'
749 link_class = self.properties[key].classname
750 # if it isn't a number, it's a key
751 if not num_re.match(value):
752 try:
753 value = self.db.classes[link_class].lookup(value)
754 except (TypeError, KeyError):
755 raise IndexError, 'new property "%s": %s not a %s'%(
756 key, value, link_class)
757 elif not self.db.getclass(link_class).hasnode(value):
758 raise IndexError, '%s has no node %s'%(link_class, value)
760 # save off the value
761 propvalues[key] = value
763 # register the link with the newly linked node
764 if self.do_journal and self.properties[key].do_journal:
765 self.db.addjournal(link_class, value, 'link',
766 (self.classname, newid, key))
768 elif isinstance(prop, Multilink):
769 if type(value) != type([]):
770 raise TypeError, 'new property "%s" not a list of ids'%key
772 # clean up and validate the list of links
773 link_class = self.properties[key].classname
774 l = []
775 for entry in value:
776 if type(entry) != type(''):
777 raise ValueError, '"%s" link value (%s) must be '\
778 'String'%(key, value)
779 # if it isn't a number, it's a key
780 if not num_re.match(entry):
781 try:
782 entry = self.db.classes[link_class].lookup(entry)
783 except (TypeError, KeyError):
784 raise IndexError, 'new property "%s": %s not a %s'%(
785 key, entry, self.properties[key].classname)
786 l.append(entry)
787 value = l
788 propvalues[key] = value
790 # handle additions
791 for nodeid in value:
792 if not self.db.getclass(link_class).hasnode(nodeid):
793 raise IndexError, '%s has no node %s'%(link_class,
794 nodeid)
795 # register the link with the newly linked node
796 if self.do_journal and self.properties[key].do_journal:
797 self.db.addjournal(link_class, nodeid, 'link',
798 (self.classname, newid, key))
800 elif isinstance(prop, String):
801 if type(value) != type(''):
802 raise TypeError, 'new property "%s" not a string'%key
804 elif isinstance(prop, Password):
805 if not isinstance(value, password.Password):
806 raise TypeError, 'new property "%s" not a Password'%key
808 elif isinstance(prop, Date):
809 if value is not None and not isinstance(value, date.Date):
810 raise TypeError, 'new property "%s" not a Date'%key
812 elif isinstance(prop, Interval):
813 if value is not None and not isinstance(value, date.Interval):
814 raise TypeError, 'new property "%s" not an Interval'%key
816 elif value is not None and isinstance(prop, Number):
817 try:
818 float(value)
819 except ValueError:
820 raise TypeError, 'new property "%s" not numeric'%key
822 elif value is not None and isinstance(prop, Boolean):
823 try:
824 int(value)
825 except ValueError:
826 raise TypeError, 'new property "%s" not boolean'%key
828 # make sure there's data where there needs to be
829 for key, prop in self.properties.items():
830 if propvalues.has_key(key):
831 continue
832 if key == self.key:
833 raise ValueError, 'key property "%s" is required'%key
834 if isinstance(prop, Multilink):
835 propvalues[key] = []
836 else:
837 propvalues[key] = None
839 # done
840 self.db.addnode(self.classname, newid, propvalues)
841 if self.do_journal:
842 self.db.addjournal(self.classname, newid, 'create', propvalues)
844 self.fireReactors('create', newid, None)
846 return newid
848 def get(self, nodeid, propname, default=_marker, cache=1):
849 """Get the value of a property on an existing node of this class.
851 'nodeid' must be the id of an existing node of this class or an
852 IndexError is raised. 'propname' must be the name of a property
853 of this class or a KeyError is raised.
855 'cache' indicates whether the transaction cache should be queried
856 for the node. If the node has been modified and you need to
857 determine what its values prior to modification are, you need to
858 set cache=0.
860 Attempts to get the "creation" or "activity" properties should
861 do the right thing.
862 """
863 if propname == 'id':
864 return nodeid
866 if propname == 'creation':
867 if not self.do_journal:
868 raise ValueError, 'Journalling is disabled for this class'
869 journal = self.db.getjournal(self.classname, nodeid)
870 if journal:
871 return self.db.getjournal(self.classname, nodeid)[0][1]
872 else:
873 # on the strange chance that there's no journal
874 return date.Date()
875 if propname == 'activity':
876 if not self.do_journal:
877 raise ValueError, 'Journalling is disabled for this class'
878 journal = self.db.getjournal(self.classname, nodeid)
879 if journal:
880 return self.db.getjournal(self.classname, nodeid)[-1][1]
881 else:
882 # on the strange chance that there's no journal
883 return date.Date()
884 if propname == 'creator':
885 if not self.do_journal:
886 raise ValueError, 'Journalling is disabled for this class'
887 journal = self.db.getjournal(self.classname, nodeid)
888 if journal:
889 name = self.db.getjournal(self.classname, nodeid)[0][2]
890 else:
891 return None
892 return self.db.user.lookup(name)
894 # get the property (raises KeyErorr if invalid)
895 prop = self.properties[propname]
897 # get the node's dict
898 d = self.db.getnode(self.classname, nodeid, cache=cache)
900 if not d.has_key(propname):
901 if default is _marker:
902 if isinstance(prop, Multilink):
903 return []
904 else:
905 return None
906 else:
907 return default
909 return d[propname]
911 # XXX not in spec
912 def getnode(self, nodeid, cache=1):
913 ''' Return a convenience wrapper for the node.
915 'nodeid' must be the id of an existing node of this class or an
916 IndexError is raised.
918 'cache' indicates whether the transaction cache should be queried
919 for the node. If the node has been modified and you need to
920 determine what its values prior to modification are, you need to
921 set cache=0.
922 '''
923 return Node(self, nodeid, cache=cache)
925 def set(self, nodeid, **propvalues):
926 """Modify a property on an existing node of this class.
928 'nodeid' must be the id of an existing node of this class or an
929 IndexError is raised.
931 Each key in 'propvalues' must be the name of a property of this
932 class or a KeyError is raised.
934 All values in 'propvalues' must be acceptable types for their
935 corresponding properties or a TypeError is raised.
937 If the value of the key property is set, it must not collide with
938 other key strings or a ValueError is raised.
940 If the value of a Link or Multilink property contains an invalid
941 node id, a ValueError is raised.
943 These operations trigger detectors and can be vetoed. Attempts
944 to modify the "creation" or "activity" properties cause a KeyError.
945 """
946 if not propvalues:
947 return propvalues
949 if propvalues.has_key('creation') or propvalues.has_key('activity'):
950 raise KeyError, '"creation" and "activity" are reserved'
952 if propvalues.has_key('id'):
953 raise KeyError, '"id" is reserved'
955 if self.db.journaltag is None:
956 raise DatabaseError, 'Database open read-only'
958 self.fireAuditors('set', nodeid, propvalues)
959 # Take a copy of the node dict so that the subsequent set
960 # operation doesn't modify the oldvalues structure.
961 try:
962 # try not using the cache initially
963 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid,
964 cache=0))
965 except IndexError:
966 # this will be needed if somone does a create() and set()
967 # with no intervening commit()
968 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
970 node = self.db.getnode(self.classname, nodeid)
971 if node.has_key(self.db.RETIRED_FLAG):
972 raise IndexError
973 num_re = re.compile('^\d+$')
975 # if the journal value is to be different, store it in here
976 journalvalues = {}
978 for propname, value in propvalues.items():
979 # check to make sure we're not duplicating an existing key
980 if propname == self.key and node[propname] != value:
981 try:
982 self.lookup(value)
983 except KeyError:
984 pass
985 else:
986 raise ValueError, 'node with key "%s" exists'%value
988 # this will raise the KeyError if the property isn't valid
989 # ... we don't use getprops() here because we only care about
990 # the writeable properties.
991 prop = self.properties[propname]
993 # if the value's the same as the existing value, no sense in
994 # doing anything
995 if node.has_key(propname) and value == node[propname]:
996 del propvalues[propname]
997 continue
999 # do stuff based on the prop type
1000 if isinstance(prop, Link):
1001 link_class = prop.classname
1002 # if it isn't a number, it's a key
1003 if value is not None and not isinstance(value, type('')):
1004 raise ValueError, 'property "%s" link value be a string'%(
1005 propname)
1006 if isinstance(value, type('')) and not num_re.match(value):
1007 try:
1008 value = self.db.classes[link_class].lookup(value)
1009 except (TypeError, KeyError):
1010 raise IndexError, 'new property "%s": %s not a %s'%(
1011 propname, value, prop.classname)
1013 if (value is not None and
1014 not self.db.getclass(link_class).hasnode(value)):
1015 raise IndexError, '%s has no node %s'%(link_class, value)
1017 if self.do_journal and prop.do_journal:
1018 # register the unlink with the old linked node
1019 if node[propname] is not None:
1020 self.db.addjournal(link_class, node[propname], 'unlink',
1021 (self.classname, nodeid, propname))
1023 # register the link with the newly linked node
1024 if value is not None:
1025 self.db.addjournal(link_class, value, 'link',
1026 (self.classname, nodeid, propname))
1028 elif isinstance(prop, Multilink):
1029 if type(value) != type([]):
1030 raise TypeError, 'new property "%s" not a list of'\
1031 ' ids'%propname
1032 link_class = self.properties[propname].classname
1033 l = []
1034 for entry in value:
1035 # if it isn't a number, it's a key
1036 if type(entry) != type(''):
1037 raise ValueError, 'new property "%s" link value ' \
1038 'must be a string'%propname
1039 if not num_re.match(entry):
1040 try:
1041 entry = self.db.classes[link_class].lookup(entry)
1042 except (TypeError, KeyError):
1043 raise IndexError, 'new property "%s": %s not a %s'%(
1044 propname, entry,
1045 self.properties[propname].classname)
1046 l.append(entry)
1047 value = l
1048 propvalues[propname] = value
1050 # figure the journal entry for this property
1051 add = []
1052 remove = []
1054 # handle removals
1055 if node.has_key(propname):
1056 l = node[propname]
1057 else:
1058 l = []
1059 for id in l[:]:
1060 if id in value:
1061 continue
1062 # register the unlink with the old linked node
1063 if self.do_journal and self.properties[propname].do_journal:
1064 self.db.addjournal(link_class, id, 'unlink',
1065 (self.classname, nodeid, propname))
1066 l.remove(id)
1067 remove.append(id)
1069 # handle additions
1070 for id in value:
1071 if not self.db.getclass(link_class).hasnode(id):
1072 raise IndexError, '%s has no node %s'%(link_class, id)
1073 if id in l:
1074 continue
1075 # register the link with the newly linked node
1076 if self.do_journal and self.properties[propname].do_journal:
1077 self.db.addjournal(link_class, id, 'link',
1078 (self.classname, nodeid, propname))
1079 l.append(id)
1080 add.append(id)
1082 # figure the journal entry
1083 l = []
1084 if add:
1085 l.append(('add', add))
1086 if remove:
1087 l.append(('remove', remove))
1088 if l:
1089 journalvalues[propname] = tuple(l)
1091 elif isinstance(prop, String):
1092 if value is not None and type(value) != type(''):
1093 raise TypeError, 'new property "%s" not a string'%propname
1095 elif isinstance(prop, Password):
1096 if not isinstance(value, password.Password):
1097 raise TypeError, 'new property "%s" not a Password'%propname
1098 propvalues[propname] = value
1100 elif value is not None and isinstance(prop, Date):
1101 if not isinstance(value, date.Date):
1102 raise TypeError, 'new property "%s" not a Date'% propname
1103 propvalues[propname] = value
1105 elif value is not None and isinstance(prop, Interval):
1106 if not isinstance(value, date.Interval):
1107 raise TypeError, 'new property "%s" not an '\
1108 'Interval'%propname
1109 propvalues[propname] = value
1111 elif value is not None and isinstance(prop, Number):
1112 try:
1113 float(value)
1114 except ValueError:
1115 raise TypeError, 'new property "%s" not numeric'%propname
1117 elif value is not None and isinstance(prop, Boolean):
1118 try:
1119 int(value)
1120 except ValueError:
1121 raise TypeError, 'new property "%s" not boolean'%propname
1123 node[propname] = value
1125 # nothing to do?
1126 if not propvalues:
1127 return propvalues
1129 # do the set, and journal it
1130 self.db.setnode(self.classname, nodeid, node)
1132 if self.do_journal:
1133 propvalues.update(journalvalues)
1134 self.db.addjournal(self.classname, nodeid, 'set', propvalues)
1136 self.fireReactors('set', nodeid, oldvalues)
1138 return propvalues
1140 def retire(self, nodeid):
1141 """Retire a node.
1143 The properties on the node remain available from the get() method,
1144 and the node's id is never reused.
1146 Retired nodes are not returned by the find(), list(), or lookup()
1147 methods, and other nodes may reuse the values of their key properties.
1149 These operations trigger detectors and can be vetoed. Attempts
1150 to modify the "creation" or "activity" properties cause a KeyError.
1151 """
1152 if self.db.journaltag is None:
1153 raise DatabaseError, 'Database open read-only'
1155 self.fireAuditors('retire', nodeid, None)
1157 node = self.db.getnode(self.classname, nodeid)
1158 node[self.db.RETIRED_FLAG] = 1
1159 self.db.setnode(self.classname, nodeid, node)
1160 if self.do_journal:
1161 self.db.addjournal(self.classname, nodeid, 'retired', None)
1163 self.fireReactors('retire', nodeid, None)
1165 def destroy(self, nodeid):
1166 """Destroy a node.
1168 WARNING: this method should never be used except in extremely rare
1169 situations where there could never be links to the node being
1170 deleted
1171 WARNING: use retire() instead
1172 WARNING: the properties of this node will not be available ever again
1173 WARNING: really, use retire() instead
1175 Well, I think that's enough warnings. This method exists mostly to
1176 support the session storage of the cgi interface.
1177 """
1178 if self.db.journaltag is None:
1179 raise DatabaseError, 'Database open read-only'
1180 self.db.destroynode(self.classname, nodeid)
1182 def history(self, nodeid):
1183 """Retrieve the journal of edits on a particular node.
1185 'nodeid' must be the id of an existing node of this class or an
1186 IndexError is raised.
1188 The returned list contains tuples of the form
1190 (date, tag, action, params)
1192 'date' is a Timestamp object specifying the time of the change and
1193 'tag' is the journaltag specified when the database was opened.
1194 """
1195 if not self.do_journal:
1196 raise ValueError, 'Journalling is disabled for this class'
1197 return self.db.getjournal(self.classname, nodeid)
1199 # Locating nodes:
1200 def hasnode(self, nodeid):
1201 '''Determine if the given nodeid actually exists
1202 '''
1203 return self.db.hasnode(self.classname, nodeid)
1205 def setkey(self, propname):
1206 """Select a String property of this class to be the key property.
1208 'propname' must be the name of a String property of this class or
1209 None, or a TypeError is raised. The values of the key property on
1210 all existing nodes must be unique or a ValueError is raised. If the
1211 property doesn't exist, KeyError is raised.
1212 """
1213 prop = self.getprops()[propname]
1214 if not isinstance(prop, String):
1215 raise TypeError, 'key properties must be String'
1216 self.key = propname
1218 def getkey(self):
1219 """Return the name of the key property for this class or None."""
1220 return self.key
1222 def labelprop(self, default_to_id=0):
1223 ''' Return the property name for a label for the given node.
1225 This method attempts to generate a consistent label for the node.
1226 It tries the following in order:
1227 1. key property
1228 2. "name" property
1229 3. "title" property
1230 4. first property from the sorted property name list
1231 '''
1232 k = self.getkey()
1233 if k:
1234 return k
1235 props = self.getprops()
1236 if props.has_key('name'):
1237 return 'name'
1238 elif props.has_key('title'):
1239 return 'title'
1240 if default_to_id:
1241 return 'id'
1242 props = props.keys()
1243 props.sort()
1244 return props[0]
1246 # TODO: set up a separate index db file for this? profile?
1247 def lookup(self, keyvalue):
1248 """Locate a particular node by its key property and return its id.
1250 If this class has no key property, a TypeError is raised. If the
1251 'keyvalue' matches one of the values for the key property among
1252 the nodes in this class, the matching node's id is returned;
1253 otherwise a KeyError is raised.
1254 """
1255 cldb = self.db.getclassdb(self.classname)
1256 try:
1257 for nodeid in self.db.getnodeids(self.classname, cldb):
1258 node = self.db.getnode(self.classname, nodeid, cldb)
1259 if node.has_key(self.db.RETIRED_FLAG):
1260 continue
1261 if node[self.key] == keyvalue:
1262 cldb.close()
1263 return nodeid
1264 finally:
1265 cldb.close()
1266 raise KeyError, keyvalue
1268 # XXX: change from spec - allows multiple props to match
1269 def find(self, **propspec):
1270 """Get the ids of nodes in this class which link to the given nodes.
1272 'propspec' consists of keyword args propname={nodeid:1,}
1273 'propname' must be the name of a property in this class, or a
1274 KeyError is raised. That property must be a Link or Multilink
1275 property, or a TypeError is raised.
1277 Any node in this class whose 'propname' property links to any of the
1278 nodeids will be returned. Used by the full text indexing, which knows
1279 that "foo" occurs in msg1, msg3 and file7, so we have hits on these issues:
1280 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1281 """
1282 propspec = propspec.items()
1283 for propname, nodeids in propspec:
1284 # check the prop is OK
1285 prop = self.properties[propname]
1286 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1287 raise TypeError, "'%s' not a Link/Multilink property"%propname
1289 # ok, now do the find
1290 cldb = self.db.getclassdb(self.classname)
1291 l = []
1292 try:
1293 for id in self.db.getnodeids(self.classname, db=cldb):
1294 node = self.db.getnode(self.classname, id, db=cldb)
1295 if node.has_key(self.db.RETIRED_FLAG):
1296 continue
1297 for propname, nodeids in propspec:
1298 # can't test if the node doesn't have this property
1299 if not node.has_key(propname):
1300 continue
1301 if type(nodeids) is type(''):
1302 nodeids = {nodeids:1}
1303 prop = self.properties[propname]
1304 value = node[propname]
1305 if isinstance(prop, Link) and nodeids.has_key(value):
1306 l.append(id)
1307 break
1308 elif isinstance(prop, Multilink):
1309 hit = 0
1310 for v in value:
1311 if nodeids.has_key(v):
1312 l.append(id)
1313 hit = 1
1314 break
1315 if hit:
1316 break
1317 finally:
1318 cldb.close()
1319 return l
1321 def stringFind(self, **requirements):
1322 """Locate a particular node by matching a set of its String
1323 properties in a caseless search.
1325 If the property is not a String property, a TypeError is raised.
1327 The return is a list of the id of all nodes that match.
1328 """
1329 for propname in requirements.keys():
1330 prop = self.properties[propname]
1331 if isinstance(not prop, String):
1332 raise TypeError, "'%s' not a String property"%propname
1333 requirements[propname] = requirements[propname].lower()
1334 l = []
1335 cldb = self.db.getclassdb(self.classname)
1336 try:
1337 for nodeid in self.db.getnodeids(self.classname, cldb):
1338 node = self.db.getnode(self.classname, nodeid, cldb)
1339 if node.has_key(self.db.RETIRED_FLAG):
1340 continue
1341 for key, value in requirements.items():
1342 if node[key] is None or node[key].lower() != value:
1343 break
1344 else:
1345 l.append(nodeid)
1346 finally:
1347 cldb.close()
1348 return l
1350 def list(self):
1351 """Return a list of the ids of the active nodes in this class."""
1352 l = []
1353 cn = self.classname
1354 cldb = self.db.getclassdb(cn)
1355 try:
1356 for nodeid in self.db.getnodeids(cn, cldb):
1357 node = self.db.getnode(cn, nodeid, cldb)
1358 if node.has_key(self.db.RETIRED_FLAG):
1359 continue
1360 l.append(nodeid)
1361 finally:
1362 cldb.close()
1363 l.sort()
1364 return l
1366 def filter(self, search_matches, filterspec, sort, group,
1367 num_re = re.compile('^\d+$')):
1368 ''' Return a list of the ids of the active nodes in this class that
1369 match the 'filter' spec, sorted by the group spec and then the
1370 sort spec.
1372 "filterspec" is {propname: value(s)}
1373 "sort" is ['+propname', '-propname', 'propname', ...]
1374 "group is ['+propname', '-propname', 'propname', ...]
1375 '''
1376 cn = self.classname
1378 # optimise filterspec
1379 l = []
1380 props = self.getprops()
1381 LINK = 0
1382 MULTILINK = 1
1383 STRING = 2
1384 OTHER = 6
1385 for k, v in filterspec.items():
1386 propclass = props[k]
1387 if isinstance(propclass, Link):
1388 if type(v) is not type([]):
1389 v = [v]
1390 # replace key values with node ids
1391 u = []
1392 link_class = self.db.classes[propclass.classname]
1393 for entry in v:
1394 if entry == '-1': entry = None
1395 elif not num_re.match(entry):
1396 try:
1397 entry = link_class.lookup(entry)
1398 except (TypeError,KeyError):
1399 raise ValueError, 'property "%s": %s not a %s'%(
1400 k, entry, self.properties[k].classname)
1401 u.append(entry)
1403 l.append((LINK, k, u))
1404 elif isinstance(propclass, Multilink):
1405 if type(v) is not type([]):
1406 v = [v]
1407 # replace key values with node ids
1408 u = []
1409 link_class = self.db.classes[propclass.classname]
1410 for entry in v:
1411 if not num_re.match(entry):
1412 try:
1413 entry = link_class.lookup(entry)
1414 except (TypeError,KeyError):
1415 raise ValueError, 'new property "%s": %s not a %s'%(
1416 k, entry, self.properties[k].classname)
1417 u.append(entry)
1418 l.append((MULTILINK, k, u))
1419 elif isinstance(propclass, String):
1420 # simple glob searching
1421 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1422 v = v.replace('?', '.')
1423 v = v.replace('*', '.*?')
1424 l.append((STRING, k, re.compile(v, re.I)))
1425 elif isinstance(propclass, Boolean):
1426 if type(v) is type(''):
1427 bv = v.lower() in ('yes', 'true', 'on', '1')
1428 else:
1429 bv = v
1430 l.append((OTHER, k, bv))
1431 elif isinstance(propclass, Number):
1432 l.append((OTHER, k, int(v)))
1433 else:
1434 l.append((OTHER, k, v))
1435 filterspec = l
1437 # now, find all the nodes that are active and pass filtering
1438 l = []
1439 cldb = self.db.getclassdb(cn)
1440 try:
1441 # TODO: only full-scan once (use items())
1442 for nodeid in self.db.getnodeids(cn, cldb):
1443 node = self.db.getnode(cn, nodeid, cldb)
1444 if node.has_key(self.db.RETIRED_FLAG):
1445 continue
1446 # apply filter
1447 for t, k, v in filterspec:
1448 # make sure the node has the property
1449 if not node.has_key(k):
1450 # this node doesn't have this property, so reject it
1451 break
1453 # now apply the property filter
1454 if t == LINK:
1455 # link - if this node's property doesn't appear in the
1456 # filterspec's nodeid list, skip it
1457 if node[k] not in v:
1458 break
1459 elif t == MULTILINK:
1460 # multilink - if any of the nodeids required by the
1461 # filterspec aren't in this node's property, then skip
1462 # it
1463 have = node[k]
1464 for want in v:
1465 if want not in have:
1466 break
1467 else:
1468 continue
1469 break
1470 elif t == STRING:
1471 # RE search
1472 if node[k] is None or not v.search(node[k]):
1473 break
1474 elif t == OTHER:
1475 # straight value comparison for the other types
1476 if node[k] != v:
1477 break
1478 else:
1479 l.append((nodeid, node))
1480 finally:
1481 cldb.close()
1482 l.sort()
1484 # filter based on full text search
1485 if search_matches is not None:
1486 k = []
1487 for v in l:
1488 if search_matches.has_key(v[0]):
1489 k.append(v)
1490 l = k
1492 # optimise sort
1493 m = []
1494 for entry in sort:
1495 if entry[0] != '-':
1496 m.append(('+', entry))
1497 else:
1498 m.append((entry[0], entry[1:]))
1499 sort = m
1501 # optimise group
1502 m = []
1503 for entry in group:
1504 if entry[0] != '-':
1505 m.append(('+', entry))
1506 else:
1507 m.append((entry[0], entry[1:]))
1508 group = m
1509 # now, sort the result
1510 def sortfun(a, b, sort=sort, group=group, properties=self.getprops(),
1511 db = self.db, cl=self):
1512 a_id, an = a
1513 b_id, bn = b
1514 # sort by group and then sort
1515 for list in group, sort:
1516 for dir, prop in list:
1517 # sorting is class-specific
1518 propclass = properties[prop]
1520 # handle the properties that might be "faked"
1521 # also, handle possible missing properties
1522 try:
1523 if not an.has_key(prop):
1524 an[prop] = cl.get(a_id, prop)
1525 av = an[prop]
1526 except KeyError:
1527 # the node doesn't have a value for this property
1528 if isinstance(propclass, Multilink): av = []
1529 else: av = ''
1530 try:
1531 if not bn.has_key(prop):
1532 bn[prop] = cl.get(b_id, prop)
1533 bv = bn[prop]
1534 except KeyError:
1535 # the node doesn't have a value for this property
1536 if isinstance(propclass, Multilink): bv = []
1537 else: bv = ''
1539 # String and Date values are sorted in the natural way
1540 if isinstance(propclass, String):
1541 # clean up the strings
1542 if av and av[0] in string.uppercase:
1543 av = an[prop] = av.lower()
1544 if bv and bv[0] in string.uppercase:
1545 bv = bn[prop] = bv.lower()
1546 if (isinstance(propclass, String) or
1547 isinstance(propclass, Date)):
1548 # it might be a string that's really an integer
1549 try:
1550 av = int(av)
1551 bv = int(bv)
1552 except:
1553 pass
1554 if dir == '+':
1555 r = cmp(av, bv)
1556 if r != 0: return r
1557 elif dir == '-':
1558 r = cmp(bv, av)
1559 if r != 0: return r
1561 # Link properties are sorted according to the value of
1562 # the "order" property on the linked nodes if it is
1563 # present; or otherwise on the key string of the linked
1564 # nodes; or finally on the node ids.
1565 elif isinstance(propclass, Link):
1566 link = db.classes[propclass.classname]
1567 if av is None and bv is not None: return -1
1568 if av is not None and bv is None: return 1
1569 if av is None and bv is None: continue
1570 if link.getprops().has_key('order'):
1571 if dir == '+':
1572 r = cmp(link.get(av, 'order'),
1573 link.get(bv, 'order'))
1574 if r != 0: return r
1575 elif dir == '-':
1576 r = cmp(link.get(bv, 'order'),
1577 link.get(av, 'order'))
1578 if r != 0: return r
1579 elif link.getkey():
1580 key = link.getkey()
1581 if dir == '+':
1582 r = cmp(link.get(av, key), link.get(bv, key))
1583 if r != 0: return r
1584 elif dir == '-':
1585 r = cmp(link.get(bv, key), link.get(av, key))
1586 if r != 0: return r
1587 else:
1588 if dir == '+':
1589 r = cmp(av, bv)
1590 if r != 0: return r
1591 elif dir == '-':
1592 r = cmp(bv, av)
1593 if r != 0: return r
1595 # Multilink properties are sorted according to how many
1596 # links are present.
1597 elif isinstance(propclass, Multilink):
1598 if dir == '+':
1599 r = cmp(len(av), len(bv))
1600 if r != 0: return r
1601 elif dir == '-':
1602 r = cmp(len(bv), len(av))
1603 if r != 0: return r
1604 elif isinstance(propclass, Number) or isinstance(propclass, Boolean):
1605 if dir == '+':
1606 r = cmp(av, bv)
1607 elif dir == '-':
1608 r = cmp(bv, av)
1610 # end for dir, prop in list:
1611 # end for list in sort, group:
1612 # if all else fails, compare the ids
1613 return cmp(a[0], b[0])
1615 l.sort(sortfun)
1616 return [i[0] for i in l]
1618 def count(self):
1619 """Get the number of nodes in this class.
1621 If the returned integer is 'numnodes', the ids of all the nodes
1622 in this class run from 1 to numnodes, and numnodes+1 will be the
1623 id of the next node to be created in this class.
1624 """
1625 return self.db.countnodes(self.classname)
1627 # Manipulating properties:
1629 def getprops(self, protected=1):
1630 """Return a dictionary mapping property names to property objects.
1631 If the "protected" flag is true, we include protected properties -
1632 those which may not be modified.
1634 In addition to the actual properties on the node, these
1635 methods provide the "creation" and "activity" properties. If the
1636 "protected" flag is true, we include protected properties - those
1637 which may not be modified.
1638 """
1639 d = self.properties.copy()
1640 if protected:
1641 d['id'] = String()
1642 d['creation'] = hyperdb.Date()
1643 d['activity'] = hyperdb.Date()
1644 d['creator'] = hyperdb.Link("user")
1645 return d
1647 def addprop(self, **properties):
1648 """Add properties to this class.
1650 The keyword arguments in 'properties' must map names to property
1651 objects, or a TypeError is raised. None of the keys in 'properties'
1652 may collide with the names of existing properties, or a ValueError
1653 is raised before any properties have been added.
1654 """
1655 for key in properties.keys():
1656 if self.properties.has_key(key):
1657 raise ValueError, key
1658 self.properties.update(properties)
1660 def index(self, nodeid):
1661 '''Add (or refresh) the node to search indexes
1662 '''
1663 # find all the String properties that have indexme
1664 for prop, propclass in self.getprops().items():
1665 if isinstance(propclass, String) and propclass.indexme:
1666 try:
1667 value = str(self.get(nodeid, prop))
1668 except IndexError:
1669 # node no longer exists - entry should be removed
1670 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1671 else:
1672 # and index them under (classname, nodeid, property)
1673 self.db.indexer.add_text((self.classname, nodeid, prop),
1674 value)
1676 #
1677 # Detector interface
1678 #
1679 def audit(self, event, detector):
1680 """Register a detector
1681 """
1682 l = self.auditors[event]
1683 if detector not in l:
1684 self.auditors[event].append(detector)
1686 def fireAuditors(self, action, nodeid, newvalues):
1687 """Fire all registered auditors.
1688 """
1689 for audit in self.auditors[action]:
1690 audit(self.db, self, nodeid, newvalues)
1692 def react(self, event, detector):
1693 """Register a detector
1694 """
1695 l = self.reactors[event]
1696 if detector not in l:
1697 self.reactors[event].append(detector)
1699 def fireReactors(self, action, nodeid, oldvalues):
1700 """Fire all registered reactors.
1701 """
1702 for react in self.reactors[action]:
1703 react(self.db, self, nodeid, oldvalues)
1705 class FileClass(Class):
1706 '''This class defines a large chunk of data. To support this, it has a
1707 mandatory String property "content" which is typically saved off
1708 externally to the hyperdb.
1710 The default MIME type of this data is defined by the
1711 "default_mime_type" class attribute, which may be overridden by each
1712 node if the class defines a "type" String property.
1713 '''
1714 default_mime_type = 'text/plain'
1716 def create(self, **propvalues):
1717 ''' snaffle the file propvalue and store in a file
1718 '''
1719 content = propvalues['content']
1720 del propvalues['content']
1721 newid = Class.create(self, **propvalues)
1722 self.db.storefile(self.classname, newid, None, content)
1723 return newid
1725 def get(self, nodeid, propname, default=_marker, cache=1):
1726 ''' trap the content propname and get it from the file
1727 '''
1729 poss_msg = 'Possibly a access right configuration problem.'
1730 if propname == 'content':
1731 try:
1732 return self.db.getfile(self.classname, nodeid, None)
1733 except IOError, (strerror):
1734 # BUG: by catching this we donot see an error in the log.
1735 return 'ERROR reading file: %s%s\n%s\n%s'%(
1736 self.classname, nodeid, poss_msg, strerror)
1737 if default is not _marker:
1738 return Class.get(self, nodeid, propname, default, cache=cache)
1739 else:
1740 return Class.get(self, nodeid, propname, cache=cache)
1742 def getprops(self, protected=1):
1743 ''' In addition to the actual properties on the node, these methods
1744 provide the "content" property. If the "protected" flag is true,
1745 we include protected properties - those which may not be
1746 modified.
1747 '''
1748 d = Class.getprops(self, protected=protected).copy()
1749 if protected:
1750 d['content'] = hyperdb.String()
1751 return d
1753 def index(self, nodeid):
1754 ''' Index the node in the search index.
1756 We want to index the content in addition to the normal String
1757 property indexing.
1758 '''
1759 # perform normal indexing
1760 Class.index(self, nodeid)
1762 # get the content to index
1763 content = self.get(nodeid, 'content')
1765 # figure the mime type
1766 if self.properties.has_key('type'):
1767 mime_type = self.get(nodeid, 'type')
1768 else:
1769 mime_type = self.default_mime_type
1771 # and index!
1772 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
1773 mime_type)
1775 # XXX deviation from spec - was called ItemClass
1776 class IssueClass(Class, roundupdb.IssueClass):
1777 # Overridden methods:
1778 def __init__(self, db, classname, **properties):
1779 """The newly-created class automatically includes the "messages",
1780 "files", "nosy", and "superseder" properties. If the 'properties'
1781 dictionary attempts to specify any of these properties or a
1782 "creation" or "activity" property, a ValueError is raised.
1783 """
1784 if not properties.has_key('title'):
1785 properties['title'] = hyperdb.String(indexme='yes')
1786 if not properties.has_key('messages'):
1787 properties['messages'] = hyperdb.Multilink("msg")
1788 if not properties.has_key('files'):
1789 properties['files'] = hyperdb.Multilink("file")
1790 if not properties.has_key('nosy'):
1791 properties['nosy'] = hyperdb.Multilink("user")
1792 if not properties.has_key('superseder'):
1793 properties['superseder'] = hyperdb.Multilink(classname)
1794 Class.__init__(self, db, classname, **properties)
1796 #
1797 #$Log: not supported by cvs2svn $
1798 #Revision 1.57 2002/07/31 23:57:36 richard
1799 # . web forms may now unset Link values (like assignedto)
1800 #
1801 #Revision 1.56 2002/07/31 22:04:33 richard
1802 #cleanup
1803 #
1804 #Revision 1.55 2002/07/30 08:22:38 richard
1805 #Session storage in the hyperdb was horribly, horribly inefficient. We use
1806 #a simple anydbm wrapper now - which could be overridden by the metakit
1807 #backend or RDB backend if necessary.
1808 #Much, much better.
1809 #
1810 #Revision 1.54 2002/07/26 08:26:59 richard
1811 #Very close now. The cgi and mailgw now use the new security API. The two
1812 #templates have been migrated to that setup. Lots of unit tests. Still some
1813 #issue in the web form for editing Roles assigned to users.
1814 #
1815 #Revision 1.53 2002/07/25 07:14:06 richard
1816 #Bugger it. Here's the current shape of the new security implementation.
1817 #Still to do:
1818 # . call the security funcs from cgi and mailgw
1819 # . change shipped templates to include correct initialisation and remove
1820 # the old config vars
1821 #... that seems like a lot. The bulk of the work has been done though. Honest :)
1822 #
1823 #Revision 1.52 2002/07/19 03:36:34 richard
1824 #Implemented the destroy() method needed by the session database (and possibly
1825 #others). At the same time, I removed the leading underscores from the hyperdb
1826 #methods that Really Didn't Need Them.
1827 #The journal also raises IndexError now for all situations where there is a
1828 #request for the journal of a node that doesn't have one. It used to return
1829 #[] in _some_ situations, but not all. This _may_ break code, but the tests
1830 #pass...
1831 #
1832 #Revision 1.51 2002/07/18 23:07:08 richard
1833 #Unit tests and a few fixes.
1834 #
1835 #Revision 1.50 2002/07/18 11:50:58 richard
1836 #added tests for number type too
1837 #
1838 #Revision 1.49 2002/07/18 11:41:10 richard
1839 #added tests for boolean type, and fixes to anydbm backend
1840 #
1841 #Revision 1.48 2002/07/18 11:17:31 gmcm
1842 #Add Number and Boolean types to hyperdb.
1843 #Add conversion cases to web, mail & admin interfaces.
1844 #Add storage/serialization cases to back_anydbm & back_metakit.
1845 #
1846 #Revision 1.47 2002/07/14 23:18:20 richard
1847 #. fixed the journal bloat from multilink changes - we just log the add or
1848 # remove operations, not the whole list
1849 #
1850 #Revision 1.46 2002/07/14 06:06:34 richard
1851 #Did some old TODOs
1852 #
1853 #Revision 1.45 2002/07/14 04:03:14 richard
1854 #Implemented a switch to disable journalling for a Class. CGI session
1855 #database now uses it.
1856 #
1857 #Revision 1.44 2002/07/14 02:05:53 richard
1858 #. all storage-specific code (ie. backend) is now implemented by the backends
1859 #
1860 #Revision 1.43 2002/07/10 06:30:30 richard
1861 #...except of course it's nice to use valid Python syntax
1862 #
1863 #Revision 1.42 2002/07/10 06:21:38 richard
1864 #Be extra safe
1865 #
1866 #Revision 1.41 2002/07/10 00:21:45 richard
1867 #explicit database closing
1868 #
1869 #Revision 1.40 2002/07/09 04:19:09 richard
1870 #Added reindex command to roundup-admin.
1871 #Fixed reindex on first access.
1872 #Also fixed reindexing of entries that change.
1873 #
1874 #Revision 1.39 2002/07/09 03:02:52 richard
1875 #More indexer work:
1876 #- all String properties may now be indexed too. Currently there's a bit of
1877 # "issue" specific code in the actual searching which needs to be
1878 # addressed. In a nutshell:
1879 # + pass 'indexme="yes"' as a String() property initialisation arg, eg:
1880 # file = FileClass(db, "file", name=String(), type=String(),
1881 # comment=String(indexme="yes"))
1882 # + the comment will then be indexed and be searchable, with the results
1883 # related back to the issue that the file is linked to
1884 #- as a result of this work, the FileClass has a default MIME type that may
1885 # be overridden in a subclass, or by the use of a "type" property as is
1886 # done in the default templates.
1887 #- the regeneration of the indexes (if necessary) is done once the schema is
1888 # set up in the dbinit.
1889 #
1890 #Revision 1.38 2002/07/08 06:58:15 richard
1891 #cleaned up the indexer code:
1892 # - it splits more words out (much simpler, faster splitter)
1893 # - removed code we'll never use (roundup.roundup_indexer has the full
1894 # implementation, and replaces roundup.indexer)
1895 # - only index text/plain and rfc822/message (ideas for other text formats to
1896 # index are welcome)
1897 # - added simple unit test for indexer. Needs more tests for regression.
1898 #
1899 #Revision 1.37 2002/06/20 23:52:35 richard
1900 #More informative error message
1901 #
1902 #Revision 1.36 2002/06/19 03:07:19 richard
1903 #Moved the file storage commit into blobfiles where it belongs.
1904 #
1905 #Revision 1.35 2002/05/25 07:16:24 rochecompaan
1906 #Merged search_indexing-branch with HEAD
1907 #
1908 #Revision 1.34 2002/05/15 06:21:21 richard
1909 # . node caching now works, and gives a small boost in performance
1910 #
1911 #As a part of this, I cleaned up the DEBUG output and implemented TRACE
1912 #output (HYPERDBTRACE='file to trace to') with checkpoints at the start of
1913 #CGI requests. Run roundup with python -O to skip all the DEBUG/TRACE stuff
1914 #(using if __debug__ which is compiled out with -O)
1915 #
1916 #Revision 1.33 2002/04/24 10:38:26 rochecompaan
1917 #All database files are now created group readable and writable.
1918 #
1919 #Revision 1.32 2002/04/15 23:25:15 richard
1920 #. node ids are now generated from a lockable store - no more race conditions
1921 #
1922 #We're using the portalocker code by Jonathan Feinberg that was contributed
1923 #to the ASPN Python cookbook. This gives us locking across Unix and Windows.
1924 #
1925 #Revision 1.31 2002/04/03 05:54:31 richard
1926 #Fixed serialisation problem by moving the serialisation step out of the
1927 #hyperdb.Class (get, set) into the hyperdb.Database.
1928 #
1929 #Also fixed htmltemplate after the showid changes I made yesterday.
1930 #
1931 #Unit tests for all of the above written.
1932 #
1933 #Revision 1.30.2.1 2002/04/03 11:55:57 rochecompaan
1934 # . Added feature #526730 - search for messages capability
1935 #
1936 #Revision 1.30 2002/02/27 03:40:59 richard
1937 #Ran it through pychecker, made fixes
1938 #
1939 #Revision 1.29 2002/02/25 14:34:31 grubert
1940 # . use blobfiles in back_anydbm which is used in back_bsddb.
1941 # change test_db as dirlist does not work for subdirectories.
1942 # ATTENTION: blobfiles now creates subdirectories for files.
1943 #
1944 #Revision 1.28 2002/02/16 09:14:17 richard
1945 # . #514854 ] History: "User" is always ticket creator
1946 #
1947 #Revision 1.27 2002/01/22 07:21:13 richard
1948 #. fixed back_bsddb so it passed the journal tests
1949 #
1950 #... it didn't seem happy using the back_anydbm _open method, which is odd.
1951 #Yet another occurrance of whichdb not being able to recognise older bsddb
1952 #databases. Yadda yadda. Made the HYPERDBDEBUG stuff more sane in the
1953 #process.
1954 #
1955 #Revision 1.26 2002/01/22 05:18:38 rochecompaan
1956 #last_set_entry was referenced before assignment
1957 #
1958 #Revision 1.25 2002/01/22 05:06:08 rochecompaan
1959 #We need to keep the last 'set' entry in the journal to preserve
1960 #information on 'activity' for nodes.
1961 #
1962 #Revision 1.24 2002/01/21 16:33:20 rochecompaan
1963 #You can now use the roundup-admin tool to pack the database
1964 #
1965 #Revision 1.23 2002/01/18 04:32:04 richard
1966 #Rollback was breaking because a message hadn't actually been written to the file. Needs
1967 #more investigation.
1968 #
1969 #Revision 1.22 2002/01/14 02:20:15 richard
1970 # . changed all config accesses so they access either the instance or the
1971 # config attriubute on the db. This means that all config is obtained from
1972 # instance_config instead of the mish-mash of classes. This will make
1973 # switching to a ConfigParser setup easier too, I hope.
1974 #
1975 #At a minimum, this makes migration a _little_ easier (a lot easier in the
1976 #0.5.0 switch, I hope!)
1977 #
1978 #Revision 1.21 2002/01/02 02:31:38 richard
1979 #Sorry for the huge checkin message - I was only intending to implement #496356
1980 #but I found a number of places where things had been broken by transactions:
1981 # . modified ROUNDUPDBSENDMAILDEBUG to be SENDMAILDEBUG and hold a filename
1982 # for _all_ roundup-generated smtp messages to be sent to.
1983 # . the transaction cache had broken the roundupdb.Class set() reactors
1984 # . newly-created author users in the mailgw weren't being committed to the db
1985 #
1986 #Stuff that made it into CHANGES.txt (ie. the stuff I was actually working
1987 #on when I found that stuff :):
1988 # . #496356 ] Use threading in messages
1989 # . detectors were being registered multiple times
1990 # . added tests for mailgw
1991 # . much better attaching of erroneous messages in the mail gateway
1992 #
1993 #Revision 1.20 2001/12/18 15:30:34 rochecompaan
1994 #Fixed bugs:
1995 # . Fixed file creation and retrieval in same transaction in anydbm
1996 # backend
1997 # . Cgi interface now renders new issue after issue creation
1998 # . Could not set issue status to resolved through cgi interface
1999 # . Mail gateway was changing status back to 'chatting' if status was
2000 # omitted as an argument
2001 #
2002 #Revision 1.19 2001/12/17 03:52:48 richard
2003 #Implemented file store rollback. As a bonus, the hyperdb is now capable of
2004 #storing more than one file per node - if a property name is supplied,
2005 #the file is called designator.property.
2006 #I decided not to migrate the existing files stored over to the new naming
2007 #scheme - the FileClass just doesn't specify the property name.
2008 #
2009 #Revision 1.18 2001/12/16 10:53:38 richard
2010 #take a copy of the node dict so that the subsequent set
2011 #operation doesn't modify the oldvalues structure
2012 #
2013 #Revision 1.17 2001/12/14 23:42:57 richard
2014 #yuck, a gdbm instance tests false :(
2015 #I've left the debugging code in - it should be removed one day if we're ever
2016 #_really_ anal about performace :)
2017 #
2018 #Revision 1.16 2001/12/12 03:23:14 richard
2019 #Cor blimey this anydbm/whichdb stuff is yecchy. Turns out that whichdb
2020 #incorrectly identifies a dbm file as a dbhash file on my system. This has
2021 #been submitted to the python bug tracker as issue #491888:
2022 #https://sourceforge.net/tracker/index.php?func=detail&aid=491888&group_id=5470&atid=105470
2023 #
2024 #Revision 1.15 2001/12/12 02:30:51 richard
2025 #I fixed the problems with people whose anydbm was using the dbm module at the
2026 #backend. It turns out the dbm module modifies the file name to append ".db"
2027 #and my check to determine if we're opening an existing or new db just
2028 #tested os.path.exists() on the filename. Well, no longer! We now perform a
2029 #much better check _and_ cope with the anydbm implementation module changing
2030 #too!
2031 #I also fixed the backends __init__ so only ImportError is squashed.
2032 #
2033 #Revision 1.14 2001/12/10 22:20:01 richard
2034 #Enabled transaction support in the bsddb backend. It uses the anydbm code
2035 #where possible, only replacing methods where the db is opened (it uses the
2036 #btree opener specifically.)
2037 #Also cleaned up some change note generation.
2038 #Made the backends package work with pydoc too.
2039 #
2040 #Revision 1.13 2001/12/02 05:06:16 richard
2041 #. We now use weakrefs in the Classes to keep the database reference, so
2042 # the close() method on the database is no longer needed.
2043 # I bumped the minimum python requirement up to 2.1 accordingly.
2044 #. #487480 ] roundup-server
2045 #. #487476 ] INSTALL.txt
2046 #
2047 #I also cleaned up the change message / post-edit stuff in the cgi client.
2048 #There's now a clearly marked "TODO: append the change note" where I believe
2049 #the change note should be added there. The "changes" list will obviously
2050 #have to be modified to be a dict of the changes, or somesuch.
2051 #
2052 #More testing needed.
2053 #
2054 #Revision 1.12 2001/12/01 07:17:50 richard
2055 #. We now have basic transaction support! Information is only written to
2056 # the database when the commit() method is called. Only the anydbm
2057 # backend is modified in this way - neither of the bsddb backends have been.
2058 # The mail, admin and cgi interfaces all use commit (except the admin tool
2059 # doesn't have a commit command, so interactive users can't commit...)
2060 #. Fixed login/registration forwarding the user to the right page (or not,
2061 # on a failure)
2062 #
2063 #Revision 1.11 2001/11/21 02:34:18 richard
2064 #Added a target version field to the extended issue schema
2065 #
2066 #Revision 1.10 2001/10/09 23:58:10 richard
2067 #Moved the data stringification up into the hyperdb.Class class' get, set
2068 #and create methods. This means that the data is also stringified for the
2069 #journal call, and removes duplication of code from the backends. The
2070 #backend code now only sees strings.
2071 #
2072 #Revision 1.9 2001/10/09 07:25:59 richard
2073 #Added the Password property type. See "pydoc roundup.password" for
2074 #implementation details. Have updated some of the documentation too.
2075 #
2076 #Revision 1.8 2001/09/29 13:27:00 richard
2077 #CGI interfaces now spit up a top-level index of all the instances they can
2078 #serve.
2079 #
2080 #Revision 1.7 2001/08/12 06:32:36 richard
2081 #using isinstance(blah, Foo) now instead of isFooType
2082 #
2083 #Revision 1.6 2001/08/07 00:24:42 richard
2084 #stupid typo
2085 #
2086 #Revision 1.5 2001/08/07 00:15:51 richard
2087 #Added the copyright/license notice to (nearly) all files at request of
2088 #Bizar Software.
2089 #
2090 #Revision 1.4 2001/07/30 01:41:36 richard
2091 #Makes schema changes mucho easier.
2092 #
2093 #Revision 1.3 2001/07/25 01:23:07 richard
2094 #Added the Roundup spec to the new documentation directory.
2095 #
2096 #Revision 1.2 2001/07/23 08:20:44 richard
2097 #Moved over to using marshal in the bsddb and anydbm backends.
2098 #roundup-admin now has a "freshen" command that'll load/save all nodes (not
2099 # retired - mod hyperdb.Class.list() so it lists retired nodes)
2100 #
2101 #