![[tokkee]](http://tokkee.org/images/avatar.png){kind=avatar}
933738dbc6d6a436c398502f4d574fb6fefc4a9c
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: back_anydbm.py,v 1.52 2002-07-19 03:36:34 richard Exp $
19 '''
20 This module defines a backend that saves the hyperdatabase in a database
21 chosen by anydbm. It is guaranteed to always be available in python
22 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
23 serious bugs, and is not available)
24 '''
26 import whichdb, anydbm, os, marshal, re, weakref, string, copy
27 from roundup import hyperdb, date, password, roundupdb
28 from blobfiles import FileStorage
29 from roundup.indexer import Indexer
30 from locking import acquire_lock, release_lock
31 from roundup.hyperdb import String, Password, Date, Interval, Link, \
32 Multilink, DatabaseError, Boolean, Number
34 #
35 # Now the database
36 #
37 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
38 """A database for storing records containing flexible data types.
40 Transaction stuff TODO:
41 . check the timestamp of the class file and nuke the cache if it's
42 modified. Do some sort of conflict checking on the dirty stuff.
43 . perhaps detect write collisions (related to above)?
45 """
46 def __init__(self, config, journaltag=None):
47 """Open a hyperdatabase given a specifier to some storage.
49 The 'storagelocator' is obtained from config.DATABASE.
50 The meaning of 'storagelocator' depends on the particular
51 implementation of the hyperdatabase. It could be a file name,
52 a directory path, a socket descriptor for a connection to a
53 database over the network, etc.
55 The 'journaltag' is a token that will be attached to the journal
56 entries for any edits done on the database. If 'journaltag' is
57 None, the database is opened in read-only mode: the Class.create(),
58 Class.set(), and Class.retire() methods are disabled.
59 """
60 self.config, self.journaltag = config, journaltag
61 self.dir = config.DATABASE
62 self.classes = {}
63 self.cache = {} # cache of nodes loaded or created
64 self.dirtynodes = {} # keep track of the dirty nodes by class
65 self.newnodes = {} # keep track of the new nodes by class
66 self.destroyednodes = {}# keep track of the destroyed nodes by class
67 self.transactions = []
68 self.indexer = Indexer(self.dir)
69 # ensure files are group readable and writable
70 os.umask(0002)
72 def post_init(self):
73 """Called once the schema initialisation has finished."""
74 # reindex the db if necessary
75 if self.indexer.should_reindex():
76 self.reindex()
78 def reindex(self):
79 for klass in self.classes.values():
80 for nodeid in klass.list():
81 klass.index(nodeid)
82 self.indexer.save_index()
84 def __repr__(self):
85 return '<back_anydbm instance at %x>'%id(self)
87 #
88 # Classes
89 #
90 def __getattr__(self, classname):
91 """A convenient way of calling self.getclass(classname)."""
92 if self.classes.has_key(classname):
93 if __debug__:
94 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
95 return self.classes[classname]
96 raise AttributeError, classname
98 def addclass(self, cl):
99 if __debug__:
100 print >>hyperdb.DEBUG, 'addclass', (self, cl)
101 cn = cl.classname
102 if self.classes.has_key(cn):
103 raise ValueError, cn
104 self.classes[cn] = cl
106 def getclasses(self):
107 """Return a list of the names of all existing classes."""
108 if __debug__:
109 print >>hyperdb.DEBUG, 'getclasses', (self,)
110 l = self.classes.keys()
111 l.sort()
112 return l
114 def getclass(self, classname):
115 """Get the Class object representing a particular class.
117 If 'classname' is not a valid class name, a KeyError is raised.
118 """
119 if __debug__:
120 print >>hyperdb.DEBUG, 'getclass', (self, classname)
121 return self.classes[classname]
123 #
124 # Class DBs
125 #
126 def clear(self):
127 '''Delete all database contents
128 '''
129 if __debug__:
130 print >>hyperdb.DEBUG, 'clear', (self,)
131 for cn in self.classes.keys():
132 for dummy in 'nodes', 'journals':
133 path = os.path.join(self.dir, 'journals.%s'%cn)
134 if os.path.exists(path):
135 os.remove(path)
136 elif os.path.exists(path+'.db'): # dbm appends .db
137 os.remove(path+'.db')
139 def getclassdb(self, classname, mode='r'):
140 ''' grab a connection to the class db that will be used for
141 multiple actions
142 '''
143 if __debug__:
144 print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode)
145 return self.opendb('nodes.%s'%classname, mode)
147 def determine_db_type(self, path):
148 ''' determine which DB wrote the class file
149 '''
150 db_type = ''
151 if os.path.exists(path):
152 db_type = whichdb.whichdb(path)
153 if not db_type:
154 raise hyperdb.DatabaseError, "Couldn't identify database type"
155 elif os.path.exists(path+'.db'):
156 # if the path ends in '.db', it's a dbm database, whether
157 # anydbm says it's dbhash or not!
158 db_type = 'dbm'
159 return db_type
161 def opendb(self, name, mode):
162 '''Low-level database opener that gets around anydbm/dbm
163 eccentricities.
164 '''
165 if __debug__:
166 print >>hyperdb.DEBUG, 'opendb', (self, name, mode)
168 # figure the class db type
169 path = os.path.join(os.getcwd(), self.dir, name)
170 db_type = self.determine_db_type(path)
172 # new database? let anydbm pick the best dbm
173 if not db_type:
174 if __debug__:
175 print >>hyperdb.DEBUG, "opendb anydbm.open(%r, 'n')"%path
176 return anydbm.open(path, 'n')
178 # open the database with the correct module
179 try:
180 dbm = __import__(db_type)
181 except ImportError:
182 raise hyperdb.DatabaseError, \
183 "Couldn't open database - the required module '%s'"\
184 " is not available"%db_type
185 if __debug__:
186 print >>hyperdb.DEBUG, "opendb %r.open(%r, %r)"%(db_type, path,
187 mode)
188 return dbm.open(path, mode)
190 def lockdb(self, name):
191 ''' Lock a database file
192 '''
193 path = os.path.join(os.getcwd(), self.dir, '%s.lock'%name)
194 return acquire_lock(path)
196 #
197 # Node IDs
198 #
199 def newid(self, classname):
200 ''' Generate a new id for the given class
201 '''
202 # open the ids DB - create if if doesn't exist
203 lock = self.lockdb('_ids')
204 db = self.opendb('_ids', 'c')
205 if db.has_key(classname):
206 newid = db[classname] = str(int(db[classname]) + 1)
207 else:
208 # the count() bit is transitional - older dbs won't start at 1
209 newid = str(self.getclass(classname).count()+1)
210 db[classname] = newid
211 db.close()
212 release_lock(lock)
213 return newid
215 #
216 # Nodes
217 #
218 def addnode(self, classname, nodeid, node):
219 ''' add the specified node to its class's db
220 '''
221 if __debug__:
222 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
223 self.newnodes.setdefault(classname, {})[nodeid] = 1
224 self.cache.setdefault(classname, {})[nodeid] = node
225 self.savenode(classname, nodeid, node)
227 def setnode(self, classname, nodeid, node):
228 ''' change the specified node
229 '''
230 if __debug__:
231 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
232 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
234 # can't set without having already loaded the node
235 self.cache[classname][nodeid] = node
236 self.savenode(classname, nodeid, node)
238 def savenode(self, classname, nodeid, node):
239 ''' perform the saving of data specified by the set/addnode
240 '''
241 if __debug__:
242 print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node)
243 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
245 def getnode(self, classname, nodeid, db=None, cache=1):
246 ''' get a node from the database
247 '''
248 if __debug__:
249 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid, db)
250 if cache:
251 # try the cache
252 cache_dict = self.cache.setdefault(classname, {})
253 if cache_dict.has_key(nodeid):
254 if __debug__:
255 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
256 nodeid)
257 return cache_dict[nodeid]
259 if __debug__:
260 print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid)
262 # get from the database and save in the cache
263 if db is None:
264 db = self.getclassdb(classname)
265 if not db.has_key(nodeid):
266 raise IndexError, "no such %s %s"%(classname, nodeid)
268 # check the uncommitted, destroyed nodes
269 if (self.destroyednodes.has_key(classname) and
270 self.destroyednodes[classname].has_key(nodeid)):
271 raise IndexError, "no such %s %s"%(classname, nodeid)
273 # decode
274 res = marshal.loads(db[nodeid])
276 # reverse the serialisation
277 res = self.unserialise(classname, res)
279 # store off in the cache dict
280 if cache:
281 cache_dict[nodeid] = res
283 return res
285 def destroynode(self, classname, nodeid):
286 '''Remove a node from the database. Called exclusively by the
287 destroy() method on Class.
288 '''
289 if __debug__:
290 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
292 # remove from cache and newnodes if it's there
293 if (self.cache.has_key(classname) and
294 self.cache[classname].has_key(nodeid)):
295 del self.cache[classname][nodeid]
296 if (self.newnodes.has_key(classname) and
297 self.newnodes[classname].has_key(nodeid)):
298 del self.newnodes[classname][nodeid]
300 # see if there's any obvious commit actions that we should get rid of
301 for entry in self.transactions[:]:
302 if entry[1][:2] == (classname, nodeid):
303 self.transactions.remove(entry)
305 # add to the destroyednodes map
306 self.destroyednodes.setdefault(classname, {})[nodeid] = 1
308 # add the destroy commit action
309 self.transactions.append((self.doDestroyNode, (classname, nodeid)))
311 def serialise(self, classname, node):
312 '''Copy the node contents, converting non-marshallable data into
313 marshallable data.
314 '''
315 if __debug__:
316 print >>hyperdb.DEBUG, 'serialise', classname, node
317 properties = self.getclass(classname).getprops()
318 d = {}
319 for k, v in node.items():
320 # if the property doesn't exist, or is the "retired" flag then
321 # it won't be in the properties dict
322 if not properties.has_key(k):
323 d[k] = v
324 continue
326 # get the property spec
327 prop = properties[k]
329 if isinstance(prop, Password):
330 d[k] = str(v)
331 elif isinstance(prop, Date) and v is not None:
332 d[k] = v.get_tuple()
333 elif isinstance(prop, Interval) and v is not None:
334 d[k] = v.get_tuple()
335 else:
336 d[k] = v
337 return d
339 def unserialise(self, classname, node):
340 '''Decode the marshalled node data
341 '''
342 if __debug__:
343 print >>hyperdb.DEBUG, 'unserialise', classname, node
344 properties = self.getclass(classname).getprops()
345 d = {}
346 for k, v in node.items():
347 # if the property doesn't exist, or is the "retired" flag then
348 # it won't be in the properties dict
349 if not properties.has_key(k):
350 d[k] = v
351 continue
353 # get the property spec
354 prop = properties[k]
356 if isinstance(prop, Date) and v is not None:
357 d[k] = date.Date(v)
358 elif isinstance(prop, Interval) and v is not None:
359 d[k] = date.Interval(v)
360 elif isinstance(prop, Password):
361 p = password.Password()
362 p.unpack(v)
363 d[k] = p
364 else:
365 d[k] = v
366 return d
368 def hasnode(self, classname, nodeid, db=None):
369 ''' determine if the database has a given node
370 '''
371 if __debug__:
372 print >>hyperdb.DEBUG, 'hasnode', (self, classname, nodeid, db)
374 # try the cache
375 cache = self.cache.setdefault(classname, {})
376 if cache.has_key(nodeid):
377 if __debug__:
378 print >>hyperdb.TRACE, 'has %s %s cached'%(classname, nodeid)
379 return 1
380 if __debug__:
381 print >>hyperdb.TRACE, 'has %s %s'%(classname, nodeid)
383 # not in the cache - check the database
384 if db is None:
385 db = self.getclassdb(classname)
386 res = db.has_key(nodeid)
387 return res
389 def countnodes(self, classname, db=None):
390 if __debug__:
391 print >>hyperdb.DEBUG, 'countnodes', (self, classname, db)
393 count = 0
395 # include the uncommitted nodes
396 if self.newnodes.has_key(classname):
397 count += len(self.newnodes[classname])
398 if self.destroyednodes.has_key(classname):
399 count -= len(self.destroyednodes[classname])
401 # and count those in the DB
402 if db is None:
403 db = self.getclassdb(classname)
404 count = count + len(db.keys())
405 return count
407 def getnodeids(self, classname, db=None):
408 if __debug__:
409 print >>hyperdb.DEBUG, 'getnodeids', (self, classname, db)
411 res = []
413 # start off with the new nodes
414 if self.newnodes.has_key(classname):
415 res += self.newnodes[classname].keys()
417 if db is None:
418 db = self.getclassdb(classname)
419 res = res + db.keys()
421 # remove the uncommitted, destroyed nodes
422 if self.destroyednodes.has_key(classname):
423 for nodeid in self.destroyednodes[classname].keys():
424 if db.has_key(nodeid):
425 res.remove(nodeid)
427 return res
430 #
431 # Files - special node properties
432 # inherited from FileStorage
434 #
435 # Journal
436 #
437 def addjournal(self, classname, nodeid, action, params):
438 ''' Journal the Action
439 'action' may be:
441 'create' or 'set' -- 'params' is a dictionary of property values
442 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
443 'retire' -- 'params' is None
444 '''
445 if __debug__:
446 print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid,
447 action, params)
448 self.transactions.append((self.doSaveJournal, (classname, nodeid,
449 action, params)))
451 def getjournal(self, classname, nodeid):
452 ''' get the journal for id
454 Raise IndexError if the node doesn't exist (as per history()'s
455 API)
456 '''
457 if __debug__:
458 print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid)
459 # attempt to open the journal - in some rare cases, the journal may
460 # not exist
461 try:
462 db = self.opendb('journals.%s'%classname, 'r')
463 except anydbm.error, error:
464 if str(error) == "need 'c' or 'n' flag to open new db":
465 raise IndexError, 'no such %s %s'%(classname, nodeid)
466 elif error.args[0] != 2:
467 raise
468 raise IndexError, 'no such %s %s'%(classname, nodeid)
469 try:
470 journal = marshal.loads(db[nodeid])
471 except KeyError:
472 db.close()
473 raise IndexError, 'no such %s %s'%(classname, nodeid)
474 db.close()
475 res = []
476 for nodeid, date_stamp, user, action, params in journal:
477 res.append((nodeid, date.Date(date_stamp), user, action, params))
478 return res
480 def pack(self, pack_before):
481 ''' delete all journal entries before 'pack_before' '''
482 if __debug__:
483 print >>hyperdb.DEBUG, 'packjournal', (self, pack_before)
485 pack_before = pack_before.get_tuple()
487 classes = self.getclasses()
489 # figure the class db type
491 for classname in classes:
492 db_name = 'journals.%s'%classname
493 path = os.path.join(os.getcwd(), self.dir, classname)
494 db_type = self.determine_db_type(path)
495 db = self.opendb(db_name, 'w')
497 for key in db.keys():
498 journal = marshal.loads(db[key])
499 l = []
500 last_set_entry = None
501 for entry in journal:
502 (nodeid, date_stamp, self.journaltag, action,
503 params) = entry
504 if date_stamp > pack_before or action == 'create':
505 l.append(entry)
506 elif action == 'set':
507 # grab the last set entry to keep information on
508 # activity
509 last_set_entry = entry
510 if last_set_entry:
511 date_stamp = last_set_entry[1]
512 # if the last set entry was made after the pack date
513 # then it is already in the list
514 if date_stamp < pack_before:
515 l.append(last_set_entry)
516 db[key] = marshal.dumps(l)
517 if db_type == 'gdbm':
518 db.reorganize()
519 db.close()
522 #
523 # Basic transaction support
524 #
525 def commit(self):
526 ''' Commit the current transactions.
527 '''
528 if __debug__:
529 print >>hyperdb.DEBUG, 'commit', (self,)
530 # TODO: lock the DB
532 # keep a handle to all the database files opened
533 self.databases = {}
535 # now, do all the transactions
536 reindex = {}
537 for method, args in self.transactions:
538 reindex[method(*args)] = 1
540 # now close all the database files
541 for db in self.databases.values():
542 db.close()
543 del self.databases
544 # TODO: unlock the DB
546 # reindex the nodes that request it
547 for classname, nodeid in filter(None, reindex.keys()):
548 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
549 self.getclass(classname).index(nodeid)
551 # save the indexer state
552 self.indexer.save_index()
554 # all transactions committed, back to normal
555 self.cache = {}
556 self.dirtynodes = {}
557 self.newnodes = {}
558 self.destroyednodes = {}
559 self.transactions = []
561 def getCachedClassDB(self, classname):
562 ''' get the class db, looking in our cache of databases for commit
563 '''
564 # get the database handle
565 db_name = 'nodes.%s'%classname
566 if not self.databases.has_key(db_name):
567 self.databases[db_name] = self.getclassdb(classname, 'c')
568 return self.databases[db_name]
570 def doSaveNode(self, classname, nodeid, node):
571 if __debug__:
572 print >>hyperdb.DEBUG, 'doSaveNode', (self, classname, nodeid,
573 node)
575 db = self.getCachedClassDB(classname)
577 # now save the marshalled data
578 db[nodeid] = marshal.dumps(self.serialise(classname, node))
580 # return the classname, nodeid so we reindex this content
581 return (classname, nodeid)
583 def getCachedJournalDB(self, classname):
584 ''' get the journal db, looking in our cache of databases for commit
585 '''
586 # get the database handle
587 db_name = 'journals.%s'%classname
588 if not self.databases.has_key(db_name):
589 self.databases[db_name] = self.opendb(db_name, 'c')
590 return self.databases[db_name]
592 def doSaveJournal(self, classname, nodeid, action, params):
593 # serialise first
594 if action in ('set', 'create'):
595 params = self.serialise(classname, params)
597 # create the journal entry
598 entry = (nodeid, date.Date().get_tuple(), self.journaltag, action,
599 params)
601 if __debug__:
602 print >>hyperdb.DEBUG, 'doSaveJournal', entry
604 db = self.getCachedJournalDB(classname)
606 # now insert the journal entry
607 if db.has_key(nodeid):
608 # append to existing
609 s = db[nodeid]
610 l = marshal.loads(s)
611 l.append(entry)
612 else:
613 l = [entry]
615 db[nodeid] = marshal.dumps(l)
617 def doDestroyNode(self, classname, nodeid):
618 if __debug__:
619 print >>hyperdb.DEBUG, 'doDestroyNode', (self, classname, nodeid)
621 # delete from the class database
622 db = self.getCachedClassDB(classname)
623 if db.has_key(nodeid):
624 del db[nodeid]
626 # delete from the database
627 db = self.getCachedJournalDB(classname)
628 if db.has_key(nodeid):
629 del db[nodeid]
631 # return the classname, nodeid so we reindex this content
632 return (classname, nodeid)
634 def rollback(self):
635 ''' Reverse all actions from the current transaction.
636 '''
637 if __debug__:
638 print >>hyperdb.DEBUG, 'rollback', (self, )
639 for method, args in self.transactions:
640 # delete temporary files
641 if method == self.doStoreFile:
642 self.rollbackStoreFile(*args)
643 self.cache = {}
644 self.dirtynodes = {}
645 self.newnodes = {}
646 self.destroyednodes = {}
647 self.transactions = []
649 _marker = []
650 class Class(hyperdb.Class):
651 """The handle to a particular class of nodes in a hyperdatabase."""
653 def __init__(self, db, classname, **properties):
654 """Create a new class with a given name and property specification.
656 'classname' must not collide with the name of an existing class,
657 or a ValueError is raised. The keyword arguments in 'properties'
658 must map names to property objects, or a TypeError is raised.
659 """
660 if (properties.has_key('creation') or properties.has_key('activity')
661 or properties.has_key('creator')):
662 raise ValueError, '"creation", "activity" and "creator" are '\
663 'reserved'
665 self.classname = classname
666 self.properties = properties
667 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
668 self.key = ''
670 # should we journal changes (default yes)
671 self.do_journal = 1
673 # do the db-related init stuff
674 db.addclass(self)
676 self.auditors = {'create': [], 'set': [], 'retire': []}
677 self.reactors = {'create': [], 'set': [], 'retire': []}
679 def enableJournalling(self):
680 '''Turn journalling on for this class
681 '''
682 self.do_journal = 1
684 def disableJournalling(self):
685 '''Turn journalling off for this class
686 '''
687 self.do_journal = 0
689 # Editing nodes:
691 def create(self, **propvalues):
692 """Create a new node of this class and return its id.
694 The keyword arguments in 'propvalues' map property names to values.
696 The values of arguments must be acceptable for the types of their
697 corresponding properties or a TypeError is raised.
699 If this class has a key property, it must be present and its value
700 must not collide with other key strings or a ValueError is raised.
702 Any other properties on this class that are missing from the
703 'propvalues' dictionary are set to None.
705 If an id in a link or multilink property does not refer to a valid
706 node, an IndexError is raised.
708 These operations trigger detectors and can be vetoed. Attempts
709 to modify the "creation" or "activity" properties cause a KeyError.
710 """
711 if propvalues.has_key('id'):
712 raise KeyError, '"id" is reserved'
714 if self.db.journaltag is None:
715 raise DatabaseError, 'Database open read-only'
717 if propvalues.has_key('creation') or propvalues.has_key('activity'):
718 raise KeyError, '"creation" and "activity" are reserved'
720 self.fireAuditors('create', None, propvalues)
722 # new node's id
723 newid = self.db.newid(self.classname)
725 # validate propvalues
726 num_re = re.compile('^\d+$')
727 for key, value in propvalues.items():
728 if key == self.key:
729 try:
730 self.lookup(value)
731 except KeyError:
732 pass
733 else:
734 raise ValueError, 'node with key "%s" exists'%value
736 # try to handle this property
737 try:
738 prop = self.properties[key]
739 except KeyError:
740 raise KeyError, '"%s" has no property "%s"'%(self.classname,
741 key)
743 if isinstance(prop, Link):
744 if type(value) != type(''):
745 raise ValueError, 'link value must be String'
746 link_class = self.properties[key].classname
747 # if it isn't a number, it's a key
748 if not num_re.match(value):
749 try:
750 value = self.db.classes[link_class].lookup(value)
751 except (TypeError, KeyError):
752 raise IndexError, 'new property "%s": %s not a %s'%(
753 key, value, link_class)
754 elif not self.db.hasnode(link_class, value):
755 raise IndexError, '%s has no node %s'%(link_class, value)
757 # save off the value
758 propvalues[key] = value
760 # register the link with the newly linked node
761 if self.do_journal and self.properties[key].do_journal:
762 self.db.addjournal(link_class, value, 'link',
763 (self.classname, newid, key))
765 elif isinstance(prop, Multilink):
766 if type(value) != type([]):
767 raise TypeError, 'new property "%s" not a list of ids'%key
769 # clean up and validate the list of links
770 link_class = self.properties[key].classname
771 l = []
772 for entry in value:
773 if type(entry) != type(''):
774 raise ValueError, '"%s" link value (%s) must be '\
775 'String'%(key, value)
776 # if it isn't a number, it's a key
777 if not num_re.match(entry):
778 try:
779 entry = self.db.classes[link_class].lookup(entry)
780 except (TypeError, KeyError):
781 raise IndexError, 'new property "%s": %s not a %s'%(
782 key, entry, self.properties[key].classname)
783 l.append(entry)
784 value = l
785 propvalues[key] = value
787 # handle additions
788 for id in value:
789 if not self.db.hasnode(link_class, id):
790 raise IndexError, '%s has no node %s'%(link_class, id)
791 # register the link with the newly linked node
792 if self.do_journal and self.properties[key].do_journal:
793 self.db.addjournal(link_class, id, 'link',
794 (self.classname, newid, key))
796 elif isinstance(prop, String):
797 if type(value) != type(''):
798 raise TypeError, 'new property "%s" not a string'%key
800 elif isinstance(prop, Password):
801 if not isinstance(value, password.Password):
802 raise TypeError, 'new property "%s" not a Password'%key
804 elif isinstance(prop, Date):
805 if value is not None and not isinstance(value, date.Date):
806 raise TypeError, 'new property "%s" not a Date'%key
808 elif isinstance(prop, Interval):
809 if value is not None and not isinstance(value, date.Interval):
810 raise TypeError, 'new property "%s" not an Interval'%key
812 elif value is not None and isinstance(prop, Number):
813 try:
814 float(value)
815 except ValueError:
816 raise TypeError, 'new property "%s" not numeric'%key
818 elif value is not None and isinstance(prop, Boolean):
819 try:
820 int(value)
821 except ValueError:
822 raise TypeError, 'new property "%s" not boolean'%key
824 # make sure there's data where there needs to be
825 for key, prop in self.properties.items():
826 if propvalues.has_key(key):
827 continue
828 if key == self.key:
829 raise ValueError, 'key property "%s" is required'%key
830 if isinstance(prop, Multilink):
831 propvalues[key] = []
832 else:
833 propvalues[key] = None
835 # done
836 self.db.addnode(self.classname, newid, propvalues)
837 if self.do_journal:
838 self.db.addjournal(self.classname, newid, 'create', propvalues)
840 self.fireReactors('create', newid, None)
842 return newid
844 def get(self, nodeid, propname, default=_marker, cache=1):
845 """Get the value of a property on an existing node of this class.
847 'nodeid' must be the id of an existing node of this class or an
848 IndexError is raised. 'propname' must be the name of a property
849 of this class or a KeyError is raised.
851 'cache' indicates whether the transaction cache should be queried
852 for the node. If the node has been modified and you need to
853 determine what its values prior to modification are, you need to
854 set cache=0.
856 Attempts to get the "creation" or "activity" properties should
857 do the right thing.
858 """
859 if propname == 'id':
860 return nodeid
862 if propname == 'creation':
863 if not self.do_journal:
864 raise ValueError, 'Journalling is disabled for this class'
865 journal = self.db.getjournal(self.classname, nodeid)
866 if journal:
867 return self.db.getjournal(self.classname, nodeid)[0][1]
868 else:
869 # on the strange chance that there's no journal
870 return date.Date()
871 if propname == 'activity':
872 if not self.do_journal:
873 raise ValueError, 'Journalling is disabled for this class'
874 journal = self.db.getjournal(self.classname, nodeid)
875 if journal:
876 return self.db.getjournal(self.classname, nodeid)[-1][1]
877 else:
878 # on the strange chance that there's no journal
879 return date.Date()
880 if propname == 'creator':
881 if not self.do_journal:
882 raise ValueError, 'Journalling is disabled for this class'
883 journal = self.db.getjournal(self.classname, nodeid)
884 if journal:
885 name = self.db.getjournal(self.classname, nodeid)[0][2]
886 else:
887 return None
888 return self.db.user.lookup(name)
890 # get the property (raises KeyErorr if invalid)
891 prop = self.properties[propname]
893 # get the node's dict
894 d = self.db.getnode(self.classname, nodeid, cache=cache)
896 if not d.has_key(propname):
897 if default is _marker:
898 if isinstance(prop, Multilink):
899 return []
900 else:
901 return None
902 else:
903 return default
905 return d[propname]
907 # XXX not in spec
908 def getnode(self, nodeid, cache=1):
909 ''' Return a convenience wrapper for the node.
911 'nodeid' must be the id of an existing node of this class or an
912 IndexError is raised.
914 'cache' indicates whether the transaction cache should be queried
915 for the node. If the node has been modified and you need to
916 determine what its values prior to modification are, you need to
917 set cache=0.
918 '''
919 return Node(self, nodeid, cache=cache)
921 def set(self, nodeid, **propvalues):
922 """Modify a property on an existing node of this class.
924 'nodeid' must be the id of an existing node of this class or an
925 IndexError is raised.
927 Each key in 'propvalues' must be the name of a property of this
928 class or a KeyError is raised.
930 All values in 'propvalues' must be acceptable types for their
931 corresponding properties or a TypeError is raised.
933 If the value of the key property is set, it must not collide with
934 other key strings or a ValueError is raised.
936 If the value of a Link or Multilink property contains an invalid
937 node id, a ValueError is raised.
939 These operations trigger detectors and can be vetoed. Attempts
940 to modify the "creation" or "activity" properties cause a KeyError.
941 """
942 if not propvalues:
943 return
945 if propvalues.has_key('creation') or propvalues.has_key('activity'):
946 raise KeyError, '"creation" and "activity" are reserved'
948 if propvalues.has_key('id'):
949 raise KeyError, '"id" is reserved'
951 if self.db.journaltag is None:
952 raise DatabaseError, 'Database open read-only'
954 self.fireAuditors('set', nodeid, propvalues)
955 # Take a copy of the node dict so that the subsequent set
956 # operation doesn't modify the oldvalues structure.
957 try:
958 # try not using the cache initially
959 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid,
960 cache=0))
961 except IndexError:
962 # this will be needed if somone does a create() and set()
963 # with no intervening commit()
964 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
966 node = self.db.getnode(self.classname, nodeid)
967 if node.has_key(self.db.RETIRED_FLAG):
968 raise IndexError
969 num_re = re.compile('^\d+$')
971 # if the journal value is to be different, store it in here
972 journalvalues = {}
974 for propname, value in propvalues.items():
975 # check to make sure we're not duplicating an existing key
976 if propname == self.key and node[propname] != value:
977 try:
978 self.lookup(value)
979 except KeyError:
980 pass
981 else:
982 raise ValueError, 'node with key "%s" exists'%value
984 # this will raise the KeyError if the property isn't valid
985 # ... we don't use getprops() here because we only care about
986 # the writeable properties.
987 prop = self.properties[propname]
989 # if the value's the same as the existing value, no sense in
990 # doing anything
991 if node.has_key(propname) and value == node[propname]:
992 del propvalues[propname]
993 continue
995 # do stuff based on the prop type
996 if isinstance(prop, Link):
997 link_class = self.properties[propname].classname
998 # if it isn't a number, it's a key
999 if type(value) != type(''):
1000 raise ValueError, 'link value must be String'
1001 if not num_re.match(value):
1002 try:
1003 value = self.db.classes[link_class].lookup(value)
1004 except (TypeError, KeyError):
1005 raise IndexError, 'new property "%s": %s not a %s'%(
1006 propname, value, self.properties[propname].classname)
1008 if not self.db.hasnode(link_class, value):
1009 raise IndexError, '%s has no node %s'%(link_class, value)
1011 if self.do_journal and self.properties[propname].do_journal:
1012 # register the unlink with the old linked node
1013 if node[propname] is not None:
1014 self.db.addjournal(link_class, node[propname], 'unlink',
1015 (self.classname, nodeid, propname))
1017 # register the link with the newly linked node
1018 if value is not None:
1019 self.db.addjournal(link_class, value, 'link',
1020 (self.classname, nodeid, propname))
1022 elif isinstance(prop, Multilink):
1023 if type(value) != type([]):
1024 raise TypeError, 'new property "%s" not a list of'\
1025 ' ids'%propname
1026 link_class = self.properties[propname].classname
1027 l = []
1028 for entry in value:
1029 # if it isn't a number, it's a key
1030 if type(entry) != type(''):
1031 raise ValueError, 'new property "%s" link value ' \
1032 'must be a string'%propname
1033 if not num_re.match(entry):
1034 try:
1035 entry = self.db.classes[link_class].lookup(entry)
1036 except (TypeError, KeyError):
1037 raise IndexError, 'new property "%s": %s not a %s'%(
1038 propname, entry,
1039 self.properties[propname].classname)
1040 l.append(entry)
1041 value = l
1042 propvalues[propname] = value
1044 # figure the journal entry for this property
1045 add = []
1046 remove = []
1048 # handle removals
1049 if node.has_key(propname):
1050 l = node[propname]
1051 else:
1052 l = []
1053 for id in l[:]:
1054 if id in value:
1055 continue
1056 # register the unlink with the old linked node
1057 if self.do_journal and self.properties[propname].do_journal:
1058 self.db.addjournal(link_class, id, 'unlink',
1059 (self.classname, nodeid, propname))
1060 l.remove(id)
1061 remove.append(id)
1063 # handle additions
1064 for id in value:
1065 if not self.db.hasnode(link_class, id):
1066 raise IndexError, '%s has no node %s'%(link_class, id)
1067 if id in l:
1068 continue
1069 # register the link with the newly linked node
1070 if self.do_journal and self.properties[propname].do_journal:
1071 self.db.addjournal(link_class, id, 'link',
1072 (self.classname, nodeid, propname))
1073 l.append(id)
1074 add.append(id)
1076 # figure the journal entry
1077 l = []
1078 if add:
1079 l.append(('add', add))
1080 if remove:
1081 l.append(('remove', remove))
1082 if l:
1083 journalvalues[propname] = tuple(l)
1085 elif isinstance(prop, String):
1086 if value is not None and type(value) != type(''):
1087 raise TypeError, 'new property "%s" not a string'%propname
1089 elif isinstance(prop, Password):
1090 if not isinstance(value, password.Password):
1091 raise TypeError, 'new property "%s" not a Password'%propname
1092 propvalues[propname] = value
1094 elif value is not None and isinstance(prop, Date):
1095 if not isinstance(value, date.Date):
1096 raise TypeError, 'new property "%s" not a Date'% propname
1097 propvalues[propname] = value
1099 elif value is not None and isinstance(prop, Interval):
1100 if not isinstance(value, date.Interval):
1101 raise TypeError, 'new property "%s" not an '\
1102 'Interval'%propname
1103 propvalues[propname] = value
1105 elif value is not None and isinstance(prop, Number):
1106 try:
1107 float(value)
1108 except ValueError:
1109 raise TypeError, 'new property "%s" not numeric'%propname
1111 elif value is not None and isinstance(prop, Boolean):
1112 try:
1113 int(value)
1114 except ValueError:
1115 raise TypeError, 'new property "%s" not boolean'%propname
1117 node[propname] = value
1119 # nothing to do?
1120 if not propvalues:
1121 return
1123 # do the set, and journal it
1124 self.db.setnode(self.classname, nodeid, node)
1126 if self.do_journal:
1127 propvalues.update(journalvalues)
1128 self.db.addjournal(self.classname, nodeid, 'set', propvalues)
1130 self.fireReactors('set', nodeid, oldvalues)
1132 def retire(self, nodeid):
1133 """Retire a node.
1135 The properties on the node remain available from the get() method,
1136 and the node's id is never reused.
1138 Retired nodes are not returned by the find(), list(), or lookup()
1139 methods, and other nodes may reuse the values of their key properties.
1141 These operations trigger detectors and can be vetoed. Attempts
1142 to modify the "creation" or "activity" properties cause a KeyError.
1143 """
1144 if self.db.journaltag is None:
1145 raise DatabaseError, 'Database open read-only'
1147 self.fireAuditors('retire', nodeid, None)
1149 node = self.db.getnode(self.classname, nodeid)
1150 node[self.db.RETIRED_FLAG] = 1
1151 self.db.setnode(self.classname, nodeid, node)
1152 if self.do_journal:
1153 self.db.addjournal(self.classname, nodeid, 'retired', None)
1155 self.fireReactors('retire', nodeid, None)
1157 def destroy(self, nodeid):
1158 """Destroy a node.
1160 WARNING: this method should never be used except in extremely rare
1161 situations where there could never be links to the node being
1162 deleted
1163 WARNING: use retire() instead
1164 WARNING: the properties of this node will not be available ever again
1165 WARNING: really, use retire() instead
1167 Well, I think that's enough warnings. This method exists mostly to
1168 support the session storage of the cgi interface.
1169 """
1170 if self.db.journaltag is None:
1171 raise DatabaseError, 'Database open read-only'
1172 self.db.destroynode(self.classname, nodeid)
1174 def history(self, nodeid):
1175 """Retrieve the journal of edits on a particular node.
1177 'nodeid' must be the id of an existing node of this class or an
1178 IndexError is raised.
1180 The returned list contains tuples of the form
1182 (date, tag, action, params)
1184 'date' is a Timestamp object specifying the time of the change and
1185 'tag' is the journaltag specified when the database was opened.
1186 """
1187 if not self.do_journal:
1188 raise ValueError, 'Journalling is disabled for this class'
1189 return self.db.getjournal(self.classname, nodeid)
1191 # Locating nodes:
1192 def hasnode(self, nodeid):
1193 '''Determine if the given nodeid actually exists
1194 '''
1195 return self.db.hasnode(self.classname, nodeid)
1197 def setkey(self, propname):
1198 """Select a String property of this class to be the key property.
1200 'propname' must be the name of a String property of this class or
1201 None, or a TypeError is raised. The values of the key property on
1202 all existing nodes must be unique or a ValueError is raised. If the
1203 property doesn't exist, KeyError is raised.
1204 """
1205 prop = self.getprops()[propname]
1206 if not isinstance(prop, String):
1207 raise TypeError, 'key properties must be String'
1208 self.key = propname
1210 def getkey(self):
1211 """Return the name of the key property for this class or None."""
1212 return self.key
1214 def labelprop(self, default_to_id=0):
1215 ''' Return the property name for a label for the given node.
1217 This method attempts to generate a consistent label for the node.
1218 It tries the following in order:
1219 1. key property
1220 2. "name" property
1221 3. "title" property
1222 4. first property from the sorted property name list
1223 '''
1224 k = self.getkey()
1225 if k:
1226 return k
1227 props = self.getprops()
1228 if props.has_key('name'):
1229 return 'name'
1230 elif props.has_key('title'):
1231 return 'title'
1232 if default_to_id:
1233 return 'id'
1234 props = props.keys()
1235 props.sort()
1236 return props[0]
1238 # TODO: set up a separate index db file for this? profile?
1239 def lookup(self, keyvalue):
1240 """Locate a particular node by its key property and return its id.
1242 If this class has no key property, a TypeError is raised. If the
1243 'keyvalue' matches one of the values for the key property among
1244 the nodes in this class, the matching node's id is returned;
1245 otherwise a KeyError is raised.
1246 """
1247 cldb = self.db.getclassdb(self.classname)
1248 try:
1249 for nodeid in self.db.getnodeids(self.classname, cldb):
1250 node = self.db.getnode(self.classname, nodeid, cldb)
1251 if node.has_key(self.db.RETIRED_FLAG):
1252 continue
1253 if node[self.key] == keyvalue:
1254 cldb.close()
1255 return nodeid
1256 finally:
1257 cldb.close()
1258 raise KeyError, keyvalue
1260 # XXX: change from spec - allows multiple props to match
1261 def find(self, **propspec):
1262 """Get the ids of nodes in this class which link to the given nodes.
1264 'propspec' consists of keyword args propname={nodeid:1,}
1265 'propname' must be the name of a property in this class, or a
1266 KeyError is raised. That property must be a Link or Multilink
1267 property, or a TypeError is raised.
1269 Any node in this class whose 'propname' property links to any of the
1270 nodeids will be returned. Used by the full text indexing, which knows
1271 that "foo" occurs in msg1, msg3 and file7, so we have hits on these issues:
1272 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1273 """
1274 propspec = propspec.items()
1275 for propname, nodeids in propspec:
1276 # check the prop is OK
1277 prop = self.properties[propname]
1278 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1279 raise TypeError, "'%s' not a Link/Multilink property"%propname
1280 #XXX edit is expensive and of questionable use
1281 #for nodeid in nodeids:
1282 # if not self.db.hasnode(prop.classname, nodeid):
1283 # raise ValueError, '%s has no node %s'%(prop.classname, nodeid)
1285 # ok, now do the find
1286 cldb = self.db.getclassdb(self.classname)
1287 l = []
1288 try:
1289 for id in self.db.getnodeids(self.classname, db=cldb):
1290 node = self.db.getnode(self.classname, id, db=cldb)
1291 if node.has_key(self.db.RETIRED_FLAG):
1292 continue
1293 for propname, nodeids in propspec:
1294 # can't test if the node doesn't have this property
1295 if not node.has_key(propname):
1296 continue
1297 if type(nodeids) is type(''):
1298 nodeids = {nodeids:1}
1299 prop = self.properties[propname]
1300 value = node[propname]
1301 if isinstance(prop, Link) and nodeids.has_key(value):
1302 l.append(id)
1303 break
1304 elif isinstance(prop, Multilink):
1305 hit = 0
1306 for v in value:
1307 if nodeids.has_key(v):
1308 l.append(id)
1309 hit = 1
1310 break
1311 if hit:
1312 break
1313 finally:
1314 cldb.close()
1315 return l
1317 def stringFind(self, **requirements):
1318 """Locate a particular node by matching a set of its String
1319 properties in a caseless search.
1321 If the property is not a String property, a TypeError is raised.
1323 The return is a list of the id of all nodes that match.
1324 """
1325 for propname in requirements.keys():
1326 prop = self.properties[propname]
1327 if isinstance(not prop, String):
1328 raise TypeError, "'%s' not a String property"%propname
1329 requirements[propname] = requirements[propname].lower()
1330 l = []
1331 cldb = self.db.getclassdb(self.classname)
1332 try:
1333 for nodeid in self.db.getnodeids(self.classname, cldb):
1334 node = self.db.getnode(self.classname, nodeid, cldb)
1335 if node.has_key(self.db.RETIRED_FLAG):
1336 continue
1337 for key, value in requirements.items():
1338 if node[key] and node[key].lower() != value:
1339 break
1340 else:
1341 l.append(nodeid)
1342 finally:
1343 cldb.close()
1344 return l
1346 def list(self):
1347 """Return a list of the ids of the active nodes in this class."""
1348 l = []
1349 cn = self.classname
1350 cldb = self.db.getclassdb(cn)
1351 try:
1352 for nodeid in self.db.getnodeids(cn, cldb):
1353 node = self.db.getnode(cn, nodeid, cldb)
1354 if node.has_key(self.db.RETIRED_FLAG):
1355 continue
1356 l.append(nodeid)
1357 finally:
1358 cldb.close()
1359 l.sort()
1360 return l
1362 # XXX not in spec
1363 def filter(self, search_matches, filterspec, sort, group,
1364 num_re = re.compile('^\d+$')):
1365 ''' Return a list of the ids of the active nodes in this class that
1366 match the 'filter' spec, sorted by the group spec and then the
1367 sort spec
1368 '''
1369 cn = self.classname
1371 # optimise filterspec
1372 l = []
1373 props = self.getprops()
1374 for k, v in filterspec.items():
1375 propclass = props[k]
1376 if isinstance(propclass, Link):
1377 if type(v) is not type([]):
1378 v = [v]
1379 # replace key values with node ids
1380 u = []
1381 link_class = self.db.classes[propclass.classname]
1382 for entry in v:
1383 if entry == '-1': entry = None
1384 elif not num_re.match(entry):
1385 try:
1386 entry = link_class.lookup(entry)
1387 except (TypeError,KeyError):
1388 raise ValueError, 'property "%s": %s not a %s'%(
1389 k, entry, self.properties[k].classname)
1390 u.append(entry)
1392 l.append((0, k, u))
1393 elif isinstance(propclass, Multilink):
1394 if type(v) is not type([]):
1395 v = [v]
1396 # replace key values with node ids
1397 u = []
1398 link_class = self.db.classes[propclass.classname]
1399 for entry in v:
1400 if not num_re.match(entry):
1401 try:
1402 entry = link_class.lookup(entry)
1403 except (TypeError,KeyError):
1404 raise ValueError, 'new property "%s": %s not a %s'%(
1405 k, entry, self.properties[k].classname)
1406 u.append(entry)
1407 l.append((1, k, u))
1408 elif isinstance(propclass, String):
1409 # simple glob searching
1410 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1411 v = v.replace('?', '.')
1412 v = v.replace('*', '.*?')
1413 l.append((2, k, re.compile(v, re.I)))
1414 elif isinstance(propclass, Boolean):
1415 if type(v) is type(''):
1416 bv = v.lower() in ('yes', 'true', 'on', '1')
1417 else:
1418 bv = v
1419 l.append((6, k, bv))
1420 elif isinstance(propclass, Number):
1421 l.append((6, k, int(v)))
1422 else:
1423 l.append((6, k, v))
1424 filterspec = l
1426 # now, find all the nodes that are active and pass filtering
1427 l = []
1428 cldb = self.db.getclassdb(cn)
1429 try:
1430 for nodeid in self.db.getnodeids(cn, cldb):
1431 node = self.db.getnode(cn, nodeid, cldb)
1432 if node.has_key(self.db.RETIRED_FLAG):
1433 continue
1434 # apply filter
1435 for t, k, v in filterspec:
1436 # this node doesn't have this property, so reject it
1437 if not node.has_key(k): break
1439 if t == 0 and node[k] not in v:
1440 # link - if this node'd property doesn't appear in the
1441 # filterspec's nodeid list, skip it
1442 break
1443 elif t == 1:
1444 # multilink - if any of the nodeids required by the
1445 # filterspec aren't in this node's property, then skip
1446 # it
1447 for value in v:
1448 if value not in node[k]:
1449 break
1450 else:
1451 continue
1452 break
1453 elif t == 2 and (node[k] is None or not v.search(node[k])):
1454 # RE search
1455 break
1456 elif t == 6 and node[k] != v:
1457 # straight value comparison for the other types
1458 break
1459 else:
1460 l.append((nodeid, node))
1461 finally:
1462 cldb.close()
1463 l.sort()
1465 # filter based on full text search
1466 if search_matches is not None:
1467 k = []
1468 l_debug = []
1469 for v in l:
1470 l_debug.append(v[0])
1471 if search_matches.has_key(v[0]):
1472 k.append(v)
1473 l = k
1475 # optimise sort
1476 m = []
1477 for entry in sort:
1478 if entry[0] != '-':
1479 m.append(('+', entry))
1480 else:
1481 m.append((entry[0], entry[1:]))
1482 sort = m
1484 # optimise group
1485 m = []
1486 for entry in group:
1487 if entry[0] != '-':
1488 m.append(('+', entry))
1489 else:
1490 m.append((entry[0], entry[1:]))
1491 group = m
1492 # now, sort the result
1493 def sortfun(a, b, sort=sort, group=group, properties=self.getprops(),
1494 db = self.db, cl=self):
1495 a_id, an = a
1496 b_id, bn = b
1497 # sort by group and then sort
1498 for list in group, sort:
1499 for dir, prop in list:
1500 # sorting is class-specific
1501 propclass = properties[prop]
1503 # handle the properties that might be "faked"
1504 # also, handle possible missing properties
1505 try:
1506 if not an.has_key(prop):
1507 an[prop] = cl.get(a_id, prop)
1508 av = an[prop]
1509 except KeyError:
1510 # the node doesn't have a value for this property
1511 if isinstance(propclass, Multilink): av = []
1512 else: av = ''
1513 try:
1514 if not bn.has_key(prop):
1515 bn[prop] = cl.get(b_id, prop)
1516 bv = bn[prop]
1517 except KeyError:
1518 # the node doesn't have a value for this property
1519 if isinstance(propclass, Multilink): bv = []
1520 else: bv = ''
1522 # String and Date values are sorted in the natural way
1523 if isinstance(propclass, String):
1524 # clean up the strings
1525 if av and av[0] in string.uppercase:
1526 av = an[prop] = av.lower()
1527 if bv and bv[0] in string.uppercase:
1528 bv = bn[prop] = bv.lower()
1529 if (isinstance(propclass, String) or
1530 isinstance(propclass, Date)):
1531 # it might be a string that's really an integer
1532 try:
1533 av = int(av)
1534 bv = int(bv)
1535 except:
1536 pass
1537 if dir == '+':
1538 r = cmp(av, bv)
1539 if r != 0: return r
1540 elif dir == '-':
1541 r = cmp(bv, av)
1542 if r != 0: return r
1544 # Link properties are sorted according to the value of
1545 # the "order" property on the linked nodes if it is
1546 # present; or otherwise on the key string of the linked
1547 # nodes; or finally on the node ids.
1548 elif isinstance(propclass, Link):
1549 link = db.classes[propclass.classname]
1550 if av is None and bv is not None: return -1
1551 if av is not None and bv is None: return 1
1552 if av is None and bv is None: continue
1553 if link.getprops().has_key('order'):
1554 if dir == '+':
1555 r = cmp(link.get(av, 'order'),
1556 link.get(bv, 'order'))
1557 if r != 0: return r
1558 elif dir == '-':
1559 r = cmp(link.get(bv, 'order'),
1560 link.get(av, 'order'))
1561 if r != 0: return r
1562 elif link.getkey():
1563 key = link.getkey()
1564 if dir == '+':
1565 r = cmp(link.get(av, key), link.get(bv, key))
1566 if r != 0: return r
1567 elif dir == '-':
1568 r = cmp(link.get(bv, key), link.get(av, key))
1569 if r != 0: return r
1570 else:
1571 if dir == '+':
1572 r = cmp(av, bv)
1573 if r != 0: return r
1574 elif dir == '-':
1575 r = cmp(bv, av)
1576 if r != 0: return r
1578 # Multilink properties are sorted according to how many
1579 # links are present.
1580 elif isinstance(propclass, Multilink):
1581 if dir == '+':
1582 r = cmp(len(av), len(bv))
1583 if r != 0: return r
1584 elif dir == '-':
1585 r = cmp(len(bv), len(av))
1586 if r != 0: return r
1587 elif isinstance(propclass, Number) or isinstance(propclass, Boolean):
1588 if dir == '+':
1589 r = cmp(av, bv)
1590 elif dir == '-':
1591 r = cmp(bv, av)
1593 # end for dir, prop in list:
1594 # end for list in sort, group:
1595 # if all else fails, compare the ids
1596 return cmp(a[0], b[0])
1598 l.sort(sortfun)
1599 return [i[0] for i in l]
1601 def count(self):
1602 """Get the number of nodes in this class.
1604 If the returned integer is 'numnodes', the ids of all the nodes
1605 in this class run from 1 to numnodes, and numnodes+1 will be the
1606 id of the next node to be created in this class.
1607 """
1608 return self.db.countnodes(self.classname)
1610 # Manipulating properties:
1612 def getprops(self, protected=1):
1613 """Return a dictionary mapping property names to property objects.
1614 If the "protected" flag is true, we include protected properties -
1615 those which may not be modified.
1617 In addition to the actual properties on the node, these
1618 methods provide the "creation" and "activity" properties. If the
1619 "protected" flag is true, we include protected properties - those
1620 which may not be modified.
1621 """
1622 d = self.properties.copy()
1623 if protected:
1624 d['id'] = String()
1625 d['creation'] = hyperdb.Date()
1626 d['activity'] = hyperdb.Date()
1627 d['creator'] = hyperdb.Link("user")
1628 return d
1630 def addprop(self, **properties):
1631 """Add properties to this class.
1633 The keyword arguments in 'properties' must map names to property
1634 objects, or a TypeError is raised. None of the keys in 'properties'
1635 may collide with the names of existing properties, or a ValueError
1636 is raised before any properties have been added.
1637 """
1638 for key in properties.keys():
1639 if self.properties.has_key(key):
1640 raise ValueError, key
1641 self.properties.update(properties)
1643 def index(self, nodeid):
1644 '''Add (or refresh) the node to search indexes
1645 '''
1646 # find all the String properties that have indexme
1647 for prop, propclass in self.getprops().items():
1648 if isinstance(propclass, String) and propclass.indexme:
1649 try:
1650 value = str(self.get(nodeid, prop))
1651 except IndexError:
1652 # node no longer exists - entry should be removed
1653 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1654 else:
1655 # and index them under (classname, nodeid, property)
1656 self.db.indexer.add_text((self.classname, nodeid, prop),
1657 value)
1659 #
1660 # Detector interface
1661 #
1662 def audit(self, event, detector):
1663 """Register a detector
1664 """
1665 l = self.auditors[event]
1666 if detector not in l:
1667 self.auditors[event].append(detector)
1669 def fireAuditors(self, action, nodeid, newvalues):
1670 """Fire all registered auditors.
1671 """
1672 for audit in self.auditors[action]:
1673 audit(self.db, self, nodeid, newvalues)
1675 def react(self, event, detector):
1676 """Register a detector
1677 """
1678 l = self.reactors[event]
1679 if detector not in l:
1680 self.reactors[event].append(detector)
1682 def fireReactors(self, action, nodeid, oldvalues):
1683 """Fire all registered reactors.
1684 """
1685 for react in self.reactors[action]:
1686 react(self.db, self, nodeid, oldvalues)
1688 class FileClass(Class):
1689 '''This class defines a large chunk of data. To support this, it has a
1690 mandatory String property "content" which is typically saved off
1691 externally to the hyperdb.
1693 The default MIME type of this data is defined by the
1694 "default_mime_type" class attribute, which may be overridden by each
1695 node if the class defines a "type" String property.
1696 '''
1697 default_mime_type = 'text/plain'
1699 def create(self, **propvalues):
1700 ''' snaffle the file propvalue and store in a file
1701 '''
1702 content = propvalues['content']
1703 del propvalues['content']
1704 newid = Class.create(self, **propvalues)
1705 self.db.storefile(self.classname, newid, None, content)
1706 return newid
1708 def get(self, nodeid, propname, default=_marker, cache=1):
1709 ''' trap the content propname and get it from the file
1710 '''
1712 poss_msg = 'Possibly a access right configuration problem.'
1713 if propname == 'content':
1714 try:
1715 return self.db.getfile(self.classname, nodeid, None)
1716 except IOError, (strerror):
1717 # BUG: by catching this we donot see an error in the log.
1718 return 'ERROR reading file: %s%s\n%s\n%s'%(
1719 self.classname, nodeid, poss_msg, strerror)
1720 if default is not _marker:
1721 return Class.get(self, nodeid, propname, default, cache=cache)
1722 else:
1723 return Class.get(self, nodeid, propname, cache=cache)
1725 def getprops(self, protected=1):
1726 ''' In addition to the actual properties on the node, these methods
1727 provide the "content" property. If the "protected" flag is true,
1728 we include protected properties - those which may not be
1729 modified.
1730 '''
1731 d = Class.getprops(self, protected=protected).copy()
1732 if protected:
1733 d['content'] = hyperdb.String()
1734 return d
1736 def index(self, nodeid):
1737 ''' Index the node in the search index.
1739 We want to index the content in addition to the normal String
1740 property indexing.
1741 '''
1742 # perform normal indexing
1743 Class.index(self, nodeid)
1745 # get the content to index
1746 content = self.get(nodeid, 'content')
1748 # figure the mime type
1749 if self.properties.has_key('type'):
1750 mime_type = self.get(nodeid, 'type')
1751 else:
1752 mime_type = self.default_mime_type
1754 # and index!
1755 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
1756 mime_type)
1758 # XXX deviation from spec - was called ItemClass
1759 class IssueClass(Class, roundupdb.IssueClass):
1760 # Overridden methods:
1761 def __init__(self, db, classname, **properties):
1762 """The newly-created class automatically includes the "messages",
1763 "files", "nosy", and "superseder" properties. If the 'properties'
1764 dictionary attempts to specify any of these properties or a
1765 "creation" or "activity" property, a ValueError is raised.
1766 """
1767 if not properties.has_key('title'):
1768 properties['title'] = hyperdb.String(indexme='yes')
1769 if not properties.has_key('messages'):
1770 properties['messages'] = hyperdb.Multilink("msg")
1771 if not properties.has_key('files'):
1772 properties['files'] = hyperdb.Multilink("file")
1773 if not properties.has_key('nosy'):
1774 properties['nosy'] = hyperdb.Multilink("user")
1775 if not properties.has_key('superseder'):
1776 properties['superseder'] = hyperdb.Multilink(classname)
1777 Class.__init__(self, db, classname, **properties)
1779 #
1780 #$Log: not supported by cvs2svn $
1781 #Revision 1.51 2002/07/18 23:07:08 richard
1782 #Unit tests and a few fixes.
1783 #
1784 #Revision 1.50 2002/07/18 11:50:58 richard
1785 #added tests for number type too
1786 #
1787 #Revision 1.49 2002/07/18 11:41:10 richard
1788 #added tests for boolean type, and fixes to anydbm backend
1789 #
1790 #Revision 1.48 2002/07/18 11:17:31 gmcm
1791 #Add Number and Boolean types to hyperdb.
1792 #Add conversion cases to web, mail & admin interfaces.
1793 #Add storage/serialization cases to back_anydbm & back_metakit.
1794 #
1795 #Revision 1.47 2002/07/14 23:18:20 richard
1796 #. fixed the journal bloat from multilink changes - we just log the add or
1797 # remove operations, not the whole list
1798 #
1799 #Revision 1.46 2002/07/14 06:06:34 richard
1800 #Did some old TODOs
1801 #
1802 #Revision 1.45 2002/07/14 04:03:14 richard
1803 #Implemented a switch to disable journalling for a Class. CGI session
1804 #database now uses it.
1805 #
1806 #Revision 1.44 2002/07/14 02:05:53 richard
1807 #. all storage-specific code (ie. backend) is now implemented by the backends
1808 #
1809 #Revision 1.43 2002/07/10 06:30:30 richard
1810 #...except of course it's nice to use valid Python syntax
1811 #
1812 #Revision 1.42 2002/07/10 06:21:38 richard
1813 #Be extra safe
1814 #
1815 #Revision 1.41 2002/07/10 00:21:45 richard
1816 #explicit database closing
1817 #
1818 #Revision 1.40 2002/07/09 04:19:09 richard
1819 #Added reindex command to roundup-admin.
1820 #Fixed reindex on first access.
1821 #Also fixed reindexing of entries that change.
1822 #
1823 #Revision 1.39 2002/07/09 03:02:52 richard
1824 #More indexer work:
1825 #- all String properties may now be indexed too. Currently there's a bit of
1826 # "issue" specific code in the actual searching which needs to be
1827 # addressed. In a nutshell:
1828 # + pass 'indexme="yes"' as a String() property initialisation arg, eg:
1829 # file = FileClass(db, "file", name=String(), type=String(),
1830 # comment=String(indexme="yes"))
1831 # + the comment will then be indexed and be searchable, with the results
1832 # related back to the issue that the file is linked to
1833 #- as a result of this work, the FileClass has a default MIME type that may
1834 # be overridden in a subclass, or by the use of a "type" property as is
1835 # done in the default templates.
1836 #- the regeneration of the indexes (if necessary) is done once the schema is
1837 # set up in the dbinit.
1838 #
1839 #Revision 1.38 2002/07/08 06:58:15 richard
1840 #cleaned up the indexer code:
1841 # - it splits more words out (much simpler, faster splitter)
1842 # - removed code we'll never use (roundup.roundup_indexer has the full
1843 # implementation, and replaces roundup.indexer)
1844 # - only index text/plain and rfc822/message (ideas for other text formats to
1845 # index are welcome)
1846 # - added simple unit test for indexer. Needs more tests for regression.
1847 #
1848 #Revision 1.37 2002/06/20 23:52:35 richard
1849 #More informative error message
1850 #
1851 #Revision 1.36 2002/06/19 03:07:19 richard
1852 #Moved the file storage commit into blobfiles where it belongs.
1853 #
1854 #Revision 1.35 2002/05/25 07:16:24 rochecompaan
1855 #Merged search_indexing-branch with HEAD
1856 #
1857 #Revision 1.34 2002/05/15 06:21:21 richard
1858 # . node caching now works, and gives a small boost in performance
1859 #
1860 #As a part of this, I cleaned up the DEBUG output and implemented TRACE
1861 #output (HYPERDBTRACE='file to trace to') with checkpoints at the start of
1862 #CGI requests. Run roundup with python -O to skip all the DEBUG/TRACE stuff
1863 #(using if __debug__ which is compiled out with -O)
1864 #
1865 #Revision 1.33 2002/04/24 10:38:26 rochecompaan
1866 #All database files are now created group readable and writable.
1867 #
1868 #Revision 1.32 2002/04/15 23:25:15 richard
1869 #. node ids are now generated from a lockable store - no more race conditions
1870 #
1871 #We're using the portalocker code by Jonathan Feinberg that was contributed
1872 #to the ASPN Python cookbook. This gives us locking across Unix and Windows.
1873 #
1874 #Revision 1.31 2002/04/03 05:54:31 richard
1875 #Fixed serialisation problem by moving the serialisation step out of the
1876 #hyperdb.Class (get, set) into the hyperdb.Database.
1877 #
1878 #Also fixed htmltemplate after the showid changes I made yesterday.
1879 #
1880 #Unit tests for all of the above written.
1881 #
1882 #Revision 1.30.2.1 2002/04/03 11:55:57 rochecompaan
1883 # . Added feature #526730 - search for messages capability
1884 #
1885 #Revision 1.30 2002/02/27 03:40:59 richard
1886 #Ran it through pychecker, made fixes
1887 #
1888 #Revision 1.29 2002/02/25 14:34:31 grubert
1889 # . use blobfiles in back_anydbm which is used in back_bsddb.
1890 # change test_db as dirlist does not work for subdirectories.
1891 # ATTENTION: blobfiles now creates subdirectories for files.
1892 #
1893 #Revision 1.28 2002/02/16 09:14:17 richard
1894 # . #514854 ] History: "User" is always ticket creator
1895 #
1896 #Revision 1.27 2002/01/22 07:21:13 richard
1897 #. fixed back_bsddb so it passed the journal tests
1898 #
1899 #... it didn't seem happy using the back_anydbm _open method, which is odd.
1900 #Yet another occurrance of whichdb not being able to recognise older bsddb
1901 #databases. Yadda yadda. Made the HYPERDBDEBUG stuff more sane in the
1902 #process.
1903 #
1904 #Revision 1.26 2002/01/22 05:18:38 rochecompaan
1905 #last_set_entry was referenced before assignment
1906 #
1907 #Revision 1.25 2002/01/22 05:06:08 rochecompaan
1908 #We need to keep the last 'set' entry in the journal to preserve
1909 #information on 'activity' for nodes.
1910 #
1911 #Revision 1.24 2002/01/21 16:33:20 rochecompaan
1912 #You can now use the roundup-admin tool to pack the database
1913 #
1914 #Revision 1.23 2002/01/18 04:32:04 richard
1915 #Rollback was breaking because a message hadn't actually been written to the file. Needs
1916 #more investigation.
1917 #
1918 #Revision 1.22 2002/01/14 02:20:15 richard
1919 # . changed all config accesses so they access either the instance or the
1920 # config attriubute on the db. This means that all config is obtained from
1921 # instance_config instead of the mish-mash of classes. This will make
1922 # switching to a ConfigParser setup easier too, I hope.
1923 #
1924 #At a minimum, this makes migration a _little_ easier (a lot easier in the
1925 #0.5.0 switch, I hope!)
1926 #
1927 #Revision 1.21 2002/01/02 02:31:38 richard
1928 #Sorry for the huge checkin message - I was only intending to implement #496356
1929 #but I found a number of places where things had been broken by transactions:
1930 # . modified ROUNDUPDBSENDMAILDEBUG to be SENDMAILDEBUG and hold a filename
1931 # for _all_ roundup-generated smtp messages to be sent to.
1932 # . the transaction cache had broken the roundupdb.Class set() reactors
1933 # . newly-created author users in the mailgw weren't being committed to the db
1934 #
1935 #Stuff that made it into CHANGES.txt (ie. the stuff I was actually working
1936 #on when I found that stuff :):
1937 # . #496356 ] Use threading in messages
1938 # . detectors were being registered multiple times
1939 # . added tests for mailgw
1940 # . much better attaching of erroneous messages in the mail gateway
1941 #
1942 #Revision 1.20 2001/12/18 15:30:34 rochecompaan
1943 #Fixed bugs:
1944 # . Fixed file creation and retrieval in same transaction in anydbm
1945 # backend
1946 # . Cgi interface now renders new issue after issue creation
1947 # . Could not set issue status to resolved through cgi interface
1948 # . Mail gateway was changing status back to 'chatting' if status was
1949 # omitted as an argument
1950 #
1951 #Revision 1.19 2001/12/17 03:52:48 richard
1952 #Implemented file store rollback. As a bonus, the hyperdb is now capable of
1953 #storing more than one file per node - if a property name is supplied,
1954 #the file is called designator.property.
1955 #I decided not to migrate the existing files stored over to the new naming
1956 #scheme - the FileClass just doesn't specify the property name.
1957 #
1958 #Revision 1.18 2001/12/16 10:53:38 richard
1959 #take a copy of the node dict so that the subsequent set
1960 #operation doesn't modify the oldvalues structure
1961 #
1962 #Revision 1.17 2001/12/14 23:42:57 richard
1963 #yuck, a gdbm instance tests false :(
1964 #I've left the debugging code in - it should be removed one day if we're ever
1965 #_really_ anal about performace :)
1966 #
1967 #Revision 1.16 2001/12/12 03:23:14 richard
1968 #Cor blimey this anydbm/whichdb stuff is yecchy. Turns out that whichdb
1969 #incorrectly identifies a dbm file as a dbhash file on my system. This has
1970 #been submitted to the python bug tracker as issue #491888:
1971 #https://sourceforge.net/tracker/index.php?func=detail&aid=491888&group_id=5470&atid=105470
1972 #
1973 #Revision 1.15 2001/12/12 02:30:51 richard
1974 #I fixed the problems with people whose anydbm was using the dbm module at the
1975 #backend. It turns out the dbm module modifies the file name to append ".db"
1976 #and my check to determine if we're opening an existing or new db just
1977 #tested os.path.exists() on the filename. Well, no longer! We now perform a
1978 #much better check _and_ cope with the anydbm implementation module changing
1979 #too!
1980 #I also fixed the backends __init__ so only ImportError is squashed.
1981 #
1982 #Revision 1.14 2001/12/10 22:20:01 richard
1983 #Enabled transaction support in the bsddb backend. It uses the anydbm code
1984 #where possible, only replacing methods where the db is opened (it uses the
1985 #btree opener specifically.)
1986 #Also cleaned up some change note generation.
1987 #Made the backends package work with pydoc too.
1988 #
1989 #Revision 1.13 2001/12/02 05:06:16 richard
1990 #. We now use weakrefs in the Classes to keep the database reference, so
1991 # the close() method on the database is no longer needed.
1992 # I bumped the minimum python requirement up to 2.1 accordingly.
1993 #. #487480 ] roundup-server
1994 #. #487476 ] INSTALL.txt
1995 #
1996 #I also cleaned up the change message / post-edit stuff in the cgi client.
1997 #There's now a clearly marked "TODO: append the change note" where I believe
1998 #the change note should be added there. The "changes" list will obviously
1999 #have to be modified to be a dict of the changes, or somesuch.
2000 #
2001 #More testing needed.
2002 #
2003 #Revision 1.12 2001/12/01 07:17:50 richard
2004 #. We now have basic transaction support! Information is only written to
2005 # the database when the commit() method is called. Only the anydbm
2006 # backend is modified in this way - neither of the bsddb backends have been.
2007 # The mail, admin and cgi interfaces all use commit (except the admin tool
2008 # doesn't have a commit command, so interactive users can't commit...)
2009 #. Fixed login/registration forwarding the user to the right page (or not,
2010 # on a failure)
2011 #
2012 #Revision 1.11 2001/11/21 02:34:18 richard
2013 #Added a target version field to the extended issue schema
2014 #
2015 #Revision 1.10 2001/10/09 23:58:10 richard
2016 #Moved the data stringification up into the hyperdb.Class class' get, set
2017 #and create methods. This means that the data is also stringified for the
2018 #journal call, and removes duplication of code from the backends. The
2019 #backend code now only sees strings.
2020 #
2021 #Revision 1.9 2001/10/09 07:25:59 richard
2022 #Added the Password property type. See "pydoc roundup.password" for
2023 #implementation details. Have updated some of the documentation too.
2024 #
2025 #Revision 1.8 2001/09/29 13:27:00 richard
2026 #CGI interfaces now spit up a top-level index of all the instances they can
2027 #serve.
2028 #
2029 #Revision 1.7 2001/08/12 06:32:36 richard
2030 #using isinstance(blah, Foo) now instead of isFooType
2031 #
2032 #Revision 1.6 2001/08/07 00:24:42 richard
2033 #stupid typo
2034 #
2035 #Revision 1.5 2001/08/07 00:15:51 richard
2036 #Added the copyright/license notice to (nearly) all files at request of
2037 #Bizar Software.
2038 #
2039 #Revision 1.4 2001/07/30 01:41:36 richard
2040 #Makes schema changes mucho easier.
2041 #
2042 #Revision 1.3 2001/07/25 01:23:07 richard
2043 #Added the Roundup spec to the new documentation directory.
2044 #
2045 #Revision 1.2 2001/07/23 08:20:44 richard
2046 #Moved over to using marshal in the bsddb and anydbm backends.
2047 #roundup-admin now has a "freshen" command that'll load/save all nodes (not
2048 # retired - mod hyperdb.Class.list() so it lists retired nodes)
2049 #
2050 #