1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: back_anydbm.py,v 1.83 2002-09-20 05:08:00 richard Exp $
19 '''
20 This module defines a backend that saves the hyperdatabase in a database
21 chosen by anydbm. It is guaranteed to always be available in python
22 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
23 serious bugs, and is not available)
24 '''
26 import whichdb, anydbm, os, marshal, re, weakref, string, copy
27 from roundup import hyperdb, date, password, roundupdb, security
28 from blobfiles import FileStorage
29 from sessions import Sessions
30 from roundup.indexer import Indexer
31 from locking import acquire_lock, release_lock
32 from roundup.hyperdb import String, Password, Date, Interval, Link, \
33 Multilink, DatabaseError, Boolean, Number
35 #
36 # Now the database
37 #
38 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
39 '''A database for storing records containing flexible data types.
41 Transaction stuff TODO:
42 . check the timestamp of the class file and nuke the cache if it's
43 modified. Do some sort of conflict checking on the dirty stuff.
44 . perhaps detect write collisions (related to above)?
46 '''
47 def __init__(self, config, journaltag=None):
48 '''Open a hyperdatabase given a specifier to some storage.
50 The 'storagelocator' is obtained from config.DATABASE.
51 The meaning of 'storagelocator' depends on the particular
52 implementation of the hyperdatabase. It could be a file name,
53 a directory path, a socket descriptor for a connection to a
54 database over the network, etc.
56 The 'journaltag' is a token that will be attached to the journal
57 entries for any edits done on the database. If 'journaltag' is
58 None, the database is opened in read-only mode: the Class.create(),
59 Class.set(), and Class.retire() methods are disabled.
60 '''
61 self.config, self.journaltag = config, journaltag
62 self.dir = config.DATABASE
63 self.classes = {}
64 self.cache = {} # cache of nodes loaded or created
65 self.dirtynodes = {} # keep track of the dirty nodes by class
66 self.newnodes = {} # keep track of the new nodes by class
67 self.destroyednodes = {}# keep track of the destroyed nodes by class
68 self.transactions = []
69 self.indexer = Indexer(self.dir)
70 self.sessions = Sessions(self.config)
71 self.security = security.Security(self)
72 # ensure files are group readable and writable
73 os.umask(0002)
75 def post_init(self):
76 ''' Called once the schema initialisation has finished.
77 '''
78 # reindex the db if necessary
79 if self.indexer.should_reindex():
80 self.reindex()
82 # figure the "curuserid"
83 if self.journaltag is None:
84 self.curuserid = None
85 elif self.journaltag == 'admin':
86 # admin user may not exist, but always has ID 1
87 self.curuserid = '1'
88 else:
89 self.curuserid = self.user.lookup(self.journaltag)
91 def reindex(self):
92 for klass in self.classes.values():
93 for nodeid in klass.list():
94 klass.index(nodeid)
95 self.indexer.save_index()
97 def __repr__(self):
98 return '<back_anydbm instance at %x>'%id(self)
100 #
101 # Classes
102 #
103 def __getattr__(self, classname):
104 '''A convenient way of calling self.getclass(classname).'''
105 if self.classes.has_key(classname):
106 if __debug__:
107 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
108 return self.classes[classname]
109 raise AttributeError, classname
111 def addclass(self, cl):
112 if __debug__:
113 print >>hyperdb.DEBUG, 'addclass', (self, cl)
114 cn = cl.classname
115 if self.classes.has_key(cn):
116 raise ValueError, cn
117 self.classes[cn] = cl
119 def getclasses(self):
120 '''Return a list of the names of all existing classes.'''
121 if __debug__:
122 print >>hyperdb.DEBUG, 'getclasses', (self,)
123 l = self.classes.keys()
124 l.sort()
125 return l
127 def getclass(self, classname):
128 '''Get the Class object representing a particular class.
130 If 'classname' is not a valid class name, a KeyError is raised.
131 '''
132 if __debug__:
133 print >>hyperdb.DEBUG, 'getclass', (self, classname)
134 try:
135 return self.classes[classname]
136 except KeyError:
137 raise KeyError, 'There is no class called "%s"'%classname
139 #
140 # Class DBs
141 #
142 def clear(self):
143 '''Delete all database contents
144 '''
145 if __debug__:
146 print >>hyperdb.DEBUG, 'clear', (self,)
147 for cn in self.classes.keys():
148 for dummy in 'nodes', 'journals':
149 path = os.path.join(self.dir, 'journals.%s'%cn)
150 if os.path.exists(path):
151 os.remove(path)
152 elif os.path.exists(path+'.db'): # dbm appends .db
153 os.remove(path+'.db')
155 def getclassdb(self, classname, mode='r'):
156 ''' grab a connection to the class db that will be used for
157 multiple actions
158 '''
159 if __debug__:
160 print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode)
161 return self.opendb('nodes.%s'%classname, mode)
163 def determine_db_type(self, path):
164 ''' determine which DB wrote the class file
165 '''
166 db_type = ''
167 if os.path.exists(path):
168 db_type = whichdb.whichdb(path)
169 if not db_type:
170 raise DatabaseError, "Couldn't identify database type"
171 elif os.path.exists(path+'.db'):
172 # if the path ends in '.db', it's a dbm database, whether
173 # anydbm says it's dbhash or not!
174 db_type = 'dbm'
175 return db_type
177 def opendb(self, name, mode):
178 '''Low-level database opener that gets around anydbm/dbm
179 eccentricities.
180 '''
181 if __debug__:
182 print >>hyperdb.DEBUG, 'opendb', (self, name, mode)
184 # figure the class db type
185 path = os.path.join(os.getcwd(), self.dir, name)
186 db_type = self.determine_db_type(path)
188 # new database? let anydbm pick the best dbm
189 if not db_type:
190 if __debug__:
191 print >>hyperdb.DEBUG, "opendb anydbm.open(%r, 'c')"%path
192 return anydbm.open(path, 'c')
194 # open the database with the correct module
195 try:
196 dbm = __import__(db_type)
197 except ImportError:
198 raise DatabaseError, \
199 "Couldn't open database - the required module '%s'"\
200 " is not available"%db_type
201 if __debug__:
202 print >>hyperdb.DEBUG, "opendb %r.open(%r, %r)"%(db_type, path,
203 mode)
204 return dbm.open(path, mode)
206 def lockdb(self, name):
207 ''' Lock a database file
208 '''
209 path = os.path.join(os.getcwd(), self.dir, '%s.lock'%name)
210 return acquire_lock(path)
212 #
213 # Node IDs
214 #
215 def newid(self, classname):
216 ''' Generate a new id for the given class
217 '''
218 # open the ids DB - create if if doesn't exist
219 lock = self.lockdb('_ids')
220 db = self.opendb('_ids', 'c')
221 if db.has_key(classname):
222 newid = db[classname] = str(int(db[classname]) + 1)
223 else:
224 # the count() bit is transitional - older dbs won't start at 1
225 newid = str(self.getclass(classname).count()+1)
226 db[classname] = newid
227 db.close()
228 release_lock(lock)
229 return newid
231 def setid(self, classname, setid):
232 ''' Set the id counter: used during import of database
233 '''
234 # open the ids DB - create if if doesn't exist
235 lock = self.lockdb('_ids')
236 db = self.opendb('_ids', 'c')
237 db[classname] = str(setid)
238 db.close()
239 release_lock(lock)
241 #
242 # Nodes
243 #
244 def addnode(self, classname, nodeid, node):
245 ''' add the specified node to its class's db
246 '''
247 if __debug__:
248 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
250 # we'll be supplied these props if we're doing an import
251 if not node.has_key('creator'):
252 # add in the "calculated" properties (dupe so we don't affect
253 # calling code's node assumptions)
254 node = node.copy()
255 node['creator'] = self.curuserid
256 node['creation'] = node['activity'] = date.Date()
258 self.newnodes.setdefault(classname, {})[nodeid] = 1
259 self.cache.setdefault(classname, {})[nodeid] = node
260 self.savenode(classname, nodeid, node)
262 def setnode(self, classname, nodeid, node):
263 ''' change the specified node
264 '''
265 if __debug__:
266 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
267 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
269 # update the activity time (dupe so we don't affect
270 # calling code's node assumptions)
271 node = node.copy()
272 node['activity'] = date.Date()
274 # can't set without having already loaded the node
275 self.cache[classname][nodeid] = node
276 self.savenode(classname, nodeid, node)
278 def savenode(self, classname, nodeid, node):
279 ''' perform the saving of data specified by the set/addnode
280 '''
281 if __debug__:
282 print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node)
283 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
285 def getnode(self, classname, nodeid, db=None, cache=1):
286 ''' get a node from the database
287 '''
288 if __debug__:
289 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid, db)
290 if cache:
291 # try the cache
292 cache_dict = self.cache.setdefault(classname, {})
293 if cache_dict.has_key(nodeid):
294 if __debug__:
295 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
296 nodeid)
297 return cache_dict[nodeid]
299 if __debug__:
300 print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid)
302 # get from the database and save in the cache
303 if db is None:
304 db = self.getclassdb(classname)
305 if not db.has_key(nodeid):
306 raise IndexError, "no such %s %s"%(classname, nodeid)
308 # check the uncommitted, destroyed nodes
309 if (self.destroyednodes.has_key(classname) and
310 self.destroyednodes[classname].has_key(nodeid)):
311 raise IndexError, "no such %s %s"%(classname, nodeid)
313 # decode
314 res = marshal.loads(db[nodeid])
316 # reverse the serialisation
317 res = self.unserialise(classname, res)
319 # store off in the cache dict
320 if cache:
321 cache_dict[nodeid] = res
323 return res
325 def destroynode(self, classname, nodeid):
326 '''Remove a node from the database. Called exclusively by the
327 destroy() method on Class.
328 '''
329 if __debug__:
330 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
332 # remove from cache and newnodes if it's there
333 if (self.cache.has_key(classname) and
334 self.cache[classname].has_key(nodeid)):
335 del self.cache[classname][nodeid]
336 if (self.newnodes.has_key(classname) and
337 self.newnodes[classname].has_key(nodeid)):
338 del self.newnodes[classname][nodeid]
340 # see if there's any obvious commit actions that we should get rid of
341 for entry in self.transactions[:]:
342 if entry[1][:2] == (classname, nodeid):
343 self.transactions.remove(entry)
345 # add to the destroyednodes map
346 self.destroyednodes.setdefault(classname, {})[nodeid] = 1
348 # add the destroy commit action
349 self.transactions.append((self.doDestroyNode, (classname, nodeid)))
351 def serialise(self, classname, node):
352 '''Copy the node contents, converting non-marshallable data into
353 marshallable data.
354 '''
355 if __debug__:
356 print >>hyperdb.DEBUG, 'serialise', classname, node
357 properties = self.getclass(classname).getprops()
358 d = {}
359 for k, v in node.items():
360 # if the property doesn't exist, or is the "retired" flag then
361 # it won't be in the properties dict
362 if not properties.has_key(k):
363 d[k] = v
364 continue
366 # get the property spec
367 prop = properties[k]
369 if isinstance(prop, Password):
370 d[k] = str(v)
371 elif isinstance(prop, Date) and v is not None:
372 d[k] = v.serialise()
373 elif isinstance(prop, Interval) and v is not None:
374 d[k] = v.serialise()
375 else:
376 d[k] = v
377 return d
379 def unserialise(self, classname, node):
380 '''Decode the marshalled node data
381 '''
382 if __debug__:
383 print >>hyperdb.DEBUG, 'unserialise', classname, node
384 properties = self.getclass(classname).getprops()
385 d = {}
386 for k, v in node.items():
387 # if the property doesn't exist, or is the "retired" flag then
388 # it won't be in the properties dict
389 if not properties.has_key(k):
390 d[k] = v
391 continue
393 # get the property spec
394 prop = properties[k]
396 if isinstance(prop, Date) and v is not None:
397 d[k] = date.Date(v)
398 elif isinstance(prop, Interval) and v is not None:
399 d[k] = date.Interval(v)
400 elif isinstance(prop, Password):
401 p = password.Password()
402 p.unpack(v)
403 d[k] = p
404 else:
405 d[k] = v
406 return d
408 def hasnode(self, classname, nodeid, db=None):
409 ''' determine if the database has a given node
410 '''
411 if __debug__:
412 print >>hyperdb.DEBUG, 'hasnode', (self, classname, nodeid, db)
414 # try the cache
415 cache = self.cache.setdefault(classname, {})
416 if cache.has_key(nodeid):
417 if __debug__:
418 print >>hyperdb.TRACE, 'has %s %s cached'%(classname, nodeid)
419 return 1
420 if __debug__:
421 print >>hyperdb.TRACE, 'has %s %s'%(classname, nodeid)
423 # not in the cache - check the database
424 if db is None:
425 db = self.getclassdb(classname)
426 res = db.has_key(nodeid)
427 return res
429 def countnodes(self, classname, db=None):
430 if __debug__:
431 print >>hyperdb.DEBUG, 'countnodes', (self, classname, db)
433 count = 0
435 # include the uncommitted nodes
436 if self.newnodes.has_key(classname):
437 count += len(self.newnodes[classname])
438 if self.destroyednodes.has_key(classname):
439 count -= len(self.destroyednodes[classname])
441 # and count those in the DB
442 if db is None:
443 db = self.getclassdb(classname)
444 count = count + len(db.keys())
445 return count
447 def getnodeids(self, classname, db=None):
448 if __debug__:
449 print >>hyperdb.DEBUG, 'getnodeids', (self, classname, db)
451 res = []
453 # start off with the new nodes
454 if self.newnodes.has_key(classname):
455 res += self.newnodes[classname].keys()
457 if db is None:
458 db = self.getclassdb(classname)
459 res = res + db.keys()
461 # remove the uncommitted, destroyed nodes
462 if self.destroyednodes.has_key(classname):
463 for nodeid in self.destroyednodes[classname].keys():
464 if db.has_key(nodeid):
465 res.remove(nodeid)
467 return res
470 #
471 # Files - special node properties
472 # inherited from FileStorage
474 #
475 # Journal
476 #
477 def addjournal(self, classname, nodeid, action, params, creator=None,
478 creation=None):
479 ''' Journal the Action
480 'action' may be:
482 'create' or 'set' -- 'params' is a dictionary of property values
483 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
484 'retire' -- 'params' is None
485 '''
486 if __debug__:
487 print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid,
488 action, params, creator, creation)
489 self.transactions.append((self.doSaveJournal, (classname, nodeid,
490 action, params, creator, creation)))
492 def getjournal(self, classname, nodeid):
493 ''' get the journal for id
495 Raise IndexError if the node doesn't exist (as per history()'s
496 API)
497 '''
498 if __debug__:
499 print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid)
500 # attempt to open the journal - in some rare cases, the journal may
501 # not exist
502 try:
503 db = self.opendb('journals.%s'%classname, 'r')
504 except anydbm.error, error:
505 if str(error) == "need 'c' or 'n' flag to open new db":
506 raise IndexError, 'no such %s %s'%(classname, nodeid)
507 elif error.args[0] != 2:
508 raise
509 raise IndexError, 'no such %s %s'%(classname, nodeid)
510 try:
511 journal = marshal.loads(db[nodeid])
512 except KeyError:
513 db.close()
514 raise IndexError, 'no such %s %s'%(classname, nodeid)
515 db.close()
516 res = []
517 for nodeid, date_stamp, user, action, params in journal:
518 res.append((nodeid, date.Date(date_stamp), user, action, params))
519 return res
521 def pack(self, pack_before):
522 ''' Delete all journal entries except "create" before 'pack_before'.
523 '''
524 if __debug__:
525 print >>hyperdb.DEBUG, 'packjournal', (self, pack_before)
527 for classname in self.getclasses():
528 # get the journal db
529 db_name = 'journals.%s'%classname
530 path = os.path.join(os.getcwd(), self.dir, classname)
531 db_type = self.determine_db_type(path)
532 db = self.opendb(db_name, 'w')
534 for key in db.keys():
535 # get the journal for this db entry
536 journal = marshal.loads(db[key])
537 l = []
538 last_set_entry = None
539 for entry in journal:
540 # unpack the entry
541 (nodeid, date_stamp, self.journaltag, action,
542 params) = entry
543 date_stamp = date.Date(date_stamp)
544 # if the entry is after the pack date, _or_ the initial
545 # create entry, then it stays
546 if date_stamp > pack_before or action == 'create':
547 l.append(entry)
548 elif action == 'set':
549 # grab the last set entry to keep information on
550 # activity
551 last_set_entry = entry
552 if last_set_entry:
553 date_stamp = last_set_entry[1]
554 # if the last set entry was made after the pack date
555 # then it is already in the list
556 if date_stamp < pack_before:
557 l.append(last_set_entry)
558 db[key] = marshal.dumps(l)
559 if db_type == 'gdbm':
560 db.reorganize()
561 db.close()
564 #
565 # Basic transaction support
566 #
567 def commit(self):
568 ''' Commit the current transactions.
569 '''
570 if __debug__:
571 print >>hyperdb.DEBUG, 'commit', (self,)
572 # TODO: lock the DB
574 # keep a handle to all the database files opened
575 self.databases = {}
577 # now, do all the transactions
578 reindex = {}
579 for method, args in self.transactions:
580 reindex[method(*args)] = 1
582 # now close all the database files
583 for db in self.databases.values():
584 db.close()
585 del self.databases
586 # TODO: unlock the DB
588 # reindex the nodes that request it
589 for classname, nodeid in filter(None, reindex.keys()):
590 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
591 self.getclass(classname).index(nodeid)
593 # save the indexer state
594 self.indexer.save_index()
596 # all transactions committed, back to normal
597 self.cache = {}
598 self.dirtynodes = {}
599 self.newnodes = {}
600 self.destroyednodes = {}
601 self.transactions = []
603 def getCachedClassDB(self, classname):
604 ''' get the class db, looking in our cache of databases for commit
605 '''
606 # get the database handle
607 db_name = 'nodes.%s'%classname
608 if not self.databases.has_key(db_name):
609 self.databases[db_name] = self.getclassdb(classname, 'c')
610 return self.databases[db_name]
612 def doSaveNode(self, classname, nodeid, node):
613 if __debug__:
614 print >>hyperdb.DEBUG, 'doSaveNode', (self, classname, nodeid,
615 node)
617 db = self.getCachedClassDB(classname)
619 # now save the marshalled data
620 db[nodeid] = marshal.dumps(self.serialise(classname, node))
622 # return the classname, nodeid so we reindex this content
623 return (classname, nodeid)
625 def getCachedJournalDB(self, classname):
626 ''' get the journal db, looking in our cache of databases for commit
627 '''
628 # get the database handle
629 db_name = 'journals.%s'%classname
630 if not self.databases.has_key(db_name):
631 self.databases[db_name] = self.opendb(db_name, 'c')
632 return self.databases[db_name]
634 def doSaveJournal(self, classname, nodeid, action, params, creator,
635 creation):
636 # serialise the parameters now if necessary
637 if isinstance(params, type({})):
638 if action in ('set', 'create'):
639 params = self.serialise(classname, params)
641 # handle supply of the special journalling parameters (usually
642 # supplied on importing an existing database)
643 if creator:
644 journaltag = creator
645 else:
646 journaltag = self.curuserid
647 if creation:
648 journaldate = creation.serialise()
649 else:
650 journaldate = date.Date().serialise()
652 # create the journal entry
653 entry = (nodeid, journaldate, journaltag, action, params)
655 if __debug__:
656 print >>hyperdb.DEBUG, 'doSaveJournal', entry
658 db = self.getCachedJournalDB(classname)
660 # now insert the journal entry
661 if db.has_key(nodeid):
662 # append to existing
663 s = db[nodeid]
664 l = marshal.loads(s)
665 l.append(entry)
666 else:
667 l = [entry]
669 db[nodeid] = marshal.dumps(l)
671 def doDestroyNode(self, classname, nodeid):
672 if __debug__:
673 print >>hyperdb.DEBUG, 'doDestroyNode', (self, classname, nodeid)
675 # delete from the class database
676 db = self.getCachedClassDB(classname)
677 if db.has_key(nodeid):
678 del db[nodeid]
680 # delete from the database
681 db = self.getCachedJournalDB(classname)
682 if db.has_key(nodeid):
683 del db[nodeid]
685 # return the classname, nodeid so we reindex this content
686 return (classname, nodeid)
688 def rollback(self):
689 ''' Reverse all actions from the current transaction.
690 '''
691 if __debug__:
692 print >>hyperdb.DEBUG, 'rollback', (self, )
693 for method, args in self.transactions:
694 # delete temporary files
695 if method == self.doStoreFile:
696 self.rollbackStoreFile(*args)
697 self.cache = {}
698 self.dirtynodes = {}
699 self.newnodes = {}
700 self.destroyednodes = {}
701 self.transactions = []
703 def close(self):
704 ''' Nothing to do
705 '''
706 pass
708 _marker = []
709 class Class(hyperdb.Class):
710 '''The handle to a particular class of nodes in a hyperdatabase.'''
712 def __init__(self, db, classname, **properties):
713 '''Create a new class with a given name and property specification.
715 'classname' must not collide with the name of an existing class,
716 or a ValueError is raised. The keyword arguments in 'properties'
717 must map names to property objects, or a TypeError is raised.
718 '''
719 if (properties.has_key('creation') or properties.has_key('activity')
720 or properties.has_key('creator')):
721 raise ValueError, '"creation", "activity" and "creator" are '\
722 'reserved'
724 self.classname = classname
725 self.properties = properties
726 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
727 self.key = ''
729 # should we journal changes (default yes)
730 self.do_journal = 1
732 # do the db-related init stuff
733 db.addclass(self)
735 self.auditors = {'create': [], 'set': [], 'retire': []}
736 self.reactors = {'create': [], 'set': [], 'retire': []}
738 def enableJournalling(self):
739 '''Turn journalling on for this class
740 '''
741 self.do_journal = 1
743 def disableJournalling(self):
744 '''Turn journalling off for this class
745 '''
746 self.do_journal = 0
748 # Editing nodes:
750 def create(self, **propvalues):
751 '''Create a new node of this class and return its id.
753 The keyword arguments in 'propvalues' map property names to values.
755 The values of arguments must be acceptable for the types of their
756 corresponding properties or a TypeError is raised.
758 If this class has a key property, it must be present and its value
759 must not collide with other key strings or a ValueError is raised.
761 Any other properties on this class that are missing from the
762 'propvalues' dictionary are set to None.
764 If an id in a link or multilink property does not refer to a valid
765 node, an IndexError is raised.
767 These operations trigger detectors and can be vetoed. Attempts
768 to modify the "creation" or "activity" properties cause a KeyError.
769 '''
770 if propvalues.has_key('id'):
771 raise KeyError, '"id" is reserved'
773 if self.db.journaltag is None:
774 raise DatabaseError, 'Database open read-only'
776 if propvalues.has_key('creation') or propvalues.has_key('activity'):
777 raise KeyError, '"creation" and "activity" are reserved'
779 self.fireAuditors('create', None, propvalues)
781 # new node's id
782 newid = self.db.newid(self.classname)
784 # validate propvalues
785 num_re = re.compile('^\d+$')
786 for key, value in propvalues.items():
787 if key == self.key:
788 try:
789 self.lookup(value)
790 except KeyError:
791 pass
792 else:
793 raise ValueError, 'node with key "%s" exists'%value
795 # try to handle this property
796 try:
797 prop = self.properties[key]
798 except KeyError:
799 raise KeyError, '"%s" has no property "%s"'%(self.classname,
800 key)
802 if value is not None and isinstance(prop, Link):
803 if type(value) != type(''):
804 raise ValueError, 'link value must be String'
805 link_class = self.properties[key].classname
806 # if it isn't a number, it's a key
807 if not num_re.match(value):
808 try:
809 value = self.db.classes[link_class].lookup(value)
810 except (TypeError, KeyError):
811 raise IndexError, 'new property "%s": %s not a %s'%(
812 key, value, link_class)
813 elif not self.db.getclass(link_class).hasnode(value):
814 raise IndexError, '%s has no node %s'%(link_class, value)
816 # save off the value
817 propvalues[key] = value
819 # register the link with the newly linked node
820 if self.do_journal and self.properties[key].do_journal:
821 self.db.addjournal(link_class, value, 'link',
822 (self.classname, newid, key))
824 elif isinstance(prop, Multilink):
825 if type(value) != type([]):
826 raise TypeError, 'new property "%s" not a list of ids'%key
828 # clean up and validate the list of links
829 link_class = self.properties[key].classname
830 l = []
831 for entry in value:
832 if type(entry) != type(''):
833 raise ValueError, '"%s" multilink value (%r) '\
834 'must contain Strings'%(key, value)
835 # if it isn't a number, it's a key
836 if not num_re.match(entry):
837 try:
838 entry = self.db.classes[link_class].lookup(entry)
839 except (TypeError, KeyError):
840 raise IndexError, 'new property "%s": %s not a %s'%(
841 key, entry, self.properties[key].classname)
842 l.append(entry)
843 value = l
844 propvalues[key] = value
846 # handle additions
847 for nodeid in value:
848 if not self.db.getclass(link_class).hasnode(nodeid):
849 raise IndexError, '%s has no node %s'%(link_class,
850 nodeid)
851 # register the link with the newly linked node
852 if self.do_journal and self.properties[key].do_journal:
853 self.db.addjournal(link_class, nodeid, 'link',
854 (self.classname, newid, key))
856 elif isinstance(prop, String):
857 if type(value) != type(''):
858 raise TypeError, 'new property "%s" not a string'%key
860 elif isinstance(prop, Password):
861 if not isinstance(value, password.Password):
862 raise TypeError, 'new property "%s" not a Password'%key
864 elif isinstance(prop, Date):
865 if value is not None and not isinstance(value, date.Date):
866 raise TypeError, 'new property "%s" not a Date'%key
868 elif isinstance(prop, Interval):
869 if value is not None and not isinstance(value, date.Interval):
870 raise TypeError, 'new property "%s" not an Interval'%key
872 elif value is not None and isinstance(prop, Number):
873 try:
874 float(value)
875 except ValueError:
876 raise TypeError, 'new property "%s" not numeric'%key
878 elif value is not None and isinstance(prop, Boolean):
879 try:
880 int(value)
881 except ValueError:
882 raise TypeError, 'new property "%s" not boolean'%key
884 # make sure there's data where there needs to be
885 for key, prop in self.properties.items():
886 if propvalues.has_key(key):
887 continue
888 if key == self.key:
889 raise ValueError, 'key property "%s" is required'%key
890 if isinstance(prop, Multilink):
891 propvalues[key] = []
892 else:
893 propvalues[key] = None
895 # done
896 self.db.addnode(self.classname, newid, propvalues)
897 if self.do_journal:
898 self.db.addjournal(self.classname, newid, 'create', propvalues)
900 self.fireReactors('create', newid, None)
902 return newid
904 def export_list(self, propnames, nodeid):
905 ''' Export a node - generate a list of CSV-able data in the order
906 specified by propnames for the given node.
907 '''
908 properties = self.getprops()
909 l = []
910 for prop in propnames:
911 proptype = properties[prop]
912 value = self.get(nodeid, prop)
913 # "marshal" data where needed
914 if value is None:
915 pass
916 elif isinstance(proptype, hyperdb.Date):
917 value = value.get_tuple()
918 elif isinstance(proptype, hyperdb.Interval):
919 value = value.get_tuple()
920 elif isinstance(proptype, hyperdb.Password):
921 value = str(value)
922 l.append(repr(value))
923 return l
925 def import_list(self, propnames, proplist):
926 ''' Import a node - all information including "id" is present and
927 should not be sanity checked. Triggers are not triggered. The
928 journal should be initialised using the "creator" and "created"
929 information.
931 Return the nodeid of the node imported.
932 '''
933 if self.db.journaltag is None:
934 raise DatabaseError, 'Database open read-only'
935 properties = self.getprops()
937 # make the new node's property map
938 d = {}
939 for i in range(len(propnames)):
940 # Use eval to reverse the repr() used to output the CSV
941 value = eval(proplist[i])
943 # Figure the property for this column
944 propname = propnames[i]
945 prop = properties[propname]
947 # "unmarshal" where necessary
948 if propname == 'id':
949 newid = value
950 continue
951 elif value is None:
952 # don't set Nones
953 continue
954 elif isinstance(prop, hyperdb.Date):
955 value = date.Date(value)
956 elif isinstance(prop, hyperdb.Interval):
957 value = date.Interval(value)
958 elif isinstance(prop, hyperdb.Password):
959 pwd = password.Password()
960 pwd.unpack(value)
961 value = pwd
962 d[propname] = value
964 # add the node and journal
965 self.db.addnode(self.classname, newid, d)
967 # extract the journalling stuff and nuke it
968 if d.has_key('creator'):
969 creator = d['creator']
970 del d['creator']
971 else:
972 creator = None
973 if d.has_key('creation'):
974 creation = d['creation']
975 del d['creation']
976 else:
977 creation = None
978 if d.has_key('activity'):
979 del d['activity']
981 self.db.addjournal(self.classname, newid, 'create', d, creator,
982 creation)
983 return newid
985 def get(self, nodeid, propname, default=_marker, cache=1):
986 '''Get the value of a property on an existing node of this class.
988 'nodeid' must be the id of an existing node of this class or an
989 IndexError is raised. 'propname' must be the name of a property
990 of this class or a KeyError is raised.
992 'cache' indicates whether the transaction cache should be queried
993 for the node. If the node has been modified and you need to
994 determine what its values prior to modification are, you need to
995 set cache=0.
997 Attempts to get the "creation" or "activity" properties should
998 do the right thing.
999 '''
1000 if propname == 'id':
1001 return nodeid
1003 # get the node's dict
1004 d = self.db.getnode(self.classname, nodeid, cache=cache)
1006 # check for one of the special props
1007 if propname == 'creation':
1008 if d.has_key('creation'):
1009 return d['creation']
1010 if not self.do_journal:
1011 raise ValueError, 'Journalling is disabled for this class'
1012 journal = self.db.getjournal(self.classname, nodeid)
1013 if journal:
1014 return self.db.getjournal(self.classname, nodeid)[0][1]
1015 else:
1016 # on the strange chance that there's no journal
1017 return date.Date()
1018 if propname == 'activity':
1019 if d.has_key('activity'):
1020 return d['activity']
1021 if not self.do_journal:
1022 raise ValueError, 'Journalling is disabled for this class'
1023 journal = self.db.getjournal(self.classname, nodeid)
1024 if journal:
1025 return self.db.getjournal(self.classname, nodeid)[-1][1]
1026 else:
1027 # on the strange chance that there's no journal
1028 return date.Date()
1029 if propname == 'creator':
1030 if d.has_key('creator'):
1031 return d['creator']
1032 if not self.do_journal:
1033 raise ValueError, 'Journalling is disabled for this class'
1034 journal = self.db.getjournal(self.classname, nodeid)
1035 if journal:
1036 num_re = re.compile('^\d+$')
1037 value = self.db.getjournal(self.classname, nodeid)[0][2]
1038 if num_re.match(value):
1039 return value
1040 else:
1041 # old-style "username" journal tag
1042 try:
1043 return self.db.user.lookup(value)
1044 except KeyError:
1045 # user's been retired, return admin
1046 return '1'
1047 else:
1048 return self.db.curuserid
1050 # get the property (raises KeyErorr if invalid)
1051 prop = self.properties[propname]
1053 if not d.has_key(propname):
1054 if default is _marker:
1055 if isinstance(prop, Multilink):
1056 return []
1057 else:
1058 return None
1059 else:
1060 return default
1062 # return a dupe of the list so code doesn't get confused
1063 if isinstance(prop, Multilink):
1064 return d[propname][:]
1066 return d[propname]
1068 # not in spec
1069 def getnode(self, nodeid, cache=1):
1070 ''' Return a convenience wrapper for the node.
1072 'nodeid' must be the id of an existing node of this class or an
1073 IndexError is raised.
1075 'cache' indicates whether the transaction cache should be queried
1076 for the node. If the node has been modified and you need to
1077 determine what its values prior to modification are, you need to
1078 set cache=0.
1079 '''
1080 return Node(self, nodeid, cache=cache)
1082 def set(self, nodeid, **propvalues):
1083 '''Modify a property on an existing node of this class.
1085 'nodeid' must be the id of an existing node of this class or an
1086 IndexError is raised.
1088 Each key in 'propvalues' must be the name of a property of this
1089 class or a KeyError is raised.
1091 All values in 'propvalues' must be acceptable types for their
1092 corresponding properties or a TypeError is raised.
1094 If the value of the key property is set, it must not collide with
1095 other key strings or a ValueError is raised.
1097 If the value of a Link or Multilink property contains an invalid
1098 node id, a ValueError is raised.
1100 These operations trigger detectors and can be vetoed. Attempts
1101 to modify the "creation" or "activity" properties cause a KeyError.
1102 '''
1103 if not propvalues:
1104 return propvalues
1106 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1107 raise KeyError, '"creation" and "activity" are reserved'
1109 if propvalues.has_key('id'):
1110 raise KeyError, '"id" is reserved'
1112 if self.db.journaltag is None:
1113 raise DatabaseError, 'Database open read-only'
1115 self.fireAuditors('set', nodeid, propvalues)
1116 # Take a copy of the node dict so that the subsequent set
1117 # operation doesn't modify the oldvalues structure.
1118 try:
1119 # try not using the cache initially
1120 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid,
1121 cache=0))
1122 except IndexError:
1123 # this will be needed if somone does a create() and set()
1124 # with no intervening commit()
1125 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1127 node = self.db.getnode(self.classname, nodeid)
1128 if node.has_key(self.db.RETIRED_FLAG):
1129 raise IndexError
1130 num_re = re.compile('^\d+$')
1132 # if the journal value is to be different, store it in here
1133 journalvalues = {}
1135 for propname, value in propvalues.items():
1136 # check to make sure we're not duplicating an existing key
1137 if propname == self.key and node[propname] != value:
1138 try:
1139 self.lookup(value)
1140 except KeyError:
1141 pass
1142 else:
1143 raise ValueError, 'node with key "%s" exists'%value
1145 # this will raise the KeyError if the property isn't valid
1146 # ... we don't use getprops() here because we only care about
1147 # the writeable properties.
1148 try:
1149 prop = self.properties[propname]
1150 except KeyError:
1151 raise KeyError, '"%s" has no property named "%s"'%(
1152 self.classname, propname)
1154 # if the value's the same as the existing value, no sense in
1155 # doing anything
1156 if node.has_key(propname) and value == node[propname]:
1157 del propvalues[propname]
1158 continue
1160 # do stuff based on the prop type
1161 if isinstance(prop, Link):
1162 link_class = prop.classname
1163 # if it isn't a number, it's a key
1164 if value is not None and not isinstance(value, type('')):
1165 raise ValueError, 'property "%s" link value be a string'%(
1166 propname)
1167 if isinstance(value, type('')) and not num_re.match(value):
1168 try:
1169 value = self.db.classes[link_class].lookup(value)
1170 except (TypeError, KeyError):
1171 raise IndexError, 'new property "%s": %s not a %s'%(
1172 propname, value, prop.classname)
1174 if (value is not None and
1175 not self.db.getclass(link_class).hasnode(value)):
1176 raise IndexError, '%s has no node %s'%(link_class, value)
1178 if self.do_journal and prop.do_journal:
1179 # register the unlink with the old linked node
1180 if node[propname] is not None:
1181 self.db.addjournal(link_class, node[propname], 'unlink',
1182 (self.classname, nodeid, propname))
1184 # register the link with the newly linked node
1185 if value is not None:
1186 self.db.addjournal(link_class, value, 'link',
1187 (self.classname, nodeid, propname))
1189 elif isinstance(prop, Multilink):
1190 if type(value) != type([]):
1191 raise TypeError, 'new property "%s" not a list of'\
1192 ' ids'%propname
1193 link_class = self.properties[propname].classname
1194 l = []
1195 for entry in value:
1196 # if it isn't a number, it's a key
1197 if type(entry) != type(''):
1198 raise ValueError, 'new property "%s" link value ' \
1199 'must be a string'%propname
1200 if not num_re.match(entry):
1201 try:
1202 entry = self.db.classes[link_class].lookup(entry)
1203 except (TypeError, KeyError):
1204 raise IndexError, 'new property "%s": %s not a %s'%(
1205 propname, entry,
1206 self.properties[propname].classname)
1207 l.append(entry)
1208 value = l
1209 propvalues[propname] = value
1211 # figure the journal entry for this property
1212 add = []
1213 remove = []
1215 # handle removals
1216 if node.has_key(propname):
1217 l = node[propname]
1218 else:
1219 l = []
1220 for id in l[:]:
1221 if id in value:
1222 continue
1223 # register the unlink with the old linked node
1224 if self.do_journal and self.properties[propname].do_journal:
1225 self.db.addjournal(link_class, id, 'unlink',
1226 (self.classname, nodeid, propname))
1227 l.remove(id)
1228 remove.append(id)
1230 # handle additions
1231 for id in value:
1232 if not self.db.getclass(link_class).hasnode(id):
1233 raise IndexError, '%s has no node %s'%(link_class, id)
1234 if id in l:
1235 continue
1236 # register the link with the newly linked node
1237 if self.do_journal and self.properties[propname].do_journal:
1238 self.db.addjournal(link_class, id, 'link',
1239 (self.classname, nodeid, propname))
1240 l.append(id)
1241 add.append(id)
1243 # figure the journal entry
1244 l = []
1245 if add:
1246 l.append(('+', add))
1247 if remove:
1248 l.append(('-', remove))
1249 if l:
1250 journalvalues[propname] = tuple(l)
1252 elif isinstance(prop, String):
1253 if value is not None and type(value) != type(''):
1254 raise TypeError, 'new property "%s" not a string'%propname
1256 elif isinstance(prop, Password):
1257 if not isinstance(value, password.Password):
1258 raise TypeError, 'new property "%s" not a Password'%propname
1259 propvalues[propname] = value
1261 elif value is not None and isinstance(prop, Date):
1262 if not isinstance(value, date.Date):
1263 raise TypeError, 'new property "%s" not a Date'% propname
1264 propvalues[propname] = value
1266 elif value is not None and isinstance(prop, Interval):
1267 if not isinstance(value, date.Interval):
1268 raise TypeError, 'new property "%s" not an '\
1269 'Interval'%propname
1270 propvalues[propname] = value
1272 elif value is not None and isinstance(prop, Number):
1273 try:
1274 float(value)
1275 except ValueError:
1276 raise TypeError, 'new property "%s" not numeric'%propname
1278 elif value is not None and isinstance(prop, Boolean):
1279 try:
1280 int(value)
1281 except ValueError:
1282 raise TypeError, 'new property "%s" not boolean'%propname
1284 node[propname] = value
1286 # nothing to do?
1287 if not propvalues:
1288 return propvalues
1290 # do the set, and journal it
1291 self.db.setnode(self.classname, nodeid, node)
1293 if self.do_journal:
1294 propvalues.update(journalvalues)
1295 self.db.addjournal(self.classname, nodeid, 'set', propvalues)
1297 self.fireReactors('set', nodeid, oldvalues)
1299 return propvalues
1301 def retire(self, nodeid):
1302 '''Retire a node.
1304 The properties on the node remain available from the get() method,
1305 and the node's id is never reused.
1307 Retired nodes are not returned by the find(), list(), or lookup()
1308 methods, and other nodes may reuse the values of their key properties.
1310 These operations trigger detectors and can be vetoed. Attempts
1311 to modify the "creation" or "activity" properties cause a KeyError.
1312 '''
1313 if self.db.journaltag is None:
1314 raise DatabaseError, 'Database open read-only'
1316 self.fireAuditors('retire', nodeid, None)
1318 node = self.db.getnode(self.classname, nodeid)
1319 node[self.db.RETIRED_FLAG] = 1
1320 self.db.setnode(self.classname, nodeid, node)
1321 if self.do_journal:
1322 self.db.addjournal(self.classname, nodeid, 'retired', None)
1324 self.fireReactors('retire', nodeid, None)
1326 def is_retired(self, nodeid):
1327 '''Return true if the node is retired.
1328 '''
1329 node = self.db.getnode(cn, nodeid, cldb)
1330 if node.has_key(self.db.RETIRED_FLAG):
1331 return 1
1332 return 0
1334 def destroy(self, nodeid):
1335 '''Destroy a node.
1337 WARNING: this method should never be used except in extremely rare
1338 situations where there could never be links to the node being
1339 deleted
1340 WARNING: use retire() instead
1341 WARNING: the properties of this node will not be available ever again
1342 WARNING: really, use retire() instead
1344 Well, I think that's enough warnings. This method exists mostly to
1345 support the session storage of the cgi interface.
1346 '''
1347 if self.db.journaltag is None:
1348 raise DatabaseError, 'Database open read-only'
1349 self.db.destroynode(self.classname, nodeid)
1351 def history(self, nodeid):
1352 '''Retrieve the journal of edits on a particular node.
1354 'nodeid' must be the id of an existing node of this class or an
1355 IndexError is raised.
1357 The returned list contains tuples of the form
1359 (date, tag, action, params)
1361 'date' is a Timestamp object specifying the time of the change and
1362 'tag' is the journaltag specified when the database was opened.
1363 '''
1364 if not self.do_journal:
1365 raise ValueError, 'Journalling is disabled for this class'
1366 return self.db.getjournal(self.classname, nodeid)
1368 # Locating nodes:
1369 def hasnode(self, nodeid):
1370 '''Determine if the given nodeid actually exists
1371 '''
1372 return self.db.hasnode(self.classname, nodeid)
1374 def setkey(self, propname):
1375 '''Select a String property of this class to be the key property.
1377 'propname' must be the name of a String property of this class or
1378 None, or a TypeError is raised. The values of the key property on
1379 all existing nodes must be unique or a ValueError is raised. If the
1380 property doesn't exist, KeyError is raised.
1381 '''
1382 prop = self.getprops()[propname]
1383 if not isinstance(prop, String):
1384 raise TypeError, 'key properties must be String'
1385 self.key = propname
1387 def getkey(self):
1388 '''Return the name of the key property for this class or None.'''
1389 return self.key
1391 def labelprop(self, default_to_id=0):
1392 ''' Return the property name for a label for the given node.
1394 This method attempts to generate a consistent label for the node.
1395 It tries the following in order:
1396 1. key property
1397 2. "name" property
1398 3. "title" property
1399 4. first property from the sorted property name list
1400 '''
1401 k = self.getkey()
1402 if k:
1403 return k
1404 props = self.getprops()
1405 if props.has_key('name'):
1406 return 'name'
1407 elif props.has_key('title'):
1408 return 'title'
1409 if default_to_id:
1410 return 'id'
1411 props = props.keys()
1412 props.sort()
1413 return props[0]
1415 # TODO: set up a separate index db file for this? profile?
1416 def lookup(self, keyvalue):
1417 '''Locate a particular node by its key property and return its id.
1419 If this class has no key property, a TypeError is raised. If the
1420 'keyvalue' matches one of the values for the key property among
1421 the nodes in this class, the matching node's id is returned;
1422 otherwise a KeyError is raised.
1423 '''
1424 if not self.key:
1425 raise TypeError, 'No key property set for class %s'%self.classname
1426 cldb = self.db.getclassdb(self.classname)
1427 try:
1428 for nodeid in self.db.getnodeids(self.classname, cldb):
1429 node = self.db.getnode(self.classname, nodeid, cldb)
1430 if node.has_key(self.db.RETIRED_FLAG):
1431 continue
1432 if node[self.key] == keyvalue:
1433 cldb.close()
1434 return nodeid
1435 finally:
1436 cldb.close()
1437 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1438 keyvalue, self.classname)
1440 # change from spec - allows multiple props to match
1441 def find(self, **propspec):
1442 '''Get the ids of nodes in this class which link to the given nodes.
1444 'propspec' consists of keyword args propname={nodeid:1,}
1445 'propname' must be the name of a property in this class, or a
1446 KeyError is raised. That property must be a Link or Multilink
1447 property, or a TypeError is raised.
1449 Any node in this class whose 'propname' property links to any of the
1450 nodeids will be returned. Used by the full text indexing, which knows
1451 that "foo" occurs in msg1, msg3 and file7, so we have hits on these issues:
1452 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1453 '''
1454 propspec = propspec.items()
1455 for propname, nodeids in propspec:
1456 # check the prop is OK
1457 prop = self.properties[propname]
1458 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1459 raise TypeError, "'%s' not a Link/Multilink property"%propname
1461 # ok, now do the find
1462 cldb = self.db.getclassdb(self.classname)
1463 l = []
1464 try:
1465 for id in self.db.getnodeids(self.classname, db=cldb):
1466 node = self.db.getnode(self.classname, id, db=cldb)
1467 if node.has_key(self.db.RETIRED_FLAG):
1468 continue
1469 for propname, nodeids in propspec:
1470 # can't test if the node doesn't have this property
1471 if not node.has_key(propname):
1472 continue
1473 if type(nodeids) is type(''):
1474 nodeids = {nodeids:1}
1475 prop = self.properties[propname]
1476 value = node[propname]
1477 if isinstance(prop, Link) and nodeids.has_key(value):
1478 l.append(id)
1479 break
1480 elif isinstance(prop, Multilink):
1481 hit = 0
1482 for v in value:
1483 if nodeids.has_key(v):
1484 l.append(id)
1485 hit = 1
1486 break
1487 if hit:
1488 break
1489 finally:
1490 cldb.close()
1491 return l
1493 def stringFind(self, **requirements):
1494 '''Locate a particular node by matching a set of its String
1495 properties in a caseless search.
1497 If the property is not a String property, a TypeError is raised.
1499 The return is a list of the id of all nodes that match.
1500 '''
1501 for propname in requirements.keys():
1502 prop = self.properties[propname]
1503 if isinstance(not prop, String):
1504 raise TypeError, "'%s' not a String property"%propname
1505 requirements[propname] = requirements[propname].lower()
1506 l = []
1507 cldb = self.db.getclassdb(self.classname)
1508 try:
1509 for nodeid in self.db.getnodeids(self.classname, cldb):
1510 node = self.db.getnode(self.classname, nodeid, cldb)
1511 if node.has_key(self.db.RETIRED_FLAG):
1512 continue
1513 for key, value in requirements.items():
1514 if node[key] is None or node[key].lower() != value:
1515 break
1516 else:
1517 l.append(nodeid)
1518 finally:
1519 cldb.close()
1520 return l
1522 def list(self):
1523 ''' Return a list of the ids of the active nodes in this class.
1524 '''
1525 l = []
1526 cn = self.classname
1527 cldb = self.db.getclassdb(cn)
1528 try:
1529 for nodeid in self.db.getnodeids(cn, cldb):
1530 node = self.db.getnode(cn, nodeid, cldb)
1531 if node.has_key(self.db.RETIRED_FLAG):
1532 continue
1533 l.append(nodeid)
1534 finally:
1535 cldb.close()
1536 l.sort()
1537 return l
1539 def filter(self, search_matches, filterspec, sort, group,
1540 num_re = re.compile('^\d+$')):
1541 ''' Return a list of the ids of the active nodes in this class that
1542 match the 'filter' spec, sorted by the group spec and then the
1543 sort spec.
1545 "filterspec" is {propname: value(s)}
1546 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1547 and prop is a prop name or None
1548 "search_matches" is {nodeid: marker}
1550 The filter must match all properties specificed - but if the
1551 property value to match is a list, any one of the values in the
1552 list may match for that property to match.
1553 '''
1554 cn = self.classname
1556 # optimise filterspec
1557 l = []
1558 props = self.getprops()
1559 LINK = 0
1560 MULTILINK = 1
1561 STRING = 2
1562 OTHER = 6
1563 for k, v in filterspec.items():
1564 propclass = props[k]
1565 if isinstance(propclass, Link):
1566 if type(v) is not type([]):
1567 v = [v]
1568 # replace key values with node ids
1569 u = []
1570 link_class = self.db.classes[propclass.classname]
1571 for entry in v:
1572 if entry == '-1': entry = None
1573 elif not num_re.match(entry):
1574 try:
1575 entry = link_class.lookup(entry)
1576 except (TypeError,KeyError):
1577 raise ValueError, 'property "%s": %s not a %s'%(
1578 k, entry, self.properties[k].classname)
1579 u.append(entry)
1581 l.append((LINK, k, u))
1582 elif isinstance(propclass, Multilink):
1583 if type(v) is not type([]):
1584 v = [v]
1585 # replace key values with node ids
1586 u = []
1587 link_class = self.db.classes[propclass.classname]
1588 for entry in v:
1589 if not num_re.match(entry):
1590 try:
1591 entry = link_class.lookup(entry)
1592 except (TypeError,KeyError):
1593 raise ValueError, 'new property "%s": %s not a %s'%(
1594 k, entry, self.properties[k].classname)
1595 u.append(entry)
1596 l.append((MULTILINK, k, u))
1597 elif isinstance(propclass, String):
1598 # simple glob searching
1599 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1600 v = v.replace('?', '.')
1601 v = v.replace('*', '.*?')
1602 l.append((STRING, k, re.compile(v, re.I)))
1603 elif isinstance(propclass, Boolean):
1604 if type(v) is type(''):
1605 bv = v.lower() in ('yes', 'true', 'on', '1')
1606 else:
1607 bv = v
1608 l.append((OTHER, k, bv))
1609 elif isinstance(propclass, Number):
1610 l.append((OTHER, k, int(v)))
1611 else:
1612 l.append((OTHER, k, v))
1613 filterspec = l
1615 # now, find all the nodes that are active and pass filtering
1616 l = []
1617 cldb = self.db.getclassdb(cn)
1618 try:
1619 # TODO: only full-scan once (use items())
1620 for nodeid in self.db.getnodeids(cn, cldb):
1621 node = self.db.getnode(cn, nodeid, cldb)
1622 if node.has_key(self.db.RETIRED_FLAG):
1623 continue
1624 # apply filter
1625 for t, k, v in filterspec:
1626 # make sure the node has the property
1627 if not node.has_key(k):
1628 # this node doesn't have this property, so reject it
1629 break
1631 # now apply the property filter
1632 if t == LINK:
1633 # link - if this node's property doesn't appear in the
1634 # filterspec's nodeid list, skip it
1635 if node[k] not in v:
1636 break
1637 elif t == MULTILINK:
1638 # multilink - if any of the nodeids required by the
1639 # filterspec aren't in this node's property, then skip
1640 # it
1641 have = node[k]
1642 for want in v:
1643 if want not in have:
1644 break
1645 else:
1646 continue
1647 break
1648 elif t == STRING:
1649 # RE search
1650 if node[k] is None or not v.search(node[k]):
1651 break
1652 elif t == OTHER:
1653 # straight value comparison for the other types
1654 if node[k] != v:
1655 break
1656 else:
1657 l.append((nodeid, node))
1658 finally:
1659 cldb.close()
1660 l.sort()
1662 # filter based on full text search
1663 if search_matches is not None:
1664 k = []
1665 for v in l:
1666 if search_matches.has_key(v[0]):
1667 k.append(v)
1668 l = k
1670 # now, sort the result
1671 def sortfun(a, b, sort=sort, group=group, properties=self.getprops(),
1672 db = self.db, cl=self):
1673 a_id, an = a
1674 b_id, bn = b
1675 # sort by group and then sort
1676 for dir, prop in group, sort:
1677 if dir is None or prop is None: continue
1679 # sorting is class-specific
1680 propclass = properties[prop]
1682 # handle the properties that might be "faked"
1683 # also, handle possible missing properties
1684 try:
1685 if not an.has_key(prop):
1686 an[prop] = cl.get(a_id, prop)
1687 av = an[prop]
1688 except KeyError:
1689 # the node doesn't have a value for this property
1690 if isinstance(propclass, Multilink): av = []
1691 else: av = ''
1692 try:
1693 if not bn.has_key(prop):
1694 bn[prop] = cl.get(b_id, prop)
1695 bv = bn[prop]
1696 except KeyError:
1697 # the node doesn't have a value for this property
1698 if isinstance(propclass, Multilink): bv = []
1699 else: bv = ''
1701 # String and Date values are sorted in the natural way
1702 if isinstance(propclass, String):
1703 # clean up the strings
1704 if av and av[0] in string.uppercase:
1705 av = an[prop] = av.lower()
1706 if bv and bv[0] in string.uppercase:
1707 bv = bn[prop] = bv.lower()
1708 if (isinstance(propclass, String) or
1709 isinstance(propclass, Date)):
1710 # it might be a string that's really an integer
1711 try:
1712 av = int(av)
1713 bv = int(bv)
1714 except:
1715 pass
1716 if dir == '+':
1717 r = cmp(av, bv)
1718 if r != 0: return r
1719 elif dir == '-':
1720 r = cmp(bv, av)
1721 if r != 0: return r
1723 # Link properties are sorted according to the value of
1724 # the "order" property on the linked nodes if it is
1725 # present; or otherwise on the key string of the linked
1726 # nodes; or finally on the node ids.
1727 elif isinstance(propclass, Link):
1728 link = db.classes[propclass.classname]
1729 if av is None and bv is not None: return -1
1730 if av is not None and bv is None: return 1
1731 if av is None and bv is None: continue
1732 if link.getprops().has_key('order'):
1733 if dir == '+':
1734 r = cmp(link.get(av, 'order'),
1735 link.get(bv, 'order'))
1736 if r != 0: return r
1737 elif dir == '-':
1738 r = cmp(link.get(bv, 'order'),
1739 link.get(av, 'order'))
1740 if r != 0: return r
1741 elif link.getkey():
1742 key = link.getkey()
1743 if dir == '+':
1744 r = cmp(link.get(av, key), link.get(bv, key))
1745 if r != 0: return r
1746 elif dir == '-':
1747 r = cmp(link.get(bv, key), link.get(av, key))
1748 if r != 0: return r
1749 else:
1750 if dir == '+':
1751 r = cmp(av, bv)
1752 if r != 0: return r
1753 elif dir == '-':
1754 r = cmp(bv, av)
1755 if r != 0: return r
1757 # Multilink properties are sorted according to how many
1758 # links are present.
1759 elif isinstance(propclass, Multilink):
1760 if dir == '+':
1761 r = cmp(len(av), len(bv))
1762 if r != 0: return r
1763 elif dir == '-':
1764 r = cmp(len(bv), len(av))
1765 if r != 0: return r
1766 elif isinstance(propclass, Number) or isinstance(propclass, Boolean):
1767 if dir == '+':
1768 r = cmp(av, bv)
1769 elif dir == '-':
1770 r = cmp(bv, av)
1772 # end for dir, prop in sort, group:
1773 # if all else fails, compare the ids
1774 return cmp(a[0], b[0])
1776 l.sort(sortfun)
1777 return [i[0] for i in l]
1779 def count(self):
1780 '''Get the number of nodes in this class.
1782 If the returned integer is 'numnodes', the ids of all the nodes
1783 in this class run from 1 to numnodes, and numnodes+1 will be the
1784 id of the next node to be created in this class.
1785 '''
1786 return self.db.countnodes(self.classname)
1788 # Manipulating properties:
1790 def getprops(self, protected=1):
1791 '''Return a dictionary mapping property names to property objects.
1792 If the "protected" flag is true, we include protected properties -
1793 those which may not be modified.
1795 In addition to the actual properties on the node, these
1796 methods provide the "creation" and "activity" properties. If the
1797 "protected" flag is true, we include protected properties - those
1798 which may not be modified.
1799 '''
1800 d = self.properties.copy()
1801 if protected:
1802 d['id'] = String()
1803 d['creation'] = hyperdb.Date()
1804 d['activity'] = hyperdb.Date()
1805 d['creator'] = hyperdb.Link('user')
1806 return d
1808 def addprop(self, **properties):
1809 '''Add properties to this class.
1811 The keyword arguments in 'properties' must map names to property
1812 objects, or a TypeError is raised. None of the keys in 'properties'
1813 may collide with the names of existing properties, or a ValueError
1814 is raised before any properties have been added.
1815 '''
1816 for key in properties.keys():
1817 if self.properties.has_key(key):
1818 raise ValueError, key
1819 self.properties.update(properties)
1821 def index(self, nodeid):
1822 '''Add (or refresh) the node to search indexes
1823 '''
1824 # find all the String properties that have indexme
1825 for prop, propclass in self.getprops().items():
1826 if isinstance(propclass, String) and propclass.indexme:
1827 try:
1828 value = str(self.get(nodeid, prop))
1829 except IndexError:
1830 # node no longer exists - entry should be removed
1831 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1832 else:
1833 # and index them under (classname, nodeid, property)
1834 self.db.indexer.add_text((self.classname, nodeid, prop),
1835 value)
1837 #
1838 # Detector interface
1839 #
1840 def audit(self, event, detector):
1841 '''Register a detector
1842 '''
1843 l = self.auditors[event]
1844 if detector not in l:
1845 self.auditors[event].append(detector)
1847 def fireAuditors(self, action, nodeid, newvalues):
1848 '''Fire all registered auditors.
1849 '''
1850 for audit in self.auditors[action]:
1851 audit(self.db, self, nodeid, newvalues)
1853 def react(self, event, detector):
1854 '''Register a detector
1855 '''
1856 l = self.reactors[event]
1857 if detector not in l:
1858 self.reactors[event].append(detector)
1860 def fireReactors(self, action, nodeid, oldvalues):
1861 '''Fire all registered reactors.
1862 '''
1863 for react in self.reactors[action]:
1864 react(self.db, self, nodeid, oldvalues)
1866 class FileClass(Class):
1867 '''This class defines a large chunk of data. To support this, it has a
1868 mandatory String property "content" which is typically saved off
1869 externally to the hyperdb.
1871 The default MIME type of this data is defined by the
1872 "default_mime_type" class attribute, which may be overridden by each
1873 node if the class defines a "type" String property.
1874 '''
1875 default_mime_type = 'text/plain'
1877 def create(self, **propvalues):
1878 ''' snaffle the file propvalue and store in a file
1879 '''
1880 content = propvalues['content']
1881 del propvalues['content']
1882 newid = Class.create(self, **propvalues)
1883 self.db.storefile(self.classname, newid, None, content)
1884 return newid
1886 def import_list(self, propnames, proplist):
1887 ''' Trap the "content" property...
1888 '''
1889 # dupe this list so we don't affect others
1890 propnames = propnames[:]
1892 # extract the "content" property from the proplist
1893 i = propnames.index('content')
1894 content = eval(proplist[i])
1895 del propnames[i]
1896 del proplist[i]
1898 # do the normal import
1899 newid = Class.import_list(self, propnames, proplist)
1901 # save off the "content" file
1902 self.db.storefile(self.classname, newid, None, content)
1903 return newid
1905 def get(self, nodeid, propname, default=_marker, cache=1):
1906 ''' trap the content propname and get it from the file
1907 '''
1909 poss_msg = 'Possibly a access right configuration problem.'
1910 if propname == 'content':
1911 try:
1912 return self.db.getfile(self.classname, nodeid, None)
1913 except IOError, (strerror):
1914 # BUG: by catching this we donot see an error in the log.
1915 return 'ERROR reading file: %s%s\n%s\n%s'%(
1916 self.classname, nodeid, poss_msg, strerror)
1917 if default is not _marker:
1918 return Class.get(self, nodeid, propname, default, cache=cache)
1919 else:
1920 return Class.get(self, nodeid, propname, cache=cache)
1922 def getprops(self, protected=1):
1923 ''' In addition to the actual properties on the node, these methods
1924 provide the "content" property. If the "protected" flag is true,
1925 we include protected properties - those which may not be
1926 modified.
1927 '''
1928 d = Class.getprops(self, protected=protected).copy()
1929 d['content'] = hyperdb.String()
1930 return d
1932 def index(self, nodeid):
1933 ''' Index the node in the search index.
1935 We want to index the content in addition to the normal String
1936 property indexing.
1937 '''
1938 # perform normal indexing
1939 Class.index(self, nodeid)
1941 # get the content to index
1942 content = self.get(nodeid, 'content')
1944 # figure the mime type
1945 if self.properties.has_key('type'):
1946 mime_type = self.get(nodeid, 'type')
1947 else:
1948 mime_type = self.default_mime_type
1950 # and index!
1951 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
1952 mime_type)
1954 # deviation from spec - was called ItemClass
1955 class IssueClass(Class, roundupdb.IssueClass):
1956 # Overridden methods:
1957 def __init__(self, db, classname, **properties):
1958 '''The newly-created class automatically includes the "messages",
1959 "files", "nosy", and "superseder" properties. If the 'properties'
1960 dictionary attempts to specify any of these properties or a
1961 "creation" or "activity" property, a ValueError is raised.
1962 '''
1963 if not properties.has_key('title'):
1964 properties['title'] = hyperdb.String(indexme='yes')
1965 if not properties.has_key('messages'):
1966 properties['messages'] = hyperdb.Multilink("msg")
1967 if not properties.has_key('files'):
1968 properties['files'] = hyperdb.Multilink("file")
1969 if not properties.has_key('nosy'):
1970 # note: journalling is turned off as it really just wastes
1971 # space. this behaviour may be overridden in an instance
1972 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
1973 if not properties.has_key('superseder'):
1974 properties['superseder'] = hyperdb.Multilink(classname)
1975 Class.__init__(self, db, classname, **properties)
1977 #