1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: back_anydbm.py,v 1.89 2002-10-08 04:11:14 richard Exp $
19 '''
20 This module defines a backend that saves the hyperdatabase in a database
21 chosen by anydbm. It is guaranteed to always be available in python
22 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
23 serious bugs, and is not available)
24 '''
26 import whichdb, anydbm, os, marshal, re, weakref, string, copy
27 from roundup import hyperdb, date, password, roundupdb, security
28 from blobfiles import FileStorage
29 from sessions import Sessions
30 from roundup.indexer import Indexer
31 from locking import acquire_lock, release_lock
32 from roundup.hyperdb import String, Password, Date, Interval, Link, \
33 Multilink, DatabaseError, Boolean, Number
35 #
36 # Now the database
37 #
38 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
39 '''A database for storing records containing flexible data types.
41 Transaction stuff TODO:
42 . check the timestamp of the class file and nuke the cache if it's
43 modified. Do some sort of conflict checking on the dirty stuff.
44 . perhaps detect write collisions (related to above)?
46 '''
47 def __init__(self, config, journaltag=None):
48 '''Open a hyperdatabase given a specifier to some storage.
50 The 'storagelocator' is obtained from config.DATABASE.
51 The meaning of 'storagelocator' depends on the particular
52 implementation of the hyperdatabase. It could be a file name,
53 a directory path, a socket descriptor for a connection to a
54 database over the network, etc.
56 The 'journaltag' is a token that will be attached to the journal
57 entries for any edits done on the database. If 'journaltag' is
58 None, the database is opened in read-only mode: the Class.create(),
59 Class.set(), and Class.retire() methods are disabled.
60 '''
61 self.config, self.journaltag = config, journaltag
62 self.dir = config.DATABASE
63 self.classes = {}
64 self.cache = {} # cache of nodes loaded or created
65 self.dirtynodes = {} # keep track of the dirty nodes by class
66 self.newnodes = {} # keep track of the new nodes by class
67 self.destroyednodes = {}# keep track of the destroyed nodes by class
68 self.transactions = []
69 self.indexer = Indexer(self.dir)
70 self.sessions = Sessions(self.config)
71 self.security = security.Security(self)
72 # ensure files are group readable and writable
73 os.umask(0002)
75 def post_init(self):
76 ''' Called once the schema initialisation has finished.
77 '''
78 # reindex the db if necessary
79 if self.indexer.should_reindex():
80 self.reindex()
82 # figure the "curuserid"
83 if self.journaltag is None:
84 self.curuserid = None
85 elif self.journaltag == 'admin':
86 # admin user may not exist, but always has ID 1
87 self.curuserid = '1'
88 else:
89 self.curuserid = self.user.lookup(self.journaltag)
91 def reindex(self):
92 for klass in self.classes.values():
93 for nodeid in klass.list():
94 klass.index(nodeid)
95 self.indexer.save_index()
97 def __repr__(self):
98 return '<back_anydbm instance at %x>'%id(self)
100 #
101 # Classes
102 #
103 def __getattr__(self, classname):
104 '''A convenient way of calling self.getclass(classname).'''
105 if self.classes.has_key(classname):
106 if __debug__:
107 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
108 return self.classes[classname]
109 raise AttributeError, classname
111 def addclass(self, cl):
112 if __debug__:
113 print >>hyperdb.DEBUG, 'addclass', (self, cl)
114 cn = cl.classname
115 if self.classes.has_key(cn):
116 raise ValueError, cn
117 self.classes[cn] = cl
119 def getclasses(self):
120 '''Return a list of the names of all existing classes.'''
121 if __debug__:
122 print >>hyperdb.DEBUG, 'getclasses', (self,)
123 l = self.classes.keys()
124 l.sort()
125 return l
127 def getclass(self, classname):
128 '''Get the Class object representing a particular class.
130 If 'classname' is not a valid class name, a KeyError is raised.
131 '''
132 if __debug__:
133 print >>hyperdb.DEBUG, 'getclass', (self, classname)
134 try:
135 return self.classes[classname]
136 except KeyError:
137 raise KeyError, 'There is no class called "%s"'%classname
139 #
140 # Class DBs
141 #
142 def clear(self):
143 '''Delete all database contents
144 '''
145 if __debug__:
146 print >>hyperdb.DEBUG, 'clear', (self,)
147 for cn in self.classes.keys():
148 for dummy in 'nodes', 'journals':
149 path = os.path.join(self.dir, 'journals.%s'%cn)
150 if os.path.exists(path):
151 os.remove(path)
152 elif os.path.exists(path+'.db'): # dbm appends .db
153 os.remove(path+'.db')
155 def getclassdb(self, classname, mode='r'):
156 ''' grab a connection to the class db that will be used for
157 multiple actions
158 '''
159 if __debug__:
160 print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode)
161 return self.opendb('nodes.%s'%classname, mode)
163 def determine_db_type(self, path):
164 ''' determine which DB wrote the class file
165 '''
166 db_type = ''
167 if os.path.exists(path):
168 db_type = whichdb.whichdb(path)
169 if not db_type:
170 raise DatabaseError, "Couldn't identify database type"
171 elif os.path.exists(path+'.db'):
172 # if the path ends in '.db', it's a dbm database, whether
173 # anydbm says it's dbhash or not!
174 db_type = 'dbm'
175 return db_type
177 def opendb(self, name, mode):
178 '''Low-level database opener that gets around anydbm/dbm
179 eccentricities.
180 '''
181 if __debug__:
182 print >>hyperdb.DEBUG, 'opendb', (self, name, mode)
184 # figure the class db type
185 path = os.path.join(os.getcwd(), self.dir, name)
186 db_type = self.determine_db_type(path)
188 # new database? let anydbm pick the best dbm
189 if not db_type:
190 if __debug__:
191 print >>hyperdb.DEBUG, "opendb anydbm.open(%r, 'c')"%path
192 return anydbm.open(path, 'c')
194 # open the database with the correct module
195 try:
196 dbm = __import__(db_type)
197 except ImportError:
198 raise DatabaseError, \
199 "Couldn't open database - the required module '%s'"\
200 " is not available"%db_type
201 if __debug__:
202 print >>hyperdb.DEBUG, "opendb %r.open(%r, %r)"%(db_type, path,
203 mode)
204 return dbm.open(path, mode)
206 def lockdb(self, name):
207 ''' Lock a database file
208 '''
209 path = os.path.join(os.getcwd(), self.dir, '%s.lock'%name)
210 return acquire_lock(path)
212 #
213 # Node IDs
214 #
215 def newid(self, classname):
216 ''' Generate a new id for the given class
217 '''
218 # open the ids DB - create if if doesn't exist
219 lock = self.lockdb('_ids')
220 db = self.opendb('_ids', 'c')
221 if db.has_key(classname):
222 newid = db[classname] = str(int(db[classname]) + 1)
223 else:
224 # the count() bit is transitional - older dbs won't start at 1
225 newid = str(self.getclass(classname).count()+1)
226 db[classname] = newid
227 db.close()
228 release_lock(lock)
229 return newid
231 def setid(self, classname, setid):
232 ''' Set the id counter: used during import of database
233 '''
234 # open the ids DB - create if if doesn't exist
235 lock = self.lockdb('_ids')
236 db = self.opendb('_ids', 'c')
237 db[classname] = str(setid)
238 db.close()
239 release_lock(lock)
241 #
242 # Nodes
243 #
244 def addnode(self, classname, nodeid, node):
245 ''' add the specified node to its class's db
246 '''
247 if __debug__:
248 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
250 # we'll be supplied these props if we're doing an import
251 if not node.has_key('creator'):
252 # add in the "calculated" properties (dupe so we don't affect
253 # calling code's node assumptions)
254 node = node.copy()
255 node['creator'] = self.curuserid
256 node['creation'] = node['activity'] = date.Date()
258 self.newnodes.setdefault(classname, {})[nodeid] = 1
259 self.cache.setdefault(classname, {})[nodeid] = node
260 self.savenode(classname, nodeid, node)
262 def setnode(self, classname, nodeid, node):
263 ''' change the specified node
264 '''
265 if __debug__:
266 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
267 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
269 # update the activity time (dupe so we don't affect
270 # calling code's node assumptions)
271 node = node.copy()
272 node['activity'] = date.Date()
274 # can't set without having already loaded the node
275 self.cache[classname][nodeid] = node
276 self.savenode(classname, nodeid, node)
278 def savenode(self, classname, nodeid, node):
279 ''' perform the saving of data specified by the set/addnode
280 '''
281 if __debug__:
282 print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node)
283 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
285 def getnode(self, classname, nodeid, db=None, cache=1):
286 ''' get a node from the database
287 '''
288 if __debug__:
289 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid, db)
290 if cache:
291 # try the cache
292 cache_dict = self.cache.setdefault(classname, {})
293 if cache_dict.has_key(nodeid):
294 if __debug__:
295 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
296 nodeid)
297 return cache_dict[nodeid]
299 if __debug__:
300 print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid)
302 # get from the database and save in the cache
303 if db is None:
304 db = self.getclassdb(classname)
305 if not db.has_key(nodeid):
306 raise IndexError, "no such %s %s"%(classname, nodeid)
308 # check the uncommitted, destroyed nodes
309 if (self.destroyednodes.has_key(classname) and
310 self.destroyednodes[classname].has_key(nodeid)):
311 raise IndexError, "no such %s %s"%(classname, nodeid)
313 # decode
314 res = marshal.loads(db[nodeid])
316 # reverse the serialisation
317 res = self.unserialise(classname, res)
319 # store off in the cache dict
320 if cache:
321 cache_dict[nodeid] = res
323 return res
325 def destroynode(self, classname, nodeid):
326 '''Remove a node from the database. Called exclusively by the
327 destroy() method on Class.
328 '''
329 if __debug__:
330 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
332 # remove from cache and newnodes if it's there
333 if (self.cache.has_key(classname) and
334 self.cache[classname].has_key(nodeid)):
335 del self.cache[classname][nodeid]
336 if (self.newnodes.has_key(classname) and
337 self.newnodes[classname].has_key(nodeid)):
338 del self.newnodes[classname][nodeid]
340 # see if there's any obvious commit actions that we should get rid of
341 for entry in self.transactions[:]:
342 if entry[1][:2] == (classname, nodeid):
343 self.transactions.remove(entry)
345 # add to the destroyednodes map
346 self.destroyednodes.setdefault(classname, {})[nodeid] = 1
348 # add the destroy commit action
349 self.transactions.append((self.doDestroyNode, (classname, nodeid)))
351 def serialise(self, classname, node):
352 '''Copy the node contents, converting non-marshallable data into
353 marshallable data.
354 '''
355 if __debug__:
356 print >>hyperdb.DEBUG, 'serialise', classname, node
357 properties = self.getclass(classname).getprops()
358 d = {}
359 for k, v in node.items():
360 # if the property doesn't exist, or is the "retired" flag then
361 # it won't be in the properties dict
362 if not properties.has_key(k):
363 d[k] = v
364 continue
366 # get the property spec
367 prop = properties[k]
369 if isinstance(prop, Password) and v is not None:
370 d[k] = str(v)
371 elif isinstance(prop, Date) and v is not None:
372 d[k] = v.serialise()
373 elif isinstance(prop, Interval) and v is not None:
374 d[k] = v.serialise()
375 else:
376 d[k] = v
377 return d
379 def unserialise(self, classname, node):
380 '''Decode the marshalled node data
381 '''
382 if __debug__:
383 print >>hyperdb.DEBUG, 'unserialise', classname, node
384 properties = self.getclass(classname).getprops()
385 d = {}
386 for k, v in node.items():
387 # if the property doesn't exist, or is the "retired" flag then
388 # it won't be in the properties dict
389 if not properties.has_key(k):
390 d[k] = v
391 continue
393 # get the property spec
394 prop = properties[k]
396 if isinstance(prop, Date) and v is not None:
397 d[k] = date.Date(v)
398 elif isinstance(prop, Interval) and v is not None:
399 d[k] = date.Interval(v)
400 elif isinstance(prop, Password) and v is not None:
401 p = password.Password()
402 p.unpack(v)
403 d[k] = p
404 else:
405 d[k] = v
406 return d
408 def hasnode(self, classname, nodeid, db=None):
409 ''' determine if the database has a given node
410 '''
411 if __debug__:
412 print >>hyperdb.DEBUG, 'hasnode', (self, classname, nodeid, db)
414 # try the cache
415 cache = self.cache.setdefault(classname, {})
416 if cache.has_key(nodeid):
417 if __debug__:
418 print >>hyperdb.TRACE, 'has %s %s cached'%(classname, nodeid)
419 return 1
420 if __debug__:
421 print >>hyperdb.TRACE, 'has %s %s'%(classname, nodeid)
423 # not in the cache - check the database
424 if db is None:
425 db = self.getclassdb(classname)
426 res = db.has_key(nodeid)
427 return res
429 def countnodes(self, classname, db=None):
430 if __debug__:
431 print >>hyperdb.DEBUG, 'countnodes', (self, classname, db)
433 count = 0
435 # include the uncommitted nodes
436 if self.newnodes.has_key(classname):
437 count += len(self.newnodes[classname])
438 if self.destroyednodes.has_key(classname):
439 count -= len(self.destroyednodes[classname])
441 # and count those in the DB
442 if db is None:
443 db = self.getclassdb(classname)
444 count = count + len(db.keys())
445 return count
447 def getnodeids(self, classname, db=None):
448 if __debug__:
449 print >>hyperdb.DEBUG, 'getnodeids', (self, classname, db)
451 res = []
453 # start off with the new nodes
454 if self.newnodes.has_key(classname):
455 res += self.newnodes[classname].keys()
457 if db is None:
458 db = self.getclassdb(classname)
459 res = res + db.keys()
461 # remove the uncommitted, destroyed nodes
462 if self.destroyednodes.has_key(classname):
463 for nodeid in self.destroyednodes[classname].keys():
464 if db.has_key(nodeid):
465 res.remove(nodeid)
467 return res
470 #
471 # Files - special node properties
472 # inherited from FileStorage
474 #
475 # Journal
476 #
477 def addjournal(self, classname, nodeid, action, params, creator=None,
478 creation=None):
479 ''' Journal the Action
480 'action' may be:
482 'create' or 'set' -- 'params' is a dictionary of property values
483 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
484 'retire' -- 'params' is None
485 '''
486 if __debug__:
487 print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid,
488 action, params, creator, creation)
489 self.transactions.append((self.doSaveJournal, (classname, nodeid,
490 action, params, creator, creation)))
492 def getjournal(self, classname, nodeid):
493 ''' get the journal for id
495 Raise IndexError if the node doesn't exist (as per history()'s
496 API)
497 '''
498 if __debug__:
499 print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid)
500 # attempt to open the journal - in some rare cases, the journal may
501 # not exist
502 try:
503 db = self.opendb('journals.%s'%classname, 'r')
504 except anydbm.error, error:
505 if str(error) == "need 'c' or 'n' flag to open new db":
506 raise IndexError, 'no such %s %s'%(classname, nodeid)
507 elif error.args[0] != 2:
508 raise
509 raise IndexError, 'no such %s %s'%(classname, nodeid)
510 try:
511 journal = marshal.loads(db[nodeid])
512 except KeyError:
513 db.close()
514 raise IndexError, 'no such %s %s'%(classname, nodeid)
515 db.close()
516 res = []
517 for nodeid, date_stamp, user, action, params in journal:
518 res.append((nodeid, date.Date(date_stamp), user, action, params))
519 return res
521 def pack(self, pack_before):
522 ''' Delete all journal entries except "create" before 'pack_before'.
523 '''
524 if __debug__:
525 print >>hyperdb.DEBUG, 'packjournal', (self, pack_before)
527 pack_before = pack_before.serialise()
528 for classname in self.getclasses():
529 # get the journal db
530 db_name = 'journals.%s'%classname
531 path = os.path.join(os.getcwd(), self.dir, classname)
532 db_type = self.determine_db_type(path)
533 db = self.opendb(db_name, 'w')
535 for key in db.keys():
536 # get the journal for this db entry
537 journal = marshal.loads(db[key])
538 l = []
539 last_set_entry = None
540 for entry in journal:
541 # unpack the entry
542 (nodeid, date_stamp, self.journaltag, action,
543 params) = entry
544 # if the entry is after the pack date, _or_ the initial
545 # create entry, then it stays
546 if date_stamp > pack_before or action == 'create':
547 l.append(entry)
548 db[key] = marshal.dumps(l)
549 if db_type == 'gdbm':
550 db.reorganize()
551 db.close()
554 #
555 # Basic transaction support
556 #
557 def commit(self):
558 ''' Commit the current transactions.
559 '''
560 if __debug__:
561 print >>hyperdb.DEBUG, 'commit', (self,)
562 # TODO: lock the DB
564 # keep a handle to all the database files opened
565 self.databases = {}
567 # now, do all the transactions
568 reindex = {}
569 for method, args in self.transactions:
570 reindex[method(*args)] = 1
572 # now close all the database files
573 for db in self.databases.values():
574 db.close()
575 del self.databases
576 # TODO: unlock the DB
578 # reindex the nodes that request it
579 for classname, nodeid in filter(None, reindex.keys()):
580 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
581 self.getclass(classname).index(nodeid)
583 # save the indexer state
584 self.indexer.save_index()
586 self.clearCache()
588 def clearCache(self):
589 # all transactions committed, back to normal
590 self.cache = {}
591 self.dirtynodes = {}
592 self.newnodes = {}
593 self.destroyednodes = {}
594 self.transactions = []
596 def getCachedClassDB(self, classname):
597 ''' get the class db, looking in our cache of databases for commit
598 '''
599 # get the database handle
600 db_name = 'nodes.%s'%classname
601 if not self.databases.has_key(db_name):
602 self.databases[db_name] = self.getclassdb(classname, 'c')
603 return self.databases[db_name]
605 def doSaveNode(self, classname, nodeid, node):
606 if __debug__:
607 print >>hyperdb.DEBUG, 'doSaveNode', (self, classname, nodeid,
608 node)
610 db = self.getCachedClassDB(classname)
612 # now save the marshalled data
613 db[nodeid] = marshal.dumps(self.serialise(classname, node))
615 # return the classname, nodeid so we reindex this content
616 return (classname, nodeid)
618 def getCachedJournalDB(self, classname):
619 ''' get the journal db, looking in our cache of databases for commit
620 '''
621 # get the database handle
622 db_name = 'journals.%s'%classname
623 if not self.databases.has_key(db_name):
624 self.databases[db_name] = self.opendb(db_name, 'c')
625 return self.databases[db_name]
627 def doSaveJournal(self, classname, nodeid, action, params, creator,
628 creation):
629 # serialise the parameters now if necessary
630 if isinstance(params, type({})):
631 if action in ('set', 'create'):
632 params = self.serialise(classname, params)
634 # handle supply of the special journalling parameters (usually
635 # supplied on importing an existing database)
636 if creator:
637 journaltag = creator
638 else:
639 journaltag = self.curuserid
640 if creation:
641 journaldate = creation.serialise()
642 else:
643 journaldate = date.Date().serialise()
645 # create the journal entry
646 entry = (nodeid, journaldate, journaltag, action, params)
648 if __debug__:
649 print >>hyperdb.DEBUG, 'doSaveJournal', entry
651 db = self.getCachedJournalDB(classname)
653 # now insert the journal entry
654 if db.has_key(nodeid):
655 # append to existing
656 s = db[nodeid]
657 l = marshal.loads(s)
658 l.append(entry)
659 else:
660 l = [entry]
662 db[nodeid] = marshal.dumps(l)
664 def doDestroyNode(self, classname, nodeid):
665 if __debug__:
666 print >>hyperdb.DEBUG, 'doDestroyNode', (self, classname, nodeid)
668 # delete from the class database
669 db = self.getCachedClassDB(classname)
670 if db.has_key(nodeid):
671 del db[nodeid]
673 # delete from the database
674 db = self.getCachedJournalDB(classname)
675 if db.has_key(nodeid):
676 del db[nodeid]
678 # return the classname, nodeid so we reindex this content
679 return (classname, nodeid)
681 def rollback(self):
682 ''' Reverse all actions from the current transaction.
683 '''
684 if __debug__:
685 print >>hyperdb.DEBUG, 'rollback', (self, )
686 for method, args in self.transactions:
687 # delete temporary files
688 if method == self.doStoreFile:
689 self.rollbackStoreFile(*args)
690 self.cache = {}
691 self.dirtynodes = {}
692 self.newnodes = {}
693 self.destroyednodes = {}
694 self.transactions = []
696 def close(self):
697 ''' Nothing to do
698 '''
699 pass
701 _marker = []
702 class Class(hyperdb.Class):
703 '''The handle to a particular class of nodes in a hyperdatabase.'''
705 def __init__(self, db, classname, **properties):
706 '''Create a new class with a given name and property specification.
708 'classname' must not collide with the name of an existing class,
709 or a ValueError is raised. The keyword arguments in 'properties'
710 must map names to property objects, or a TypeError is raised.
711 '''
712 if (properties.has_key('creation') or properties.has_key('activity')
713 or properties.has_key('creator')):
714 raise ValueError, '"creation", "activity" and "creator" are '\
715 'reserved'
717 self.classname = classname
718 self.properties = properties
719 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
720 self.key = ''
722 # should we journal changes (default yes)
723 self.do_journal = 1
725 # do the db-related init stuff
726 db.addclass(self)
728 self.auditors = {'create': [], 'set': [], 'retire': []}
729 self.reactors = {'create': [], 'set': [], 'retire': []}
731 def enableJournalling(self):
732 '''Turn journalling on for this class
733 '''
734 self.do_journal = 1
736 def disableJournalling(self):
737 '''Turn journalling off for this class
738 '''
739 self.do_journal = 0
741 # Editing nodes:
743 def create(self, **propvalues):
744 '''Create a new node of this class and return its id.
746 The keyword arguments in 'propvalues' map property names to values.
748 The values of arguments must be acceptable for the types of their
749 corresponding properties or a TypeError is raised.
751 If this class has a key property, it must be present and its value
752 must not collide with other key strings or a ValueError is raised.
754 Any other properties on this class that are missing from the
755 'propvalues' dictionary are set to None.
757 If an id in a link or multilink property does not refer to a valid
758 node, an IndexError is raised.
760 These operations trigger detectors and can be vetoed. Attempts
761 to modify the "creation" or "activity" properties cause a KeyError.
762 '''
763 if propvalues.has_key('id'):
764 raise KeyError, '"id" is reserved'
766 if self.db.journaltag is None:
767 raise DatabaseError, 'Database open read-only'
769 if propvalues.has_key('creation') or propvalues.has_key('activity'):
770 raise KeyError, '"creation" and "activity" are reserved'
772 self.fireAuditors('create', None, propvalues)
774 # new node's id
775 newid = self.db.newid(self.classname)
777 # validate propvalues
778 num_re = re.compile('^\d+$')
779 for key, value in propvalues.items():
780 if key == self.key:
781 try:
782 self.lookup(value)
783 except KeyError:
784 pass
785 else:
786 raise ValueError, 'node with key "%s" exists'%value
788 # try to handle this property
789 try:
790 prop = self.properties[key]
791 except KeyError:
792 raise KeyError, '"%s" has no property "%s"'%(self.classname,
793 key)
795 if value is not None and isinstance(prop, Link):
796 if type(value) != type(''):
797 raise ValueError, 'link value must be String'
798 link_class = self.properties[key].classname
799 # if it isn't a number, it's a key
800 if not num_re.match(value):
801 try:
802 value = self.db.classes[link_class].lookup(value)
803 except (TypeError, KeyError):
804 raise IndexError, 'new property "%s": %s not a %s'%(
805 key, value, link_class)
806 elif not self.db.getclass(link_class).hasnode(value):
807 raise IndexError, '%s has no node %s'%(link_class, value)
809 # save off the value
810 propvalues[key] = value
812 # register the link with the newly linked node
813 if self.do_journal and self.properties[key].do_journal:
814 self.db.addjournal(link_class, value, 'link',
815 (self.classname, newid, key))
817 elif isinstance(prop, Multilink):
818 if type(value) != type([]):
819 raise TypeError, 'new property "%s" not a list of ids'%key
821 # clean up and validate the list of links
822 link_class = self.properties[key].classname
823 l = []
824 for entry in value:
825 if type(entry) != type(''):
826 raise ValueError, '"%s" multilink value (%r) '\
827 'must contain Strings'%(key, value)
828 # if it isn't a number, it's a key
829 if not num_re.match(entry):
830 try:
831 entry = self.db.classes[link_class].lookup(entry)
832 except (TypeError, KeyError):
833 raise IndexError, 'new property "%s": %s not a %s'%(
834 key, entry, self.properties[key].classname)
835 l.append(entry)
836 value = l
837 propvalues[key] = value
839 # handle additions
840 for nodeid in value:
841 if not self.db.getclass(link_class).hasnode(nodeid):
842 raise IndexError, '%s has no node %s'%(link_class,
843 nodeid)
844 # register the link with the newly linked node
845 if self.do_journal and self.properties[key].do_journal:
846 self.db.addjournal(link_class, nodeid, 'link',
847 (self.classname, newid, key))
849 elif isinstance(prop, String):
850 if type(value) != type(''):
851 raise TypeError, 'new property "%s" not a string'%key
853 elif isinstance(prop, Password):
854 if not isinstance(value, password.Password):
855 raise TypeError, 'new property "%s" not a Password'%key
857 elif isinstance(prop, Date):
858 if value is not None and not isinstance(value, date.Date):
859 raise TypeError, 'new property "%s" not a Date'%key
861 elif isinstance(prop, Interval):
862 if value is not None and not isinstance(value, date.Interval):
863 raise TypeError, 'new property "%s" not an Interval'%key
865 elif value is not None and isinstance(prop, Number):
866 try:
867 float(value)
868 except ValueError:
869 raise TypeError, 'new property "%s" not numeric'%key
871 elif value is not None and isinstance(prop, Boolean):
872 try:
873 int(value)
874 except ValueError:
875 raise TypeError, 'new property "%s" not boolean'%key
877 # make sure there's data where there needs to be
878 for key, prop in self.properties.items():
879 if propvalues.has_key(key):
880 continue
881 if key == self.key:
882 raise ValueError, 'key property "%s" is required'%key
883 if isinstance(prop, Multilink):
884 propvalues[key] = []
885 else:
886 propvalues[key] = None
888 # done
889 self.db.addnode(self.classname, newid, propvalues)
890 if self.do_journal:
891 self.db.addjournal(self.classname, newid, 'create', propvalues)
893 self.fireReactors('create', newid, None)
895 return newid
897 def export_list(self, propnames, nodeid):
898 ''' Export a node - generate a list of CSV-able data in the order
899 specified by propnames for the given node.
900 '''
901 properties = self.getprops()
902 l = []
903 for prop in propnames:
904 proptype = properties[prop]
905 value = self.get(nodeid, prop)
906 # "marshal" data where needed
907 if value is None:
908 pass
909 elif isinstance(proptype, hyperdb.Date):
910 value = value.get_tuple()
911 elif isinstance(proptype, hyperdb.Interval):
912 value = value.get_tuple()
913 elif isinstance(proptype, hyperdb.Password):
914 value = str(value)
915 l.append(repr(value))
916 return l
918 def import_list(self, propnames, proplist):
919 ''' Import a node - all information including "id" is present and
920 should not be sanity checked. Triggers are not triggered. The
921 journal should be initialised using the "creator" and "created"
922 information.
924 Return the nodeid of the node imported.
925 '''
926 if self.db.journaltag is None:
927 raise DatabaseError, 'Database open read-only'
928 properties = self.getprops()
930 # make the new node's property map
931 d = {}
932 for i in range(len(propnames)):
933 # Use eval to reverse the repr() used to output the CSV
934 value = eval(proplist[i])
936 # Figure the property for this column
937 propname = propnames[i]
938 prop = properties[propname]
940 # "unmarshal" where necessary
941 if propname == 'id':
942 newid = value
943 continue
944 elif value is None:
945 # don't set Nones
946 continue
947 elif isinstance(prop, hyperdb.Date):
948 value = date.Date(value)
949 elif isinstance(prop, hyperdb.Interval):
950 value = date.Interval(value)
951 elif isinstance(prop, hyperdb.Password):
952 pwd = password.Password()
953 pwd.unpack(value)
954 value = pwd
955 d[propname] = value
957 # add the node and journal
958 self.db.addnode(self.classname, newid, d)
960 # extract the journalling stuff and nuke it
961 if d.has_key('creator'):
962 creator = d['creator']
963 del d['creator']
964 else:
965 creator = None
966 if d.has_key('creation'):
967 creation = d['creation']
968 del d['creation']
969 else:
970 creation = None
971 if d.has_key('activity'):
972 del d['activity']
973 self.db.addjournal(self.classname, newid, 'create', d, creator,
974 creation)
975 return newid
977 def get(self, nodeid, propname, default=_marker, cache=1):
978 '''Get the value of a property on an existing node of this class.
980 'nodeid' must be the id of an existing node of this class or an
981 IndexError is raised. 'propname' must be the name of a property
982 of this class or a KeyError is raised.
984 'cache' indicates whether the transaction cache should be queried
985 for the node. If the node has been modified and you need to
986 determine what its values prior to modification are, you need to
987 set cache=0.
989 Attempts to get the "creation" or "activity" properties should
990 do the right thing.
991 '''
992 if propname == 'id':
993 return nodeid
995 # get the node's dict
996 d = self.db.getnode(self.classname, nodeid, cache=cache)
998 # check for one of the special props
999 if propname == 'creation':
1000 if d.has_key('creation'):
1001 return d['creation']
1002 if not self.do_journal:
1003 raise ValueError, 'Journalling is disabled for this class'
1004 journal = self.db.getjournal(self.classname, nodeid)
1005 if journal:
1006 return self.db.getjournal(self.classname, nodeid)[0][1]
1007 else:
1008 # on the strange chance that there's no journal
1009 return date.Date()
1010 if propname == 'activity':
1011 if d.has_key('activity'):
1012 return d['activity']
1013 if not self.do_journal:
1014 raise ValueError, 'Journalling is disabled for this class'
1015 journal = self.db.getjournal(self.classname, nodeid)
1016 if journal:
1017 return self.db.getjournal(self.classname, nodeid)[-1][1]
1018 else:
1019 # on the strange chance that there's no journal
1020 return date.Date()
1021 if propname == 'creator':
1022 if d.has_key('creator'):
1023 return d['creator']
1024 if not self.do_journal:
1025 raise ValueError, 'Journalling is disabled for this class'
1026 journal = self.db.getjournal(self.classname, nodeid)
1027 if journal:
1028 num_re = re.compile('^\d+$')
1029 value = self.db.getjournal(self.classname, nodeid)[0][2]
1030 if num_re.match(value):
1031 return value
1032 else:
1033 # old-style "username" journal tag
1034 try:
1035 return self.db.user.lookup(value)
1036 except KeyError:
1037 # user's been retired, return admin
1038 return '1'
1039 else:
1040 return self.db.curuserid
1042 # get the property (raises KeyErorr if invalid)
1043 prop = self.properties[propname]
1045 if not d.has_key(propname):
1046 if default is _marker:
1047 if isinstance(prop, Multilink):
1048 return []
1049 else:
1050 return None
1051 else:
1052 return default
1054 # return a dupe of the list so code doesn't get confused
1055 if isinstance(prop, Multilink):
1056 return d[propname][:]
1058 return d[propname]
1060 # not in spec
1061 def getnode(self, nodeid, cache=1):
1062 ''' Return a convenience wrapper for the node.
1064 'nodeid' must be the id of an existing node of this class or an
1065 IndexError is raised.
1067 'cache' indicates whether the transaction cache should be queried
1068 for the node. If the node has been modified and you need to
1069 determine what its values prior to modification are, you need to
1070 set cache=0.
1071 '''
1072 return Node(self, nodeid, cache=cache)
1074 def set(self, nodeid, **propvalues):
1075 '''Modify a property on an existing node of this class.
1077 'nodeid' must be the id of an existing node of this class or an
1078 IndexError is raised.
1080 Each key in 'propvalues' must be the name of a property of this
1081 class or a KeyError is raised.
1083 All values in 'propvalues' must be acceptable types for their
1084 corresponding properties or a TypeError is raised.
1086 If the value of the key property is set, it must not collide with
1087 other key strings or a ValueError is raised.
1089 If the value of a Link or Multilink property contains an invalid
1090 node id, a ValueError is raised.
1092 These operations trigger detectors and can be vetoed. Attempts
1093 to modify the "creation" or "activity" properties cause a KeyError.
1094 '''
1095 if not propvalues:
1096 return propvalues
1098 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1099 raise KeyError, '"creation" and "activity" are reserved'
1101 if propvalues.has_key('id'):
1102 raise KeyError, '"id" is reserved'
1104 if self.db.journaltag is None:
1105 raise DatabaseError, 'Database open read-only'
1107 self.fireAuditors('set', nodeid, propvalues)
1108 # Take a copy of the node dict so that the subsequent set
1109 # operation doesn't modify the oldvalues structure.
1110 try:
1111 # try not using the cache initially
1112 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid,
1113 cache=0))
1114 except IndexError:
1115 # this will be needed if somone does a create() and set()
1116 # with no intervening commit()
1117 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1119 node = self.db.getnode(self.classname, nodeid)
1120 if node.has_key(self.db.RETIRED_FLAG):
1121 raise IndexError
1122 num_re = re.compile('^\d+$')
1124 # if the journal value is to be different, store it in here
1125 journalvalues = {}
1127 for propname, value in propvalues.items():
1128 # check to make sure we're not duplicating an existing key
1129 if propname == self.key and node[propname] != value:
1130 try:
1131 self.lookup(value)
1132 except KeyError:
1133 pass
1134 else:
1135 raise ValueError, 'node with key "%s" exists'%value
1137 # this will raise the KeyError if the property isn't valid
1138 # ... we don't use getprops() here because we only care about
1139 # the writeable properties.
1140 try:
1141 prop = self.properties[propname]
1142 except KeyError:
1143 raise KeyError, '"%s" has no property named "%s"'%(
1144 self.classname, propname)
1146 # if the value's the same as the existing value, no sense in
1147 # doing anything
1148 if node.has_key(propname) and value == node[propname]:
1149 del propvalues[propname]
1150 continue
1152 # do stuff based on the prop type
1153 if isinstance(prop, Link):
1154 link_class = prop.classname
1155 # if it isn't a number, it's a key
1156 if value is not None and not isinstance(value, type('')):
1157 raise ValueError, 'property "%s" link value be a string'%(
1158 propname)
1159 if isinstance(value, type('')) and not num_re.match(value):
1160 try:
1161 value = self.db.classes[link_class].lookup(value)
1162 except (TypeError, KeyError):
1163 raise IndexError, 'new property "%s": %s not a %s'%(
1164 propname, value, prop.classname)
1166 if (value is not None and
1167 not self.db.getclass(link_class).hasnode(value)):
1168 raise IndexError, '%s has no node %s'%(link_class, value)
1170 if self.do_journal and prop.do_journal:
1171 # register the unlink with the old linked node
1172 if node.has_key(propname) and node[propname] is not None:
1173 self.db.addjournal(link_class, node[propname], 'unlink',
1174 (self.classname, nodeid, propname))
1176 # register the link with the newly linked node
1177 if value is not None:
1178 self.db.addjournal(link_class, value, 'link',
1179 (self.classname, nodeid, propname))
1181 elif isinstance(prop, Multilink):
1182 if type(value) != type([]):
1183 raise TypeError, 'new property "%s" not a list of'\
1184 ' ids'%propname
1185 link_class = self.properties[propname].classname
1186 l = []
1187 for entry in value:
1188 # if it isn't a number, it's a key
1189 if type(entry) != type(''):
1190 raise ValueError, 'new property "%s" link value ' \
1191 'must be a string'%propname
1192 if not num_re.match(entry):
1193 try:
1194 entry = self.db.classes[link_class].lookup(entry)
1195 except (TypeError, KeyError):
1196 raise IndexError, 'new property "%s": %s not a %s'%(
1197 propname, entry,
1198 self.properties[propname].classname)
1199 l.append(entry)
1200 value = l
1201 propvalues[propname] = value
1203 # figure the journal entry for this property
1204 add = []
1205 remove = []
1207 # handle removals
1208 if node.has_key(propname):
1209 l = node[propname]
1210 else:
1211 l = []
1212 for id in l[:]:
1213 if id in value:
1214 continue
1215 # register the unlink with the old linked node
1216 if self.do_journal and self.properties[propname].do_journal:
1217 self.db.addjournal(link_class, id, 'unlink',
1218 (self.classname, nodeid, propname))
1219 l.remove(id)
1220 remove.append(id)
1222 # handle additions
1223 for id in value:
1224 if not self.db.getclass(link_class).hasnode(id):
1225 raise IndexError, '%s has no node %s'%(link_class, id)
1226 if id in l:
1227 continue
1228 # register the link with the newly linked node
1229 if self.do_journal and self.properties[propname].do_journal:
1230 self.db.addjournal(link_class, id, 'link',
1231 (self.classname, nodeid, propname))
1232 l.append(id)
1233 add.append(id)
1235 # figure the journal entry
1236 l = []
1237 if add:
1238 l.append(('+', add))
1239 if remove:
1240 l.append(('-', remove))
1241 if l:
1242 journalvalues[propname] = tuple(l)
1244 elif isinstance(prop, String):
1245 if value is not None and type(value) != type(''):
1246 raise TypeError, 'new property "%s" not a string'%propname
1248 elif isinstance(prop, Password):
1249 if not isinstance(value, password.Password):
1250 raise TypeError, 'new property "%s" not a Password'%propname
1251 propvalues[propname] = value
1253 elif value is not None and isinstance(prop, Date):
1254 if not isinstance(value, date.Date):
1255 raise TypeError, 'new property "%s" not a Date'% propname
1256 propvalues[propname] = value
1258 elif value is not None and isinstance(prop, Interval):
1259 if not isinstance(value, date.Interval):
1260 raise TypeError, 'new property "%s" not an '\
1261 'Interval'%propname
1262 propvalues[propname] = value
1264 elif value is not None and isinstance(prop, Number):
1265 try:
1266 float(value)
1267 except ValueError:
1268 raise TypeError, 'new property "%s" not numeric'%propname
1270 elif value is not None and isinstance(prop, Boolean):
1271 try:
1272 int(value)
1273 except ValueError:
1274 raise TypeError, 'new property "%s" not boolean'%propname
1276 node[propname] = value
1278 # nothing to do?
1279 if not propvalues:
1280 return propvalues
1282 # do the set, and journal it
1283 self.db.setnode(self.classname, nodeid, node)
1285 if self.do_journal:
1286 propvalues.update(journalvalues)
1287 self.db.addjournal(self.classname, nodeid, 'set', propvalues)
1289 self.fireReactors('set', nodeid, oldvalues)
1291 return propvalues
1293 def retire(self, nodeid):
1294 '''Retire a node.
1296 The properties on the node remain available from the get() method,
1297 and the node's id is never reused.
1299 Retired nodes are not returned by the find(), list(), or lookup()
1300 methods, and other nodes may reuse the values of their key properties.
1302 These operations trigger detectors and can be vetoed. Attempts
1303 to modify the "creation" or "activity" properties cause a KeyError.
1304 '''
1305 if self.db.journaltag is None:
1306 raise DatabaseError, 'Database open read-only'
1308 self.fireAuditors('retire', nodeid, None)
1310 node = self.db.getnode(self.classname, nodeid)
1311 node[self.db.RETIRED_FLAG] = 1
1312 self.db.setnode(self.classname, nodeid, node)
1313 if self.do_journal:
1314 self.db.addjournal(self.classname, nodeid, 'retired', None)
1316 self.fireReactors('retire', nodeid, None)
1318 def is_retired(self, nodeid):
1319 '''Return true if the node is retired.
1320 '''
1321 node = self.db.getnode(cn, nodeid, cldb)
1322 if node.has_key(self.db.RETIRED_FLAG):
1323 return 1
1324 return 0
1326 def destroy(self, nodeid):
1327 '''Destroy a node.
1329 WARNING: this method should never be used except in extremely rare
1330 situations where there could never be links to the node being
1331 deleted
1332 WARNING: use retire() instead
1333 WARNING: the properties of this node will not be available ever again
1334 WARNING: really, use retire() instead
1336 Well, I think that's enough warnings. This method exists mostly to
1337 support the session storage of the cgi interface.
1338 '''
1339 if self.db.journaltag is None:
1340 raise DatabaseError, 'Database open read-only'
1341 self.db.destroynode(self.classname, nodeid)
1343 def history(self, nodeid):
1344 '''Retrieve the journal of edits on a particular node.
1346 'nodeid' must be the id of an existing node of this class or an
1347 IndexError is raised.
1349 The returned list contains tuples of the form
1351 (date, tag, action, params)
1353 'date' is a Timestamp object specifying the time of the change and
1354 'tag' is the journaltag specified when the database was opened.
1355 '''
1356 if not self.do_journal:
1357 raise ValueError, 'Journalling is disabled for this class'
1358 return self.db.getjournal(self.classname, nodeid)
1360 # Locating nodes:
1361 def hasnode(self, nodeid):
1362 '''Determine if the given nodeid actually exists
1363 '''
1364 return self.db.hasnode(self.classname, nodeid)
1366 def setkey(self, propname):
1367 '''Select a String property of this class to be the key property.
1369 'propname' must be the name of a String property of this class or
1370 None, or a TypeError is raised. The values of the key property on
1371 all existing nodes must be unique or a ValueError is raised. If the
1372 property doesn't exist, KeyError is raised.
1373 '''
1374 prop = self.getprops()[propname]
1375 if not isinstance(prop, String):
1376 raise TypeError, 'key properties must be String'
1377 self.key = propname
1379 def getkey(self):
1380 '''Return the name of the key property for this class or None.'''
1381 return self.key
1383 def labelprop(self, default_to_id=0):
1384 ''' Return the property name for a label for the given node.
1386 This method attempts to generate a consistent label for the node.
1387 It tries the following in order:
1388 1. key property
1389 2. "name" property
1390 3. "title" property
1391 4. first property from the sorted property name list
1392 '''
1393 k = self.getkey()
1394 if k:
1395 return k
1396 props = self.getprops()
1397 if props.has_key('name'):
1398 return 'name'
1399 elif props.has_key('title'):
1400 return 'title'
1401 if default_to_id:
1402 return 'id'
1403 props = props.keys()
1404 props.sort()
1405 return props[0]
1407 # TODO: set up a separate index db file for this? profile?
1408 def lookup(self, keyvalue):
1409 '''Locate a particular node by its key property and return its id.
1411 If this class has no key property, a TypeError is raised. If the
1412 'keyvalue' matches one of the values for the key property among
1413 the nodes in this class, the matching node's id is returned;
1414 otherwise a KeyError is raised.
1415 '''
1416 if not self.key:
1417 raise TypeError, 'No key property set for class %s'%self.classname
1418 cldb = self.db.getclassdb(self.classname)
1419 try:
1420 for nodeid in self.db.getnodeids(self.classname, cldb):
1421 node = self.db.getnode(self.classname, nodeid, cldb)
1422 if node.has_key(self.db.RETIRED_FLAG):
1423 continue
1424 if node[self.key] == keyvalue:
1425 cldb.close()
1426 return nodeid
1427 finally:
1428 cldb.close()
1429 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1430 keyvalue, self.classname)
1432 # change from spec - allows multiple props to match
1433 def find(self, **propspec):
1434 '''Get the ids of nodes in this class which link to the given nodes.
1436 'propspec' consists of keyword args propname=nodeid or
1437 propname={nodeid:1, }
1438 'propname' must be the name of a property in this class, or a
1439 KeyError is raised. That property must be a Link or
1440 Multilink property, or a TypeError is raised.
1442 Any node in this class whose 'propname' property links to any of the
1443 nodeids will be returned. Used by the full text indexing, which knows
1444 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1445 issues:
1447 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1448 '''
1449 propspec = propspec.items()
1450 for propname, nodeids in propspec:
1451 # check the prop is OK
1452 prop = self.properties[propname]
1453 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1454 raise TypeError, "'%s' not a Link/Multilink property"%propname
1456 # ok, now do the find
1457 cldb = self.db.getclassdb(self.classname)
1458 l = []
1459 try:
1460 for id in self.db.getnodeids(self.classname, db=cldb):
1461 node = self.db.getnode(self.classname, id, db=cldb)
1462 if node.has_key(self.db.RETIRED_FLAG):
1463 continue
1464 for propname, nodeids in propspec:
1465 # can't test if the node doesn't have this property
1466 if not node.has_key(propname):
1467 continue
1468 if type(nodeids) is type(''):
1469 nodeids = {nodeids:1}
1470 prop = self.properties[propname]
1471 value = node[propname]
1472 if isinstance(prop, Link) and nodeids.has_key(value):
1473 l.append(id)
1474 break
1475 elif isinstance(prop, Multilink):
1476 hit = 0
1477 for v in value:
1478 if nodeids.has_key(v):
1479 l.append(id)
1480 hit = 1
1481 break
1482 if hit:
1483 break
1484 finally:
1485 cldb.close()
1486 return l
1488 def stringFind(self, **requirements):
1489 '''Locate a particular node by matching a set of its String
1490 properties in a caseless search.
1492 If the property is not a String property, a TypeError is raised.
1494 The return is a list of the id of all nodes that match.
1495 '''
1496 for propname in requirements.keys():
1497 prop = self.properties[propname]
1498 if isinstance(not prop, String):
1499 raise TypeError, "'%s' not a String property"%propname
1500 requirements[propname] = requirements[propname].lower()
1501 l = []
1502 cldb = self.db.getclassdb(self.classname)
1503 try:
1504 for nodeid in self.db.getnodeids(self.classname, cldb):
1505 node = self.db.getnode(self.classname, nodeid, cldb)
1506 if node.has_key(self.db.RETIRED_FLAG):
1507 continue
1508 for key, value in requirements.items():
1509 if node[key] is None or node[key].lower() != value:
1510 break
1511 else:
1512 l.append(nodeid)
1513 finally:
1514 cldb.close()
1515 return l
1517 def list(self):
1518 ''' Return a list of the ids of the active nodes in this class.
1519 '''
1520 l = []
1521 cn = self.classname
1522 cldb = self.db.getclassdb(cn)
1523 try:
1524 for nodeid in self.db.getnodeids(cn, cldb):
1525 node = self.db.getnode(cn, nodeid, cldb)
1526 if node.has_key(self.db.RETIRED_FLAG):
1527 continue
1528 l.append(nodeid)
1529 finally:
1530 cldb.close()
1531 l.sort()
1532 return l
1534 def filter(self, search_matches, filterspec, sort=(None,None),
1535 group=(None,None), num_re = re.compile('^\d+$')):
1536 ''' Return a list of the ids of the active nodes in this class that
1537 match the 'filter' spec, sorted by the group spec and then the
1538 sort spec.
1540 "filterspec" is {propname: value(s)}
1541 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1542 and prop is a prop name or None
1543 "search_matches" is {nodeid: marker}
1545 The filter must match all properties specificed - but if the
1546 property value to match is a list, any one of the values in the
1547 list may match for that property to match.
1548 '''
1549 cn = self.classname
1551 # optimise filterspec
1552 l = []
1553 props = self.getprops()
1554 LINK = 0
1555 MULTILINK = 1
1556 STRING = 2
1557 OTHER = 6
1558 for k, v in filterspec.items():
1559 propclass = props[k]
1560 if isinstance(propclass, Link):
1561 if type(v) is not type([]):
1562 v = [v]
1563 # replace key values with node ids
1564 u = []
1565 link_class = self.db.classes[propclass.classname]
1566 for entry in v:
1567 if entry == '-1': entry = None
1568 elif not num_re.match(entry):
1569 try:
1570 entry = link_class.lookup(entry)
1571 except (TypeError,KeyError):
1572 raise ValueError, 'property "%s": %s not a %s'%(
1573 k, entry, self.properties[k].classname)
1574 u.append(entry)
1576 l.append((LINK, k, u))
1577 elif isinstance(propclass, Multilink):
1578 if type(v) is not type([]):
1579 v = [v]
1580 # replace key values with node ids
1581 u = []
1582 link_class = self.db.classes[propclass.classname]
1583 for entry in v:
1584 if not num_re.match(entry):
1585 try:
1586 entry = link_class.lookup(entry)
1587 except (TypeError,KeyError):
1588 raise ValueError, 'new property "%s": %s not a %s'%(
1589 k, entry, self.properties[k].classname)
1590 u.append(entry)
1591 l.append((MULTILINK, k, u))
1592 elif isinstance(propclass, String):
1593 # simple glob searching
1594 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1595 v = v.replace('?', '.')
1596 v = v.replace('*', '.*?')
1597 l.append((STRING, k, re.compile(v, re.I)))
1598 elif isinstance(propclass, Boolean):
1599 if type(v) is type(''):
1600 bv = v.lower() in ('yes', 'true', 'on', '1')
1601 else:
1602 bv = v
1603 l.append((OTHER, k, bv))
1604 elif isinstance(propclass, Number):
1605 l.append((OTHER, k, int(v)))
1606 else:
1607 l.append((OTHER, k, v))
1608 filterspec = l
1610 # now, find all the nodes that are active and pass filtering
1611 l = []
1612 cldb = self.db.getclassdb(cn)
1613 try:
1614 # TODO: only full-scan once (use items())
1615 for nodeid in self.db.getnodeids(cn, cldb):
1616 node = self.db.getnode(cn, nodeid, cldb)
1617 if node.has_key(self.db.RETIRED_FLAG):
1618 continue
1619 # apply filter
1620 for t, k, v in filterspec:
1621 # make sure the node has the property
1622 if not node.has_key(k):
1623 # this node doesn't have this property, so reject it
1624 break
1626 # now apply the property filter
1627 if t == LINK:
1628 # link - if this node's property doesn't appear in the
1629 # filterspec's nodeid list, skip it
1630 if node[k] not in v:
1631 break
1632 elif t == MULTILINK:
1633 # multilink - if any of the nodeids required by the
1634 # filterspec aren't in this node's property, then skip
1635 # it
1636 have = node[k]
1637 for want in v:
1638 if want not in have:
1639 break
1640 else:
1641 continue
1642 break
1643 elif t == STRING:
1644 # RE search
1645 if node[k] is None or not v.search(node[k]):
1646 break
1647 elif t == OTHER:
1648 # straight value comparison for the other types
1649 if node[k] != v:
1650 break
1651 else:
1652 l.append((nodeid, node))
1653 finally:
1654 cldb.close()
1655 l.sort()
1657 # filter based on full text search
1658 if search_matches is not None:
1659 k = []
1660 for v in l:
1661 if search_matches.has_key(v[0]):
1662 k.append(v)
1663 l = k
1665 # now, sort the result
1666 def sortfun(a, b, sort=sort, group=group, properties=self.getprops(),
1667 db = self.db, cl=self):
1668 a_id, an = a
1669 b_id, bn = b
1670 # sort by group and then sort
1671 for dir, prop in group, sort:
1672 if dir is None or prop is None: continue
1674 # sorting is class-specific
1675 propclass = properties[prop]
1677 # handle the properties that might be "faked"
1678 # also, handle possible missing properties
1679 try:
1680 if not an.has_key(prop):
1681 an[prop] = cl.get(a_id, prop)
1682 av = an[prop]
1683 except KeyError:
1684 # the node doesn't have a value for this property
1685 if isinstance(propclass, Multilink): av = []
1686 else: av = ''
1687 try:
1688 if not bn.has_key(prop):
1689 bn[prop] = cl.get(b_id, prop)
1690 bv = bn[prop]
1691 except KeyError:
1692 # the node doesn't have a value for this property
1693 if isinstance(propclass, Multilink): bv = []
1694 else: bv = ''
1696 # String and Date values are sorted in the natural way
1697 if isinstance(propclass, String):
1698 # clean up the strings
1699 if av and av[0] in string.uppercase:
1700 av = an[prop] = av.lower()
1701 if bv and bv[0] in string.uppercase:
1702 bv = bn[prop] = bv.lower()
1703 if (isinstance(propclass, String) or
1704 isinstance(propclass, Date)):
1705 # it might be a string that's really an integer
1706 try:
1707 av = int(av)
1708 bv = int(bv)
1709 except:
1710 pass
1711 if dir == '+':
1712 r = cmp(av, bv)
1713 if r != 0: return r
1714 elif dir == '-':
1715 r = cmp(bv, av)
1716 if r != 0: return r
1718 # Link properties are sorted according to the value of
1719 # the "order" property on the linked nodes if it is
1720 # present; or otherwise on the key string of the linked
1721 # nodes; or finally on the node ids.
1722 elif isinstance(propclass, Link):
1723 link = db.classes[propclass.classname]
1724 if av is None and bv is not None: return -1
1725 if av is not None and bv is None: return 1
1726 if av is None and bv is None: continue
1727 if link.getprops().has_key('order'):
1728 if dir == '+':
1729 r = cmp(link.get(av, 'order'),
1730 link.get(bv, 'order'))
1731 if r != 0: return r
1732 elif dir == '-':
1733 r = cmp(link.get(bv, 'order'),
1734 link.get(av, 'order'))
1735 if r != 0: return r
1736 elif link.getkey():
1737 key = link.getkey()
1738 if dir == '+':
1739 r = cmp(link.get(av, key), link.get(bv, key))
1740 if r != 0: return r
1741 elif dir == '-':
1742 r = cmp(link.get(bv, key), link.get(av, key))
1743 if r != 0: return r
1744 else:
1745 if dir == '+':
1746 r = cmp(av, bv)
1747 if r != 0: return r
1748 elif dir == '-':
1749 r = cmp(bv, av)
1750 if r != 0: return r
1752 # Multilink properties are sorted according to how many
1753 # links are present.
1754 elif isinstance(propclass, Multilink):
1755 if dir == '+':
1756 r = cmp(len(av), len(bv))
1757 if r != 0: return r
1758 elif dir == '-':
1759 r = cmp(len(bv), len(av))
1760 if r != 0: return r
1761 elif isinstance(propclass, Number) or isinstance(propclass, Boolean):
1762 if dir == '+':
1763 r = cmp(av, bv)
1764 elif dir == '-':
1765 r = cmp(bv, av)
1767 # end for dir, prop in sort, group:
1768 # if all else fails, compare the ids
1769 return cmp(a[0], b[0])
1771 l.sort(sortfun)
1772 return [i[0] for i in l]
1774 def count(self):
1775 '''Get the number of nodes in this class.
1777 If the returned integer is 'numnodes', the ids of all the nodes
1778 in this class run from 1 to numnodes, and numnodes+1 will be the
1779 id of the next node to be created in this class.
1780 '''
1781 return self.db.countnodes(self.classname)
1783 # Manipulating properties:
1785 def getprops(self, protected=1):
1786 '''Return a dictionary mapping property names to property objects.
1787 If the "protected" flag is true, we include protected properties -
1788 those which may not be modified.
1790 In addition to the actual properties on the node, these
1791 methods provide the "creation" and "activity" properties. If the
1792 "protected" flag is true, we include protected properties - those
1793 which may not be modified.
1794 '''
1795 d = self.properties.copy()
1796 if protected:
1797 d['id'] = String()
1798 d['creation'] = hyperdb.Date()
1799 d['activity'] = hyperdb.Date()
1800 d['creator'] = hyperdb.Link('user')
1801 return d
1803 def addprop(self, **properties):
1804 '''Add properties to this class.
1806 The keyword arguments in 'properties' must map names to property
1807 objects, or a TypeError is raised. None of the keys in 'properties'
1808 may collide with the names of existing properties, or a ValueError
1809 is raised before any properties have been added.
1810 '''
1811 for key in properties.keys():
1812 if self.properties.has_key(key):
1813 raise ValueError, key
1814 self.properties.update(properties)
1816 def index(self, nodeid):
1817 '''Add (or refresh) the node to search indexes
1818 '''
1819 # find all the String properties that have indexme
1820 for prop, propclass in self.getprops().items():
1821 if isinstance(propclass, String) and propclass.indexme:
1822 try:
1823 value = str(self.get(nodeid, prop))
1824 except IndexError:
1825 # node no longer exists - entry should be removed
1826 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1827 else:
1828 # and index them under (classname, nodeid, property)
1829 self.db.indexer.add_text((self.classname, nodeid, prop),
1830 value)
1832 #
1833 # Detector interface
1834 #
1835 def audit(self, event, detector):
1836 '''Register a detector
1837 '''
1838 l = self.auditors[event]
1839 if detector not in l:
1840 self.auditors[event].append(detector)
1842 def fireAuditors(self, action, nodeid, newvalues):
1843 '''Fire all registered auditors.
1844 '''
1845 for audit in self.auditors[action]:
1846 audit(self.db, self, nodeid, newvalues)
1848 def react(self, event, detector):
1849 '''Register a detector
1850 '''
1851 l = self.reactors[event]
1852 if detector not in l:
1853 self.reactors[event].append(detector)
1855 def fireReactors(self, action, nodeid, oldvalues):
1856 '''Fire all registered reactors.
1857 '''
1858 for react in self.reactors[action]:
1859 react(self.db, self, nodeid, oldvalues)
1861 class FileClass(Class):
1862 '''This class defines a large chunk of data. To support this, it has a
1863 mandatory String property "content" which is typically saved off
1864 externally to the hyperdb.
1866 The default MIME type of this data is defined by the
1867 "default_mime_type" class attribute, which may be overridden by each
1868 node if the class defines a "type" String property.
1869 '''
1870 default_mime_type = 'text/plain'
1872 def create(self, **propvalues):
1873 ''' snaffle the file propvalue and store in a file
1874 '''
1875 content = propvalues['content']
1876 del propvalues['content']
1877 newid = Class.create(self, **propvalues)
1878 self.db.storefile(self.classname, newid, None, content)
1879 return newid
1881 def import_list(self, propnames, proplist):
1882 ''' Trap the "content" property...
1883 '''
1884 # dupe this list so we don't affect others
1885 propnames = propnames[:]
1887 # extract the "content" property from the proplist
1888 i = propnames.index('content')
1889 content = eval(proplist[i])
1890 del propnames[i]
1891 del proplist[i]
1893 # do the normal import
1894 newid = Class.import_list(self, propnames, proplist)
1896 # save off the "content" file
1897 self.db.storefile(self.classname, newid, None, content)
1898 return newid
1900 def get(self, nodeid, propname, default=_marker, cache=1):
1901 ''' trap the content propname and get it from the file
1902 '''
1904 poss_msg = 'Possibly a access right configuration problem.'
1905 if propname == 'content':
1906 try:
1907 return self.db.getfile(self.classname, nodeid, None)
1908 except IOError, (strerror):
1909 # BUG: by catching this we donot see an error in the log.
1910 return 'ERROR reading file: %s%s\n%s\n%s'%(
1911 self.classname, nodeid, poss_msg, strerror)
1912 if default is not _marker:
1913 return Class.get(self, nodeid, propname, default, cache=cache)
1914 else:
1915 return Class.get(self, nodeid, propname, cache=cache)
1917 def getprops(self, protected=1):
1918 ''' In addition to the actual properties on the node, these methods
1919 provide the "content" property. If the "protected" flag is true,
1920 we include protected properties - those which may not be
1921 modified.
1922 '''
1923 d = Class.getprops(self, protected=protected).copy()
1924 d['content'] = hyperdb.String()
1925 return d
1927 def index(self, nodeid):
1928 ''' Index the node in the search index.
1930 We want to index the content in addition to the normal String
1931 property indexing.
1932 '''
1933 # perform normal indexing
1934 Class.index(self, nodeid)
1936 # get the content to index
1937 content = self.get(nodeid, 'content')
1939 # figure the mime type
1940 if self.properties.has_key('type'):
1941 mime_type = self.get(nodeid, 'type')
1942 else:
1943 mime_type = self.default_mime_type
1945 # and index!
1946 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
1947 mime_type)
1949 # deviation from spec - was called ItemClass
1950 class IssueClass(Class, roundupdb.IssueClass):
1951 # Overridden methods:
1952 def __init__(self, db, classname, **properties):
1953 '''The newly-created class automatically includes the "messages",
1954 "files", "nosy", and "superseder" properties. If the 'properties'
1955 dictionary attempts to specify any of these properties or a
1956 "creation" or "activity" property, a ValueError is raised.
1957 '''
1958 if not properties.has_key('title'):
1959 properties['title'] = hyperdb.String(indexme='yes')
1960 if not properties.has_key('messages'):
1961 properties['messages'] = hyperdb.Multilink("msg")
1962 if not properties.has_key('files'):
1963 properties['files'] = hyperdb.Multilink("file")
1964 if not properties.has_key('nosy'):
1965 # note: journalling is turned off as it really just wastes
1966 # space. this behaviour may be overridden in an instance
1967 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
1968 if not properties.has_key('superseder'):
1969 properties['superseder'] = hyperdb.Multilink(classname)
1970 Class.__init__(self, db, classname, **properties)
1972 #