1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: back_anydbm.py,v 1.97 2003-01-15 22:17:19 kedder Exp $
19 '''
20 This module defines a backend that saves the hyperdatabase in a database
21 chosen by anydbm. It is guaranteed to always be available in python
22 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
23 serious bugs, and is not available)
24 '''
26 import whichdb, anydbm, os, marshal, re, weakref, string, copy
27 from roundup import hyperdb, date, password, roundupdb, security
28 from blobfiles import FileStorage
29 from sessions import Sessions
30 from roundup.indexer import Indexer
31 from roundup.backends import locking
32 from roundup.hyperdb import String, Password, Date, Interval, Link, \
33 Multilink, DatabaseError, Boolean, Number
35 #
36 # Now the database
37 #
38 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
39 '''A database for storing records containing flexible data types.
41 Transaction stuff TODO:
42 . check the timestamp of the class file and nuke the cache if it's
43 modified. Do some sort of conflict checking on the dirty stuff.
44 . perhaps detect write collisions (related to above)?
46 '''
47 def __init__(self, config, journaltag=None):
48 '''Open a hyperdatabase given a specifier to some storage.
50 The 'storagelocator' is obtained from config.DATABASE.
51 The meaning of 'storagelocator' depends on the particular
52 implementation of the hyperdatabase. It could be a file name,
53 a directory path, a socket descriptor for a connection to a
54 database over the network, etc.
56 The 'journaltag' is a token that will be attached to the journal
57 entries for any edits done on the database. If 'journaltag' is
58 None, the database is opened in read-only mode: the Class.create(),
59 Class.set(), and Class.retire() methods are disabled.
60 '''
61 self.config, self.journaltag = config, journaltag
62 self.dir = config.DATABASE
63 self.classes = {}
64 self.cache = {} # cache of nodes loaded or created
65 self.dirtynodes = {} # keep track of the dirty nodes by class
66 self.newnodes = {} # keep track of the new nodes by class
67 self.destroyednodes = {}# keep track of the destroyed nodes by class
68 self.transactions = []
69 self.indexer = Indexer(self.dir)
70 self.sessions = Sessions(self.config)
71 self.security = security.Security(self)
72 # ensure files are group readable and writable
73 os.umask(0002)
75 # lock it
76 lockfilenm = os.path.join(self.dir, 'lock')
77 self.lockfile = locking.acquire_lock(lockfilenm)
78 self.lockfile.write(str(os.getpid()))
79 self.lockfile.flush()
81 def post_init(self):
82 ''' Called once the schema initialisation has finished.
83 '''
84 # reindex the db if necessary
85 if self.indexer.should_reindex():
86 self.reindex()
88 # figure the "curuserid"
89 if self.journaltag is None:
90 self.curuserid = None
91 elif self.journaltag == 'admin':
92 # admin user may not exist, but always has ID 1
93 self.curuserid = '1'
94 else:
95 self.curuserid = self.user.lookup(self.journaltag)
97 def reindex(self):
98 for klass in self.classes.values():
99 for nodeid in klass.list():
100 klass.index(nodeid)
101 self.indexer.save_index()
103 def __repr__(self):
104 return '<back_anydbm instance at %x>'%id(self)
106 #
107 # Classes
108 #
109 def __getattr__(self, classname):
110 '''A convenient way of calling self.getclass(classname).'''
111 if self.classes.has_key(classname):
112 if __debug__:
113 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
114 return self.classes[classname]
115 raise AttributeError, classname
117 def addclass(self, cl):
118 if __debug__:
119 print >>hyperdb.DEBUG, 'addclass', (self, cl)
120 cn = cl.classname
121 if self.classes.has_key(cn):
122 raise ValueError, cn
123 self.classes[cn] = cl
125 def getclasses(self):
126 '''Return a list of the names of all existing classes.'''
127 if __debug__:
128 print >>hyperdb.DEBUG, 'getclasses', (self,)
129 l = self.classes.keys()
130 l.sort()
131 return l
133 def getclass(self, classname):
134 '''Get the Class object representing a particular class.
136 If 'classname' is not a valid class name, a KeyError is raised.
137 '''
138 if __debug__:
139 print >>hyperdb.DEBUG, 'getclass', (self, classname)
140 try:
141 return self.classes[classname]
142 except KeyError:
143 raise KeyError, 'There is no class called "%s"'%classname
145 #
146 # Class DBs
147 #
148 def clear(self):
149 '''Delete all database contents
150 '''
151 if __debug__:
152 print >>hyperdb.DEBUG, 'clear', (self,)
153 for cn in self.classes.keys():
154 for dummy in 'nodes', 'journals':
155 path = os.path.join(self.dir, 'journals.%s'%cn)
156 if os.path.exists(path):
157 os.remove(path)
158 elif os.path.exists(path+'.db'): # dbm appends .db
159 os.remove(path+'.db')
161 def getclassdb(self, classname, mode='r'):
162 ''' grab a connection to the class db that will be used for
163 multiple actions
164 '''
165 if __debug__:
166 print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode)
167 return self.opendb('nodes.%s'%classname, mode)
169 def determine_db_type(self, path):
170 ''' determine which DB wrote the class file
171 '''
172 db_type = ''
173 if os.path.exists(path):
174 db_type = whichdb.whichdb(path)
175 if not db_type:
176 raise DatabaseError, "Couldn't identify database type"
177 elif os.path.exists(path+'.db'):
178 # if the path ends in '.db', it's a dbm database, whether
179 # anydbm says it's dbhash or not!
180 db_type = 'dbm'
181 return db_type
183 def opendb(self, name, mode):
184 '''Low-level database opener that gets around anydbm/dbm
185 eccentricities.
186 '''
187 if __debug__:
188 print >>hyperdb.DEBUG, 'opendb', (self, name, mode)
190 # figure the class db type
191 path = os.path.join(os.getcwd(), self.dir, name)
192 db_type = self.determine_db_type(path)
194 # new database? let anydbm pick the best dbm
195 if not db_type:
196 if __debug__:
197 print >>hyperdb.DEBUG, "opendb anydbm.open(%r, 'c')"%path
198 return anydbm.open(path, 'c')
200 # open the database with the correct module
201 try:
202 dbm = __import__(db_type)
203 except ImportError:
204 raise DatabaseError, \
205 "Couldn't open database - the required module '%s'"\
206 " is not available"%db_type
207 if __debug__:
208 print >>hyperdb.DEBUG, "opendb %r.open(%r, %r)"%(db_type, path,
209 mode)
210 return dbm.open(path, mode)
212 #
213 # Node IDs
214 #
215 def newid(self, classname):
216 ''' Generate a new id for the given class
217 '''
218 # open the ids DB - create if if doesn't exist
219 db = self.opendb('_ids', 'c')
220 if db.has_key(classname):
221 newid = db[classname] = str(int(db[classname]) + 1)
222 else:
223 # the count() bit is transitional - older dbs won't start at 1
224 newid = str(self.getclass(classname).count()+1)
225 db[classname] = newid
226 db.close()
227 return newid
229 def setid(self, classname, setid):
230 ''' Set the id counter: used during import of database
231 '''
232 # open the ids DB - create if if doesn't exist
233 db = self.opendb('_ids', 'c')
234 db[classname] = str(setid)
235 db.close()
237 #
238 # Nodes
239 #
240 def addnode(self, classname, nodeid, node):
241 ''' add the specified node to its class's db
242 '''
243 if __debug__:
244 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
246 # we'll be supplied these props if we're doing an import
247 if not node.has_key('creator'):
248 # add in the "calculated" properties (dupe so we don't affect
249 # calling code's node assumptions)
250 node = node.copy()
251 node['creator'] = self.curuserid
252 node['creation'] = node['activity'] = date.Date()
254 self.newnodes.setdefault(classname, {})[nodeid] = 1
255 self.cache.setdefault(classname, {})[nodeid] = node
256 self.savenode(classname, nodeid, node)
258 def setnode(self, classname, nodeid, node):
259 ''' change the specified node
260 '''
261 if __debug__:
262 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
263 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
265 # update the activity time (dupe so we don't affect
266 # calling code's node assumptions)
267 node = node.copy()
268 node['activity'] = date.Date()
270 # can't set without having already loaded the node
271 self.cache[classname][nodeid] = node
272 self.savenode(classname, nodeid, node)
274 def savenode(self, classname, nodeid, node):
275 ''' perform the saving of data specified by the set/addnode
276 '''
277 if __debug__:
278 print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node)
279 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
281 def getnode(self, classname, nodeid, db=None, cache=1):
282 ''' get a node from the database
283 '''
284 if __debug__:
285 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid, db)
286 if cache:
287 # try the cache
288 cache_dict = self.cache.setdefault(classname, {})
289 if cache_dict.has_key(nodeid):
290 if __debug__:
291 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
292 nodeid)
293 return cache_dict[nodeid]
295 if __debug__:
296 print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid)
298 # get from the database and save in the cache
299 if db is None:
300 db = self.getclassdb(classname)
301 if not db.has_key(nodeid):
302 raise IndexError, "no such %s %s"%(classname, nodeid)
304 # check the uncommitted, destroyed nodes
305 if (self.destroyednodes.has_key(classname) and
306 self.destroyednodes[classname].has_key(nodeid)):
307 raise IndexError, "no such %s %s"%(classname, nodeid)
309 # decode
310 res = marshal.loads(db[nodeid])
312 # reverse the serialisation
313 res = self.unserialise(classname, res)
315 # store off in the cache dict
316 if cache:
317 cache_dict[nodeid] = res
319 return res
321 def destroynode(self, classname, nodeid):
322 '''Remove a node from the database. Called exclusively by the
323 destroy() method on Class.
324 '''
325 if __debug__:
326 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
328 # remove from cache and newnodes if it's there
329 if (self.cache.has_key(classname) and
330 self.cache[classname].has_key(nodeid)):
331 del self.cache[classname][nodeid]
332 if (self.newnodes.has_key(classname) and
333 self.newnodes[classname].has_key(nodeid)):
334 del self.newnodes[classname][nodeid]
336 # see if there's any obvious commit actions that we should get rid of
337 for entry in self.transactions[:]:
338 if entry[1][:2] == (classname, nodeid):
339 self.transactions.remove(entry)
341 # add to the destroyednodes map
342 self.destroyednodes.setdefault(classname, {})[nodeid] = 1
344 # add the destroy commit action
345 self.transactions.append((self.doDestroyNode, (classname, nodeid)))
347 def serialise(self, classname, node):
348 '''Copy the node contents, converting non-marshallable data into
349 marshallable data.
350 '''
351 if __debug__:
352 print >>hyperdb.DEBUG, 'serialise', classname, node
353 properties = self.getclass(classname).getprops()
354 d = {}
355 for k, v in node.items():
356 # if the property doesn't exist, or is the "retired" flag then
357 # it won't be in the properties dict
358 if not properties.has_key(k):
359 d[k] = v
360 continue
362 # get the property spec
363 prop = properties[k]
365 if isinstance(prop, Password) and v is not None:
366 d[k] = str(v)
367 elif isinstance(prop, Date) and v is not None:
368 d[k] = v.serialise()
369 elif isinstance(prop, Interval) and v is not None:
370 d[k] = v.serialise()
371 else:
372 d[k] = v
373 return d
375 def unserialise(self, classname, node):
376 '''Decode the marshalled node data
377 '''
378 if __debug__:
379 print >>hyperdb.DEBUG, 'unserialise', classname, node
380 properties = self.getclass(classname).getprops()
381 d = {}
382 for k, v in node.items():
383 # if the property doesn't exist, or is the "retired" flag then
384 # it won't be in the properties dict
385 if not properties.has_key(k):
386 d[k] = v
387 continue
389 # get the property spec
390 prop = properties[k]
392 if isinstance(prop, Date) and v is not None:
393 d[k] = date.Date(v)
394 elif isinstance(prop, Interval) and v is not None:
395 d[k] = date.Interval(v)
396 elif isinstance(prop, Password) and v is not None:
397 p = password.Password()
398 p.unpack(v)
399 d[k] = p
400 else:
401 d[k] = v
402 return d
404 def hasnode(self, classname, nodeid, db=None):
405 ''' determine if the database has a given node
406 '''
407 if __debug__:
408 print >>hyperdb.DEBUG, 'hasnode', (self, classname, nodeid, db)
410 # try the cache
411 cache = self.cache.setdefault(classname, {})
412 if cache.has_key(nodeid):
413 if __debug__:
414 print >>hyperdb.TRACE, 'has %s %s cached'%(classname, nodeid)
415 return 1
416 if __debug__:
417 print >>hyperdb.TRACE, 'has %s %s'%(classname, nodeid)
419 # not in the cache - check the database
420 if db is None:
421 db = self.getclassdb(classname)
422 res = db.has_key(nodeid)
423 return res
425 def countnodes(self, classname, db=None):
426 if __debug__:
427 print >>hyperdb.DEBUG, 'countnodes', (self, classname, db)
429 count = 0
431 # include the uncommitted nodes
432 if self.newnodes.has_key(classname):
433 count += len(self.newnodes[classname])
434 if self.destroyednodes.has_key(classname):
435 count -= len(self.destroyednodes[classname])
437 # and count those in the DB
438 if db is None:
439 db = self.getclassdb(classname)
440 count = count + len(db.keys())
441 return count
443 def getnodeids(self, classname, db=None):
444 if __debug__:
445 print >>hyperdb.DEBUG, 'getnodeids', (self, classname, db)
447 res = []
449 # start off with the new nodes
450 if self.newnodes.has_key(classname):
451 res += self.newnodes[classname].keys()
453 if db is None:
454 db = self.getclassdb(classname)
455 res = res + db.keys()
457 # remove the uncommitted, destroyed nodes
458 if self.destroyednodes.has_key(classname):
459 for nodeid in self.destroyednodes[classname].keys():
460 if db.has_key(nodeid):
461 res.remove(nodeid)
463 return res
466 #
467 # Files - special node properties
468 # inherited from FileStorage
470 #
471 # Journal
472 #
473 def addjournal(self, classname, nodeid, action, params, creator=None,
474 creation=None):
475 ''' Journal the Action
476 'action' may be:
478 'create' or 'set' -- 'params' is a dictionary of property values
479 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
480 'retire' -- 'params' is None
481 '''
482 if __debug__:
483 print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid,
484 action, params, creator, creation)
485 self.transactions.append((self.doSaveJournal, (classname, nodeid,
486 action, params, creator, creation)))
488 def getjournal(self, classname, nodeid):
489 ''' get the journal for id
491 Raise IndexError if the node doesn't exist (as per history()'s
492 API)
493 '''
494 if __debug__:
495 print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid)
496 # attempt to open the journal - in some rare cases, the journal may
497 # not exist
498 try:
499 db = self.opendb('journals.%s'%classname, 'r')
500 except anydbm.error, error:
501 if str(error) == "need 'c' or 'n' flag to open new db":
502 raise IndexError, 'no such %s %s'%(classname, nodeid)
503 elif error.args[0] != 2:
504 raise
505 raise IndexError, 'no such %s %s'%(classname, nodeid)
506 try:
507 journal = marshal.loads(db[nodeid])
508 except KeyError:
509 db.close()
510 raise IndexError, 'no such %s %s'%(classname, nodeid)
511 db.close()
512 res = []
513 for nodeid, date_stamp, user, action, params in journal:
514 res.append((nodeid, date.Date(date_stamp), user, action, params))
515 return res
517 def pack(self, pack_before):
518 ''' Delete all journal entries except "create" before 'pack_before'.
519 '''
520 if __debug__:
521 print >>hyperdb.DEBUG, 'packjournal', (self, pack_before)
523 pack_before = pack_before.serialise()
524 for classname in self.getclasses():
525 # get the journal db
526 db_name = 'journals.%s'%classname
527 path = os.path.join(os.getcwd(), self.dir, classname)
528 db_type = self.determine_db_type(path)
529 db = self.opendb(db_name, 'w')
531 for key in db.keys():
532 # get the journal for this db entry
533 journal = marshal.loads(db[key])
534 l = []
535 last_set_entry = None
536 for entry in journal:
537 # unpack the entry
538 (nodeid, date_stamp, self.journaltag, action,
539 params) = entry
540 # if the entry is after the pack date, _or_ the initial
541 # create entry, then it stays
542 if date_stamp > pack_before or action == 'create':
543 l.append(entry)
544 db[key] = marshal.dumps(l)
545 if db_type == 'gdbm':
546 db.reorganize()
547 db.close()
550 #
551 # Basic transaction support
552 #
553 def commit(self):
554 ''' Commit the current transactions.
555 '''
556 if __debug__:
557 print >>hyperdb.DEBUG, 'commit', (self,)
558 # TODO: lock the DB
560 # keep a handle to all the database files opened
561 self.databases = {}
563 # now, do all the transactions
564 reindex = {}
565 for method, args in self.transactions:
566 reindex[method(*args)] = 1
568 # now close all the database files
569 for db in self.databases.values():
570 db.close()
571 del self.databases
572 # TODO: unlock the DB
574 # reindex the nodes that request it
575 for classname, nodeid in filter(None, reindex.keys()):
576 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
577 self.getclass(classname).index(nodeid)
579 # save the indexer state
580 self.indexer.save_index()
582 self.clearCache()
584 def clearCache(self):
585 # all transactions committed, back to normal
586 self.cache = {}
587 self.dirtynodes = {}
588 self.newnodes = {}
589 self.destroyednodes = {}
590 self.transactions = []
592 def getCachedClassDB(self, classname):
593 ''' get the class db, looking in our cache of databases for commit
594 '''
595 # get the database handle
596 db_name = 'nodes.%s'%classname
597 if not self.databases.has_key(db_name):
598 self.databases[db_name] = self.getclassdb(classname, 'c')
599 return self.databases[db_name]
601 def doSaveNode(self, classname, nodeid, node):
602 if __debug__:
603 print >>hyperdb.DEBUG, 'doSaveNode', (self, classname, nodeid,
604 node)
606 db = self.getCachedClassDB(classname)
608 # now save the marshalled data
609 db[nodeid] = marshal.dumps(self.serialise(classname, node))
611 # return the classname, nodeid so we reindex this content
612 return (classname, nodeid)
614 def getCachedJournalDB(self, classname):
615 ''' get the journal db, looking in our cache of databases for commit
616 '''
617 # get the database handle
618 db_name = 'journals.%s'%classname
619 if not self.databases.has_key(db_name):
620 self.databases[db_name] = self.opendb(db_name, 'c')
621 return self.databases[db_name]
623 def doSaveJournal(self, classname, nodeid, action, params, creator,
624 creation):
625 # serialise the parameters now if necessary
626 if isinstance(params, type({})):
627 if action in ('set', 'create'):
628 params = self.serialise(classname, params)
630 # handle supply of the special journalling parameters (usually
631 # supplied on importing an existing database)
632 if creator:
633 journaltag = creator
634 else:
635 journaltag = self.curuserid
636 if creation:
637 journaldate = creation.serialise()
638 else:
639 journaldate = date.Date().serialise()
641 # create the journal entry
642 entry = (nodeid, journaldate, journaltag, action, params)
644 if __debug__:
645 print >>hyperdb.DEBUG, 'doSaveJournal', entry
647 db = self.getCachedJournalDB(classname)
649 # now insert the journal entry
650 if db.has_key(nodeid):
651 # append to existing
652 s = db[nodeid]
653 l = marshal.loads(s)
654 l.append(entry)
655 else:
656 l = [entry]
658 db[nodeid] = marshal.dumps(l)
660 def doDestroyNode(self, classname, nodeid):
661 if __debug__:
662 print >>hyperdb.DEBUG, 'doDestroyNode', (self, classname, nodeid)
664 # delete from the class database
665 db = self.getCachedClassDB(classname)
666 if db.has_key(nodeid):
667 del db[nodeid]
669 # delete from the database
670 db = self.getCachedJournalDB(classname)
671 if db.has_key(nodeid):
672 del db[nodeid]
674 # return the classname, nodeid so we reindex this content
675 return (classname, nodeid)
677 def rollback(self):
678 ''' Reverse all actions from the current transaction.
679 '''
680 if __debug__:
681 print >>hyperdb.DEBUG, 'rollback', (self, )
682 for method, args in self.transactions:
683 # delete temporary files
684 if method == self.doStoreFile:
685 self.rollbackStoreFile(*args)
686 self.cache = {}
687 self.dirtynodes = {}
688 self.newnodes = {}
689 self.destroyednodes = {}
690 self.transactions = []
692 def close(self):
693 ''' Nothing to do
694 '''
695 if self.lockfile is not None:
696 locking.release_lock(self.lockfile)
697 if self.lockfile is not None:
698 self.lockfile.close()
699 self.lockfile = None
701 _marker = []
702 class Class(hyperdb.Class):
703 '''The handle to a particular class of nodes in a hyperdatabase.'''
705 def __init__(self, db, classname, **properties):
706 '''Create a new class with a given name and property specification.
708 'classname' must not collide with the name of an existing class,
709 or a ValueError is raised. The keyword arguments in 'properties'
710 must map names to property objects, or a TypeError is raised.
711 '''
712 if (properties.has_key('creation') or properties.has_key('activity')
713 or properties.has_key('creator')):
714 raise ValueError, '"creation", "activity" and "creator" are '\
715 'reserved'
717 self.classname = classname
718 self.properties = properties
719 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
720 self.key = ''
722 # should we journal changes (default yes)
723 self.do_journal = 1
725 # do the db-related init stuff
726 db.addclass(self)
728 self.auditors = {'create': [], 'set': [], 'retire': []}
729 self.reactors = {'create': [], 'set': [], 'retire': []}
731 def enableJournalling(self):
732 '''Turn journalling on for this class
733 '''
734 self.do_journal = 1
736 def disableJournalling(self):
737 '''Turn journalling off for this class
738 '''
739 self.do_journal = 0
741 # Editing nodes:
743 def create(self, **propvalues):
744 '''Create a new node of this class and return its id.
746 The keyword arguments in 'propvalues' map property names to values.
748 The values of arguments must be acceptable for the types of their
749 corresponding properties or a TypeError is raised.
751 If this class has a key property, it must be present and its value
752 must not collide with other key strings or a ValueError is raised.
754 Any other properties on this class that are missing from the
755 'propvalues' dictionary are set to None.
757 If an id in a link or multilink property does not refer to a valid
758 node, an IndexError is raised.
760 These operations trigger detectors and can be vetoed. Attempts
761 to modify the "creation" or "activity" properties cause a KeyError.
762 '''
763 if propvalues.has_key('id'):
764 raise KeyError, '"id" is reserved'
766 if self.db.journaltag is None:
767 raise DatabaseError, 'Database open read-only'
769 if propvalues.has_key('creation') or propvalues.has_key('activity'):
770 raise KeyError, '"creation" and "activity" are reserved'
772 self.fireAuditors('create', None, propvalues)
774 # new node's id
775 newid = self.db.newid(self.classname)
777 # validate propvalues
778 num_re = re.compile('^\d+$')
779 for key, value in propvalues.items():
780 if key == self.key:
781 try:
782 self.lookup(value)
783 except KeyError:
784 pass
785 else:
786 raise ValueError, 'node with key "%s" exists'%value
788 # try to handle this property
789 try:
790 prop = self.properties[key]
791 except KeyError:
792 raise KeyError, '"%s" has no property "%s"'%(self.classname,
793 key)
795 if value is not None and isinstance(prop, Link):
796 if type(value) != type(''):
797 raise ValueError, 'link value must be String'
798 link_class = self.properties[key].classname
799 # if it isn't a number, it's a key
800 if not num_re.match(value):
801 try:
802 value = self.db.classes[link_class].lookup(value)
803 except (TypeError, KeyError):
804 raise IndexError, 'new property "%s": %s not a %s'%(
805 key, value, link_class)
806 elif not self.db.getclass(link_class).hasnode(value):
807 raise IndexError, '%s has no node %s'%(link_class, value)
809 # save off the value
810 propvalues[key] = value
812 # register the link with the newly linked node
813 if self.do_journal and self.properties[key].do_journal:
814 self.db.addjournal(link_class, value, 'link',
815 (self.classname, newid, key))
817 elif isinstance(prop, Multilink):
818 if type(value) != type([]):
819 raise TypeError, 'new property "%s" not a list of ids'%key
821 # clean up and validate the list of links
822 link_class = self.properties[key].classname
823 l = []
824 for entry in value:
825 if type(entry) != type(''):
826 raise ValueError, '"%s" multilink value (%r) '\
827 'must contain Strings'%(key, value)
828 # if it isn't a number, it's a key
829 if not num_re.match(entry):
830 try:
831 entry = self.db.classes[link_class].lookup(entry)
832 except (TypeError, KeyError):
833 raise IndexError, 'new property "%s": %s not a %s'%(
834 key, entry, self.properties[key].classname)
835 l.append(entry)
836 value = l
837 propvalues[key] = value
839 # handle additions
840 for nodeid in value:
841 if not self.db.getclass(link_class).hasnode(nodeid):
842 raise IndexError, '%s has no node %s'%(link_class,
843 nodeid)
844 # register the link with the newly linked node
845 if self.do_journal and self.properties[key].do_journal:
846 self.db.addjournal(link_class, nodeid, 'link',
847 (self.classname, newid, key))
849 elif isinstance(prop, String):
850 if type(value) != type('') and type(value) != type(u''):
851 raise TypeError, 'new property "%s" not a string'%key
853 elif isinstance(prop, Password):
854 if not isinstance(value, password.Password):
855 raise TypeError, 'new property "%s" not a Password'%key
857 elif isinstance(prop, Date):
858 if value is not None and not isinstance(value, date.Date):
859 raise TypeError, 'new property "%s" not a Date'%key
861 elif isinstance(prop, Interval):
862 if value is not None and not isinstance(value, date.Interval):
863 raise TypeError, 'new property "%s" not an Interval'%key
865 elif value is not None and isinstance(prop, Number):
866 try:
867 float(value)
868 except ValueError:
869 raise TypeError, 'new property "%s" not numeric'%key
871 elif value is not None and isinstance(prop, Boolean):
872 try:
873 int(value)
874 except ValueError:
875 raise TypeError, 'new property "%s" not boolean'%key
877 # make sure there's data where there needs to be
878 for key, prop in self.properties.items():
879 if propvalues.has_key(key):
880 continue
881 if key == self.key:
882 raise ValueError, 'key property "%s" is required'%key
883 if isinstance(prop, Multilink):
884 propvalues[key] = []
885 else:
886 propvalues[key] = None
888 # done
889 self.db.addnode(self.classname, newid, propvalues)
890 if self.do_journal:
891 self.db.addjournal(self.classname, newid, 'create', {})
893 self.fireReactors('create', newid, None)
895 return newid
897 def export_list(self, propnames, nodeid):
898 ''' Export a node - generate a list of CSV-able data in the order
899 specified by propnames for the given node.
900 '''
901 properties = self.getprops()
902 l = []
903 for prop in propnames:
904 proptype = properties[prop]
905 value = self.get(nodeid, prop)
906 # "marshal" data where needed
907 if value is None:
908 pass
909 elif isinstance(proptype, hyperdb.Date):
910 value = value.get_tuple()
911 elif isinstance(proptype, hyperdb.Interval):
912 value = value.get_tuple()
913 elif isinstance(proptype, hyperdb.Password):
914 value = str(value)
915 l.append(repr(value))
916 return l
918 def import_list(self, propnames, proplist):
919 ''' Import a node - all information including "id" is present and
920 should not be sanity checked. Triggers are not triggered. The
921 journal should be initialised using the "creator" and "created"
922 information.
924 Return the nodeid of the node imported.
925 '''
926 if self.db.journaltag is None:
927 raise DatabaseError, 'Database open read-only'
928 properties = self.getprops()
930 # make the new node's property map
931 d = {}
932 for i in range(len(propnames)):
933 # Use eval to reverse the repr() used to output the CSV
934 value = eval(proplist[i])
936 # Figure the property for this column
937 propname = propnames[i]
938 prop = properties[propname]
940 # "unmarshal" where necessary
941 if propname == 'id':
942 newid = value
943 continue
944 elif value is None:
945 # don't set Nones
946 continue
947 elif isinstance(prop, hyperdb.Date):
948 value = date.Date(value)
949 elif isinstance(prop, hyperdb.Interval):
950 value = date.Interval(value)
951 elif isinstance(prop, hyperdb.Password):
952 pwd = password.Password()
953 pwd.unpack(value)
954 value = pwd
955 d[propname] = value
957 # add the node and journal
958 self.db.addnode(self.classname, newid, d)
960 # extract the journalling stuff and nuke it
961 if d.has_key('creator'):
962 creator = d['creator']
963 del d['creator']
964 else:
965 creator = None
966 if d.has_key('creation'):
967 creation = d['creation']
968 del d['creation']
969 else:
970 creation = None
971 if d.has_key('activity'):
972 del d['activity']
973 self.db.addjournal(self.classname, newid, 'create', {}, creator,
974 creation)
975 return newid
977 def get(self, nodeid, propname, default=_marker, cache=1):
978 '''Get the value of a property on an existing node of this class.
980 'nodeid' must be the id of an existing node of this class or an
981 IndexError is raised. 'propname' must be the name of a property
982 of this class or a KeyError is raised.
984 'cache' indicates whether the transaction cache should be queried
985 for the node. If the node has been modified and you need to
986 determine what its values prior to modification are, you need to
987 set cache=0.
989 Attempts to get the "creation" or "activity" properties should
990 do the right thing.
991 '''
992 if propname == 'id':
993 return nodeid
995 # get the node's dict
996 d = self.db.getnode(self.classname, nodeid, cache=cache)
998 # check for one of the special props
999 if propname == 'creation':
1000 if d.has_key('creation'):
1001 return d['creation']
1002 if not self.do_journal:
1003 raise ValueError, 'Journalling is disabled for this class'
1004 journal = self.db.getjournal(self.classname, nodeid)
1005 if journal:
1006 return self.db.getjournal(self.classname, nodeid)[0][1]
1007 else:
1008 # on the strange chance that there's no journal
1009 return date.Date()
1010 if propname == 'activity':
1011 if d.has_key('activity'):
1012 return d['activity']
1013 if not self.do_journal:
1014 raise ValueError, 'Journalling is disabled for this class'
1015 journal = self.db.getjournal(self.classname, nodeid)
1016 if journal:
1017 return self.db.getjournal(self.classname, nodeid)[-1][1]
1018 else:
1019 # on the strange chance that there's no journal
1020 return date.Date()
1021 if propname == 'creator':
1022 if d.has_key('creator'):
1023 return d['creator']
1024 if not self.do_journal:
1025 raise ValueError, 'Journalling is disabled for this class'
1026 journal = self.db.getjournal(self.classname, nodeid)
1027 if journal:
1028 num_re = re.compile('^\d+$')
1029 value = self.db.getjournal(self.classname, nodeid)[0][2]
1030 if num_re.match(value):
1031 return value
1032 else:
1033 # old-style "username" journal tag
1034 try:
1035 return self.db.user.lookup(value)
1036 except KeyError:
1037 # user's been retired, return admin
1038 return '1'
1039 else:
1040 return self.db.curuserid
1042 # get the property (raises KeyErorr if invalid)
1043 prop = self.properties[propname]
1045 if not d.has_key(propname):
1046 if default is _marker:
1047 if isinstance(prop, Multilink):
1048 return []
1049 else:
1050 return None
1051 else:
1052 return default
1054 # return a dupe of the list so code doesn't get confused
1055 if isinstance(prop, Multilink):
1056 return d[propname][:]
1058 return d[propname]
1060 # not in spec
1061 def getnode(self, nodeid, cache=1):
1062 ''' Return a convenience wrapper for the node.
1064 'nodeid' must be the id of an existing node of this class or an
1065 IndexError is raised.
1067 'cache' indicates whether the transaction cache should be queried
1068 for the node. If the node has been modified and you need to
1069 determine what its values prior to modification are, you need to
1070 set cache=0.
1071 '''
1072 return Node(self, nodeid, cache=cache)
1074 def set(self, nodeid, **propvalues):
1075 '''Modify a property on an existing node of this class.
1077 'nodeid' must be the id of an existing node of this class or an
1078 IndexError is raised.
1080 Each key in 'propvalues' must be the name of a property of this
1081 class or a KeyError is raised.
1083 All values in 'propvalues' must be acceptable types for their
1084 corresponding properties or a TypeError is raised.
1086 If the value of the key property is set, it must not collide with
1087 other key strings or a ValueError is raised.
1089 If the value of a Link or Multilink property contains an invalid
1090 node id, a ValueError is raised.
1092 These operations trigger detectors and can be vetoed. Attempts
1093 to modify the "creation" or "activity" properties cause a KeyError.
1094 '''
1095 if not propvalues:
1096 return propvalues
1098 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1099 raise KeyError, '"creation" and "activity" are reserved'
1101 if propvalues.has_key('id'):
1102 raise KeyError, '"id" is reserved'
1104 if self.db.journaltag is None:
1105 raise DatabaseError, 'Database open read-only'
1107 self.fireAuditors('set', nodeid, propvalues)
1108 # Take a copy of the node dict so that the subsequent set
1109 # operation doesn't modify the oldvalues structure.
1110 try:
1111 # try not using the cache initially
1112 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid,
1113 cache=0))
1114 except IndexError:
1115 # this will be needed if somone does a create() and set()
1116 # with no intervening commit()
1117 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1119 node = self.db.getnode(self.classname, nodeid)
1120 if node.has_key(self.db.RETIRED_FLAG):
1121 raise IndexError
1122 num_re = re.compile('^\d+$')
1124 # if the journal value is to be different, store it in here
1125 journalvalues = {}
1127 for propname, value in propvalues.items():
1128 # check to make sure we're not duplicating an existing key
1129 if propname == self.key and node[propname] != value:
1130 try:
1131 self.lookup(value)
1132 except KeyError:
1133 pass
1134 else:
1135 raise ValueError, 'node with key "%s" exists'%value
1137 # this will raise the KeyError if the property isn't valid
1138 # ... we don't use getprops() here because we only care about
1139 # the writeable properties.
1140 try:
1141 prop = self.properties[propname]
1142 except KeyError:
1143 raise KeyError, '"%s" has no property named "%s"'%(
1144 self.classname, propname)
1146 # if the value's the same as the existing value, no sense in
1147 # doing anything
1148 current = node.get(propname, None)
1149 if value == current:
1150 del propvalues[propname]
1151 continue
1152 journalvalues[propname] = current
1154 # do stuff based on the prop type
1155 if isinstance(prop, Link):
1156 link_class = prop.classname
1157 # if it isn't a number, it's a key
1158 if value is not None and not isinstance(value, type('')):
1159 raise ValueError, 'property "%s" link value be a string'%(
1160 propname)
1161 if isinstance(value, type('')) and not num_re.match(value):
1162 try:
1163 value = self.db.classes[link_class].lookup(value)
1164 except (TypeError, KeyError):
1165 raise IndexError, 'new property "%s": %s not a %s'%(
1166 propname, value, prop.classname)
1168 if (value is not None and
1169 not self.db.getclass(link_class).hasnode(value)):
1170 raise IndexError, '%s has no node %s'%(link_class, value)
1172 if self.do_journal and prop.do_journal:
1173 # register the unlink with the old linked node
1174 if node.has_key(propname) and node[propname] is not None:
1175 self.db.addjournal(link_class, node[propname], 'unlink',
1176 (self.classname, nodeid, propname))
1178 # register the link with the newly linked node
1179 if value is not None:
1180 self.db.addjournal(link_class, value, 'link',
1181 (self.classname, nodeid, propname))
1183 elif isinstance(prop, Multilink):
1184 if type(value) != type([]):
1185 raise TypeError, 'new property "%s" not a list of'\
1186 ' ids'%propname
1187 link_class = self.properties[propname].classname
1188 l = []
1189 for entry in value:
1190 # if it isn't a number, it's a key
1191 if type(entry) != type(''):
1192 raise ValueError, 'new property "%s" link value ' \
1193 'must be a string'%propname
1194 if not num_re.match(entry):
1195 try:
1196 entry = self.db.classes[link_class].lookup(entry)
1197 except (TypeError, KeyError):
1198 raise IndexError, 'new property "%s": %s not a %s'%(
1199 propname, entry,
1200 self.properties[propname].classname)
1201 l.append(entry)
1202 value = l
1203 propvalues[propname] = value
1205 # figure the journal entry for this property
1206 add = []
1207 remove = []
1209 # handle removals
1210 if node.has_key(propname):
1211 l = node[propname]
1212 else:
1213 l = []
1214 for id in l[:]:
1215 if id in value:
1216 continue
1217 # register the unlink with the old linked node
1218 if self.do_journal and self.properties[propname].do_journal:
1219 self.db.addjournal(link_class, id, 'unlink',
1220 (self.classname, nodeid, propname))
1221 l.remove(id)
1222 remove.append(id)
1224 # handle additions
1225 for id in value:
1226 if not self.db.getclass(link_class).hasnode(id):
1227 raise IndexError, '%s has no node %s'%(link_class, id)
1228 if id in l:
1229 continue
1230 # register the link with the newly linked node
1231 if self.do_journal and self.properties[propname].do_journal:
1232 self.db.addjournal(link_class, id, 'link',
1233 (self.classname, nodeid, propname))
1234 l.append(id)
1235 add.append(id)
1237 # figure the journal entry
1238 l = []
1239 if add:
1240 l.append(('+', add))
1241 if remove:
1242 l.append(('-', remove))
1243 if l:
1244 journalvalues[propname] = tuple(l)
1246 elif isinstance(prop, String):
1247 if value is not None and type(value) != type('') and type(value) != type(u''):
1248 raise TypeError, 'new property "%s" not a string'%propname
1250 elif isinstance(prop, Password):
1251 if not isinstance(value, password.Password):
1252 raise TypeError, 'new property "%s" not a Password'%propname
1253 propvalues[propname] = value
1255 elif value is not None and isinstance(prop, Date):
1256 if not isinstance(value, date.Date):
1257 raise TypeError, 'new property "%s" not a Date'% propname
1258 propvalues[propname] = value
1260 elif value is not None and isinstance(prop, Interval):
1261 if not isinstance(value, date.Interval):
1262 raise TypeError, 'new property "%s" not an '\
1263 'Interval'%propname
1264 propvalues[propname] = value
1266 elif value is not None and isinstance(prop, Number):
1267 try:
1268 float(value)
1269 except ValueError:
1270 raise TypeError, 'new property "%s" not numeric'%propname
1272 elif value is not None and isinstance(prop, Boolean):
1273 try:
1274 int(value)
1275 except ValueError:
1276 raise TypeError, 'new property "%s" not boolean'%propname
1278 node[propname] = value
1280 # nothing to do?
1281 if not propvalues:
1282 return propvalues
1284 # do the set, and journal it
1285 self.db.setnode(self.classname, nodeid, node)
1287 if self.do_journal:
1288 self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
1290 self.fireReactors('set', nodeid, oldvalues)
1292 return propvalues
1294 def retire(self, nodeid):
1295 '''Retire a node.
1297 The properties on the node remain available from the get() method,
1298 and the node's id is never reused.
1300 Retired nodes are not returned by the find(), list(), or lookup()
1301 methods, and other nodes may reuse the values of their key properties.
1303 These operations trigger detectors and can be vetoed. Attempts
1304 to modify the "creation" or "activity" properties cause a KeyError.
1305 '''
1306 if self.db.journaltag is None:
1307 raise DatabaseError, 'Database open read-only'
1309 self.fireAuditors('retire', nodeid, None)
1311 node = self.db.getnode(self.classname, nodeid)
1312 node[self.db.RETIRED_FLAG] = 1
1313 self.db.setnode(self.classname, nodeid, node)
1314 if self.do_journal:
1315 self.db.addjournal(self.classname, nodeid, 'retired', None)
1317 self.fireReactors('retire', nodeid, None)
1319 def is_retired(self, nodeid):
1320 '''Return true if the node is retired.
1321 '''
1322 node = self.db.getnode(cn, nodeid, cldb)
1323 if node.has_key(self.db.RETIRED_FLAG):
1324 return 1
1325 return 0
1327 def destroy(self, nodeid):
1328 '''Destroy a node.
1330 WARNING: this method should never be used except in extremely rare
1331 situations where there could never be links to the node being
1332 deleted
1333 WARNING: use retire() instead
1334 WARNING: the properties of this node will not be available ever again
1335 WARNING: really, use retire() instead
1337 Well, I think that's enough warnings. This method exists mostly to
1338 support the session storage of the cgi interface.
1339 '''
1340 if self.db.journaltag is None:
1341 raise DatabaseError, 'Database open read-only'
1342 self.db.destroynode(self.classname, nodeid)
1344 def history(self, nodeid):
1345 '''Retrieve the journal of edits on a particular node.
1347 'nodeid' must be the id of an existing node of this class or an
1348 IndexError is raised.
1350 The returned list contains tuples of the form
1352 (date, tag, action, params)
1354 'date' is a Timestamp object specifying the time of the change and
1355 'tag' is the journaltag specified when the database was opened.
1356 '''
1357 if not self.do_journal:
1358 raise ValueError, 'Journalling is disabled for this class'
1359 return self.db.getjournal(self.classname, nodeid)
1361 # Locating nodes:
1362 def hasnode(self, nodeid):
1363 '''Determine if the given nodeid actually exists
1364 '''
1365 return self.db.hasnode(self.classname, nodeid)
1367 def setkey(self, propname):
1368 '''Select a String property of this class to be the key property.
1370 'propname' must be the name of a String property of this class or
1371 None, or a TypeError is raised. The values of the key property on
1372 all existing nodes must be unique or a ValueError is raised. If the
1373 property doesn't exist, KeyError is raised.
1374 '''
1375 prop = self.getprops()[propname]
1376 if not isinstance(prop, String):
1377 raise TypeError, 'key properties must be String'
1378 self.key = propname
1380 def getkey(self):
1381 '''Return the name of the key property for this class or None.'''
1382 return self.key
1384 def labelprop(self, default_to_id=0):
1385 ''' Return the property name for a label for the given node.
1387 This method attempts to generate a consistent label for the node.
1388 It tries the following in order:
1389 1. key property
1390 2. "name" property
1391 3. "title" property
1392 4. first property from the sorted property name list
1393 '''
1394 k = self.getkey()
1395 if k:
1396 return k
1397 props = self.getprops()
1398 if props.has_key('name'):
1399 return 'name'
1400 elif props.has_key('title'):
1401 return 'title'
1402 if default_to_id:
1403 return 'id'
1404 props = props.keys()
1405 props.sort()
1406 return props[0]
1408 # TODO: set up a separate index db file for this? profile?
1409 def lookup(self, keyvalue):
1410 '''Locate a particular node by its key property and return its id.
1412 If this class has no key property, a TypeError is raised. If the
1413 'keyvalue' matches one of the values for the key property among
1414 the nodes in this class, the matching node's id is returned;
1415 otherwise a KeyError is raised.
1416 '''
1417 if not self.key:
1418 raise TypeError, 'No key property set for class %s'%self.classname
1419 cldb = self.db.getclassdb(self.classname)
1420 try:
1421 for nodeid in self.db.getnodeids(self.classname, cldb):
1422 node = self.db.getnode(self.classname, nodeid, cldb)
1423 if node.has_key(self.db.RETIRED_FLAG):
1424 continue
1425 if node[self.key] == keyvalue:
1426 return nodeid
1427 finally:
1428 cldb.close()
1429 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1430 keyvalue, self.classname)
1432 # change from spec - allows multiple props to match
1433 def find(self, **propspec):
1434 '''Get the ids of nodes in this class which link to the given nodes.
1436 'propspec' consists of keyword args propname=nodeid or
1437 propname={nodeid:1, }
1438 'propname' must be the name of a property in this class, or a
1439 KeyError is raised. That property must be a Link or
1440 Multilink property, or a TypeError is raised.
1442 Any node in this class whose 'propname' property links to any of the
1443 nodeids will be returned. Used by the full text indexing, which knows
1444 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1445 issues:
1447 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1448 '''
1449 propspec = propspec.items()
1450 for propname, nodeids in propspec:
1451 # check the prop is OK
1452 prop = self.properties[propname]
1453 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1454 raise TypeError, "'%s' not a Link/Multilink property"%propname
1456 # ok, now do the find
1457 cldb = self.db.getclassdb(self.classname)
1458 l = []
1459 try:
1460 for id in self.db.getnodeids(self.classname, db=cldb):
1461 node = self.db.getnode(self.classname, id, db=cldb)
1462 if node.has_key(self.db.RETIRED_FLAG):
1463 continue
1464 for propname, nodeids in propspec:
1465 # can't test if the node doesn't have this property
1466 if not node.has_key(propname):
1467 continue
1468 if type(nodeids) is type(''):
1469 nodeids = {nodeids:1}
1470 prop = self.properties[propname]
1471 value = node[propname]
1472 if isinstance(prop, Link) and nodeids.has_key(value):
1473 l.append(id)
1474 break
1475 elif isinstance(prop, Multilink):
1476 hit = 0
1477 for v in value:
1478 if nodeids.has_key(v):
1479 l.append(id)
1480 hit = 1
1481 break
1482 if hit:
1483 break
1484 finally:
1485 cldb.close()
1486 return l
1488 def stringFind(self, **requirements):
1489 '''Locate a particular node by matching a set of its String
1490 properties in a caseless search.
1492 If the property is not a String property, a TypeError is raised.
1494 The return is a list of the id of all nodes that match.
1495 '''
1496 for propname in requirements.keys():
1497 prop = self.properties[propname]
1498 if isinstance(not prop, String):
1499 raise TypeError, "'%s' not a String property"%propname
1500 requirements[propname] = requirements[propname].lower()
1501 l = []
1502 cldb = self.db.getclassdb(self.classname)
1503 try:
1504 for nodeid in self.db.getnodeids(self.classname, cldb):
1505 node = self.db.getnode(self.classname, nodeid, cldb)
1506 if node.has_key(self.db.RETIRED_FLAG):
1507 continue
1508 for key, value in requirements.items():
1509 if not node.has_key(key):
1510 break
1511 if node[key] is None or node[key].lower() != value:
1512 break
1513 else:
1514 l.append(nodeid)
1515 finally:
1516 cldb.close()
1517 return l
1519 def list(self):
1520 ''' Return a list of the ids of the active nodes in this class.
1521 '''
1522 l = []
1523 cn = self.classname
1524 cldb = self.db.getclassdb(cn)
1525 try:
1526 for nodeid in self.db.getnodeids(cn, cldb):
1527 node = self.db.getnode(cn, nodeid, cldb)
1528 if node.has_key(self.db.RETIRED_FLAG):
1529 continue
1530 l.append(nodeid)
1531 finally:
1532 cldb.close()
1533 l.sort()
1534 return l
1536 def filter(self, search_matches, filterspec, sort=(None,None),
1537 group=(None,None), num_re = re.compile('^\d+$')):
1538 ''' Return a list of the ids of the active nodes in this class that
1539 match the 'filter' spec, sorted by the group spec and then the
1540 sort spec.
1542 "filterspec" is {propname: value(s)}
1543 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1544 and prop is a prop name or None
1545 "search_matches" is {nodeid: marker}
1547 The filter must match all properties specificed - but if the
1548 property value to match is a list, any one of the values in the
1549 list may match for that property to match.
1550 '''
1551 cn = self.classname
1553 # optimise filterspec
1554 l = []
1555 props = self.getprops()
1556 LINK = 0
1557 MULTILINK = 1
1558 STRING = 2
1559 OTHER = 6
1560 for k, v in filterspec.items():
1561 propclass = props[k]
1562 if isinstance(propclass, Link):
1563 if type(v) is not type([]):
1564 v = [v]
1565 # replace key values with node ids
1566 u = []
1567 link_class = self.db.classes[propclass.classname]
1568 for entry in v:
1569 if entry == '-1': entry = None
1570 elif not num_re.match(entry):
1571 try:
1572 entry = link_class.lookup(entry)
1573 except (TypeError,KeyError):
1574 raise ValueError, 'property "%s": %s not a %s'%(
1575 k, entry, self.properties[k].classname)
1576 u.append(entry)
1578 l.append((LINK, k, u))
1579 elif isinstance(propclass, Multilink):
1580 if type(v) is not type([]):
1581 v = [v]
1582 # replace key values with node ids
1583 u = []
1584 link_class = self.db.classes[propclass.classname]
1585 for entry in v:
1586 if not num_re.match(entry):
1587 try:
1588 entry = link_class.lookup(entry)
1589 except (TypeError,KeyError):
1590 raise ValueError, 'new property "%s": %s not a %s'%(
1591 k, entry, self.properties[k].classname)
1592 u.append(entry)
1593 l.append((MULTILINK, k, u))
1594 elif isinstance(propclass, String) and k != 'id':
1595 # simple glob searching
1596 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1597 v = v.replace('?', '.')
1598 v = v.replace('*', '.*?')
1599 l.append((STRING, k, re.compile(v, re.I)))
1600 elif isinstance(propclass, Boolean):
1601 if type(v) is type(''):
1602 bv = v.lower() in ('yes', 'true', 'on', '1')
1603 else:
1604 bv = v
1605 l.append((OTHER, k, bv))
1606 elif isinstance(propclass, Date):
1607 l.append((OTHER, k, date.Date(v)))
1608 elif isinstance(propclass, Interval):
1609 l.append((OTHER, k, date.Interval(v)))
1610 elif isinstance(propclass, Number):
1611 l.append((OTHER, k, int(v)))
1612 else:
1613 l.append((OTHER, k, v))
1614 filterspec = l
1616 # now, find all the nodes that are active and pass filtering
1617 l = []
1618 cldb = self.db.getclassdb(cn)
1619 try:
1620 # TODO: only full-scan once (use items())
1621 for nodeid in self.db.getnodeids(cn, cldb):
1622 node = self.db.getnode(cn, nodeid, cldb)
1623 if node.has_key(self.db.RETIRED_FLAG):
1624 continue
1625 # apply filter
1626 for t, k, v in filterspec:
1627 # handle the id prop
1628 if k == 'id' and v == nodeid:
1629 continue
1631 # make sure the node has the property
1632 if not node.has_key(k):
1633 # this node doesn't have this property, so reject it
1634 break
1636 # now apply the property filter
1637 if t == LINK:
1638 # link - if this node's property doesn't appear in the
1639 # filterspec's nodeid list, skip it
1640 if node[k] not in v:
1641 break
1642 elif t == MULTILINK:
1643 # multilink - if any of the nodeids required by the
1644 # filterspec aren't in this node's property, then skip
1645 # it
1646 have = node[k]
1647 for want in v:
1648 if want not in have:
1649 break
1650 else:
1651 continue
1652 break
1653 elif t == STRING:
1654 # RE search
1655 if node[k] is None or not v.search(node[k]):
1656 break
1657 elif t == OTHER:
1658 # straight value comparison for the other types
1659 if node[k] != v:
1660 break
1661 else:
1662 l.append((nodeid, node))
1663 finally:
1664 cldb.close()
1665 l.sort()
1667 # filter based on full text search
1668 if search_matches is not None:
1669 k = []
1670 for v in l:
1671 if search_matches.has_key(v[0]):
1672 k.append(v)
1673 l = k
1675 # now, sort the result
1676 def sortfun(a, b, sort=sort, group=group, properties=self.getprops(),
1677 db = self.db, cl=self):
1678 a_id, an = a
1679 b_id, bn = b
1680 # sort by group and then sort
1681 for dir, prop in group, sort:
1682 if dir is None or prop is None: continue
1684 # sorting is class-specific
1685 propclass = properties[prop]
1687 # handle the properties that might be "faked"
1688 # also, handle possible missing properties
1689 try:
1690 if not an.has_key(prop):
1691 an[prop] = cl.get(a_id, prop)
1692 av = an[prop]
1693 except KeyError:
1694 # the node doesn't have a value for this property
1695 if isinstance(propclass, Multilink): av = []
1696 else: av = ''
1697 try:
1698 if not bn.has_key(prop):
1699 bn[prop] = cl.get(b_id, prop)
1700 bv = bn[prop]
1701 except KeyError:
1702 # the node doesn't have a value for this property
1703 if isinstance(propclass, Multilink): bv = []
1704 else: bv = ''
1706 # String and Date values are sorted in the natural way
1707 if isinstance(propclass, String):
1708 # clean up the strings
1709 if av and av[0] in string.uppercase:
1710 av = av.lower()
1711 if bv and bv[0] in string.uppercase:
1712 bv = bv.lower()
1713 if (isinstance(propclass, String) or
1714 isinstance(propclass, Date)):
1715 # it might be a string that's really an integer
1716 try:
1717 av = int(av)
1718 bv = int(bv)
1719 except:
1720 pass
1721 if dir == '+':
1722 r = cmp(av, bv)
1723 if r != 0: return r
1724 elif dir == '-':
1725 r = cmp(bv, av)
1726 if r != 0: return r
1728 # Link properties are sorted according to the value of
1729 # the "order" property on the linked nodes if it is
1730 # present; or otherwise on the key string of the linked
1731 # nodes; or finally on the node ids.
1732 elif isinstance(propclass, Link):
1733 link = db.classes[propclass.classname]
1734 if av is None and bv is not None: return -1
1735 if av is not None and bv is None: return 1
1736 if av is None and bv is None: continue
1737 if link.getprops().has_key('order'):
1738 if dir == '+':
1739 r = cmp(link.get(av, 'order'),
1740 link.get(bv, 'order'))
1741 if r != 0: return r
1742 elif dir == '-':
1743 r = cmp(link.get(bv, 'order'),
1744 link.get(av, 'order'))
1745 if r != 0: return r
1746 elif link.getkey():
1747 key = link.getkey()
1748 if dir == '+':
1749 r = cmp(link.get(av, key), link.get(bv, key))
1750 if r != 0: return r
1751 elif dir == '-':
1752 r = cmp(link.get(bv, key), link.get(av, key))
1753 if r != 0: return r
1754 else:
1755 if dir == '+':
1756 r = cmp(av, bv)
1757 if r != 0: return r
1758 elif dir == '-':
1759 r = cmp(bv, av)
1760 if r != 0: return r
1762 # Multilink properties are sorted according to how many
1763 # links are present.
1764 elif isinstance(propclass, Multilink):
1765 if dir == '+':
1766 r = cmp(len(av), len(bv))
1767 if r != 0: return r
1768 elif dir == '-':
1769 r = cmp(len(bv), len(av))
1770 if r != 0: return r
1771 elif isinstance(propclass, Number) or isinstance(propclass, Boolean):
1772 if dir == '+':
1773 r = cmp(av, bv)
1774 elif dir == '-':
1775 r = cmp(bv, av)
1777 # end for dir, prop in sort, group:
1778 # if all else fails, compare the ids
1779 return cmp(a[0], b[0])
1781 l.sort(sortfun)
1782 return [i[0] for i in l]
1784 def count(self):
1785 '''Get the number of nodes in this class.
1787 If the returned integer is 'numnodes', the ids of all the nodes
1788 in this class run from 1 to numnodes, and numnodes+1 will be the
1789 id of the next node to be created in this class.
1790 '''
1791 return self.db.countnodes(self.classname)
1793 # Manipulating properties:
1795 def getprops(self, protected=1):
1796 '''Return a dictionary mapping property names to property objects.
1797 If the "protected" flag is true, we include protected properties -
1798 those which may not be modified.
1800 In addition to the actual properties on the node, these
1801 methods provide the "creation" and "activity" properties. If the
1802 "protected" flag is true, we include protected properties - those
1803 which may not be modified.
1804 '''
1805 d = self.properties.copy()
1806 if protected:
1807 d['id'] = String()
1808 d['creation'] = hyperdb.Date()
1809 d['activity'] = hyperdb.Date()
1810 d['creator'] = hyperdb.Link('user')
1811 return d
1813 def addprop(self, **properties):
1814 '''Add properties to this class.
1816 The keyword arguments in 'properties' must map names to property
1817 objects, or a TypeError is raised. None of the keys in 'properties'
1818 may collide with the names of existing properties, or a ValueError
1819 is raised before any properties have been added.
1820 '''
1821 for key in properties.keys():
1822 if self.properties.has_key(key):
1823 raise ValueError, key
1824 self.properties.update(properties)
1826 def index(self, nodeid):
1827 '''Add (or refresh) the node to search indexes
1828 '''
1829 # find all the String properties that have indexme
1830 for prop, propclass in self.getprops().items():
1831 if isinstance(propclass, String) and propclass.indexme:
1832 try:
1833 value = str(self.get(nodeid, prop))
1834 except IndexError:
1835 # node no longer exists - entry should be removed
1836 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1837 else:
1838 # and index them under (classname, nodeid, property)
1839 self.db.indexer.add_text((self.classname, nodeid, prop),
1840 value)
1842 #
1843 # Detector interface
1844 #
1845 def audit(self, event, detector):
1846 '''Register a detector
1847 '''
1848 l = self.auditors[event]
1849 if detector not in l:
1850 self.auditors[event].append(detector)
1852 def fireAuditors(self, action, nodeid, newvalues):
1853 '''Fire all registered auditors.
1854 '''
1855 for audit in self.auditors[action]:
1856 audit(self.db, self, nodeid, newvalues)
1858 def react(self, event, detector):
1859 '''Register a detector
1860 '''
1861 l = self.reactors[event]
1862 if detector not in l:
1863 self.reactors[event].append(detector)
1865 def fireReactors(self, action, nodeid, oldvalues):
1866 '''Fire all registered reactors.
1867 '''
1868 for react in self.reactors[action]:
1869 react(self.db, self, nodeid, oldvalues)
1871 class FileClass(Class):
1872 '''This class defines a large chunk of data. To support this, it has a
1873 mandatory String property "content" which is typically saved off
1874 externally to the hyperdb.
1876 The default MIME type of this data is defined by the
1877 "default_mime_type" class attribute, which may be overridden by each
1878 node if the class defines a "type" String property.
1879 '''
1880 default_mime_type = 'text/plain'
1882 def create(self, **propvalues):
1883 ''' snaffle the file propvalue and store in a file
1884 '''
1885 content = propvalues['content']
1886 del propvalues['content']
1887 newid = Class.create(self, **propvalues)
1888 self.db.storefile(self.classname, newid, None, content)
1889 return newid
1891 def import_list(self, propnames, proplist):
1892 ''' Trap the "content" property...
1893 '''
1894 # dupe this list so we don't affect others
1895 propnames = propnames[:]
1897 # extract the "content" property from the proplist
1898 i = propnames.index('content')
1899 content = eval(proplist[i])
1900 del propnames[i]
1901 del proplist[i]
1903 # do the normal import
1904 newid = Class.import_list(self, propnames, proplist)
1906 # save off the "content" file
1907 self.db.storefile(self.classname, newid, None, content)
1908 return newid
1910 def get(self, nodeid, propname, default=_marker, cache=1):
1911 ''' trap the content propname and get it from the file
1912 '''
1913 poss_msg = 'Possibly an access right configuration problem.'
1914 if propname == 'content':
1915 try:
1916 return self.db.getfile(self.classname, nodeid, None)
1917 except IOError, (strerror):
1918 # XXX by catching this we donot see an error in the log.
1919 return 'ERROR reading file: %s%s\n%s\n%s'%(
1920 self.classname, nodeid, poss_msg, strerror)
1921 if default is not _marker:
1922 return Class.get(self, nodeid, propname, default, cache=cache)
1923 else:
1924 return Class.get(self, nodeid, propname, cache=cache)
1926 def getprops(self, protected=1):
1927 ''' In addition to the actual properties on the node, these methods
1928 provide the "content" property. If the "protected" flag is true,
1929 we include protected properties - those which may not be
1930 modified.
1931 '''
1932 d = Class.getprops(self, protected=protected).copy()
1933 d['content'] = hyperdb.String()
1934 return d
1936 def index(self, nodeid):
1937 ''' Index the node in the search index.
1939 We want to index the content in addition to the normal String
1940 property indexing.
1941 '''
1942 # perform normal indexing
1943 Class.index(self, nodeid)
1945 # get the content to index
1946 content = self.get(nodeid, 'content')
1948 # figure the mime type
1949 if self.properties.has_key('type'):
1950 mime_type = self.get(nodeid, 'type')
1951 else:
1952 mime_type = self.default_mime_type
1954 # and index!
1955 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
1956 mime_type)
1958 # deviation from spec - was called ItemClass
1959 class IssueClass(Class, roundupdb.IssueClass):
1960 # Overridden methods:
1961 def __init__(self, db, classname, **properties):
1962 '''The newly-created class automatically includes the "messages",
1963 "files", "nosy", and "superseder" properties. If the 'properties'
1964 dictionary attempts to specify any of these properties or a
1965 "creation" or "activity" property, a ValueError is raised.
1966 '''
1967 if not properties.has_key('title'):
1968 properties['title'] = hyperdb.String(indexme='yes')
1969 if not properties.has_key('messages'):
1970 properties['messages'] = hyperdb.Multilink("msg")
1971 if not properties.has_key('files'):
1972 properties['files'] = hyperdb.Multilink("file")
1973 if not properties.has_key('nosy'):
1974 # note: journalling is turned off as it really just wastes
1975 # space. this behaviour may be overridden in an instance
1976 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
1977 if not properties.has_key('superseder'):
1978 properties['superseder'] = hyperdb.Multilink(classname)
1979 Class.__init__(self, db, classname, **properties)
1981 #