1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: back_anydbm.py,v 1.100 2003-02-06 05:43:47 richard Exp $
19 '''
20 This module defines a backend that saves the hyperdatabase in a database
21 chosen by anydbm. It is guaranteed to always be available in python
22 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
23 serious bugs, and is not available)
24 '''
26 import whichdb, anydbm, os, marshal, re, weakref, string, copy
27 from roundup import hyperdb, date, password, roundupdb, security
28 from blobfiles import FileStorage
29 from sessions import Sessions
30 from roundup.indexer import Indexer
31 from roundup.backends import locking
32 from roundup.hyperdb import String, Password, Date, Interval, Link, \
33 Multilink, DatabaseError, Boolean, Number
35 #
36 # Now the database
37 #
38 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
39 '''A database for storing records containing flexible data types.
41 Transaction stuff TODO:
42 . check the timestamp of the class file and nuke the cache if it's
43 modified. Do some sort of conflict checking on the dirty stuff.
44 . perhaps detect write collisions (related to above)?
46 '''
47 def __init__(self, config, journaltag=None):
48 '''Open a hyperdatabase given a specifier to some storage.
50 The 'storagelocator' is obtained from config.DATABASE.
51 The meaning of 'storagelocator' depends on the particular
52 implementation of the hyperdatabase. It could be a file name,
53 a directory path, a socket descriptor for a connection to a
54 database over the network, etc.
56 The 'journaltag' is a token that will be attached to the journal
57 entries for any edits done on the database. If 'journaltag' is
58 None, the database is opened in read-only mode: the Class.create(),
59 Class.set(), and Class.retire() methods are disabled.
60 '''
61 self.config, self.journaltag = config, journaltag
62 self.dir = config.DATABASE
63 self.classes = {}
64 self.cache = {} # cache of nodes loaded or created
65 self.dirtynodes = {} # keep track of the dirty nodes by class
66 self.newnodes = {} # keep track of the new nodes by class
67 self.destroyednodes = {}# keep track of the destroyed nodes by class
68 self.transactions = []
69 self.indexer = Indexer(self.dir)
70 self.sessions = Sessions(self.config)
71 self.security = security.Security(self)
72 # ensure files are group readable and writable
73 os.umask(0002)
75 # lock it
76 lockfilenm = os.path.join(self.dir, 'lock')
77 self.lockfile = locking.acquire_lock(lockfilenm)
78 self.lockfile.write(str(os.getpid()))
79 self.lockfile.flush()
81 def post_init(self):
82 ''' Called once the schema initialisation has finished.
83 '''
84 # reindex the db if necessary
85 if self.indexer.should_reindex():
86 self.reindex()
88 # figure the "curuserid"
89 if self.journaltag is None:
90 self.curuserid = None
91 elif self.journaltag == 'admin':
92 # admin user may not exist, but always has ID 1
93 self.curuserid = '1'
94 else:
95 self.curuserid = self.user.lookup(self.journaltag)
97 def reindex(self):
98 for klass in self.classes.values():
99 for nodeid in klass.list():
100 klass.index(nodeid)
101 self.indexer.save_index()
103 def __repr__(self):
104 return '<back_anydbm instance at %x>'%id(self)
106 #
107 # Classes
108 #
109 def __getattr__(self, classname):
110 '''A convenient way of calling self.getclass(classname).'''
111 if self.classes.has_key(classname):
112 if __debug__:
113 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
114 return self.classes[classname]
115 raise AttributeError, classname
117 def addclass(self, cl):
118 if __debug__:
119 print >>hyperdb.DEBUG, 'addclass', (self, cl)
120 cn = cl.classname
121 if self.classes.has_key(cn):
122 raise ValueError, cn
123 self.classes[cn] = cl
125 def getclasses(self):
126 '''Return a list of the names of all existing classes.'''
127 if __debug__:
128 print >>hyperdb.DEBUG, 'getclasses', (self,)
129 l = self.classes.keys()
130 l.sort()
131 return l
133 def getclass(self, classname):
134 '''Get the Class object representing a particular class.
136 If 'classname' is not a valid class name, a KeyError is raised.
137 '''
138 if __debug__:
139 print >>hyperdb.DEBUG, 'getclass', (self, classname)
140 try:
141 return self.classes[classname]
142 except KeyError:
143 raise KeyError, 'There is no class called "%s"'%classname
145 #
146 # Class DBs
147 #
148 def clear(self):
149 '''Delete all database contents
150 '''
151 if __debug__:
152 print >>hyperdb.DEBUG, 'clear', (self,)
153 for cn in self.classes.keys():
154 for dummy in 'nodes', 'journals':
155 path = os.path.join(self.dir, 'journals.%s'%cn)
156 if os.path.exists(path):
157 os.remove(path)
158 elif os.path.exists(path+'.db'): # dbm appends .db
159 os.remove(path+'.db')
161 def getclassdb(self, classname, mode='r'):
162 ''' grab a connection to the class db that will be used for
163 multiple actions
164 '''
165 if __debug__:
166 print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode)
167 return self.opendb('nodes.%s'%classname, mode)
169 def determine_db_type(self, path):
170 ''' determine which DB wrote the class file
171 '''
172 db_type = ''
173 if os.path.exists(path):
174 db_type = whichdb.whichdb(path)
175 if not db_type:
176 raise DatabaseError, "Couldn't identify database type"
177 elif os.path.exists(path+'.db'):
178 # if the path ends in '.db', it's a dbm database, whether
179 # anydbm says it's dbhash or not!
180 db_type = 'dbm'
181 return db_type
183 def opendb(self, name, mode):
184 '''Low-level database opener that gets around anydbm/dbm
185 eccentricities.
186 '''
187 if __debug__:
188 print >>hyperdb.DEBUG, 'opendb', (self, name, mode)
190 # figure the class db type
191 path = os.path.join(os.getcwd(), self.dir, name)
192 db_type = self.determine_db_type(path)
194 # new database? let anydbm pick the best dbm
195 if not db_type:
196 if __debug__:
197 print >>hyperdb.DEBUG, "opendb anydbm.open(%r, 'c')"%path
198 return anydbm.open(path, 'c')
200 # open the database with the correct module
201 try:
202 dbm = __import__(db_type)
203 except ImportError:
204 raise DatabaseError, \
205 "Couldn't open database - the required module '%s'"\
206 " is not available"%db_type
207 if __debug__:
208 print >>hyperdb.DEBUG, "opendb %r.open(%r, %r)"%(db_type, path,
209 mode)
210 return dbm.open(path, mode)
212 #
213 # Node IDs
214 #
215 def newid(self, classname):
216 ''' Generate a new id for the given class
217 '''
218 # open the ids DB - create if if doesn't exist
219 db = self.opendb('_ids', 'c')
220 if db.has_key(classname):
221 newid = db[classname] = str(int(db[classname]) + 1)
222 else:
223 # the count() bit is transitional - older dbs won't start at 1
224 newid = str(self.getclass(classname).count()+1)
225 db[classname] = newid
226 db.close()
227 return newid
229 def setid(self, classname, setid):
230 ''' Set the id counter: used during import of database
231 '''
232 # open the ids DB - create if if doesn't exist
233 db = self.opendb('_ids', 'c')
234 db[classname] = str(setid)
235 db.close()
237 #
238 # Nodes
239 #
240 def addnode(self, classname, nodeid, node):
241 ''' add the specified node to its class's db
242 '''
243 if __debug__:
244 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
246 # we'll be supplied these props if we're doing an import
247 if not node.has_key('creator'):
248 # add in the "calculated" properties (dupe so we don't affect
249 # calling code's node assumptions)
250 node = node.copy()
251 node['creator'] = self.curuserid
252 node['creation'] = node['activity'] = date.Date()
254 self.newnodes.setdefault(classname, {})[nodeid] = 1
255 self.cache.setdefault(classname, {})[nodeid] = node
256 self.savenode(classname, nodeid, node)
258 def setnode(self, classname, nodeid, node):
259 ''' change the specified node
260 '''
261 if __debug__:
262 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
263 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
265 # update the activity time (dupe so we don't affect
266 # calling code's node assumptions)
267 node = node.copy()
268 node['activity'] = date.Date()
270 # can't set without having already loaded the node
271 self.cache[classname][nodeid] = node
272 self.savenode(classname, nodeid, node)
274 def savenode(self, classname, nodeid, node):
275 ''' perform the saving of data specified by the set/addnode
276 '''
277 if __debug__:
278 print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node)
279 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
281 def getnode(self, classname, nodeid, db=None, cache=1):
282 ''' get a node from the database
283 '''
284 if __debug__:
285 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid, db)
286 if cache:
287 # try the cache
288 cache_dict = self.cache.setdefault(classname, {})
289 if cache_dict.has_key(nodeid):
290 if __debug__:
291 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
292 nodeid)
293 return cache_dict[nodeid]
295 if __debug__:
296 print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid)
298 # get from the database and save in the cache
299 if db is None:
300 db = self.getclassdb(classname)
301 if not db.has_key(nodeid):
302 raise IndexError, "no such %s %s"%(classname, nodeid)
304 # check the uncommitted, destroyed nodes
305 if (self.destroyednodes.has_key(classname) and
306 self.destroyednodes[classname].has_key(nodeid)):
307 raise IndexError, "no such %s %s"%(classname, nodeid)
309 # decode
310 res = marshal.loads(db[nodeid])
312 # reverse the serialisation
313 res = self.unserialise(classname, res)
315 # store off in the cache dict
316 if cache:
317 cache_dict[nodeid] = res
319 return res
321 def destroynode(self, classname, nodeid):
322 '''Remove a node from the database. Called exclusively by the
323 destroy() method on Class.
324 '''
325 if __debug__:
326 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
328 # remove from cache and newnodes if it's there
329 if (self.cache.has_key(classname) and
330 self.cache[classname].has_key(nodeid)):
331 del self.cache[classname][nodeid]
332 if (self.newnodes.has_key(classname) and
333 self.newnodes[classname].has_key(nodeid)):
334 del self.newnodes[classname][nodeid]
336 # see if there's any obvious commit actions that we should get rid of
337 for entry in self.transactions[:]:
338 if entry[1][:2] == (classname, nodeid):
339 self.transactions.remove(entry)
341 # add to the destroyednodes map
342 self.destroyednodes.setdefault(classname, {})[nodeid] = 1
344 # add the destroy commit action
345 self.transactions.append((self.doDestroyNode, (classname, nodeid)))
347 def serialise(self, classname, node):
348 '''Copy the node contents, converting non-marshallable data into
349 marshallable data.
350 '''
351 if __debug__:
352 print >>hyperdb.DEBUG, 'serialise', classname, node
353 properties = self.getclass(classname).getprops()
354 d = {}
355 for k, v in node.items():
356 # if the property doesn't exist, or is the "retired" flag then
357 # it won't be in the properties dict
358 if not properties.has_key(k):
359 d[k] = v
360 continue
362 # get the property spec
363 prop = properties[k]
365 if isinstance(prop, Password) and v is not None:
366 d[k] = str(v)
367 elif isinstance(prop, Date) and v is not None:
368 d[k] = v.serialise()
369 elif isinstance(prop, Interval) and v is not None:
370 d[k] = v.serialise()
371 else:
372 d[k] = v
373 return d
375 def unserialise(self, classname, node):
376 '''Decode the marshalled node data
377 '''
378 if __debug__:
379 print >>hyperdb.DEBUG, 'unserialise', classname, node
380 properties = self.getclass(classname).getprops()
381 d = {}
382 for k, v in node.items():
383 # if the property doesn't exist, or is the "retired" flag then
384 # it won't be in the properties dict
385 if not properties.has_key(k):
386 d[k] = v
387 continue
389 # get the property spec
390 prop = properties[k]
392 if isinstance(prop, Date) and v is not None:
393 d[k] = date.Date(v)
394 elif isinstance(prop, Interval) and v is not None:
395 d[k] = date.Interval(v)
396 elif isinstance(prop, Password) and v is not None:
397 p = password.Password()
398 p.unpack(v)
399 d[k] = p
400 else:
401 d[k] = v
402 return d
404 def hasnode(self, classname, nodeid, db=None):
405 ''' determine if the database has a given node
406 '''
407 if __debug__:
408 print >>hyperdb.DEBUG, 'hasnode', (self, classname, nodeid, db)
410 # try the cache
411 cache = self.cache.setdefault(classname, {})
412 if cache.has_key(nodeid):
413 if __debug__:
414 print >>hyperdb.TRACE, 'has %s %s cached'%(classname, nodeid)
415 return 1
416 if __debug__:
417 print >>hyperdb.TRACE, 'has %s %s'%(classname, nodeid)
419 # not in the cache - check the database
420 if db is None:
421 db = self.getclassdb(classname)
422 res = db.has_key(nodeid)
423 return res
425 def countnodes(self, classname, db=None):
426 if __debug__:
427 print >>hyperdb.DEBUG, 'countnodes', (self, classname, db)
429 count = 0
431 # include the uncommitted nodes
432 if self.newnodes.has_key(classname):
433 count += len(self.newnodes[classname])
434 if self.destroyednodes.has_key(classname):
435 count -= len(self.destroyednodes[classname])
437 # and count those in the DB
438 if db is None:
439 db = self.getclassdb(classname)
440 count = count + len(db.keys())
441 return count
443 def getnodeids(self, classname, db=None):
444 if __debug__:
445 print >>hyperdb.DEBUG, 'getnodeids', (self, classname, db)
447 res = []
449 # start off with the new nodes
450 if self.newnodes.has_key(classname):
451 res += self.newnodes[classname].keys()
453 if db is None:
454 db = self.getclassdb(classname)
455 res = res + db.keys()
457 # remove the uncommitted, destroyed nodes
458 if self.destroyednodes.has_key(classname):
459 for nodeid in self.destroyednodes[classname].keys():
460 if db.has_key(nodeid):
461 res.remove(nodeid)
463 return res
466 #
467 # Files - special node properties
468 # inherited from FileStorage
470 #
471 # Journal
472 #
473 def addjournal(self, classname, nodeid, action, params, creator=None,
474 creation=None):
475 ''' Journal the Action
476 'action' may be:
478 'create' or 'set' -- 'params' is a dictionary of property values
479 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
480 'retire' -- 'params' is None
481 '''
482 if __debug__:
483 print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid,
484 action, params, creator, creation)
485 self.transactions.append((self.doSaveJournal, (classname, nodeid,
486 action, params, creator, creation)))
488 def getjournal(self, classname, nodeid):
489 ''' get the journal for id
491 Raise IndexError if the node doesn't exist (as per history()'s
492 API)
493 '''
494 if __debug__:
495 print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid)
496 # attempt to open the journal - in some rare cases, the journal may
497 # not exist
498 try:
499 db = self.opendb('journals.%s'%classname, 'r')
500 except anydbm.error, error:
501 if str(error) == "need 'c' or 'n' flag to open new db":
502 raise IndexError, 'no such %s %s'%(classname, nodeid)
503 elif error.args[0] != 2:
504 raise
505 raise IndexError, 'no such %s %s'%(classname, nodeid)
506 try:
507 journal = marshal.loads(db[nodeid])
508 except KeyError:
509 db.close()
510 raise IndexError, 'no such %s %s'%(classname, nodeid)
511 db.close()
512 res = []
513 for nodeid, date_stamp, user, action, params in journal:
514 res.append((nodeid, date.Date(date_stamp), user, action, params))
515 return res
517 def pack(self, pack_before):
518 ''' Delete all journal entries except "create" before 'pack_before'.
519 '''
520 if __debug__:
521 print >>hyperdb.DEBUG, 'packjournal', (self, pack_before)
523 pack_before = pack_before.serialise()
524 for classname in self.getclasses():
525 # get the journal db
526 db_name = 'journals.%s'%classname
527 path = os.path.join(os.getcwd(), self.dir, classname)
528 db_type = self.determine_db_type(path)
529 db = self.opendb(db_name, 'w')
531 for key in db.keys():
532 # get the journal for this db entry
533 journal = marshal.loads(db[key])
534 l = []
535 last_set_entry = None
536 for entry in journal:
537 # unpack the entry
538 (nodeid, date_stamp, self.journaltag, action,
539 params) = entry
540 # if the entry is after the pack date, _or_ the initial
541 # create entry, then it stays
542 if date_stamp > pack_before or action == 'create':
543 l.append(entry)
544 db[key] = marshal.dumps(l)
545 if db_type == 'gdbm':
546 db.reorganize()
547 db.close()
550 #
551 # Basic transaction support
552 #
553 def commit(self):
554 ''' Commit the current transactions.
555 '''
556 if __debug__:
557 print >>hyperdb.DEBUG, 'commit', (self,)
559 # keep a handle to all the database files opened
560 self.databases = {}
562 # now, do all the transactions
563 reindex = {}
564 for method, args in self.transactions:
565 reindex[method(*args)] = 1
567 # now close all the database files
568 for db in self.databases.values():
569 db.close()
570 del self.databases
572 # reindex the nodes that request it
573 for classname, nodeid in filter(None, reindex.keys()):
574 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
575 self.getclass(classname).index(nodeid)
577 # save the indexer state
578 self.indexer.save_index()
580 self.clearCache()
582 def clearCache(self):
583 # all transactions committed, back to normal
584 self.cache = {}
585 self.dirtynodes = {}
586 self.newnodes = {}
587 self.destroyednodes = {}
588 self.transactions = []
590 def getCachedClassDB(self, classname):
591 ''' get the class db, looking in our cache of databases for commit
592 '''
593 # get the database handle
594 db_name = 'nodes.%s'%classname
595 if not self.databases.has_key(db_name):
596 self.databases[db_name] = self.getclassdb(classname, 'c')
597 return self.databases[db_name]
599 def doSaveNode(self, classname, nodeid, node):
600 if __debug__:
601 print >>hyperdb.DEBUG, 'doSaveNode', (self, classname, nodeid,
602 node)
604 db = self.getCachedClassDB(classname)
606 # now save the marshalled data
607 db[nodeid] = marshal.dumps(self.serialise(classname, node))
609 # return the classname, nodeid so we reindex this content
610 return (classname, nodeid)
612 def getCachedJournalDB(self, classname):
613 ''' get the journal db, looking in our cache of databases for commit
614 '''
615 # get the database handle
616 db_name = 'journals.%s'%classname
617 if not self.databases.has_key(db_name):
618 self.databases[db_name] = self.opendb(db_name, 'c')
619 return self.databases[db_name]
621 def doSaveJournal(self, classname, nodeid, action, params, creator,
622 creation):
623 # serialise the parameters now if necessary
624 if isinstance(params, type({})):
625 if action in ('set', 'create'):
626 params = self.serialise(classname, params)
628 # handle supply of the special journalling parameters (usually
629 # supplied on importing an existing database)
630 if creator:
631 journaltag = creator
632 else:
633 journaltag = self.curuserid
634 if creation:
635 journaldate = creation.serialise()
636 else:
637 journaldate = date.Date().serialise()
639 # create the journal entry
640 entry = (nodeid, journaldate, journaltag, action, params)
642 if __debug__:
643 print >>hyperdb.DEBUG, 'doSaveJournal', entry
645 db = self.getCachedJournalDB(classname)
647 # now insert the journal entry
648 if db.has_key(nodeid):
649 # append to existing
650 s = db[nodeid]
651 l = marshal.loads(s)
652 l.append(entry)
653 else:
654 l = [entry]
656 db[nodeid] = marshal.dumps(l)
658 def doDestroyNode(self, classname, nodeid):
659 if __debug__:
660 print >>hyperdb.DEBUG, 'doDestroyNode', (self, classname, nodeid)
662 # delete from the class database
663 db = self.getCachedClassDB(classname)
664 if db.has_key(nodeid):
665 del db[nodeid]
667 # delete from the database
668 db = self.getCachedJournalDB(classname)
669 if db.has_key(nodeid):
670 del db[nodeid]
672 # return the classname, nodeid so we reindex this content
673 return (classname, nodeid)
675 def rollback(self):
676 ''' Reverse all actions from the current transaction.
677 '''
678 if __debug__:
679 print >>hyperdb.DEBUG, 'rollback', (self, )
680 for method, args in self.transactions:
681 # delete temporary files
682 if method == self.doStoreFile:
683 self.rollbackStoreFile(*args)
684 self.cache = {}
685 self.dirtynodes = {}
686 self.newnodes = {}
687 self.destroyednodes = {}
688 self.transactions = []
690 def close(self):
691 ''' Nothing to do
692 '''
693 if self.lockfile is not None:
694 locking.release_lock(self.lockfile)
695 if self.lockfile is not None:
696 self.lockfile.close()
697 self.lockfile = None
699 _marker = []
700 class Class(hyperdb.Class):
701 '''The handle to a particular class of nodes in a hyperdatabase.'''
703 def __init__(self, db, classname, **properties):
704 '''Create a new class with a given name and property specification.
706 'classname' must not collide with the name of an existing class,
707 or a ValueError is raised. The keyword arguments in 'properties'
708 must map names to property objects, or a TypeError is raised.
709 '''
710 if (properties.has_key('creation') or properties.has_key('activity')
711 or properties.has_key('creator')):
712 raise ValueError, '"creation", "activity" and "creator" are '\
713 'reserved'
715 self.classname = classname
716 self.properties = properties
717 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
718 self.key = ''
720 # should we journal changes (default yes)
721 self.do_journal = 1
723 # do the db-related init stuff
724 db.addclass(self)
726 self.auditors = {'create': [], 'set': [], 'retire': []}
727 self.reactors = {'create': [], 'set': [], 'retire': []}
729 def enableJournalling(self):
730 '''Turn journalling on for this class
731 '''
732 self.do_journal = 1
734 def disableJournalling(self):
735 '''Turn journalling off for this class
736 '''
737 self.do_journal = 0
739 # Editing nodes:
741 def create(self, **propvalues):
742 '''Create a new node of this class and return its id.
744 The keyword arguments in 'propvalues' map property names to values.
746 The values of arguments must be acceptable for the types of their
747 corresponding properties or a TypeError is raised.
749 If this class has a key property, it must be present and its value
750 must not collide with other key strings or a ValueError is raised.
752 Any other properties on this class that are missing from the
753 'propvalues' dictionary are set to None.
755 If an id in a link or multilink property does not refer to a valid
756 node, an IndexError is raised.
758 These operations trigger detectors and can be vetoed. Attempts
759 to modify the "creation" or "activity" properties cause a KeyError.
760 '''
761 if propvalues.has_key('id'):
762 raise KeyError, '"id" is reserved'
764 if self.db.journaltag is None:
765 raise DatabaseError, 'Database open read-only'
767 if propvalues.has_key('creation') or propvalues.has_key('activity'):
768 raise KeyError, '"creation" and "activity" are reserved'
770 self.fireAuditors('create', None, propvalues)
772 # new node's id
773 newid = self.db.newid(self.classname)
775 # validate propvalues
776 num_re = re.compile('^\d+$')
777 for key, value in propvalues.items():
778 if key == self.key:
779 try:
780 self.lookup(value)
781 except KeyError:
782 pass
783 else:
784 raise ValueError, 'node with key "%s" exists'%value
786 # try to handle this property
787 try:
788 prop = self.properties[key]
789 except KeyError:
790 raise KeyError, '"%s" has no property "%s"'%(self.classname,
791 key)
793 if value is not None and isinstance(prop, Link):
794 if type(value) != type(''):
795 raise ValueError, 'link value must be String'
796 link_class = self.properties[key].classname
797 # if it isn't a number, it's a key
798 if not num_re.match(value):
799 try:
800 value = self.db.classes[link_class].lookup(value)
801 except (TypeError, KeyError):
802 raise IndexError, 'new property "%s": %s not a %s'%(
803 key, value, link_class)
804 elif not self.db.getclass(link_class).hasnode(value):
805 raise IndexError, '%s has no node %s'%(link_class, value)
807 # save off the value
808 propvalues[key] = value
810 # register the link with the newly linked node
811 if self.do_journal and self.properties[key].do_journal:
812 self.db.addjournal(link_class, value, 'link',
813 (self.classname, newid, key))
815 elif isinstance(prop, Multilink):
816 if type(value) != type([]):
817 raise TypeError, 'new property "%s" not a list of ids'%key
819 # clean up and validate the list of links
820 link_class = self.properties[key].classname
821 l = []
822 for entry in value:
823 if type(entry) != type(''):
824 raise ValueError, '"%s" multilink value (%r) '\
825 'must contain Strings'%(key, value)
826 # if it isn't a number, it's a key
827 if not num_re.match(entry):
828 try:
829 entry = self.db.classes[link_class].lookup(entry)
830 except (TypeError, KeyError):
831 raise IndexError, 'new property "%s": %s not a %s'%(
832 key, entry, self.properties[key].classname)
833 l.append(entry)
834 value = l
835 propvalues[key] = value
837 # handle additions
838 for nodeid in value:
839 if not self.db.getclass(link_class).hasnode(nodeid):
840 raise IndexError, '%s has no node %s'%(link_class,
841 nodeid)
842 # register the link with the newly linked node
843 if self.do_journal and self.properties[key].do_journal:
844 self.db.addjournal(link_class, nodeid, 'link',
845 (self.classname, newid, key))
847 elif isinstance(prop, String):
848 if type(value) != type('') and type(value) != type(u''):
849 raise TypeError, 'new property "%s" not a string'%key
851 elif isinstance(prop, Password):
852 if not isinstance(value, password.Password):
853 raise TypeError, 'new property "%s" not a Password'%key
855 elif isinstance(prop, Date):
856 if value is not None and not isinstance(value, date.Date):
857 raise TypeError, 'new property "%s" not a Date'%key
859 elif isinstance(prop, Interval):
860 if value is not None and not isinstance(value, date.Interval):
861 raise TypeError, 'new property "%s" not an Interval'%key
863 elif value is not None and isinstance(prop, Number):
864 try:
865 float(value)
866 except ValueError:
867 raise TypeError, 'new property "%s" not numeric'%key
869 elif value is not None and isinstance(prop, Boolean):
870 try:
871 int(value)
872 except ValueError:
873 raise TypeError, 'new property "%s" not boolean'%key
875 # make sure there's data where there needs to be
876 for key, prop in self.properties.items():
877 if propvalues.has_key(key):
878 continue
879 if key == self.key:
880 raise ValueError, 'key property "%s" is required'%key
881 if isinstance(prop, Multilink):
882 propvalues[key] = []
883 else:
884 propvalues[key] = None
886 # done
887 self.db.addnode(self.classname, newid, propvalues)
888 if self.do_journal:
889 self.db.addjournal(self.classname, newid, 'create', {})
891 self.fireReactors('create', newid, None)
893 return newid
895 def export_list(self, propnames, nodeid):
896 ''' Export a node - generate a list of CSV-able data in the order
897 specified by propnames for the given node.
898 '''
899 properties = self.getprops()
900 l = []
901 for prop in propnames:
902 proptype = properties[prop]
903 value = self.get(nodeid, prop)
904 # "marshal" data where needed
905 if value is None:
906 pass
907 elif isinstance(proptype, hyperdb.Date):
908 value = value.get_tuple()
909 elif isinstance(proptype, hyperdb.Interval):
910 value = value.get_tuple()
911 elif isinstance(proptype, hyperdb.Password):
912 value = str(value)
913 l.append(repr(value))
914 return l
916 def import_list(self, propnames, proplist):
917 ''' Import a node - all information including "id" is present and
918 should not be sanity checked. Triggers are not triggered. The
919 journal should be initialised using the "creator" and "created"
920 information.
922 Return the nodeid of the node imported.
923 '''
924 if self.db.journaltag is None:
925 raise DatabaseError, 'Database open read-only'
926 properties = self.getprops()
928 # make the new node's property map
929 d = {}
930 for i in range(len(propnames)):
931 # Use eval to reverse the repr() used to output the CSV
932 value = eval(proplist[i])
934 # Figure the property for this column
935 propname = propnames[i]
936 prop = properties[propname]
938 # "unmarshal" where necessary
939 if propname == 'id':
940 newid = value
941 continue
942 elif value is None:
943 # don't set Nones
944 continue
945 elif isinstance(prop, hyperdb.Date):
946 value = date.Date(value)
947 elif isinstance(prop, hyperdb.Interval):
948 value = date.Interval(value)
949 elif isinstance(prop, hyperdb.Password):
950 pwd = password.Password()
951 pwd.unpack(value)
952 value = pwd
953 d[propname] = value
955 # add the node and journal
956 self.db.addnode(self.classname, newid, d)
958 # extract the journalling stuff and nuke it
959 if d.has_key('creator'):
960 creator = d['creator']
961 del d['creator']
962 else:
963 creator = None
964 if d.has_key('creation'):
965 creation = d['creation']
966 del d['creation']
967 else:
968 creation = None
969 if d.has_key('activity'):
970 del d['activity']
971 self.db.addjournal(self.classname, newid, 'create', {}, creator,
972 creation)
973 return newid
975 def get(self, nodeid, propname, default=_marker, cache=1):
976 '''Get the value of a property on an existing node of this class.
978 'nodeid' must be the id of an existing node of this class or an
979 IndexError is raised. 'propname' must be the name of a property
980 of this class or a KeyError is raised.
982 'cache' indicates whether the transaction cache should be queried
983 for the node. If the node has been modified and you need to
984 determine what its values prior to modification are, you need to
985 set cache=0.
987 Attempts to get the "creation" or "activity" properties should
988 do the right thing.
989 '''
990 if propname == 'id':
991 return nodeid
993 # get the node's dict
994 d = self.db.getnode(self.classname, nodeid, cache=cache)
996 # check for one of the special props
997 if propname == 'creation':
998 if d.has_key('creation'):
999 return d['creation']
1000 if not self.do_journal:
1001 raise ValueError, 'Journalling is disabled for this class'
1002 journal = self.db.getjournal(self.classname, nodeid)
1003 if journal:
1004 return self.db.getjournal(self.classname, nodeid)[0][1]
1005 else:
1006 # on the strange chance that there's no journal
1007 return date.Date()
1008 if propname == 'activity':
1009 if d.has_key('activity'):
1010 return d['activity']
1011 if not self.do_journal:
1012 raise ValueError, 'Journalling is disabled for this class'
1013 journal = self.db.getjournal(self.classname, nodeid)
1014 if journal:
1015 return self.db.getjournal(self.classname, nodeid)[-1][1]
1016 else:
1017 # on the strange chance that there's no journal
1018 return date.Date()
1019 if propname == 'creator':
1020 if d.has_key('creator'):
1021 return d['creator']
1022 if not self.do_journal:
1023 raise ValueError, 'Journalling is disabled for this class'
1024 journal = self.db.getjournal(self.classname, nodeid)
1025 if journal:
1026 num_re = re.compile('^\d+$')
1027 value = self.db.getjournal(self.classname, nodeid)[0][2]
1028 if num_re.match(value):
1029 return value
1030 else:
1031 # old-style "username" journal tag
1032 try:
1033 return self.db.user.lookup(value)
1034 except KeyError:
1035 # user's been retired, return admin
1036 return '1'
1037 else:
1038 return self.db.curuserid
1040 # get the property (raises KeyErorr if invalid)
1041 prop = self.properties[propname]
1043 if not d.has_key(propname):
1044 if default is _marker:
1045 if isinstance(prop, Multilink):
1046 return []
1047 else:
1048 return None
1049 else:
1050 return default
1052 # return a dupe of the list so code doesn't get confused
1053 if isinstance(prop, Multilink):
1054 return d[propname][:]
1056 return d[propname]
1058 # not in spec
1059 def getnode(self, nodeid, cache=1):
1060 ''' Return a convenience wrapper for the node.
1062 'nodeid' must be the id of an existing node of this class or an
1063 IndexError is raised.
1065 'cache' indicates whether the transaction cache should be queried
1066 for the node. If the node has been modified and you need to
1067 determine what its values prior to modification are, you need to
1068 set cache=0.
1069 '''
1070 return Node(self, nodeid, cache=cache)
1072 def set(self, nodeid, **propvalues):
1073 '''Modify a property on an existing node of this class.
1075 'nodeid' must be the id of an existing node of this class or an
1076 IndexError is raised.
1078 Each key in 'propvalues' must be the name of a property of this
1079 class or a KeyError is raised.
1081 All values in 'propvalues' must be acceptable types for their
1082 corresponding properties or a TypeError is raised.
1084 If the value of the key property is set, it must not collide with
1085 other key strings or a ValueError is raised.
1087 If the value of a Link or Multilink property contains an invalid
1088 node id, a ValueError is raised.
1090 These operations trigger detectors and can be vetoed. Attempts
1091 to modify the "creation" or "activity" properties cause a KeyError.
1092 '''
1093 if not propvalues:
1094 return propvalues
1096 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1097 raise KeyError, '"creation" and "activity" are reserved'
1099 if propvalues.has_key('id'):
1100 raise KeyError, '"id" is reserved'
1102 if self.db.journaltag is None:
1103 raise DatabaseError, 'Database open read-only'
1105 self.fireAuditors('set', nodeid, propvalues)
1106 # Take a copy of the node dict so that the subsequent set
1107 # operation doesn't modify the oldvalues structure.
1108 try:
1109 # try not using the cache initially
1110 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid,
1111 cache=0))
1112 except IndexError:
1113 # this will be needed if somone does a create() and set()
1114 # with no intervening commit()
1115 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1117 node = self.db.getnode(self.classname, nodeid)
1118 if node.has_key(self.db.RETIRED_FLAG):
1119 raise IndexError
1120 num_re = re.compile('^\d+$')
1122 # if the journal value is to be different, store it in here
1123 journalvalues = {}
1125 for propname, value in propvalues.items():
1126 # check to make sure we're not duplicating an existing key
1127 if propname == self.key and node[propname] != value:
1128 try:
1129 self.lookup(value)
1130 except KeyError:
1131 pass
1132 else:
1133 raise ValueError, 'node with key "%s" exists'%value
1135 # this will raise the KeyError if the property isn't valid
1136 # ... we don't use getprops() here because we only care about
1137 # the writeable properties.
1138 try:
1139 prop = self.properties[propname]
1140 except KeyError:
1141 raise KeyError, '"%s" has no property named "%s"'%(
1142 self.classname, propname)
1144 # if the value's the same as the existing value, no sense in
1145 # doing anything
1146 current = node.get(propname, None)
1147 if value == current:
1148 del propvalues[propname]
1149 continue
1150 journalvalues[propname] = current
1152 # do stuff based on the prop type
1153 if isinstance(prop, Link):
1154 link_class = prop.classname
1155 # if it isn't a number, it's a key
1156 if value is not None and not isinstance(value, type('')):
1157 raise ValueError, 'property "%s" link value be a string'%(
1158 propname)
1159 if isinstance(value, type('')) and not num_re.match(value):
1160 try:
1161 value = self.db.classes[link_class].lookup(value)
1162 except (TypeError, KeyError):
1163 raise IndexError, 'new property "%s": %s not a %s'%(
1164 propname, value, prop.classname)
1166 if (value is not None and
1167 not self.db.getclass(link_class).hasnode(value)):
1168 raise IndexError, '%s has no node %s'%(link_class, value)
1170 if self.do_journal and prop.do_journal:
1171 # register the unlink with the old linked node
1172 if node.has_key(propname) and node[propname] is not None:
1173 self.db.addjournal(link_class, node[propname], 'unlink',
1174 (self.classname, nodeid, propname))
1176 # register the link with the newly linked node
1177 if value is not None:
1178 self.db.addjournal(link_class, value, 'link',
1179 (self.classname, nodeid, propname))
1181 elif isinstance(prop, Multilink):
1182 if type(value) != type([]):
1183 raise TypeError, 'new property "%s" not a list of'\
1184 ' ids'%propname
1185 link_class = self.properties[propname].classname
1186 l = []
1187 for entry in value:
1188 # if it isn't a number, it's a key
1189 if type(entry) != type(''):
1190 raise ValueError, 'new property "%s" link value ' \
1191 'must be a string'%propname
1192 if not num_re.match(entry):
1193 try:
1194 entry = self.db.classes[link_class].lookup(entry)
1195 except (TypeError, KeyError):
1196 raise IndexError, 'new property "%s": %s not a %s'%(
1197 propname, entry,
1198 self.properties[propname].classname)
1199 l.append(entry)
1200 value = l
1201 propvalues[propname] = value
1203 # figure the journal entry for this property
1204 add = []
1205 remove = []
1207 # handle removals
1208 if node.has_key(propname):
1209 l = node[propname]
1210 else:
1211 l = []
1212 for id in l[:]:
1213 if id in value:
1214 continue
1215 # register the unlink with the old linked node
1216 if self.do_journal and self.properties[propname].do_journal:
1217 self.db.addjournal(link_class, id, 'unlink',
1218 (self.classname, nodeid, propname))
1219 l.remove(id)
1220 remove.append(id)
1222 # handle additions
1223 for id in value:
1224 if not self.db.getclass(link_class).hasnode(id):
1225 raise IndexError, '%s has no node %s'%(link_class, id)
1226 if id in l:
1227 continue
1228 # register the link with the newly linked node
1229 if self.do_journal and self.properties[propname].do_journal:
1230 self.db.addjournal(link_class, id, 'link',
1231 (self.classname, nodeid, propname))
1232 l.append(id)
1233 add.append(id)
1235 # figure the journal entry
1236 l = []
1237 if add:
1238 l.append(('+', add))
1239 if remove:
1240 l.append(('-', remove))
1241 if l:
1242 journalvalues[propname] = tuple(l)
1244 elif isinstance(prop, String):
1245 if value is not None and type(value) != type('') and type(value) != type(u''):
1246 raise TypeError, 'new property "%s" not a string'%propname
1248 elif isinstance(prop, Password):
1249 if not isinstance(value, password.Password):
1250 raise TypeError, 'new property "%s" not a Password'%propname
1251 propvalues[propname] = value
1253 elif value is not None and isinstance(prop, Date):
1254 if not isinstance(value, date.Date):
1255 raise TypeError, 'new property "%s" not a Date'% propname
1256 propvalues[propname] = value
1258 elif value is not None and isinstance(prop, Interval):
1259 if not isinstance(value, date.Interval):
1260 raise TypeError, 'new property "%s" not an '\
1261 'Interval'%propname
1262 propvalues[propname] = value
1264 elif value is not None and isinstance(prop, Number):
1265 try:
1266 float(value)
1267 except ValueError:
1268 raise TypeError, 'new property "%s" not numeric'%propname
1270 elif value is not None and isinstance(prop, Boolean):
1271 try:
1272 int(value)
1273 except ValueError:
1274 raise TypeError, 'new property "%s" not boolean'%propname
1276 node[propname] = value
1278 # nothing to do?
1279 if not propvalues:
1280 return propvalues
1282 # do the set, and journal it
1283 self.db.setnode(self.classname, nodeid, node)
1285 if self.do_journal:
1286 self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
1288 self.fireReactors('set', nodeid, oldvalues)
1290 return propvalues
1292 def retire(self, nodeid):
1293 '''Retire a node.
1295 The properties on the node remain available from the get() method,
1296 and the node's id is never reused.
1298 Retired nodes are not returned by the find(), list(), or lookup()
1299 methods, and other nodes may reuse the values of their key properties.
1301 These operations trigger detectors and can be vetoed. Attempts
1302 to modify the "creation" or "activity" properties cause a KeyError.
1303 '''
1304 if self.db.journaltag is None:
1305 raise DatabaseError, 'Database open read-only'
1307 self.fireAuditors('retire', nodeid, None)
1309 node = self.db.getnode(self.classname, nodeid)
1310 node[self.db.RETIRED_FLAG] = 1
1311 self.db.setnode(self.classname, nodeid, node)
1312 if self.do_journal:
1313 self.db.addjournal(self.classname, nodeid, 'retired', None)
1315 self.fireReactors('retire', nodeid, None)
1317 def is_retired(self, nodeid):
1318 '''Return true if the node is retired.
1319 '''
1320 node = self.db.getnode(cn, nodeid, cldb)
1321 if node.has_key(self.db.RETIRED_FLAG):
1322 return 1
1323 return 0
1325 def destroy(self, nodeid):
1326 '''Destroy a node.
1328 WARNING: this method should never be used except in extremely rare
1329 situations where there could never be links to the node being
1330 deleted
1331 WARNING: use retire() instead
1332 WARNING: the properties of this node will not be available ever again
1333 WARNING: really, use retire() instead
1335 Well, I think that's enough warnings. This method exists mostly to
1336 support the session storage of the cgi interface.
1337 '''
1338 if self.db.journaltag is None:
1339 raise DatabaseError, 'Database open read-only'
1340 self.db.destroynode(self.classname, nodeid)
1342 def history(self, nodeid):
1343 '''Retrieve the journal of edits on a particular node.
1345 'nodeid' must be the id of an existing node of this class or an
1346 IndexError is raised.
1348 The returned list contains tuples of the form
1350 (nodeid, date, tag, action, params)
1352 'date' is a Timestamp object specifying the time of the change and
1353 'tag' is the journaltag specified when the database was opened.
1354 '''
1355 if not self.do_journal:
1356 raise ValueError, 'Journalling is disabled for this class'
1357 return self.db.getjournal(self.classname, nodeid)
1359 # Locating nodes:
1360 def hasnode(self, nodeid):
1361 '''Determine if the given nodeid actually exists
1362 '''
1363 return self.db.hasnode(self.classname, nodeid)
1365 def setkey(self, propname):
1366 '''Select a String property of this class to be the key property.
1368 'propname' must be the name of a String property of this class or
1369 None, or a TypeError is raised. The values of the key property on
1370 all existing nodes must be unique or a ValueError is raised. If the
1371 property doesn't exist, KeyError is raised.
1372 '''
1373 prop = self.getprops()[propname]
1374 if not isinstance(prop, String):
1375 raise TypeError, 'key properties must be String'
1376 self.key = propname
1378 def getkey(self):
1379 '''Return the name of the key property for this class or None.'''
1380 return self.key
1382 def labelprop(self, default_to_id=0):
1383 ''' Return the property name for a label for the given node.
1385 This method attempts to generate a consistent label for the node.
1386 It tries the following in order:
1387 1. key property
1388 2. "name" property
1389 3. "title" property
1390 4. first property from the sorted property name list
1391 '''
1392 k = self.getkey()
1393 if k:
1394 return k
1395 props = self.getprops()
1396 if props.has_key('name'):
1397 return 'name'
1398 elif props.has_key('title'):
1399 return 'title'
1400 if default_to_id:
1401 return 'id'
1402 props = props.keys()
1403 props.sort()
1404 return props[0]
1406 # TODO: set up a separate index db file for this? profile?
1407 def lookup(self, keyvalue):
1408 '''Locate a particular node by its key property and return its id.
1410 If this class has no key property, a TypeError is raised. If the
1411 'keyvalue' matches one of the values for the key property among
1412 the nodes in this class, the matching node's id is returned;
1413 otherwise a KeyError is raised.
1414 '''
1415 if not self.key:
1416 raise TypeError, 'No key property set for class %s'%self.classname
1417 cldb = self.db.getclassdb(self.classname)
1418 try:
1419 for nodeid in self.db.getnodeids(self.classname, cldb):
1420 node = self.db.getnode(self.classname, nodeid, cldb)
1421 if node.has_key(self.db.RETIRED_FLAG):
1422 continue
1423 if node[self.key] == keyvalue:
1424 return nodeid
1425 finally:
1426 cldb.close()
1427 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1428 keyvalue, self.classname)
1430 # change from spec - allows multiple props to match
1431 def find(self, **propspec):
1432 '''Get the ids of nodes in this class which link to the given nodes.
1434 'propspec' consists of keyword args propname=nodeid or
1435 propname={nodeid:1, }
1436 'propname' must be the name of a property in this class, or a
1437 KeyError is raised. That property must be a Link or
1438 Multilink property, or a TypeError is raised.
1440 Any node in this class whose 'propname' property links to any of the
1441 nodeids will be returned. Used by the full text indexing, which knows
1442 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1443 issues:
1445 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1446 '''
1447 propspec = propspec.items()
1448 for propname, nodeids in propspec:
1449 # check the prop is OK
1450 prop = self.properties[propname]
1451 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1452 raise TypeError, "'%s' not a Link/Multilink property"%propname
1454 # ok, now do the find
1455 cldb = self.db.getclassdb(self.classname)
1456 l = []
1457 try:
1458 for id in self.db.getnodeids(self.classname, db=cldb):
1459 node = self.db.getnode(self.classname, id, db=cldb)
1460 if node.has_key(self.db.RETIRED_FLAG):
1461 continue
1462 for propname, nodeids in propspec:
1463 # can't test if the node doesn't have this property
1464 if not node.has_key(propname):
1465 continue
1466 if type(nodeids) is type(''):
1467 nodeids = {nodeids:1}
1468 prop = self.properties[propname]
1469 value = node[propname]
1470 if isinstance(prop, Link) and nodeids.has_key(value):
1471 l.append(id)
1472 break
1473 elif isinstance(prop, Multilink):
1474 hit = 0
1475 for v in value:
1476 if nodeids.has_key(v):
1477 l.append(id)
1478 hit = 1
1479 break
1480 if hit:
1481 break
1482 finally:
1483 cldb.close()
1484 return l
1486 def stringFind(self, **requirements):
1487 '''Locate a particular node by matching a set of its String
1488 properties in a caseless search.
1490 If the property is not a String property, a TypeError is raised.
1492 The return is a list of the id of all nodes that match.
1493 '''
1494 for propname in requirements.keys():
1495 prop = self.properties[propname]
1496 if isinstance(not prop, String):
1497 raise TypeError, "'%s' not a String property"%propname
1498 requirements[propname] = requirements[propname].lower()
1499 l = []
1500 cldb = self.db.getclassdb(self.classname)
1501 try:
1502 for nodeid in self.db.getnodeids(self.classname, cldb):
1503 node = self.db.getnode(self.classname, nodeid, cldb)
1504 if node.has_key(self.db.RETIRED_FLAG):
1505 continue
1506 for key, value in requirements.items():
1507 if not node.has_key(key):
1508 break
1509 if node[key] is None or node[key].lower() != value:
1510 break
1511 else:
1512 l.append(nodeid)
1513 finally:
1514 cldb.close()
1515 return l
1517 def list(self):
1518 ''' Return a list of the ids of the active nodes in this class.
1519 '''
1520 l = []
1521 cn = self.classname
1522 cldb = self.db.getclassdb(cn)
1523 try:
1524 for nodeid in self.db.getnodeids(cn, cldb):
1525 node = self.db.getnode(cn, nodeid, cldb)
1526 if node.has_key(self.db.RETIRED_FLAG):
1527 continue
1528 l.append(nodeid)
1529 finally:
1530 cldb.close()
1531 l.sort()
1532 return l
1534 def filter(self, search_matches, filterspec, sort=(None,None),
1535 group=(None,None), num_re = re.compile('^\d+$')):
1536 ''' Return a list of the ids of the active nodes in this class that
1537 match the 'filter' spec, sorted by the group spec and then the
1538 sort spec.
1540 "filterspec" is {propname: value(s)}
1541 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1542 and prop is a prop name or None
1543 "search_matches" is {nodeid: marker}
1545 The filter must match all properties specificed - but if the
1546 property value to match is a list, any one of the values in the
1547 list may match for that property to match.
1548 '''
1549 cn = self.classname
1551 # optimise filterspec
1552 l = []
1553 props = self.getprops()
1554 LINK = 0
1555 MULTILINK = 1
1556 STRING = 2
1557 OTHER = 6
1558 for k, v in filterspec.items():
1559 propclass = props[k]
1560 if isinstance(propclass, Link):
1561 if type(v) is not type([]):
1562 v = [v]
1563 # replace key values with node ids
1564 u = []
1565 link_class = self.db.classes[propclass.classname]
1566 for entry in v:
1567 if entry == '-1': entry = None
1568 elif not num_re.match(entry):
1569 try:
1570 entry = link_class.lookup(entry)
1571 except (TypeError,KeyError):
1572 raise ValueError, 'property "%s": %s not a %s'%(
1573 k, entry, self.properties[k].classname)
1574 u.append(entry)
1576 l.append((LINK, k, u))
1577 elif isinstance(propclass, Multilink):
1578 if type(v) is not type([]):
1579 v = [v]
1580 # replace key values with node ids
1581 u = []
1582 link_class = self.db.classes[propclass.classname]
1583 for entry in v:
1584 if not num_re.match(entry):
1585 try:
1586 entry = link_class.lookup(entry)
1587 except (TypeError,KeyError):
1588 raise ValueError, 'new property "%s": %s not a %s'%(
1589 k, entry, self.properties[k].classname)
1590 u.append(entry)
1591 l.append((MULTILINK, k, u))
1592 elif isinstance(propclass, String) and k != 'id':
1593 # simple glob searching
1594 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1595 v = v.replace('?', '.')
1596 v = v.replace('*', '.*?')
1597 l.append((STRING, k, re.compile(v, re.I)))
1598 elif isinstance(propclass, Boolean):
1599 if type(v) is type(''):
1600 bv = v.lower() in ('yes', 'true', 'on', '1')
1601 else:
1602 bv = v
1603 l.append((OTHER, k, bv))
1604 elif isinstance(propclass, Date):
1605 l.append((OTHER, k, date.Date(v)))
1606 elif isinstance(propclass, Interval):
1607 l.append((OTHER, k, date.Interval(v)))
1608 elif isinstance(propclass, Number):
1609 l.append((OTHER, k, int(v)))
1610 else:
1611 l.append((OTHER, k, v))
1612 filterspec = l
1614 # now, find all the nodes that are active and pass filtering
1615 l = []
1616 cldb = self.db.getclassdb(cn)
1617 try:
1618 # TODO: only full-scan once (use items())
1619 for nodeid in self.db.getnodeids(cn, cldb):
1620 node = self.db.getnode(cn, nodeid, cldb)
1621 if node.has_key(self.db.RETIRED_FLAG):
1622 continue
1623 # apply filter
1624 for t, k, v in filterspec:
1625 # handle the id prop
1626 if k == 'id' and v == nodeid:
1627 continue
1629 # make sure the node has the property
1630 if not node.has_key(k):
1631 # this node doesn't have this property, so reject it
1632 break
1634 # now apply the property filter
1635 if t == LINK:
1636 # link - if this node's property doesn't appear in the
1637 # filterspec's nodeid list, skip it
1638 if node[k] not in v:
1639 break
1640 elif t == MULTILINK:
1641 # multilink - if any of the nodeids required by the
1642 # filterspec aren't in this node's property, then skip
1643 # it
1644 have = node[k]
1645 for want in v:
1646 if want not in have:
1647 break
1648 else:
1649 continue
1650 break
1651 elif t == STRING:
1652 # RE search
1653 if node[k] is None or not v.search(node[k]):
1654 break
1655 elif t == OTHER:
1656 # straight value comparison for the other types
1657 if node[k] != v:
1658 break
1659 else:
1660 l.append((nodeid, node))
1661 finally:
1662 cldb.close()
1663 l.sort()
1665 # filter based on full text search
1666 if search_matches is not None:
1667 k = []
1668 for v in l:
1669 if search_matches.has_key(v[0]):
1670 k.append(v)
1671 l = k
1673 # now, sort the result
1674 def sortfun(a, b, sort=sort, group=group, properties=self.getprops(),
1675 db = self.db, cl=self):
1676 a_id, an = a
1677 b_id, bn = b
1678 # sort by group and then sort
1679 for dir, prop in group, sort:
1680 if dir is None or prop is None: continue
1682 # sorting is class-specific
1683 propclass = properties[prop]
1685 # handle the properties that might be "faked"
1686 # also, handle possible missing properties
1687 try:
1688 if not an.has_key(prop):
1689 an[prop] = cl.get(a_id, prop)
1690 av = an[prop]
1691 except KeyError:
1692 # the node doesn't have a value for this property
1693 if isinstance(propclass, Multilink): av = []
1694 else: av = ''
1695 try:
1696 if not bn.has_key(prop):
1697 bn[prop] = cl.get(b_id, prop)
1698 bv = bn[prop]
1699 except KeyError:
1700 # the node doesn't have a value for this property
1701 if isinstance(propclass, Multilink): bv = []
1702 else: bv = ''
1704 # String and Date values are sorted in the natural way
1705 if isinstance(propclass, String):
1706 # clean up the strings
1707 if av and av[0] in string.uppercase:
1708 av = av.lower()
1709 if bv and bv[0] in string.uppercase:
1710 bv = bv.lower()
1711 if (isinstance(propclass, String) or
1712 isinstance(propclass, Date)):
1713 # it might be a string that's really an integer
1714 try:
1715 av = int(av)
1716 bv = int(bv)
1717 except:
1718 pass
1719 if dir == '+':
1720 r = cmp(av, bv)
1721 if r != 0: return r
1722 elif dir == '-':
1723 r = cmp(bv, av)
1724 if r != 0: return r
1726 # Link properties are sorted according to the value of
1727 # the "order" property on the linked nodes if it is
1728 # present; or otherwise on the key string of the linked
1729 # nodes; or finally on the node ids.
1730 elif isinstance(propclass, Link):
1731 link = db.classes[propclass.classname]
1732 if av is None and bv is not None: return -1
1733 if av is not None and bv is None: return 1
1734 if av is None and bv is None: continue
1735 if link.getprops().has_key('order'):
1736 if dir == '+':
1737 r = cmp(link.get(av, 'order'),
1738 link.get(bv, 'order'))
1739 if r != 0: return r
1740 elif dir == '-':
1741 r = cmp(link.get(bv, 'order'),
1742 link.get(av, 'order'))
1743 if r != 0: return r
1744 elif link.getkey():
1745 key = link.getkey()
1746 if dir == '+':
1747 r = cmp(link.get(av, key), link.get(bv, key))
1748 if r != 0: return r
1749 elif dir == '-':
1750 r = cmp(link.get(bv, key), link.get(av, key))
1751 if r != 0: return r
1752 else:
1753 if dir == '+':
1754 r = cmp(av, bv)
1755 if r != 0: return r
1756 elif dir == '-':
1757 r = cmp(bv, av)
1758 if r != 0: return r
1760 # Multilink properties are sorted according to how many
1761 # links are present.
1762 elif isinstance(propclass, Multilink):
1763 r = cmp(len(av), len(bv))
1764 if r == 0:
1765 # Compare contents of multilink property if lenghts is
1766 # equal
1767 r = cmp ('.'.join(av), '.'.join(bv))
1768 if dir == '+':
1769 return r
1770 elif dir == '-':
1771 return -r
1772 elif isinstance(propclass, Number) or isinstance(propclass, Boolean):
1773 if dir == '+':
1774 r = cmp(av, bv)
1775 elif dir == '-':
1776 r = cmp(bv, av)
1778 # end for dir, prop in sort, group:
1779 # if all else fails, compare the ids
1780 return cmp(a[0], b[0])
1782 l.sort(sortfun)
1783 return [i[0] for i in l]
1785 def count(self):
1786 '''Get the number of nodes in this class.
1788 If the returned integer is 'numnodes', the ids of all the nodes
1789 in this class run from 1 to numnodes, and numnodes+1 will be the
1790 id of the next node to be created in this class.
1791 '''
1792 return self.db.countnodes(self.classname)
1794 # Manipulating properties:
1796 def getprops(self, protected=1):
1797 '''Return a dictionary mapping property names to property objects.
1798 If the "protected" flag is true, we include protected properties -
1799 those which may not be modified.
1801 In addition to the actual properties on the node, these
1802 methods provide the "creation" and "activity" properties. If the
1803 "protected" flag is true, we include protected properties - those
1804 which may not be modified.
1805 '''
1806 d = self.properties.copy()
1807 if protected:
1808 d['id'] = String()
1809 d['creation'] = hyperdb.Date()
1810 d['activity'] = hyperdb.Date()
1811 d['creator'] = hyperdb.Link('user')
1812 return d
1814 def addprop(self, **properties):
1815 '''Add properties to this class.
1817 The keyword arguments in 'properties' must map names to property
1818 objects, or a TypeError is raised. None of the keys in 'properties'
1819 may collide with the names of existing properties, or a ValueError
1820 is raised before any properties have been added.
1821 '''
1822 for key in properties.keys():
1823 if self.properties.has_key(key):
1824 raise ValueError, key
1825 self.properties.update(properties)
1827 def index(self, nodeid):
1828 '''Add (or refresh) the node to search indexes
1829 '''
1830 # find all the String properties that have indexme
1831 for prop, propclass in self.getprops().items():
1832 if isinstance(propclass, String) and propclass.indexme:
1833 try:
1834 value = str(self.get(nodeid, prop))
1835 except IndexError:
1836 # node no longer exists - entry should be removed
1837 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1838 else:
1839 # and index them under (classname, nodeid, property)
1840 self.db.indexer.add_text((self.classname, nodeid, prop),
1841 value)
1843 #
1844 # Detector interface
1845 #
1846 def audit(self, event, detector):
1847 '''Register a detector
1848 '''
1849 l = self.auditors[event]
1850 if detector not in l:
1851 self.auditors[event].append(detector)
1853 def fireAuditors(self, action, nodeid, newvalues):
1854 '''Fire all registered auditors.
1855 '''
1856 for audit in self.auditors[action]:
1857 audit(self.db, self, nodeid, newvalues)
1859 def react(self, event, detector):
1860 '''Register a detector
1861 '''
1862 l = self.reactors[event]
1863 if detector not in l:
1864 self.reactors[event].append(detector)
1866 def fireReactors(self, action, nodeid, oldvalues):
1867 '''Fire all registered reactors.
1868 '''
1869 for react in self.reactors[action]:
1870 react(self.db, self, nodeid, oldvalues)
1872 class FileClass(Class):
1873 '''This class defines a large chunk of data. To support this, it has a
1874 mandatory String property "content" which is typically saved off
1875 externally to the hyperdb.
1877 The default MIME type of this data is defined by the
1878 "default_mime_type" class attribute, which may be overridden by each
1879 node if the class defines a "type" String property.
1880 '''
1881 default_mime_type = 'text/plain'
1883 def create(self, **propvalues):
1884 ''' snaffle the file propvalue and store in a file
1885 '''
1886 content = propvalues['content']
1887 del propvalues['content']
1888 newid = Class.create(self, **propvalues)
1889 self.db.storefile(self.classname, newid, None, content)
1890 return newid
1892 def import_list(self, propnames, proplist):
1893 ''' Trap the "content" property...
1894 '''
1895 # dupe this list so we don't affect others
1896 propnames = propnames[:]
1898 # extract the "content" property from the proplist
1899 i = propnames.index('content')
1900 content = eval(proplist[i])
1901 del propnames[i]
1902 del proplist[i]
1904 # do the normal import
1905 newid = Class.import_list(self, propnames, proplist)
1907 # save off the "content" file
1908 self.db.storefile(self.classname, newid, None, content)
1909 return newid
1911 def get(self, nodeid, propname, default=_marker, cache=1):
1912 ''' trap the content propname and get it from the file
1913 '''
1914 poss_msg = 'Possibly an access right configuration problem.'
1915 if propname == 'content':
1916 try:
1917 return self.db.getfile(self.classname, nodeid, None)
1918 except IOError, (strerror):
1919 # XXX by catching this we donot see an error in the log.
1920 return 'ERROR reading file: %s%s\n%s\n%s'%(
1921 self.classname, nodeid, poss_msg, strerror)
1922 if default is not _marker:
1923 return Class.get(self, nodeid, propname, default, cache=cache)
1924 else:
1925 return Class.get(self, nodeid, propname, cache=cache)
1927 def getprops(self, protected=1):
1928 ''' In addition to the actual properties on the node, these methods
1929 provide the "content" property. If the "protected" flag is true,
1930 we include protected properties - those which may not be
1931 modified.
1932 '''
1933 d = Class.getprops(self, protected=protected).copy()
1934 d['content'] = hyperdb.String()
1935 return d
1937 def index(self, nodeid):
1938 ''' Index the node in the search index.
1940 We want to index the content in addition to the normal String
1941 property indexing.
1942 '''
1943 # perform normal indexing
1944 Class.index(self, nodeid)
1946 # get the content to index
1947 content = self.get(nodeid, 'content')
1949 # figure the mime type
1950 if self.properties.has_key('type'):
1951 mime_type = self.get(nodeid, 'type')
1952 else:
1953 mime_type = self.default_mime_type
1955 # and index!
1956 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
1957 mime_type)
1959 # deviation from spec - was called ItemClass
1960 class IssueClass(Class, roundupdb.IssueClass):
1961 # Overridden methods:
1962 def __init__(self, db, classname, **properties):
1963 '''The newly-created class automatically includes the "messages",
1964 "files", "nosy", and "superseder" properties. If the 'properties'
1965 dictionary attempts to specify any of these properties or a
1966 "creation" or "activity" property, a ValueError is raised.
1967 '''
1968 if not properties.has_key('title'):
1969 properties['title'] = hyperdb.String(indexme='yes')
1970 if not properties.has_key('messages'):
1971 properties['messages'] = hyperdb.Multilink("msg")
1972 if not properties.has_key('files'):
1973 properties['files'] = hyperdb.Multilink("file")
1974 if not properties.has_key('nosy'):
1975 # note: journalling is turned off as it really just wastes
1976 # space. this behaviour may be overridden in an instance
1977 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
1978 if not properties.has_key('superseder'):
1979 properties['superseder'] = hyperdb.Multilink(classname)
1980 Class.__init__(self, db, classname, **properties)
1982 #