f36213d0ecab9f70b8d3ca937793c68405212493
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: back_anydbm.py,v 1.78 2002-09-13 08:20:07 richard Exp $
19 '''
20 This module defines a backend that saves the hyperdatabase in a database
21 chosen by anydbm. It is guaranteed to always be available in python
22 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
23 serious bugs, and is not available)
24 '''
26 import whichdb, anydbm, os, marshal, re, weakref, string, copy
27 from roundup import hyperdb, date, password, roundupdb, security
28 from blobfiles import FileStorage
29 from sessions import Sessions
30 from roundup.indexer import Indexer
31 from locking import acquire_lock, release_lock
32 from roundup.hyperdb import String, Password, Date, Interval, Link, \
33 Multilink, DatabaseError, Boolean, Number
35 #
36 # Now the database
37 #
38 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
39 '''A database for storing records containing flexible data types.
41 Transaction stuff TODO:
42 . check the timestamp of the class file and nuke the cache if it's
43 modified. Do some sort of conflict checking on the dirty stuff.
44 . perhaps detect write collisions (related to above)?
46 '''
47 def __init__(self, config, journaltag=None):
48 '''Open a hyperdatabase given a specifier to some storage.
50 The 'storagelocator' is obtained from config.DATABASE.
51 The meaning of 'storagelocator' depends on the particular
52 implementation of the hyperdatabase. It could be a file name,
53 a directory path, a socket descriptor for a connection to a
54 database over the network, etc.
56 The 'journaltag' is a token that will be attached to the journal
57 entries for any edits done on the database. If 'journaltag' is
58 None, the database is opened in read-only mode: the Class.create(),
59 Class.set(), and Class.retire() methods are disabled.
60 '''
61 self.config, self.journaltag = config, journaltag
62 self.dir = config.DATABASE
63 self.classes = {}
64 self.cache = {} # cache of nodes loaded or created
65 self.dirtynodes = {} # keep track of the dirty nodes by class
66 self.newnodes = {} # keep track of the new nodes by class
67 self.destroyednodes = {}# keep track of the destroyed nodes by class
68 self.transactions = []
69 self.indexer = Indexer(self.dir)
70 self.sessions = Sessions(self.config)
71 self.security = security.Security(self)
72 # ensure files are group readable and writable
73 os.umask(0002)
75 def post_init(self):
76 '''Called once the schema initialisation has finished.'''
77 # reindex the db if necessary
78 if self.indexer.should_reindex():
79 self.reindex()
81 def reindex(self):
82 for klass in self.classes.values():
83 for nodeid in klass.list():
84 klass.index(nodeid)
85 self.indexer.save_index()
87 def __repr__(self):
88 return '<back_anydbm instance at %x>'%id(self)
90 #
91 # Classes
92 #
93 def __getattr__(self, classname):
94 '''A convenient way of calling self.getclass(classname).'''
95 if self.classes.has_key(classname):
96 if __debug__:
97 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
98 return self.classes[classname]
99 raise AttributeError, classname
101 def addclass(self, cl):
102 if __debug__:
103 print >>hyperdb.DEBUG, 'addclass', (self, cl)
104 cn = cl.classname
105 if self.classes.has_key(cn):
106 raise ValueError, cn
107 self.classes[cn] = cl
109 def getclasses(self):
110 '''Return a list of the names of all existing classes.'''
111 if __debug__:
112 print >>hyperdb.DEBUG, 'getclasses', (self,)
113 l = self.classes.keys()
114 l.sort()
115 return l
117 def getclass(self, classname):
118 '''Get the Class object representing a particular class.
120 If 'classname' is not a valid class name, a KeyError is raised.
121 '''
122 if __debug__:
123 print >>hyperdb.DEBUG, 'getclass', (self, classname)
124 return self.classes[classname]
126 #
127 # Class DBs
128 #
129 def clear(self):
130 '''Delete all database contents
131 '''
132 if __debug__:
133 print >>hyperdb.DEBUG, 'clear', (self,)
134 for cn in self.classes.keys():
135 for dummy in 'nodes', 'journals':
136 path = os.path.join(self.dir, 'journals.%s'%cn)
137 if os.path.exists(path):
138 os.remove(path)
139 elif os.path.exists(path+'.db'): # dbm appends .db
140 os.remove(path+'.db')
142 def getclassdb(self, classname, mode='r'):
143 ''' grab a connection to the class db that will be used for
144 multiple actions
145 '''
146 if __debug__:
147 print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode)
148 return self.opendb('nodes.%s'%classname, mode)
150 def determine_db_type(self, path):
151 ''' determine which DB wrote the class file
152 '''
153 db_type = ''
154 if os.path.exists(path):
155 db_type = whichdb.whichdb(path)
156 if not db_type:
157 raise DatabaseError, "Couldn't identify database type"
158 elif os.path.exists(path+'.db'):
159 # if the path ends in '.db', it's a dbm database, whether
160 # anydbm says it's dbhash or not!
161 db_type = 'dbm'
162 return db_type
164 def opendb(self, name, mode):
165 '''Low-level database opener that gets around anydbm/dbm
166 eccentricities.
167 '''
168 if __debug__:
169 print >>hyperdb.DEBUG, 'opendb', (self, name, mode)
171 # figure the class db type
172 path = os.path.join(os.getcwd(), self.dir, name)
173 db_type = self.determine_db_type(path)
175 # new database? let anydbm pick the best dbm
176 if not db_type:
177 if __debug__:
178 print >>hyperdb.DEBUG, "opendb anydbm.open(%r, 'c')"%path
179 return anydbm.open(path, 'c')
181 # open the database with the correct module
182 try:
183 dbm = __import__(db_type)
184 except ImportError:
185 raise DatabaseError, \
186 "Couldn't open database - the required module '%s'"\
187 " is not available"%db_type
188 if __debug__:
189 print >>hyperdb.DEBUG, "opendb %r.open(%r, %r)"%(db_type, path,
190 mode)
191 return dbm.open(path, mode)
193 def lockdb(self, name):
194 ''' Lock a database file
195 '''
196 path = os.path.join(os.getcwd(), self.dir, '%s.lock'%name)
197 return acquire_lock(path)
199 #
200 # Node IDs
201 #
202 def newid(self, classname):
203 ''' Generate a new id for the given class
204 '''
205 # open the ids DB - create if if doesn't exist
206 lock = self.lockdb('_ids')
207 db = self.opendb('_ids', 'c')
208 if db.has_key(classname):
209 newid = db[classname] = str(int(db[classname]) + 1)
210 else:
211 # the count() bit is transitional - older dbs won't start at 1
212 newid = str(self.getclass(classname).count()+1)
213 db[classname] = newid
214 db.close()
215 release_lock(lock)
216 return newid
218 def setid(self, classname, setid):
219 ''' Set the id counter: used during import of database
220 '''
221 # open the ids DB - create if if doesn't exist
222 lock = self.lockdb('_ids')
223 db = self.opendb('_ids', 'c')
224 db[classname] = str(setid)
225 db.close()
226 release_lock(lock)
228 #
229 # Nodes
230 #
231 def addnode(self, classname, nodeid, node):
232 ''' add the specified node to its class's db
233 '''
234 if __debug__:
235 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
236 self.newnodes.setdefault(classname, {})[nodeid] = 1
237 self.cache.setdefault(classname, {})[nodeid] = node
238 self.savenode(classname, nodeid, node)
240 def setnode(self, classname, nodeid, node):
241 ''' change the specified node
242 '''
243 if __debug__:
244 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
245 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
247 # can't set without having already loaded the node
248 self.cache[classname][nodeid] = node
249 self.savenode(classname, nodeid, node)
251 def savenode(self, classname, nodeid, node):
252 ''' perform the saving of data specified by the set/addnode
253 '''
254 if __debug__:
255 print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node)
256 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
258 def getnode(self, classname, nodeid, db=None, cache=1):
259 ''' get a node from the database
260 '''
261 if __debug__:
262 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid, db)
263 if cache:
264 # try the cache
265 cache_dict = self.cache.setdefault(classname, {})
266 if cache_dict.has_key(nodeid):
267 if __debug__:
268 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
269 nodeid)
270 return cache_dict[nodeid]
272 if __debug__:
273 print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid)
275 # get from the database and save in the cache
276 if db is None:
277 db = self.getclassdb(classname)
278 if not db.has_key(nodeid):
279 raise IndexError, "no such %s %s"%(classname, nodeid)
281 # check the uncommitted, destroyed nodes
282 if (self.destroyednodes.has_key(classname) and
283 self.destroyednodes[classname].has_key(nodeid)):
284 raise IndexError, "no such %s %s"%(classname, nodeid)
286 # decode
287 res = marshal.loads(db[nodeid])
289 # reverse the serialisation
290 res = self.unserialise(classname, res)
292 # store off in the cache dict
293 if cache:
294 cache_dict[nodeid] = res
296 return res
298 def destroynode(self, classname, nodeid):
299 '''Remove a node from the database. Called exclusively by the
300 destroy() method on Class.
301 '''
302 if __debug__:
303 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
305 # remove from cache and newnodes if it's there
306 if (self.cache.has_key(classname) and
307 self.cache[classname].has_key(nodeid)):
308 del self.cache[classname][nodeid]
309 if (self.newnodes.has_key(classname) and
310 self.newnodes[classname].has_key(nodeid)):
311 del self.newnodes[classname][nodeid]
313 # see if there's any obvious commit actions that we should get rid of
314 for entry in self.transactions[:]:
315 if entry[1][:2] == (classname, nodeid):
316 self.transactions.remove(entry)
318 # add to the destroyednodes map
319 self.destroyednodes.setdefault(classname, {})[nodeid] = 1
321 # add the destroy commit action
322 self.transactions.append((self.doDestroyNode, (classname, nodeid)))
324 def serialise(self, classname, node):
325 '''Copy the node contents, converting non-marshallable data into
326 marshallable data.
327 '''
328 if __debug__:
329 print >>hyperdb.DEBUG, 'serialise', classname, node
330 properties = self.getclass(classname).getprops()
331 d = {}
332 for k, v in node.items():
333 # if the property doesn't exist, or is the "retired" flag then
334 # it won't be in the properties dict
335 if not properties.has_key(k):
336 d[k] = v
337 continue
339 # get the property spec
340 prop = properties[k]
342 if isinstance(prop, Password):
343 d[k] = str(v)
344 elif isinstance(prop, Date) and v is not None:
345 d[k] = v.serialise()
346 elif isinstance(prop, Interval) and v is not None:
347 d[k] = v.serialise()
348 else:
349 d[k] = v
350 return d
352 def unserialise(self, classname, node):
353 '''Decode the marshalled node data
354 '''
355 if __debug__:
356 print >>hyperdb.DEBUG, 'unserialise', classname, node
357 properties = self.getclass(classname).getprops()
358 d = {}
359 for k, v in node.items():
360 # if the property doesn't exist, or is the "retired" flag then
361 # it won't be in the properties dict
362 if not properties.has_key(k):
363 d[k] = v
364 continue
366 # get the property spec
367 prop = properties[k]
369 if isinstance(prop, Date) and v is not None:
370 d[k] = date.Date(v)
371 elif isinstance(prop, Interval) and v is not None:
372 d[k] = date.Interval(v)
373 elif isinstance(prop, Password):
374 p = password.Password()
375 p.unpack(v)
376 d[k] = p
377 else:
378 d[k] = v
379 return d
381 def hasnode(self, classname, nodeid, db=None):
382 ''' determine if the database has a given node
383 '''
384 if __debug__:
385 print >>hyperdb.DEBUG, 'hasnode', (self, classname, nodeid, db)
387 # try the cache
388 cache = self.cache.setdefault(classname, {})
389 if cache.has_key(nodeid):
390 if __debug__:
391 print >>hyperdb.TRACE, 'has %s %s cached'%(classname, nodeid)
392 return 1
393 if __debug__:
394 print >>hyperdb.TRACE, 'has %s %s'%(classname, nodeid)
396 # not in the cache - check the database
397 if db is None:
398 db = self.getclassdb(classname)
399 res = db.has_key(nodeid)
400 return res
402 def countnodes(self, classname, db=None):
403 if __debug__:
404 print >>hyperdb.DEBUG, 'countnodes', (self, classname, db)
406 count = 0
408 # include the uncommitted nodes
409 if self.newnodes.has_key(classname):
410 count += len(self.newnodes[classname])
411 if self.destroyednodes.has_key(classname):
412 count -= len(self.destroyednodes[classname])
414 # and count those in the DB
415 if db is None:
416 db = self.getclassdb(classname)
417 count = count + len(db.keys())
418 return count
420 def getnodeids(self, classname, db=None):
421 if __debug__:
422 print >>hyperdb.DEBUG, 'getnodeids', (self, classname, db)
424 res = []
426 # start off with the new nodes
427 if self.newnodes.has_key(classname):
428 res += self.newnodes[classname].keys()
430 if db is None:
431 db = self.getclassdb(classname)
432 res = res + db.keys()
434 # remove the uncommitted, destroyed nodes
435 if self.destroyednodes.has_key(classname):
436 for nodeid in self.destroyednodes[classname].keys():
437 if db.has_key(nodeid):
438 res.remove(nodeid)
440 return res
443 #
444 # Files - special node properties
445 # inherited from FileStorage
447 #
448 # Journal
449 #
450 def addjournal(self, classname, nodeid, action, params, creator=None,
451 creation=None):
452 ''' Journal the Action
453 'action' may be:
455 'create' or 'set' -- 'params' is a dictionary of property values
456 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
457 'retire' -- 'params' is None
458 '''
459 if __debug__:
460 print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid,
461 action, params, creator, creation)
462 self.transactions.append((self.doSaveJournal, (classname, nodeid,
463 action, params, creator, creation)))
465 def getjournal(self, classname, nodeid):
466 ''' get the journal for id
468 Raise IndexError if the node doesn't exist (as per history()'s
469 API)
470 '''
471 if __debug__:
472 print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid)
473 # attempt to open the journal - in some rare cases, the journal may
474 # not exist
475 try:
476 db = self.opendb('journals.%s'%classname, 'r')
477 except anydbm.error, error:
478 if str(error) == "need 'c' or 'n' flag to open new db":
479 raise IndexError, 'no such %s %s'%(classname, nodeid)
480 elif error.args[0] != 2:
481 raise
482 raise IndexError, 'no such %s %s'%(classname, nodeid)
483 try:
484 journal = marshal.loads(db[nodeid])
485 except KeyError:
486 db.close()
487 raise IndexError, 'no such %s %s'%(classname, nodeid)
488 db.close()
489 res = []
490 for nodeid, date_stamp, user, action, params in journal:
491 res.append((nodeid, date.Date(date_stamp), user, action, params))
492 return res
494 def pack(self, pack_before):
495 ''' Delete all journal entries except "create" before 'pack_before'.
496 '''
497 if __debug__:
498 print >>hyperdb.DEBUG, 'packjournal', (self, pack_before)
500 for classname in self.getclasses():
501 # get the journal db
502 db_name = 'journals.%s'%classname
503 path = os.path.join(os.getcwd(), self.dir, classname)
504 db_type = self.determine_db_type(path)
505 db = self.opendb(db_name, 'w')
507 for key in db.keys():
508 # get the journal for this db entry
509 journal = marshal.loads(db[key])
510 l = []
511 last_set_entry = None
512 for entry in journal:
513 # unpack the entry
514 (nodeid, date_stamp, self.journaltag, action,
515 params) = entry
516 date_stamp = date.Date(date_stamp)
517 # if the entry is after the pack date, _or_ the initial
518 # create entry, then it stays
519 if date_stamp > pack_before or action == 'create':
520 l.append(entry)
521 elif action == 'set':
522 # grab the last set entry to keep information on
523 # activity
524 last_set_entry = entry
525 if last_set_entry:
526 date_stamp = last_set_entry[1]
527 # if the last set entry was made after the pack date
528 # then it is already in the list
529 if date_stamp < pack_before:
530 l.append(last_set_entry)
531 db[key] = marshal.dumps(l)
532 if db_type == 'gdbm':
533 db.reorganize()
534 db.close()
537 #
538 # Basic transaction support
539 #
540 def commit(self):
541 ''' Commit the current transactions.
542 '''
543 if __debug__:
544 print >>hyperdb.DEBUG, 'commit', (self,)
545 # TODO: lock the DB
547 # keep a handle to all the database files opened
548 self.databases = {}
550 # now, do all the transactions
551 reindex = {}
552 for method, args in self.transactions:
553 reindex[method(*args)] = 1
555 # now close all the database files
556 for db in self.databases.values():
557 db.close()
558 del self.databases
559 # TODO: unlock the DB
561 # reindex the nodes that request it
562 for classname, nodeid in filter(None, reindex.keys()):
563 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
564 self.getclass(classname).index(nodeid)
566 # save the indexer state
567 self.indexer.save_index()
569 # all transactions committed, back to normal
570 self.cache = {}
571 self.dirtynodes = {}
572 self.newnodes = {}
573 self.destroyednodes = {}
574 self.transactions = []
576 def getCachedClassDB(self, classname):
577 ''' get the class db, looking in our cache of databases for commit
578 '''
579 # get the database handle
580 db_name = 'nodes.%s'%classname
581 if not self.databases.has_key(db_name):
582 self.databases[db_name] = self.getclassdb(classname, 'c')
583 return self.databases[db_name]
585 def doSaveNode(self, classname, nodeid, node):
586 if __debug__:
587 print >>hyperdb.DEBUG, 'doSaveNode', (self, classname, nodeid,
588 node)
590 db = self.getCachedClassDB(classname)
592 # now save the marshalled data
593 db[nodeid] = marshal.dumps(self.serialise(classname, node))
595 # return the classname, nodeid so we reindex this content
596 return (classname, nodeid)
598 def getCachedJournalDB(self, classname):
599 ''' get the journal db, looking in our cache of databases for commit
600 '''
601 # get the database handle
602 db_name = 'journals.%s'%classname
603 if not self.databases.has_key(db_name):
604 self.databases[db_name] = self.opendb(db_name, 'c')
605 return self.databases[db_name]
607 def doSaveJournal(self, classname, nodeid, action, params, creator,
608 creation):
609 # serialise the parameters now if necessary
610 if isinstance(params, type({})):
611 if action in ('set', 'create'):
612 params = self.serialise(classname, params)
614 # handle supply of the special journalling parameters (usually
615 # supplied on importing an existing database)
616 if creator:
617 journaltag = creator
618 else:
619 journaltag = self.journaltag
620 if creation:
621 journaldate = creation.serialise()
622 else:
623 journaldate = date.Date().serialise()
625 # create the journal entry
626 entry = (nodeid, journaldate, journaltag, action, params)
628 if __debug__:
629 print >>hyperdb.DEBUG, 'doSaveJournal', entry
631 db = self.getCachedJournalDB(classname)
633 # now insert the journal entry
634 if db.has_key(nodeid):
635 # append to existing
636 s = db[nodeid]
637 l = marshal.loads(s)
638 l.append(entry)
639 else:
640 l = [entry]
642 db[nodeid] = marshal.dumps(l)
644 def doDestroyNode(self, classname, nodeid):
645 if __debug__:
646 print >>hyperdb.DEBUG, 'doDestroyNode', (self, classname, nodeid)
648 # delete from the class database
649 db = self.getCachedClassDB(classname)
650 if db.has_key(nodeid):
651 del db[nodeid]
653 # delete from the database
654 db = self.getCachedJournalDB(classname)
655 if db.has_key(nodeid):
656 del db[nodeid]
658 # return the classname, nodeid so we reindex this content
659 return (classname, nodeid)
661 def rollback(self):
662 ''' Reverse all actions from the current transaction.
663 '''
664 if __debug__:
665 print >>hyperdb.DEBUG, 'rollback', (self, )
666 for method, args in self.transactions:
667 # delete temporary files
668 if method == self.doStoreFile:
669 self.rollbackStoreFile(*args)
670 self.cache = {}
671 self.dirtynodes = {}
672 self.newnodes = {}
673 self.destroyednodes = {}
674 self.transactions = []
676 def close(self):
677 ''' Nothing to do
678 '''
679 pass
681 _marker = []
682 class Class(hyperdb.Class):
683 '''The handle to a particular class of nodes in a hyperdatabase.'''
685 def __init__(self, db, classname, **properties):
686 '''Create a new class with a given name and property specification.
688 'classname' must not collide with the name of an existing class,
689 or a ValueError is raised. The keyword arguments in 'properties'
690 must map names to property objects, or a TypeError is raised.
691 '''
692 if (properties.has_key('creation') or properties.has_key('activity')
693 or properties.has_key('creator')):
694 raise ValueError, '"creation", "activity" and "creator" are '\
695 'reserved'
697 self.classname = classname
698 self.properties = properties
699 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
700 self.key = ''
702 # should we journal changes (default yes)
703 self.do_journal = 1
705 # do the db-related init stuff
706 db.addclass(self)
708 self.auditors = {'create': [], 'set': [], 'retire': []}
709 self.reactors = {'create': [], 'set': [], 'retire': []}
711 def enableJournalling(self):
712 '''Turn journalling on for this class
713 '''
714 self.do_journal = 1
716 def disableJournalling(self):
717 '''Turn journalling off for this class
718 '''
719 self.do_journal = 0
721 # Editing nodes:
723 def create(self, **propvalues):
724 '''Create a new node of this class and return its id.
726 The keyword arguments in 'propvalues' map property names to values.
728 The values of arguments must be acceptable for the types of their
729 corresponding properties or a TypeError is raised.
731 If this class has a key property, it must be present and its value
732 must not collide with other key strings or a ValueError is raised.
734 Any other properties on this class that are missing from the
735 'propvalues' dictionary are set to None.
737 If an id in a link or multilink property does not refer to a valid
738 node, an IndexError is raised.
740 These operations trigger detectors and can be vetoed. Attempts
741 to modify the "creation" or "activity" properties cause a KeyError.
742 '''
743 if propvalues.has_key('id'):
744 raise KeyError, '"id" is reserved'
746 if self.db.journaltag is None:
747 raise DatabaseError, 'Database open read-only'
749 if propvalues.has_key('creation') or propvalues.has_key('activity'):
750 raise KeyError, '"creation" and "activity" are reserved'
752 self.fireAuditors('create', None, propvalues)
754 # new node's id
755 newid = self.db.newid(self.classname)
757 # validate propvalues
758 num_re = re.compile('^\d+$')
759 for key, value in propvalues.items():
760 if key == self.key:
761 try:
762 self.lookup(value)
763 except KeyError:
764 pass
765 else:
766 raise ValueError, 'node with key "%s" exists'%value
768 # try to handle this property
769 try:
770 prop = self.properties[key]
771 except KeyError:
772 raise KeyError, '"%s" has no property "%s"'%(self.classname,
773 key)
775 if value is not None and isinstance(prop, Link):
776 if type(value) != type(''):
777 raise ValueError, 'link value must be String'
778 link_class = self.properties[key].classname
779 # if it isn't a number, it's a key
780 if not num_re.match(value):
781 try:
782 value = self.db.classes[link_class].lookup(value)
783 except (TypeError, KeyError):
784 raise IndexError, 'new property "%s": %s not a %s'%(
785 key, value, link_class)
786 elif not self.db.getclass(link_class).hasnode(value):
787 raise IndexError, '%s has no node %s'%(link_class, value)
789 # save off the value
790 propvalues[key] = value
792 # register the link with the newly linked node
793 if self.do_journal and self.properties[key].do_journal:
794 self.db.addjournal(link_class, value, 'link',
795 (self.classname, newid, key))
797 elif isinstance(prop, Multilink):
798 if type(value) != type([]):
799 raise TypeError, 'new property "%s" not a list of ids'%key
801 # clean up and validate the list of links
802 link_class = self.properties[key].classname
803 l = []
804 for entry in value:
805 if type(entry) != type(''):
806 raise ValueError, '"%s" multilink value (%r) '\
807 'must contain Strings'%(key, value)
808 # if it isn't a number, it's a key
809 if not num_re.match(entry):
810 try:
811 entry = self.db.classes[link_class].lookup(entry)
812 except (TypeError, KeyError):
813 raise IndexError, 'new property "%s": %s not a %s'%(
814 key, entry, self.properties[key].classname)
815 l.append(entry)
816 value = l
817 propvalues[key] = value
819 # handle additions
820 for nodeid in value:
821 if not self.db.getclass(link_class).hasnode(nodeid):
822 raise IndexError, '%s has no node %s'%(link_class,
823 nodeid)
824 # register the link with the newly linked node
825 if self.do_journal and self.properties[key].do_journal:
826 self.db.addjournal(link_class, nodeid, 'link',
827 (self.classname, newid, key))
829 elif isinstance(prop, String):
830 if type(value) != type(''):
831 raise TypeError, 'new property "%s" not a string'%key
833 elif isinstance(prop, Password):
834 if not isinstance(value, password.Password):
835 raise TypeError, 'new property "%s" not a Password'%key
837 elif isinstance(prop, Date):
838 if value is not None and not isinstance(value, date.Date):
839 raise TypeError, 'new property "%s" not a Date'%key
841 elif isinstance(prop, Interval):
842 if value is not None and not isinstance(value, date.Interval):
843 raise TypeError, 'new property "%s" not an Interval'%key
845 elif value is not None and isinstance(prop, Number):
846 try:
847 float(value)
848 except ValueError:
849 raise TypeError, 'new property "%s" not numeric'%key
851 elif value is not None and isinstance(prop, Boolean):
852 try:
853 int(value)
854 except ValueError:
855 raise TypeError, 'new property "%s" not boolean'%key
857 # make sure there's data where there needs to be
858 for key, prop in self.properties.items():
859 if propvalues.has_key(key):
860 continue
861 if key == self.key:
862 raise ValueError, 'key property "%s" is required'%key
863 if isinstance(prop, Multilink):
864 propvalues[key] = []
865 else:
866 propvalues[key] = None
868 # done
869 self.db.addnode(self.classname, newid, propvalues)
870 if self.do_journal:
871 self.db.addjournal(self.classname, newid, 'create', propvalues)
873 self.fireReactors('create', newid, None)
875 return newid
877 def export_list(self, propnames, nodeid):
878 ''' Export a node - generate a list of CSV-able data in the order
879 specified by propnames for the given node.
880 '''
881 properties = self.getprops()
882 l = []
883 for prop in propnames:
884 proptype = properties[prop]
885 value = self.get(nodeid, prop)
886 # "marshal" data where needed
887 if value is None:
888 pass
889 elif isinstance(proptype, hyperdb.Date):
890 value = value.get_tuple()
891 elif isinstance(proptype, hyperdb.Interval):
892 value = value.get_tuple()
893 elif isinstance(proptype, hyperdb.Password):
894 value = str(value)
895 l.append(repr(value))
896 return l
898 def import_list(self, propnames, proplist):
899 ''' Import a node - all information including "id" is present and
900 should not be sanity checked. Triggers are not triggered. The
901 journal should be initialised using the "creator" and "created"
902 information.
904 Return the nodeid of the node imported.
905 '''
906 if self.db.journaltag is None:
907 raise DatabaseError, 'Database open read-only'
908 properties = self.getprops()
910 # make the new node's property map
911 d = {}
912 for i in range(len(propnames)):
913 # Use eval to reverse the repr() used to output the CSV
914 value = eval(proplist[i])
916 # Figure the property for this column
917 propname = propnames[i]
918 prop = properties[propname]
920 # "unmarshal" where necessary
921 if propname == 'id':
922 newid = value
923 continue
924 elif value is None:
925 # don't set Nones
926 continue
927 elif isinstance(prop, hyperdb.Date):
928 value = date.Date(value)
929 elif isinstance(prop, hyperdb.Interval):
930 value = date.Interval(value)
931 elif isinstance(prop, hyperdb.Password):
932 pwd = password.Password()
933 pwd.unpack(value)
934 value = pwd
935 d[propname] = value
937 # extract the extraneous journalling gumpf and nuke it
938 if d.has_key('creator'):
939 creator = d['creator']
940 del d['creator']
941 else:
942 creator = None
943 if d.has_key('creation'):
944 creation = d['creation']
945 del d['creation']
946 else:
947 creation = None
948 if d.has_key('activity'):
949 del d['activity']
951 # add the node and journal
952 self.db.addnode(self.classname, newid, d)
953 self.db.addjournal(self.classname, newid, 'create', d, creator,
954 creation)
955 return newid
957 def get(self, nodeid, propname, default=_marker, cache=1):
958 '''Get the value of a property on an existing node of this class.
960 'nodeid' must be the id of an existing node of this class or an
961 IndexError is raised. 'propname' must be the name of a property
962 of this class or a KeyError is raised.
964 'cache' indicates whether the transaction cache should be queried
965 for the node. If the node has been modified and you need to
966 determine what its values prior to modification are, you need to
967 set cache=0.
969 Attempts to get the "creation" or "activity" properties should
970 do the right thing.
971 '''
972 if propname == 'id':
973 return nodeid
975 if propname == 'creation':
976 if not self.do_journal:
977 raise ValueError, 'Journalling is disabled for this class'
978 journal = self.db.getjournal(self.classname, nodeid)
979 if journal:
980 return self.db.getjournal(self.classname, nodeid)[0][1]
981 else:
982 # on the strange chance that there's no journal
983 return date.Date()
984 if propname == 'activity':
985 if not self.do_journal:
986 raise ValueError, 'Journalling is disabled for this class'
987 journal = self.db.getjournal(self.classname, nodeid)
988 if journal:
989 return self.db.getjournal(self.classname, nodeid)[-1][1]
990 else:
991 # on the strange chance that there's no journal
992 return date.Date()
993 if propname == 'creator':
994 if not self.do_journal:
995 raise ValueError, 'Journalling is disabled for this class'
996 journal = self.db.getjournal(self.classname, nodeid)
997 if journal:
998 return self.db.getjournal(self.classname, nodeid)[0][2]
999 else:
1000 return self.db.journaltag
1002 # get the property (raises KeyErorr if invalid)
1003 prop = self.properties[propname]
1005 # get the node's dict
1006 d = self.db.getnode(self.classname, nodeid, cache=cache)
1008 if not d.has_key(propname):
1009 if default is _marker:
1010 if isinstance(prop, Multilink):
1011 return []
1012 else:
1013 return None
1014 else:
1015 return default
1017 # return a dupe of the list so code doesn't get confused
1018 if isinstance(prop, Multilink):
1019 return d[propname][:]
1021 return d[propname]
1023 # not in spec
1024 def getnode(self, nodeid, cache=1):
1025 ''' Return a convenience wrapper for the node.
1027 'nodeid' must be the id of an existing node of this class or an
1028 IndexError is raised.
1030 'cache' indicates whether the transaction cache should be queried
1031 for the node. If the node has been modified and you need to
1032 determine what its values prior to modification are, you need to
1033 set cache=0.
1034 '''
1035 return Node(self, nodeid, cache=cache)
1037 def set(self, nodeid, **propvalues):
1038 '''Modify a property on an existing node of this class.
1040 'nodeid' must be the id of an existing node of this class or an
1041 IndexError is raised.
1043 Each key in 'propvalues' must be the name of a property of this
1044 class or a KeyError is raised.
1046 All values in 'propvalues' must be acceptable types for their
1047 corresponding properties or a TypeError is raised.
1049 If the value of the key property is set, it must not collide with
1050 other key strings or a ValueError is raised.
1052 If the value of a Link or Multilink property contains an invalid
1053 node id, a ValueError is raised.
1055 These operations trigger detectors and can be vetoed. Attempts
1056 to modify the "creation" or "activity" properties cause a KeyError.
1057 '''
1058 if not propvalues:
1059 return propvalues
1061 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1062 raise KeyError, '"creation" and "activity" are reserved'
1064 if propvalues.has_key('id'):
1065 raise KeyError, '"id" is reserved'
1067 if self.db.journaltag is None:
1068 raise DatabaseError, 'Database open read-only'
1070 self.fireAuditors('set', nodeid, propvalues)
1071 # Take a copy of the node dict so that the subsequent set
1072 # operation doesn't modify the oldvalues structure.
1073 try:
1074 # try not using the cache initially
1075 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid,
1076 cache=0))
1077 except IndexError:
1078 # this will be needed if somone does a create() and set()
1079 # with no intervening commit()
1080 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1082 node = self.db.getnode(self.classname, nodeid)
1083 if node.has_key(self.db.RETIRED_FLAG):
1084 raise IndexError
1085 num_re = re.compile('^\d+$')
1087 # if the journal value is to be different, store it in here
1088 journalvalues = {}
1090 for propname, value in propvalues.items():
1091 # check to make sure we're not duplicating an existing key
1092 if propname == self.key and node[propname] != value:
1093 try:
1094 self.lookup(value)
1095 except KeyError:
1096 pass
1097 else:
1098 raise ValueError, 'node with key "%s" exists'%value
1100 # this will raise the KeyError if the property isn't valid
1101 # ... we don't use getprops() here because we only care about
1102 # the writeable properties.
1103 prop = self.properties[propname]
1105 # if the value's the same as the existing value, no sense in
1106 # doing anything
1107 if node.has_key(propname) and value == node[propname]:
1108 del propvalues[propname]
1109 continue
1111 # do stuff based on the prop type
1112 if isinstance(prop, Link):
1113 link_class = prop.classname
1114 # if it isn't a number, it's a key
1115 if value is not None and not isinstance(value, type('')):
1116 raise ValueError, 'property "%s" link value be a string'%(
1117 propname)
1118 if isinstance(value, type('')) and not num_re.match(value):
1119 try:
1120 value = self.db.classes[link_class].lookup(value)
1121 except (TypeError, KeyError):
1122 raise IndexError, 'new property "%s": %s not a %s'%(
1123 propname, value, prop.classname)
1125 if (value is not None and
1126 not self.db.getclass(link_class).hasnode(value)):
1127 raise IndexError, '%s has no node %s'%(link_class, value)
1129 if self.do_journal and prop.do_journal:
1130 # register the unlink with the old linked node
1131 if node[propname] is not None:
1132 self.db.addjournal(link_class, node[propname], 'unlink',
1133 (self.classname, nodeid, propname))
1135 # register the link with the newly linked node
1136 if value is not None:
1137 self.db.addjournal(link_class, value, 'link',
1138 (self.classname, nodeid, propname))
1140 elif isinstance(prop, Multilink):
1141 if type(value) != type([]):
1142 raise TypeError, 'new property "%s" not a list of'\
1143 ' ids'%propname
1144 link_class = self.properties[propname].classname
1145 l = []
1146 for entry in value:
1147 # if it isn't a number, it's a key
1148 if type(entry) != type(''):
1149 raise ValueError, 'new property "%s" link value ' \
1150 'must be a string'%propname
1151 if not num_re.match(entry):
1152 try:
1153 entry = self.db.classes[link_class].lookup(entry)
1154 except (TypeError, KeyError):
1155 raise IndexError, 'new property "%s": %s not a %s'%(
1156 propname, entry,
1157 self.properties[propname].classname)
1158 l.append(entry)
1159 value = l
1160 propvalues[propname] = value
1162 # figure the journal entry for this property
1163 add = []
1164 remove = []
1166 # handle removals
1167 if node.has_key(propname):
1168 l = node[propname]
1169 else:
1170 l = []
1171 for id in l[:]:
1172 if id in value:
1173 continue
1174 # register the unlink with the old linked node
1175 if self.do_journal and self.properties[propname].do_journal:
1176 self.db.addjournal(link_class, id, 'unlink',
1177 (self.classname, nodeid, propname))
1178 l.remove(id)
1179 remove.append(id)
1181 # handle additions
1182 for id in value:
1183 if not self.db.getclass(link_class).hasnode(id):
1184 raise IndexError, '%s has no node %s'%(link_class, id)
1185 if id in l:
1186 continue
1187 # register the link with the newly linked node
1188 if self.do_journal and self.properties[propname].do_journal:
1189 self.db.addjournal(link_class, id, 'link',
1190 (self.classname, nodeid, propname))
1191 l.append(id)
1192 add.append(id)
1194 # figure the journal entry
1195 l = []
1196 if add:
1197 l.append(('+', add))
1198 if remove:
1199 l.append(('-', remove))
1200 if l:
1201 journalvalues[propname] = tuple(l)
1203 elif isinstance(prop, String):
1204 if value is not None and type(value) != type(''):
1205 raise TypeError, 'new property "%s" not a string'%propname
1207 elif isinstance(prop, Password):
1208 if not isinstance(value, password.Password):
1209 raise TypeError, 'new property "%s" not a Password'%propname
1210 propvalues[propname] = value
1212 elif value is not None and isinstance(prop, Date):
1213 if not isinstance(value, date.Date):
1214 raise TypeError, 'new property "%s" not a Date'% propname
1215 propvalues[propname] = value
1217 elif value is not None and isinstance(prop, Interval):
1218 if not isinstance(value, date.Interval):
1219 raise TypeError, 'new property "%s" not an '\
1220 'Interval'%propname
1221 propvalues[propname] = value
1223 elif value is not None and isinstance(prop, Number):
1224 try:
1225 float(value)
1226 except ValueError:
1227 raise TypeError, 'new property "%s" not numeric'%propname
1229 elif value is not None and isinstance(prop, Boolean):
1230 try:
1231 int(value)
1232 except ValueError:
1233 raise TypeError, 'new property "%s" not boolean'%propname
1235 node[propname] = value
1237 # nothing to do?
1238 if not propvalues:
1239 return propvalues
1241 # do the set, and journal it
1242 self.db.setnode(self.classname, nodeid, node)
1244 if self.do_journal:
1245 propvalues.update(journalvalues)
1246 self.db.addjournal(self.classname, nodeid, 'set', propvalues)
1248 self.fireReactors('set', nodeid, oldvalues)
1250 return propvalues
1252 def retire(self, nodeid):
1253 '''Retire a node.
1255 The properties on the node remain available from the get() method,
1256 and the node's id is never reused.
1258 Retired nodes are not returned by the find(), list(), or lookup()
1259 methods, and other nodes may reuse the values of their key properties.
1261 These operations trigger detectors and can be vetoed. Attempts
1262 to modify the "creation" or "activity" properties cause a KeyError.
1263 '''
1264 if self.db.journaltag is None:
1265 raise DatabaseError, 'Database open read-only'
1267 self.fireAuditors('retire', nodeid, None)
1269 node = self.db.getnode(self.classname, nodeid)
1270 node[self.db.RETIRED_FLAG] = 1
1271 self.db.setnode(self.classname, nodeid, node)
1272 if self.do_journal:
1273 self.db.addjournal(self.classname, nodeid, 'retired', None)
1275 self.fireReactors('retire', nodeid, None)
1277 def is_retired(self, nodeid):
1278 '''Return true if the node is retired.
1279 '''
1280 node = self.db.getnode(cn, nodeid, cldb)
1281 if node.has_key(self.db.RETIRED_FLAG):
1282 return 1
1283 return 0
1285 def destroy(self, nodeid):
1286 '''Destroy a node.
1288 WARNING: this method should never be used except in extremely rare
1289 situations where there could never be links to the node being
1290 deleted
1291 WARNING: use retire() instead
1292 WARNING: the properties of this node will not be available ever again
1293 WARNING: really, use retire() instead
1295 Well, I think that's enough warnings. This method exists mostly to
1296 support the session storage of the cgi interface.
1297 '''
1298 if self.db.journaltag is None:
1299 raise DatabaseError, 'Database open read-only'
1300 self.db.destroynode(self.classname, nodeid)
1302 def history(self, nodeid):
1303 '''Retrieve the journal of edits on a particular node.
1305 'nodeid' must be the id of an existing node of this class or an
1306 IndexError is raised.
1308 The returned list contains tuples of the form
1310 (date, tag, action, params)
1312 'date' is a Timestamp object specifying the time of the change and
1313 'tag' is the journaltag specified when the database was opened.
1314 '''
1315 if not self.do_journal:
1316 raise ValueError, 'Journalling is disabled for this class'
1317 return self.db.getjournal(self.classname, nodeid)
1319 # Locating nodes:
1320 def hasnode(self, nodeid):
1321 '''Determine if the given nodeid actually exists
1322 '''
1323 return self.db.hasnode(self.classname, nodeid)
1325 def setkey(self, propname):
1326 '''Select a String property of this class to be the key property.
1328 'propname' must be the name of a String property of this class or
1329 None, or a TypeError is raised. The values of the key property on
1330 all existing nodes must be unique or a ValueError is raised. If the
1331 property doesn't exist, KeyError is raised.
1332 '''
1333 prop = self.getprops()[propname]
1334 if not isinstance(prop, String):
1335 raise TypeError, 'key properties must be String'
1336 self.key = propname
1338 def getkey(self):
1339 '''Return the name of the key property for this class or None.'''
1340 return self.key
1342 def labelprop(self, default_to_id=0):
1343 ''' Return the property name for a label for the given node.
1345 This method attempts to generate a consistent label for the node.
1346 It tries the following in order:
1347 1. key property
1348 2. "name" property
1349 3. "title" property
1350 4. first property from the sorted property name list
1351 '''
1352 k = self.getkey()
1353 if k:
1354 return k
1355 props = self.getprops()
1356 if props.has_key('name'):
1357 return 'name'
1358 elif props.has_key('title'):
1359 return 'title'
1360 if default_to_id:
1361 return 'id'
1362 props = props.keys()
1363 props.sort()
1364 return props[0]
1366 # TODO: set up a separate index db file for this? profile?
1367 def lookup(self, keyvalue):
1368 '''Locate a particular node by its key property and return its id.
1370 If this class has no key property, a TypeError is raised. If the
1371 'keyvalue' matches one of the values for the key property among
1372 the nodes in this class, the matching node's id is returned;
1373 otherwise a KeyError is raised.
1374 '''
1375 if not self.key:
1376 raise TypeError, 'No key property set for class %s'%self.classname
1377 cldb = self.db.getclassdb(self.classname)
1378 try:
1379 for nodeid in self.db.getnodeids(self.classname, cldb):
1380 node = self.db.getnode(self.classname, nodeid, cldb)
1381 if node.has_key(self.db.RETIRED_FLAG):
1382 continue
1383 if node[self.key] == keyvalue:
1384 cldb.close()
1385 return nodeid
1386 finally:
1387 cldb.close()
1388 raise KeyError, keyvalue
1390 # change from spec - allows multiple props to match
1391 def find(self, **propspec):
1392 '''Get the ids of nodes in this class which link to the given nodes.
1394 'propspec' consists of keyword args propname={nodeid:1,}
1395 'propname' must be the name of a property in this class, or a
1396 KeyError is raised. That property must be a Link or Multilink
1397 property, or a TypeError is raised.
1399 Any node in this class whose 'propname' property links to any of the
1400 nodeids will be returned. Used by the full text indexing, which knows
1401 that "foo" occurs in msg1, msg3 and file7, so we have hits on these issues:
1402 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1403 '''
1404 propspec = propspec.items()
1405 for propname, nodeids in propspec:
1406 # check the prop is OK
1407 prop = self.properties[propname]
1408 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1409 raise TypeError, "'%s' not a Link/Multilink property"%propname
1411 # ok, now do the find
1412 cldb = self.db.getclassdb(self.classname)
1413 l = []
1414 try:
1415 for id in self.db.getnodeids(self.classname, db=cldb):
1416 node = self.db.getnode(self.classname, id, db=cldb)
1417 if node.has_key(self.db.RETIRED_FLAG):
1418 continue
1419 for propname, nodeids in propspec:
1420 # can't test if the node doesn't have this property
1421 if not node.has_key(propname):
1422 continue
1423 if type(nodeids) is type(''):
1424 nodeids = {nodeids:1}
1425 prop = self.properties[propname]
1426 value = node[propname]
1427 if isinstance(prop, Link) and nodeids.has_key(value):
1428 l.append(id)
1429 break
1430 elif isinstance(prop, Multilink):
1431 hit = 0
1432 for v in value:
1433 if nodeids.has_key(v):
1434 l.append(id)
1435 hit = 1
1436 break
1437 if hit:
1438 break
1439 finally:
1440 cldb.close()
1441 return l
1443 def stringFind(self, **requirements):
1444 '''Locate a particular node by matching a set of its String
1445 properties in a caseless search.
1447 If the property is not a String property, a TypeError is raised.
1449 The return is a list of the id of all nodes that match.
1450 '''
1451 for propname in requirements.keys():
1452 prop = self.properties[propname]
1453 if isinstance(not prop, String):
1454 raise TypeError, "'%s' not a String property"%propname
1455 requirements[propname] = requirements[propname].lower()
1456 l = []
1457 cldb = self.db.getclassdb(self.classname)
1458 try:
1459 for nodeid in self.db.getnodeids(self.classname, cldb):
1460 node = self.db.getnode(self.classname, nodeid, cldb)
1461 if node.has_key(self.db.RETIRED_FLAG):
1462 continue
1463 for key, value in requirements.items():
1464 if node[key] is None or node[key].lower() != value:
1465 break
1466 else:
1467 l.append(nodeid)
1468 finally:
1469 cldb.close()
1470 return l
1472 def list(self):
1473 ''' Return a list of the ids of the active nodes in this class.
1474 '''
1475 l = []
1476 cn = self.classname
1477 cldb = self.db.getclassdb(cn)
1478 try:
1479 for nodeid in self.db.getnodeids(cn, cldb):
1480 node = self.db.getnode(cn, nodeid, cldb)
1481 if node.has_key(self.db.RETIRED_FLAG):
1482 continue
1483 l.append(nodeid)
1484 finally:
1485 cldb.close()
1486 l.sort()
1487 return l
1489 def filter(self, search_matches, filterspec, sort, group,
1490 num_re = re.compile('^\d+$')):
1491 ''' Return a list of the ids of the active nodes in this class that
1492 match the 'filter' spec, sorted by the group spec and then the
1493 sort spec.
1495 "filterspec" is {propname: value(s)}
1496 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1497 and prop is a prop name or None
1498 "search_matches" is {nodeid: marker}
1499 '''
1500 cn = self.classname
1502 # optimise filterspec
1503 l = []
1504 props = self.getprops()
1505 LINK = 0
1506 MULTILINK = 1
1507 STRING = 2
1508 OTHER = 6
1509 for k, v in filterspec.items():
1510 propclass = props[k]
1511 if isinstance(propclass, Link):
1512 if type(v) is not type([]):
1513 v = [v]
1514 # replace key values with node ids
1515 u = []
1516 link_class = self.db.classes[propclass.classname]
1517 for entry in v:
1518 if entry == '-1': entry = None
1519 elif not num_re.match(entry):
1520 try:
1521 entry = link_class.lookup(entry)
1522 except (TypeError,KeyError):
1523 raise ValueError, 'property "%s": %s not a %s'%(
1524 k, entry, self.properties[k].classname)
1525 u.append(entry)
1527 l.append((LINK, k, u))
1528 elif isinstance(propclass, Multilink):
1529 if type(v) is not type([]):
1530 v = [v]
1531 # replace key values with node ids
1532 u = []
1533 link_class = self.db.classes[propclass.classname]
1534 for entry in v:
1535 if not num_re.match(entry):
1536 try:
1537 entry = link_class.lookup(entry)
1538 except (TypeError,KeyError):
1539 raise ValueError, 'new property "%s": %s not a %s'%(
1540 k, entry, self.properties[k].classname)
1541 u.append(entry)
1542 l.append((MULTILINK, k, u))
1543 elif isinstance(propclass, String):
1544 # simple glob searching
1545 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1546 v = v.replace('?', '.')
1547 v = v.replace('*', '.*?')
1548 l.append((STRING, k, re.compile(v, re.I)))
1549 elif isinstance(propclass, Boolean):
1550 if type(v) is type(''):
1551 bv = v.lower() in ('yes', 'true', 'on', '1')
1552 else:
1553 bv = v
1554 l.append((OTHER, k, bv))
1555 elif isinstance(propclass, Number):
1556 l.append((OTHER, k, int(v)))
1557 else:
1558 l.append((OTHER, k, v))
1559 filterspec = l
1561 # now, find all the nodes that are active and pass filtering
1562 l = []
1563 cldb = self.db.getclassdb(cn)
1564 try:
1565 # TODO: only full-scan once (use items())
1566 for nodeid in self.db.getnodeids(cn, cldb):
1567 node = self.db.getnode(cn, nodeid, cldb)
1568 if node.has_key(self.db.RETIRED_FLAG):
1569 continue
1570 # apply filter
1571 for t, k, v in filterspec:
1572 # make sure the node has the property
1573 if not node.has_key(k):
1574 # this node doesn't have this property, so reject it
1575 break
1577 # now apply the property filter
1578 if t == LINK:
1579 # link - if this node's property doesn't appear in the
1580 # filterspec's nodeid list, skip it
1581 if node[k] not in v:
1582 break
1583 elif t == MULTILINK:
1584 # multilink - if any of the nodeids required by the
1585 # filterspec aren't in this node's property, then skip
1586 # it
1587 have = node[k]
1588 for want in v:
1589 if want not in have:
1590 break
1591 else:
1592 continue
1593 break
1594 elif t == STRING:
1595 # RE search
1596 if node[k] is None or not v.search(node[k]):
1597 break
1598 elif t == OTHER:
1599 # straight value comparison for the other types
1600 if node[k] != v:
1601 break
1602 else:
1603 l.append((nodeid, node))
1604 finally:
1605 cldb.close()
1606 l.sort()
1608 # filter based on full text search
1609 if search_matches is not None:
1610 k = []
1611 for v in l:
1612 if search_matches.has_key(v[0]):
1613 k.append(v)
1614 l = k
1616 # now, sort the result
1617 def sortfun(a, b, sort=sort, group=group, properties=self.getprops(),
1618 db = self.db, cl=self):
1619 a_id, an = a
1620 b_id, bn = b
1621 # sort by group and then sort
1622 for dir, prop in group, sort:
1623 if dir is None or prop is None: continue
1625 # sorting is class-specific
1626 propclass = properties[prop]
1628 # handle the properties that might be "faked"
1629 # also, handle possible missing properties
1630 try:
1631 if not an.has_key(prop):
1632 an[prop] = cl.get(a_id, prop)
1633 av = an[prop]
1634 except KeyError:
1635 # the node doesn't have a value for this property
1636 if isinstance(propclass, Multilink): av = []
1637 else: av = ''
1638 try:
1639 if not bn.has_key(prop):
1640 bn[prop] = cl.get(b_id, prop)
1641 bv = bn[prop]
1642 except KeyError:
1643 # the node doesn't have a value for this property
1644 if isinstance(propclass, Multilink): bv = []
1645 else: bv = ''
1647 # String and Date values are sorted in the natural way
1648 if isinstance(propclass, String):
1649 # clean up the strings
1650 if av and av[0] in string.uppercase:
1651 av = an[prop] = av.lower()
1652 if bv and bv[0] in string.uppercase:
1653 bv = bn[prop] = bv.lower()
1654 if (isinstance(propclass, String) or
1655 isinstance(propclass, Date)):
1656 # it might be a string that's really an integer
1657 try:
1658 av = int(av)
1659 bv = int(bv)
1660 except:
1661 pass
1662 if dir == '+':
1663 r = cmp(av, bv)
1664 if r != 0: return r
1665 elif dir == '-':
1666 r = cmp(bv, av)
1667 if r != 0: return r
1669 # Link properties are sorted according to the value of
1670 # the "order" property on the linked nodes if it is
1671 # present; or otherwise on the key string of the linked
1672 # nodes; or finally on the node ids.
1673 elif isinstance(propclass, Link):
1674 link = db.classes[propclass.classname]
1675 if av is None and bv is not None: return -1
1676 if av is not None and bv is None: return 1
1677 if av is None and bv is None: continue
1678 if link.getprops().has_key('order'):
1679 if dir == '+':
1680 r = cmp(link.get(av, 'order'),
1681 link.get(bv, 'order'))
1682 if r != 0: return r
1683 elif dir == '-':
1684 r = cmp(link.get(bv, 'order'),
1685 link.get(av, 'order'))
1686 if r != 0: return r
1687 elif link.getkey():
1688 key = link.getkey()
1689 if dir == '+':
1690 r = cmp(link.get(av, key), link.get(bv, key))
1691 if r != 0: return r
1692 elif dir == '-':
1693 r = cmp(link.get(bv, key), link.get(av, key))
1694 if r != 0: return r
1695 else:
1696 if dir == '+':
1697 r = cmp(av, bv)
1698 if r != 0: return r
1699 elif dir == '-':
1700 r = cmp(bv, av)
1701 if r != 0: return r
1703 # Multilink properties are sorted according to how many
1704 # links are present.
1705 elif isinstance(propclass, Multilink):
1706 if dir == '+':
1707 r = cmp(len(av), len(bv))
1708 if r != 0: return r
1709 elif dir == '-':
1710 r = cmp(len(bv), len(av))
1711 if r != 0: return r
1712 elif isinstance(propclass, Number) or isinstance(propclass, Boolean):
1713 if dir == '+':
1714 r = cmp(av, bv)
1715 elif dir == '-':
1716 r = cmp(bv, av)
1718 # end for dir, prop in sort, group:
1719 # if all else fails, compare the ids
1720 return cmp(a[0], b[0])
1722 l.sort(sortfun)
1723 return [i[0] for i in l]
1725 def count(self):
1726 '''Get the number of nodes in this class.
1728 If the returned integer is 'numnodes', the ids of all the nodes
1729 in this class run from 1 to numnodes, and numnodes+1 will be the
1730 id of the next node to be created in this class.
1731 '''
1732 return self.db.countnodes(self.classname)
1734 # Manipulating properties:
1736 def getprops(self, protected=1):
1737 '''Return a dictionary mapping property names to property objects.
1738 If the "protected" flag is true, we include protected properties -
1739 those which may not be modified.
1741 In addition to the actual properties on the node, these
1742 methods provide the "creation" and "activity" properties. If the
1743 "protected" flag is true, we include protected properties - those
1744 which may not be modified.
1745 '''
1746 d = self.properties.copy()
1747 if protected:
1748 d['id'] = String()
1749 d['creation'] = hyperdb.Date()
1750 d['activity'] = hyperdb.Date()
1751 # can't be a link to user because the user might have been
1752 # retired since the journal entry was created
1753 d['creator'] = hyperdb.String()
1754 return d
1756 def addprop(self, **properties):
1757 '''Add properties to this class.
1759 The keyword arguments in 'properties' must map names to property
1760 objects, or a TypeError is raised. None of the keys in 'properties'
1761 may collide with the names of existing properties, or a ValueError
1762 is raised before any properties have been added.
1763 '''
1764 for key in properties.keys():
1765 if self.properties.has_key(key):
1766 raise ValueError, key
1767 self.properties.update(properties)
1769 def index(self, nodeid):
1770 '''Add (or refresh) the node to search indexes
1771 '''
1772 # find all the String properties that have indexme
1773 for prop, propclass in self.getprops().items():
1774 if isinstance(propclass, String) and propclass.indexme:
1775 try:
1776 value = str(self.get(nodeid, prop))
1777 except IndexError:
1778 # node no longer exists - entry should be removed
1779 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1780 else:
1781 # and index them under (classname, nodeid, property)
1782 self.db.indexer.add_text((self.classname, nodeid, prop),
1783 value)
1785 #
1786 # Detector interface
1787 #
1788 def audit(self, event, detector):
1789 '''Register a detector
1790 '''
1791 l = self.auditors[event]
1792 if detector not in l:
1793 self.auditors[event].append(detector)
1795 def fireAuditors(self, action, nodeid, newvalues):
1796 '''Fire all registered auditors.
1797 '''
1798 for audit in self.auditors[action]:
1799 audit(self.db, self, nodeid, newvalues)
1801 def react(self, event, detector):
1802 '''Register a detector
1803 '''
1804 l = self.reactors[event]
1805 if detector not in l:
1806 self.reactors[event].append(detector)
1808 def fireReactors(self, action, nodeid, oldvalues):
1809 '''Fire all registered reactors.
1810 '''
1811 for react in self.reactors[action]:
1812 react(self.db, self, nodeid, oldvalues)
1814 class FileClass(Class):
1815 '''This class defines a large chunk of data. To support this, it has a
1816 mandatory String property "content" which is typically saved off
1817 externally to the hyperdb.
1819 The default MIME type of this data is defined by the
1820 "default_mime_type" class attribute, which may be overridden by each
1821 node if the class defines a "type" String property.
1822 '''
1823 default_mime_type = 'text/plain'
1825 def create(self, **propvalues):
1826 ''' snaffle the file propvalue and store in a file
1827 '''
1828 content = propvalues['content']
1829 del propvalues['content']
1830 newid = Class.create(self, **propvalues)
1831 self.db.storefile(self.classname, newid, None, content)
1832 return newid
1834 def import_list(self, propnames, proplist):
1835 ''' Trap the "content" property...
1836 '''
1837 # dupe this list so we don't affect others
1838 propnames = propnames[:]
1840 # extract the "content" property from the proplist
1841 i = propnames.index('content')
1842 content = eval(proplist[i])
1843 del propnames[i]
1844 del proplist[i]
1846 # do the normal import
1847 newid = Class.import_list(self, propnames, proplist)
1849 # save off the "content" file
1850 self.db.storefile(self.classname, newid, None, content)
1851 return newid
1853 def get(self, nodeid, propname, default=_marker, cache=1):
1854 ''' trap the content propname and get it from the file
1855 '''
1857 poss_msg = 'Possibly a access right configuration problem.'
1858 if propname == 'content':
1859 try:
1860 return self.db.getfile(self.classname, nodeid, None)
1861 except IOError, (strerror):
1862 # BUG: by catching this we donot see an error in the log.
1863 return 'ERROR reading file: %s%s\n%s\n%s'%(
1864 self.classname, nodeid, poss_msg, strerror)
1865 if default is not _marker:
1866 return Class.get(self, nodeid, propname, default, cache=cache)
1867 else:
1868 return Class.get(self, nodeid, propname, cache=cache)
1870 def getprops(self, protected=1):
1871 ''' In addition to the actual properties on the node, these methods
1872 provide the "content" property. If the "protected" flag is true,
1873 we include protected properties - those which may not be
1874 modified.
1875 '''
1876 d = Class.getprops(self, protected=protected).copy()
1877 if protected:
1878 d['content'] = hyperdb.String()
1879 return d
1881 def index(self, nodeid):
1882 ''' Index the node in the search index.
1884 We want to index the content in addition to the normal String
1885 property indexing.
1886 '''
1887 # perform normal indexing
1888 Class.index(self, nodeid)
1890 # get the content to index
1891 content = self.get(nodeid, 'content')
1893 # figure the mime type
1894 if self.properties.has_key('type'):
1895 mime_type = self.get(nodeid, 'type')
1896 else:
1897 mime_type = self.default_mime_type
1899 # and index!
1900 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
1901 mime_type)
1903 # deviation from spec - was called ItemClass
1904 class IssueClass(Class, roundupdb.IssueClass):
1905 # Overridden methods:
1906 def __init__(self, db, classname, **properties):
1907 '''The newly-created class automatically includes the "messages",
1908 "files", "nosy", and "superseder" properties. If the 'properties'
1909 dictionary attempts to specify any of these properties or a
1910 "creation" or "activity" property, a ValueError is raised.
1911 '''
1912 if not properties.has_key('title'):
1913 properties['title'] = hyperdb.String(indexme='yes')
1914 if not properties.has_key('messages'):
1915 properties['messages'] = hyperdb.Multilink("msg")
1916 if not properties.has_key('files'):
1917 properties['files'] = hyperdb.Multilink("file")
1918 if not properties.has_key('nosy'):
1919 # note: journalling is turned off as it really just wastes
1920 # space. this behaviour may be overridden in an instance
1921 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
1922 if not properties.has_key('superseder'):
1923 properties['superseder'] = hyperdb.Multilink(classname)
1924 Class.__init__(self, db, classname, **properties)
1926 #