1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: back_anydbm.py,v 1.75 2002-09-10 12:44:42 richard Exp $
19 '''
20 This module defines a backend that saves the hyperdatabase in a database
21 chosen by anydbm. It is guaranteed to always be available in python
22 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
23 serious bugs, and is not available)
24 '''
26 import whichdb, anydbm, os, marshal, re, weakref, string, copy
27 from roundup import hyperdb, date, password, roundupdb, security
28 from blobfiles import FileStorage
29 from sessions import Sessions
30 from roundup.indexer import Indexer
31 from locking import acquire_lock, release_lock
32 from roundup.hyperdb import String, Password, Date, Interval, Link, \
33 Multilink, DatabaseError, Boolean, Number
35 #
36 # Now the database
37 #
38 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
39 '''A database for storing records containing flexible data types.
41 Transaction stuff TODO:
42 . check the timestamp of the class file and nuke the cache if it's
43 modified. Do some sort of conflict checking on the dirty stuff.
44 . perhaps detect write collisions (related to above)?
46 '''
47 def __init__(self, config, journaltag=None):
48 '''Open a hyperdatabase given a specifier to some storage.
50 The 'storagelocator' is obtained from config.DATABASE.
51 The meaning of 'storagelocator' depends on the particular
52 implementation of the hyperdatabase. It could be a file name,
53 a directory path, a socket descriptor for a connection to a
54 database over the network, etc.
56 The 'journaltag' is a token that will be attached to the journal
57 entries for any edits done on the database. If 'journaltag' is
58 None, the database is opened in read-only mode: the Class.create(),
59 Class.set(), and Class.retire() methods are disabled.
60 '''
61 self.config, self.journaltag = config, journaltag
62 self.dir = config.DATABASE
63 self.classes = {}
64 self.cache = {} # cache of nodes loaded or created
65 self.dirtynodes = {} # keep track of the dirty nodes by class
66 self.newnodes = {} # keep track of the new nodes by class
67 self.destroyednodes = {}# keep track of the destroyed nodes by class
68 self.transactions = []
69 self.indexer = Indexer(self.dir)
70 self.sessions = Sessions(self.config)
71 self.security = security.Security(self)
72 # ensure files are group readable and writable
73 os.umask(0002)
75 def post_init(self):
76 '''Called once the schema initialisation has finished.'''
77 # reindex the db if necessary
78 if self.indexer.should_reindex():
79 self.reindex()
81 def reindex(self):
82 for klass in self.classes.values():
83 for nodeid in klass.list():
84 klass.index(nodeid)
85 self.indexer.save_index()
87 def __repr__(self):
88 return '<back_anydbm instance at %x>'%id(self)
90 #
91 # Classes
92 #
93 def __getattr__(self, classname):
94 '''A convenient way of calling self.getclass(classname).'''
95 if self.classes.has_key(classname):
96 if __debug__:
97 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
98 return self.classes[classname]
99 raise AttributeError, classname
101 def addclass(self, cl):
102 if __debug__:
103 print >>hyperdb.DEBUG, 'addclass', (self, cl)
104 cn = cl.classname
105 if self.classes.has_key(cn):
106 raise ValueError, cn
107 self.classes[cn] = cl
109 def getclasses(self):
110 '''Return a list of the names of all existing classes.'''
111 if __debug__:
112 print >>hyperdb.DEBUG, 'getclasses', (self,)
113 l = self.classes.keys()
114 l.sort()
115 return l
117 def getclass(self, classname):
118 '''Get the Class object representing a particular class.
120 If 'classname' is not a valid class name, a KeyError is raised.
121 '''
122 if __debug__:
123 print >>hyperdb.DEBUG, 'getclass', (self, classname)
124 return self.classes[classname]
126 #
127 # Class DBs
128 #
129 def clear(self):
130 '''Delete all database contents
131 '''
132 if __debug__:
133 print >>hyperdb.DEBUG, 'clear', (self,)
134 for cn in self.classes.keys():
135 for dummy in 'nodes', 'journals':
136 path = os.path.join(self.dir, 'journals.%s'%cn)
137 if os.path.exists(path):
138 os.remove(path)
139 elif os.path.exists(path+'.db'): # dbm appends .db
140 os.remove(path+'.db')
142 def getclassdb(self, classname, mode='r'):
143 ''' grab a connection to the class db that will be used for
144 multiple actions
145 '''
146 if __debug__:
147 print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode)
148 return self.opendb('nodes.%s'%classname, mode)
150 def determine_db_type(self, path):
151 ''' determine which DB wrote the class file
152 '''
153 db_type = ''
154 if os.path.exists(path):
155 db_type = whichdb.whichdb(path)
156 if not db_type:
157 raise hyperdb.DatabaseError, "Couldn't identify database type"
158 elif os.path.exists(path+'.db'):
159 # if the path ends in '.db', it's a dbm database, whether
160 # anydbm says it's dbhash or not!
161 db_type = 'dbm'
162 return db_type
164 def opendb(self, name, mode):
165 '''Low-level database opener that gets around anydbm/dbm
166 eccentricities.
167 '''
168 if __debug__:
169 print >>hyperdb.DEBUG, 'opendb', (self, name, mode)
171 # figure the class db type
172 path = os.path.join(os.getcwd(), self.dir, name)
173 db_type = self.determine_db_type(path)
175 # new database? let anydbm pick the best dbm
176 if not db_type:
177 if __debug__:
178 print >>hyperdb.DEBUG, "opendb anydbm.open(%r, 'c')"%path
179 return anydbm.open(path, 'c')
181 # open the database with the correct module
182 try:
183 dbm = __import__(db_type)
184 except ImportError:
185 raise hyperdb.DatabaseError, \
186 "Couldn't open database - the required module '%s'"\
187 " is not available"%db_type
188 if __debug__:
189 print >>hyperdb.DEBUG, "opendb %r.open(%r, %r)"%(db_type, path,
190 mode)
191 return dbm.open(path, mode)
193 def lockdb(self, name):
194 ''' Lock a database file
195 '''
196 path = os.path.join(os.getcwd(), self.dir, '%s.lock'%name)
197 return acquire_lock(path)
199 #
200 # Node IDs
201 #
202 def newid(self, classname):
203 ''' Generate a new id for the given class
204 '''
205 # open the ids DB - create if if doesn't exist
206 lock = self.lockdb('_ids')
207 db = self.opendb('_ids', 'c')
208 if db.has_key(classname):
209 newid = db[classname] = str(int(db[classname]) + 1)
210 else:
211 # the count() bit is transitional - older dbs won't start at 1
212 newid = str(self.getclass(classname).count()+1)
213 db[classname] = newid
214 db.close()
215 release_lock(lock)
216 return newid
218 def setid(self, classname, setid):
219 ''' Set the id counter: used during import of database
220 '''
221 # open the ids DB - create if if doesn't exist
222 lock = self.lockdb('_ids')
223 db = self.opendb('_ids', 'c')
224 db[classname] = str(setid)
225 db.close()
226 release_lock(lock)
228 #
229 # Nodes
230 #
231 def addnode(self, classname, nodeid, node):
232 ''' add the specified node to its class's db
233 '''
234 if __debug__:
235 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
236 self.newnodes.setdefault(classname, {})[nodeid] = 1
237 self.cache.setdefault(classname, {})[nodeid] = node
238 self.savenode(classname, nodeid, node)
240 def setnode(self, classname, nodeid, node):
241 ''' change the specified node
242 '''
243 if __debug__:
244 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
245 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
247 # can't set without having already loaded the node
248 self.cache[classname][nodeid] = node
249 self.savenode(classname, nodeid, node)
251 def savenode(self, classname, nodeid, node):
252 ''' perform the saving of data specified by the set/addnode
253 '''
254 if __debug__:
255 print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node)
256 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
258 def getnode(self, classname, nodeid, db=None, cache=1):
259 ''' get a node from the database
260 '''
261 if __debug__:
262 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid, db)
263 if cache:
264 # try the cache
265 cache_dict = self.cache.setdefault(classname, {})
266 if cache_dict.has_key(nodeid):
267 if __debug__:
268 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
269 nodeid)
270 return cache_dict[nodeid]
272 if __debug__:
273 print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid)
275 # get from the database and save in the cache
276 if db is None:
277 db = self.getclassdb(classname)
278 if not db.has_key(nodeid):
279 raise IndexError, "no such %s %s"%(classname, nodeid)
281 # check the uncommitted, destroyed nodes
282 if (self.destroyednodes.has_key(classname) and
283 self.destroyednodes[classname].has_key(nodeid)):
284 raise IndexError, "no such %s %s"%(classname, nodeid)
286 # decode
287 res = marshal.loads(db[nodeid])
289 # reverse the serialisation
290 res = self.unserialise(classname, res)
292 # store off in the cache dict
293 if cache:
294 cache_dict[nodeid] = res
296 return res
298 def destroynode(self, classname, nodeid):
299 '''Remove a node from the database. Called exclusively by the
300 destroy() method on Class.
301 '''
302 if __debug__:
303 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
305 # remove from cache and newnodes if it's there
306 if (self.cache.has_key(classname) and
307 self.cache[classname].has_key(nodeid)):
308 del self.cache[classname][nodeid]
309 if (self.newnodes.has_key(classname) and
310 self.newnodes[classname].has_key(nodeid)):
311 del self.newnodes[classname][nodeid]
313 # see if there's any obvious commit actions that we should get rid of
314 for entry in self.transactions[:]:
315 if entry[1][:2] == (classname, nodeid):
316 self.transactions.remove(entry)
318 # add to the destroyednodes map
319 self.destroyednodes.setdefault(classname, {})[nodeid] = 1
321 # add the destroy commit action
322 self.transactions.append((self.doDestroyNode, (classname, nodeid)))
324 def serialise(self, classname, node):
325 '''Copy the node contents, converting non-marshallable data into
326 marshallable data.
327 '''
328 if __debug__:
329 print >>hyperdb.DEBUG, 'serialise', classname, node
330 properties = self.getclass(classname).getprops()
331 d = {}
332 for k, v in node.items():
333 # if the property doesn't exist, or is the "retired" flag then
334 # it won't be in the properties dict
335 if not properties.has_key(k):
336 d[k] = v
337 continue
339 # get the property spec
340 prop = properties[k]
342 if isinstance(prop, Password):
343 d[k] = str(v)
344 elif isinstance(prop, Date) and v is not None:
345 d[k] = v.serialise()
346 elif isinstance(prop, Interval) and v is not None:
347 d[k] = v.serialise()
348 else:
349 d[k] = v
350 return d
352 def unserialise(self, classname, node):
353 '''Decode the marshalled node data
354 '''
355 if __debug__:
356 print >>hyperdb.DEBUG, 'unserialise', classname, node
357 properties = self.getclass(classname).getprops()
358 d = {}
359 for k, v in node.items():
360 # if the property doesn't exist, or is the "retired" flag then
361 # it won't be in the properties dict
362 if not properties.has_key(k):
363 d[k] = v
364 continue
366 # get the property spec
367 prop = properties[k]
369 if isinstance(prop, Date) and v is not None:
370 d[k] = date.Date(v)
371 elif isinstance(prop, Interval) and v is not None:
372 d[k] = date.Interval(v)
373 elif isinstance(prop, Password):
374 p = password.Password()
375 p.unpack(v)
376 d[k] = p
377 else:
378 d[k] = v
379 return d
381 def hasnode(self, classname, nodeid, db=None):
382 ''' determine if the database has a given node
383 '''
384 if __debug__:
385 print >>hyperdb.DEBUG, 'hasnode', (self, classname, nodeid, db)
387 # try the cache
388 cache = self.cache.setdefault(classname, {})
389 if cache.has_key(nodeid):
390 if __debug__:
391 print >>hyperdb.TRACE, 'has %s %s cached'%(classname, nodeid)
392 return 1
393 if __debug__:
394 print >>hyperdb.TRACE, 'has %s %s'%(classname, nodeid)
396 # not in the cache - check the database
397 if db is None:
398 db = self.getclassdb(classname)
399 res = db.has_key(nodeid)
400 return res
402 def countnodes(self, classname, db=None):
403 if __debug__:
404 print >>hyperdb.DEBUG, 'countnodes', (self, classname, db)
406 count = 0
408 # include the uncommitted nodes
409 if self.newnodes.has_key(classname):
410 count += len(self.newnodes[classname])
411 if self.destroyednodes.has_key(classname):
412 count -= len(self.destroyednodes[classname])
414 # and count those in the DB
415 if db is None:
416 db = self.getclassdb(classname)
417 count = count + len(db.keys())
418 return count
420 def getnodeids(self, classname, db=None):
421 if __debug__:
422 print >>hyperdb.DEBUG, 'getnodeids', (self, classname, db)
424 res = []
426 # start off with the new nodes
427 if self.newnodes.has_key(classname):
428 res += self.newnodes[classname].keys()
430 if db is None:
431 db = self.getclassdb(classname)
432 res = res + db.keys()
434 # remove the uncommitted, destroyed nodes
435 if self.destroyednodes.has_key(classname):
436 for nodeid in self.destroyednodes[classname].keys():
437 if db.has_key(nodeid):
438 res.remove(nodeid)
440 return res
443 #
444 # Files - special node properties
445 # inherited from FileStorage
447 #
448 # Journal
449 #
450 def addjournal(self, classname, nodeid, action, params):
451 ''' Journal the Action
452 'action' may be:
454 'create' or 'set' -- 'params' is a dictionary of property values
455 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
456 'retire' -- 'params' is None
457 '''
458 if __debug__:
459 print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid,
460 action, params)
461 self.transactions.append((self.doSaveJournal, (classname, nodeid,
462 action, params)))
464 def getjournal(self, classname, nodeid):
465 ''' get the journal for id
467 Raise IndexError if the node doesn't exist (as per history()'s
468 API)
469 '''
470 if __debug__:
471 print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid)
472 # attempt to open the journal - in some rare cases, the journal may
473 # not exist
474 try:
475 db = self.opendb('journals.%s'%classname, 'r')
476 except anydbm.error, error:
477 if str(error) == "need 'c' or 'n' flag to open new db":
478 raise IndexError, 'no such %s %s'%(classname, nodeid)
479 elif error.args[0] != 2:
480 raise
481 raise IndexError, 'no such %s %s'%(classname, nodeid)
482 try:
483 journal = marshal.loads(db[nodeid])
484 except KeyError:
485 db.close()
486 raise IndexError, 'no such %s %s'%(classname, nodeid)
487 db.close()
488 res = []
489 for nodeid, date_stamp, user, action, params in journal:
490 res.append((nodeid, date.Date(date_stamp), user, action, params))
491 return res
493 def pack(self, pack_before):
494 ''' Delete all journal entries except "create" before 'pack_before'.
495 '''
496 if __debug__:
497 print >>hyperdb.DEBUG, 'packjournal', (self, pack_before)
499 for classname in self.getclasses():
500 # get the journal db
501 db_name = 'journals.%s'%classname
502 path = os.path.join(os.getcwd(), self.dir, classname)
503 db_type = self.determine_db_type(path)
504 db = self.opendb(db_name, 'w')
506 for key in db.keys():
507 # get the journal for this db entry
508 journal = marshal.loads(db[key])
509 l = []
510 last_set_entry = None
511 for entry in journal:
512 # unpack the entry
513 (nodeid, date_stamp, self.journaltag, action,
514 params) = entry
515 date_stamp = date.Date(date_stamp)
516 # if the entry is after the pack date, _or_ the initial
517 # create entry, then it stays
518 if date_stamp > pack_before or action == 'create':
519 l.append(entry)
520 elif action == 'set':
521 # grab the last set entry to keep information on
522 # activity
523 last_set_entry = entry
524 if last_set_entry:
525 date_stamp = last_set_entry[1]
526 # if the last set entry was made after the pack date
527 # then it is already in the list
528 if date_stamp < pack_before:
529 l.append(last_set_entry)
530 db[key] = marshal.dumps(l)
531 if db_type == 'gdbm':
532 db.reorganize()
533 db.close()
536 #
537 # Basic transaction support
538 #
539 def commit(self):
540 ''' Commit the current transactions.
541 '''
542 if __debug__:
543 print >>hyperdb.DEBUG, 'commit', (self,)
544 # TODO: lock the DB
546 # keep a handle to all the database files opened
547 self.databases = {}
549 # now, do all the transactions
550 reindex = {}
551 for method, args in self.transactions:
552 reindex[method(*args)] = 1
554 # now close all the database files
555 for db in self.databases.values():
556 db.close()
557 del self.databases
558 # TODO: unlock the DB
560 # reindex the nodes that request it
561 for classname, nodeid in filter(None, reindex.keys()):
562 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
563 self.getclass(classname).index(nodeid)
565 # save the indexer state
566 self.indexer.save_index()
568 # all transactions committed, back to normal
569 self.cache = {}
570 self.dirtynodes = {}
571 self.newnodes = {}
572 self.destroyednodes = {}
573 self.transactions = []
575 def getCachedClassDB(self, classname):
576 ''' get the class db, looking in our cache of databases for commit
577 '''
578 # get the database handle
579 db_name = 'nodes.%s'%classname
580 if not self.databases.has_key(db_name):
581 self.databases[db_name] = self.getclassdb(classname, 'c')
582 return self.databases[db_name]
584 def doSaveNode(self, classname, nodeid, node):
585 if __debug__:
586 print >>hyperdb.DEBUG, 'doSaveNode', (self, classname, nodeid,
587 node)
589 db = self.getCachedClassDB(classname)
591 # now save the marshalled data
592 db[nodeid] = marshal.dumps(self.serialise(classname, node))
594 # return the classname, nodeid so we reindex this content
595 return (classname, nodeid)
597 def getCachedJournalDB(self, classname):
598 ''' get the journal db, looking in our cache of databases for commit
599 '''
600 # get the database handle
601 db_name = 'journals.%s'%classname
602 if not self.databases.has_key(db_name):
603 self.databases[db_name] = self.opendb(db_name, 'c')
604 return self.databases[db_name]
606 def doSaveJournal(self, classname, nodeid, action, params):
607 # handle supply of the special journalling parameters (usually
608 # supplied on importing an existing database)
609 if isinstance(params, type({})):
610 if params.has_key('creator'):
611 journaltag = self.user.get(params['creator'], 'username')
612 del params['creator']
613 else:
614 journaltag = self.journaltag
615 if params.has_key('created'):
616 journaldate = params['created'].serialise()
617 del params['created']
618 else:
619 journaldate = date.Date().serialise()
620 if params.has_key('activity'):
621 del params['activity']
623 # serialise the parameters now
624 if action in ('set', 'create'):
625 params = self.serialise(classname, params)
626 else:
627 journaltag = self.journaltag
628 journaldate = date.Date().serialise()
630 # create the journal entry
631 entry = (nodeid, journaldate, journaltag, action, params)
633 if __debug__:
634 print >>hyperdb.DEBUG, 'doSaveJournal', entry
636 db = self.getCachedJournalDB(classname)
638 # now insert the journal entry
639 if db.has_key(nodeid):
640 # append to existing
641 s = db[nodeid]
642 l = marshal.loads(s)
643 l.append(entry)
644 else:
645 l = [entry]
647 db[nodeid] = marshal.dumps(l)
649 def doDestroyNode(self, classname, nodeid):
650 if __debug__:
651 print >>hyperdb.DEBUG, 'doDestroyNode', (self, classname, nodeid)
653 # delete from the class database
654 db = self.getCachedClassDB(classname)
655 if db.has_key(nodeid):
656 del db[nodeid]
658 # delete from the database
659 db = self.getCachedJournalDB(classname)
660 if db.has_key(nodeid):
661 del db[nodeid]
663 # return the classname, nodeid so we reindex this content
664 return (classname, nodeid)
666 def rollback(self):
667 ''' Reverse all actions from the current transaction.
668 '''
669 if __debug__:
670 print >>hyperdb.DEBUG, 'rollback', (self, )
671 for method, args in self.transactions:
672 # delete temporary files
673 if method == self.doStoreFile:
674 self.rollbackStoreFile(*args)
675 self.cache = {}
676 self.dirtynodes = {}
677 self.newnodes = {}
678 self.destroyednodes = {}
679 self.transactions = []
681 _marker = []
682 class Class(hyperdb.Class):
683 '''The handle to a particular class of nodes in a hyperdatabase.'''
685 def __init__(self, db, classname, **properties):
686 '''Create a new class with a given name and property specification.
688 'classname' must not collide with the name of an existing class,
689 or a ValueError is raised. The keyword arguments in 'properties'
690 must map names to property objects, or a TypeError is raised.
691 '''
692 if (properties.has_key('creation') or properties.has_key('activity')
693 or properties.has_key('creator')):
694 raise ValueError, '"creation", "activity" and "creator" are '\
695 'reserved'
697 self.classname = classname
698 self.properties = properties
699 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
700 self.key = ''
702 # should we journal changes (default yes)
703 self.do_journal = 1
705 # do the db-related init stuff
706 db.addclass(self)
708 self.auditors = {'create': [], 'set': [], 'retire': []}
709 self.reactors = {'create': [], 'set': [], 'retire': []}
711 def enableJournalling(self):
712 '''Turn journalling on for this class
713 '''
714 self.do_journal = 1
716 def disableJournalling(self):
717 '''Turn journalling off for this class
718 '''
719 self.do_journal = 0
721 # Editing nodes:
723 def create(self, **propvalues):
724 '''Create a new node of this class and return its id.
726 The keyword arguments in 'propvalues' map property names to values.
728 The values of arguments must be acceptable for the types of their
729 corresponding properties or a TypeError is raised.
731 If this class has a key property, it must be present and its value
732 must not collide with other key strings or a ValueError is raised.
734 Any other properties on this class that are missing from the
735 'propvalues' dictionary are set to None.
737 If an id in a link or multilink property does not refer to a valid
738 node, an IndexError is raised.
740 These operations trigger detectors and can be vetoed. Attempts
741 to modify the "creation" or "activity" properties cause a KeyError.
742 '''
743 if propvalues.has_key('id'):
744 raise KeyError, '"id" is reserved'
746 if self.db.journaltag is None:
747 raise DatabaseError, 'Database open read-only'
749 if propvalues.has_key('creation') or propvalues.has_key('activity'):
750 raise KeyError, '"creation" and "activity" are reserved'
752 self.fireAuditors('create', None, propvalues)
754 # new node's id
755 newid = self.db.newid(self.classname)
757 # validate propvalues
758 num_re = re.compile('^\d+$')
759 for key, value in propvalues.items():
760 if key == self.key:
761 try:
762 self.lookup(value)
763 except KeyError:
764 pass
765 else:
766 raise ValueError, 'node with key "%s" exists'%value
768 # try to handle this property
769 try:
770 prop = self.properties[key]
771 except KeyError:
772 raise KeyError, '"%s" has no property "%s"'%(self.classname,
773 key)
775 if value is not None and isinstance(prop, Link):
776 if type(value) != type(''):
777 raise ValueError, 'link value must be String'
778 link_class = self.properties[key].classname
779 # if it isn't a number, it's a key
780 if not num_re.match(value):
781 try:
782 value = self.db.classes[link_class].lookup(value)
783 except (TypeError, KeyError):
784 raise IndexError, 'new property "%s": %s not a %s'%(
785 key, value, link_class)
786 elif not self.db.getclass(link_class).hasnode(value):
787 raise IndexError, '%s has no node %s'%(link_class, value)
789 # save off the value
790 propvalues[key] = value
792 # register the link with the newly linked node
793 if self.do_journal and self.properties[key].do_journal:
794 self.db.addjournal(link_class, value, 'link',
795 (self.classname, newid, key))
797 elif isinstance(prop, Multilink):
798 if type(value) != type([]):
799 raise TypeError, 'new property "%s" not a list of ids'%key
801 # clean up and validate the list of links
802 link_class = self.properties[key].classname
803 l = []
804 for entry in value:
805 if type(entry) != type(''):
806 raise ValueError, '"%s" multilink value (%r) '\
807 'must contain Strings'%(key, value)
808 # if it isn't a number, it's a key
809 if not num_re.match(entry):
810 try:
811 entry = self.db.classes[link_class].lookup(entry)
812 except (TypeError, KeyError):
813 raise IndexError, 'new property "%s": %s not a %s'%(
814 key, entry, self.properties[key].classname)
815 l.append(entry)
816 value = l
817 propvalues[key] = value
819 # handle additions
820 for nodeid in value:
821 if not self.db.getclass(link_class).hasnode(nodeid):
822 raise IndexError, '%s has no node %s'%(link_class,
823 nodeid)
824 # register the link with the newly linked node
825 if self.do_journal and self.properties[key].do_journal:
826 self.db.addjournal(link_class, nodeid, 'link',
827 (self.classname, newid, key))
829 elif isinstance(prop, String):
830 if type(value) != type(''):
831 raise TypeError, 'new property "%s" not a string'%key
833 elif isinstance(prop, Password):
834 if not isinstance(value, password.Password):
835 raise TypeError, 'new property "%s" not a Password'%key
837 elif isinstance(prop, Date):
838 if value is not None and not isinstance(value, date.Date):
839 raise TypeError, 'new property "%s" not a Date'%key
841 elif isinstance(prop, Interval):
842 if value is not None and not isinstance(value, date.Interval):
843 raise TypeError, 'new property "%s" not an Interval'%key
845 elif value is not None and isinstance(prop, Number):
846 try:
847 float(value)
848 except ValueError:
849 raise TypeError, 'new property "%s" not numeric'%key
851 elif value is not None and isinstance(prop, Boolean):
852 try:
853 int(value)
854 except ValueError:
855 raise TypeError, 'new property "%s" not boolean'%key
857 # make sure there's data where there needs to be
858 for key, prop in self.properties.items():
859 if propvalues.has_key(key):
860 continue
861 if key == self.key:
862 raise ValueError, 'key property "%s" is required'%key
863 if isinstance(prop, Multilink):
864 propvalues[key] = []
865 else:
866 propvalues[key] = None
868 # done
869 self.db.addnode(self.classname, newid, propvalues)
870 if self.do_journal:
871 self.db.addjournal(self.classname, newid, 'create', propvalues)
873 self.fireReactors('create', newid, None)
875 return newid
877 def export_list(self, propnames, nodeid):
878 ''' Export a node - generate a list of CSV-able data in the order
879 specified by propnames for the given node.
880 '''
881 properties = self.getprops()
882 l = []
883 for prop in propnames:
884 proptype = properties[prop]
885 value = self.get(nodeid, prop)
886 # "marshal" data where needed
887 if isinstance(proptype, hyperdb.Date):
888 value = value.get_tuple()
889 elif isinstance(proptype, hyperdb.Interval):
890 value = value.get_tuple()
891 elif isinstance(proptype, hyperdb.Password):
892 value = str(value)
893 l.append(repr(value))
894 return l
896 def import_list(self, propnames, proplist):
897 ''' Import a node - all information including "id" is present and
898 should not be sanity checked. Triggers are not triggered. The
899 journal should be initialised using the "creator" and "created"
900 information.
902 Return the nodeid of the node imported.
903 '''
904 if self.db.journaltag is None:
905 raise DatabaseError, 'Database open read-only'
906 properties = self.getprops()
908 # make the new node's property map
909 d = {}
910 for i in range(len(propnames)):
911 # Use eval to reverse the repr() used to output the CSV
912 value = eval(proplist[i])
914 # Figure the property for this column
915 propname = propnames[i]
916 prop = properties[propname]
918 # "unmarshal" where necessary
919 if propname == 'id':
920 newid = value
921 continue
922 elif isinstance(prop, hyperdb.Date):
923 value = date.Date(value)
924 elif isinstance(prop, hyperdb.Interval):
925 value = date.Interval(value)
926 elif isinstance(prop, hyperdb.Password):
927 pwd = password.Password()
928 pwd.unpack(value)
929 value = pwd
930 if value is not None:
931 d[propname] = value
933 # add
934 self.db.addnode(self.classname, newid, d)
935 self.db.addjournal(self.classname, newid, 'create', d)
936 return newid
938 def get(self, nodeid, propname, default=_marker, cache=1):
939 '''Get the value of a property on an existing node of this class.
941 'nodeid' must be the id of an existing node of this class or an
942 IndexError is raised. 'propname' must be the name of a property
943 of this class or a KeyError is raised.
945 'cache' indicates whether the transaction cache should be queried
946 for the node. If the node has been modified and you need to
947 determine what its values prior to modification are, you need to
948 set cache=0.
950 Attempts to get the "creation" or "activity" properties should
951 do the right thing.
952 '''
953 if propname == 'id':
954 return nodeid
956 if propname == 'creation':
957 if not self.do_journal:
958 raise ValueError, 'Journalling is disabled for this class'
959 journal = self.db.getjournal(self.classname, nodeid)
960 if journal:
961 return self.db.getjournal(self.classname, nodeid)[0][1]
962 else:
963 # on the strange chance that there's no journal
964 return date.Date()
965 if propname == 'activity':
966 if not self.do_journal:
967 raise ValueError, 'Journalling is disabled for this class'
968 journal = self.db.getjournal(self.classname, nodeid)
969 if journal:
970 return self.db.getjournal(self.classname, nodeid)[-1][1]
971 else:
972 # on the strange chance that there's no journal
973 return date.Date()
974 if propname == 'creator':
975 if not self.do_journal:
976 raise ValueError, 'Journalling is disabled for this class'
977 journal = self.db.getjournal(self.classname, nodeid)
978 if journal:
979 return self.db.getjournal(self.classname, nodeid)[0][2]
980 else:
981 return self.db.journaltag
983 # get the property (raises KeyErorr if invalid)
984 prop = self.properties[propname]
986 # get the node's dict
987 d = self.db.getnode(self.classname, nodeid, cache=cache)
989 if not d.has_key(propname):
990 if default is _marker:
991 if isinstance(prop, Multilink):
992 return []
993 else:
994 return None
995 else:
996 return default
998 # return a dupe of the list so code doesn't get confused
999 if isinstance(prop, Multilink):
1000 return d[propname][:]
1002 return d[propname]
1004 # not in spec
1005 def getnode(self, nodeid, cache=1):
1006 ''' Return a convenience wrapper for the node.
1008 'nodeid' must be the id of an existing node of this class or an
1009 IndexError is raised.
1011 'cache' indicates whether the transaction cache should be queried
1012 for the node. If the node has been modified and you need to
1013 determine what its values prior to modification are, you need to
1014 set cache=0.
1015 '''
1016 return Node(self, nodeid, cache=cache)
1018 def set(self, nodeid, **propvalues):
1019 '''Modify a property on an existing node of this class.
1021 'nodeid' must be the id of an existing node of this class or an
1022 IndexError is raised.
1024 Each key in 'propvalues' must be the name of a property of this
1025 class or a KeyError is raised.
1027 All values in 'propvalues' must be acceptable types for their
1028 corresponding properties or a TypeError is raised.
1030 If the value of the key property is set, it must not collide with
1031 other key strings or a ValueError is raised.
1033 If the value of a Link or Multilink property contains an invalid
1034 node id, a ValueError is raised.
1036 These operations trigger detectors and can be vetoed. Attempts
1037 to modify the "creation" or "activity" properties cause a KeyError.
1038 '''
1039 if not propvalues:
1040 return propvalues
1042 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1043 raise KeyError, '"creation" and "activity" are reserved'
1045 if propvalues.has_key('id'):
1046 raise KeyError, '"id" is reserved'
1048 if self.db.journaltag is None:
1049 raise DatabaseError, 'Database open read-only'
1051 self.fireAuditors('set', nodeid, propvalues)
1052 # Take a copy of the node dict so that the subsequent set
1053 # operation doesn't modify the oldvalues structure.
1054 try:
1055 # try not using the cache initially
1056 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid,
1057 cache=0))
1058 except IndexError:
1059 # this will be needed if somone does a create() and set()
1060 # with no intervening commit()
1061 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1063 node = self.db.getnode(self.classname, nodeid)
1064 if node.has_key(self.db.RETIRED_FLAG):
1065 raise IndexError
1066 num_re = re.compile('^\d+$')
1068 # if the journal value is to be different, store it in here
1069 journalvalues = {}
1071 for propname, value in propvalues.items():
1072 # check to make sure we're not duplicating an existing key
1073 if propname == self.key and node[propname] != value:
1074 try:
1075 self.lookup(value)
1076 except KeyError:
1077 pass
1078 else:
1079 raise ValueError, 'node with key "%s" exists'%value
1081 # this will raise the KeyError if the property isn't valid
1082 # ... we don't use getprops() here because we only care about
1083 # the writeable properties.
1084 prop = self.properties[propname]
1086 # if the value's the same as the existing value, no sense in
1087 # doing anything
1088 if node.has_key(propname) and value == node[propname]:
1089 del propvalues[propname]
1090 continue
1092 # do stuff based on the prop type
1093 if isinstance(prop, Link):
1094 link_class = prop.classname
1095 # if it isn't a number, it's a key
1096 if value is not None and not isinstance(value, type('')):
1097 raise ValueError, 'property "%s" link value be a string'%(
1098 propname)
1099 if isinstance(value, type('')) and not num_re.match(value):
1100 try:
1101 value = self.db.classes[link_class].lookup(value)
1102 except (TypeError, KeyError):
1103 raise IndexError, 'new property "%s": %s not a %s'%(
1104 propname, value, prop.classname)
1106 if (value is not None and
1107 not self.db.getclass(link_class).hasnode(value)):
1108 raise IndexError, '%s has no node %s'%(link_class, value)
1110 if self.do_journal and prop.do_journal:
1111 # register the unlink with the old linked node
1112 if node[propname] is not None:
1113 self.db.addjournal(link_class, node[propname], 'unlink',
1114 (self.classname, nodeid, propname))
1116 # register the link with the newly linked node
1117 if value is not None:
1118 self.db.addjournal(link_class, value, 'link',
1119 (self.classname, nodeid, propname))
1121 elif isinstance(prop, Multilink):
1122 if type(value) != type([]):
1123 raise TypeError, 'new property "%s" not a list of'\
1124 ' ids'%propname
1125 link_class = self.properties[propname].classname
1126 l = []
1127 for entry in value:
1128 # if it isn't a number, it's a key
1129 if type(entry) != type(''):
1130 raise ValueError, 'new property "%s" link value ' \
1131 'must be a string'%propname
1132 if not num_re.match(entry):
1133 try:
1134 entry = self.db.classes[link_class].lookup(entry)
1135 except (TypeError, KeyError):
1136 raise IndexError, 'new property "%s": %s not a %s'%(
1137 propname, entry,
1138 self.properties[propname].classname)
1139 l.append(entry)
1140 value = l
1141 propvalues[propname] = value
1143 # figure the journal entry for this property
1144 add = []
1145 remove = []
1147 # handle removals
1148 if node.has_key(propname):
1149 l = node[propname]
1150 else:
1151 l = []
1152 for id in l[:]:
1153 if id in value:
1154 continue
1155 # register the unlink with the old linked node
1156 if self.do_journal and self.properties[propname].do_journal:
1157 self.db.addjournal(link_class, id, 'unlink',
1158 (self.classname, nodeid, propname))
1159 l.remove(id)
1160 remove.append(id)
1162 # handle additions
1163 for id in value:
1164 if not self.db.getclass(link_class).hasnode(id):
1165 raise IndexError, '%s has no node %s'%(link_class, id)
1166 if id in l:
1167 continue
1168 # register the link with the newly linked node
1169 if self.do_journal and self.properties[propname].do_journal:
1170 self.db.addjournal(link_class, id, 'link',
1171 (self.classname, nodeid, propname))
1172 l.append(id)
1173 add.append(id)
1175 # figure the journal entry
1176 l = []
1177 if add:
1178 l.append(('+', add))
1179 if remove:
1180 l.append(('-', remove))
1181 if l:
1182 journalvalues[propname] = tuple(l)
1184 elif isinstance(prop, String):
1185 if value is not None and type(value) != type(''):
1186 raise TypeError, 'new property "%s" not a string'%propname
1188 elif isinstance(prop, Password):
1189 if not isinstance(value, password.Password):
1190 raise TypeError, 'new property "%s" not a Password'%propname
1191 propvalues[propname] = value
1193 elif value is not None and isinstance(prop, Date):
1194 if not isinstance(value, date.Date):
1195 raise TypeError, 'new property "%s" not a Date'% propname
1196 propvalues[propname] = value
1198 elif value is not None and isinstance(prop, Interval):
1199 if not isinstance(value, date.Interval):
1200 raise TypeError, 'new property "%s" not an '\
1201 'Interval'%propname
1202 propvalues[propname] = value
1204 elif value is not None and isinstance(prop, Number):
1205 try:
1206 float(value)
1207 except ValueError:
1208 raise TypeError, 'new property "%s" not numeric'%propname
1210 elif value is not None and isinstance(prop, Boolean):
1211 try:
1212 int(value)
1213 except ValueError:
1214 raise TypeError, 'new property "%s" not boolean'%propname
1216 node[propname] = value
1218 # nothing to do?
1219 if not propvalues:
1220 return propvalues
1222 # do the set, and journal it
1223 self.db.setnode(self.classname, nodeid, node)
1225 if self.do_journal:
1226 propvalues.update(journalvalues)
1227 self.db.addjournal(self.classname, nodeid, 'set', propvalues)
1229 self.fireReactors('set', nodeid, oldvalues)
1231 return propvalues
1233 def retire(self, nodeid):
1234 '''Retire a node.
1236 The properties on the node remain available from the get() method,
1237 and the node's id is never reused.
1239 Retired nodes are not returned by the find(), list(), or lookup()
1240 methods, and other nodes may reuse the values of their key properties.
1242 These operations trigger detectors and can be vetoed. Attempts
1243 to modify the "creation" or "activity" properties cause a KeyError.
1244 '''
1245 if self.db.journaltag is None:
1246 raise DatabaseError, 'Database open read-only'
1248 self.fireAuditors('retire', nodeid, None)
1250 node = self.db.getnode(self.classname, nodeid)
1251 node[self.db.RETIRED_FLAG] = 1
1252 self.db.setnode(self.classname, nodeid, node)
1253 if self.do_journal:
1254 self.db.addjournal(self.classname, nodeid, 'retired', None)
1256 self.fireReactors('retire', nodeid, None)
1258 def is_retired(self, nodeid):
1259 '''Return true if the node is retired.
1260 '''
1261 node = self.db.getnode(cn, nodeid, cldb)
1262 if node.has_key(self.db.RETIRED_FLAG):
1263 return 1
1264 return 0
1266 def destroy(self, nodeid):
1267 '''Destroy a node.
1269 WARNING: this method should never be used except in extremely rare
1270 situations where there could never be links to the node being
1271 deleted
1272 WARNING: use retire() instead
1273 WARNING: the properties of this node will not be available ever again
1274 WARNING: really, use retire() instead
1276 Well, I think that's enough warnings. This method exists mostly to
1277 support the session storage of the cgi interface.
1278 '''
1279 if self.db.journaltag is None:
1280 raise DatabaseError, 'Database open read-only'
1281 self.db.destroynode(self.classname, nodeid)
1283 def history(self, nodeid):
1284 '''Retrieve the journal of edits on a particular node.
1286 'nodeid' must be the id of an existing node of this class or an
1287 IndexError is raised.
1289 The returned list contains tuples of the form
1291 (date, tag, action, params)
1293 'date' is a Timestamp object specifying the time of the change and
1294 'tag' is the journaltag specified when the database was opened.
1295 '''
1296 if not self.do_journal:
1297 raise ValueError, 'Journalling is disabled for this class'
1298 return self.db.getjournal(self.classname, nodeid)
1300 # Locating nodes:
1301 def hasnode(self, nodeid):
1302 '''Determine if the given nodeid actually exists
1303 '''
1304 return self.db.hasnode(self.classname, nodeid)
1306 def setkey(self, propname):
1307 '''Select a String property of this class to be the key property.
1309 'propname' must be the name of a String property of this class or
1310 None, or a TypeError is raised. The values of the key property on
1311 all existing nodes must be unique or a ValueError is raised. If the
1312 property doesn't exist, KeyError is raised.
1313 '''
1314 prop = self.getprops()[propname]
1315 if not isinstance(prop, String):
1316 raise TypeError, 'key properties must be String'
1317 self.key = propname
1319 def getkey(self):
1320 '''Return the name of the key property for this class or None.'''
1321 return self.key
1323 def labelprop(self, default_to_id=0):
1324 ''' Return the property name for a label for the given node.
1326 This method attempts to generate a consistent label for the node.
1327 It tries the following in order:
1328 1. key property
1329 2. "name" property
1330 3. "title" property
1331 4. first property from the sorted property name list
1332 '''
1333 k = self.getkey()
1334 if k:
1335 return k
1336 props = self.getprops()
1337 if props.has_key('name'):
1338 return 'name'
1339 elif props.has_key('title'):
1340 return 'title'
1341 if default_to_id:
1342 return 'id'
1343 props = props.keys()
1344 props.sort()
1345 return props[0]
1347 # TODO: set up a separate index db file for this? profile?
1348 def lookup(self, keyvalue):
1349 '''Locate a particular node by its key property and return its id.
1351 If this class has no key property, a TypeError is raised. If the
1352 'keyvalue' matches one of the values for the key property among
1353 the nodes in this class, the matching node's id is returned;
1354 otherwise a KeyError is raised.
1355 '''
1356 if not self.key:
1357 raise TypeError, 'No key property set'
1358 cldb = self.db.getclassdb(self.classname)
1359 try:
1360 for nodeid in self.db.getnodeids(self.classname, cldb):
1361 node = self.db.getnode(self.classname, nodeid, cldb)
1362 if node.has_key(self.db.RETIRED_FLAG):
1363 continue
1364 if node[self.key] == keyvalue:
1365 cldb.close()
1366 return nodeid
1367 finally:
1368 cldb.close()
1369 raise KeyError, keyvalue
1371 # change from spec - allows multiple props to match
1372 def find(self, **propspec):
1373 '''Get the ids of nodes in this class which link to the given nodes.
1375 'propspec' consists of keyword args propname={nodeid:1,}
1376 'propname' must be the name of a property in this class, or a
1377 KeyError is raised. That property must be a Link or Multilink
1378 property, or a TypeError is raised.
1380 Any node in this class whose 'propname' property links to any of the
1381 nodeids will be returned. Used by the full text indexing, which knows
1382 that "foo" occurs in msg1, msg3 and file7, so we have hits on these issues:
1383 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1384 '''
1385 propspec = propspec.items()
1386 for propname, nodeids in propspec:
1387 # check the prop is OK
1388 prop = self.properties[propname]
1389 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1390 raise TypeError, "'%s' not a Link/Multilink property"%propname
1392 # ok, now do the find
1393 cldb = self.db.getclassdb(self.classname)
1394 l = []
1395 try:
1396 for id in self.db.getnodeids(self.classname, db=cldb):
1397 node = self.db.getnode(self.classname, id, db=cldb)
1398 if node.has_key(self.db.RETIRED_FLAG):
1399 continue
1400 for propname, nodeids in propspec:
1401 # can't test if the node doesn't have this property
1402 if not node.has_key(propname):
1403 continue
1404 if type(nodeids) is type(''):
1405 nodeids = {nodeids:1}
1406 prop = self.properties[propname]
1407 value = node[propname]
1408 if isinstance(prop, Link) and nodeids.has_key(value):
1409 l.append(id)
1410 break
1411 elif isinstance(prop, Multilink):
1412 hit = 0
1413 for v in value:
1414 if nodeids.has_key(v):
1415 l.append(id)
1416 hit = 1
1417 break
1418 if hit:
1419 break
1420 finally:
1421 cldb.close()
1422 return l
1424 def stringFind(self, **requirements):
1425 '''Locate a particular node by matching a set of its String
1426 properties in a caseless search.
1428 If the property is not a String property, a TypeError is raised.
1430 The return is a list of the id of all nodes that match.
1431 '''
1432 for propname in requirements.keys():
1433 prop = self.properties[propname]
1434 if isinstance(not prop, String):
1435 raise TypeError, "'%s' not a String property"%propname
1436 requirements[propname] = requirements[propname].lower()
1437 l = []
1438 cldb = self.db.getclassdb(self.classname)
1439 try:
1440 for nodeid in self.db.getnodeids(self.classname, cldb):
1441 node = self.db.getnode(self.classname, nodeid, cldb)
1442 if node.has_key(self.db.RETIRED_FLAG):
1443 continue
1444 for key, value in requirements.items():
1445 if node[key] is None or node[key].lower() != value:
1446 break
1447 else:
1448 l.append(nodeid)
1449 finally:
1450 cldb.close()
1451 return l
1453 def list(self):
1454 ''' Return a list of the ids of the active nodes in this class.
1455 '''
1456 l = []
1457 cn = self.classname
1458 cldb = self.db.getclassdb(cn)
1459 try:
1460 for nodeid in self.db.getnodeids(cn, cldb):
1461 node = self.db.getnode(cn, nodeid, cldb)
1462 if node.has_key(self.db.RETIRED_FLAG):
1463 continue
1464 l.append(nodeid)
1465 finally:
1466 cldb.close()
1467 l.sort()
1468 return l
1470 def filter(self, search_matches, filterspec, sort, group,
1471 num_re = re.compile('^\d+$')):
1472 ''' Return a list of the ids of the active nodes in this class that
1473 match the 'filter' spec, sorted by the group spec and then the
1474 sort spec.
1476 "filterspec" is {propname: value(s)}
1477 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1478 and prop is a prop name or None
1479 "search_matches" is {nodeid: marker}
1480 '''
1481 cn = self.classname
1483 # optimise filterspec
1484 l = []
1485 props = self.getprops()
1486 LINK = 0
1487 MULTILINK = 1
1488 STRING = 2
1489 OTHER = 6
1490 for k, v in filterspec.items():
1491 propclass = props[k]
1492 if isinstance(propclass, Link):
1493 if type(v) is not type([]):
1494 v = [v]
1495 # replace key values with node ids
1496 u = []
1497 link_class = self.db.classes[propclass.classname]
1498 for entry in v:
1499 if entry == '-1': entry = None
1500 elif not num_re.match(entry):
1501 try:
1502 entry = link_class.lookup(entry)
1503 except (TypeError,KeyError):
1504 raise ValueError, 'property "%s": %s not a %s'%(
1505 k, entry, self.properties[k].classname)
1506 u.append(entry)
1508 l.append((LINK, k, u))
1509 elif isinstance(propclass, Multilink):
1510 if type(v) is not type([]):
1511 v = [v]
1512 # replace key values with node ids
1513 u = []
1514 link_class = self.db.classes[propclass.classname]
1515 for entry in v:
1516 if not num_re.match(entry):
1517 try:
1518 entry = link_class.lookup(entry)
1519 except (TypeError,KeyError):
1520 raise ValueError, 'new property "%s": %s not a %s'%(
1521 k, entry, self.properties[k].classname)
1522 u.append(entry)
1523 l.append((MULTILINK, k, u))
1524 elif isinstance(propclass, String):
1525 # simple glob searching
1526 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1527 v = v.replace('?', '.')
1528 v = v.replace('*', '.*?')
1529 l.append((STRING, k, re.compile(v, re.I)))
1530 elif isinstance(propclass, Boolean):
1531 if type(v) is type(''):
1532 bv = v.lower() in ('yes', 'true', 'on', '1')
1533 else:
1534 bv = v
1535 l.append((OTHER, k, bv))
1536 elif isinstance(propclass, Number):
1537 l.append((OTHER, k, int(v)))
1538 else:
1539 l.append((OTHER, k, v))
1540 filterspec = l
1542 # now, find all the nodes that are active and pass filtering
1543 l = []
1544 cldb = self.db.getclassdb(cn)
1545 try:
1546 # TODO: only full-scan once (use items())
1547 for nodeid in self.db.getnodeids(cn, cldb):
1548 node = self.db.getnode(cn, nodeid, cldb)
1549 if node.has_key(self.db.RETIRED_FLAG):
1550 continue
1551 # apply filter
1552 for t, k, v in filterspec:
1553 # make sure the node has the property
1554 if not node.has_key(k):
1555 # this node doesn't have this property, so reject it
1556 break
1558 # now apply the property filter
1559 if t == LINK:
1560 # link - if this node's property doesn't appear in the
1561 # filterspec's nodeid list, skip it
1562 if node[k] not in v:
1563 break
1564 elif t == MULTILINK:
1565 # multilink - if any of the nodeids required by the
1566 # filterspec aren't in this node's property, then skip
1567 # it
1568 have = node[k]
1569 for want in v:
1570 if want not in have:
1571 break
1572 else:
1573 continue
1574 break
1575 elif t == STRING:
1576 # RE search
1577 if node[k] is None or not v.search(node[k]):
1578 break
1579 elif t == OTHER:
1580 # straight value comparison for the other types
1581 if node[k] != v:
1582 break
1583 else:
1584 l.append((nodeid, node))
1585 finally:
1586 cldb.close()
1587 l.sort()
1589 # filter based on full text search
1590 if search_matches is not None:
1591 k = []
1592 for v in l:
1593 if search_matches.has_key(v[0]):
1594 k.append(v)
1595 l = k
1597 # now, sort the result
1598 def sortfun(a, b, sort=sort, group=group, properties=self.getprops(),
1599 db = self.db, cl=self):
1600 a_id, an = a
1601 b_id, bn = b
1602 # sort by group and then sort
1603 for dir, prop in group, sort:
1604 if dir is None or prop is None: continue
1606 # sorting is class-specific
1607 propclass = properties[prop]
1609 # handle the properties that might be "faked"
1610 # also, handle possible missing properties
1611 try:
1612 if not an.has_key(prop):
1613 an[prop] = cl.get(a_id, prop)
1614 av = an[prop]
1615 except KeyError:
1616 # the node doesn't have a value for this property
1617 if isinstance(propclass, Multilink): av = []
1618 else: av = ''
1619 try:
1620 if not bn.has_key(prop):
1621 bn[prop] = cl.get(b_id, prop)
1622 bv = bn[prop]
1623 except KeyError:
1624 # the node doesn't have a value for this property
1625 if isinstance(propclass, Multilink): bv = []
1626 else: bv = ''
1628 # String and Date values are sorted in the natural way
1629 if isinstance(propclass, String):
1630 # clean up the strings
1631 if av and av[0] in string.uppercase:
1632 av = an[prop] = av.lower()
1633 if bv and bv[0] in string.uppercase:
1634 bv = bn[prop] = bv.lower()
1635 if (isinstance(propclass, String) or
1636 isinstance(propclass, Date)):
1637 # it might be a string that's really an integer
1638 try:
1639 av = int(av)
1640 bv = int(bv)
1641 except:
1642 pass
1643 if dir == '+':
1644 r = cmp(av, bv)
1645 if r != 0: return r
1646 elif dir == '-':
1647 r = cmp(bv, av)
1648 if r != 0: return r
1650 # Link properties are sorted according to the value of
1651 # the "order" property on the linked nodes if it is
1652 # present; or otherwise on the key string of the linked
1653 # nodes; or finally on the node ids.
1654 elif isinstance(propclass, Link):
1655 link = db.classes[propclass.classname]
1656 if av is None and bv is not None: return -1
1657 if av is not None and bv is None: return 1
1658 if av is None and bv is None: continue
1659 if link.getprops().has_key('order'):
1660 if dir == '+':
1661 r = cmp(link.get(av, 'order'),
1662 link.get(bv, 'order'))
1663 if r != 0: return r
1664 elif dir == '-':
1665 r = cmp(link.get(bv, 'order'),
1666 link.get(av, 'order'))
1667 if r != 0: return r
1668 elif link.getkey():
1669 key = link.getkey()
1670 if dir == '+':
1671 r = cmp(link.get(av, key), link.get(bv, key))
1672 if r != 0: return r
1673 elif dir == '-':
1674 r = cmp(link.get(bv, key), link.get(av, key))
1675 if r != 0: return r
1676 else:
1677 if dir == '+':
1678 r = cmp(av, bv)
1679 if r != 0: return r
1680 elif dir == '-':
1681 r = cmp(bv, av)
1682 if r != 0: return r
1684 # Multilink properties are sorted according to how many
1685 # links are present.
1686 elif isinstance(propclass, Multilink):
1687 if dir == '+':
1688 r = cmp(len(av), len(bv))
1689 if r != 0: return r
1690 elif dir == '-':
1691 r = cmp(len(bv), len(av))
1692 if r != 0: return r
1693 elif isinstance(propclass, Number) or isinstance(propclass, Boolean):
1694 if dir == '+':
1695 r = cmp(av, bv)
1696 elif dir == '-':
1697 r = cmp(bv, av)
1699 # end for dir, prop in sort, group:
1700 # if all else fails, compare the ids
1701 return cmp(a[0], b[0])
1703 l.sort(sortfun)
1704 return [i[0] for i in l]
1706 def count(self):
1707 '''Get the number of nodes in this class.
1709 If the returned integer is 'numnodes', the ids of all the nodes
1710 in this class run from 1 to numnodes, and numnodes+1 will be the
1711 id of the next node to be created in this class.
1712 '''
1713 return self.db.countnodes(self.classname)
1715 # Manipulating properties:
1717 def getprops(self, protected=1):
1718 '''Return a dictionary mapping property names to property objects.
1719 If the "protected" flag is true, we include protected properties -
1720 those which may not be modified.
1722 In addition to the actual properties on the node, these
1723 methods provide the "creation" and "activity" properties. If the
1724 "protected" flag is true, we include protected properties - those
1725 which may not be modified.
1726 '''
1727 d = self.properties.copy()
1728 if protected:
1729 d['id'] = String()
1730 d['creation'] = hyperdb.Date()
1731 d['activity'] = hyperdb.Date()
1732 # can't be a link to user because the user might have been
1733 # retired since the journal entry was created
1734 d['creator'] = hyperdb.String()
1735 return d
1737 def addprop(self, **properties):
1738 '''Add properties to this class.
1740 The keyword arguments in 'properties' must map names to property
1741 objects, or a TypeError is raised. None of the keys in 'properties'
1742 may collide with the names of existing properties, or a ValueError
1743 is raised before any properties have been added.
1744 '''
1745 for key in properties.keys():
1746 if self.properties.has_key(key):
1747 raise ValueError, key
1748 self.properties.update(properties)
1750 def index(self, nodeid):
1751 '''Add (or refresh) the node to search indexes
1752 '''
1753 # find all the String properties that have indexme
1754 for prop, propclass in self.getprops().items():
1755 if isinstance(propclass, String) and propclass.indexme:
1756 try:
1757 value = str(self.get(nodeid, prop))
1758 except IndexError:
1759 # node no longer exists - entry should be removed
1760 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1761 else:
1762 # and index them under (classname, nodeid, property)
1763 self.db.indexer.add_text((self.classname, nodeid, prop),
1764 value)
1766 #
1767 # Detector interface
1768 #
1769 def audit(self, event, detector):
1770 '''Register a detector
1771 '''
1772 l = self.auditors[event]
1773 if detector not in l:
1774 self.auditors[event].append(detector)
1776 def fireAuditors(self, action, nodeid, newvalues):
1777 '''Fire all registered auditors.
1778 '''
1779 for audit in self.auditors[action]:
1780 audit(self.db, self, nodeid, newvalues)
1782 def react(self, event, detector):
1783 '''Register a detector
1784 '''
1785 l = self.reactors[event]
1786 if detector not in l:
1787 self.reactors[event].append(detector)
1789 def fireReactors(self, action, nodeid, oldvalues):
1790 '''Fire all registered reactors.
1791 '''
1792 for react in self.reactors[action]:
1793 react(self.db, self, nodeid, oldvalues)
1795 class FileClass(Class):
1796 '''This class defines a large chunk of data. To support this, it has a
1797 mandatory String property "content" which is typically saved off
1798 externally to the hyperdb.
1800 The default MIME type of this data is defined by the
1801 "default_mime_type" class attribute, which may be overridden by each
1802 node if the class defines a "type" String property.
1803 '''
1804 default_mime_type = 'text/plain'
1806 def create(self, **propvalues):
1807 ''' snaffle the file propvalue and store in a file
1808 '''
1809 content = propvalues['content']
1810 del propvalues['content']
1811 newid = Class.create(self, **propvalues)
1812 self.db.storefile(self.classname, newid, None, content)
1813 return newid
1815 def import_list(self, propnames, proplist):
1816 ''' Trap the "content" property...
1817 '''
1818 # dupe this list so we don't affect others
1819 propnames = propnames[:]
1821 # extract the "content" property from the proplist
1822 i = propnames.index('content')
1823 content = proplist[i]
1824 del propnames[i]
1825 del proplist[i]
1827 # do the normal import
1828 newid = Class.import_list(self, propnames, proplist)
1830 # save off the "content" file
1831 self.db.storefile(self.classname, newid, None, content)
1832 return newid
1834 def get(self, nodeid, propname, default=_marker, cache=1):
1835 ''' trap the content propname and get it from the file
1836 '''
1838 poss_msg = 'Possibly a access right configuration problem.'
1839 if propname == 'content':
1840 try:
1841 return self.db.getfile(self.classname, nodeid, None)
1842 except IOError, (strerror):
1843 # BUG: by catching this we donot see an error in the log.
1844 return 'ERROR reading file: %s%s\n%s\n%s'%(
1845 self.classname, nodeid, poss_msg, strerror)
1846 if default is not _marker:
1847 return Class.get(self, nodeid, propname, default, cache=cache)
1848 else:
1849 return Class.get(self, nodeid, propname, cache=cache)
1851 def getprops(self, protected=1):
1852 ''' In addition to the actual properties on the node, these methods
1853 provide the "content" property. If the "protected" flag is true,
1854 we include protected properties - those which may not be
1855 modified.
1856 '''
1857 d = Class.getprops(self, protected=protected).copy()
1858 if protected:
1859 d['content'] = hyperdb.String()
1860 return d
1862 def index(self, nodeid):
1863 ''' Index the node in the search index.
1865 We want to index the content in addition to the normal String
1866 property indexing.
1867 '''
1868 # perform normal indexing
1869 Class.index(self, nodeid)
1871 # get the content to index
1872 content = self.get(nodeid, 'content')
1874 # figure the mime type
1875 if self.properties.has_key('type'):
1876 mime_type = self.get(nodeid, 'type')
1877 else:
1878 mime_type = self.default_mime_type
1880 # and index!
1881 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
1882 mime_type)
1884 # deviation from spec - was called ItemClass
1885 class IssueClass(Class, roundupdb.IssueClass):
1886 # Overridden methods:
1887 def __init__(self, db, classname, **properties):
1888 '''The newly-created class automatically includes the "messages",
1889 "files", "nosy", and "superseder" properties. If the 'properties'
1890 dictionary attempts to specify any of these properties or a
1891 "creation" or "activity" property, a ValueError is raised.
1892 '''
1893 if not properties.has_key('title'):
1894 properties['title'] = hyperdb.String(indexme='yes')
1895 if not properties.has_key('messages'):
1896 properties['messages'] = hyperdb.Multilink("msg")
1897 if not properties.has_key('files'):
1898 properties['files'] = hyperdb.Multilink("file")
1899 if not properties.has_key('nosy'):
1900 # note: journalling is turned off as it really just wastes
1901 # space. this behaviour may be overridden in an instance
1902 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
1903 if not properties.has_key('superseder'):
1904 properties['superseder'] = hyperdb.Multilink(classname)
1905 Class.__init__(self, db, classname, **properties)
1907 #