61f2226b86717dfd919b15e3050c9cc05cb37249
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: back_anydbm.py,v 1.130 2003-11-10 03:56:39 richard Exp $
19 '''
20 This module defines a backend that saves the hyperdatabase in a database
21 chosen by anydbm. It is guaranteed to always be available in python
22 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
23 serious bugs, and is not available)
24 '''
26 try:
27 import anydbm, sys
28 # dumbdbm only works in python 2.1.2+
29 if sys.version_info < (2,1,2):
30 import dumbdbm
31 assert anydbm._defaultmod != dumbdbm
32 del dumbdbm
33 except AssertionError:
34 print "WARNING: you should upgrade to python 2.1.3"
36 import whichdb, os, marshal, re, weakref, string, copy
37 from roundup import hyperdb, date, password, roundupdb, security
38 from blobfiles import FileStorage
39 from sessions import Sessions, OneTimeKeys
40 from roundup.indexer import Indexer
41 from roundup.backends import locking
42 from roundup.hyperdb import String, Password, Date, Interval, Link, \
43 Multilink, DatabaseError, Boolean, Number, Node
44 from roundup.date import Range
46 #
47 # Now the database
48 #
49 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
50 '''A database for storing records containing flexible data types.
52 Transaction stuff TODO:
53 . check the timestamp of the class file and nuke the cache if it's
54 modified. Do some sort of conflict checking on the dirty stuff.
55 . perhaps detect write collisions (related to above)?
57 '''
58 def __init__(self, config, journaltag=None):
59 '''Open a hyperdatabase given a specifier to some storage.
61 The 'storagelocator' is obtained from config.DATABASE.
62 The meaning of 'storagelocator' depends on the particular
63 implementation of the hyperdatabase. It could be a file name,
64 a directory path, a socket descriptor for a connection to a
65 database over the network, etc.
67 The 'journaltag' is a token that will be attached to the journal
68 entries for any edits done on the database. If 'journaltag' is
69 None, the database is opened in read-only mode: the Class.create(),
70 Class.set(), Class.retire(), and Class.restore() methods are
71 disabled.
72 '''
73 self.config, self.journaltag = config, journaltag
74 self.dir = config.DATABASE
75 self.classes = {}
76 self.cache = {} # cache of nodes loaded or created
77 self.dirtynodes = {} # keep track of the dirty nodes by class
78 self.newnodes = {} # keep track of the new nodes by class
79 self.destroyednodes = {}# keep track of the destroyed nodes by class
80 self.transactions = []
81 self.indexer = Indexer(self.dir)
82 self.sessions = Sessions(self.config)
83 self.otks = OneTimeKeys(self.config)
84 self.security = security.Security(self)
85 # ensure files are group readable and writable
86 os.umask(0002)
88 # lock it
89 lockfilenm = os.path.join(self.dir, 'lock')
90 self.lockfile = locking.acquire_lock(lockfilenm)
91 self.lockfile.write(str(os.getpid()))
92 self.lockfile.flush()
94 def post_init(self):
95 ''' Called once the schema initialisation has finished.
96 '''
97 # reindex the db if necessary
98 if self.indexer.should_reindex():
99 self.reindex()
101 def refresh_database(self):
102 "Rebuild the database"
103 self.reindex()
105 def reindex(self):
106 for klass in self.classes.values():
107 for nodeid in klass.list():
108 klass.index(nodeid)
109 self.indexer.save_index()
111 def __repr__(self):
112 return '<back_anydbm instance at %x>'%id(self)
114 #
115 # Classes
116 #
117 def __getattr__(self, classname):
118 '''A convenient way of calling self.getclass(classname).'''
119 if self.classes.has_key(classname):
120 if __debug__:
121 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
122 return self.classes[classname]
123 raise AttributeError, classname
125 def addclass(self, cl):
126 if __debug__:
127 print >>hyperdb.DEBUG, 'addclass', (self, cl)
128 cn = cl.classname
129 if self.classes.has_key(cn):
130 raise ValueError, cn
131 self.classes[cn] = cl
133 def getclasses(self):
134 '''Return a list of the names of all existing classes.'''
135 if __debug__:
136 print >>hyperdb.DEBUG, 'getclasses', (self,)
137 l = self.classes.keys()
138 l.sort()
139 return l
141 def getclass(self, classname):
142 '''Get the Class object representing a particular class.
144 If 'classname' is not a valid class name, a KeyError is raised.
145 '''
146 if __debug__:
147 print >>hyperdb.DEBUG, 'getclass', (self, classname)
148 try:
149 return self.classes[classname]
150 except KeyError:
151 raise KeyError, 'There is no class called "%s"'%classname
153 #
154 # Class DBs
155 #
156 def clear(self):
157 '''Delete all database contents
158 '''
159 if __debug__:
160 print >>hyperdb.DEBUG, 'clear', (self,)
161 for cn in self.classes.keys():
162 for dummy in 'nodes', 'journals':
163 path = os.path.join(self.dir, 'journals.%s'%cn)
164 if os.path.exists(path):
165 os.remove(path)
166 elif os.path.exists(path+'.db'): # dbm appends .db
167 os.remove(path+'.db')
169 def getclassdb(self, classname, mode='r'):
170 ''' grab a connection to the class db that will be used for
171 multiple actions
172 '''
173 if __debug__:
174 print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode)
175 return self.opendb('nodes.%s'%classname, mode)
177 def determine_db_type(self, path):
178 ''' determine which DB wrote the class file
179 '''
180 db_type = ''
181 if os.path.exists(path):
182 db_type = whichdb.whichdb(path)
183 if not db_type:
184 raise DatabaseError, "Couldn't identify database type"
185 elif os.path.exists(path+'.db'):
186 # if the path ends in '.db', it's a dbm database, whether
187 # anydbm says it's dbhash or not!
188 db_type = 'dbm'
189 return db_type
191 def opendb(self, name, mode):
192 '''Low-level database opener that gets around anydbm/dbm
193 eccentricities.
194 '''
195 if __debug__:
196 print >>hyperdb.DEBUG, 'opendb', (self, name, mode)
198 # figure the class db type
199 path = os.path.join(os.getcwd(), self.dir, name)
200 db_type = self.determine_db_type(path)
202 # new database? let anydbm pick the best dbm
203 if not db_type:
204 if __debug__:
205 print >>hyperdb.DEBUG, "opendb anydbm.open(%r, 'c')"%path
206 return anydbm.open(path, 'c')
208 # open the database with the correct module
209 try:
210 dbm = __import__(db_type)
211 except ImportError:
212 raise DatabaseError, \
213 "Couldn't open database - the required module '%s'"\
214 " is not available"%db_type
215 if __debug__:
216 print >>hyperdb.DEBUG, "opendb %r.open(%r, %r)"%(db_type, path,
217 mode)
218 return dbm.open(path, mode)
220 #
221 # Node IDs
222 #
223 def newid(self, classname):
224 ''' Generate a new id for the given class
225 '''
226 # open the ids DB - create if if doesn't exist
227 db = self.opendb('_ids', 'c')
228 if db.has_key(classname):
229 newid = db[classname] = str(int(db[classname]) + 1)
230 else:
231 # the count() bit is transitional - older dbs won't start at 1
232 newid = str(self.getclass(classname).count()+1)
233 db[classname] = newid
234 db.close()
235 return newid
237 def setid(self, classname, setid):
238 ''' Set the id counter: used during import of database
239 '''
240 # open the ids DB - create if if doesn't exist
241 db = self.opendb('_ids', 'c')
242 db[classname] = str(setid)
243 db.close()
245 #
246 # Nodes
247 #
248 def addnode(self, classname, nodeid, node):
249 ''' add the specified node to its class's db
250 '''
251 if __debug__:
252 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
254 # we'll be supplied these props if we're doing an import
255 if not node.has_key('creator'):
256 # add in the "calculated" properties (dupe so we don't affect
257 # calling code's node assumptions)
258 node = node.copy()
259 node['creator'] = self.getuid()
260 node['creation'] = node['activity'] = date.Date()
262 self.newnodes.setdefault(classname, {})[nodeid] = 1
263 self.cache.setdefault(classname, {})[nodeid] = node
264 self.savenode(classname, nodeid, node)
266 def setnode(self, classname, nodeid, node):
267 ''' change the specified node
268 '''
269 if __debug__:
270 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
271 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
273 # update the activity time (dupe so we don't affect
274 # calling code's node assumptions)
275 node = node.copy()
276 node['activity'] = date.Date()
278 # can't set without having already loaded the node
279 self.cache[classname][nodeid] = node
280 self.savenode(classname, nodeid, node)
282 def savenode(self, classname, nodeid, node):
283 ''' perform the saving of data specified by the set/addnode
284 '''
285 if __debug__:
286 print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node)
287 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
289 def getnode(self, classname, nodeid, db=None, cache=1):
290 ''' get a node from the database
292 Note the "cache" parameter is not used, and exists purely for
293 backward compatibility!
294 '''
295 if __debug__:
296 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid, db)
298 # try the cache
299 cache_dict = self.cache.setdefault(classname, {})
300 if cache_dict.has_key(nodeid):
301 if __debug__:
302 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
303 nodeid)
304 return cache_dict[nodeid]
306 if __debug__:
307 print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid)
309 # get from the database and save in the cache
310 if db is None:
311 db = self.getclassdb(classname)
312 if not db.has_key(nodeid):
313 # try the cache - might be a brand-new node
314 cache_dict = self.cache.setdefault(classname, {})
315 if cache_dict.has_key(nodeid):
316 if __debug__:
317 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
318 nodeid)
319 return cache_dict[nodeid]
320 raise IndexError, "no such %s %s"%(classname, nodeid)
322 # check the uncommitted, destroyed nodes
323 if (self.destroyednodes.has_key(classname) and
324 self.destroyednodes[classname].has_key(nodeid)):
325 raise IndexError, "no such %s %s"%(classname, nodeid)
327 # decode
328 res = marshal.loads(db[nodeid])
330 # reverse the serialisation
331 res = self.unserialise(classname, res)
333 # store off in the cache dict
334 if cache:
335 cache_dict[nodeid] = res
337 return res
339 def destroynode(self, classname, nodeid):
340 '''Remove a node from the database. Called exclusively by the
341 destroy() method on Class.
342 '''
343 if __debug__:
344 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
346 # remove from cache and newnodes if it's there
347 if (self.cache.has_key(classname) and
348 self.cache[classname].has_key(nodeid)):
349 del self.cache[classname][nodeid]
350 if (self.newnodes.has_key(classname) and
351 self.newnodes[classname].has_key(nodeid)):
352 del self.newnodes[classname][nodeid]
354 # see if there's any obvious commit actions that we should get rid of
355 for entry in self.transactions[:]:
356 if entry[1][:2] == (classname, nodeid):
357 self.transactions.remove(entry)
359 # add to the destroyednodes map
360 self.destroyednodes.setdefault(classname, {})[nodeid] = 1
362 # add the destroy commit action
363 self.transactions.append((self.doDestroyNode, (classname, nodeid)))
365 def serialise(self, classname, node):
366 '''Copy the node contents, converting non-marshallable data into
367 marshallable data.
368 '''
369 if __debug__:
370 print >>hyperdb.DEBUG, 'serialise', classname, node
371 properties = self.getclass(classname).getprops()
372 d = {}
373 for k, v in node.items():
374 # if the property doesn't exist, or is the "retired" flag then
375 # it won't be in the properties dict
376 if not properties.has_key(k):
377 d[k] = v
378 continue
380 # get the property spec
381 prop = properties[k]
383 if isinstance(prop, Password) and v is not None:
384 d[k] = str(v)
385 elif isinstance(prop, Date) and v is not None:
386 d[k] = v.serialise()
387 elif isinstance(prop, Interval) and v is not None:
388 d[k] = v.serialise()
389 else:
390 d[k] = v
391 return d
393 def unserialise(self, classname, node):
394 '''Decode the marshalled node data
395 '''
396 if __debug__:
397 print >>hyperdb.DEBUG, 'unserialise', classname, node
398 properties = self.getclass(classname).getprops()
399 d = {}
400 for k, v in node.items():
401 # if the property doesn't exist, or is the "retired" flag then
402 # it won't be in the properties dict
403 if not properties.has_key(k):
404 d[k] = v
405 continue
407 # get the property spec
408 prop = properties[k]
410 if isinstance(prop, Date) and v is not None:
411 d[k] = date.Date(v)
412 elif isinstance(prop, Interval) and v is not None:
413 d[k] = date.Interval(v)
414 elif isinstance(prop, Password) and v is not None:
415 p = password.Password()
416 p.unpack(v)
417 d[k] = p
418 else:
419 d[k] = v
420 return d
422 def hasnode(self, classname, nodeid, db=None):
423 ''' determine if the database has a given node
424 '''
425 if __debug__:
426 print >>hyperdb.DEBUG, 'hasnode', (self, classname, nodeid, db)
428 # try the cache
429 cache = self.cache.setdefault(classname, {})
430 if cache.has_key(nodeid):
431 if __debug__:
432 print >>hyperdb.TRACE, 'has %s %s cached'%(classname, nodeid)
433 return 1
434 if __debug__:
435 print >>hyperdb.TRACE, 'has %s %s'%(classname, nodeid)
437 # not in the cache - check the database
438 if db is None:
439 db = self.getclassdb(classname)
440 res = db.has_key(nodeid)
441 return res
443 def countnodes(self, classname, db=None):
444 if __debug__:
445 print >>hyperdb.DEBUG, 'countnodes', (self, classname, db)
447 count = 0
449 # include the uncommitted nodes
450 if self.newnodes.has_key(classname):
451 count += len(self.newnodes[classname])
452 if self.destroyednodes.has_key(classname):
453 count -= len(self.destroyednodes[classname])
455 # and count those in the DB
456 if db is None:
457 db = self.getclassdb(classname)
458 count = count + len(db.keys())
459 return count
462 #
463 # Files - special node properties
464 # inherited from FileStorage
466 #
467 # Journal
468 #
469 def addjournal(self, classname, nodeid, action, params, creator=None,
470 creation=None):
471 ''' Journal the Action
472 'action' may be:
474 'create' or 'set' -- 'params' is a dictionary of property values
475 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
476 'retire' -- 'params' is None
477 '''
478 if __debug__:
479 print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid,
480 action, params, creator, creation)
481 self.transactions.append((self.doSaveJournal, (classname, nodeid,
482 action, params, creator, creation)))
484 def getjournal(self, classname, nodeid):
485 ''' get the journal for id
487 Raise IndexError if the node doesn't exist (as per history()'s
488 API)
489 '''
490 if __debug__:
491 print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid)
493 # our journal result
494 res = []
496 # add any journal entries for transactions not committed to the
497 # database
498 for method, args in self.transactions:
499 if method != self.doSaveJournal:
500 continue
501 (cache_classname, cache_nodeid, cache_action, cache_params,
502 cache_creator, cache_creation) = args
503 if cache_classname == classname and cache_nodeid == nodeid:
504 if not cache_creator:
505 cache_creator = self.getuid()
506 if not cache_creation:
507 cache_creation = date.Date()
508 res.append((cache_nodeid, cache_creation, cache_creator,
509 cache_action, cache_params))
511 # attempt to open the journal - in some rare cases, the journal may
512 # not exist
513 try:
514 db = self.opendb('journals.%s'%classname, 'r')
515 except anydbm.error, error:
516 if str(error) == "need 'c' or 'n' flag to open new db":
517 raise IndexError, 'no such %s %s'%(classname, nodeid)
518 elif error.args[0] != 2:
519 raise
520 raise IndexError, 'no such %s %s'%(classname, nodeid)
521 try:
522 journal = marshal.loads(db[nodeid])
523 except KeyError:
524 db.close()
525 if res:
526 # we have some unsaved journal entries, be happy!
527 return res
528 raise IndexError, 'no such %s %s'%(classname, nodeid)
529 db.close()
531 # add all the saved journal entries for this node
532 for nodeid, date_stamp, user, action, params in journal:
533 res.append((nodeid, date.Date(date_stamp), user, action, params))
534 return res
536 def pack(self, pack_before):
537 ''' Delete all journal entries except "create" before 'pack_before'.
538 '''
539 if __debug__:
540 print >>hyperdb.DEBUG, 'packjournal', (self, pack_before)
542 pack_before = pack_before.serialise()
543 for classname in self.getclasses():
544 # get the journal db
545 db_name = 'journals.%s'%classname
546 path = os.path.join(os.getcwd(), self.dir, classname)
547 db_type = self.determine_db_type(path)
548 db = self.opendb(db_name, 'w')
550 for key in db.keys():
551 # get the journal for this db entry
552 journal = marshal.loads(db[key])
553 l = []
554 last_set_entry = None
555 for entry in journal:
556 # unpack the entry
557 (nodeid, date_stamp, self.journaltag, action,
558 params) = entry
559 # if the entry is after the pack date, _or_ the initial
560 # create entry, then it stays
561 if date_stamp > pack_before or action == 'create':
562 l.append(entry)
563 db[key] = marshal.dumps(l)
564 if db_type == 'gdbm':
565 db.reorganize()
566 db.close()
569 #
570 # Basic transaction support
571 #
572 def commit(self):
573 ''' Commit the current transactions.
574 '''
575 if __debug__:
576 print >>hyperdb.DEBUG, 'commit', (self,)
578 # keep a handle to all the database files opened
579 self.databases = {}
581 try:
582 # now, do all the transactions
583 reindex = {}
584 for method, args in self.transactions:
585 reindex[method(*args)] = 1
586 finally:
587 # make sure we close all the database files
588 for db in self.databases.values():
589 db.close()
590 del self.databases
592 # reindex the nodes that request it
593 for classname, nodeid in filter(None, reindex.keys()):
594 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
595 self.getclass(classname).index(nodeid)
597 # save the indexer state
598 self.indexer.save_index()
600 self.clearCache()
602 def clearCache(self):
603 # all transactions committed, back to normal
604 self.cache = {}
605 self.dirtynodes = {}
606 self.newnodes = {}
607 self.destroyednodes = {}
608 self.transactions = []
610 def getCachedClassDB(self, classname):
611 ''' get the class db, looking in our cache of databases for commit
612 '''
613 # get the database handle
614 db_name = 'nodes.%s'%classname
615 if not self.databases.has_key(db_name):
616 self.databases[db_name] = self.getclassdb(classname, 'c')
617 return self.databases[db_name]
619 def doSaveNode(self, classname, nodeid, node):
620 if __debug__:
621 print >>hyperdb.DEBUG, 'doSaveNode', (self, classname, nodeid,
622 node)
624 db = self.getCachedClassDB(classname)
626 # now save the marshalled data
627 db[nodeid] = marshal.dumps(self.serialise(classname, node))
629 # return the classname, nodeid so we reindex this content
630 return (classname, nodeid)
632 def getCachedJournalDB(self, classname):
633 ''' get the journal db, looking in our cache of databases for commit
634 '''
635 # get the database handle
636 db_name = 'journals.%s'%classname
637 if not self.databases.has_key(db_name):
638 self.databases[db_name] = self.opendb(db_name, 'c')
639 return self.databases[db_name]
641 def doSaveJournal(self, classname, nodeid, action, params, creator,
642 creation):
643 # serialise the parameters now if necessary
644 if isinstance(params, type({})):
645 if action in ('set', 'create'):
646 params = self.serialise(classname, params)
648 # handle supply of the special journalling parameters (usually
649 # supplied on importing an existing database)
650 if creator:
651 journaltag = creator
652 else:
653 journaltag = self.getuid()
654 if creation:
655 journaldate = creation.serialise()
656 else:
657 journaldate = date.Date().serialise()
659 # create the journal entry
660 entry = (nodeid, journaldate, journaltag, action, params)
662 if __debug__:
663 print >>hyperdb.DEBUG, 'doSaveJournal', entry
665 db = self.getCachedJournalDB(classname)
667 # now insert the journal entry
668 if db.has_key(nodeid):
669 # append to existing
670 s = db[nodeid]
671 l = marshal.loads(s)
672 l.append(entry)
673 else:
674 l = [entry]
676 db[nodeid] = marshal.dumps(l)
678 def doDestroyNode(self, classname, nodeid):
679 if __debug__:
680 print >>hyperdb.DEBUG, 'doDestroyNode', (self, classname, nodeid)
682 # delete from the class database
683 db = self.getCachedClassDB(classname)
684 if db.has_key(nodeid):
685 del db[nodeid]
687 # delete from the database
688 db = self.getCachedJournalDB(classname)
689 if db.has_key(nodeid):
690 del db[nodeid]
692 # return the classname, nodeid so we reindex this content
693 return (classname, nodeid)
695 def rollback(self):
696 ''' Reverse all actions from the current transaction.
697 '''
698 if __debug__:
699 print >>hyperdb.DEBUG, 'rollback', (self, )
700 for method, args in self.transactions:
701 # delete temporary files
702 if method == self.doStoreFile:
703 self.rollbackStoreFile(*args)
704 self.cache = {}
705 self.dirtynodes = {}
706 self.newnodes = {}
707 self.destroyednodes = {}
708 self.transactions = []
710 def close(self):
711 ''' Nothing to do
712 '''
713 if self.lockfile is not None:
714 locking.release_lock(self.lockfile)
715 if self.lockfile is not None:
716 self.lockfile.close()
717 self.lockfile = None
719 _marker = []
720 class Class(hyperdb.Class):
721 '''The handle to a particular class of nodes in a hyperdatabase.'''
723 def __init__(self, db, classname, **properties):
724 '''Create a new class with a given name and property specification.
726 'classname' must not collide with the name of an existing class,
727 or a ValueError is raised. The keyword arguments in 'properties'
728 must map names to property objects, or a TypeError is raised.
729 '''
730 if (properties.has_key('creation') or properties.has_key('activity')
731 or properties.has_key('creator')):
732 raise ValueError, '"creation", "activity" and "creator" are '\
733 'reserved'
735 self.classname = classname
736 self.properties = properties
737 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
738 self.key = ''
740 # should we journal changes (default yes)
741 self.do_journal = 1
743 # do the db-related init stuff
744 db.addclass(self)
746 self.auditors = {'create': [], 'set': [], 'retire': [], 'restore': []}
747 self.reactors = {'create': [], 'set': [], 'retire': [], 'restore': []}
749 def enableJournalling(self):
750 '''Turn journalling on for this class
751 '''
752 self.do_journal = 1
754 def disableJournalling(self):
755 '''Turn journalling off for this class
756 '''
757 self.do_journal = 0
759 # Editing nodes:
761 def create(self, **propvalues):
762 '''Create a new node of this class and return its id.
764 The keyword arguments in 'propvalues' map property names to values.
766 The values of arguments must be acceptable for the types of their
767 corresponding properties or a TypeError is raised.
769 If this class has a key property, it must be present and its value
770 must not collide with other key strings or a ValueError is raised.
772 Any other properties on this class that are missing from the
773 'propvalues' dictionary are set to None.
775 If an id in a link or multilink property does not refer to a valid
776 node, an IndexError is raised.
778 These operations trigger detectors and can be vetoed. Attempts
779 to modify the "creation" or "activity" properties cause a KeyError.
780 '''
781 self.fireAuditors('create', None, propvalues)
782 newid = self.create_inner(**propvalues)
783 self.fireReactors('create', newid, None)
784 return newid
786 def create_inner(self, **propvalues):
787 ''' Called by create, in-between the audit and react calls.
788 '''
789 if propvalues.has_key('id'):
790 raise KeyError, '"id" is reserved'
792 if self.db.journaltag is None:
793 raise DatabaseError, 'Database open read-only'
795 if propvalues.has_key('creation') or propvalues.has_key('activity'):
796 raise KeyError, '"creation" and "activity" are reserved'
797 # new node's id
798 newid = self.db.newid(self.classname)
800 # validate propvalues
801 num_re = re.compile('^\d+$')
802 for key, value in propvalues.items():
803 if key == self.key:
804 try:
805 self.lookup(value)
806 except KeyError:
807 pass
808 else:
809 raise ValueError, 'node with key "%s" exists'%value
811 # try to handle this property
812 try:
813 prop = self.properties[key]
814 except KeyError:
815 raise KeyError, '"%s" has no property "%s"'%(self.classname,
816 key)
818 if value is not None and isinstance(prop, Link):
819 if type(value) != type(''):
820 raise ValueError, 'link value must be String'
821 link_class = self.properties[key].classname
822 # if it isn't a number, it's a key
823 if not num_re.match(value):
824 try:
825 value = self.db.classes[link_class].lookup(value)
826 except (TypeError, KeyError):
827 raise IndexError, 'new property "%s": %s not a %s'%(
828 key, value, link_class)
829 elif not self.db.getclass(link_class).hasnode(value):
830 raise IndexError, '%s has no node %s'%(link_class, value)
832 # save off the value
833 propvalues[key] = value
835 # register the link with the newly linked node
836 if self.do_journal and self.properties[key].do_journal:
837 self.db.addjournal(link_class, value, 'link',
838 (self.classname, newid, key))
840 elif isinstance(prop, Multilink):
841 if type(value) != type([]):
842 raise TypeError, 'new property "%s" not a list of ids'%key
844 # clean up and validate the list of links
845 link_class = self.properties[key].classname
846 l = []
847 for entry in value:
848 if type(entry) != type(''):
849 raise ValueError, '"%s" multilink value (%r) '\
850 'must contain Strings'%(key, value)
851 # if it isn't a number, it's a key
852 if not num_re.match(entry):
853 try:
854 entry = self.db.classes[link_class].lookup(entry)
855 except (TypeError, KeyError):
856 raise IndexError, 'new property "%s": %s not a %s'%(
857 key, entry, self.properties[key].classname)
858 l.append(entry)
859 value = l
860 propvalues[key] = value
862 # handle additions
863 for nodeid in value:
864 if not self.db.getclass(link_class).hasnode(nodeid):
865 raise IndexError, '%s has no node %s'%(link_class,
866 nodeid)
867 # register the link with the newly linked node
868 if self.do_journal and self.properties[key].do_journal:
869 self.db.addjournal(link_class, nodeid, 'link',
870 (self.classname, newid, key))
872 elif isinstance(prop, String):
873 if type(value) != type('') and type(value) != type(u''):
874 raise TypeError, 'new property "%s" not a string'%key
876 elif isinstance(prop, Password):
877 if not isinstance(value, password.Password):
878 raise TypeError, 'new property "%s" not a Password'%key
880 elif isinstance(prop, Date):
881 if value is not None and not isinstance(value, date.Date):
882 raise TypeError, 'new property "%s" not a Date'%key
884 elif isinstance(prop, Interval):
885 if value is not None and not isinstance(value, date.Interval):
886 raise TypeError, 'new property "%s" not an Interval'%key
888 elif value is not None and isinstance(prop, Number):
889 try:
890 float(value)
891 except ValueError:
892 raise TypeError, 'new property "%s" not numeric'%key
894 elif value is not None and isinstance(prop, Boolean):
895 try:
896 int(value)
897 except ValueError:
898 raise TypeError, 'new property "%s" not boolean'%key
900 # make sure there's data where there needs to be
901 for key, prop in self.properties.items():
902 if propvalues.has_key(key):
903 continue
904 if key == self.key:
905 raise ValueError, 'key property "%s" is required'%key
906 if isinstance(prop, Multilink):
907 propvalues[key] = []
908 else:
909 propvalues[key] = None
911 # done
912 self.db.addnode(self.classname, newid, propvalues)
913 if self.do_journal:
914 self.db.addjournal(self.classname, newid, 'create', {})
916 return newid
918 def export_list(self, propnames, nodeid):
919 ''' Export a node - generate a list of CSV-able data in the order
920 specified by propnames for the given node.
921 '''
922 properties = self.getprops()
923 l = []
924 for prop in propnames:
925 proptype = properties[prop]
926 value = self.get(nodeid, prop)
927 # "marshal" data where needed
928 if value is None:
929 pass
930 elif isinstance(proptype, hyperdb.Date):
931 value = value.get_tuple()
932 elif isinstance(proptype, hyperdb.Interval):
933 value = value.get_tuple()
934 elif isinstance(proptype, hyperdb.Password):
935 value = str(value)
936 l.append(repr(value))
938 # append retired flag
939 l.append(self.is_retired(nodeid))
941 return l
943 def import_list(self, propnames, proplist):
944 ''' Import a node - all information including "id" is present and
945 should not be sanity checked. Triggers are not triggered. The
946 journal should be initialised using the "creator" and "created"
947 information.
949 Return the nodeid of the node imported.
950 '''
951 if self.db.journaltag is None:
952 raise DatabaseError, 'Database open read-only'
953 properties = self.getprops()
955 # make the new node's property map
956 d = {}
957 newid = None
958 for i in range(len(propnames)):
959 # Figure the property for this column
960 propname = propnames[i]
962 # Use eval to reverse the repr() used to output the CSV
963 value = eval(proplist[i])
965 # "unmarshal" where necessary
966 if propname == 'id':
967 newid = value
968 continue
969 elif propname == 'is retired':
970 # is the item retired?
971 if int(value):
972 d[self.db.RETIRED_FLAG] = 1
973 continue
974 elif value is None:
975 d[propname] = None
976 continue
978 prop = properties[propname]
979 if isinstance(prop, hyperdb.Date):
980 value = date.Date(value)
981 elif isinstance(prop, hyperdb.Interval):
982 value = date.Interval(value)
983 elif isinstance(prop, hyperdb.Password):
984 pwd = password.Password()
985 pwd.unpack(value)
986 value = pwd
987 d[propname] = value
989 # get a new id if necessary
990 if newid is None:
991 newid = self.db.newid(self.classname)
993 # add the node and journal
994 self.db.addnode(self.classname, newid, d)
996 # extract the journalling stuff and nuke it
997 if d.has_key('creator'):
998 creator = d['creator']
999 del d['creator']
1000 else:
1001 creator = None
1002 if d.has_key('creation'):
1003 creation = d['creation']
1004 del d['creation']
1005 else:
1006 creation = None
1007 if d.has_key('activity'):
1008 del d['activity']
1009 self.db.addjournal(self.classname, newid, 'create', {}, creator,
1010 creation)
1011 return newid
1013 def get(self, nodeid, propname, default=_marker, cache=1):
1014 '''Get the value of a property on an existing node of this class.
1016 'nodeid' must be the id of an existing node of this class or an
1017 IndexError is raised. 'propname' must be the name of a property
1018 of this class or a KeyError is raised.
1020 'cache' exists for backward compatibility, and is not used.
1022 Attempts to get the "creation" or "activity" properties should
1023 do the right thing.
1024 '''
1025 if propname == 'id':
1026 return nodeid
1028 # get the node's dict
1029 d = self.db.getnode(self.classname, nodeid)
1031 # check for one of the special props
1032 if propname == 'creation':
1033 if d.has_key('creation'):
1034 return d['creation']
1035 if not self.do_journal:
1036 raise ValueError, 'Journalling is disabled for this class'
1037 journal = self.db.getjournal(self.classname, nodeid)
1038 if journal:
1039 return self.db.getjournal(self.classname, nodeid)[0][1]
1040 else:
1041 # on the strange chance that there's no journal
1042 return date.Date()
1043 if propname == 'activity':
1044 if d.has_key('activity'):
1045 return d['activity']
1046 if not self.do_journal:
1047 raise ValueError, 'Journalling is disabled for this class'
1048 journal = self.db.getjournal(self.classname, nodeid)
1049 if journal:
1050 return self.db.getjournal(self.classname, nodeid)[-1][1]
1051 else:
1052 # on the strange chance that there's no journal
1053 return date.Date()
1054 if propname == 'creator':
1055 if d.has_key('creator'):
1056 return d['creator']
1057 if not self.do_journal:
1058 raise ValueError, 'Journalling is disabled for this class'
1059 journal = self.db.getjournal(self.classname, nodeid)
1060 if journal:
1061 num_re = re.compile('^\d+$')
1062 value = self.db.getjournal(self.classname, nodeid)[0][2]
1063 if num_re.match(value):
1064 return value
1065 else:
1066 # old-style "username" journal tag
1067 try:
1068 return self.db.user.lookup(value)
1069 except KeyError:
1070 # user's been retired, return admin
1071 return '1'
1072 else:
1073 return self.db.getuid()
1075 # get the property (raises KeyErorr if invalid)
1076 prop = self.properties[propname]
1078 if not d.has_key(propname):
1079 if default is _marker:
1080 if isinstance(prop, Multilink):
1081 return []
1082 else:
1083 return None
1084 else:
1085 return default
1087 # return a dupe of the list so code doesn't get confused
1088 if isinstance(prop, Multilink):
1089 return d[propname][:]
1091 return d[propname]
1093 # not in spec
1094 def getnode(self, nodeid, cache=1):
1095 ''' Return a convenience wrapper for the node.
1097 'nodeid' must be the id of an existing node of this class or an
1098 IndexError is raised.
1100 'cache' exists for backwards compatibility, and is not used.
1101 '''
1102 return Node(self, nodeid)
1104 def set(self, nodeid, **propvalues):
1105 '''Modify a property on an existing node of this class.
1107 'nodeid' must be the id of an existing node of this class or an
1108 IndexError is raised.
1110 Each key in 'propvalues' must be the name of a property of this
1111 class or a KeyError is raised.
1113 All values in 'propvalues' must be acceptable types for their
1114 corresponding properties or a TypeError is raised.
1116 If the value of the key property is set, it must not collide with
1117 other key strings or a ValueError is raised.
1119 If the value of a Link or Multilink property contains an invalid
1120 node id, a ValueError is raised.
1122 These operations trigger detectors and can be vetoed. Attempts
1123 to modify the "creation" or "activity" properties cause a KeyError.
1124 '''
1125 if not propvalues:
1126 return propvalues
1128 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1129 raise KeyError, '"creation" and "activity" are reserved'
1131 if propvalues.has_key('id'):
1132 raise KeyError, '"id" is reserved'
1134 if self.db.journaltag is None:
1135 raise DatabaseError, 'Database open read-only'
1137 self.fireAuditors('set', nodeid, propvalues)
1138 # Take a copy of the node dict so that the subsequent set
1139 # operation doesn't modify the oldvalues structure.
1140 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1142 node = self.db.getnode(self.classname, nodeid)
1143 if node.has_key(self.db.RETIRED_FLAG):
1144 raise IndexError
1145 num_re = re.compile('^\d+$')
1147 # if the journal value is to be different, store it in here
1148 journalvalues = {}
1150 for propname, value in propvalues.items():
1151 # check to make sure we're not duplicating an existing key
1152 if propname == self.key and node[propname] != value:
1153 try:
1154 self.lookup(value)
1155 except KeyError:
1156 pass
1157 else:
1158 raise ValueError, 'node with key "%s" exists'%value
1160 # this will raise the KeyError if the property isn't valid
1161 # ... we don't use getprops() here because we only care about
1162 # the writeable properties.
1163 try:
1164 prop = self.properties[propname]
1165 except KeyError:
1166 raise KeyError, '"%s" has no property named "%s"'%(
1167 self.classname, propname)
1169 # if the value's the same as the existing value, no sense in
1170 # doing anything
1171 current = node.get(propname, None)
1172 if value == current:
1173 del propvalues[propname]
1174 continue
1175 journalvalues[propname] = current
1177 # do stuff based on the prop type
1178 if isinstance(prop, Link):
1179 link_class = prop.classname
1180 # if it isn't a number, it's a key
1181 if value is not None and not isinstance(value, type('')):
1182 raise ValueError, 'property "%s" link value be a string'%(
1183 propname)
1184 if isinstance(value, type('')) and not num_re.match(value):
1185 try:
1186 value = self.db.classes[link_class].lookup(value)
1187 except (TypeError, KeyError):
1188 raise IndexError, 'new property "%s": %s not a %s'%(
1189 propname, value, prop.classname)
1191 if (value is not None and
1192 not self.db.getclass(link_class).hasnode(value)):
1193 raise IndexError, '%s has no node %s'%(link_class, value)
1195 if self.do_journal and prop.do_journal:
1196 # register the unlink with the old linked node
1197 if node.has_key(propname) and node[propname] is not None:
1198 self.db.addjournal(link_class, node[propname], 'unlink',
1199 (self.classname, nodeid, propname))
1201 # register the link with the newly linked node
1202 if value is not None:
1203 self.db.addjournal(link_class, value, 'link',
1204 (self.classname, nodeid, propname))
1206 elif isinstance(prop, Multilink):
1207 if type(value) != type([]):
1208 raise TypeError, 'new property "%s" not a list of'\
1209 ' ids'%propname
1210 link_class = self.properties[propname].classname
1211 l = []
1212 for entry in value:
1213 # if it isn't a number, it's a key
1214 if type(entry) != type(''):
1215 raise ValueError, 'new property "%s" link value ' \
1216 'must be a string'%propname
1217 if not num_re.match(entry):
1218 try:
1219 entry = self.db.classes[link_class].lookup(entry)
1220 except (TypeError, KeyError):
1221 raise IndexError, 'new property "%s": %s not a %s'%(
1222 propname, entry,
1223 self.properties[propname].classname)
1224 l.append(entry)
1225 value = l
1226 propvalues[propname] = value
1228 # figure the journal entry for this property
1229 add = []
1230 remove = []
1232 # handle removals
1233 if node.has_key(propname):
1234 l = node[propname]
1235 else:
1236 l = []
1237 for id in l[:]:
1238 if id in value:
1239 continue
1240 # register the unlink with the old linked node
1241 if self.do_journal and self.properties[propname].do_journal:
1242 self.db.addjournal(link_class, id, 'unlink',
1243 (self.classname, nodeid, propname))
1244 l.remove(id)
1245 remove.append(id)
1247 # handle additions
1248 for id in value:
1249 if not self.db.getclass(link_class).hasnode(id):
1250 raise IndexError, '%s has no node %s'%(link_class, id)
1251 if id in l:
1252 continue
1253 # register the link with the newly linked node
1254 if self.do_journal and self.properties[propname].do_journal:
1255 self.db.addjournal(link_class, id, 'link',
1256 (self.classname, nodeid, propname))
1257 l.append(id)
1258 add.append(id)
1260 # figure the journal entry
1261 l = []
1262 if add:
1263 l.append(('+', add))
1264 if remove:
1265 l.append(('-', remove))
1266 if l:
1267 journalvalues[propname] = tuple(l)
1269 elif isinstance(prop, String):
1270 if value is not None and type(value) != type('') and type(value) != type(u''):
1271 raise TypeError, 'new property "%s" not a string'%propname
1273 elif isinstance(prop, Password):
1274 if not isinstance(value, password.Password):
1275 raise TypeError, 'new property "%s" not a Password'%propname
1276 propvalues[propname] = value
1278 elif value is not None and isinstance(prop, Date):
1279 if not isinstance(value, date.Date):
1280 raise TypeError, 'new property "%s" not a Date'% propname
1281 propvalues[propname] = value
1283 elif value is not None and isinstance(prop, Interval):
1284 if not isinstance(value, date.Interval):
1285 raise TypeError, 'new property "%s" not an '\
1286 'Interval'%propname
1287 propvalues[propname] = value
1289 elif value is not None and isinstance(prop, Number):
1290 try:
1291 float(value)
1292 except ValueError:
1293 raise TypeError, 'new property "%s" not numeric'%propname
1295 elif value is not None and isinstance(prop, Boolean):
1296 try:
1297 int(value)
1298 except ValueError:
1299 raise TypeError, 'new property "%s" not boolean'%propname
1301 node[propname] = value
1303 # nothing to do?
1304 if not propvalues:
1305 return propvalues
1307 # do the set, and journal it
1308 self.db.setnode(self.classname, nodeid, node)
1310 if self.do_journal:
1311 self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
1313 self.fireReactors('set', nodeid, oldvalues)
1315 return propvalues
1317 def retire(self, nodeid):
1318 '''Retire a node.
1320 The properties on the node remain available from the get() method,
1321 and the node's id is never reused.
1323 Retired nodes are not returned by the find(), list(), or lookup()
1324 methods, and other nodes may reuse the values of their key properties.
1326 These operations trigger detectors and can be vetoed. Attempts
1327 to modify the "creation" or "activity" properties cause a KeyError.
1328 '''
1329 if self.db.journaltag is None:
1330 raise DatabaseError, 'Database open read-only'
1332 self.fireAuditors('retire', nodeid, None)
1334 node = self.db.getnode(self.classname, nodeid)
1335 node[self.db.RETIRED_FLAG] = 1
1336 self.db.setnode(self.classname, nodeid, node)
1337 if self.do_journal:
1338 self.db.addjournal(self.classname, nodeid, 'retired', None)
1340 self.fireReactors('retire', nodeid, None)
1342 def restore(self, nodeid):
1343 '''Restpre a retired node.
1345 Make node available for all operations like it was before retirement.
1346 '''
1347 if self.db.journaltag is None:
1348 raise DatabaseError, 'Database open read-only'
1350 node = self.db.getnode(self.classname, nodeid)
1351 # check if key property was overrided
1352 key = self.getkey()
1353 try:
1354 id = self.lookup(node[key])
1355 except KeyError:
1356 pass
1357 else:
1358 raise KeyError, "Key property (%s) of retired node clashes with \
1359 existing one (%s)" % (key, node[key])
1360 # Now we can safely restore node
1361 self.fireAuditors('restore', nodeid, None)
1362 del node[self.db.RETIRED_FLAG]
1363 self.db.setnode(self.classname, nodeid, node)
1364 if self.do_journal:
1365 self.db.addjournal(self.classname, nodeid, 'restored', None)
1367 self.fireReactors('restore', nodeid, None)
1369 def is_retired(self, nodeid, cldb=None):
1370 '''Return true if the node is retired.
1371 '''
1372 node = self.db.getnode(self.classname, nodeid, cldb)
1373 if node.has_key(self.db.RETIRED_FLAG):
1374 return 1
1375 return 0
1377 def destroy(self, nodeid):
1378 '''Destroy a node.
1380 WARNING: this method should never be used except in extremely rare
1381 situations where there could never be links to the node being
1382 deleted
1383 WARNING: use retire() instead
1384 WARNING: the properties of this node will not be available ever again
1385 WARNING: really, use retire() instead
1387 Well, I think that's enough warnings. This method exists mostly to
1388 support the session storage of the cgi interface.
1389 '''
1390 if self.db.journaltag is None:
1391 raise DatabaseError, 'Database open read-only'
1392 self.db.destroynode(self.classname, nodeid)
1394 def history(self, nodeid):
1395 '''Retrieve the journal of edits on a particular node.
1397 'nodeid' must be the id of an existing node of this class or an
1398 IndexError is raised.
1400 The returned list contains tuples of the form
1402 (nodeid, date, tag, action, params)
1404 'date' is a Timestamp object specifying the time of the change and
1405 'tag' is the journaltag specified when the database was opened.
1406 '''
1407 if not self.do_journal:
1408 raise ValueError, 'Journalling is disabled for this class'
1409 return self.db.getjournal(self.classname, nodeid)
1411 # Locating nodes:
1412 def hasnode(self, nodeid):
1413 '''Determine if the given nodeid actually exists
1414 '''
1415 return self.db.hasnode(self.classname, nodeid)
1417 def setkey(self, propname):
1418 '''Select a String property of this class to be the key property.
1420 'propname' must be the name of a String property of this class or
1421 None, or a TypeError is raised. The values of the key property on
1422 all existing nodes must be unique or a ValueError is raised. If the
1423 property doesn't exist, KeyError is raised.
1424 '''
1425 prop = self.getprops()[propname]
1426 if not isinstance(prop, String):
1427 raise TypeError, 'key properties must be String'
1428 self.key = propname
1430 def getkey(self):
1431 '''Return the name of the key property for this class or None.'''
1432 return self.key
1434 def labelprop(self, default_to_id=0):
1435 ''' Return the property name for a label for the given node.
1437 This method attempts to generate a consistent label for the node.
1438 It tries the following in order:
1439 1. key property
1440 2. "name" property
1441 3. "title" property
1442 4. first property from the sorted property name list
1443 '''
1444 k = self.getkey()
1445 if k:
1446 return k
1447 props = self.getprops()
1448 if props.has_key('name'):
1449 return 'name'
1450 elif props.has_key('title'):
1451 return 'title'
1452 if default_to_id:
1453 return 'id'
1454 props = props.keys()
1455 props.sort()
1456 return props[0]
1458 # TODO: set up a separate index db file for this? profile?
1459 def lookup(self, keyvalue):
1460 '''Locate a particular node by its key property and return its id.
1462 If this class has no key property, a TypeError is raised. If the
1463 'keyvalue' matches one of the values for the key property among
1464 the nodes in this class, the matching node's id is returned;
1465 otherwise a KeyError is raised.
1466 '''
1467 if not self.key:
1468 raise TypeError, 'No key property set for class %s'%self.classname
1469 cldb = self.db.getclassdb(self.classname)
1470 try:
1471 for nodeid in self.getnodeids(cldb):
1472 node = self.db.getnode(self.classname, nodeid, cldb)
1473 if node.has_key(self.db.RETIRED_FLAG):
1474 continue
1475 if node[self.key] == keyvalue:
1476 return nodeid
1477 finally:
1478 cldb.close()
1479 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1480 keyvalue, self.classname)
1482 # change from spec - allows multiple props to match
1483 def find(self, **propspec):
1484 '''Get the ids of items in this class which link to the given items.
1486 'propspec' consists of keyword args propname=itemid or
1487 propname={itemid:1, }
1488 'propname' must be the name of a property in this class, or a
1489 KeyError is raised. That property must be a Link or
1490 Multilink property, or a TypeError is raised.
1492 Any item in this class whose 'propname' property links to any of the
1493 itemids will be returned. Used by the full text indexing, which knows
1494 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1495 issues:
1497 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1498 '''
1499 propspec = propspec.items()
1500 for propname, itemids in propspec:
1501 # check the prop is OK
1502 prop = self.properties[propname]
1503 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1504 raise TypeError, "'%s' not a Link/Multilink property"%propname
1506 # ok, now do the find
1507 cldb = self.db.getclassdb(self.classname)
1508 l = []
1509 try:
1510 for id in self.getnodeids(db=cldb):
1511 item = self.db.getnode(self.classname, id, db=cldb)
1512 if item.has_key(self.db.RETIRED_FLAG):
1513 continue
1514 for propname, itemids in propspec:
1515 # can't test if the item doesn't have this property
1516 if not item.has_key(propname):
1517 continue
1518 if type(itemids) is not type({}):
1519 itemids = {itemids:1}
1521 # grab the property definition and its value on this item
1522 prop = self.properties[propname]
1523 value = item[propname]
1524 if isinstance(prop, Link) and itemids.has_key(value):
1525 l.append(id)
1526 break
1527 elif isinstance(prop, Multilink):
1528 hit = 0
1529 for v in value:
1530 if itemids.has_key(v):
1531 l.append(id)
1532 hit = 1
1533 break
1534 if hit:
1535 break
1536 finally:
1537 cldb.close()
1538 return l
1540 def stringFind(self, **requirements):
1541 '''Locate a particular node by matching a set of its String
1542 properties in a caseless search.
1544 If the property is not a String property, a TypeError is raised.
1546 The return is a list of the id of all nodes that match.
1547 '''
1548 for propname in requirements.keys():
1549 prop = self.properties[propname]
1550 if isinstance(not prop, String):
1551 raise TypeError, "'%s' not a String property"%propname
1552 requirements[propname] = requirements[propname].lower()
1553 l = []
1554 cldb = self.db.getclassdb(self.classname)
1555 try:
1556 for nodeid in self.getnodeids(cldb):
1557 node = self.db.getnode(self.classname, nodeid, cldb)
1558 if node.has_key(self.db.RETIRED_FLAG):
1559 continue
1560 for key, value in requirements.items():
1561 if not node.has_key(key):
1562 break
1563 if node[key] is None or node[key].lower() != value:
1564 break
1565 else:
1566 l.append(nodeid)
1567 finally:
1568 cldb.close()
1569 return l
1571 def list(self):
1572 ''' Return a list of the ids of the active nodes in this class.
1573 '''
1574 l = []
1575 cn = self.classname
1576 cldb = self.db.getclassdb(cn)
1577 try:
1578 for nodeid in self.getnodeids(cldb):
1579 node = self.db.getnode(cn, nodeid, cldb)
1580 if node.has_key(self.db.RETIRED_FLAG):
1581 continue
1582 l.append(nodeid)
1583 finally:
1584 cldb.close()
1585 l.sort()
1586 return l
1588 def getnodeids(self, db=None):
1589 ''' Return a list of ALL nodeids
1590 '''
1591 if __debug__:
1592 print >>hyperdb.DEBUG, 'getnodeids', (self, self.classname, db)
1594 res = []
1596 # start off with the new nodes
1597 if self.db.newnodes.has_key(self.classname):
1598 res += self.db.newnodes[self.classname].keys()
1600 if db is None:
1601 db = self.db.getclassdb(self.classname)
1602 res = res + db.keys()
1604 # remove the uncommitted, destroyed nodes
1605 if self.db.destroyednodes.has_key(self.classname):
1606 for nodeid in self.db.destroyednodes[self.classname].keys():
1607 if db.has_key(nodeid):
1608 res.remove(nodeid)
1610 return res
1612 def filter(self, search_matches, filterspec, sort=(None,None),
1613 group=(None,None), num_re = re.compile('^\d+$')):
1614 ''' Return a list of the ids of the active nodes in this class that
1615 match the 'filter' spec, sorted by the group spec and then the
1616 sort spec.
1618 "filterspec" is {propname: value(s)}
1619 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1620 and prop is a prop name or None
1621 "search_matches" is {nodeid: marker}
1623 The filter must match all properties specificed - but if the
1624 property value to match is a list, any one of the values in the
1625 list may match for that property to match. Unless the property
1626 is a Multilink, in which case the item's property list must
1627 match the filterspec list.
1628 '''
1629 cn = self.classname
1631 # optimise filterspec
1632 l = []
1633 props = self.getprops()
1634 LINK = 0
1635 MULTILINK = 1
1636 STRING = 2
1637 DATE = 3
1638 INTERVAL = 4
1639 OTHER = 6
1641 timezone = self.db.getUserTimezone()
1642 for k, v in filterspec.items():
1643 propclass = props[k]
1644 if isinstance(propclass, Link):
1645 if type(v) is not type([]):
1646 v = [v]
1647 # replace key values with node ids
1648 u = []
1649 link_class = self.db.classes[propclass.classname]
1650 for entry in v:
1651 # the value -1 is a special "not set" sentinel
1652 if entry == '-1':
1653 entry = None
1654 elif not num_re.match(entry):
1655 try:
1656 entry = link_class.lookup(entry)
1657 except (TypeError,KeyError):
1658 raise ValueError, 'property "%s": %s not a %s'%(
1659 k, entry, self.properties[k].classname)
1660 u.append(entry)
1662 l.append((LINK, k, u))
1663 elif isinstance(propclass, Multilink):
1664 # the value -1 is a special "not set" sentinel
1665 if v in ('-1', ['-1']):
1666 v = []
1667 elif type(v) is not type([]):
1668 v = [v]
1670 # replace key values with node ids
1671 u = []
1672 link_class = self.db.classes[propclass.classname]
1673 for entry in v:
1674 if not num_re.match(entry):
1675 try:
1676 entry = link_class.lookup(entry)
1677 except (TypeError,KeyError):
1678 raise ValueError, 'new property "%s": %s not a %s'%(
1679 k, entry, self.properties[k].classname)
1680 u.append(entry)
1681 u.sort()
1682 l.append((MULTILINK, k, u))
1683 elif isinstance(propclass, String) and k != 'id':
1684 if type(v) is not type([]):
1685 v = [v]
1686 m = []
1687 for v in v:
1688 # simple glob searching
1689 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1690 v = v.replace('?', '.')
1691 v = v.replace('*', '.*?')
1692 m.append(v)
1693 m = re.compile('(%s)'%('|'.join(m)), re.I)
1694 l.append((STRING, k, m))
1695 elif isinstance(propclass, Date):
1696 try:
1697 date_rng = Range(v, date.Date, offset=timezone)
1698 l.append((DATE, k, date_rng))
1699 except ValueError:
1700 # If range creation fails - ignore that search parameter
1701 pass
1702 elif isinstance(propclass, Interval):
1703 try:
1704 intv_rng = Range(v, date.Interval)
1705 l.append((INTERVAL, k, intv_rng))
1706 except ValueError:
1707 # If range creation fails - ignore that search parameter
1708 pass
1710 elif isinstance(propclass, Boolean):
1711 if type(v) is type(''):
1712 bv = v.lower() in ('yes', 'true', 'on', '1')
1713 else:
1714 bv = v
1715 l.append((OTHER, k, bv))
1716 elif isinstance(propclass, Number):
1717 l.append((OTHER, k, int(v)))
1718 else:
1719 l.append((OTHER, k, v))
1720 filterspec = l
1722 # now, find all the nodes that are active and pass filtering
1723 l = []
1724 cldb = self.db.getclassdb(cn)
1725 try:
1726 # TODO: only full-scan once (use items())
1727 for nodeid in self.getnodeids(cldb):
1728 node = self.db.getnode(cn, nodeid, cldb)
1729 if node.has_key(self.db.RETIRED_FLAG):
1730 continue
1731 # apply filter
1732 for t, k, v in filterspec:
1733 # handle the id prop
1734 if k == 'id' and v == nodeid:
1735 continue
1737 # make sure the node has the property
1738 if not node.has_key(k):
1739 # this node doesn't have this property, so reject it
1740 break
1742 # now apply the property filter
1743 if t == LINK:
1744 # link - if this node's property doesn't appear in the
1745 # filterspec's nodeid list, skip it
1746 if node[k] not in v:
1747 break
1748 elif t == MULTILINK:
1749 # multilink - if any of the nodeids required by the
1750 # filterspec aren't in this node's property, then skip
1751 # it
1752 have = node[k]
1753 # check for matching the absence of multilink values
1754 if not v and have:
1755 break
1757 # othewise, make sure this node has each of the
1758 # required values
1759 for want in v:
1760 if want not in have:
1761 break
1762 else:
1763 continue
1764 break
1765 elif t == STRING:
1766 if node[k] is None:
1767 break
1768 # RE search
1769 if not v.search(node[k]):
1770 break
1771 elif t == DATE or t == INTERVAL:
1772 if node[k] is None:
1773 break
1774 if v.to_value:
1775 if not (v.from_value <= node[k] and v.to_value >= node[k]):
1776 break
1777 else:
1778 if not (v.from_value <= node[k]):
1779 break
1780 elif t == OTHER:
1781 # straight value comparison for the other types
1782 if node[k] != v:
1783 break
1784 else:
1785 l.append((nodeid, node))
1786 finally:
1787 cldb.close()
1788 l.sort()
1790 # filter based on full text search
1791 if search_matches is not None:
1792 k = []
1793 for v in l:
1794 if search_matches.has_key(v[0]):
1795 k.append(v)
1796 l = k
1798 # now, sort the result
1799 def sortfun(a, b, sort=sort, group=group, properties=self.getprops(),
1800 db = self.db, cl=self):
1801 a_id, an = a
1802 b_id, bn = b
1803 # sort by group and then sort
1804 for dir, prop in group, sort:
1805 if dir is None or prop is None: continue
1807 # sorting is class-specific
1808 propclass = properties[prop]
1810 # handle the properties that might be "faked"
1811 # also, handle possible missing properties
1812 try:
1813 if not an.has_key(prop):
1814 an[prop] = cl.get(a_id, prop)
1815 av = an[prop]
1816 except KeyError:
1817 # the node doesn't have a value for this property
1818 if isinstance(propclass, Multilink): av = []
1819 else: av = ''
1820 try:
1821 if not bn.has_key(prop):
1822 bn[prop] = cl.get(b_id, prop)
1823 bv = bn[prop]
1824 except KeyError:
1825 # the node doesn't have a value for this property
1826 if isinstance(propclass, Multilink): bv = []
1827 else: bv = ''
1829 # String and Date values are sorted in the natural way
1830 if isinstance(propclass, String):
1831 # clean up the strings
1832 if av and av[0] in string.uppercase:
1833 av = av.lower()
1834 if bv and bv[0] in string.uppercase:
1835 bv = bv.lower()
1836 if (isinstance(propclass, String) or
1837 isinstance(propclass, Date)):
1838 # it might be a string that's really an integer
1839 try:
1840 av = int(av)
1841 bv = int(bv)
1842 except:
1843 pass
1844 if dir == '+':
1845 r = cmp(av, bv)
1846 if r != 0: return r
1847 elif dir == '-':
1848 r = cmp(bv, av)
1849 if r != 0: return r
1851 # Link properties are sorted according to the value of
1852 # the "order" property on the linked nodes if it is
1853 # present; or otherwise on the key string of the linked
1854 # nodes; or finally on the node ids.
1855 elif isinstance(propclass, Link):
1856 link = db.classes[propclass.classname]
1857 if av is None and bv is not None: return -1
1858 if av is not None and bv is None: return 1
1859 if av is None and bv is None: continue
1860 if link.getprops().has_key('order'):
1861 if dir == '+':
1862 r = cmp(link.get(av, 'order'),
1863 link.get(bv, 'order'))
1864 if r != 0: return r
1865 elif dir == '-':
1866 r = cmp(link.get(bv, 'order'),
1867 link.get(av, 'order'))
1868 if r != 0: return r
1869 elif link.getkey():
1870 key = link.getkey()
1871 if dir == '+':
1872 r = cmp(link.get(av, key), link.get(bv, key))
1873 if r != 0: return r
1874 elif dir == '-':
1875 r = cmp(link.get(bv, key), link.get(av, key))
1876 if r != 0: return r
1877 else:
1878 if dir == '+':
1879 r = cmp(av, bv)
1880 if r != 0: return r
1881 elif dir == '-':
1882 r = cmp(bv, av)
1883 if r != 0: return r
1885 else:
1886 # all other types just compare
1887 if dir == '+':
1888 r = cmp(av, bv)
1889 elif dir == '-':
1890 r = cmp(bv, av)
1891 if r != 0: return r
1893 # end for dir, prop in sort, group:
1894 # if all else fails, compare the ids
1895 return cmp(a[0], b[0])
1897 l.sort(sortfun)
1898 return [i[0] for i in l]
1900 def count(self):
1901 '''Get the number of nodes in this class.
1903 If the returned integer is 'numnodes', the ids of all the nodes
1904 in this class run from 1 to numnodes, and numnodes+1 will be the
1905 id of the next node to be created in this class.
1906 '''
1907 return self.db.countnodes(self.classname)
1909 # Manipulating properties:
1911 def getprops(self, protected=1):
1912 '''Return a dictionary mapping property names to property objects.
1913 If the "protected" flag is true, we include protected properties -
1914 those which may not be modified.
1916 In addition to the actual properties on the node, these
1917 methods provide the "creation" and "activity" properties. If the
1918 "protected" flag is true, we include protected properties - those
1919 which may not be modified.
1920 '''
1921 d = self.properties.copy()
1922 if protected:
1923 d['id'] = String()
1924 d['creation'] = hyperdb.Date()
1925 d['activity'] = hyperdb.Date()
1926 d['creator'] = hyperdb.Link('user')
1927 return d
1929 def addprop(self, **properties):
1930 '''Add properties to this class.
1932 The keyword arguments in 'properties' must map names to property
1933 objects, or a TypeError is raised. None of the keys in 'properties'
1934 may collide with the names of existing properties, or a ValueError
1935 is raised before any properties have been added.
1936 '''
1937 for key in properties.keys():
1938 if self.properties.has_key(key):
1939 raise ValueError, key
1940 self.properties.update(properties)
1942 def index(self, nodeid):
1943 '''Add (or refresh) the node to search indexes
1944 '''
1945 # find all the String properties that have indexme
1946 for prop, propclass in self.getprops().items():
1947 if isinstance(propclass, String) and propclass.indexme:
1948 try:
1949 value = str(self.get(nodeid, prop))
1950 except IndexError:
1951 # node no longer exists - entry should be removed
1952 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1953 else:
1954 # and index them under (classname, nodeid, property)
1955 self.db.indexer.add_text((self.classname, nodeid, prop),
1956 value)
1958 #
1959 # Detector interface
1960 #
1961 def audit(self, event, detector):
1962 '''Register a detector
1963 '''
1964 l = self.auditors[event]
1965 if detector not in l:
1966 self.auditors[event].append(detector)
1968 def fireAuditors(self, action, nodeid, newvalues):
1969 '''Fire all registered auditors.
1970 '''
1971 for audit in self.auditors[action]:
1972 audit(self.db, self, nodeid, newvalues)
1974 def react(self, event, detector):
1975 '''Register a detector
1976 '''
1977 l = self.reactors[event]
1978 if detector not in l:
1979 self.reactors[event].append(detector)
1981 def fireReactors(self, action, nodeid, oldvalues):
1982 '''Fire all registered reactors.
1983 '''
1984 for react in self.reactors[action]:
1985 react(self.db, self, nodeid, oldvalues)
1987 class FileClass(Class, hyperdb.FileClass):
1988 '''This class defines a large chunk of data. To support this, it has a
1989 mandatory String property "content" which is typically saved off
1990 externally to the hyperdb.
1992 The default MIME type of this data is defined by the
1993 "default_mime_type" class attribute, which may be overridden by each
1994 node if the class defines a "type" String property.
1995 '''
1996 default_mime_type = 'text/plain'
1998 def create(self, **propvalues):
1999 ''' Snarf the "content" propvalue and store in a file
2000 '''
2001 # we need to fire the auditors now, or the content property won't
2002 # be in propvalues for the auditors to play with
2003 self.fireAuditors('create', None, propvalues)
2005 # now remove the content property so it's not stored in the db
2006 content = propvalues['content']
2007 del propvalues['content']
2009 # do the database create
2010 newid = Class.create_inner(self, **propvalues)
2012 # fire reactors
2013 self.fireReactors('create', newid, None)
2015 # store off the content as a file
2016 self.db.storefile(self.classname, newid, None, content)
2017 return newid
2019 def import_list(self, propnames, proplist):
2020 ''' Trap the "content" property...
2021 '''
2022 # dupe this list so we don't affect others
2023 propnames = propnames[:]
2025 # extract the "content" property from the proplist
2026 i = propnames.index('content')
2027 content = eval(proplist[i])
2028 del propnames[i]
2029 del proplist[i]
2031 # do the normal import
2032 newid = Class.import_list(self, propnames, proplist)
2034 # save off the "content" file
2035 self.db.storefile(self.classname, newid, None, content)
2036 return newid
2038 def get(self, nodeid, propname, default=_marker, cache=1):
2039 ''' Trap the content propname and get it from the file
2041 'cache' exists for backwards compatibility, and is not used.
2042 '''
2043 poss_msg = 'Possibly an access right configuration problem.'
2044 if propname == 'content':
2045 try:
2046 return self.db.getfile(self.classname, nodeid, None)
2047 except IOError, (strerror):
2048 # XXX by catching this we donot see an error in the log.
2049 return 'ERROR reading file: %s%s\n%s\n%s'%(
2050 self.classname, nodeid, poss_msg, strerror)
2051 if default is not _marker:
2052 return Class.get(self, nodeid, propname, default)
2053 else:
2054 return Class.get(self, nodeid, propname)
2056 def getprops(self, protected=1):
2057 ''' In addition to the actual properties on the node, these methods
2058 provide the "content" property. If the "protected" flag is true,
2059 we include protected properties - those which may not be
2060 modified.
2061 '''
2062 d = Class.getprops(self, protected=protected).copy()
2063 d['content'] = hyperdb.String()
2064 return d
2066 def index(self, nodeid):
2067 ''' Index the node in the search index.
2069 We want to index the content in addition to the normal String
2070 property indexing.
2071 '''
2072 # perform normal indexing
2073 Class.index(self, nodeid)
2075 # get the content to index
2076 content = self.get(nodeid, 'content')
2078 # figure the mime type
2079 if self.properties.has_key('type'):
2080 mime_type = self.get(nodeid, 'type')
2081 else:
2082 mime_type = self.default_mime_type
2084 # and index!
2085 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
2086 mime_type)
2088 # deviation from spec - was called ItemClass
2089 class IssueClass(Class, roundupdb.IssueClass):
2090 # Overridden methods:
2091 def __init__(self, db, classname, **properties):
2092 '''The newly-created class automatically includes the "messages",
2093 "files", "nosy", and "superseder" properties. If the 'properties'
2094 dictionary attempts to specify any of these properties or a
2095 "creation" or "activity" property, a ValueError is raised.
2096 '''
2097 if not properties.has_key('title'):
2098 properties['title'] = hyperdb.String(indexme='yes')
2099 if not properties.has_key('messages'):
2100 properties['messages'] = hyperdb.Multilink("msg")
2101 if not properties.has_key('files'):
2102 properties['files'] = hyperdb.Multilink("file")
2103 if not properties.has_key('nosy'):
2104 # note: journalling is turned off as it really just wastes
2105 # space. this behaviour may be overridden in an instance
2106 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2107 if not properties.has_key('superseder'):
2108 properties['superseder'] = hyperdb.Multilink(classname)
2109 Class.__init__(self, db, classname, **properties)
2111 #