1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: back_anydbm.py,v 1.118 2003-03-26 11:19:28 richard Exp $
19 '''
20 This module defines a backend that saves the hyperdatabase in a database
21 chosen by anydbm. It is guaranteed to always be available in python
22 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
23 serious bugs, and is not available)
24 '''
26 import whichdb, anydbm, os, marshal, re, weakref, string, copy
27 from roundup import hyperdb, date, password, roundupdb, security
28 from blobfiles import FileStorage
29 from sessions import Sessions, OneTimeKeys
30 from roundup.indexer import Indexer
31 from roundup.backends import locking
32 from roundup.hyperdb import String, Password, Date, Interval, Link, \
33 Multilink, DatabaseError, Boolean, Number, Node
34 from roundup.date import Range
36 #
37 # Now the database
38 #
39 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
40 '''A database for storing records containing flexible data types.
42 Transaction stuff TODO:
43 . check the timestamp of the class file and nuke the cache if it's
44 modified. Do some sort of conflict checking on the dirty stuff.
45 . perhaps detect write collisions (related to above)?
47 '''
48 def __init__(self, config, journaltag=None):
49 '''Open a hyperdatabase given a specifier to some storage.
51 The 'storagelocator' is obtained from config.DATABASE.
52 The meaning of 'storagelocator' depends on the particular
53 implementation of the hyperdatabase. It could be a file name,
54 a directory path, a socket descriptor for a connection to a
55 database over the network, etc.
57 The 'journaltag' is a token that will be attached to the journal
58 entries for any edits done on the database. If 'journaltag' is
59 None, the database is opened in read-only mode: the Class.create(),
60 Class.set(), Class.retire(), and Class.restore() methods are
61 disabled.
62 '''
63 self.config, self.journaltag = config, journaltag
64 self.dir = config.DATABASE
65 self.classes = {}
66 self.cache = {} # cache of nodes loaded or created
67 self.dirtynodes = {} # keep track of the dirty nodes by class
68 self.newnodes = {} # keep track of the new nodes by class
69 self.destroyednodes = {}# keep track of the destroyed nodes by class
70 self.transactions = []
71 self.indexer = Indexer(self.dir)
72 self.sessions = Sessions(self.config)
73 self.otks = OneTimeKeys(self.config)
74 self.security = security.Security(self)
75 # ensure files are group readable and writable
76 os.umask(0002)
78 # lock it
79 lockfilenm = os.path.join(self.dir, 'lock')
80 self.lockfile = locking.acquire_lock(lockfilenm)
81 self.lockfile.write(str(os.getpid()))
82 self.lockfile.flush()
84 def post_init(self):
85 ''' Called once the schema initialisation has finished.
86 '''
87 # reindex the db if necessary
88 if self.indexer.should_reindex():
89 self.reindex()
91 # figure the "curuserid"
92 if self.journaltag is None:
93 self.curuserid = None
94 elif self.journaltag == 'admin':
95 # admin user may not exist, but always has ID 1
96 self.curuserid = '1'
97 else:
98 self.curuserid = self.user.lookup(self.journaltag)
100 def reindex(self):
101 for klass in self.classes.values():
102 for nodeid in klass.list():
103 klass.index(nodeid)
104 self.indexer.save_index()
106 def __repr__(self):
107 return '<back_anydbm instance at %x>'%id(self)
109 #
110 # Classes
111 #
112 def __getattr__(self, classname):
113 '''A convenient way of calling self.getclass(classname).'''
114 if self.classes.has_key(classname):
115 if __debug__:
116 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
117 return self.classes[classname]
118 raise AttributeError, classname
120 def addclass(self, cl):
121 if __debug__:
122 print >>hyperdb.DEBUG, 'addclass', (self, cl)
123 cn = cl.classname
124 if self.classes.has_key(cn):
125 raise ValueError, cn
126 self.classes[cn] = cl
128 def getclasses(self):
129 '''Return a list of the names of all existing classes.'''
130 if __debug__:
131 print >>hyperdb.DEBUG, 'getclasses', (self,)
132 l = self.classes.keys()
133 l.sort()
134 return l
136 def getclass(self, classname):
137 '''Get the Class object representing a particular class.
139 If 'classname' is not a valid class name, a KeyError is raised.
140 '''
141 if __debug__:
142 print >>hyperdb.DEBUG, 'getclass', (self, classname)
143 try:
144 return self.classes[classname]
145 except KeyError:
146 raise KeyError, 'There is no class called "%s"'%classname
148 #
149 # Class DBs
150 #
151 def clear(self):
152 '''Delete all database contents
153 '''
154 if __debug__:
155 print >>hyperdb.DEBUG, 'clear', (self,)
156 for cn in self.classes.keys():
157 for dummy in 'nodes', 'journals':
158 path = os.path.join(self.dir, 'journals.%s'%cn)
159 if os.path.exists(path):
160 os.remove(path)
161 elif os.path.exists(path+'.db'): # dbm appends .db
162 os.remove(path+'.db')
164 def getclassdb(self, classname, mode='r'):
165 ''' grab a connection to the class db that will be used for
166 multiple actions
167 '''
168 if __debug__:
169 print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode)
170 return self.opendb('nodes.%s'%classname, mode)
172 def determine_db_type(self, path):
173 ''' determine which DB wrote the class file
174 '''
175 db_type = ''
176 if os.path.exists(path):
177 db_type = whichdb.whichdb(path)
178 if not db_type:
179 raise DatabaseError, "Couldn't identify database type"
180 elif os.path.exists(path+'.db'):
181 # if the path ends in '.db', it's a dbm database, whether
182 # anydbm says it's dbhash or not!
183 db_type = 'dbm'
184 return db_type
186 def opendb(self, name, mode):
187 '''Low-level database opener that gets around anydbm/dbm
188 eccentricities.
189 '''
190 if __debug__:
191 print >>hyperdb.DEBUG, 'opendb', (self, name, mode)
193 # figure the class db type
194 path = os.path.join(os.getcwd(), self.dir, name)
195 db_type = self.determine_db_type(path)
197 # new database? let anydbm pick the best dbm
198 if not db_type:
199 if __debug__:
200 print >>hyperdb.DEBUG, "opendb anydbm.open(%r, 'c')"%path
201 return anydbm.open(path, 'c')
203 # open the database with the correct module
204 try:
205 dbm = __import__(db_type)
206 except ImportError:
207 raise DatabaseError, \
208 "Couldn't open database - the required module '%s'"\
209 " is not available"%db_type
210 if __debug__:
211 print >>hyperdb.DEBUG, "opendb %r.open(%r, %r)"%(db_type, path,
212 mode)
213 return dbm.open(path, mode)
215 #
216 # Node IDs
217 #
218 def newid(self, classname):
219 ''' Generate a new id for the given class
220 '''
221 # open the ids DB - create if if doesn't exist
222 db = self.opendb('_ids', 'c')
223 if db.has_key(classname):
224 newid = db[classname] = str(int(db[classname]) + 1)
225 else:
226 # the count() bit is transitional - older dbs won't start at 1
227 newid = str(self.getclass(classname).count()+1)
228 db[classname] = newid
229 db.close()
230 return newid
232 def setid(self, classname, setid):
233 ''' Set the id counter: used during import of database
234 '''
235 # open the ids DB - create if if doesn't exist
236 db = self.opendb('_ids', 'c')
237 db[classname] = str(setid)
238 db.close()
240 #
241 # Nodes
242 #
243 def addnode(self, classname, nodeid, node):
244 ''' add the specified node to its class's db
245 '''
246 if __debug__:
247 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
249 # we'll be supplied these props if we're doing an import
250 if not node.has_key('creator'):
251 # add in the "calculated" properties (dupe so we don't affect
252 # calling code's node assumptions)
253 node = node.copy()
254 node['creator'] = self.curuserid
255 node['creation'] = node['activity'] = date.Date()
257 self.newnodes.setdefault(classname, {})[nodeid] = 1
258 self.cache.setdefault(classname, {})[nodeid] = node
259 self.savenode(classname, nodeid, node)
261 def setnode(self, classname, nodeid, node):
262 ''' change the specified node
263 '''
264 if __debug__:
265 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
266 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
268 # update the activity time (dupe so we don't affect
269 # calling code's node assumptions)
270 node = node.copy()
271 node['activity'] = date.Date()
273 # can't set without having already loaded the node
274 self.cache[classname][nodeid] = node
275 self.savenode(classname, nodeid, node)
277 def savenode(self, classname, nodeid, node):
278 ''' perform the saving of data specified by the set/addnode
279 '''
280 if __debug__:
281 print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node)
282 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
284 def getnode(self, classname, nodeid, db=None, cache=1):
285 ''' get a node from the database
286 '''
287 if __debug__:
288 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid, db)
289 if cache:
290 # try the cache
291 cache_dict = self.cache.setdefault(classname, {})
292 if cache_dict.has_key(nodeid):
293 if __debug__:
294 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
295 nodeid)
296 return cache_dict[nodeid]
298 if __debug__:
299 print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid)
301 # get from the database and save in the cache
302 if db is None:
303 db = self.getclassdb(classname)
304 if not db.has_key(nodeid):
305 # try the cache - might be a brand-new node
306 cache_dict = self.cache.setdefault(classname, {})
307 if cache_dict.has_key(nodeid):
308 if __debug__:
309 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
310 nodeid)
311 return cache_dict[nodeid]
312 raise IndexError, "no such %s %s"%(classname, nodeid)
314 # check the uncommitted, destroyed nodes
315 if (self.destroyednodes.has_key(classname) and
316 self.destroyednodes[classname].has_key(nodeid)):
317 raise IndexError, "no such %s %s"%(classname, nodeid)
319 # decode
320 res = marshal.loads(db[nodeid])
322 # reverse the serialisation
323 res = self.unserialise(classname, res)
325 # store off in the cache dict
326 if cache:
327 cache_dict[nodeid] = res
329 return res
331 def destroynode(self, classname, nodeid):
332 '''Remove a node from the database. Called exclusively by the
333 destroy() method on Class.
334 '''
335 if __debug__:
336 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
338 # remove from cache and newnodes if it's there
339 if (self.cache.has_key(classname) and
340 self.cache[classname].has_key(nodeid)):
341 del self.cache[classname][nodeid]
342 if (self.newnodes.has_key(classname) and
343 self.newnodes[classname].has_key(nodeid)):
344 del self.newnodes[classname][nodeid]
346 # see if there's any obvious commit actions that we should get rid of
347 for entry in self.transactions[:]:
348 if entry[1][:2] == (classname, nodeid):
349 self.transactions.remove(entry)
351 # add to the destroyednodes map
352 self.destroyednodes.setdefault(classname, {})[nodeid] = 1
354 # add the destroy commit action
355 self.transactions.append((self.doDestroyNode, (classname, nodeid)))
357 def serialise(self, classname, node):
358 '''Copy the node contents, converting non-marshallable data into
359 marshallable data.
360 '''
361 if __debug__:
362 print >>hyperdb.DEBUG, 'serialise', classname, node
363 properties = self.getclass(classname).getprops()
364 d = {}
365 for k, v in node.items():
366 # if the property doesn't exist, or is the "retired" flag then
367 # it won't be in the properties dict
368 if not properties.has_key(k):
369 d[k] = v
370 continue
372 # get the property spec
373 prop = properties[k]
375 if isinstance(prop, Password) and v is not None:
376 d[k] = str(v)
377 elif isinstance(prop, Date) and v is not None:
378 d[k] = v.serialise()
379 elif isinstance(prop, Interval) and v is not None:
380 d[k] = v.serialise()
381 else:
382 d[k] = v
383 return d
385 def unserialise(self, classname, node):
386 '''Decode the marshalled node data
387 '''
388 if __debug__:
389 print >>hyperdb.DEBUG, 'unserialise', classname, node
390 properties = self.getclass(classname).getprops()
391 d = {}
392 for k, v in node.items():
393 # if the property doesn't exist, or is the "retired" flag then
394 # it won't be in the properties dict
395 if not properties.has_key(k):
396 d[k] = v
397 continue
399 # get the property spec
400 prop = properties[k]
402 if isinstance(prop, Date) and v is not None:
403 d[k] = date.Date(v)
404 elif isinstance(prop, Interval) and v is not None:
405 d[k] = date.Interval(v)
406 elif isinstance(prop, Password) and v is not None:
407 p = password.Password()
408 p.unpack(v)
409 d[k] = p
410 else:
411 d[k] = v
412 return d
414 def hasnode(self, classname, nodeid, db=None):
415 ''' determine if the database has a given node
416 '''
417 if __debug__:
418 print >>hyperdb.DEBUG, 'hasnode', (self, classname, nodeid, db)
420 # try the cache
421 cache = self.cache.setdefault(classname, {})
422 if cache.has_key(nodeid):
423 if __debug__:
424 print >>hyperdb.TRACE, 'has %s %s cached'%(classname, nodeid)
425 return 1
426 if __debug__:
427 print >>hyperdb.TRACE, 'has %s %s'%(classname, nodeid)
429 # not in the cache - check the database
430 if db is None:
431 db = self.getclassdb(classname)
432 res = db.has_key(nodeid)
433 return res
435 def countnodes(self, classname, db=None):
436 if __debug__:
437 print >>hyperdb.DEBUG, 'countnodes', (self, classname, db)
439 count = 0
441 # include the uncommitted nodes
442 if self.newnodes.has_key(classname):
443 count += len(self.newnodes[classname])
444 if self.destroyednodes.has_key(classname):
445 count -= len(self.destroyednodes[classname])
447 # and count those in the DB
448 if db is None:
449 db = self.getclassdb(classname)
450 count = count + len(db.keys())
451 return count
454 #
455 # Files - special node properties
456 # inherited from FileStorage
458 #
459 # Journal
460 #
461 def addjournal(self, classname, nodeid, action, params, creator=None,
462 creation=None):
463 ''' Journal the Action
464 'action' may be:
466 'create' or 'set' -- 'params' is a dictionary of property values
467 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
468 'retire' -- 'params' is None
469 '''
470 if __debug__:
471 print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid,
472 action, params, creator, creation)
473 self.transactions.append((self.doSaveJournal, (classname, nodeid,
474 action, params, creator, creation)))
476 def getjournal(self, classname, nodeid):
477 ''' get the journal for id
479 Raise IndexError if the node doesn't exist (as per history()'s
480 API)
481 '''
482 if __debug__:
483 print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid)
485 # our journal result
486 res = []
488 # add any journal entries for transactions not committed to the
489 # database
490 for method, args in self.transactions:
491 if method != self.doSaveJournal:
492 continue
493 (cache_classname, cache_nodeid, cache_action, cache_params,
494 cache_creator, cache_creation) = args
495 if cache_classname == classname and cache_nodeid == nodeid:
496 if not cache_creator:
497 cache_creator = self.curuserid
498 if not cache_creation:
499 cache_creation = date.Date()
500 res.append((cache_nodeid, cache_creation, cache_creator,
501 cache_action, cache_params))
503 # attempt to open the journal - in some rare cases, the journal may
504 # not exist
505 try:
506 db = self.opendb('journals.%s'%classname, 'r')
507 except anydbm.error, error:
508 if str(error) == "need 'c' or 'n' flag to open new db":
509 raise IndexError, 'no such %s %s'%(classname, nodeid)
510 elif error.args[0] != 2:
511 raise
512 raise IndexError, 'no such %s %s'%(classname, nodeid)
513 try:
514 journal = marshal.loads(db[nodeid])
515 except KeyError:
516 db.close()
517 if res:
518 # we have some unsaved journal entries, be happy!
519 return res
520 raise IndexError, 'no such %s %s'%(classname, nodeid)
521 db.close()
523 # add all the saved journal entries for this node
524 for nodeid, date_stamp, user, action, params in journal:
525 res.append((nodeid, date.Date(date_stamp), user, action, params))
526 return res
528 def pack(self, pack_before):
529 ''' Delete all journal entries except "create" before 'pack_before'.
530 '''
531 if __debug__:
532 print >>hyperdb.DEBUG, 'packjournal', (self, pack_before)
534 pack_before = pack_before.serialise()
535 for classname in self.getclasses():
536 # get the journal db
537 db_name = 'journals.%s'%classname
538 path = os.path.join(os.getcwd(), self.dir, classname)
539 db_type = self.determine_db_type(path)
540 db = self.opendb(db_name, 'w')
542 for key in db.keys():
543 # get the journal for this db entry
544 journal = marshal.loads(db[key])
545 l = []
546 last_set_entry = None
547 for entry in journal:
548 # unpack the entry
549 (nodeid, date_stamp, self.journaltag, action,
550 params) = entry
551 # if the entry is after the pack date, _or_ the initial
552 # create entry, then it stays
553 if date_stamp > pack_before or action == 'create':
554 l.append(entry)
555 db[key] = marshal.dumps(l)
556 if db_type == 'gdbm':
557 db.reorganize()
558 db.close()
561 #
562 # Basic transaction support
563 #
564 def commit(self):
565 ''' Commit the current transactions.
566 '''
567 if __debug__:
568 print >>hyperdb.DEBUG, 'commit', (self,)
570 # keep a handle to all the database files opened
571 self.databases = {}
573 # now, do all the transactions
574 reindex = {}
575 for method, args in self.transactions:
576 reindex[method(*args)] = 1
578 # now close all the database files
579 for db in self.databases.values():
580 db.close()
581 del self.databases
583 # reindex the nodes that request it
584 for classname, nodeid in filter(None, reindex.keys()):
585 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
586 self.getclass(classname).index(nodeid)
588 # save the indexer state
589 self.indexer.save_index()
591 self.clearCache()
593 def clearCache(self):
594 # all transactions committed, back to normal
595 self.cache = {}
596 self.dirtynodes = {}
597 self.newnodes = {}
598 self.destroyednodes = {}
599 self.transactions = []
601 def getCachedClassDB(self, classname):
602 ''' get the class db, looking in our cache of databases for commit
603 '''
604 # get the database handle
605 db_name = 'nodes.%s'%classname
606 if not self.databases.has_key(db_name):
607 self.databases[db_name] = self.getclassdb(classname, 'c')
608 return self.databases[db_name]
610 def doSaveNode(self, classname, nodeid, node):
611 if __debug__:
612 print >>hyperdb.DEBUG, 'doSaveNode', (self, classname, nodeid,
613 node)
615 db = self.getCachedClassDB(classname)
617 # now save the marshalled data
618 db[nodeid] = marshal.dumps(self.serialise(classname, node))
620 # return the classname, nodeid so we reindex this content
621 return (classname, nodeid)
623 def getCachedJournalDB(self, classname):
624 ''' get the journal db, looking in our cache of databases for commit
625 '''
626 # get the database handle
627 db_name = 'journals.%s'%classname
628 if not self.databases.has_key(db_name):
629 self.databases[db_name] = self.opendb(db_name, 'c')
630 return self.databases[db_name]
632 def doSaveJournal(self, classname, nodeid, action, params, creator,
633 creation):
634 # serialise the parameters now if necessary
635 if isinstance(params, type({})):
636 if action in ('set', 'create'):
637 params = self.serialise(classname, params)
639 # handle supply of the special journalling parameters (usually
640 # supplied on importing an existing database)
641 if creator:
642 journaltag = creator
643 else:
644 journaltag = self.curuserid
645 if creation:
646 journaldate = creation.serialise()
647 else:
648 journaldate = date.Date().serialise()
650 # create the journal entry
651 entry = (nodeid, journaldate, journaltag, action, params)
653 if __debug__:
654 print >>hyperdb.DEBUG, 'doSaveJournal', entry
656 db = self.getCachedJournalDB(classname)
658 # now insert the journal entry
659 if db.has_key(nodeid):
660 # append to existing
661 s = db[nodeid]
662 l = marshal.loads(s)
663 l.append(entry)
664 else:
665 l = [entry]
667 db[nodeid] = marshal.dumps(l)
669 def doDestroyNode(self, classname, nodeid):
670 if __debug__:
671 print >>hyperdb.DEBUG, 'doDestroyNode', (self, classname, nodeid)
673 # delete from the class database
674 db = self.getCachedClassDB(classname)
675 if db.has_key(nodeid):
676 del db[nodeid]
678 # delete from the database
679 db = self.getCachedJournalDB(classname)
680 if db.has_key(nodeid):
681 del db[nodeid]
683 # return the classname, nodeid so we reindex this content
684 return (classname, nodeid)
686 def rollback(self):
687 ''' Reverse all actions from the current transaction.
688 '''
689 if __debug__:
690 print >>hyperdb.DEBUG, 'rollback', (self, )
691 for method, args in self.transactions:
692 # delete temporary files
693 if method == self.doStoreFile:
694 self.rollbackStoreFile(*args)
695 self.cache = {}
696 self.dirtynodes = {}
697 self.newnodes = {}
698 self.destroyednodes = {}
699 self.transactions = []
701 def close(self):
702 ''' Nothing to do
703 '''
704 if self.lockfile is not None:
705 locking.release_lock(self.lockfile)
706 if self.lockfile is not None:
707 self.lockfile.close()
708 self.lockfile = None
710 _marker = []
711 class Class(hyperdb.Class):
712 '''The handle to a particular class of nodes in a hyperdatabase.'''
714 def __init__(self, db, classname, **properties):
715 '''Create a new class with a given name and property specification.
717 'classname' must not collide with the name of an existing class,
718 or a ValueError is raised. The keyword arguments in 'properties'
719 must map names to property objects, or a TypeError is raised.
720 '''
721 if (properties.has_key('creation') or properties.has_key('activity')
722 or properties.has_key('creator')):
723 raise ValueError, '"creation", "activity" and "creator" are '\
724 'reserved'
726 self.classname = classname
727 self.properties = properties
728 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
729 self.key = ''
731 # should we journal changes (default yes)
732 self.do_journal = 1
734 # do the db-related init stuff
735 db.addclass(self)
737 self.auditors = {'create': [], 'set': [], 'retire': [], 'restore': []}
738 self.reactors = {'create': [], 'set': [], 'retire': [], 'restore': []}
740 def enableJournalling(self):
741 '''Turn journalling on for this class
742 '''
743 self.do_journal = 1
745 def disableJournalling(self):
746 '''Turn journalling off for this class
747 '''
748 self.do_journal = 0
750 # Editing nodes:
752 def create(self, **propvalues):
753 '''Create a new node of this class and return its id.
755 The keyword arguments in 'propvalues' map property names to values.
757 The values of arguments must be acceptable for the types of their
758 corresponding properties or a TypeError is raised.
760 If this class has a key property, it must be present and its value
761 must not collide with other key strings or a ValueError is raised.
763 Any other properties on this class that are missing from the
764 'propvalues' dictionary are set to None.
766 If an id in a link or multilink property does not refer to a valid
767 node, an IndexError is raised.
769 These operations trigger detectors and can be vetoed. Attempts
770 to modify the "creation" or "activity" properties cause a KeyError.
771 '''
772 self.fireAuditors('create', None, propvalues)
773 newid = self.create_inner(**propvalues)
774 self.fireReactors('create', newid, None)
775 return newid
777 def create_inner(self, **propvalues):
778 ''' Called by create, in-between the audit and react calls.
779 '''
780 if propvalues.has_key('id'):
781 raise KeyError, '"id" is reserved'
783 if self.db.journaltag is None:
784 raise DatabaseError, 'Database open read-only'
786 if propvalues.has_key('creation') or propvalues.has_key('activity'):
787 raise KeyError, '"creation" and "activity" are reserved'
788 # new node's id
789 newid = self.db.newid(self.classname)
791 # validate propvalues
792 num_re = re.compile('^\d+$')
793 for key, value in propvalues.items():
794 if key == self.key:
795 try:
796 self.lookup(value)
797 except KeyError:
798 pass
799 else:
800 raise ValueError, 'node with key "%s" exists'%value
802 # try to handle this property
803 try:
804 prop = self.properties[key]
805 except KeyError:
806 raise KeyError, '"%s" has no property "%s"'%(self.classname,
807 key)
809 if value is not None and isinstance(prop, Link):
810 if type(value) != type(''):
811 raise ValueError, 'link value must be String'
812 link_class = self.properties[key].classname
813 # if it isn't a number, it's a key
814 if not num_re.match(value):
815 try:
816 value = self.db.classes[link_class].lookup(value)
817 except (TypeError, KeyError):
818 raise IndexError, 'new property "%s": %s not a %s'%(
819 key, value, link_class)
820 elif not self.db.getclass(link_class).hasnode(value):
821 raise IndexError, '%s has no node %s'%(link_class, value)
823 # save off the value
824 propvalues[key] = value
826 # register the link with the newly linked node
827 if self.do_journal and self.properties[key].do_journal:
828 self.db.addjournal(link_class, value, 'link',
829 (self.classname, newid, key))
831 elif isinstance(prop, Multilink):
832 if type(value) != type([]):
833 raise TypeError, 'new property "%s" not a list of ids'%key
835 # clean up and validate the list of links
836 link_class = self.properties[key].classname
837 l = []
838 for entry in value:
839 if type(entry) != type(''):
840 raise ValueError, '"%s" multilink value (%r) '\
841 'must contain Strings'%(key, value)
842 # if it isn't a number, it's a key
843 if not num_re.match(entry):
844 try:
845 entry = self.db.classes[link_class].lookup(entry)
846 except (TypeError, KeyError):
847 raise IndexError, 'new property "%s": %s not a %s'%(
848 key, entry, self.properties[key].classname)
849 l.append(entry)
850 value = l
851 propvalues[key] = value
853 # handle additions
854 for nodeid in value:
855 if not self.db.getclass(link_class).hasnode(nodeid):
856 raise IndexError, '%s has no node %s'%(link_class,
857 nodeid)
858 # register the link with the newly linked node
859 if self.do_journal and self.properties[key].do_journal:
860 self.db.addjournal(link_class, nodeid, 'link',
861 (self.classname, newid, key))
863 elif isinstance(prop, String):
864 if type(value) != type('') and type(value) != type(u''):
865 raise TypeError, 'new property "%s" not a string'%key
867 elif isinstance(prop, Password):
868 if not isinstance(value, password.Password):
869 raise TypeError, 'new property "%s" not a Password'%key
871 elif isinstance(prop, Date):
872 if value is not None and not isinstance(value, date.Date):
873 raise TypeError, 'new property "%s" not a Date'%key
875 elif isinstance(prop, Interval):
876 if value is not None and not isinstance(value, date.Interval):
877 raise TypeError, 'new property "%s" not an Interval'%key
879 elif value is not None and isinstance(prop, Number):
880 try:
881 float(value)
882 except ValueError:
883 raise TypeError, 'new property "%s" not numeric'%key
885 elif value is not None and isinstance(prop, Boolean):
886 try:
887 int(value)
888 except ValueError:
889 raise TypeError, 'new property "%s" not boolean'%key
891 # make sure there's data where there needs to be
892 for key, prop in self.properties.items():
893 if propvalues.has_key(key):
894 continue
895 if key == self.key:
896 raise ValueError, 'key property "%s" is required'%key
897 if isinstance(prop, Multilink):
898 propvalues[key] = []
899 else:
900 propvalues[key] = None
902 # done
903 self.db.addnode(self.classname, newid, propvalues)
904 if self.do_journal:
905 self.db.addjournal(self.classname, newid, 'create', {})
907 return newid
909 def export_list(self, propnames, nodeid):
910 ''' Export a node - generate a list of CSV-able data in the order
911 specified by propnames for the given node.
912 '''
913 properties = self.getprops()
914 l = []
915 for prop in propnames:
916 proptype = properties[prop]
917 value = self.get(nodeid, prop)
918 # "marshal" data where needed
919 if value is None:
920 pass
921 elif isinstance(proptype, hyperdb.Date):
922 value = value.get_tuple()
923 elif isinstance(proptype, hyperdb.Interval):
924 value = value.get_tuple()
925 elif isinstance(proptype, hyperdb.Password):
926 value = str(value)
927 l.append(repr(value))
929 # append retired flag
930 l.append(self.is_retired(nodeid))
932 return l
934 def import_list(self, propnames, proplist):
935 ''' Import a node - all information including "id" is present and
936 should not be sanity checked. Triggers are not triggered. The
937 journal should be initialised using the "creator" and "created"
938 information.
940 Return the nodeid of the node imported.
941 '''
942 if self.db.journaltag is None:
943 raise DatabaseError, 'Database open read-only'
944 properties = self.getprops()
946 # make the new node's property map
947 d = {}
948 newid = None
949 for i in range(len(propnames)):
950 # Figure the property for this column
951 propname = propnames[i]
953 # Use eval to reverse the repr() used to output the CSV
954 value = eval(proplist[i])
956 # "unmarshal" where necessary
957 if propname == 'id':
958 newid = value
959 continue
960 elif propname == 'is retired':
961 # is the item retired?
962 if int(value):
963 d[self.db.RETIRED_FLAG] = 1
964 continue
965 elif value is None:
966 # don't set Nones
967 continue
969 prop = properties[propname]
970 if isinstance(prop, hyperdb.Date):
971 value = date.Date(value)
972 elif isinstance(prop, hyperdb.Interval):
973 value = date.Interval(value)
974 elif isinstance(prop, hyperdb.Password):
975 pwd = password.Password()
976 pwd.unpack(value)
977 value = pwd
978 d[propname] = value
980 # get a new id if necessary
981 if newid is None:
982 newid = self.db.newid(self.classname)
984 # add the node and journal
985 self.db.addnode(self.classname, newid, d)
987 # extract the journalling stuff and nuke it
988 if d.has_key('creator'):
989 creator = d['creator']
990 del d['creator']
991 else:
992 creator = None
993 if d.has_key('creation'):
994 creation = d['creation']
995 del d['creation']
996 else:
997 creation = None
998 if d.has_key('activity'):
999 del d['activity']
1000 self.db.addjournal(self.classname, newid, 'create', {}, creator,
1001 creation)
1002 return newid
1004 def get(self, nodeid, propname, default=_marker, cache=1):
1005 '''Get the value of a property on an existing node of this class.
1007 'nodeid' must be the id of an existing node of this class or an
1008 IndexError is raised. 'propname' must be the name of a property
1009 of this class or a KeyError is raised.
1011 'cache' indicates whether the transaction cache should be queried
1012 for the node. If the node has been modified and you need to
1013 determine what its values prior to modification are, you need to
1014 set cache=0.
1016 Attempts to get the "creation" or "activity" properties should
1017 do the right thing.
1018 '''
1019 if propname == 'id':
1020 return nodeid
1022 # get the node's dict
1023 d = self.db.getnode(self.classname, nodeid, cache=cache)
1025 # check for one of the special props
1026 if propname == 'creation':
1027 if d.has_key('creation'):
1028 return d['creation']
1029 if not self.do_journal:
1030 raise ValueError, 'Journalling is disabled for this class'
1031 journal = self.db.getjournal(self.classname, nodeid)
1032 if journal:
1033 return self.db.getjournal(self.classname, nodeid)[0][1]
1034 else:
1035 # on the strange chance that there's no journal
1036 return date.Date()
1037 if propname == 'activity':
1038 if d.has_key('activity'):
1039 return d['activity']
1040 if not self.do_journal:
1041 raise ValueError, 'Journalling is disabled for this class'
1042 journal = self.db.getjournal(self.classname, nodeid)
1043 if journal:
1044 return self.db.getjournal(self.classname, nodeid)[-1][1]
1045 else:
1046 # on the strange chance that there's no journal
1047 return date.Date()
1048 if propname == 'creator':
1049 if d.has_key('creator'):
1050 return d['creator']
1051 if not self.do_journal:
1052 raise ValueError, 'Journalling is disabled for this class'
1053 journal = self.db.getjournal(self.classname, nodeid)
1054 if journal:
1055 num_re = re.compile('^\d+$')
1056 value = self.db.getjournal(self.classname, nodeid)[0][2]
1057 if num_re.match(value):
1058 return value
1059 else:
1060 # old-style "username" journal tag
1061 try:
1062 return self.db.user.lookup(value)
1063 except KeyError:
1064 # user's been retired, return admin
1065 return '1'
1066 else:
1067 return self.db.curuserid
1069 # get the property (raises KeyErorr if invalid)
1070 prop = self.properties[propname]
1072 if not d.has_key(propname):
1073 if default is _marker:
1074 if isinstance(prop, Multilink):
1075 return []
1076 else:
1077 return None
1078 else:
1079 return default
1081 # return a dupe of the list so code doesn't get confused
1082 if isinstance(prop, Multilink):
1083 return d[propname][:]
1085 return d[propname]
1087 # not in spec
1088 def getnode(self, nodeid, cache=1):
1089 ''' Return a convenience wrapper for the node.
1091 'nodeid' must be the id of an existing node of this class or an
1092 IndexError is raised.
1094 'cache' indicates whether the transaction cache should be queried
1095 for the node. If the node has been modified and you need to
1096 determine what its values prior to modification are, you need to
1097 set cache=0.
1098 '''
1099 return Node(self, nodeid, cache=cache)
1101 def set(self, nodeid, **propvalues):
1102 '''Modify a property on an existing node of this class.
1104 'nodeid' must be the id of an existing node of this class or an
1105 IndexError is raised.
1107 Each key in 'propvalues' must be the name of a property of this
1108 class or a KeyError is raised.
1110 All values in 'propvalues' must be acceptable types for their
1111 corresponding properties or a TypeError is raised.
1113 If the value of the key property is set, it must not collide with
1114 other key strings or a ValueError is raised.
1116 If the value of a Link or Multilink property contains an invalid
1117 node id, a ValueError is raised.
1119 These operations trigger detectors and can be vetoed. Attempts
1120 to modify the "creation" or "activity" properties cause a KeyError.
1121 '''
1122 if not propvalues:
1123 return propvalues
1125 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1126 raise KeyError, '"creation" and "activity" are reserved'
1128 if propvalues.has_key('id'):
1129 raise KeyError, '"id" is reserved'
1131 if self.db.journaltag is None:
1132 raise DatabaseError, 'Database open read-only'
1134 self.fireAuditors('set', nodeid, propvalues)
1135 # Take a copy of the node dict so that the subsequent set
1136 # operation doesn't modify the oldvalues structure.
1137 try:
1138 # try not using the cache initially
1139 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid,
1140 cache=0))
1141 except IndexError:
1142 # this will be needed if somone does a create() and set()
1143 # with no intervening commit()
1144 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1146 node = self.db.getnode(self.classname, nodeid)
1147 if node.has_key(self.db.RETIRED_FLAG):
1148 raise IndexError
1149 num_re = re.compile('^\d+$')
1151 # if the journal value is to be different, store it in here
1152 journalvalues = {}
1154 for propname, value in propvalues.items():
1155 # check to make sure we're not duplicating an existing key
1156 if propname == self.key and node[propname] != value:
1157 try:
1158 self.lookup(value)
1159 except KeyError:
1160 pass
1161 else:
1162 raise ValueError, 'node with key "%s" exists'%value
1164 # this will raise the KeyError if the property isn't valid
1165 # ... we don't use getprops() here because we only care about
1166 # the writeable properties.
1167 try:
1168 prop = self.properties[propname]
1169 except KeyError:
1170 raise KeyError, '"%s" has no property named "%s"'%(
1171 self.classname, propname)
1173 # if the value's the same as the existing value, no sense in
1174 # doing anything
1175 current = node.get(propname, None)
1176 if value == current:
1177 del propvalues[propname]
1178 continue
1179 journalvalues[propname] = current
1181 # do stuff based on the prop type
1182 if isinstance(prop, Link):
1183 link_class = prop.classname
1184 # if it isn't a number, it's a key
1185 if value is not None and not isinstance(value, type('')):
1186 raise ValueError, 'property "%s" link value be a string'%(
1187 propname)
1188 if isinstance(value, type('')) and not num_re.match(value):
1189 try:
1190 value = self.db.classes[link_class].lookup(value)
1191 except (TypeError, KeyError):
1192 raise IndexError, 'new property "%s": %s not a %s'%(
1193 propname, value, prop.classname)
1195 if (value is not None and
1196 not self.db.getclass(link_class).hasnode(value)):
1197 raise IndexError, '%s has no node %s'%(link_class, value)
1199 if self.do_journal and prop.do_journal:
1200 # register the unlink with the old linked node
1201 if node.has_key(propname) and node[propname] is not None:
1202 self.db.addjournal(link_class, node[propname], 'unlink',
1203 (self.classname, nodeid, propname))
1205 # register the link with the newly linked node
1206 if value is not None:
1207 self.db.addjournal(link_class, value, 'link',
1208 (self.classname, nodeid, propname))
1210 elif isinstance(prop, Multilink):
1211 if type(value) != type([]):
1212 raise TypeError, 'new property "%s" not a list of'\
1213 ' ids'%propname
1214 link_class = self.properties[propname].classname
1215 l = []
1216 for entry in value:
1217 # if it isn't a number, it's a key
1218 if type(entry) != type(''):
1219 raise ValueError, 'new property "%s" link value ' \
1220 'must be a string'%propname
1221 if not num_re.match(entry):
1222 try:
1223 entry = self.db.classes[link_class].lookup(entry)
1224 except (TypeError, KeyError):
1225 raise IndexError, 'new property "%s": %s not a %s'%(
1226 propname, entry,
1227 self.properties[propname].classname)
1228 l.append(entry)
1229 value = l
1230 propvalues[propname] = value
1232 # figure the journal entry for this property
1233 add = []
1234 remove = []
1236 # handle removals
1237 if node.has_key(propname):
1238 l = node[propname]
1239 else:
1240 l = []
1241 for id in l[:]:
1242 if id in value:
1243 continue
1244 # register the unlink with the old linked node
1245 if self.do_journal and self.properties[propname].do_journal:
1246 self.db.addjournal(link_class, id, 'unlink',
1247 (self.classname, nodeid, propname))
1248 l.remove(id)
1249 remove.append(id)
1251 # handle additions
1252 for id in value:
1253 if not self.db.getclass(link_class).hasnode(id):
1254 raise IndexError, '%s has no node %s'%(link_class, id)
1255 if id in l:
1256 continue
1257 # register the link with the newly linked node
1258 if self.do_journal and self.properties[propname].do_journal:
1259 self.db.addjournal(link_class, id, 'link',
1260 (self.classname, nodeid, propname))
1261 l.append(id)
1262 add.append(id)
1264 # figure the journal entry
1265 l = []
1266 if add:
1267 l.append(('+', add))
1268 if remove:
1269 l.append(('-', remove))
1270 if l:
1271 journalvalues[propname] = tuple(l)
1273 elif isinstance(prop, String):
1274 if value is not None and type(value) != type('') and type(value) != type(u''):
1275 raise TypeError, 'new property "%s" not a string'%propname
1277 elif isinstance(prop, Password):
1278 if not isinstance(value, password.Password):
1279 raise TypeError, 'new property "%s" not a Password'%propname
1280 propvalues[propname] = value
1282 elif value is not None and isinstance(prop, Date):
1283 if not isinstance(value, date.Date):
1284 raise TypeError, 'new property "%s" not a Date'% propname
1285 propvalues[propname] = value
1287 elif value is not None and isinstance(prop, Interval):
1288 if not isinstance(value, date.Interval):
1289 raise TypeError, 'new property "%s" not an '\
1290 'Interval'%propname
1291 propvalues[propname] = value
1293 elif value is not None and isinstance(prop, Number):
1294 try:
1295 float(value)
1296 except ValueError:
1297 raise TypeError, 'new property "%s" not numeric'%propname
1299 elif value is not None and isinstance(prop, Boolean):
1300 try:
1301 int(value)
1302 except ValueError:
1303 raise TypeError, 'new property "%s" not boolean'%propname
1305 node[propname] = value
1307 # nothing to do?
1308 if not propvalues:
1309 return propvalues
1311 # do the set, and journal it
1312 self.db.setnode(self.classname, nodeid, node)
1314 if self.do_journal:
1315 self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
1317 self.fireReactors('set', nodeid, oldvalues)
1319 return propvalues
1321 def retire(self, nodeid):
1322 '''Retire a node.
1324 The properties on the node remain available from the get() method,
1325 and the node's id is never reused.
1327 Retired nodes are not returned by the find(), list(), or lookup()
1328 methods, and other nodes may reuse the values of their key properties.
1330 These operations trigger detectors and can be vetoed. Attempts
1331 to modify the "creation" or "activity" properties cause a KeyError.
1332 '''
1333 if self.db.journaltag is None:
1334 raise DatabaseError, 'Database open read-only'
1336 self.fireAuditors('retire', nodeid, None)
1338 node = self.db.getnode(self.classname, nodeid)
1339 node[self.db.RETIRED_FLAG] = 1
1340 self.db.setnode(self.classname, nodeid, node)
1341 if self.do_journal:
1342 self.db.addjournal(self.classname, nodeid, 'retired', None)
1344 self.fireReactors('retire', nodeid, None)
1346 def restore(self, nodeid):
1347 '''Restpre a retired node.
1349 Make node available for all operations like it was before retirement.
1350 '''
1351 if self.db.journaltag is None:
1352 raise DatabaseError, 'Database open read-only'
1354 node = self.db.getnode(self.classname, nodeid)
1355 # check if key property was overrided
1356 key = self.getkey()
1357 try:
1358 id = self.lookup(node[key])
1359 except KeyError:
1360 pass
1361 else:
1362 raise KeyError, "Key property (%s) of retired node clashes with \
1363 existing one (%s)" % (key, node[key])
1364 # Now we can safely restore node
1365 self.fireAuditors('restore', nodeid, None)
1366 del node[self.db.RETIRED_FLAG]
1367 self.db.setnode(self.classname, nodeid, node)
1368 if self.do_journal:
1369 self.db.addjournal(self.classname, nodeid, 'restored', None)
1371 self.fireReactors('restore', nodeid, None)
1373 def is_retired(self, nodeid, cldb=None):
1374 '''Return true if the node is retired.
1375 '''
1376 node = self.db.getnode(self.classname, nodeid, cldb)
1377 if node.has_key(self.db.RETIRED_FLAG):
1378 return 1
1379 return 0
1381 def destroy(self, nodeid):
1382 '''Destroy a node.
1384 WARNING: this method should never be used except in extremely rare
1385 situations where there could never be links to the node being
1386 deleted
1387 WARNING: use retire() instead
1388 WARNING: the properties of this node will not be available ever again
1389 WARNING: really, use retire() instead
1391 Well, I think that's enough warnings. This method exists mostly to
1392 support the session storage of the cgi interface.
1393 '''
1394 if self.db.journaltag is None:
1395 raise DatabaseError, 'Database open read-only'
1396 self.db.destroynode(self.classname, nodeid)
1398 def history(self, nodeid):
1399 '''Retrieve the journal of edits on a particular node.
1401 'nodeid' must be the id of an existing node of this class or an
1402 IndexError is raised.
1404 The returned list contains tuples of the form
1406 (nodeid, date, tag, action, params)
1408 'date' is a Timestamp object specifying the time of the change and
1409 'tag' is the journaltag specified when the database was opened.
1410 '''
1411 if not self.do_journal:
1412 raise ValueError, 'Journalling is disabled for this class'
1413 return self.db.getjournal(self.classname, nodeid)
1415 # Locating nodes:
1416 def hasnode(self, nodeid):
1417 '''Determine if the given nodeid actually exists
1418 '''
1419 return self.db.hasnode(self.classname, nodeid)
1421 def setkey(self, propname):
1422 '''Select a String property of this class to be the key property.
1424 'propname' must be the name of a String property of this class or
1425 None, or a TypeError is raised. The values of the key property on
1426 all existing nodes must be unique or a ValueError is raised. If the
1427 property doesn't exist, KeyError is raised.
1428 '''
1429 prop = self.getprops()[propname]
1430 if not isinstance(prop, String):
1431 raise TypeError, 'key properties must be String'
1432 self.key = propname
1434 def getkey(self):
1435 '''Return the name of the key property for this class or None.'''
1436 return self.key
1438 def labelprop(self, default_to_id=0):
1439 ''' Return the property name for a label for the given node.
1441 This method attempts to generate a consistent label for the node.
1442 It tries the following in order:
1443 1. key property
1444 2. "name" property
1445 3. "title" property
1446 4. first property from the sorted property name list
1447 '''
1448 k = self.getkey()
1449 if k:
1450 return k
1451 props = self.getprops()
1452 if props.has_key('name'):
1453 return 'name'
1454 elif props.has_key('title'):
1455 return 'title'
1456 if default_to_id:
1457 return 'id'
1458 props = props.keys()
1459 props.sort()
1460 return props[0]
1462 # TODO: set up a separate index db file for this? profile?
1463 def lookup(self, keyvalue):
1464 '''Locate a particular node by its key property and return its id.
1466 If this class has no key property, a TypeError is raised. If the
1467 'keyvalue' matches one of the values for the key property among
1468 the nodes in this class, the matching node's id is returned;
1469 otherwise a KeyError is raised.
1470 '''
1471 if not self.key:
1472 raise TypeError, 'No key property set for class %s'%self.classname
1473 cldb = self.db.getclassdb(self.classname)
1474 try:
1475 for nodeid in self.getnodeids(cldb):
1476 node = self.db.getnode(self.classname, nodeid, cldb)
1477 if node.has_key(self.db.RETIRED_FLAG):
1478 continue
1479 if node[self.key] == keyvalue:
1480 return nodeid
1481 finally:
1482 cldb.close()
1483 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1484 keyvalue, self.classname)
1486 # change from spec - allows multiple props to match
1487 def find(self, **propspec):
1488 '''Get the ids of items in this class which link to the given items.
1490 'propspec' consists of keyword args propname=itemid or
1491 propname={itemid:1, }
1492 'propname' must be the name of a property in this class, or a
1493 KeyError is raised. That property must be a Link or
1494 Multilink property, or a TypeError is raised.
1496 Any item in this class whose 'propname' property links to any of the
1497 itemids will be returned. Used by the full text indexing, which knows
1498 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1499 issues:
1501 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1502 '''
1503 propspec = propspec.items()
1504 for propname, itemids in propspec:
1505 # check the prop is OK
1506 prop = self.properties[propname]
1507 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1508 raise TypeError, "'%s' not a Link/Multilink property"%propname
1510 # ok, now do the find
1511 cldb = self.db.getclassdb(self.classname)
1512 l = []
1513 try:
1514 for id in self.getnodeids(db=cldb):
1515 item = self.db.getnode(self.classname, id, db=cldb)
1516 if item.has_key(self.db.RETIRED_FLAG):
1517 continue
1518 for propname, itemids in propspec:
1519 # can't test if the item doesn't have this property
1520 if not item.has_key(propname):
1521 continue
1522 if type(itemids) is not type({}):
1523 itemids = {itemids:1}
1525 # grab the property definition and its value on this item
1526 prop = self.properties[propname]
1527 value = item[propname]
1528 if isinstance(prop, Link) and itemids.has_key(value):
1529 l.append(id)
1530 break
1531 elif isinstance(prop, Multilink):
1532 hit = 0
1533 for v in value:
1534 if itemids.has_key(v):
1535 l.append(id)
1536 hit = 1
1537 break
1538 if hit:
1539 break
1540 finally:
1541 cldb.close()
1542 return l
1544 def stringFind(self, **requirements):
1545 '''Locate a particular node by matching a set of its String
1546 properties in a caseless search.
1548 If the property is not a String property, a TypeError is raised.
1550 The return is a list of the id of all nodes that match.
1551 '''
1552 for propname in requirements.keys():
1553 prop = self.properties[propname]
1554 if isinstance(not prop, String):
1555 raise TypeError, "'%s' not a String property"%propname
1556 requirements[propname] = requirements[propname].lower()
1557 l = []
1558 cldb = self.db.getclassdb(self.classname)
1559 try:
1560 for nodeid in self.getnodeids(cldb):
1561 node = self.db.getnode(self.classname, nodeid, cldb)
1562 if node.has_key(self.db.RETIRED_FLAG):
1563 continue
1564 for key, value in requirements.items():
1565 if not node.has_key(key):
1566 break
1567 if node[key] is None or node[key].lower() != value:
1568 break
1569 else:
1570 l.append(nodeid)
1571 finally:
1572 cldb.close()
1573 return l
1575 def list(self):
1576 ''' Return a list of the ids of the active nodes in this class.
1577 '''
1578 l = []
1579 cn = self.classname
1580 cldb = self.db.getclassdb(cn)
1581 try:
1582 for nodeid in self.getnodeids(cldb):
1583 node = self.db.getnode(cn, nodeid, cldb)
1584 if node.has_key(self.db.RETIRED_FLAG):
1585 continue
1586 l.append(nodeid)
1587 finally:
1588 cldb.close()
1589 l.sort()
1590 return l
1592 def getnodeids(self, db=None):
1593 ''' Return a list of ALL nodeids
1594 '''
1595 if __debug__:
1596 print >>hyperdb.DEBUG, 'getnodeids', (self, self.classname, db)
1598 res = []
1600 # start off with the new nodes
1601 if self.db.newnodes.has_key(self.classname):
1602 res += self.db.newnodes[self.classname].keys()
1604 if db is None:
1605 db = self.db.getclassdb(self.classname)
1606 res = res + db.keys()
1608 # remove the uncommitted, destroyed nodes
1609 if self.db.destroyednodes.has_key(self.classname):
1610 for nodeid in self.db.destroyednodes[self.classname].keys():
1611 if db.has_key(nodeid):
1612 res.remove(nodeid)
1614 return res
1616 def filter(self, search_matches, filterspec, sort=(None,None),
1617 group=(None,None), num_re = re.compile('^\d+$')):
1618 ''' Return a list of the ids of the active nodes in this class that
1619 match the 'filter' spec, sorted by the group spec and then the
1620 sort spec.
1622 "filterspec" is {propname: value(s)}
1623 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1624 and prop is a prop name or None
1625 "search_matches" is {nodeid: marker}
1627 The filter must match all properties specificed - but if the
1628 property value to match is a list, any one of the values in the
1629 list may match for that property to match. Unless the property
1630 is a Multilink, in which case the item's property list must
1631 match the filterspec list.
1632 '''
1633 cn = self.classname
1635 # optimise filterspec
1636 l = []
1637 props = self.getprops()
1638 LINK = 0
1639 MULTILINK = 1
1640 STRING = 2
1641 DATE = 3
1642 OTHER = 6
1644 timezone = self.db.getUserTimezone()
1645 for k, v in filterspec.items():
1646 propclass = props[k]
1647 if isinstance(propclass, Link):
1648 if type(v) is not type([]):
1649 v = [v]
1650 # replace key values with node ids
1651 u = []
1652 link_class = self.db.classes[propclass.classname]
1653 for entry in v:
1654 # the value -1 is a special "not set" sentinel
1655 if entry == '-1':
1656 entry = None
1657 elif not num_re.match(entry):
1658 try:
1659 entry = link_class.lookup(entry)
1660 except (TypeError,KeyError):
1661 raise ValueError, 'property "%s": %s not a %s'%(
1662 k, entry, self.properties[k].classname)
1663 u.append(entry)
1665 l.append((LINK, k, u))
1666 elif isinstance(propclass, Multilink):
1667 # the value -1 is a special "not set" sentinel
1668 if v in ('-1', ['-1']):
1669 v = []
1670 elif type(v) is not type([]):
1671 v = [v]
1673 # replace key values with node ids
1674 u = []
1675 link_class = self.db.classes[propclass.classname]
1676 for entry in v:
1677 if not num_re.match(entry):
1678 try:
1679 entry = link_class.lookup(entry)
1680 except (TypeError,KeyError):
1681 raise ValueError, 'new property "%s": %s not a %s'%(
1682 k, entry, self.properties[k].classname)
1683 u.append(entry)
1684 u.sort()
1685 l.append((MULTILINK, k, u))
1686 elif isinstance(propclass, String) and k != 'id':
1687 # simple glob searching
1688 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1689 v = v.replace('?', '.')
1690 v = v.replace('*', '.*?')
1691 l.append((STRING, k, re.compile(v, re.I)))
1692 elif isinstance(propclass, Date):
1693 try:
1694 date_rng = Range(v, date.Date, offset=timezone)
1695 l.append((DATE, k, date_rng))
1696 except ValueError:
1697 # If range creation fails - ignore that search parameter
1698 pass
1699 elif isinstance(propclass, Boolean):
1700 if type(v) is type(''):
1701 bv = v.lower() in ('yes', 'true', 'on', '1')
1702 else:
1703 bv = v
1704 l.append((OTHER, k, bv))
1705 # kedder: dates are filtered by ranges
1706 #elif isinstance(propclass, Date):
1707 # l.append((OTHER, k, date.Date(v)))
1708 elif isinstance(propclass, Interval):
1709 l.append((OTHER, k, date.Interval(v)))
1710 elif isinstance(propclass, Number):
1711 l.append((OTHER, k, int(v)))
1712 else:
1713 l.append((OTHER, k, v))
1714 filterspec = l
1716 # now, find all the nodes that are active and pass filtering
1717 l = []
1718 cldb = self.db.getclassdb(cn)
1719 try:
1720 # TODO: only full-scan once (use items())
1721 for nodeid in self.getnodeids(cldb):
1722 node = self.db.getnode(cn, nodeid, cldb)
1723 if node.has_key(self.db.RETIRED_FLAG):
1724 continue
1725 # apply filter
1726 for t, k, v in filterspec:
1727 # handle the id prop
1728 if k == 'id' and v == nodeid:
1729 continue
1731 # make sure the node has the property
1732 if not node.has_key(k):
1733 # this node doesn't have this property, so reject it
1734 break
1736 # now apply the property filter
1737 if t == LINK:
1738 # link - if this node's property doesn't appear in the
1739 # filterspec's nodeid list, skip it
1740 if node[k] not in v:
1741 break
1742 elif t == MULTILINK:
1743 # multilink - if any of the nodeids required by the
1744 # filterspec aren't in this node's property, then skip
1745 # it
1746 have = node[k]
1747 # check for matching the absence of multilink values
1748 if not v and have:
1749 break
1751 # othewise, make sure this node has each of the
1752 # required values
1753 for want in v:
1754 if want not in have:
1755 break
1756 else:
1757 continue
1758 break
1759 elif t == STRING:
1760 # RE search
1761 if node[k] is None or not v.search(node[k]):
1762 break
1763 elif t == DATE:
1764 if node[k] is None: break
1765 if v.to_value:
1766 if not (v.from_value < node[k] and v.to_value > node[k]):
1767 break
1768 else:
1769 if not (v.from_value < node[k]):
1770 break
1771 elif t == OTHER:
1772 # straight value comparison for the other types
1773 if node[k] != v:
1774 break
1775 else:
1776 l.append((nodeid, node))
1777 finally:
1778 cldb.close()
1779 l.sort()
1781 # filter based on full text search
1782 if search_matches is not None:
1783 k = []
1784 for v in l:
1785 if search_matches.has_key(v[0]):
1786 k.append(v)
1787 l = k
1789 # now, sort the result
1790 def sortfun(a, b, sort=sort, group=group, properties=self.getprops(),
1791 db = self.db, cl=self):
1792 a_id, an = a
1793 b_id, bn = b
1794 # sort by group and then sort
1795 for dir, prop in group, sort:
1796 if dir is None or prop is None: continue
1798 # sorting is class-specific
1799 propclass = properties[prop]
1801 # handle the properties that might be "faked"
1802 # also, handle possible missing properties
1803 try:
1804 if not an.has_key(prop):
1805 an[prop] = cl.get(a_id, prop)
1806 av = an[prop]
1807 except KeyError:
1808 # the node doesn't have a value for this property
1809 if isinstance(propclass, Multilink): av = []
1810 else: av = ''
1811 try:
1812 if not bn.has_key(prop):
1813 bn[prop] = cl.get(b_id, prop)
1814 bv = bn[prop]
1815 except KeyError:
1816 # the node doesn't have a value for this property
1817 if isinstance(propclass, Multilink): bv = []
1818 else: bv = ''
1820 # String and Date values are sorted in the natural way
1821 if isinstance(propclass, String):
1822 # clean up the strings
1823 if av and av[0] in string.uppercase:
1824 av = av.lower()
1825 if bv and bv[0] in string.uppercase:
1826 bv = bv.lower()
1827 if (isinstance(propclass, String) or
1828 isinstance(propclass, Date)):
1829 # it might be a string that's really an integer
1830 try:
1831 av = int(av)
1832 bv = int(bv)
1833 except:
1834 pass
1835 if dir == '+':
1836 r = cmp(av, bv)
1837 if r != 0: return r
1838 elif dir == '-':
1839 r = cmp(bv, av)
1840 if r != 0: return r
1842 # Link properties are sorted according to the value of
1843 # the "order" property on the linked nodes if it is
1844 # present; or otherwise on the key string of the linked
1845 # nodes; or finally on the node ids.
1846 elif isinstance(propclass, Link):
1847 link = db.classes[propclass.classname]
1848 if av is None and bv is not None: return -1
1849 if av is not None and bv is None: return 1
1850 if av is None and bv is None: continue
1851 if link.getprops().has_key('order'):
1852 if dir == '+':
1853 r = cmp(link.get(av, 'order'),
1854 link.get(bv, 'order'))
1855 if r != 0: return r
1856 elif dir == '-':
1857 r = cmp(link.get(bv, 'order'),
1858 link.get(av, 'order'))
1859 if r != 0: return r
1860 elif link.getkey():
1861 key = link.getkey()
1862 if dir == '+':
1863 r = cmp(link.get(av, key), link.get(bv, key))
1864 if r != 0: return r
1865 elif dir == '-':
1866 r = cmp(link.get(bv, key), link.get(av, key))
1867 if r != 0: return r
1868 else:
1869 if dir == '+':
1870 r = cmp(av, bv)
1871 if r != 0: return r
1872 elif dir == '-':
1873 r = cmp(bv, av)
1874 if r != 0: return r
1876 # Multilink properties are sorted according to how many
1877 # links are present.
1878 elif isinstance(propclass, Multilink):
1879 r = cmp(len(av), len(bv))
1880 if r == 0:
1881 # Compare contents of multilink property if lenghts is
1882 # equal
1883 r = cmp ('.'.join(av), '.'.join(bv))
1884 if r:
1885 if dir == '+':
1886 return r
1887 else:
1888 return -r
1890 else:
1891 # all other types just compare
1892 if dir == '+':
1893 r = cmp(av, bv)
1894 elif dir == '-':
1895 r = cmp(bv, av)
1896 if r != 0: return r
1898 # end for dir, prop in sort, group:
1899 # if all else fails, compare the ids
1900 return cmp(a[0], b[0])
1902 l.sort(sortfun)
1903 return [i[0] for i in l]
1905 def count(self):
1906 '''Get the number of nodes in this class.
1908 If the returned integer is 'numnodes', the ids of all the nodes
1909 in this class run from 1 to numnodes, and numnodes+1 will be the
1910 id of the next node to be created in this class.
1911 '''
1912 return self.db.countnodes(self.classname)
1914 # Manipulating properties:
1916 def getprops(self, protected=1):
1917 '''Return a dictionary mapping property names to property objects.
1918 If the "protected" flag is true, we include protected properties -
1919 those which may not be modified.
1921 In addition to the actual properties on the node, these
1922 methods provide the "creation" and "activity" properties. If the
1923 "protected" flag is true, we include protected properties - those
1924 which may not be modified.
1925 '''
1926 d = self.properties.copy()
1927 if protected:
1928 d['id'] = String()
1929 d['creation'] = hyperdb.Date()
1930 d['activity'] = hyperdb.Date()
1931 d['creator'] = hyperdb.Link('user')
1932 return d
1934 def addprop(self, **properties):
1935 '''Add properties to this class.
1937 The keyword arguments in 'properties' must map names to property
1938 objects, or a TypeError is raised. None of the keys in 'properties'
1939 may collide with the names of existing properties, or a ValueError
1940 is raised before any properties have been added.
1941 '''
1942 for key in properties.keys():
1943 if self.properties.has_key(key):
1944 raise ValueError, key
1945 self.properties.update(properties)
1947 def index(self, nodeid):
1948 '''Add (or refresh) the node to search indexes
1949 '''
1950 # find all the String properties that have indexme
1951 for prop, propclass in self.getprops().items():
1952 if isinstance(propclass, String) and propclass.indexme:
1953 try:
1954 value = str(self.get(nodeid, prop))
1955 except IndexError:
1956 # node no longer exists - entry should be removed
1957 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1958 else:
1959 # and index them under (classname, nodeid, property)
1960 self.db.indexer.add_text((self.classname, nodeid, prop),
1961 value)
1963 #
1964 # Detector interface
1965 #
1966 def audit(self, event, detector):
1967 '''Register a detector
1968 '''
1969 l = self.auditors[event]
1970 if detector not in l:
1971 self.auditors[event].append(detector)
1973 def fireAuditors(self, action, nodeid, newvalues):
1974 '''Fire all registered auditors.
1975 '''
1976 for audit in self.auditors[action]:
1977 audit(self.db, self, nodeid, newvalues)
1979 def react(self, event, detector):
1980 '''Register a detector
1981 '''
1982 l = self.reactors[event]
1983 if detector not in l:
1984 self.reactors[event].append(detector)
1986 def fireReactors(self, action, nodeid, oldvalues):
1987 '''Fire all registered reactors.
1988 '''
1989 for react in self.reactors[action]:
1990 react(self.db, self, nodeid, oldvalues)
1992 class FileClass(Class, hyperdb.FileClass):
1993 '''This class defines a large chunk of data. To support this, it has a
1994 mandatory String property "content" which is typically saved off
1995 externally to the hyperdb.
1997 The default MIME type of this data is defined by the
1998 "default_mime_type" class attribute, which may be overridden by each
1999 node if the class defines a "type" String property.
2000 '''
2001 default_mime_type = 'text/plain'
2003 def create(self, **propvalues):
2004 ''' Snarf the "content" propvalue and store in a file
2005 '''
2006 # we need to fire the auditors now, or the content property won't
2007 # be in propvalues for the auditors to play with
2008 self.fireAuditors('create', None, propvalues)
2010 # now remove the content property so it's not stored in the db
2011 content = propvalues['content']
2012 del propvalues['content']
2014 # do the database create
2015 newid = Class.create_inner(self, **propvalues)
2017 # fire reactors
2018 self.fireReactors('create', newid, None)
2020 # store off the content as a file
2021 self.db.storefile(self.classname, newid, None, content)
2022 return newid
2024 def import_list(self, propnames, proplist):
2025 ''' Trap the "content" property...
2026 '''
2027 # dupe this list so we don't affect others
2028 propnames = propnames[:]
2030 # extract the "content" property from the proplist
2031 i = propnames.index('content')
2032 content = eval(proplist[i])
2033 del propnames[i]
2034 del proplist[i]
2036 # do the normal import
2037 newid = Class.import_list(self, propnames, proplist)
2039 # save off the "content" file
2040 self.db.storefile(self.classname, newid, None, content)
2041 return newid
2043 def get(self, nodeid, propname, default=_marker, cache=1):
2044 ''' trap the content propname and get it from the file
2045 '''
2046 poss_msg = 'Possibly an access right configuration problem.'
2047 if propname == 'content':
2048 try:
2049 return self.db.getfile(self.classname, nodeid, None)
2050 except IOError, (strerror):
2051 # XXX by catching this we donot see an error in the log.
2052 return 'ERROR reading file: %s%s\n%s\n%s'%(
2053 self.classname, nodeid, poss_msg, strerror)
2054 if default is not _marker:
2055 return Class.get(self, nodeid, propname, default, cache=cache)
2056 else:
2057 return Class.get(self, nodeid, propname, cache=cache)
2059 def getprops(self, protected=1):
2060 ''' In addition to the actual properties on the node, these methods
2061 provide the "content" property. If the "protected" flag is true,
2062 we include protected properties - those which may not be
2063 modified.
2064 '''
2065 d = Class.getprops(self, protected=protected).copy()
2066 d['content'] = hyperdb.String()
2067 return d
2069 def index(self, nodeid):
2070 ''' Index the node in the search index.
2072 We want to index the content in addition to the normal String
2073 property indexing.
2074 '''
2075 # perform normal indexing
2076 Class.index(self, nodeid)
2078 # get the content to index
2079 content = self.get(nodeid, 'content')
2081 # figure the mime type
2082 if self.properties.has_key('type'):
2083 mime_type = self.get(nodeid, 'type')
2084 else:
2085 mime_type = self.default_mime_type
2087 # and index!
2088 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
2089 mime_type)
2091 # deviation from spec - was called ItemClass
2092 class IssueClass(Class, roundupdb.IssueClass):
2093 # Overridden methods:
2094 def __init__(self, db, classname, **properties):
2095 '''The newly-created class automatically includes the "messages",
2096 "files", "nosy", and "superseder" properties. If the 'properties'
2097 dictionary attempts to specify any of these properties or a
2098 "creation" or "activity" property, a ValueError is raised.
2099 '''
2100 if not properties.has_key('title'):
2101 properties['title'] = hyperdb.String(indexme='yes')
2102 if not properties.has_key('messages'):
2103 properties['messages'] = hyperdb.Multilink("msg")
2104 if not properties.has_key('files'):
2105 properties['files'] = hyperdb.Multilink("file")
2106 if not properties.has_key('nosy'):
2107 # note: journalling is turned off as it really just wastes
2108 # space. this behaviour may be overridden in an instance
2109 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2110 if not properties.has_key('superseder'):
2111 properties['superseder'] = hyperdb.Multilink(classname)
2112 Class.__init__(self, db, classname, **properties)
2114 #