258b4daf1ee7e947db0eeb0735ff9a06afcc71c5
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: back_anydbm.py,v 1.112 2003-03-16 22:24:54 kedder Exp $
19 '''
20 This module defines a backend that saves the hyperdatabase in a database
21 chosen by anydbm. It is guaranteed to always be available in python
22 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
23 serious bugs, and is not available)
24 '''
26 import whichdb, anydbm, os, marshal, re, weakref, string, copy
27 from roundup import hyperdb, date, password, roundupdb, security
28 from blobfiles import FileStorage
29 from sessions import Sessions, OneTimeKeys
30 from roundup.indexer import Indexer
31 from roundup.backends import locking
32 from roundup.hyperdb import String, Password, Date, Interval, Link, \
33 Multilink, DatabaseError, Boolean, Number, Node
34 from roundup.date import Range
36 #
37 # Now the database
38 #
39 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
40 '''A database for storing records containing flexible data types.
42 Transaction stuff TODO:
43 . check the timestamp of the class file and nuke the cache if it's
44 modified. Do some sort of conflict checking on the dirty stuff.
45 . perhaps detect write collisions (related to above)?
47 '''
48 def __init__(self, config, journaltag=None):
49 '''Open a hyperdatabase given a specifier to some storage.
51 The 'storagelocator' is obtained from config.DATABASE.
52 The meaning of 'storagelocator' depends on the particular
53 implementation of the hyperdatabase. It could be a file name,
54 a directory path, a socket descriptor for a connection to a
55 database over the network, etc.
57 The 'journaltag' is a token that will be attached to the journal
58 entries for any edits done on the database. If 'journaltag' is
59 None, the database is opened in read-only mode: the Class.create(),
60 Class.set(), Class.retire(), and Class.restore() methods are
61 disabled.
62 '''
63 self.config, self.journaltag = config, journaltag
64 self.dir = config.DATABASE
65 self.classes = {}
66 self.cache = {} # cache of nodes loaded or created
67 self.dirtynodes = {} # keep track of the dirty nodes by class
68 self.newnodes = {} # keep track of the new nodes by class
69 self.destroyednodes = {}# keep track of the destroyed nodes by class
70 self.transactions = []
71 self.indexer = Indexer(self.dir)
72 self.sessions = Sessions(self.config)
73 self.otks = OneTimeKeys(self.config)
74 self.security = security.Security(self)
75 # ensure files are group readable and writable
76 os.umask(0002)
78 # lock it
79 lockfilenm = os.path.join(self.dir, 'lock')
80 self.lockfile = locking.acquire_lock(lockfilenm)
81 self.lockfile.write(str(os.getpid()))
82 self.lockfile.flush()
84 def post_init(self):
85 ''' Called once the schema initialisation has finished.
86 '''
87 # reindex the db if necessary
88 if self.indexer.should_reindex():
89 self.reindex()
91 # figure the "curuserid"
92 if self.journaltag is None:
93 self.curuserid = None
94 elif self.journaltag == 'admin':
95 # admin user may not exist, but always has ID 1
96 self.curuserid = '1'
97 else:
98 self.curuserid = self.user.lookup(self.journaltag)
100 def reindex(self):
101 for klass in self.classes.values():
102 for nodeid in klass.list():
103 klass.index(nodeid)
104 self.indexer.save_index()
106 def __repr__(self):
107 return '<back_anydbm instance at %x>'%id(self)
109 #
110 # Classes
111 #
112 def __getattr__(self, classname):
113 '''A convenient way of calling self.getclass(classname).'''
114 if self.classes.has_key(classname):
115 if __debug__:
116 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
117 return self.classes[classname]
118 raise AttributeError, classname
120 def addclass(self, cl):
121 if __debug__:
122 print >>hyperdb.DEBUG, 'addclass', (self, cl)
123 cn = cl.classname
124 if self.classes.has_key(cn):
125 raise ValueError, cn
126 self.classes[cn] = cl
128 def getclasses(self):
129 '''Return a list of the names of all existing classes.'''
130 if __debug__:
131 print >>hyperdb.DEBUG, 'getclasses', (self,)
132 l = self.classes.keys()
133 l.sort()
134 return l
136 def getclass(self, classname):
137 '''Get the Class object representing a particular class.
139 If 'classname' is not a valid class name, a KeyError is raised.
140 '''
141 if __debug__:
142 print >>hyperdb.DEBUG, 'getclass', (self, classname)
143 try:
144 return self.classes[classname]
145 except KeyError:
146 raise KeyError, 'There is no class called "%s"'%classname
148 #
149 # Class DBs
150 #
151 def clear(self):
152 '''Delete all database contents
153 '''
154 if __debug__:
155 print >>hyperdb.DEBUG, 'clear', (self,)
156 for cn in self.classes.keys():
157 for dummy in 'nodes', 'journals':
158 path = os.path.join(self.dir, 'journals.%s'%cn)
159 if os.path.exists(path):
160 os.remove(path)
161 elif os.path.exists(path+'.db'): # dbm appends .db
162 os.remove(path+'.db')
164 def getclassdb(self, classname, mode='r'):
165 ''' grab a connection to the class db that will be used for
166 multiple actions
167 '''
168 if __debug__:
169 print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode)
170 return self.opendb('nodes.%s'%classname, mode)
172 def determine_db_type(self, path):
173 ''' determine which DB wrote the class file
174 '''
175 db_type = ''
176 if os.path.exists(path):
177 db_type = whichdb.whichdb(path)
178 if not db_type:
179 raise DatabaseError, "Couldn't identify database type"
180 elif os.path.exists(path+'.db'):
181 # if the path ends in '.db', it's a dbm database, whether
182 # anydbm says it's dbhash or not!
183 db_type = 'dbm'
184 return db_type
186 def opendb(self, name, mode):
187 '''Low-level database opener that gets around anydbm/dbm
188 eccentricities.
189 '''
190 if __debug__:
191 print >>hyperdb.DEBUG, 'opendb', (self, name, mode)
193 # figure the class db type
194 path = os.path.join(os.getcwd(), self.dir, name)
195 db_type = self.determine_db_type(path)
197 # new database? let anydbm pick the best dbm
198 if not db_type:
199 if __debug__:
200 print >>hyperdb.DEBUG, "opendb anydbm.open(%r, 'c')"%path
201 return anydbm.open(path, 'c')
203 # open the database with the correct module
204 try:
205 dbm = __import__(db_type)
206 except ImportError:
207 raise DatabaseError, \
208 "Couldn't open database - the required module '%s'"\
209 " is not available"%db_type
210 if __debug__:
211 print >>hyperdb.DEBUG, "opendb %r.open(%r, %r)"%(db_type, path,
212 mode)
213 return dbm.open(path, mode)
215 #
216 # Node IDs
217 #
218 def newid(self, classname):
219 ''' Generate a new id for the given class
220 '''
221 # open the ids DB - create if if doesn't exist
222 db = self.opendb('_ids', 'c')
223 if db.has_key(classname):
224 newid = db[classname] = str(int(db[classname]) + 1)
225 else:
226 # the count() bit is transitional - older dbs won't start at 1
227 newid = str(self.getclass(classname).count()+1)
228 db[classname] = newid
229 db.close()
230 return newid
232 def setid(self, classname, setid):
233 ''' Set the id counter: used during import of database
234 '''
235 # open the ids DB - create if if doesn't exist
236 db = self.opendb('_ids', 'c')
237 db[classname] = str(setid)
238 db.close()
240 #
241 # Nodes
242 #
243 def addnode(self, classname, nodeid, node):
244 ''' add the specified node to its class's db
245 '''
246 if __debug__:
247 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
249 # we'll be supplied these props if we're doing an import
250 if not node.has_key('creator'):
251 # add in the "calculated" properties (dupe so we don't affect
252 # calling code's node assumptions)
253 node = node.copy()
254 node['creator'] = self.curuserid
255 node['creation'] = node['activity'] = date.Date()
257 self.newnodes.setdefault(classname, {})[nodeid] = 1
258 self.cache.setdefault(classname, {})[nodeid] = node
259 self.savenode(classname, nodeid, node)
261 def setnode(self, classname, nodeid, node):
262 ''' change the specified node
263 '''
264 if __debug__:
265 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
266 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
268 # update the activity time (dupe so we don't affect
269 # calling code's node assumptions)
270 node = node.copy()
271 node['activity'] = date.Date()
273 # can't set without having already loaded the node
274 self.cache[classname][nodeid] = node
275 self.savenode(classname, nodeid, node)
277 def savenode(self, classname, nodeid, node):
278 ''' perform the saving of data specified by the set/addnode
279 '''
280 if __debug__:
281 print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node)
282 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
284 def getnode(self, classname, nodeid, db=None, cache=1):
285 ''' get a node from the database
286 '''
287 if __debug__:
288 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid, db)
289 if cache:
290 # try the cache
291 cache_dict = self.cache.setdefault(classname, {})
292 if cache_dict.has_key(nodeid):
293 if __debug__:
294 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
295 nodeid)
296 return cache_dict[nodeid]
298 if __debug__:
299 print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid)
301 # get from the database and save in the cache
302 if db is None:
303 db = self.getclassdb(classname)
304 if not db.has_key(nodeid):
305 # try the cache - might be a brand-new node
306 cache_dict = self.cache.setdefault(classname, {})
307 if cache_dict.has_key(nodeid):
308 if __debug__:
309 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
310 nodeid)
311 return cache_dict[nodeid]
312 raise IndexError, "no such %s %s"%(classname, nodeid)
314 # check the uncommitted, destroyed nodes
315 if (self.destroyednodes.has_key(classname) and
316 self.destroyednodes[classname].has_key(nodeid)):
317 raise IndexError, "no such %s %s"%(classname, nodeid)
319 # decode
320 res = marshal.loads(db[nodeid])
322 # reverse the serialisation
323 res = self.unserialise(classname, res)
325 # store off in the cache dict
326 if cache:
327 cache_dict[nodeid] = res
329 return res
331 def destroynode(self, classname, nodeid):
332 '''Remove a node from the database. Called exclusively by the
333 destroy() method on Class.
334 '''
335 if __debug__:
336 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
338 # remove from cache and newnodes if it's there
339 if (self.cache.has_key(classname) and
340 self.cache[classname].has_key(nodeid)):
341 del self.cache[classname][nodeid]
342 if (self.newnodes.has_key(classname) and
343 self.newnodes[classname].has_key(nodeid)):
344 del self.newnodes[classname][nodeid]
346 # see if there's any obvious commit actions that we should get rid of
347 for entry in self.transactions[:]:
348 if entry[1][:2] == (classname, nodeid):
349 self.transactions.remove(entry)
351 # add to the destroyednodes map
352 self.destroyednodes.setdefault(classname, {})[nodeid] = 1
354 # add the destroy commit action
355 self.transactions.append((self.doDestroyNode, (classname, nodeid)))
357 def serialise(self, classname, node):
358 '''Copy the node contents, converting non-marshallable data into
359 marshallable data.
360 '''
361 if __debug__:
362 print >>hyperdb.DEBUG, 'serialise', classname, node
363 properties = self.getclass(classname).getprops()
364 d = {}
365 for k, v in node.items():
366 # if the property doesn't exist, or is the "retired" flag then
367 # it won't be in the properties dict
368 if not properties.has_key(k):
369 d[k] = v
370 continue
372 # get the property spec
373 prop = properties[k]
375 if isinstance(prop, Password) and v is not None:
376 d[k] = str(v)
377 elif isinstance(prop, Date) and v is not None:
378 d[k] = v.serialise()
379 elif isinstance(prop, Interval) and v is not None:
380 d[k] = v.serialise()
381 else:
382 d[k] = v
383 return d
385 def unserialise(self, classname, node):
386 '''Decode the marshalled node data
387 '''
388 if __debug__:
389 print >>hyperdb.DEBUG, 'unserialise', classname, node
390 properties = self.getclass(classname).getprops()
391 d = {}
392 for k, v in node.items():
393 # if the property doesn't exist, or is the "retired" flag then
394 # it won't be in the properties dict
395 if not properties.has_key(k):
396 d[k] = v
397 continue
399 # get the property spec
400 prop = properties[k]
402 if isinstance(prop, Date) and v is not None:
403 d[k] = date.Date(v)
404 elif isinstance(prop, Interval) and v is not None:
405 d[k] = date.Interval(v)
406 elif isinstance(prop, Password) and v is not None:
407 p = password.Password()
408 p.unpack(v)
409 d[k] = p
410 else:
411 d[k] = v
412 return d
414 def hasnode(self, classname, nodeid, db=None):
415 ''' determine if the database has a given node
416 '''
417 if __debug__:
418 print >>hyperdb.DEBUG, 'hasnode', (self, classname, nodeid, db)
420 # try the cache
421 cache = self.cache.setdefault(classname, {})
422 if cache.has_key(nodeid):
423 if __debug__:
424 print >>hyperdb.TRACE, 'has %s %s cached'%(classname, nodeid)
425 return 1
426 if __debug__:
427 print >>hyperdb.TRACE, 'has %s %s'%(classname, nodeid)
429 # not in the cache - check the database
430 if db is None:
431 db = self.getclassdb(classname)
432 res = db.has_key(nodeid)
433 return res
435 def countnodes(self, classname, db=None):
436 if __debug__:
437 print >>hyperdb.DEBUG, 'countnodes', (self, classname, db)
439 count = 0
441 # include the uncommitted nodes
442 if self.newnodes.has_key(classname):
443 count += len(self.newnodes[classname])
444 if self.destroyednodes.has_key(classname):
445 count -= len(self.destroyednodes[classname])
447 # and count those in the DB
448 if db is None:
449 db = self.getclassdb(classname)
450 count = count + len(db.keys())
451 return count
454 #
455 # Files - special node properties
456 # inherited from FileStorage
458 #
459 # Journal
460 #
461 def addjournal(self, classname, nodeid, action, params, creator=None,
462 creation=None):
463 ''' Journal the Action
464 'action' may be:
466 'create' or 'set' -- 'params' is a dictionary of property values
467 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
468 'retire' -- 'params' is None
469 '''
470 if __debug__:
471 print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid,
472 action, params, creator, creation)
473 self.transactions.append((self.doSaveJournal, (classname, nodeid,
474 action, params, creator, creation)))
476 def getjournal(self, classname, nodeid):
477 ''' get the journal for id
479 Raise IndexError if the node doesn't exist (as per history()'s
480 API)
481 '''
482 if __debug__:
483 print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid)
484 # attempt to open the journal - in some rare cases, the journal may
485 # not exist
486 try:
487 db = self.opendb('journals.%s'%classname, 'r')
488 except anydbm.error, error:
489 if str(error) == "need 'c' or 'n' flag to open new db":
490 raise IndexError, 'no such %s %s'%(classname, nodeid)
491 elif error.args[0] != 2:
492 raise
493 raise IndexError, 'no such %s %s'%(classname, nodeid)
494 try:
495 journal = marshal.loads(db[nodeid])
496 except KeyError:
497 db.close()
498 raise IndexError, 'no such %s %s'%(classname, nodeid)
499 db.close()
500 res = []
501 for nodeid, date_stamp, user, action, params in journal:
502 res.append((nodeid, date.Date(date_stamp), user, action, params))
503 return res
505 def pack(self, pack_before):
506 ''' Delete all journal entries except "create" before 'pack_before'.
507 '''
508 if __debug__:
509 print >>hyperdb.DEBUG, 'packjournal', (self, pack_before)
511 pack_before = pack_before.serialise()
512 for classname in self.getclasses():
513 # get the journal db
514 db_name = 'journals.%s'%classname
515 path = os.path.join(os.getcwd(), self.dir, classname)
516 db_type = self.determine_db_type(path)
517 db = self.opendb(db_name, 'w')
519 for key in db.keys():
520 # get the journal for this db entry
521 journal = marshal.loads(db[key])
522 l = []
523 last_set_entry = None
524 for entry in journal:
525 # unpack the entry
526 (nodeid, date_stamp, self.journaltag, action,
527 params) = entry
528 # if the entry is after the pack date, _or_ the initial
529 # create entry, then it stays
530 if date_stamp > pack_before or action == 'create':
531 l.append(entry)
532 db[key] = marshal.dumps(l)
533 if db_type == 'gdbm':
534 db.reorganize()
535 db.close()
538 #
539 # Basic transaction support
540 #
541 def commit(self):
542 ''' Commit the current transactions.
543 '''
544 if __debug__:
545 print >>hyperdb.DEBUG, 'commit', (self,)
547 # keep a handle to all the database files opened
548 self.databases = {}
550 # now, do all the transactions
551 reindex = {}
552 for method, args in self.transactions:
553 reindex[method(*args)] = 1
555 # now close all the database files
556 for db in self.databases.values():
557 db.close()
558 del self.databases
560 # reindex the nodes that request it
561 for classname, nodeid in filter(None, reindex.keys()):
562 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
563 self.getclass(classname).index(nodeid)
565 # save the indexer state
566 self.indexer.save_index()
568 self.clearCache()
570 def clearCache(self):
571 # all transactions committed, back to normal
572 self.cache = {}
573 self.dirtynodes = {}
574 self.newnodes = {}
575 self.destroyednodes = {}
576 self.transactions = []
578 def getCachedClassDB(self, classname):
579 ''' get the class db, looking in our cache of databases for commit
580 '''
581 # get the database handle
582 db_name = 'nodes.%s'%classname
583 if not self.databases.has_key(db_name):
584 self.databases[db_name] = self.getclassdb(classname, 'c')
585 return self.databases[db_name]
587 def doSaveNode(self, classname, nodeid, node):
588 if __debug__:
589 print >>hyperdb.DEBUG, 'doSaveNode', (self, classname, nodeid,
590 node)
592 db = self.getCachedClassDB(classname)
594 # now save the marshalled data
595 db[nodeid] = marshal.dumps(self.serialise(classname, node))
597 # return the classname, nodeid so we reindex this content
598 return (classname, nodeid)
600 def getCachedJournalDB(self, classname):
601 ''' get the journal db, looking in our cache of databases for commit
602 '''
603 # get the database handle
604 db_name = 'journals.%s'%classname
605 if not self.databases.has_key(db_name):
606 self.databases[db_name] = self.opendb(db_name, 'c')
607 return self.databases[db_name]
609 def doSaveJournal(self, classname, nodeid, action, params, creator,
610 creation):
611 # serialise the parameters now if necessary
612 if isinstance(params, type({})):
613 if action in ('set', 'create'):
614 params = self.serialise(classname, params)
616 # handle supply of the special journalling parameters (usually
617 # supplied on importing an existing database)
618 if creator:
619 journaltag = creator
620 else:
621 journaltag = self.curuserid
622 if creation:
623 journaldate = creation.serialise()
624 else:
625 journaldate = date.Date().serialise()
627 # create the journal entry
628 entry = (nodeid, journaldate, journaltag, action, params)
630 if __debug__:
631 print >>hyperdb.DEBUG, 'doSaveJournal', entry
633 db = self.getCachedJournalDB(classname)
635 # now insert the journal entry
636 if db.has_key(nodeid):
637 # append to existing
638 s = db[nodeid]
639 l = marshal.loads(s)
640 l.append(entry)
641 else:
642 l = [entry]
644 db[nodeid] = marshal.dumps(l)
646 def doDestroyNode(self, classname, nodeid):
647 if __debug__:
648 print >>hyperdb.DEBUG, 'doDestroyNode', (self, classname, nodeid)
650 # delete from the class database
651 db = self.getCachedClassDB(classname)
652 if db.has_key(nodeid):
653 del db[nodeid]
655 # delete from the database
656 db = self.getCachedJournalDB(classname)
657 if db.has_key(nodeid):
658 del db[nodeid]
660 # return the classname, nodeid so we reindex this content
661 return (classname, nodeid)
663 def rollback(self):
664 ''' Reverse all actions from the current transaction.
665 '''
666 if __debug__:
667 print >>hyperdb.DEBUG, 'rollback', (self, )
668 for method, args in self.transactions:
669 # delete temporary files
670 if method == self.doStoreFile:
671 self.rollbackStoreFile(*args)
672 self.cache = {}
673 self.dirtynodes = {}
674 self.newnodes = {}
675 self.destroyednodes = {}
676 self.transactions = []
678 def close(self):
679 ''' Nothing to do
680 '''
681 if self.lockfile is not None:
682 locking.release_lock(self.lockfile)
683 if self.lockfile is not None:
684 self.lockfile.close()
685 self.lockfile = None
687 _marker = []
688 class Class(hyperdb.Class):
689 '''The handle to a particular class of nodes in a hyperdatabase.'''
691 def __init__(self, db, classname, **properties):
692 '''Create a new class with a given name and property specification.
694 'classname' must not collide with the name of an existing class,
695 or a ValueError is raised. The keyword arguments in 'properties'
696 must map names to property objects, or a TypeError is raised.
697 '''
698 if (properties.has_key('creation') or properties.has_key('activity')
699 or properties.has_key('creator')):
700 raise ValueError, '"creation", "activity" and "creator" are '\
701 'reserved'
703 self.classname = classname
704 self.properties = properties
705 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
706 self.key = ''
708 # should we journal changes (default yes)
709 self.do_journal = 1
711 # do the db-related init stuff
712 db.addclass(self)
714 self.auditors = {'create': [], 'set': [], 'retire': [], 'restore': []}
715 self.reactors = {'create': [], 'set': [], 'retire': [], 'restore': []}
717 def enableJournalling(self):
718 '''Turn journalling on for this class
719 '''
720 self.do_journal = 1
722 def disableJournalling(self):
723 '''Turn journalling off for this class
724 '''
725 self.do_journal = 0
727 # Editing nodes:
729 def create(self, **propvalues):
730 '''Create a new node of this class and return its id.
732 The keyword arguments in 'propvalues' map property names to values.
734 The values of arguments must be acceptable for the types of their
735 corresponding properties or a TypeError is raised.
737 If this class has a key property, it must be present and its value
738 must not collide with other key strings or a ValueError is raised.
740 Any other properties on this class that are missing from the
741 'propvalues' dictionary are set to None.
743 If an id in a link or multilink property does not refer to a valid
744 node, an IndexError is raised.
746 These operations trigger detectors and can be vetoed. Attempts
747 to modify the "creation" or "activity" properties cause a KeyError.
748 '''
749 self.fireAuditors('create', None, propvalues)
750 newid = self.create_inner(**propvalues)
751 self.fireReactors('create', newid, None)
752 return newid
754 def create_inner(self, **propvalues):
755 ''' Called by create, in-between the audit and react calls.
756 '''
757 if propvalues.has_key('id'):
758 raise KeyError, '"id" is reserved'
760 if self.db.journaltag is None:
761 raise DatabaseError, 'Database open read-only'
763 if propvalues.has_key('creation') or propvalues.has_key('activity'):
764 raise KeyError, '"creation" and "activity" are reserved'
765 # new node's id
766 newid = self.db.newid(self.classname)
768 # validate propvalues
769 num_re = re.compile('^\d+$')
770 for key, value in propvalues.items():
771 if key == self.key:
772 try:
773 self.lookup(value)
774 except KeyError:
775 pass
776 else:
777 raise ValueError, 'node with key "%s" exists'%value
779 # try to handle this property
780 try:
781 prop = self.properties[key]
782 except KeyError:
783 raise KeyError, '"%s" has no property "%s"'%(self.classname,
784 key)
786 if value is not None and isinstance(prop, Link):
787 if type(value) != type(''):
788 raise ValueError, 'link value must be String'
789 link_class = self.properties[key].classname
790 # if it isn't a number, it's a key
791 if not num_re.match(value):
792 try:
793 value = self.db.classes[link_class].lookup(value)
794 except (TypeError, KeyError):
795 raise IndexError, 'new property "%s": %s not a %s'%(
796 key, value, link_class)
797 elif not self.db.getclass(link_class).hasnode(value):
798 raise IndexError, '%s has no node %s'%(link_class, value)
800 # save off the value
801 propvalues[key] = value
803 # register the link with the newly linked node
804 if self.do_journal and self.properties[key].do_journal:
805 self.db.addjournal(link_class, value, 'link',
806 (self.classname, newid, key))
808 elif isinstance(prop, Multilink):
809 if type(value) != type([]):
810 raise TypeError, 'new property "%s" not a list of ids'%key
812 # clean up and validate the list of links
813 link_class = self.properties[key].classname
814 l = []
815 for entry in value:
816 if type(entry) != type(''):
817 raise ValueError, '"%s" multilink value (%r) '\
818 'must contain Strings'%(key, value)
819 # if it isn't a number, it's a key
820 if not num_re.match(entry):
821 try:
822 entry = self.db.classes[link_class].lookup(entry)
823 except (TypeError, KeyError):
824 raise IndexError, 'new property "%s": %s not a %s'%(
825 key, entry, self.properties[key].classname)
826 l.append(entry)
827 value = l
828 propvalues[key] = value
830 # handle additions
831 for nodeid in value:
832 if not self.db.getclass(link_class).hasnode(nodeid):
833 raise IndexError, '%s has no node %s'%(link_class,
834 nodeid)
835 # register the link with the newly linked node
836 if self.do_journal and self.properties[key].do_journal:
837 self.db.addjournal(link_class, nodeid, 'link',
838 (self.classname, newid, key))
840 elif isinstance(prop, String):
841 if type(value) != type('') and type(value) != type(u''):
842 raise TypeError, 'new property "%s" not a string'%key
844 elif isinstance(prop, Password):
845 if not isinstance(value, password.Password):
846 raise TypeError, 'new property "%s" not a Password'%key
848 elif isinstance(prop, Date):
849 if value is not None and not isinstance(value, date.Date):
850 raise TypeError, 'new property "%s" not a Date'%key
852 elif isinstance(prop, Interval):
853 if value is not None and not isinstance(value, date.Interval):
854 raise TypeError, 'new property "%s" not an Interval'%key
856 elif value is not None and isinstance(prop, Number):
857 try:
858 float(value)
859 except ValueError:
860 raise TypeError, 'new property "%s" not numeric'%key
862 elif value is not None and isinstance(prop, Boolean):
863 try:
864 int(value)
865 except ValueError:
866 raise TypeError, 'new property "%s" not boolean'%key
868 # make sure there's data where there needs to be
869 for key, prop in self.properties.items():
870 if propvalues.has_key(key):
871 continue
872 if key == self.key:
873 raise ValueError, 'key property "%s" is required'%key
874 if isinstance(prop, Multilink):
875 propvalues[key] = []
876 else:
877 propvalues[key] = None
879 # done
880 self.db.addnode(self.classname, newid, propvalues)
881 if self.do_journal:
882 self.db.addjournal(self.classname, newid, 'create', {})
884 return newid
886 def export_list(self, propnames, nodeid):
887 ''' Export a node - generate a list of CSV-able data in the order
888 specified by propnames for the given node.
889 '''
890 properties = self.getprops()
891 l = []
892 for prop in propnames:
893 proptype = properties[prop]
894 value = self.get(nodeid, prop)
895 # "marshal" data where needed
896 if value is None:
897 pass
898 elif isinstance(proptype, hyperdb.Date):
899 value = value.get_tuple()
900 elif isinstance(proptype, hyperdb.Interval):
901 value = value.get_tuple()
902 elif isinstance(proptype, hyperdb.Password):
903 value = str(value)
904 l.append(repr(value))
906 # append retired flag
907 l.append(self.is_retired(nodeid))
909 return l
911 def import_list(self, propnames, proplist):
912 ''' Import a node - all information including "id" is present and
913 should not be sanity checked. Triggers are not triggered. The
914 journal should be initialised using the "creator" and "created"
915 information.
917 Return the nodeid of the node imported.
918 '''
919 if self.db.journaltag is None:
920 raise DatabaseError, 'Database open read-only'
921 properties = self.getprops()
923 # make the new node's property map
924 d = {}
925 newid = None
926 for i in range(len(propnames)):
927 # Figure the property for this column
928 propname = propnames[i]
930 # Use eval to reverse the repr() used to output the CSV
931 value = eval(proplist[i])
933 # "unmarshal" where necessary
934 if propname == 'id':
935 newid = value
936 continue
937 elif propname == 'is retired':
938 # is the item retired?
939 if int(value):
940 d[self.db.RETIRED_FLAG] = 1
941 continue
942 elif value is None:
943 # don't set Nones
944 continue
946 prop = properties[propname]
947 if isinstance(prop, hyperdb.Date):
948 value = date.Date(value)
949 elif isinstance(prop, hyperdb.Interval):
950 value = date.Interval(value)
951 elif isinstance(prop, hyperdb.Password):
952 pwd = password.Password()
953 pwd.unpack(value)
954 value = pwd
955 d[propname] = value
957 # get a new id if necessary
958 if newid is None:
959 newid = self.db.newid(self.classname)
961 # add the node and journal
962 self.db.addnode(self.classname, newid, d)
964 # extract the journalling stuff and nuke it
965 if d.has_key('creator'):
966 creator = d['creator']
967 del d['creator']
968 else:
969 creator = None
970 if d.has_key('creation'):
971 creation = d['creation']
972 del d['creation']
973 else:
974 creation = None
975 if d.has_key('activity'):
976 del d['activity']
977 self.db.addjournal(self.classname, newid, 'create', {}, creator,
978 creation)
979 return newid
981 def get(self, nodeid, propname, default=_marker, cache=1):
982 '''Get the value of a property on an existing node of this class.
984 'nodeid' must be the id of an existing node of this class or an
985 IndexError is raised. 'propname' must be the name of a property
986 of this class or a KeyError is raised.
988 'cache' indicates whether the transaction cache should be queried
989 for the node. If the node has been modified and you need to
990 determine what its values prior to modification are, you need to
991 set cache=0.
993 Attempts to get the "creation" or "activity" properties should
994 do the right thing.
995 '''
996 if propname == 'id':
997 return nodeid
999 # get the node's dict
1000 d = self.db.getnode(self.classname, nodeid, cache=cache)
1002 # check for one of the special props
1003 if propname == 'creation':
1004 if d.has_key('creation'):
1005 return d['creation']
1006 if not self.do_journal:
1007 raise ValueError, 'Journalling is disabled for this class'
1008 journal = self.db.getjournal(self.classname, nodeid)
1009 if journal:
1010 return self.db.getjournal(self.classname, nodeid)[0][1]
1011 else:
1012 # on the strange chance that there's no journal
1013 return date.Date()
1014 if propname == 'activity':
1015 if d.has_key('activity'):
1016 return d['activity']
1017 if not self.do_journal:
1018 raise ValueError, 'Journalling is disabled for this class'
1019 journal = self.db.getjournal(self.classname, nodeid)
1020 if journal:
1021 return self.db.getjournal(self.classname, nodeid)[-1][1]
1022 else:
1023 # on the strange chance that there's no journal
1024 return date.Date()
1025 if propname == 'creator':
1026 if d.has_key('creator'):
1027 return d['creator']
1028 if not self.do_journal:
1029 raise ValueError, 'Journalling is disabled for this class'
1030 journal = self.db.getjournal(self.classname, nodeid)
1031 if journal:
1032 num_re = re.compile('^\d+$')
1033 value = self.db.getjournal(self.classname, nodeid)[0][2]
1034 if num_re.match(value):
1035 return value
1036 else:
1037 # old-style "username" journal tag
1038 try:
1039 return self.db.user.lookup(value)
1040 except KeyError:
1041 # user's been retired, return admin
1042 return '1'
1043 else:
1044 return self.db.curuserid
1046 # get the property (raises KeyErorr if invalid)
1047 prop = self.properties[propname]
1049 if not d.has_key(propname):
1050 if default is _marker:
1051 if isinstance(prop, Multilink):
1052 return []
1053 else:
1054 return None
1055 else:
1056 return default
1058 # return a dupe of the list so code doesn't get confused
1059 if isinstance(prop, Multilink):
1060 return d[propname][:]
1062 return d[propname]
1064 # not in spec
1065 def getnode(self, nodeid, cache=1):
1066 ''' Return a convenience wrapper for the node.
1068 'nodeid' must be the id of an existing node of this class or an
1069 IndexError is raised.
1071 'cache' indicates whether the transaction cache should be queried
1072 for the node. If the node has been modified and you need to
1073 determine what its values prior to modification are, you need to
1074 set cache=0.
1075 '''
1076 return Node(self, nodeid, cache=cache)
1078 def set(self, nodeid, **propvalues):
1079 '''Modify a property on an existing node of this class.
1081 'nodeid' must be the id of an existing node of this class or an
1082 IndexError is raised.
1084 Each key in 'propvalues' must be the name of a property of this
1085 class or a KeyError is raised.
1087 All values in 'propvalues' must be acceptable types for their
1088 corresponding properties or a TypeError is raised.
1090 If the value of the key property is set, it must not collide with
1091 other key strings or a ValueError is raised.
1093 If the value of a Link or Multilink property contains an invalid
1094 node id, a ValueError is raised.
1096 These operations trigger detectors and can be vetoed. Attempts
1097 to modify the "creation" or "activity" properties cause a KeyError.
1098 '''
1099 if not propvalues:
1100 return propvalues
1102 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1103 raise KeyError, '"creation" and "activity" are reserved'
1105 if propvalues.has_key('id'):
1106 raise KeyError, '"id" is reserved'
1108 if self.db.journaltag is None:
1109 raise DatabaseError, 'Database open read-only'
1111 self.fireAuditors('set', nodeid, propvalues)
1112 # Take a copy of the node dict so that the subsequent set
1113 # operation doesn't modify the oldvalues structure.
1114 try:
1115 # try not using the cache initially
1116 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid,
1117 cache=0))
1118 except IndexError:
1119 # this will be needed if somone does a create() and set()
1120 # with no intervening commit()
1121 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1123 node = self.db.getnode(self.classname, nodeid)
1124 if node.has_key(self.db.RETIRED_FLAG):
1125 raise IndexError
1126 num_re = re.compile('^\d+$')
1128 # if the journal value is to be different, store it in here
1129 journalvalues = {}
1131 for propname, value in propvalues.items():
1132 # check to make sure we're not duplicating an existing key
1133 if propname == self.key and node[propname] != value:
1134 try:
1135 self.lookup(value)
1136 except KeyError:
1137 pass
1138 else:
1139 raise ValueError, 'node with key "%s" exists'%value
1141 # this will raise the KeyError if the property isn't valid
1142 # ... we don't use getprops() here because we only care about
1143 # the writeable properties.
1144 try:
1145 prop = self.properties[propname]
1146 except KeyError:
1147 raise KeyError, '"%s" has no property named "%s"'%(
1148 self.classname, propname)
1150 # if the value's the same as the existing value, no sense in
1151 # doing anything
1152 current = node.get(propname, None)
1153 if value == current:
1154 del propvalues[propname]
1155 continue
1156 journalvalues[propname] = current
1158 # do stuff based on the prop type
1159 if isinstance(prop, Link):
1160 link_class = prop.classname
1161 # if it isn't a number, it's a key
1162 if value is not None and not isinstance(value, type('')):
1163 raise ValueError, 'property "%s" link value be a string'%(
1164 propname)
1165 if isinstance(value, type('')) and not num_re.match(value):
1166 try:
1167 value = self.db.classes[link_class].lookup(value)
1168 except (TypeError, KeyError):
1169 raise IndexError, 'new property "%s": %s not a %s'%(
1170 propname, value, prop.classname)
1172 if (value is not None and
1173 not self.db.getclass(link_class).hasnode(value)):
1174 raise IndexError, '%s has no node %s'%(link_class, value)
1176 if self.do_journal and prop.do_journal:
1177 # register the unlink with the old linked node
1178 if node.has_key(propname) and node[propname] is not None:
1179 self.db.addjournal(link_class, node[propname], 'unlink',
1180 (self.classname, nodeid, propname))
1182 # register the link with the newly linked node
1183 if value is not None:
1184 self.db.addjournal(link_class, value, 'link',
1185 (self.classname, nodeid, propname))
1187 elif isinstance(prop, Multilink):
1188 if type(value) != type([]):
1189 raise TypeError, 'new property "%s" not a list of'\
1190 ' ids'%propname
1191 link_class = self.properties[propname].classname
1192 l = []
1193 for entry in value:
1194 # if it isn't a number, it's a key
1195 if type(entry) != type(''):
1196 raise ValueError, 'new property "%s" link value ' \
1197 'must be a string'%propname
1198 if not num_re.match(entry):
1199 try:
1200 entry = self.db.classes[link_class].lookup(entry)
1201 except (TypeError, KeyError):
1202 raise IndexError, 'new property "%s": %s not a %s'%(
1203 propname, entry,
1204 self.properties[propname].classname)
1205 l.append(entry)
1206 value = l
1207 propvalues[propname] = value
1209 # figure the journal entry for this property
1210 add = []
1211 remove = []
1213 # handle removals
1214 if node.has_key(propname):
1215 l = node[propname]
1216 else:
1217 l = []
1218 for id in l[:]:
1219 if id in value:
1220 continue
1221 # register the unlink with the old linked node
1222 if self.do_journal and self.properties[propname].do_journal:
1223 self.db.addjournal(link_class, id, 'unlink',
1224 (self.classname, nodeid, propname))
1225 l.remove(id)
1226 remove.append(id)
1228 # handle additions
1229 for id in value:
1230 if not self.db.getclass(link_class).hasnode(id):
1231 raise IndexError, '%s has no node %s'%(link_class, id)
1232 if id in l:
1233 continue
1234 # register the link with the newly linked node
1235 if self.do_journal and self.properties[propname].do_journal:
1236 self.db.addjournal(link_class, id, 'link',
1237 (self.classname, nodeid, propname))
1238 l.append(id)
1239 add.append(id)
1241 # figure the journal entry
1242 l = []
1243 if add:
1244 l.append(('+', add))
1245 if remove:
1246 l.append(('-', remove))
1247 if l:
1248 journalvalues[propname] = tuple(l)
1250 elif isinstance(prop, String):
1251 if value is not None and type(value) != type('') and type(value) != type(u''):
1252 raise TypeError, 'new property "%s" not a string'%propname
1254 elif isinstance(prop, Password):
1255 if not isinstance(value, password.Password):
1256 raise TypeError, 'new property "%s" not a Password'%propname
1257 propvalues[propname] = value
1259 elif value is not None and isinstance(prop, Date):
1260 if not isinstance(value, date.Date):
1261 raise TypeError, 'new property "%s" not a Date'% propname
1262 propvalues[propname] = value
1264 elif value is not None and isinstance(prop, Interval):
1265 if not isinstance(value, date.Interval):
1266 raise TypeError, 'new property "%s" not an '\
1267 'Interval'%propname
1268 propvalues[propname] = value
1270 elif value is not None and isinstance(prop, Number):
1271 try:
1272 float(value)
1273 except ValueError:
1274 raise TypeError, 'new property "%s" not numeric'%propname
1276 elif value is not None and isinstance(prop, Boolean):
1277 try:
1278 int(value)
1279 except ValueError:
1280 raise TypeError, 'new property "%s" not boolean'%propname
1282 node[propname] = value
1284 # nothing to do?
1285 if not propvalues:
1286 return propvalues
1288 # do the set, and journal it
1289 self.db.setnode(self.classname, nodeid, node)
1291 if self.do_journal:
1292 self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
1294 self.fireReactors('set', nodeid, oldvalues)
1296 return propvalues
1298 def retire(self, nodeid):
1299 '''Retire a node.
1301 The properties on the node remain available from the get() method,
1302 and the node's id is never reused.
1304 Retired nodes are not returned by the find(), list(), or lookup()
1305 methods, and other nodes may reuse the values of their key properties.
1307 These operations trigger detectors and can be vetoed. Attempts
1308 to modify the "creation" or "activity" properties cause a KeyError.
1309 '''
1310 if self.db.journaltag is None:
1311 raise DatabaseError, 'Database open read-only'
1313 self.fireAuditors('retire', nodeid, None)
1315 node = self.db.getnode(self.classname, nodeid)
1316 node[self.db.RETIRED_FLAG] = 1
1317 self.db.setnode(self.classname, nodeid, node)
1318 if self.do_journal:
1319 self.db.addjournal(self.classname, nodeid, 'retired', None)
1321 self.fireReactors('retire', nodeid, None)
1323 def restore(self, nodeid):
1324 '''Restpre a retired node.
1326 Make node available for all operations like it was before retirement.
1327 '''
1328 if self.db.journaltag is None:
1329 raise DatabaseError, 'Database open read-only'
1331 self.fireAuditors('restore', nodeid, None)
1333 node = self.db.getnode(self.classname, nodeid)
1334 del node[self.db.RETIRED_FLAG]
1335 self.db.setnode(self.classname, nodeid, node)
1336 if self.do_journal:
1337 self.db.addjournal(self.classname, nodeid, 'restored', None)
1339 self.fireReactors('restore', nodeid, None)
1341 def is_retired(self, nodeid, cldb=None):
1342 '''Return true if the node is retired.
1343 '''
1344 node = self.db.getnode(self.classname, nodeid, cldb)
1345 if node.has_key(self.db.RETIRED_FLAG):
1346 return 1
1347 return 0
1349 def destroy(self, nodeid):
1350 '''Destroy a node.
1352 WARNING: this method should never be used except in extremely rare
1353 situations where there could never be links to the node being
1354 deleted
1355 WARNING: use retire() instead
1356 WARNING: the properties of this node will not be available ever again
1357 WARNING: really, use retire() instead
1359 Well, I think that's enough warnings. This method exists mostly to
1360 support the session storage of the cgi interface.
1361 '''
1362 if self.db.journaltag is None:
1363 raise DatabaseError, 'Database open read-only'
1364 self.db.destroynode(self.classname, nodeid)
1366 def history(self, nodeid):
1367 '''Retrieve the journal of edits on a particular node.
1369 'nodeid' must be the id of an existing node of this class or an
1370 IndexError is raised.
1372 The returned list contains tuples of the form
1374 (nodeid, date, tag, action, params)
1376 'date' is a Timestamp object specifying the time of the change and
1377 'tag' is the journaltag specified when the database was opened.
1378 '''
1379 if not self.do_journal:
1380 raise ValueError, 'Journalling is disabled for this class'
1381 return self.db.getjournal(self.classname, nodeid)
1383 # Locating nodes:
1384 def hasnode(self, nodeid):
1385 '''Determine if the given nodeid actually exists
1386 '''
1387 return self.db.hasnode(self.classname, nodeid)
1389 def setkey(self, propname):
1390 '''Select a String property of this class to be the key property.
1392 'propname' must be the name of a String property of this class or
1393 None, or a TypeError is raised. The values of the key property on
1394 all existing nodes must be unique or a ValueError is raised. If the
1395 property doesn't exist, KeyError is raised.
1396 '''
1397 prop = self.getprops()[propname]
1398 if not isinstance(prop, String):
1399 raise TypeError, 'key properties must be String'
1400 self.key = propname
1402 def getkey(self):
1403 '''Return the name of the key property for this class or None.'''
1404 return self.key
1406 def labelprop(self, default_to_id=0):
1407 ''' Return the property name for a label for the given node.
1409 This method attempts to generate a consistent label for the node.
1410 It tries the following in order:
1411 1. key property
1412 2. "name" property
1413 3. "title" property
1414 4. first property from the sorted property name list
1415 '''
1416 k = self.getkey()
1417 if k:
1418 return k
1419 props = self.getprops()
1420 if props.has_key('name'):
1421 return 'name'
1422 elif props.has_key('title'):
1423 return 'title'
1424 if default_to_id:
1425 return 'id'
1426 props = props.keys()
1427 props.sort()
1428 return props[0]
1430 # TODO: set up a separate index db file for this? profile?
1431 def lookup(self, keyvalue):
1432 '''Locate a particular node by its key property and return its id.
1434 If this class has no key property, a TypeError is raised. If the
1435 'keyvalue' matches one of the values for the key property among
1436 the nodes in this class, the matching node's id is returned;
1437 otherwise a KeyError is raised.
1438 '''
1439 if not self.key:
1440 raise TypeError, 'No key property set for class %s'%self.classname
1441 cldb = self.db.getclassdb(self.classname)
1442 try:
1443 for nodeid in self.getnodeids(cldb):
1444 node = self.db.getnode(self.classname, nodeid, cldb)
1445 if node.has_key(self.db.RETIRED_FLAG):
1446 continue
1447 if node[self.key] == keyvalue:
1448 return nodeid
1449 finally:
1450 cldb.close()
1451 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1452 keyvalue, self.classname)
1454 # change from spec - allows multiple props to match
1455 def find(self, **propspec):
1456 '''Get the ids of nodes in this class which link to the given nodes.
1458 'propspec' consists of keyword args propname=nodeid or
1459 propname={nodeid:1, }
1460 'propname' must be the name of a property in this class, or a
1461 KeyError is raised. That property must be a Link or
1462 Multilink property, or a TypeError is raised.
1464 Any node in this class whose 'propname' property links to any of the
1465 nodeids will be returned. Used by the full text indexing, which knows
1466 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1467 issues:
1469 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1470 '''
1471 propspec = propspec.items()
1472 for propname, nodeids in propspec:
1473 # check the prop is OK
1474 prop = self.properties[propname]
1475 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1476 raise TypeError, "'%s' not a Link/Multilink property"%propname
1478 # ok, now do the find
1479 cldb = self.db.getclassdb(self.classname)
1480 l = []
1481 try:
1482 for id in self.getnodeids(db=cldb):
1483 node = self.db.getnode(self.classname, id, db=cldb)
1484 if node.has_key(self.db.RETIRED_FLAG):
1485 continue
1486 for propname, nodeids in propspec:
1487 # can't test if the node doesn't have this property
1488 if not node.has_key(propname):
1489 continue
1490 if type(nodeids) is type(''):
1491 nodeids = {nodeids:1}
1492 prop = self.properties[propname]
1493 value = node[propname]
1494 if isinstance(prop, Link) and nodeids.has_key(value):
1495 l.append(id)
1496 break
1497 elif isinstance(prop, Multilink):
1498 hit = 0
1499 for v in value:
1500 if nodeids.has_key(v):
1501 l.append(id)
1502 hit = 1
1503 break
1504 if hit:
1505 break
1506 finally:
1507 cldb.close()
1508 return l
1510 def stringFind(self, **requirements):
1511 '''Locate a particular node by matching a set of its String
1512 properties in a caseless search.
1514 If the property is not a String property, a TypeError is raised.
1516 The return is a list of the id of all nodes that match.
1517 '''
1518 for propname in requirements.keys():
1519 prop = self.properties[propname]
1520 if isinstance(not prop, String):
1521 raise TypeError, "'%s' not a String property"%propname
1522 requirements[propname] = requirements[propname].lower()
1523 l = []
1524 cldb = self.db.getclassdb(self.classname)
1525 try:
1526 for nodeid in self.getnodeids(cldb):
1527 node = self.db.getnode(self.classname, nodeid, cldb)
1528 if node.has_key(self.db.RETIRED_FLAG):
1529 continue
1530 for key, value in requirements.items():
1531 if not node.has_key(key):
1532 break
1533 if node[key] is None or node[key].lower() != value:
1534 break
1535 else:
1536 l.append(nodeid)
1537 finally:
1538 cldb.close()
1539 return l
1541 def list(self):
1542 ''' Return a list of the ids of the active nodes in this class.
1543 '''
1544 l = []
1545 cn = self.classname
1546 cldb = self.db.getclassdb(cn)
1547 try:
1548 for nodeid in self.getnodeids(cldb):
1549 node = self.db.getnode(cn, nodeid, cldb)
1550 if node.has_key(self.db.RETIRED_FLAG):
1551 continue
1552 l.append(nodeid)
1553 finally:
1554 cldb.close()
1555 l.sort()
1556 return l
1558 def getnodeids(self, db=None):
1559 ''' Return a list of ALL nodeids
1560 '''
1561 if __debug__:
1562 print >>hyperdb.DEBUG, 'getnodeids', (self, self.classname, db)
1564 res = []
1566 # start off with the new nodes
1567 if self.db.newnodes.has_key(self.classname):
1568 res += self.db.newnodes[self.classname].keys()
1570 if db is None:
1571 db = self.db.getclassdb(self.classname)
1572 res = res + db.keys()
1574 # remove the uncommitted, destroyed nodes
1575 if self.db.destroyednodes.has_key(self.classname):
1576 for nodeid in self.db.destroyednodes[self.classname].keys():
1577 if db.has_key(nodeid):
1578 res.remove(nodeid)
1580 return res
1582 def filter(self, search_matches, filterspec, sort=(None,None),
1583 group=(None,None), num_re = re.compile('^\d+$')):
1584 ''' Return a list of the ids of the active nodes in this class that
1585 match the 'filter' spec, sorted by the group spec and then the
1586 sort spec.
1588 "filterspec" is {propname: value(s)}
1589 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1590 and prop is a prop name or None
1591 "search_matches" is {nodeid: marker}
1593 The filter must match all properties specificed - but if the
1594 property value to match is a list, any one of the values in the
1595 list may match for that property to match.
1596 '''
1597 cn = self.classname
1599 # optimise filterspec
1600 l = []
1601 props = self.getprops()
1602 LINK = 0
1603 MULTILINK = 1
1604 STRING = 2
1605 DATE = 3
1606 OTHER = 6
1608 timezone = self.db.getUserTimezone()
1609 for k, v in filterspec.items():
1610 propclass = props[k]
1611 if isinstance(propclass, Link):
1612 if type(v) is not type([]):
1613 v = [v]
1614 # replace key values with node ids
1615 u = []
1616 link_class = self.db.classes[propclass.classname]
1617 for entry in v:
1618 if entry == '-1': entry = None
1619 elif not num_re.match(entry):
1620 try:
1621 entry = link_class.lookup(entry)
1622 except (TypeError,KeyError):
1623 raise ValueError, 'property "%s": %s not a %s'%(
1624 k, entry, self.properties[k].classname)
1625 u.append(entry)
1627 l.append((LINK, k, u))
1628 elif isinstance(propclass, Multilink):
1629 if type(v) is not type([]):
1630 v = [v]
1631 # replace key values with node ids
1632 u = []
1633 link_class = self.db.classes[propclass.classname]
1634 for entry in v:
1635 if not num_re.match(entry):
1636 try:
1637 entry = link_class.lookup(entry)
1638 except (TypeError,KeyError):
1639 raise ValueError, 'new property "%s": %s not a %s'%(
1640 k, entry, self.properties[k].classname)
1641 u.append(entry)
1642 l.append((MULTILINK, k, u))
1643 elif isinstance(propclass, String) and k != 'id':
1644 # simple glob searching
1645 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1646 v = v.replace('?', '.')
1647 v = v.replace('*', '.*?')
1648 l.append((STRING, k, re.compile(v, re.I)))
1649 elif isinstance(propclass, Date):
1650 try:
1651 date_rng = Range(v, date.Date, offset=timezone)
1652 l.append((DATE, k, date_rng))
1653 except ValueError:
1654 # If range creation fails - ignore that search parameter
1655 pass
1656 elif isinstance(propclass, Boolean):
1657 if type(v) is type(''):
1658 bv = v.lower() in ('yes', 'true', 'on', '1')
1659 else:
1660 bv = v
1661 l.append((OTHER, k, bv))
1662 # kedder: dates are filtered by ranges
1663 #elif isinstance(propclass, Date):
1664 # l.append((OTHER, k, date.Date(v)))
1665 elif isinstance(propclass, Interval):
1666 l.append((OTHER, k, date.Interval(v)))
1667 elif isinstance(propclass, Number):
1668 l.append((OTHER, k, int(v)))
1669 else:
1670 l.append((OTHER, k, v))
1671 filterspec = l
1673 # now, find all the nodes that are active and pass filtering
1674 l = []
1675 cldb = self.db.getclassdb(cn)
1676 try:
1677 # TODO: only full-scan once (use items())
1678 for nodeid in self.getnodeids(cldb):
1679 node = self.db.getnode(cn, nodeid, cldb)
1680 if node.has_key(self.db.RETIRED_FLAG):
1681 continue
1682 # apply filter
1683 for t, k, v in filterspec:
1684 # handle the id prop
1685 if k == 'id' and v == nodeid:
1686 continue
1688 # make sure the node has the property
1689 if not node.has_key(k):
1690 # this node doesn't have this property, so reject it
1691 break
1693 # now apply the property filter
1694 if t == LINK:
1695 # link - if this node's property doesn't appear in the
1696 # filterspec's nodeid list, skip it
1697 if node[k] not in v:
1698 break
1699 elif t == MULTILINK:
1700 # multilink - if any of the nodeids required by the
1701 # filterspec aren't in this node's property, then skip
1702 # it
1703 have = node[k]
1704 for want in v:
1705 if want not in have:
1706 break
1707 else:
1708 continue
1709 break
1710 elif t == STRING:
1711 # RE search
1712 if node[k] is None or not v.search(node[k]):
1713 break
1714 elif t == DATE:
1715 if node[k] is None: break
1716 if v.to_value:
1717 if not (v.from_value < node[k] and v.to_value > node[k]):
1718 break
1719 else:
1720 if not (v.from_value < node[k]):
1721 break
1722 elif t == OTHER:
1723 # straight value comparison for the other types
1724 if node[k] != v:
1725 break
1726 else:
1727 l.append((nodeid, node))
1728 finally:
1729 cldb.close()
1730 l.sort()
1732 # filter based on full text search
1733 if search_matches is not None:
1734 k = []
1735 for v in l:
1736 if search_matches.has_key(v[0]):
1737 k.append(v)
1738 l = k
1740 # now, sort the result
1741 def sortfun(a, b, sort=sort, group=group, properties=self.getprops(),
1742 db = self.db, cl=self):
1743 a_id, an = a
1744 b_id, bn = b
1745 # sort by group and then sort
1746 for dir, prop in group, sort:
1747 if dir is None or prop is None: continue
1749 # sorting is class-specific
1750 propclass = properties[prop]
1752 # handle the properties that might be "faked"
1753 # also, handle possible missing properties
1754 try:
1755 if not an.has_key(prop):
1756 an[prop] = cl.get(a_id, prop)
1757 av = an[prop]
1758 except KeyError:
1759 # the node doesn't have a value for this property
1760 if isinstance(propclass, Multilink): av = []
1761 else: av = ''
1762 try:
1763 if not bn.has_key(prop):
1764 bn[prop] = cl.get(b_id, prop)
1765 bv = bn[prop]
1766 except KeyError:
1767 # the node doesn't have a value for this property
1768 if isinstance(propclass, Multilink): bv = []
1769 else: bv = ''
1771 # String and Date values are sorted in the natural way
1772 if isinstance(propclass, String):
1773 # clean up the strings
1774 if av and av[0] in string.uppercase:
1775 av = av.lower()
1776 if bv and bv[0] in string.uppercase:
1777 bv = bv.lower()
1778 if (isinstance(propclass, String) or
1779 isinstance(propclass, Date)):
1780 # it might be a string that's really an integer
1781 try:
1782 av = int(av)
1783 bv = int(bv)
1784 except:
1785 pass
1786 if dir == '+':
1787 r = cmp(av, bv)
1788 if r != 0: return r
1789 elif dir == '-':
1790 r = cmp(bv, av)
1791 if r != 0: return r
1793 # Link properties are sorted according to the value of
1794 # the "order" property on the linked nodes if it is
1795 # present; or otherwise on the key string of the linked
1796 # nodes; or finally on the node ids.
1797 elif isinstance(propclass, Link):
1798 link = db.classes[propclass.classname]
1799 if av is None and bv is not None: return -1
1800 if av is not None and bv is None: return 1
1801 if av is None and bv is None: continue
1802 if link.getprops().has_key('order'):
1803 if dir == '+':
1804 r = cmp(link.get(av, 'order'),
1805 link.get(bv, 'order'))
1806 if r != 0: return r
1807 elif dir == '-':
1808 r = cmp(link.get(bv, 'order'),
1809 link.get(av, 'order'))
1810 if r != 0: return r
1811 elif link.getkey():
1812 key = link.getkey()
1813 if dir == '+':
1814 r = cmp(link.get(av, key), link.get(bv, key))
1815 if r != 0: return r
1816 elif dir == '-':
1817 r = cmp(link.get(bv, key), link.get(av, key))
1818 if r != 0: return r
1819 else:
1820 if dir == '+':
1821 r = cmp(av, bv)
1822 if r != 0: return r
1823 elif dir == '-':
1824 r = cmp(bv, av)
1825 if r != 0: return r
1827 # Multilink properties are sorted according to how many
1828 # links are present.
1829 elif isinstance(propclass, Multilink):
1830 r = cmp(len(av), len(bv))
1831 if r == 0:
1832 # Compare contents of multilink property if lenghts is
1833 # equal
1834 r = cmp ('.'.join(av), '.'.join(bv))
1835 if r:
1836 if dir == '+':
1837 return r
1838 else:
1839 return -r
1841 else:
1842 # all other types just compare
1843 if dir == '+':
1844 r = cmp(av, bv)
1845 elif dir == '-':
1846 r = cmp(bv, av)
1847 if r != 0: return r
1849 # end for dir, prop in sort, group:
1850 # if all else fails, compare the ids
1851 return cmp(a[0], b[0])
1853 l.sort(sortfun)
1854 return [i[0] for i in l]
1856 def count(self):
1857 '''Get the number of nodes in this class.
1859 If the returned integer is 'numnodes', the ids of all the nodes
1860 in this class run from 1 to numnodes, and numnodes+1 will be the
1861 id of the next node to be created in this class.
1862 '''
1863 return self.db.countnodes(self.classname)
1865 # Manipulating properties:
1867 def getprops(self, protected=1):
1868 '''Return a dictionary mapping property names to property objects.
1869 If the "protected" flag is true, we include protected properties -
1870 those which may not be modified.
1872 In addition to the actual properties on the node, these
1873 methods provide the "creation" and "activity" properties. If the
1874 "protected" flag is true, we include protected properties - those
1875 which may not be modified.
1876 '''
1877 d = self.properties.copy()
1878 if protected:
1879 d['id'] = String()
1880 d['creation'] = hyperdb.Date()
1881 d['activity'] = hyperdb.Date()
1882 d['creator'] = hyperdb.Link('user')
1883 return d
1885 def addprop(self, **properties):
1886 '''Add properties to this class.
1888 The keyword arguments in 'properties' must map names to property
1889 objects, or a TypeError is raised. None of the keys in 'properties'
1890 may collide with the names of existing properties, or a ValueError
1891 is raised before any properties have been added.
1892 '''
1893 for key in properties.keys():
1894 if self.properties.has_key(key):
1895 raise ValueError, key
1896 self.properties.update(properties)
1898 def index(self, nodeid):
1899 '''Add (or refresh) the node to search indexes
1900 '''
1901 # find all the String properties that have indexme
1902 for prop, propclass in self.getprops().items():
1903 if isinstance(propclass, String) and propclass.indexme:
1904 try:
1905 value = str(self.get(nodeid, prop))
1906 except IndexError:
1907 # node no longer exists - entry should be removed
1908 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1909 else:
1910 # and index them under (classname, nodeid, property)
1911 self.db.indexer.add_text((self.classname, nodeid, prop),
1912 value)
1914 #
1915 # Detector interface
1916 #
1917 def audit(self, event, detector):
1918 '''Register a detector
1919 '''
1920 l = self.auditors[event]
1921 if detector not in l:
1922 self.auditors[event].append(detector)
1924 def fireAuditors(self, action, nodeid, newvalues):
1925 '''Fire all registered auditors.
1926 '''
1927 for audit in self.auditors[action]:
1928 audit(self.db, self, nodeid, newvalues)
1930 def react(self, event, detector):
1931 '''Register a detector
1932 '''
1933 l = self.reactors[event]
1934 if detector not in l:
1935 self.reactors[event].append(detector)
1937 def fireReactors(self, action, nodeid, oldvalues):
1938 '''Fire all registered reactors.
1939 '''
1940 for react in self.reactors[action]:
1941 react(self.db, self, nodeid, oldvalues)
1943 class FileClass(Class, hyperdb.FileClass):
1944 '''This class defines a large chunk of data. To support this, it has a
1945 mandatory String property "content" which is typically saved off
1946 externally to the hyperdb.
1948 The default MIME type of this data is defined by the
1949 "default_mime_type" class attribute, which may be overridden by each
1950 node if the class defines a "type" String property.
1951 '''
1952 default_mime_type = 'text/plain'
1954 def create(self, **propvalues):
1955 ''' Snarf the "content" propvalue and store in a file
1956 '''
1957 # we need to fire the auditors now, or the content property won't
1958 # be in propvalues for the auditors to play with
1959 self.fireAuditors('create', None, propvalues)
1961 # now remove the content property so it's not stored in the db
1962 content = propvalues['content']
1963 del propvalues['content']
1965 # do the database create
1966 newid = Class.create_inner(self, **propvalues)
1968 # fire reactors
1969 self.fireReactors('create', newid, None)
1971 # store off the content as a file
1972 self.db.storefile(self.classname, newid, None, content)
1973 return newid
1975 def import_list(self, propnames, proplist):
1976 ''' Trap the "content" property...
1977 '''
1978 # dupe this list so we don't affect others
1979 propnames = propnames[:]
1981 # extract the "content" property from the proplist
1982 i = propnames.index('content')
1983 content = eval(proplist[i])
1984 del propnames[i]
1985 del proplist[i]
1987 # do the normal import
1988 newid = Class.import_list(self, propnames, proplist)
1990 # save off the "content" file
1991 self.db.storefile(self.classname, newid, None, content)
1992 return newid
1994 def get(self, nodeid, propname, default=_marker, cache=1):
1995 ''' trap the content propname and get it from the file
1996 '''
1997 poss_msg = 'Possibly an access right configuration problem.'
1998 if propname == 'content':
1999 try:
2000 return self.db.getfile(self.classname, nodeid, None)
2001 except IOError, (strerror):
2002 # XXX by catching this we donot see an error in the log.
2003 return 'ERROR reading file: %s%s\n%s\n%s'%(
2004 self.classname, nodeid, poss_msg, strerror)
2005 if default is not _marker:
2006 return Class.get(self, nodeid, propname, default, cache=cache)
2007 else:
2008 return Class.get(self, nodeid, propname, cache=cache)
2010 def getprops(self, protected=1):
2011 ''' In addition to the actual properties on the node, these methods
2012 provide the "content" property. If the "protected" flag is true,
2013 we include protected properties - those which may not be
2014 modified.
2015 '''
2016 d = Class.getprops(self, protected=protected).copy()
2017 d['content'] = hyperdb.String()
2018 return d
2020 def index(self, nodeid):
2021 ''' Index the node in the search index.
2023 We want to index the content in addition to the normal String
2024 property indexing.
2025 '''
2026 # perform normal indexing
2027 Class.index(self, nodeid)
2029 # get the content to index
2030 content = self.get(nodeid, 'content')
2032 # figure the mime type
2033 if self.properties.has_key('type'):
2034 mime_type = self.get(nodeid, 'type')
2035 else:
2036 mime_type = self.default_mime_type
2038 # and index!
2039 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
2040 mime_type)
2042 # deviation from spec - was called ItemClass
2043 class IssueClass(Class, roundupdb.IssueClass):
2044 # Overridden methods:
2045 def __init__(self, db, classname, **properties):
2046 '''The newly-created class automatically includes the "messages",
2047 "files", "nosy", and "superseder" properties. If the 'properties'
2048 dictionary attempts to specify any of these properties or a
2049 "creation" or "activity" property, a ValueError is raised.
2050 '''
2051 if not properties.has_key('title'):
2052 properties['title'] = hyperdb.String(indexme='yes')
2053 if not properties.has_key('messages'):
2054 properties['messages'] = hyperdb.Multilink("msg")
2055 if not properties.has_key('files'):
2056 properties['files'] = hyperdb.Multilink("file")
2057 if not properties.has_key('nosy'):
2058 # note: journalling is turned off as it really just wastes
2059 # space. this behaviour may be overridden in an instance
2060 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2061 if not properties.has_key('superseder'):
2062 properties['superseder'] = hyperdb.Multilink(classname)
2063 Class.__init__(self, db, classname, **properties)
2065 #