1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: back_anydbm.py,v 1.105 2003-02-25 10:19:31 richard Exp $
19 '''
20 This module defines a backend that saves the hyperdatabase in a database
21 chosen by anydbm. It is guaranteed to always be available in python
22 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
23 serious bugs, and is not available)
24 '''
26 import whichdb, anydbm, os, marshal, re, weakref, string, copy
27 from roundup import hyperdb, date, password, roundupdb, security
28 from blobfiles import FileStorage
29 from sessions import Sessions, OneTimeKeys
30 from roundup.indexer import Indexer
31 from roundup.backends import locking
32 from roundup.hyperdb import String, Password, Date, Interval, Link, \
33 Multilink, DatabaseError, Boolean, Number, Node
35 #
36 # Now the database
37 #
38 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
39 '''A database for storing records containing flexible data types.
41 Transaction stuff TODO:
42 . check the timestamp of the class file and nuke the cache if it's
43 modified. Do some sort of conflict checking on the dirty stuff.
44 . perhaps detect write collisions (related to above)?
46 '''
47 def __init__(self, config, journaltag=None):
48 '''Open a hyperdatabase given a specifier to some storage.
50 The 'storagelocator' is obtained from config.DATABASE.
51 The meaning of 'storagelocator' depends on the particular
52 implementation of the hyperdatabase. It could be a file name,
53 a directory path, a socket descriptor for a connection to a
54 database over the network, etc.
56 The 'journaltag' is a token that will be attached to the journal
57 entries for any edits done on the database. If 'journaltag' is
58 None, the database is opened in read-only mode: the Class.create(),
59 Class.set(), and Class.retire() methods are disabled.
60 '''
61 self.config, self.journaltag = config, journaltag
62 self.dir = config.DATABASE
63 self.classes = {}
64 self.cache = {} # cache of nodes loaded or created
65 self.dirtynodes = {} # keep track of the dirty nodes by class
66 self.newnodes = {} # keep track of the new nodes by class
67 self.destroyednodes = {}# keep track of the destroyed nodes by class
68 self.transactions = []
69 self.indexer = Indexer(self.dir)
70 self.sessions = Sessions(self.config)
71 self.otks = OneTimeKeys(self.config)
72 self.security = security.Security(self)
73 # ensure files are group readable and writable
74 os.umask(0002)
76 # lock it
77 lockfilenm = os.path.join(self.dir, 'lock')
78 self.lockfile = locking.acquire_lock(lockfilenm)
79 self.lockfile.write(str(os.getpid()))
80 self.lockfile.flush()
82 def post_init(self):
83 ''' Called once the schema initialisation has finished.
84 '''
85 # reindex the db if necessary
86 if self.indexer.should_reindex():
87 self.reindex()
89 # figure the "curuserid"
90 if self.journaltag is None:
91 self.curuserid = None
92 elif self.journaltag == 'admin':
93 # admin user may not exist, but always has ID 1
94 self.curuserid = '1'
95 else:
96 self.curuserid = self.user.lookup(self.journaltag)
98 def reindex(self):
99 for klass in self.classes.values():
100 for nodeid in klass.list():
101 klass.index(nodeid)
102 self.indexer.save_index()
104 def __repr__(self):
105 return '<back_anydbm instance at %x>'%id(self)
107 #
108 # Classes
109 #
110 def __getattr__(self, classname):
111 '''A convenient way of calling self.getclass(classname).'''
112 if self.classes.has_key(classname):
113 if __debug__:
114 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
115 return self.classes[classname]
116 raise AttributeError, classname
118 def addclass(self, cl):
119 if __debug__:
120 print >>hyperdb.DEBUG, 'addclass', (self, cl)
121 cn = cl.classname
122 if self.classes.has_key(cn):
123 raise ValueError, cn
124 self.classes[cn] = cl
126 def getclasses(self):
127 '''Return a list of the names of all existing classes.'''
128 if __debug__:
129 print >>hyperdb.DEBUG, 'getclasses', (self,)
130 l = self.classes.keys()
131 l.sort()
132 return l
134 def getclass(self, classname):
135 '''Get the Class object representing a particular class.
137 If 'classname' is not a valid class name, a KeyError is raised.
138 '''
139 if __debug__:
140 print >>hyperdb.DEBUG, 'getclass', (self, classname)
141 try:
142 return self.classes[classname]
143 except KeyError:
144 raise KeyError, 'There is no class called "%s"'%classname
146 #
147 # Class DBs
148 #
149 def clear(self):
150 '''Delete all database contents
151 '''
152 if __debug__:
153 print >>hyperdb.DEBUG, 'clear', (self,)
154 for cn in self.classes.keys():
155 for dummy in 'nodes', 'journals':
156 path = os.path.join(self.dir, 'journals.%s'%cn)
157 if os.path.exists(path):
158 os.remove(path)
159 elif os.path.exists(path+'.db'): # dbm appends .db
160 os.remove(path+'.db')
162 def getclassdb(self, classname, mode='r'):
163 ''' grab a connection to the class db that will be used for
164 multiple actions
165 '''
166 if __debug__:
167 print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode)
168 return self.opendb('nodes.%s'%classname, mode)
170 def determine_db_type(self, path):
171 ''' determine which DB wrote the class file
172 '''
173 db_type = ''
174 if os.path.exists(path):
175 db_type = whichdb.whichdb(path)
176 if not db_type:
177 raise DatabaseError, "Couldn't identify database type"
178 elif os.path.exists(path+'.db'):
179 # if the path ends in '.db', it's a dbm database, whether
180 # anydbm says it's dbhash or not!
181 db_type = 'dbm'
182 return db_type
184 def opendb(self, name, mode):
185 '''Low-level database opener that gets around anydbm/dbm
186 eccentricities.
187 '''
188 if __debug__:
189 print >>hyperdb.DEBUG, 'opendb', (self, name, mode)
191 # figure the class db type
192 path = os.path.join(os.getcwd(), self.dir, name)
193 db_type = self.determine_db_type(path)
195 # new database? let anydbm pick the best dbm
196 if not db_type:
197 if __debug__:
198 print >>hyperdb.DEBUG, "opendb anydbm.open(%r, 'c')"%path
199 return anydbm.open(path, 'c')
201 # open the database with the correct module
202 try:
203 dbm = __import__(db_type)
204 except ImportError:
205 raise DatabaseError, \
206 "Couldn't open database - the required module '%s'"\
207 " is not available"%db_type
208 if __debug__:
209 print >>hyperdb.DEBUG, "opendb %r.open(%r, %r)"%(db_type, path,
210 mode)
211 return dbm.open(path, mode)
213 #
214 # Node IDs
215 #
216 def newid(self, classname):
217 ''' Generate a new id for the given class
218 '''
219 # open the ids DB - create if if doesn't exist
220 db = self.opendb('_ids', 'c')
221 if db.has_key(classname):
222 newid = db[classname] = str(int(db[classname]) + 1)
223 else:
224 # the count() bit is transitional - older dbs won't start at 1
225 newid = str(self.getclass(classname).count()+1)
226 db[classname] = newid
227 db.close()
228 return newid
230 def setid(self, classname, setid):
231 ''' Set the id counter: used during import of database
232 '''
233 # open the ids DB - create if if doesn't exist
234 db = self.opendb('_ids', 'c')
235 db[classname] = str(setid)
236 db.close()
238 #
239 # Nodes
240 #
241 def addnode(self, classname, nodeid, node):
242 ''' add the specified node to its class's db
243 '''
244 if __debug__:
245 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
247 # we'll be supplied these props if we're doing an import
248 if not node.has_key('creator'):
249 # add in the "calculated" properties (dupe so we don't affect
250 # calling code's node assumptions)
251 node = node.copy()
252 node['creator'] = self.curuserid
253 node['creation'] = node['activity'] = date.Date()
255 self.newnodes.setdefault(classname, {})[nodeid] = 1
256 self.cache.setdefault(classname, {})[nodeid] = node
257 self.savenode(classname, nodeid, node)
259 def setnode(self, classname, nodeid, node):
260 ''' change the specified node
261 '''
262 if __debug__:
263 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
264 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
266 # update the activity time (dupe so we don't affect
267 # calling code's node assumptions)
268 node = node.copy()
269 node['activity'] = date.Date()
271 # can't set without having already loaded the node
272 self.cache[classname][nodeid] = node
273 self.savenode(classname, nodeid, node)
275 def savenode(self, classname, nodeid, node):
276 ''' perform the saving of data specified by the set/addnode
277 '''
278 if __debug__:
279 print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node)
280 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
282 def getnode(self, classname, nodeid, db=None, cache=1):
283 ''' get a node from the database
284 '''
285 if __debug__:
286 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid, db)
287 if cache:
288 # try the cache
289 cache_dict = self.cache.setdefault(classname, {})
290 if cache_dict.has_key(nodeid):
291 if __debug__:
292 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
293 nodeid)
294 return cache_dict[nodeid]
296 if __debug__:
297 print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid)
299 # get from the database and save in the cache
300 if db is None:
301 db = self.getclassdb(classname)
302 if not db.has_key(nodeid):
303 # try the cache - might be a brand-new node
304 cache_dict = self.cache.setdefault(classname, {})
305 if cache_dict.has_key(nodeid):
306 if __debug__:
307 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
308 nodeid)
309 return cache_dict[nodeid]
310 raise IndexError, "no such %s %s"%(classname, nodeid)
312 # check the uncommitted, destroyed nodes
313 if (self.destroyednodes.has_key(classname) and
314 self.destroyednodes[classname].has_key(nodeid)):
315 raise IndexError, "no such %s %s"%(classname, nodeid)
317 # decode
318 res = marshal.loads(db[nodeid])
320 # reverse the serialisation
321 res = self.unserialise(classname, res)
323 # store off in the cache dict
324 if cache:
325 cache_dict[nodeid] = res
327 return res
329 def destroynode(self, classname, nodeid):
330 '''Remove a node from the database. Called exclusively by the
331 destroy() method on Class.
332 '''
333 if __debug__:
334 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
336 # remove from cache and newnodes if it's there
337 if (self.cache.has_key(classname) and
338 self.cache[classname].has_key(nodeid)):
339 del self.cache[classname][nodeid]
340 if (self.newnodes.has_key(classname) and
341 self.newnodes[classname].has_key(nodeid)):
342 del self.newnodes[classname][nodeid]
344 # see if there's any obvious commit actions that we should get rid of
345 for entry in self.transactions[:]:
346 if entry[1][:2] == (classname, nodeid):
347 self.transactions.remove(entry)
349 # add to the destroyednodes map
350 self.destroyednodes.setdefault(classname, {})[nodeid] = 1
352 # add the destroy commit action
353 self.transactions.append((self.doDestroyNode, (classname, nodeid)))
355 def serialise(self, classname, node):
356 '''Copy the node contents, converting non-marshallable data into
357 marshallable data.
358 '''
359 if __debug__:
360 print >>hyperdb.DEBUG, 'serialise', classname, node
361 properties = self.getclass(classname).getprops()
362 d = {}
363 for k, v in node.items():
364 # if the property doesn't exist, or is the "retired" flag then
365 # it won't be in the properties dict
366 if not properties.has_key(k):
367 d[k] = v
368 continue
370 # get the property spec
371 prop = properties[k]
373 if isinstance(prop, Password) and v is not None:
374 d[k] = str(v)
375 elif isinstance(prop, Date) and v is not None:
376 d[k] = v.serialise()
377 elif isinstance(prop, Interval) and v is not None:
378 d[k] = v.serialise()
379 else:
380 d[k] = v
381 return d
383 def unserialise(self, classname, node):
384 '''Decode the marshalled node data
385 '''
386 if __debug__:
387 print >>hyperdb.DEBUG, 'unserialise', classname, node
388 properties = self.getclass(classname).getprops()
389 d = {}
390 for k, v in node.items():
391 # if the property doesn't exist, or is the "retired" flag then
392 # it won't be in the properties dict
393 if not properties.has_key(k):
394 d[k] = v
395 continue
397 # get the property spec
398 prop = properties[k]
400 if isinstance(prop, Date) and v is not None:
401 d[k] = date.Date(v)
402 elif isinstance(prop, Interval) and v is not None:
403 d[k] = date.Interval(v)
404 elif isinstance(prop, Password) and v is not None:
405 p = password.Password()
406 p.unpack(v)
407 d[k] = p
408 else:
409 d[k] = v
410 return d
412 def hasnode(self, classname, nodeid, db=None):
413 ''' determine if the database has a given node
414 '''
415 if __debug__:
416 print >>hyperdb.DEBUG, 'hasnode', (self, classname, nodeid, db)
418 # try the cache
419 cache = self.cache.setdefault(classname, {})
420 if cache.has_key(nodeid):
421 if __debug__:
422 print >>hyperdb.TRACE, 'has %s %s cached'%(classname, nodeid)
423 return 1
424 if __debug__:
425 print >>hyperdb.TRACE, 'has %s %s'%(classname, nodeid)
427 # not in the cache - check the database
428 if db is None:
429 db = self.getclassdb(classname)
430 res = db.has_key(nodeid)
431 return res
433 def countnodes(self, classname, db=None):
434 if __debug__:
435 print >>hyperdb.DEBUG, 'countnodes', (self, classname, db)
437 count = 0
439 # include the uncommitted nodes
440 if self.newnodes.has_key(classname):
441 count += len(self.newnodes[classname])
442 if self.destroyednodes.has_key(classname):
443 count -= len(self.destroyednodes[classname])
445 # and count those in the DB
446 if db is None:
447 db = self.getclassdb(classname)
448 count = count + len(db.keys())
449 return count
451 def getnodeids(self, classname, db=None):
452 if __debug__:
453 print >>hyperdb.DEBUG, 'getnodeids', (self, classname, db)
455 res = []
457 # start off with the new nodes
458 if self.newnodes.has_key(classname):
459 res += self.newnodes[classname].keys()
461 if db is None:
462 db = self.getclassdb(classname)
463 res = res + db.keys()
465 # remove the uncommitted, destroyed nodes
466 if self.destroyednodes.has_key(classname):
467 for nodeid in self.destroyednodes[classname].keys():
468 if db.has_key(nodeid):
469 res.remove(nodeid)
471 return res
474 #
475 # Files - special node properties
476 # inherited from FileStorage
478 #
479 # Journal
480 #
481 def addjournal(self, classname, nodeid, action, params, creator=None,
482 creation=None):
483 ''' Journal the Action
484 'action' may be:
486 'create' or 'set' -- 'params' is a dictionary of property values
487 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
488 'retire' -- 'params' is None
489 '''
490 if __debug__:
491 print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid,
492 action, params, creator, creation)
493 self.transactions.append((self.doSaveJournal, (classname, nodeid,
494 action, params, creator, creation)))
496 def getjournal(self, classname, nodeid):
497 ''' get the journal for id
499 Raise IndexError if the node doesn't exist (as per history()'s
500 API)
501 '''
502 if __debug__:
503 print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid)
504 # attempt to open the journal - in some rare cases, the journal may
505 # not exist
506 try:
507 db = self.opendb('journals.%s'%classname, 'r')
508 except anydbm.error, error:
509 if str(error) == "need 'c' or 'n' flag to open new db":
510 raise IndexError, 'no such %s %s'%(classname, nodeid)
511 elif error.args[0] != 2:
512 raise
513 raise IndexError, 'no such %s %s'%(classname, nodeid)
514 try:
515 journal = marshal.loads(db[nodeid])
516 except KeyError:
517 db.close()
518 raise IndexError, 'no such %s %s'%(classname, nodeid)
519 db.close()
520 res = []
521 for nodeid, date_stamp, user, action, params in journal:
522 res.append((nodeid, date.Date(date_stamp), user, action, params))
523 return res
525 def pack(self, pack_before):
526 ''' Delete all journal entries except "create" before 'pack_before'.
527 '''
528 if __debug__:
529 print >>hyperdb.DEBUG, 'packjournal', (self, pack_before)
531 pack_before = pack_before.serialise()
532 for classname in self.getclasses():
533 # get the journal db
534 db_name = 'journals.%s'%classname
535 path = os.path.join(os.getcwd(), self.dir, classname)
536 db_type = self.determine_db_type(path)
537 db = self.opendb(db_name, 'w')
539 for key in db.keys():
540 # get the journal for this db entry
541 journal = marshal.loads(db[key])
542 l = []
543 last_set_entry = None
544 for entry in journal:
545 # unpack the entry
546 (nodeid, date_stamp, self.journaltag, action,
547 params) = entry
548 # if the entry is after the pack date, _or_ the initial
549 # create entry, then it stays
550 if date_stamp > pack_before or action == 'create':
551 l.append(entry)
552 db[key] = marshal.dumps(l)
553 if db_type == 'gdbm':
554 db.reorganize()
555 db.close()
558 #
559 # Basic transaction support
560 #
561 def commit(self):
562 ''' Commit the current transactions.
563 '''
564 if __debug__:
565 print >>hyperdb.DEBUG, 'commit', (self,)
567 # keep a handle to all the database files opened
568 self.databases = {}
570 # now, do all the transactions
571 reindex = {}
572 for method, args in self.transactions:
573 reindex[method(*args)] = 1
575 # now close all the database files
576 for db in self.databases.values():
577 db.close()
578 del self.databases
580 # reindex the nodes that request it
581 for classname, nodeid in filter(None, reindex.keys()):
582 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
583 self.getclass(classname).index(nodeid)
585 # save the indexer state
586 self.indexer.save_index()
588 self.clearCache()
590 def clearCache(self):
591 # all transactions committed, back to normal
592 self.cache = {}
593 self.dirtynodes = {}
594 self.newnodes = {}
595 self.destroyednodes = {}
596 self.transactions = []
598 def getCachedClassDB(self, classname):
599 ''' get the class db, looking in our cache of databases for commit
600 '''
601 # get the database handle
602 db_name = 'nodes.%s'%classname
603 if not self.databases.has_key(db_name):
604 self.databases[db_name] = self.getclassdb(classname, 'c')
605 return self.databases[db_name]
607 def doSaveNode(self, classname, nodeid, node):
608 if __debug__:
609 print >>hyperdb.DEBUG, 'doSaveNode', (self, classname, nodeid,
610 node)
612 db = self.getCachedClassDB(classname)
614 # now save the marshalled data
615 db[nodeid] = marshal.dumps(self.serialise(classname, node))
617 # return the classname, nodeid so we reindex this content
618 return (classname, nodeid)
620 def getCachedJournalDB(self, classname):
621 ''' get the journal db, looking in our cache of databases for commit
622 '''
623 # get the database handle
624 db_name = 'journals.%s'%classname
625 if not self.databases.has_key(db_name):
626 self.databases[db_name] = self.opendb(db_name, 'c')
627 return self.databases[db_name]
629 def doSaveJournal(self, classname, nodeid, action, params, creator,
630 creation):
631 # serialise the parameters now if necessary
632 if isinstance(params, type({})):
633 if action in ('set', 'create'):
634 params = self.serialise(classname, params)
636 # handle supply of the special journalling parameters (usually
637 # supplied on importing an existing database)
638 if creator:
639 journaltag = creator
640 else:
641 journaltag = self.curuserid
642 if creation:
643 journaldate = creation.serialise()
644 else:
645 journaldate = date.Date().serialise()
647 # create the journal entry
648 entry = (nodeid, journaldate, journaltag, action, params)
650 if __debug__:
651 print >>hyperdb.DEBUG, 'doSaveJournal', entry
653 db = self.getCachedJournalDB(classname)
655 # now insert the journal entry
656 if db.has_key(nodeid):
657 # append to existing
658 s = db[nodeid]
659 l = marshal.loads(s)
660 l.append(entry)
661 else:
662 l = [entry]
664 db[nodeid] = marshal.dumps(l)
666 def doDestroyNode(self, classname, nodeid):
667 if __debug__:
668 print >>hyperdb.DEBUG, 'doDestroyNode', (self, classname, nodeid)
670 # delete from the class database
671 db = self.getCachedClassDB(classname)
672 if db.has_key(nodeid):
673 del db[nodeid]
675 # delete from the database
676 db = self.getCachedJournalDB(classname)
677 if db.has_key(nodeid):
678 del db[nodeid]
680 # return the classname, nodeid so we reindex this content
681 return (classname, nodeid)
683 def rollback(self):
684 ''' Reverse all actions from the current transaction.
685 '''
686 if __debug__:
687 print >>hyperdb.DEBUG, 'rollback', (self, )
688 for method, args in self.transactions:
689 # delete temporary files
690 if method == self.doStoreFile:
691 self.rollbackStoreFile(*args)
692 self.cache = {}
693 self.dirtynodes = {}
694 self.newnodes = {}
695 self.destroyednodes = {}
696 self.transactions = []
698 def close(self):
699 ''' Nothing to do
700 '''
701 if self.lockfile is not None:
702 locking.release_lock(self.lockfile)
703 if self.lockfile is not None:
704 self.lockfile.close()
705 self.lockfile = None
707 _marker = []
708 class Class(hyperdb.Class):
709 '''The handle to a particular class of nodes in a hyperdatabase.'''
711 def __init__(self, db, classname, **properties):
712 '''Create a new class with a given name and property specification.
714 'classname' must not collide with the name of an existing class,
715 or a ValueError is raised. The keyword arguments in 'properties'
716 must map names to property objects, or a TypeError is raised.
717 '''
718 if (properties.has_key('creation') or properties.has_key('activity')
719 or properties.has_key('creator')):
720 raise ValueError, '"creation", "activity" and "creator" are '\
721 'reserved'
723 self.classname = classname
724 self.properties = properties
725 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
726 self.key = ''
728 # should we journal changes (default yes)
729 self.do_journal = 1
731 # do the db-related init stuff
732 db.addclass(self)
734 self.auditors = {'create': [], 'set': [], 'retire': []}
735 self.reactors = {'create': [], 'set': [], 'retire': []}
737 def enableJournalling(self):
738 '''Turn journalling on for this class
739 '''
740 self.do_journal = 1
742 def disableJournalling(self):
743 '''Turn journalling off for this class
744 '''
745 self.do_journal = 0
747 # Editing nodes:
749 def create(self, **propvalues):
750 '''Create a new node of this class and return its id.
752 The keyword arguments in 'propvalues' map property names to values.
754 The values of arguments must be acceptable for the types of their
755 corresponding properties or a TypeError is raised.
757 If this class has a key property, it must be present and its value
758 must not collide with other key strings or a ValueError is raised.
760 Any other properties on this class that are missing from the
761 'propvalues' dictionary are set to None.
763 If an id in a link or multilink property does not refer to a valid
764 node, an IndexError is raised.
766 These operations trigger detectors and can be vetoed. Attempts
767 to modify the "creation" or "activity" properties cause a KeyError.
768 '''
769 self.fireAuditors('create', None, propvalues)
770 newid = self.create_inner(**propvalues)
771 self.fireReactors('create', newid, None)
772 return newid
774 def create_inner(self, **propvalues):
775 ''' Called by create, in-between the audit and react calls.
776 '''
777 if propvalues.has_key('id'):
778 raise KeyError, '"id" is reserved'
780 if self.db.journaltag is None:
781 raise DatabaseError, 'Database open read-only'
783 if propvalues.has_key('creation') or propvalues.has_key('activity'):
784 raise KeyError, '"creation" and "activity" are reserved'
785 # new node's id
786 newid = self.db.newid(self.classname)
788 # validate propvalues
789 num_re = re.compile('^\d+$')
790 for key, value in propvalues.items():
791 if key == self.key:
792 try:
793 self.lookup(value)
794 except KeyError:
795 pass
796 else:
797 raise ValueError, 'node with key "%s" exists'%value
799 # try to handle this property
800 try:
801 prop = self.properties[key]
802 except KeyError:
803 raise KeyError, '"%s" has no property "%s"'%(self.classname,
804 key)
806 if value is not None and isinstance(prop, Link):
807 if type(value) != type(''):
808 raise ValueError, 'link value must be String'
809 link_class = self.properties[key].classname
810 # if it isn't a number, it's a key
811 if not num_re.match(value):
812 try:
813 value = self.db.classes[link_class].lookup(value)
814 except (TypeError, KeyError):
815 raise IndexError, 'new property "%s": %s not a %s'%(
816 key, value, link_class)
817 elif not self.db.getclass(link_class).hasnode(value):
818 raise IndexError, '%s has no node %s'%(link_class, value)
820 # save off the value
821 propvalues[key] = value
823 # register the link with the newly linked node
824 if self.do_journal and self.properties[key].do_journal:
825 self.db.addjournal(link_class, value, 'link',
826 (self.classname, newid, key))
828 elif isinstance(prop, Multilink):
829 if type(value) != type([]):
830 raise TypeError, 'new property "%s" not a list of ids'%key
832 # clean up and validate the list of links
833 link_class = self.properties[key].classname
834 l = []
835 for entry in value:
836 if type(entry) != type(''):
837 raise ValueError, '"%s" multilink value (%r) '\
838 'must contain Strings'%(key, value)
839 # if it isn't a number, it's a key
840 if not num_re.match(entry):
841 try:
842 entry = self.db.classes[link_class].lookup(entry)
843 except (TypeError, KeyError):
844 raise IndexError, 'new property "%s": %s not a %s'%(
845 key, entry, self.properties[key].classname)
846 l.append(entry)
847 value = l
848 propvalues[key] = value
850 # handle additions
851 for nodeid in value:
852 if not self.db.getclass(link_class).hasnode(nodeid):
853 raise IndexError, '%s has no node %s'%(link_class,
854 nodeid)
855 # register the link with the newly linked node
856 if self.do_journal and self.properties[key].do_journal:
857 self.db.addjournal(link_class, nodeid, 'link',
858 (self.classname, newid, key))
860 elif isinstance(prop, String):
861 if type(value) != type('') and type(value) != type(u''):
862 raise TypeError, 'new property "%s" not a string'%key
864 elif isinstance(prop, Password):
865 if not isinstance(value, password.Password):
866 raise TypeError, 'new property "%s" not a Password'%key
868 elif isinstance(prop, Date):
869 if value is not None and not isinstance(value, date.Date):
870 raise TypeError, 'new property "%s" not a Date'%key
872 elif isinstance(prop, Interval):
873 if value is not None and not isinstance(value, date.Interval):
874 raise TypeError, 'new property "%s" not an Interval'%key
876 elif value is not None and isinstance(prop, Number):
877 try:
878 float(value)
879 except ValueError:
880 raise TypeError, 'new property "%s" not numeric'%key
882 elif value is not None and isinstance(prop, Boolean):
883 try:
884 int(value)
885 except ValueError:
886 raise TypeError, 'new property "%s" not boolean'%key
888 # make sure there's data where there needs to be
889 for key, prop in self.properties.items():
890 if propvalues.has_key(key):
891 continue
892 if key == self.key:
893 raise ValueError, 'key property "%s" is required'%key
894 if isinstance(prop, Multilink):
895 propvalues[key] = []
896 else:
897 propvalues[key] = None
899 # done
900 self.db.addnode(self.classname, newid, propvalues)
901 if self.do_journal:
902 self.db.addjournal(self.classname, newid, 'create', {})
904 return newid
906 def export_list(self, propnames, nodeid):
907 ''' Export a node - generate a list of CSV-able data in the order
908 specified by propnames for the given node.
909 '''
910 properties = self.getprops()
911 l = []
912 for prop in propnames:
913 proptype = properties[prop]
914 value = self.get(nodeid, prop)
915 # "marshal" data where needed
916 if value is None:
917 pass
918 elif isinstance(proptype, hyperdb.Date):
919 value = value.get_tuple()
920 elif isinstance(proptype, hyperdb.Interval):
921 value = value.get_tuple()
922 elif isinstance(proptype, hyperdb.Password):
923 value = str(value)
924 l.append(repr(value))
925 return l
927 def import_list(self, propnames, proplist):
928 ''' Import a node - all information including "id" is present and
929 should not be sanity checked. Triggers are not triggered. The
930 journal should be initialised using the "creator" and "created"
931 information.
933 Return the nodeid of the node imported.
934 '''
935 if self.db.journaltag is None:
936 raise DatabaseError, 'Database open read-only'
937 properties = self.getprops()
939 # make the new node's property map
940 d = {}
941 for i in range(len(propnames)):
942 # Use eval to reverse the repr() used to output the CSV
943 value = eval(proplist[i])
945 # Figure the property for this column
946 propname = propnames[i]
947 prop = properties[propname]
949 # "unmarshal" where necessary
950 if propname == 'id':
951 newid = value
952 continue
953 elif value is None:
954 # don't set Nones
955 continue
956 elif isinstance(prop, hyperdb.Date):
957 value = date.Date(value)
958 elif isinstance(prop, hyperdb.Interval):
959 value = date.Interval(value)
960 elif isinstance(prop, hyperdb.Password):
961 pwd = password.Password()
962 pwd.unpack(value)
963 value = pwd
964 d[propname] = value
966 # add the node and journal
967 self.db.addnode(self.classname, newid, d)
969 # extract the journalling stuff and nuke it
970 if d.has_key('creator'):
971 creator = d['creator']
972 del d['creator']
973 else:
974 creator = None
975 if d.has_key('creation'):
976 creation = d['creation']
977 del d['creation']
978 else:
979 creation = None
980 if d.has_key('activity'):
981 del d['activity']
982 self.db.addjournal(self.classname, newid, 'create', {}, creator,
983 creation)
984 return newid
986 def get(self, nodeid, propname, default=_marker, cache=1):
987 '''Get the value of a property on an existing node of this class.
989 'nodeid' must be the id of an existing node of this class or an
990 IndexError is raised. 'propname' must be the name of a property
991 of this class or a KeyError is raised.
993 'cache' indicates whether the transaction cache should be queried
994 for the node. If the node has been modified and you need to
995 determine what its values prior to modification are, you need to
996 set cache=0.
998 Attempts to get the "creation" or "activity" properties should
999 do the right thing.
1000 '''
1001 if propname == 'id':
1002 return nodeid
1004 # get the node's dict
1005 d = self.db.getnode(self.classname, nodeid, cache=cache)
1007 # check for one of the special props
1008 if propname == 'creation':
1009 if d.has_key('creation'):
1010 return d['creation']
1011 if not self.do_journal:
1012 raise ValueError, 'Journalling is disabled for this class'
1013 journal = self.db.getjournal(self.classname, nodeid)
1014 if journal:
1015 return self.db.getjournal(self.classname, nodeid)[0][1]
1016 else:
1017 # on the strange chance that there's no journal
1018 return date.Date()
1019 if propname == 'activity':
1020 if d.has_key('activity'):
1021 return d['activity']
1022 if not self.do_journal:
1023 raise ValueError, 'Journalling is disabled for this class'
1024 journal = self.db.getjournal(self.classname, nodeid)
1025 if journal:
1026 return self.db.getjournal(self.classname, nodeid)[-1][1]
1027 else:
1028 # on the strange chance that there's no journal
1029 return date.Date()
1030 if propname == 'creator':
1031 if d.has_key('creator'):
1032 return d['creator']
1033 if not self.do_journal:
1034 raise ValueError, 'Journalling is disabled for this class'
1035 journal = self.db.getjournal(self.classname, nodeid)
1036 if journal:
1037 num_re = re.compile('^\d+$')
1038 value = self.db.getjournal(self.classname, nodeid)[0][2]
1039 if num_re.match(value):
1040 return value
1041 else:
1042 # old-style "username" journal tag
1043 try:
1044 return self.db.user.lookup(value)
1045 except KeyError:
1046 # user's been retired, return admin
1047 return '1'
1048 else:
1049 return self.db.curuserid
1051 # get the property (raises KeyErorr if invalid)
1052 prop = self.properties[propname]
1054 if not d.has_key(propname):
1055 if default is _marker:
1056 if isinstance(prop, Multilink):
1057 return []
1058 else:
1059 return None
1060 else:
1061 return default
1063 # return a dupe of the list so code doesn't get confused
1064 if isinstance(prop, Multilink):
1065 return d[propname][:]
1067 return d[propname]
1069 # not in spec
1070 def getnode(self, nodeid, cache=1):
1071 ''' Return a convenience wrapper for the node.
1073 'nodeid' must be the id of an existing node of this class or an
1074 IndexError is raised.
1076 'cache' indicates whether the transaction cache should be queried
1077 for the node. If the node has been modified and you need to
1078 determine what its values prior to modification are, you need to
1079 set cache=0.
1080 '''
1081 return Node(self, nodeid, cache=cache)
1083 def set(self, nodeid, **propvalues):
1084 '''Modify a property on an existing node of this class.
1086 'nodeid' must be the id of an existing node of this class or an
1087 IndexError is raised.
1089 Each key in 'propvalues' must be the name of a property of this
1090 class or a KeyError is raised.
1092 All values in 'propvalues' must be acceptable types for their
1093 corresponding properties or a TypeError is raised.
1095 If the value of the key property is set, it must not collide with
1096 other key strings or a ValueError is raised.
1098 If the value of a Link or Multilink property contains an invalid
1099 node id, a ValueError is raised.
1101 These operations trigger detectors and can be vetoed. Attempts
1102 to modify the "creation" or "activity" properties cause a KeyError.
1103 '''
1104 if not propvalues:
1105 return propvalues
1107 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1108 raise KeyError, '"creation" and "activity" are reserved'
1110 if propvalues.has_key('id'):
1111 raise KeyError, '"id" is reserved'
1113 if self.db.journaltag is None:
1114 raise DatabaseError, 'Database open read-only'
1116 self.fireAuditors('set', nodeid, propvalues)
1117 # Take a copy of the node dict so that the subsequent set
1118 # operation doesn't modify the oldvalues structure.
1119 try:
1120 # try not using the cache initially
1121 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid,
1122 cache=0))
1123 except IndexError:
1124 # this will be needed if somone does a create() and set()
1125 # with no intervening commit()
1126 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1128 node = self.db.getnode(self.classname, nodeid)
1129 if node.has_key(self.db.RETIRED_FLAG):
1130 raise IndexError
1131 num_re = re.compile('^\d+$')
1133 # if the journal value is to be different, store it in here
1134 journalvalues = {}
1136 for propname, value in propvalues.items():
1137 # check to make sure we're not duplicating an existing key
1138 if propname == self.key and node[propname] != value:
1139 try:
1140 self.lookup(value)
1141 except KeyError:
1142 pass
1143 else:
1144 raise ValueError, 'node with key "%s" exists'%value
1146 # this will raise the KeyError if the property isn't valid
1147 # ... we don't use getprops() here because we only care about
1148 # the writeable properties.
1149 try:
1150 prop = self.properties[propname]
1151 except KeyError:
1152 raise KeyError, '"%s" has no property named "%s"'%(
1153 self.classname, propname)
1155 # if the value's the same as the existing value, no sense in
1156 # doing anything
1157 current = node.get(propname, None)
1158 if value == current:
1159 del propvalues[propname]
1160 continue
1161 journalvalues[propname] = current
1163 # do stuff based on the prop type
1164 if isinstance(prop, Link):
1165 link_class = prop.classname
1166 # if it isn't a number, it's a key
1167 if value is not None and not isinstance(value, type('')):
1168 raise ValueError, 'property "%s" link value be a string'%(
1169 propname)
1170 if isinstance(value, type('')) and not num_re.match(value):
1171 try:
1172 value = self.db.classes[link_class].lookup(value)
1173 except (TypeError, KeyError):
1174 raise IndexError, 'new property "%s": %s not a %s'%(
1175 propname, value, prop.classname)
1177 if (value is not None and
1178 not self.db.getclass(link_class).hasnode(value)):
1179 raise IndexError, '%s has no node %s'%(link_class, value)
1181 if self.do_journal and prop.do_journal:
1182 # register the unlink with the old linked node
1183 if node.has_key(propname) and node[propname] is not None:
1184 self.db.addjournal(link_class, node[propname], 'unlink',
1185 (self.classname, nodeid, propname))
1187 # register the link with the newly linked node
1188 if value is not None:
1189 self.db.addjournal(link_class, value, 'link',
1190 (self.classname, nodeid, propname))
1192 elif isinstance(prop, Multilink):
1193 if type(value) != type([]):
1194 raise TypeError, 'new property "%s" not a list of'\
1195 ' ids'%propname
1196 link_class = self.properties[propname].classname
1197 l = []
1198 for entry in value:
1199 # if it isn't a number, it's a key
1200 if type(entry) != type(''):
1201 raise ValueError, 'new property "%s" link value ' \
1202 'must be a string'%propname
1203 if not num_re.match(entry):
1204 try:
1205 entry = self.db.classes[link_class].lookup(entry)
1206 except (TypeError, KeyError):
1207 raise IndexError, 'new property "%s": %s not a %s'%(
1208 propname, entry,
1209 self.properties[propname].classname)
1210 l.append(entry)
1211 value = l
1212 propvalues[propname] = value
1214 # figure the journal entry for this property
1215 add = []
1216 remove = []
1218 # handle removals
1219 if node.has_key(propname):
1220 l = node[propname]
1221 else:
1222 l = []
1223 for id in l[:]:
1224 if id in value:
1225 continue
1226 # register the unlink with the old linked node
1227 if self.do_journal and self.properties[propname].do_journal:
1228 self.db.addjournal(link_class, id, 'unlink',
1229 (self.classname, nodeid, propname))
1230 l.remove(id)
1231 remove.append(id)
1233 # handle additions
1234 for id in value:
1235 if not self.db.getclass(link_class).hasnode(id):
1236 raise IndexError, '%s has no node %s'%(link_class, id)
1237 if id in l:
1238 continue
1239 # register the link with the newly linked node
1240 if self.do_journal and self.properties[propname].do_journal:
1241 self.db.addjournal(link_class, id, 'link',
1242 (self.classname, nodeid, propname))
1243 l.append(id)
1244 add.append(id)
1246 # figure the journal entry
1247 l = []
1248 if add:
1249 l.append(('+', add))
1250 if remove:
1251 l.append(('-', remove))
1252 if l:
1253 journalvalues[propname] = tuple(l)
1255 elif isinstance(prop, String):
1256 if value is not None and type(value) != type('') and type(value) != type(u''):
1257 raise TypeError, 'new property "%s" not a string'%propname
1259 elif isinstance(prop, Password):
1260 if not isinstance(value, password.Password):
1261 raise TypeError, 'new property "%s" not a Password'%propname
1262 propvalues[propname] = value
1264 elif value is not None and isinstance(prop, Date):
1265 if not isinstance(value, date.Date):
1266 raise TypeError, 'new property "%s" not a Date'% propname
1267 propvalues[propname] = value
1269 elif value is not None and isinstance(prop, Interval):
1270 if not isinstance(value, date.Interval):
1271 raise TypeError, 'new property "%s" not an '\
1272 'Interval'%propname
1273 propvalues[propname] = value
1275 elif value is not None and isinstance(prop, Number):
1276 try:
1277 float(value)
1278 except ValueError:
1279 raise TypeError, 'new property "%s" not numeric'%propname
1281 elif value is not None and isinstance(prop, Boolean):
1282 try:
1283 int(value)
1284 except ValueError:
1285 raise TypeError, 'new property "%s" not boolean'%propname
1287 node[propname] = value
1289 # nothing to do?
1290 if not propvalues:
1291 return propvalues
1293 # do the set, and journal it
1294 self.db.setnode(self.classname, nodeid, node)
1296 if self.do_journal:
1297 self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
1299 self.fireReactors('set', nodeid, oldvalues)
1301 return propvalues
1303 def retire(self, nodeid):
1304 '''Retire a node.
1306 The properties on the node remain available from the get() method,
1307 and the node's id is never reused.
1309 Retired nodes are not returned by the find(), list(), or lookup()
1310 methods, and other nodes may reuse the values of their key properties.
1312 These operations trigger detectors and can be vetoed. Attempts
1313 to modify the "creation" or "activity" properties cause a KeyError.
1314 '''
1315 if self.db.journaltag is None:
1316 raise DatabaseError, 'Database open read-only'
1318 self.fireAuditors('retire', nodeid, None)
1320 node = self.db.getnode(self.classname, nodeid)
1321 node[self.db.RETIRED_FLAG] = 1
1322 self.db.setnode(self.classname, nodeid, node)
1323 if self.do_journal:
1324 self.db.addjournal(self.classname, nodeid, 'retired', None)
1326 self.fireReactors('retire', nodeid, None)
1328 def is_retired(self, nodeid):
1329 '''Return true if the node is retired.
1330 '''
1331 node = self.db.getnode(cn, nodeid, cldb)
1332 if node.has_key(self.db.RETIRED_FLAG):
1333 return 1
1334 return 0
1336 def destroy(self, nodeid):
1337 '''Destroy a node.
1339 WARNING: this method should never be used except in extremely rare
1340 situations where there could never be links to the node being
1341 deleted
1342 WARNING: use retire() instead
1343 WARNING: the properties of this node will not be available ever again
1344 WARNING: really, use retire() instead
1346 Well, I think that's enough warnings. This method exists mostly to
1347 support the session storage of the cgi interface.
1348 '''
1349 if self.db.journaltag is None:
1350 raise DatabaseError, 'Database open read-only'
1351 self.db.destroynode(self.classname, nodeid)
1353 def history(self, nodeid):
1354 '''Retrieve the journal of edits on a particular node.
1356 'nodeid' must be the id of an existing node of this class or an
1357 IndexError is raised.
1359 The returned list contains tuples of the form
1361 (nodeid, date, tag, action, params)
1363 'date' is a Timestamp object specifying the time of the change and
1364 'tag' is the journaltag specified when the database was opened.
1365 '''
1366 if not self.do_journal:
1367 raise ValueError, 'Journalling is disabled for this class'
1368 return self.db.getjournal(self.classname, nodeid)
1370 # Locating nodes:
1371 def hasnode(self, nodeid):
1372 '''Determine if the given nodeid actually exists
1373 '''
1374 return self.db.hasnode(self.classname, nodeid)
1376 def setkey(self, propname):
1377 '''Select a String property of this class to be the key property.
1379 'propname' must be the name of a String property of this class or
1380 None, or a TypeError is raised. The values of the key property on
1381 all existing nodes must be unique or a ValueError is raised. If the
1382 property doesn't exist, KeyError is raised.
1383 '''
1384 prop = self.getprops()[propname]
1385 if not isinstance(prop, String):
1386 raise TypeError, 'key properties must be String'
1387 self.key = propname
1389 def getkey(self):
1390 '''Return the name of the key property for this class or None.'''
1391 return self.key
1393 def labelprop(self, default_to_id=0):
1394 ''' Return the property name for a label for the given node.
1396 This method attempts to generate a consistent label for the node.
1397 It tries the following in order:
1398 1. key property
1399 2. "name" property
1400 3. "title" property
1401 4. first property from the sorted property name list
1402 '''
1403 k = self.getkey()
1404 if k:
1405 return k
1406 props = self.getprops()
1407 if props.has_key('name'):
1408 return 'name'
1409 elif props.has_key('title'):
1410 return 'title'
1411 if default_to_id:
1412 return 'id'
1413 props = props.keys()
1414 props.sort()
1415 return props[0]
1417 # TODO: set up a separate index db file for this? profile?
1418 def lookup(self, keyvalue):
1419 '''Locate a particular node by its key property and return its id.
1421 If this class has no key property, a TypeError is raised. If the
1422 'keyvalue' matches one of the values for the key property among
1423 the nodes in this class, the matching node's id is returned;
1424 otherwise a KeyError is raised.
1425 '''
1426 if not self.key:
1427 raise TypeError, 'No key property set for class %s'%self.classname
1428 cldb = self.db.getclassdb(self.classname)
1429 try:
1430 for nodeid in self.db.getnodeids(self.classname, cldb):
1431 node = self.db.getnode(self.classname, nodeid, cldb)
1432 if node.has_key(self.db.RETIRED_FLAG):
1433 continue
1434 if node[self.key] == keyvalue:
1435 return nodeid
1436 finally:
1437 cldb.close()
1438 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1439 keyvalue, self.classname)
1441 # change from spec - allows multiple props to match
1442 def find(self, **propspec):
1443 '''Get the ids of nodes in this class which link to the given nodes.
1445 'propspec' consists of keyword args propname=nodeid or
1446 propname={nodeid:1, }
1447 'propname' must be the name of a property in this class, or a
1448 KeyError is raised. That property must be a Link or
1449 Multilink property, or a TypeError is raised.
1451 Any node in this class whose 'propname' property links to any of the
1452 nodeids will be returned. Used by the full text indexing, which knows
1453 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1454 issues:
1456 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1457 '''
1458 propspec = propspec.items()
1459 for propname, nodeids in propspec:
1460 # check the prop is OK
1461 prop = self.properties[propname]
1462 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1463 raise TypeError, "'%s' not a Link/Multilink property"%propname
1465 # ok, now do the find
1466 cldb = self.db.getclassdb(self.classname)
1467 l = []
1468 try:
1469 for id in self.db.getnodeids(self.classname, db=cldb):
1470 node = self.db.getnode(self.classname, id, db=cldb)
1471 if node.has_key(self.db.RETIRED_FLAG):
1472 continue
1473 for propname, nodeids in propspec:
1474 # can't test if the node doesn't have this property
1475 if not node.has_key(propname):
1476 continue
1477 if type(nodeids) is type(''):
1478 nodeids = {nodeids:1}
1479 prop = self.properties[propname]
1480 value = node[propname]
1481 if isinstance(prop, Link) and nodeids.has_key(value):
1482 l.append(id)
1483 break
1484 elif isinstance(prop, Multilink):
1485 hit = 0
1486 for v in value:
1487 if nodeids.has_key(v):
1488 l.append(id)
1489 hit = 1
1490 break
1491 if hit:
1492 break
1493 finally:
1494 cldb.close()
1495 return l
1497 def stringFind(self, **requirements):
1498 '''Locate a particular node by matching a set of its String
1499 properties in a caseless search.
1501 If the property is not a String property, a TypeError is raised.
1503 The return is a list of the id of all nodes that match.
1504 '''
1505 for propname in requirements.keys():
1506 prop = self.properties[propname]
1507 if isinstance(not prop, String):
1508 raise TypeError, "'%s' not a String property"%propname
1509 requirements[propname] = requirements[propname].lower()
1510 l = []
1511 cldb = self.db.getclassdb(self.classname)
1512 try:
1513 for nodeid in self.db.getnodeids(self.classname, cldb):
1514 node = self.db.getnode(self.classname, nodeid, cldb)
1515 if node.has_key(self.db.RETIRED_FLAG):
1516 continue
1517 for key, value in requirements.items():
1518 if not node.has_key(key):
1519 break
1520 if node[key] is None or node[key].lower() != value:
1521 break
1522 else:
1523 l.append(nodeid)
1524 finally:
1525 cldb.close()
1526 return l
1528 def list(self):
1529 ''' Return a list of the ids of the active nodes in this class.
1530 '''
1531 l = []
1532 cn = self.classname
1533 cldb = self.db.getclassdb(cn)
1534 try:
1535 for nodeid in self.db.getnodeids(cn, cldb):
1536 node = self.db.getnode(cn, nodeid, cldb)
1537 if node.has_key(self.db.RETIRED_FLAG):
1538 continue
1539 l.append(nodeid)
1540 finally:
1541 cldb.close()
1542 l.sort()
1543 return l
1545 def filter(self, search_matches, filterspec, sort=(None,None),
1546 group=(None,None), num_re = re.compile('^\d+$')):
1547 ''' Return a list of the ids of the active nodes in this class that
1548 match the 'filter' spec, sorted by the group spec and then the
1549 sort spec.
1551 "filterspec" is {propname: value(s)}
1552 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1553 and prop is a prop name or None
1554 "search_matches" is {nodeid: marker}
1556 The filter must match all properties specificed - but if the
1557 property value to match is a list, any one of the values in the
1558 list may match for that property to match.
1559 '''
1560 cn = self.classname
1562 # optimise filterspec
1563 l = []
1564 props = self.getprops()
1565 LINK = 0
1566 MULTILINK = 1
1567 STRING = 2
1568 OTHER = 6
1569 for k, v in filterspec.items():
1570 propclass = props[k]
1571 if isinstance(propclass, Link):
1572 if type(v) is not type([]):
1573 v = [v]
1574 # replace key values with node ids
1575 u = []
1576 link_class = self.db.classes[propclass.classname]
1577 for entry in v:
1578 if entry == '-1': entry = None
1579 elif not num_re.match(entry):
1580 try:
1581 entry = link_class.lookup(entry)
1582 except (TypeError,KeyError):
1583 raise ValueError, 'property "%s": %s not a %s'%(
1584 k, entry, self.properties[k].classname)
1585 u.append(entry)
1587 l.append((LINK, k, u))
1588 elif isinstance(propclass, Multilink):
1589 if type(v) is not type([]):
1590 v = [v]
1591 # replace key values with node ids
1592 u = []
1593 link_class = self.db.classes[propclass.classname]
1594 for entry in v:
1595 if not num_re.match(entry):
1596 try:
1597 entry = link_class.lookup(entry)
1598 except (TypeError,KeyError):
1599 raise ValueError, 'new property "%s": %s not a %s'%(
1600 k, entry, self.properties[k].classname)
1601 u.append(entry)
1602 l.append((MULTILINK, k, u))
1603 elif isinstance(propclass, String) and k != 'id':
1604 # simple glob searching
1605 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1606 v = v.replace('?', '.')
1607 v = v.replace('*', '.*?')
1608 l.append((STRING, k, re.compile(v, re.I)))
1609 elif isinstance(propclass, Boolean):
1610 if type(v) is type(''):
1611 bv = v.lower() in ('yes', 'true', 'on', '1')
1612 else:
1613 bv = v
1614 l.append((OTHER, k, bv))
1615 elif isinstance(propclass, Date):
1616 l.append((OTHER, k, date.Date(v)))
1617 elif isinstance(propclass, Interval):
1618 l.append((OTHER, k, date.Interval(v)))
1619 elif isinstance(propclass, Number):
1620 l.append((OTHER, k, int(v)))
1621 else:
1622 l.append((OTHER, k, v))
1623 filterspec = l
1625 # now, find all the nodes that are active and pass filtering
1626 l = []
1627 cldb = self.db.getclassdb(cn)
1628 try:
1629 # TODO: only full-scan once (use items())
1630 for nodeid in self.db.getnodeids(cn, cldb):
1631 node = self.db.getnode(cn, nodeid, cldb)
1632 if node.has_key(self.db.RETIRED_FLAG):
1633 continue
1634 # apply filter
1635 for t, k, v in filterspec:
1636 # handle the id prop
1637 if k == 'id' and v == nodeid:
1638 continue
1640 # make sure the node has the property
1641 if not node.has_key(k):
1642 # this node doesn't have this property, so reject it
1643 break
1645 # now apply the property filter
1646 if t == LINK:
1647 # link - if this node's property doesn't appear in the
1648 # filterspec's nodeid list, skip it
1649 if node[k] not in v:
1650 break
1651 elif t == MULTILINK:
1652 # multilink - if any of the nodeids required by the
1653 # filterspec aren't in this node's property, then skip
1654 # it
1655 have = node[k]
1656 for want in v:
1657 if want not in have:
1658 break
1659 else:
1660 continue
1661 break
1662 elif t == STRING:
1663 # RE search
1664 if node[k] is None or not v.search(node[k]):
1665 break
1666 elif t == OTHER:
1667 # straight value comparison for the other types
1668 if node[k] != v:
1669 break
1670 else:
1671 l.append((nodeid, node))
1672 finally:
1673 cldb.close()
1674 l.sort()
1676 # filter based on full text search
1677 if search_matches is not None:
1678 k = []
1679 for v in l:
1680 if search_matches.has_key(v[0]):
1681 k.append(v)
1682 l = k
1684 # now, sort the result
1685 def sortfun(a, b, sort=sort, group=group, properties=self.getprops(),
1686 db = self.db, cl=self):
1687 a_id, an = a
1688 b_id, bn = b
1689 # sort by group and then sort
1690 for dir, prop in group, sort:
1691 if dir is None or prop is None: continue
1693 # sorting is class-specific
1694 propclass = properties[prop]
1696 # handle the properties that might be "faked"
1697 # also, handle possible missing properties
1698 try:
1699 if not an.has_key(prop):
1700 an[prop] = cl.get(a_id, prop)
1701 av = an[prop]
1702 except KeyError:
1703 # the node doesn't have a value for this property
1704 if isinstance(propclass, Multilink): av = []
1705 else: av = ''
1706 try:
1707 if not bn.has_key(prop):
1708 bn[prop] = cl.get(b_id, prop)
1709 bv = bn[prop]
1710 except KeyError:
1711 # the node doesn't have a value for this property
1712 if isinstance(propclass, Multilink): bv = []
1713 else: bv = ''
1715 # String and Date values are sorted in the natural way
1716 if isinstance(propclass, String):
1717 # clean up the strings
1718 if av and av[0] in string.uppercase:
1719 av = av.lower()
1720 if bv and bv[0] in string.uppercase:
1721 bv = bv.lower()
1722 if (isinstance(propclass, String) or
1723 isinstance(propclass, Date)):
1724 # it might be a string that's really an integer
1725 try:
1726 av = int(av)
1727 bv = int(bv)
1728 except:
1729 pass
1730 if dir == '+':
1731 r = cmp(av, bv)
1732 if r != 0: return r
1733 elif dir == '-':
1734 r = cmp(bv, av)
1735 if r != 0: return r
1737 # Link properties are sorted according to the value of
1738 # the "order" property on the linked nodes if it is
1739 # present; or otherwise on the key string of the linked
1740 # nodes; or finally on the node ids.
1741 elif isinstance(propclass, Link):
1742 link = db.classes[propclass.classname]
1743 if av is None and bv is not None: return -1
1744 if av is not None and bv is None: return 1
1745 if av is None and bv is None: continue
1746 if link.getprops().has_key('order'):
1747 if dir == '+':
1748 r = cmp(link.get(av, 'order'),
1749 link.get(bv, 'order'))
1750 if r != 0: return r
1751 elif dir == '-':
1752 r = cmp(link.get(bv, 'order'),
1753 link.get(av, 'order'))
1754 if r != 0: return r
1755 elif link.getkey():
1756 key = link.getkey()
1757 if dir == '+':
1758 r = cmp(link.get(av, key), link.get(bv, key))
1759 if r != 0: return r
1760 elif dir == '-':
1761 r = cmp(link.get(bv, key), link.get(av, key))
1762 if r != 0: return r
1763 else:
1764 if dir == '+':
1765 r = cmp(av, bv)
1766 if r != 0: return r
1767 elif dir == '-':
1768 r = cmp(bv, av)
1769 if r != 0: return r
1771 # Multilink properties are sorted according to how many
1772 # links are present.
1773 elif isinstance(propclass, Multilink):
1774 r = cmp(len(av), len(bv))
1775 if r == 0:
1776 # Compare contents of multilink property if lenghts is
1777 # equal
1778 r = cmp ('.'.join(av), '.'.join(bv))
1779 if dir == '+':
1780 return r
1781 elif dir == '-':
1782 return -r
1783 elif isinstance(propclass, Number) or isinstance(propclass, Boolean):
1784 if dir == '+':
1785 r = cmp(av, bv)
1786 elif dir == '-':
1787 r = cmp(bv, av)
1789 # end for dir, prop in sort, group:
1790 # if all else fails, compare the ids
1791 return cmp(a[0], b[0])
1793 l.sort(sortfun)
1794 return [i[0] for i in l]
1796 def count(self):
1797 '''Get the number of nodes in this class.
1799 If the returned integer is 'numnodes', the ids of all the nodes
1800 in this class run from 1 to numnodes, and numnodes+1 will be the
1801 id of the next node to be created in this class.
1802 '''
1803 return self.db.countnodes(self.classname)
1805 # Manipulating properties:
1807 def getprops(self, protected=1):
1808 '''Return a dictionary mapping property names to property objects.
1809 If the "protected" flag is true, we include protected properties -
1810 those which may not be modified.
1812 In addition to the actual properties on the node, these
1813 methods provide the "creation" and "activity" properties. If the
1814 "protected" flag is true, we include protected properties - those
1815 which may not be modified.
1816 '''
1817 d = self.properties.copy()
1818 if protected:
1819 d['id'] = String()
1820 d['creation'] = hyperdb.Date()
1821 d['activity'] = hyperdb.Date()
1822 d['creator'] = hyperdb.Link('user')
1823 return d
1825 def addprop(self, **properties):
1826 '''Add properties to this class.
1828 The keyword arguments in 'properties' must map names to property
1829 objects, or a TypeError is raised. None of the keys in 'properties'
1830 may collide with the names of existing properties, or a ValueError
1831 is raised before any properties have been added.
1832 '''
1833 for key in properties.keys():
1834 if self.properties.has_key(key):
1835 raise ValueError, key
1836 self.properties.update(properties)
1838 def index(self, nodeid):
1839 '''Add (or refresh) the node to search indexes
1840 '''
1841 # find all the String properties that have indexme
1842 for prop, propclass in self.getprops().items():
1843 if isinstance(propclass, String) and propclass.indexme:
1844 try:
1845 value = str(self.get(nodeid, prop))
1846 except IndexError:
1847 # node no longer exists - entry should be removed
1848 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1849 else:
1850 # and index them under (classname, nodeid, property)
1851 self.db.indexer.add_text((self.classname, nodeid, prop),
1852 value)
1854 #
1855 # Detector interface
1856 #
1857 def audit(self, event, detector):
1858 '''Register a detector
1859 '''
1860 l = self.auditors[event]
1861 if detector not in l:
1862 self.auditors[event].append(detector)
1864 def fireAuditors(self, action, nodeid, newvalues):
1865 '''Fire all registered auditors.
1866 '''
1867 for audit in self.auditors[action]:
1868 audit(self.db, self, nodeid, newvalues)
1870 def react(self, event, detector):
1871 '''Register a detector
1872 '''
1873 l = self.reactors[event]
1874 if detector not in l:
1875 self.reactors[event].append(detector)
1877 def fireReactors(self, action, nodeid, oldvalues):
1878 '''Fire all registered reactors.
1879 '''
1880 for react in self.reactors[action]:
1881 react(self.db, self, nodeid, oldvalues)
1883 class FileClass(Class, hyperdb.FileClass):
1884 '''This class defines a large chunk of data. To support this, it has a
1885 mandatory String property "content" which is typically saved off
1886 externally to the hyperdb.
1888 The default MIME type of this data is defined by the
1889 "default_mime_type" class attribute, which may be overridden by each
1890 node if the class defines a "type" String property.
1891 '''
1892 default_mime_type = 'text/plain'
1894 def create(self, **propvalues):
1895 ''' Snarf the "content" propvalue and store in a file
1896 '''
1897 # we need to fire the auditors now, or the content property won't
1898 # be in propvalues for the auditors to play with
1899 self.fireAuditors('create', None, propvalues)
1901 # now remove the content property so it's not stored in the db
1902 content = propvalues['content']
1903 del propvalues['content']
1905 # do the database create
1906 newid = Class.create_inner(self, **propvalues)
1908 # fire reactors
1909 self.fireReactors('create', newid, None)
1911 # store off the content as a file
1912 self.db.storefile(self.classname, newid, None, content)
1913 return newid
1915 def import_list(self, propnames, proplist):
1916 ''' Trap the "content" property...
1917 '''
1918 # dupe this list so we don't affect others
1919 propnames = propnames[:]
1921 # extract the "content" property from the proplist
1922 i = propnames.index('content')
1923 content = eval(proplist[i])
1924 del propnames[i]
1925 del proplist[i]
1927 # do the normal import
1928 newid = Class.import_list(self, propnames, proplist)
1930 # save off the "content" file
1931 self.db.storefile(self.classname, newid, None, content)
1932 return newid
1934 def get(self, nodeid, propname, default=_marker, cache=1):
1935 ''' trap the content propname and get it from the file
1936 '''
1937 poss_msg = 'Possibly an access right configuration problem.'
1938 if propname == 'content':
1939 try:
1940 return self.db.getfile(self.classname, nodeid, None)
1941 except IOError, (strerror):
1942 # XXX by catching this we donot see an error in the log.
1943 return 'ERROR reading file: %s%s\n%s\n%s'%(
1944 self.classname, nodeid, poss_msg, strerror)
1945 if default is not _marker:
1946 return Class.get(self, nodeid, propname, default, cache=cache)
1947 else:
1948 return Class.get(self, nodeid, propname, cache=cache)
1950 def getprops(self, protected=1):
1951 ''' In addition to the actual properties on the node, these methods
1952 provide the "content" property. If the "protected" flag is true,
1953 we include protected properties - those which may not be
1954 modified.
1955 '''
1956 d = Class.getprops(self, protected=protected).copy()
1957 d['content'] = hyperdb.String()
1958 return d
1960 def index(self, nodeid):
1961 ''' Index the node in the search index.
1963 We want to index the content in addition to the normal String
1964 property indexing.
1965 '''
1966 # perform normal indexing
1967 Class.index(self, nodeid)
1969 # get the content to index
1970 content = self.get(nodeid, 'content')
1972 # figure the mime type
1973 if self.properties.has_key('type'):
1974 mime_type = self.get(nodeid, 'type')
1975 else:
1976 mime_type = self.default_mime_type
1978 # and index!
1979 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
1980 mime_type)
1982 # deviation from spec - was called ItemClass
1983 class IssueClass(Class, roundupdb.IssueClass):
1984 # Overridden methods:
1985 def __init__(self, db, classname, **properties):
1986 '''The newly-created class automatically includes the "messages",
1987 "files", "nosy", and "superseder" properties. If the 'properties'
1988 dictionary attempts to specify any of these properties or a
1989 "creation" or "activity" property, a ValueError is raised.
1990 '''
1991 if not properties.has_key('title'):
1992 properties['title'] = hyperdb.String(indexme='yes')
1993 if not properties.has_key('messages'):
1994 properties['messages'] = hyperdb.Multilink("msg")
1995 if not properties.has_key('files'):
1996 properties['files'] = hyperdb.Multilink("file")
1997 if not properties.has_key('nosy'):
1998 # note: journalling is turned off as it really just wastes
1999 # space. this behaviour may be overridden in an instance
2000 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2001 if not properties.has_key('superseder'):
2002 properties['superseder'] = hyperdb.Multilink(classname)
2003 Class.__init__(self, db, classname, **properties)
2005 #