1971ccade4254cd4a81a29da76f441e07a960de6
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: back_anydbm.py,v 1.103 2003-02-14 00:31:44 richard Exp $
19 '''
20 This module defines a backend that saves the hyperdatabase in a database
21 chosen by anydbm. It is guaranteed to always be available in python
22 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
23 serious bugs, and is not available)
24 '''
26 import whichdb, anydbm, os, marshal, re, weakref, string, copy
27 from roundup import hyperdb, date, password, roundupdb, security
28 from blobfiles import FileStorage
29 from sessions import Sessions
30 from roundup.indexer import Indexer
31 from roundup.backends import locking
32 from roundup.hyperdb import String, Password, Date, Interval, Link, \
33 Multilink, DatabaseError, Boolean, Number, Node
35 #
36 # Now the database
37 #
38 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
39 '''A database for storing records containing flexible data types.
41 Transaction stuff TODO:
42 . check the timestamp of the class file and nuke the cache if it's
43 modified. Do some sort of conflict checking on the dirty stuff.
44 . perhaps detect write collisions (related to above)?
46 '''
47 def __init__(self, config, journaltag=None):
48 '''Open a hyperdatabase given a specifier to some storage.
50 The 'storagelocator' is obtained from config.DATABASE.
51 The meaning of 'storagelocator' depends on the particular
52 implementation of the hyperdatabase. It could be a file name,
53 a directory path, a socket descriptor for a connection to a
54 database over the network, etc.
56 The 'journaltag' is a token that will be attached to the journal
57 entries for any edits done on the database. If 'journaltag' is
58 None, the database is opened in read-only mode: the Class.create(),
59 Class.set(), and Class.retire() methods are disabled.
60 '''
61 self.config, self.journaltag = config, journaltag
62 self.dir = config.DATABASE
63 self.classes = {}
64 self.cache = {} # cache of nodes loaded or created
65 self.dirtynodes = {} # keep track of the dirty nodes by class
66 self.newnodes = {} # keep track of the new nodes by class
67 self.destroyednodes = {}# keep track of the destroyed nodes by class
68 self.transactions = []
69 self.indexer = Indexer(self.dir)
70 self.sessions = Sessions(self.config)
71 self.security = security.Security(self)
72 # ensure files are group readable and writable
73 os.umask(0002)
75 # lock it
76 lockfilenm = os.path.join(self.dir, 'lock')
77 self.lockfile = locking.acquire_lock(lockfilenm)
78 self.lockfile.write(str(os.getpid()))
79 self.lockfile.flush()
81 def post_init(self):
82 ''' Called once the schema initialisation has finished.
83 '''
84 # reindex the db if necessary
85 if self.indexer.should_reindex():
86 self.reindex()
88 # figure the "curuserid"
89 if self.journaltag is None:
90 self.curuserid = None
91 elif self.journaltag == 'admin':
92 # admin user may not exist, but always has ID 1
93 self.curuserid = '1'
94 else:
95 self.curuserid = self.user.lookup(self.journaltag)
97 def reindex(self):
98 for klass in self.classes.values():
99 for nodeid in klass.list():
100 klass.index(nodeid)
101 self.indexer.save_index()
103 def __repr__(self):
104 return '<back_anydbm instance at %x>'%id(self)
106 #
107 # Classes
108 #
109 def __getattr__(self, classname):
110 '''A convenient way of calling self.getclass(classname).'''
111 if self.classes.has_key(classname):
112 if __debug__:
113 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
114 return self.classes[classname]
115 raise AttributeError, classname
117 def addclass(self, cl):
118 if __debug__:
119 print >>hyperdb.DEBUG, 'addclass', (self, cl)
120 cn = cl.classname
121 if self.classes.has_key(cn):
122 raise ValueError, cn
123 self.classes[cn] = cl
125 def getclasses(self):
126 '''Return a list of the names of all existing classes.'''
127 if __debug__:
128 print >>hyperdb.DEBUG, 'getclasses', (self,)
129 l = self.classes.keys()
130 l.sort()
131 return l
133 def getclass(self, classname):
134 '''Get the Class object representing a particular class.
136 If 'classname' is not a valid class name, a KeyError is raised.
137 '''
138 if __debug__:
139 print >>hyperdb.DEBUG, 'getclass', (self, classname)
140 try:
141 return self.classes[classname]
142 except KeyError:
143 raise KeyError, 'There is no class called "%s"'%classname
145 #
146 # Class DBs
147 #
148 def clear(self):
149 '''Delete all database contents
150 '''
151 if __debug__:
152 print >>hyperdb.DEBUG, 'clear', (self,)
153 for cn in self.classes.keys():
154 for dummy in 'nodes', 'journals':
155 path = os.path.join(self.dir, 'journals.%s'%cn)
156 if os.path.exists(path):
157 os.remove(path)
158 elif os.path.exists(path+'.db'): # dbm appends .db
159 os.remove(path+'.db')
161 def getclassdb(self, classname, mode='r'):
162 ''' grab a connection to the class db that will be used for
163 multiple actions
164 '''
165 if __debug__:
166 print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode)
167 return self.opendb('nodes.%s'%classname, mode)
169 def determine_db_type(self, path):
170 ''' determine which DB wrote the class file
171 '''
172 db_type = ''
173 if os.path.exists(path):
174 db_type = whichdb.whichdb(path)
175 if not db_type:
176 raise DatabaseError, "Couldn't identify database type"
177 elif os.path.exists(path+'.db'):
178 # if the path ends in '.db', it's a dbm database, whether
179 # anydbm says it's dbhash or not!
180 db_type = 'dbm'
181 return db_type
183 def opendb(self, name, mode):
184 '''Low-level database opener that gets around anydbm/dbm
185 eccentricities.
186 '''
187 if __debug__:
188 print >>hyperdb.DEBUG, 'opendb', (self, name, mode)
190 # figure the class db type
191 path = os.path.join(os.getcwd(), self.dir, name)
192 db_type = self.determine_db_type(path)
194 # new database? let anydbm pick the best dbm
195 if not db_type:
196 if __debug__:
197 print >>hyperdb.DEBUG, "opendb anydbm.open(%r, 'c')"%path
198 return anydbm.open(path, 'c')
200 # open the database with the correct module
201 try:
202 dbm = __import__(db_type)
203 except ImportError:
204 raise DatabaseError, \
205 "Couldn't open database - the required module '%s'"\
206 " is not available"%db_type
207 if __debug__:
208 print >>hyperdb.DEBUG, "opendb %r.open(%r, %r)"%(db_type, path,
209 mode)
210 return dbm.open(path, mode)
212 #
213 # Node IDs
214 #
215 def newid(self, classname):
216 ''' Generate a new id for the given class
217 '''
218 # open the ids DB - create if if doesn't exist
219 db = self.opendb('_ids', 'c')
220 if db.has_key(classname):
221 newid = db[classname] = str(int(db[classname]) + 1)
222 else:
223 # the count() bit is transitional - older dbs won't start at 1
224 newid = str(self.getclass(classname).count()+1)
225 db[classname] = newid
226 db.close()
227 return newid
229 def setid(self, classname, setid):
230 ''' Set the id counter: used during import of database
231 '''
232 # open the ids DB - create if if doesn't exist
233 db = self.opendb('_ids', 'c')
234 db[classname] = str(setid)
235 db.close()
237 #
238 # Nodes
239 #
240 def addnode(self, classname, nodeid, node):
241 ''' add the specified node to its class's db
242 '''
243 if __debug__:
244 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
246 # we'll be supplied these props if we're doing an import
247 if not node.has_key('creator'):
248 # add in the "calculated" properties (dupe so we don't affect
249 # calling code's node assumptions)
250 node = node.copy()
251 node['creator'] = self.curuserid
252 node['creation'] = node['activity'] = date.Date()
254 self.newnodes.setdefault(classname, {})[nodeid] = 1
255 self.cache.setdefault(classname, {})[nodeid] = node
256 self.savenode(classname, nodeid, node)
258 def setnode(self, classname, nodeid, node):
259 ''' change the specified node
260 '''
261 if __debug__:
262 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
263 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
265 # update the activity time (dupe so we don't affect
266 # calling code's node assumptions)
267 node = node.copy()
268 node['activity'] = date.Date()
270 # can't set without having already loaded the node
271 self.cache[classname][nodeid] = node
272 self.savenode(classname, nodeid, node)
274 def savenode(self, classname, nodeid, node):
275 ''' perform the saving of data specified by the set/addnode
276 '''
277 if __debug__:
278 print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node)
279 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
281 def getnode(self, classname, nodeid, db=None, cache=1):
282 ''' get a node from the database
283 '''
284 if __debug__:
285 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid, db)
286 if cache:
287 # try the cache
288 cache_dict = self.cache.setdefault(classname, {})
289 if cache_dict.has_key(nodeid):
290 if __debug__:
291 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
292 nodeid)
293 return cache_dict[nodeid]
295 if __debug__:
296 print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid)
298 # get from the database and save in the cache
299 if db is None:
300 db = self.getclassdb(classname)
301 if not db.has_key(nodeid):
302 # try the cache - might be a brand-new node
303 cache_dict = self.cache.setdefault(classname, {})
304 if cache_dict.has_key(nodeid):
305 if __debug__:
306 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
307 nodeid)
308 return cache_dict[nodeid]
309 raise IndexError, "no such %s %s"%(classname, nodeid)
311 # check the uncommitted, destroyed nodes
312 if (self.destroyednodes.has_key(classname) and
313 self.destroyednodes[classname].has_key(nodeid)):
314 raise IndexError, "no such %s %s"%(classname, nodeid)
316 # decode
317 res = marshal.loads(db[nodeid])
319 # reverse the serialisation
320 res = self.unserialise(classname, res)
322 # store off in the cache dict
323 if cache:
324 cache_dict[nodeid] = res
326 return res
328 def destroynode(self, classname, nodeid):
329 '''Remove a node from the database. Called exclusively by the
330 destroy() method on Class.
331 '''
332 if __debug__:
333 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
335 # remove from cache and newnodes if it's there
336 if (self.cache.has_key(classname) and
337 self.cache[classname].has_key(nodeid)):
338 del self.cache[classname][nodeid]
339 if (self.newnodes.has_key(classname) and
340 self.newnodes[classname].has_key(nodeid)):
341 del self.newnodes[classname][nodeid]
343 # see if there's any obvious commit actions that we should get rid of
344 for entry in self.transactions[:]:
345 if entry[1][:2] == (classname, nodeid):
346 self.transactions.remove(entry)
348 # add to the destroyednodes map
349 self.destroyednodes.setdefault(classname, {})[nodeid] = 1
351 # add the destroy commit action
352 self.transactions.append((self.doDestroyNode, (classname, nodeid)))
354 def serialise(self, classname, node):
355 '''Copy the node contents, converting non-marshallable data into
356 marshallable data.
357 '''
358 if __debug__:
359 print >>hyperdb.DEBUG, 'serialise', classname, node
360 properties = self.getclass(classname).getprops()
361 d = {}
362 for k, v in node.items():
363 # if the property doesn't exist, or is the "retired" flag then
364 # it won't be in the properties dict
365 if not properties.has_key(k):
366 d[k] = v
367 continue
369 # get the property spec
370 prop = properties[k]
372 if isinstance(prop, Password) and v is not None:
373 d[k] = str(v)
374 elif isinstance(prop, Date) and v is not None:
375 d[k] = v.serialise()
376 elif isinstance(prop, Interval) and v is not None:
377 d[k] = v.serialise()
378 else:
379 d[k] = v
380 return d
382 def unserialise(self, classname, node):
383 '''Decode the marshalled node data
384 '''
385 if __debug__:
386 print >>hyperdb.DEBUG, 'unserialise', classname, node
387 properties = self.getclass(classname).getprops()
388 d = {}
389 for k, v in node.items():
390 # if the property doesn't exist, or is the "retired" flag then
391 # it won't be in the properties dict
392 if not properties.has_key(k):
393 d[k] = v
394 continue
396 # get the property spec
397 prop = properties[k]
399 if isinstance(prop, Date) and v is not None:
400 d[k] = date.Date(v)
401 elif isinstance(prop, Interval) and v is not None:
402 d[k] = date.Interval(v)
403 elif isinstance(prop, Password) and v is not None:
404 p = password.Password()
405 p.unpack(v)
406 d[k] = p
407 else:
408 d[k] = v
409 return d
411 def hasnode(self, classname, nodeid, db=None):
412 ''' determine if the database has a given node
413 '''
414 if __debug__:
415 print >>hyperdb.DEBUG, 'hasnode', (self, classname, nodeid, db)
417 # try the cache
418 cache = self.cache.setdefault(classname, {})
419 if cache.has_key(nodeid):
420 if __debug__:
421 print >>hyperdb.TRACE, 'has %s %s cached'%(classname, nodeid)
422 return 1
423 if __debug__:
424 print >>hyperdb.TRACE, 'has %s %s'%(classname, nodeid)
426 # not in the cache - check the database
427 if db is None:
428 db = self.getclassdb(classname)
429 res = db.has_key(nodeid)
430 return res
432 def countnodes(self, classname, db=None):
433 if __debug__:
434 print >>hyperdb.DEBUG, 'countnodes', (self, classname, db)
436 count = 0
438 # include the uncommitted nodes
439 if self.newnodes.has_key(classname):
440 count += len(self.newnodes[classname])
441 if self.destroyednodes.has_key(classname):
442 count -= len(self.destroyednodes[classname])
444 # and count those in the DB
445 if db is None:
446 db = self.getclassdb(classname)
447 count = count + len(db.keys())
448 return count
450 def getnodeids(self, classname, db=None):
451 if __debug__:
452 print >>hyperdb.DEBUG, 'getnodeids', (self, classname, db)
454 res = []
456 # start off with the new nodes
457 if self.newnodes.has_key(classname):
458 res += self.newnodes[classname].keys()
460 if db is None:
461 db = self.getclassdb(classname)
462 res = res + db.keys()
464 # remove the uncommitted, destroyed nodes
465 if self.destroyednodes.has_key(classname):
466 for nodeid in self.destroyednodes[classname].keys():
467 if db.has_key(nodeid):
468 res.remove(nodeid)
470 return res
473 #
474 # Files - special node properties
475 # inherited from FileStorage
477 #
478 # Journal
479 #
480 def addjournal(self, classname, nodeid, action, params, creator=None,
481 creation=None):
482 ''' Journal the Action
483 'action' may be:
485 'create' or 'set' -- 'params' is a dictionary of property values
486 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
487 'retire' -- 'params' is None
488 '''
489 if __debug__:
490 print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid,
491 action, params, creator, creation)
492 self.transactions.append((self.doSaveJournal, (classname, nodeid,
493 action, params, creator, creation)))
495 def getjournal(self, classname, nodeid):
496 ''' get the journal for id
498 Raise IndexError if the node doesn't exist (as per history()'s
499 API)
500 '''
501 if __debug__:
502 print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid)
503 # attempt to open the journal - in some rare cases, the journal may
504 # not exist
505 try:
506 db = self.opendb('journals.%s'%classname, 'r')
507 except anydbm.error, error:
508 if str(error) == "need 'c' or 'n' flag to open new db":
509 raise IndexError, 'no such %s %s'%(classname, nodeid)
510 elif error.args[0] != 2:
511 raise
512 raise IndexError, 'no such %s %s'%(classname, nodeid)
513 try:
514 journal = marshal.loads(db[nodeid])
515 except KeyError:
516 db.close()
517 raise IndexError, 'no such %s %s'%(classname, nodeid)
518 db.close()
519 res = []
520 for nodeid, date_stamp, user, action, params in journal:
521 res.append((nodeid, date.Date(date_stamp), user, action, params))
522 return res
524 def pack(self, pack_before):
525 ''' Delete all journal entries except "create" before 'pack_before'.
526 '''
527 if __debug__:
528 print >>hyperdb.DEBUG, 'packjournal', (self, pack_before)
530 pack_before = pack_before.serialise()
531 for classname in self.getclasses():
532 # get the journal db
533 db_name = 'journals.%s'%classname
534 path = os.path.join(os.getcwd(), self.dir, classname)
535 db_type = self.determine_db_type(path)
536 db = self.opendb(db_name, 'w')
538 for key in db.keys():
539 # get the journal for this db entry
540 journal = marshal.loads(db[key])
541 l = []
542 last_set_entry = None
543 for entry in journal:
544 # unpack the entry
545 (nodeid, date_stamp, self.journaltag, action,
546 params) = entry
547 # if the entry is after the pack date, _or_ the initial
548 # create entry, then it stays
549 if date_stamp > pack_before or action == 'create':
550 l.append(entry)
551 db[key] = marshal.dumps(l)
552 if db_type == 'gdbm':
553 db.reorganize()
554 db.close()
557 #
558 # Basic transaction support
559 #
560 def commit(self):
561 ''' Commit the current transactions.
562 '''
563 if __debug__:
564 print >>hyperdb.DEBUG, 'commit', (self,)
566 # keep a handle to all the database files opened
567 self.databases = {}
569 # now, do all the transactions
570 reindex = {}
571 for method, args in self.transactions:
572 reindex[method(*args)] = 1
574 # now close all the database files
575 for db in self.databases.values():
576 db.close()
577 del self.databases
579 # reindex the nodes that request it
580 for classname, nodeid in filter(None, reindex.keys()):
581 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
582 self.getclass(classname).index(nodeid)
584 # save the indexer state
585 self.indexer.save_index()
587 self.clearCache()
589 def clearCache(self):
590 # all transactions committed, back to normal
591 self.cache = {}
592 self.dirtynodes = {}
593 self.newnodes = {}
594 self.destroyednodes = {}
595 self.transactions = []
597 def getCachedClassDB(self, classname):
598 ''' get the class db, looking in our cache of databases for commit
599 '''
600 # get the database handle
601 db_name = 'nodes.%s'%classname
602 if not self.databases.has_key(db_name):
603 self.databases[db_name] = self.getclassdb(classname, 'c')
604 return self.databases[db_name]
606 def doSaveNode(self, classname, nodeid, node):
607 if __debug__:
608 print >>hyperdb.DEBUG, 'doSaveNode', (self, classname, nodeid,
609 node)
611 db = self.getCachedClassDB(classname)
613 # now save the marshalled data
614 db[nodeid] = marshal.dumps(self.serialise(classname, node))
616 # return the classname, nodeid so we reindex this content
617 return (classname, nodeid)
619 def getCachedJournalDB(self, classname):
620 ''' get the journal db, looking in our cache of databases for commit
621 '''
622 # get the database handle
623 db_name = 'journals.%s'%classname
624 if not self.databases.has_key(db_name):
625 self.databases[db_name] = self.opendb(db_name, 'c')
626 return self.databases[db_name]
628 def doSaveJournal(self, classname, nodeid, action, params, creator,
629 creation):
630 # serialise the parameters now if necessary
631 if isinstance(params, type({})):
632 if action in ('set', 'create'):
633 params = self.serialise(classname, params)
635 # handle supply of the special journalling parameters (usually
636 # supplied on importing an existing database)
637 if creator:
638 journaltag = creator
639 else:
640 journaltag = self.curuserid
641 if creation:
642 journaldate = creation.serialise()
643 else:
644 journaldate = date.Date().serialise()
646 # create the journal entry
647 entry = (nodeid, journaldate, journaltag, action, params)
649 if __debug__:
650 print >>hyperdb.DEBUG, 'doSaveJournal', entry
652 db = self.getCachedJournalDB(classname)
654 # now insert the journal entry
655 if db.has_key(nodeid):
656 # append to existing
657 s = db[nodeid]
658 l = marshal.loads(s)
659 l.append(entry)
660 else:
661 l = [entry]
663 db[nodeid] = marshal.dumps(l)
665 def doDestroyNode(self, classname, nodeid):
666 if __debug__:
667 print >>hyperdb.DEBUG, 'doDestroyNode', (self, classname, nodeid)
669 # delete from the class database
670 db = self.getCachedClassDB(classname)
671 if db.has_key(nodeid):
672 del db[nodeid]
674 # delete from the database
675 db = self.getCachedJournalDB(classname)
676 if db.has_key(nodeid):
677 del db[nodeid]
679 # return the classname, nodeid so we reindex this content
680 return (classname, nodeid)
682 def rollback(self):
683 ''' Reverse all actions from the current transaction.
684 '''
685 if __debug__:
686 print >>hyperdb.DEBUG, 'rollback', (self, )
687 for method, args in self.transactions:
688 # delete temporary files
689 if method == self.doStoreFile:
690 self.rollbackStoreFile(*args)
691 self.cache = {}
692 self.dirtynodes = {}
693 self.newnodes = {}
694 self.destroyednodes = {}
695 self.transactions = []
697 def close(self):
698 ''' Nothing to do
699 '''
700 if self.lockfile is not None:
701 locking.release_lock(self.lockfile)
702 if self.lockfile is not None:
703 self.lockfile.close()
704 self.lockfile = None
706 _marker = []
707 class Class(hyperdb.Class):
708 '''The handle to a particular class of nodes in a hyperdatabase.'''
710 def __init__(self, db, classname, **properties):
711 '''Create a new class with a given name and property specification.
713 'classname' must not collide with the name of an existing class,
714 or a ValueError is raised. The keyword arguments in 'properties'
715 must map names to property objects, or a TypeError is raised.
716 '''
717 if (properties.has_key('creation') or properties.has_key('activity')
718 or properties.has_key('creator')):
719 raise ValueError, '"creation", "activity" and "creator" are '\
720 'reserved'
722 self.classname = classname
723 self.properties = properties
724 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
725 self.key = ''
727 # should we journal changes (default yes)
728 self.do_journal = 1
730 # do the db-related init stuff
731 db.addclass(self)
733 self.auditors = {'create': [], 'set': [], 'retire': []}
734 self.reactors = {'create': [], 'set': [], 'retire': []}
736 def enableJournalling(self):
737 '''Turn journalling on for this class
738 '''
739 self.do_journal = 1
741 def disableJournalling(self):
742 '''Turn journalling off for this class
743 '''
744 self.do_journal = 0
746 # Editing nodes:
748 def create(self, **propvalues):
749 '''Create a new node of this class and return its id.
751 The keyword arguments in 'propvalues' map property names to values.
753 The values of arguments must be acceptable for the types of their
754 corresponding properties or a TypeError is raised.
756 If this class has a key property, it must be present and its value
757 must not collide with other key strings or a ValueError is raised.
759 Any other properties on this class that are missing from the
760 'propvalues' dictionary are set to None.
762 If an id in a link or multilink property does not refer to a valid
763 node, an IndexError is raised.
765 These operations trigger detectors and can be vetoed. Attempts
766 to modify the "creation" or "activity" properties cause a KeyError.
767 '''
768 self.fireAuditors('create', None, propvalues)
769 newid = self.create_inner(**propvalues)
770 self.fireReactors('create', newid, None)
771 return newid
773 def create_inner(self, **propvalues):
774 ''' Called by create, in-between the audit and react calls.
775 '''
776 if propvalues.has_key('id'):
777 raise KeyError, '"id" is reserved'
779 if self.db.journaltag is None:
780 raise DatabaseError, 'Database open read-only'
782 if propvalues.has_key('creation') or propvalues.has_key('activity'):
783 raise KeyError, '"creation" and "activity" are reserved'
784 # new node's id
785 newid = self.db.newid(self.classname)
787 # validate propvalues
788 num_re = re.compile('^\d+$')
789 for key, value in propvalues.items():
790 if key == self.key:
791 try:
792 self.lookup(value)
793 except KeyError:
794 pass
795 else:
796 raise ValueError, 'node with key "%s" exists'%value
798 # try to handle this property
799 try:
800 prop = self.properties[key]
801 except KeyError:
802 raise KeyError, '"%s" has no property "%s"'%(self.classname,
803 key)
805 if value is not None and isinstance(prop, Link):
806 if type(value) != type(''):
807 raise ValueError, 'link value must be String'
808 link_class = self.properties[key].classname
809 # if it isn't a number, it's a key
810 if not num_re.match(value):
811 try:
812 value = self.db.classes[link_class].lookup(value)
813 except (TypeError, KeyError):
814 raise IndexError, 'new property "%s": %s not a %s'%(
815 key, value, link_class)
816 elif not self.db.getclass(link_class).hasnode(value):
817 raise IndexError, '%s has no node %s'%(link_class, value)
819 # save off the value
820 propvalues[key] = value
822 # register the link with the newly linked node
823 if self.do_journal and self.properties[key].do_journal:
824 self.db.addjournal(link_class, value, 'link',
825 (self.classname, newid, key))
827 elif isinstance(prop, Multilink):
828 if type(value) != type([]):
829 raise TypeError, 'new property "%s" not a list of ids'%key
831 # clean up and validate the list of links
832 link_class = self.properties[key].classname
833 l = []
834 for entry in value:
835 if type(entry) != type(''):
836 raise ValueError, '"%s" multilink value (%r) '\
837 'must contain Strings'%(key, value)
838 # if it isn't a number, it's a key
839 if not num_re.match(entry):
840 try:
841 entry = self.db.classes[link_class].lookup(entry)
842 except (TypeError, KeyError):
843 raise IndexError, 'new property "%s": %s not a %s'%(
844 key, entry, self.properties[key].classname)
845 l.append(entry)
846 value = l
847 propvalues[key] = value
849 # handle additions
850 for nodeid in value:
851 if not self.db.getclass(link_class).hasnode(nodeid):
852 raise IndexError, '%s has no node %s'%(link_class,
853 nodeid)
854 # register the link with the newly linked node
855 if self.do_journal and self.properties[key].do_journal:
856 self.db.addjournal(link_class, nodeid, 'link',
857 (self.classname, newid, key))
859 elif isinstance(prop, String):
860 if type(value) != type('') and type(value) != type(u''):
861 raise TypeError, 'new property "%s" not a string'%key
863 elif isinstance(prop, Password):
864 if not isinstance(value, password.Password):
865 raise TypeError, 'new property "%s" not a Password'%key
867 elif isinstance(prop, Date):
868 if value is not None and not isinstance(value, date.Date):
869 raise TypeError, 'new property "%s" not a Date'%key
871 elif isinstance(prop, Interval):
872 if value is not None and not isinstance(value, date.Interval):
873 raise TypeError, 'new property "%s" not an Interval'%key
875 elif value is not None and isinstance(prop, Number):
876 try:
877 float(value)
878 except ValueError:
879 raise TypeError, 'new property "%s" not numeric'%key
881 elif value is not None and isinstance(prop, Boolean):
882 try:
883 int(value)
884 except ValueError:
885 raise TypeError, 'new property "%s" not boolean'%key
887 # make sure there's data where there needs to be
888 for key, prop in self.properties.items():
889 if propvalues.has_key(key):
890 continue
891 if key == self.key:
892 raise ValueError, 'key property "%s" is required'%key
893 if isinstance(prop, Multilink):
894 propvalues[key] = []
895 else:
896 propvalues[key] = None
898 # done
899 self.db.addnode(self.classname, newid, propvalues)
900 if self.do_journal:
901 self.db.addjournal(self.classname, newid, 'create', {})
903 return newid
905 def export_list(self, propnames, nodeid):
906 ''' Export a node - generate a list of CSV-able data in the order
907 specified by propnames for the given node.
908 '''
909 properties = self.getprops()
910 l = []
911 for prop in propnames:
912 proptype = properties[prop]
913 value = self.get(nodeid, prop)
914 # "marshal" data where needed
915 if value is None:
916 pass
917 elif isinstance(proptype, hyperdb.Date):
918 value = value.get_tuple()
919 elif isinstance(proptype, hyperdb.Interval):
920 value = value.get_tuple()
921 elif isinstance(proptype, hyperdb.Password):
922 value = str(value)
923 l.append(repr(value))
924 return l
926 def import_list(self, propnames, proplist):
927 ''' Import a node - all information including "id" is present and
928 should not be sanity checked. Triggers are not triggered. The
929 journal should be initialised using the "creator" and "created"
930 information.
932 Return the nodeid of the node imported.
933 '''
934 if self.db.journaltag is None:
935 raise DatabaseError, 'Database open read-only'
936 properties = self.getprops()
938 # make the new node's property map
939 d = {}
940 for i in range(len(propnames)):
941 # Use eval to reverse the repr() used to output the CSV
942 value = eval(proplist[i])
944 # Figure the property for this column
945 propname = propnames[i]
946 prop = properties[propname]
948 # "unmarshal" where necessary
949 if propname == 'id':
950 newid = value
951 continue
952 elif value is None:
953 # don't set Nones
954 continue
955 elif isinstance(prop, hyperdb.Date):
956 value = date.Date(value)
957 elif isinstance(prop, hyperdb.Interval):
958 value = date.Interval(value)
959 elif isinstance(prop, hyperdb.Password):
960 pwd = password.Password()
961 pwd.unpack(value)
962 value = pwd
963 d[propname] = value
965 # add the node and journal
966 self.db.addnode(self.classname, newid, d)
968 # extract the journalling stuff and nuke it
969 if d.has_key('creator'):
970 creator = d['creator']
971 del d['creator']
972 else:
973 creator = None
974 if d.has_key('creation'):
975 creation = d['creation']
976 del d['creation']
977 else:
978 creation = None
979 if d.has_key('activity'):
980 del d['activity']
981 self.db.addjournal(self.classname, newid, 'create', {}, creator,
982 creation)
983 return newid
985 def get(self, nodeid, propname, default=_marker, cache=1):
986 '''Get the value of a property on an existing node of this class.
988 'nodeid' must be the id of an existing node of this class or an
989 IndexError is raised. 'propname' must be the name of a property
990 of this class or a KeyError is raised.
992 'cache' indicates whether the transaction cache should be queried
993 for the node. If the node has been modified and you need to
994 determine what its values prior to modification are, you need to
995 set cache=0.
997 Attempts to get the "creation" or "activity" properties should
998 do the right thing.
999 '''
1000 if propname == 'id':
1001 return nodeid
1003 # get the node's dict
1004 d = self.db.getnode(self.classname, nodeid, cache=cache)
1006 # check for one of the special props
1007 if propname == 'creation':
1008 if d.has_key('creation'):
1009 return d['creation']
1010 if not self.do_journal:
1011 raise ValueError, 'Journalling is disabled for this class'
1012 journal = self.db.getjournal(self.classname, nodeid)
1013 if journal:
1014 return self.db.getjournal(self.classname, nodeid)[0][1]
1015 else:
1016 # on the strange chance that there's no journal
1017 return date.Date()
1018 if propname == 'activity':
1019 if d.has_key('activity'):
1020 return d['activity']
1021 if not self.do_journal:
1022 raise ValueError, 'Journalling is disabled for this class'
1023 journal = self.db.getjournal(self.classname, nodeid)
1024 if journal:
1025 return self.db.getjournal(self.classname, nodeid)[-1][1]
1026 else:
1027 # on the strange chance that there's no journal
1028 return date.Date()
1029 if propname == 'creator':
1030 if d.has_key('creator'):
1031 return d['creator']
1032 if not self.do_journal:
1033 raise ValueError, 'Journalling is disabled for this class'
1034 journal = self.db.getjournal(self.classname, nodeid)
1035 if journal:
1036 num_re = re.compile('^\d+$')
1037 value = self.db.getjournal(self.classname, nodeid)[0][2]
1038 if num_re.match(value):
1039 return value
1040 else:
1041 # old-style "username" journal tag
1042 try:
1043 return self.db.user.lookup(value)
1044 except KeyError:
1045 # user's been retired, return admin
1046 return '1'
1047 else:
1048 return self.db.curuserid
1050 # get the property (raises KeyErorr if invalid)
1051 prop = self.properties[propname]
1053 if not d.has_key(propname):
1054 if default is _marker:
1055 if isinstance(prop, Multilink):
1056 return []
1057 else:
1058 return None
1059 else:
1060 return default
1062 # return a dupe of the list so code doesn't get confused
1063 if isinstance(prop, Multilink):
1064 return d[propname][:]
1066 return d[propname]
1068 # not in spec
1069 def getnode(self, nodeid, cache=1):
1070 ''' Return a convenience wrapper for the node.
1072 'nodeid' must be the id of an existing node of this class or an
1073 IndexError is raised.
1075 'cache' indicates whether the transaction cache should be queried
1076 for the node. If the node has been modified and you need to
1077 determine what its values prior to modification are, you need to
1078 set cache=0.
1079 '''
1080 return Node(self, nodeid, cache=cache)
1082 def set(self, nodeid, **propvalues):
1083 '''Modify a property on an existing node of this class.
1085 'nodeid' must be the id of an existing node of this class or an
1086 IndexError is raised.
1088 Each key in 'propvalues' must be the name of a property of this
1089 class or a KeyError is raised.
1091 All values in 'propvalues' must be acceptable types for their
1092 corresponding properties or a TypeError is raised.
1094 If the value of the key property is set, it must not collide with
1095 other key strings or a ValueError is raised.
1097 If the value of a Link or Multilink property contains an invalid
1098 node id, a ValueError is raised.
1100 These operations trigger detectors and can be vetoed. Attempts
1101 to modify the "creation" or "activity" properties cause a KeyError.
1102 '''
1103 if not propvalues:
1104 return propvalues
1106 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1107 raise KeyError, '"creation" and "activity" are reserved'
1109 if propvalues.has_key('id'):
1110 raise KeyError, '"id" is reserved'
1112 if self.db.journaltag is None:
1113 raise DatabaseError, 'Database open read-only'
1115 self.fireAuditors('set', nodeid, propvalues)
1116 # Take a copy of the node dict so that the subsequent set
1117 # operation doesn't modify the oldvalues structure.
1118 try:
1119 # try not using the cache initially
1120 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid,
1121 cache=0))
1122 except IndexError:
1123 # this will be needed if somone does a create() and set()
1124 # with no intervening commit()
1125 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1127 node = self.db.getnode(self.classname, nodeid)
1128 if node.has_key(self.db.RETIRED_FLAG):
1129 raise IndexError
1130 num_re = re.compile('^\d+$')
1132 # if the journal value is to be different, store it in here
1133 journalvalues = {}
1135 for propname, value in propvalues.items():
1136 # check to make sure we're not duplicating an existing key
1137 if propname == self.key and node[propname] != value:
1138 try:
1139 self.lookup(value)
1140 except KeyError:
1141 pass
1142 else:
1143 raise ValueError, 'node with key "%s" exists'%value
1145 # this will raise the KeyError if the property isn't valid
1146 # ... we don't use getprops() here because we only care about
1147 # the writeable properties.
1148 try:
1149 prop = self.properties[propname]
1150 except KeyError:
1151 raise KeyError, '"%s" has no property named "%s"'%(
1152 self.classname, propname)
1154 # if the value's the same as the existing value, no sense in
1155 # doing anything
1156 current = node.get(propname, None)
1157 if value == current:
1158 del propvalues[propname]
1159 continue
1160 journalvalues[propname] = current
1162 # do stuff based on the prop type
1163 if isinstance(prop, Link):
1164 link_class = prop.classname
1165 # if it isn't a number, it's a key
1166 if value is not None and not isinstance(value, type('')):
1167 raise ValueError, 'property "%s" link value be a string'%(
1168 propname)
1169 if isinstance(value, type('')) and not num_re.match(value):
1170 try:
1171 value = self.db.classes[link_class].lookup(value)
1172 except (TypeError, KeyError):
1173 raise IndexError, 'new property "%s": %s not a %s'%(
1174 propname, value, prop.classname)
1176 if (value is not None and
1177 not self.db.getclass(link_class).hasnode(value)):
1178 raise IndexError, '%s has no node %s'%(link_class, value)
1180 if self.do_journal and prop.do_journal:
1181 # register the unlink with the old linked node
1182 if node.has_key(propname) and node[propname] is not None:
1183 self.db.addjournal(link_class, node[propname], 'unlink',
1184 (self.classname, nodeid, propname))
1186 # register the link with the newly linked node
1187 if value is not None:
1188 self.db.addjournal(link_class, value, 'link',
1189 (self.classname, nodeid, propname))
1191 elif isinstance(prop, Multilink):
1192 if type(value) != type([]):
1193 raise TypeError, 'new property "%s" not a list of'\
1194 ' ids'%propname
1195 link_class = self.properties[propname].classname
1196 l = []
1197 for entry in value:
1198 # if it isn't a number, it's a key
1199 if type(entry) != type(''):
1200 raise ValueError, 'new property "%s" link value ' \
1201 'must be a string'%propname
1202 if not num_re.match(entry):
1203 try:
1204 entry = self.db.classes[link_class].lookup(entry)
1205 except (TypeError, KeyError):
1206 raise IndexError, 'new property "%s": %s not a %s'%(
1207 propname, entry,
1208 self.properties[propname].classname)
1209 l.append(entry)
1210 value = l
1211 propvalues[propname] = value
1213 # figure the journal entry for this property
1214 add = []
1215 remove = []
1217 # handle removals
1218 if node.has_key(propname):
1219 l = node[propname]
1220 else:
1221 l = []
1222 for id in l[:]:
1223 if id in value:
1224 continue
1225 # register the unlink with the old linked node
1226 if self.do_journal and self.properties[propname].do_journal:
1227 self.db.addjournal(link_class, id, 'unlink',
1228 (self.classname, nodeid, propname))
1229 l.remove(id)
1230 remove.append(id)
1232 # handle additions
1233 for id in value:
1234 if not self.db.getclass(link_class).hasnode(id):
1235 raise IndexError, '%s has no node %s'%(link_class, id)
1236 if id in l:
1237 continue
1238 # register the link with the newly linked node
1239 if self.do_journal and self.properties[propname].do_journal:
1240 self.db.addjournal(link_class, id, 'link',
1241 (self.classname, nodeid, propname))
1242 l.append(id)
1243 add.append(id)
1245 # figure the journal entry
1246 l = []
1247 if add:
1248 l.append(('+', add))
1249 if remove:
1250 l.append(('-', remove))
1251 if l:
1252 journalvalues[propname] = tuple(l)
1254 elif isinstance(prop, String):
1255 if value is not None and type(value) != type('') and type(value) != type(u''):
1256 raise TypeError, 'new property "%s" not a string'%propname
1258 elif isinstance(prop, Password):
1259 if not isinstance(value, password.Password):
1260 raise TypeError, 'new property "%s" not a Password'%propname
1261 propvalues[propname] = value
1263 elif value is not None and isinstance(prop, Date):
1264 if not isinstance(value, date.Date):
1265 raise TypeError, 'new property "%s" not a Date'% propname
1266 propvalues[propname] = value
1268 elif value is not None and isinstance(prop, Interval):
1269 if not isinstance(value, date.Interval):
1270 raise TypeError, 'new property "%s" not an '\
1271 'Interval'%propname
1272 propvalues[propname] = value
1274 elif value is not None and isinstance(prop, Number):
1275 try:
1276 float(value)
1277 except ValueError:
1278 raise TypeError, 'new property "%s" not numeric'%propname
1280 elif value is not None and isinstance(prop, Boolean):
1281 try:
1282 int(value)
1283 except ValueError:
1284 raise TypeError, 'new property "%s" not boolean'%propname
1286 node[propname] = value
1288 # nothing to do?
1289 if not propvalues:
1290 return propvalues
1292 # do the set, and journal it
1293 self.db.setnode(self.classname, nodeid, node)
1295 if self.do_journal:
1296 self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
1298 self.fireReactors('set', nodeid, oldvalues)
1300 return propvalues
1302 def retire(self, nodeid):
1303 '''Retire a node.
1305 The properties on the node remain available from the get() method,
1306 and the node's id is never reused.
1308 Retired nodes are not returned by the find(), list(), or lookup()
1309 methods, and other nodes may reuse the values of their key properties.
1311 These operations trigger detectors and can be vetoed. Attempts
1312 to modify the "creation" or "activity" properties cause a KeyError.
1313 '''
1314 if self.db.journaltag is None:
1315 raise DatabaseError, 'Database open read-only'
1317 self.fireAuditors('retire', nodeid, None)
1319 node = self.db.getnode(self.classname, nodeid)
1320 node[self.db.RETIRED_FLAG] = 1
1321 self.db.setnode(self.classname, nodeid, node)
1322 if self.do_journal:
1323 self.db.addjournal(self.classname, nodeid, 'retired', None)
1325 self.fireReactors('retire', nodeid, None)
1327 def is_retired(self, nodeid):
1328 '''Return true if the node is retired.
1329 '''
1330 node = self.db.getnode(cn, nodeid, cldb)
1331 if node.has_key(self.db.RETIRED_FLAG):
1332 return 1
1333 return 0
1335 def destroy(self, nodeid):
1336 '''Destroy a node.
1338 WARNING: this method should never be used except in extremely rare
1339 situations where there could never be links to the node being
1340 deleted
1341 WARNING: use retire() instead
1342 WARNING: the properties of this node will not be available ever again
1343 WARNING: really, use retire() instead
1345 Well, I think that's enough warnings. This method exists mostly to
1346 support the session storage of the cgi interface.
1347 '''
1348 if self.db.journaltag is None:
1349 raise DatabaseError, 'Database open read-only'
1350 self.db.destroynode(self.classname, nodeid)
1352 def history(self, nodeid):
1353 '''Retrieve the journal of edits on a particular node.
1355 'nodeid' must be the id of an existing node of this class or an
1356 IndexError is raised.
1358 The returned list contains tuples of the form
1360 (nodeid, date, tag, action, params)
1362 'date' is a Timestamp object specifying the time of the change and
1363 'tag' is the journaltag specified when the database was opened.
1364 '''
1365 if not self.do_journal:
1366 raise ValueError, 'Journalling is disabled for this class'
1367 return self.db.getjournal(self.classname, nodeid)
1369 # Locating nodes:
1370 def hasnode(self, nodeid):
1371 '''Determine if the given nodeid actually exists
1372 '''
1373 return self.db.hasnode(self.classname, nodeid)
1375 def setkey(self, propname):
1376 '''Select a String property of this class to be the key property.
1378 'propname' must be the name of a String property of this class or
1379 None, or a TypeError is raised. The values of the key property on
1380 all existing nodes must be unique or a ValueError is raised. If the
1381 property doesn't exist, KeyError is raised.
1382 '''
1383 prop = self.getprops()[propname]
1384 if not isinstance(prop, String):
1385 raise TypeError, 'key properties must be String'
1386 self.key = propname
1388 def getkey(self):
1389 '''Return the name of the key property for this class or None.'''
1390 return self.key
1392 def labelprop(self, default_to_id=0):
1393 ''' Return the property name for a label for the given node.
1395 This method attempts to generate a consistent label for the node.
1396 It tries the following in order:
1397 1. key property
1398 2. "name" property
1399 3. "title" property
1400 4. first property from the sorted property name list
1401 '''
1402 k = self.getkey()
1403 if k:
1404 return k
1405 props = self.getprops()
1406 if props.has_key('name'):
1407 return 'name'
1408 elif props.has_key('title'):
1409 return 'title'
1410 if default_to_id:
1411 return 'id'
1412 props = props.keys()
1413 props.sort()
1414 return props[0]
1416 # TODO: set up a separate index db file for this? profile?
1417 def lookup(self, keyvalue):
1418 '''Locate a particular node by its key property and return its id.
1420 If this class has no key property, a TypeError is raised. If the
1421 'keyvalue' matches one of the values for the key property among
1422 the nodes in this class, the matching node's id is returned;
1423 otherwise a KeyError is raised.
1424 '''
1425 if not self.key:
1426 raise TypeError, 'No key property set for class %s'%self.classname
1427 cldb = self.db.getclassdb(self.classname)
1428 try:
1429 for nodeid in self.db.getnodeids(self.classname, cldb):
1430 node = self.db.getnode(self.classname, nodeid, cldb)
1431 if node.has_key(self.db.RETIRED_FLAG):
1432 continue
1433 if node[self.key] == keyvalue:
1434 return nodeid
1435 finally:
1436 cldb.close()
1437 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1438 keyvalue, self.classname)
1440 # change from spec - allows multiple props to match
1441 def find(self, **propspec):
1442 '''Get the ids of nodes in this class which link to the given nodes.
1444 'propspec' consists of keyword args propname=nodeid or
1445 propname={nodeid:1, }
1446 'propname' must be the name of a property in this class, or a
1447 KeyError is raised. That property must be a Link or
1448 Multilink property, or a TypeError is raised.
1450 Any node in this class whose 'propname' property links to any of the
1451 nodeids will be returned. Used by the full text indexing, which knows
1452 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1453 issues:
1455 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1456 '''
1457 propspec = propspec.items()
1458 for propname, nodeids in propspec:
1459 # check the prop is OK
1460 prop = self.properties[propname]
1461 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1462 raise TypeError, "'%s' not a Link/Multilink property"%propname
1464 # ok, now do the find
1465 cldb = self.db.getclassdb(self.classname)
1466 l = []
1467 try:
1468 for id in self.db.getnodeids(self.classname, db=cldb):
1469 node = self.db.getnode(self.classname, id, db=cldb)
1470 if node.has_key(self.db.RETIRED_FLAG):
1471 continue
1472 for propname, nodeids in propspec:
1473 # can't test if the node doesn't have this property
1474 if not node.has_key(propname):
1475 continue
1476 if type(nodeids) is type(''):
1477 nodeids = {nodeids:1}
1478 prop = self.properties[propname]
1479 value = node[propname]
1480 if isinstance(prop, Link) and nodeids.has_key(value):
1481 l.append(id)
1482 break
1483 elif isinstance(prop, Multilink):
1484 hit = 0
1485 for v in value:
1486 if nodeids.has_key(v):
1487 l.append(id)
1488 hit = 1
1489 break
1490 if hit:
1491 break
1492 finally:
1493 cldb.close()
1494 return l
1496 def stringFind(self, **requirements):
1497 '''Locate a particular node by matching a set of its String
1498 properties in a caseless search.
1500 If the property is not a String property, a TypeError is raised.
1502 The return is a list of the id of all nodes that match.
1503 '''
1504 for propname in requirements.keys():
1505 prop = self.properties[propname]
1506 if isinstance(not prop, String):
1507 raise TypeError, "'%s' not a String property"%propname
1508 requirements[propname] = requirements[propname].lower()
1509 l = []
1510 cldb = self.db.getclassdb(self.classname)
1511 try:
1512 for nodeid in self.db.getnodeids(self.classname, cldb):
1513 node = self.db.getnode(self.classname, nodeid, cldb)
1514 if node.has_key(self.db.RETIRED_FLAG):
1515 continue
1516 for key, value in requirements.items():
1517 if not node.has_key(key):
1518 break
1519 if node[key] is None or node[key].lower() != value:
1520 break
1521 else:
1522 l.append(nodeid)
1523 finally:
1524 cldb.close()
1525 return l
1527 def list(self):
1528 ''' Return a list of the ids of the active nodes in this class.
1529 '''
1530 l = []
1531 cn = self.classname
1532 cldb = self.db.getclassdb(cn)
1533 try:
1534 for nodeid in self.db.getnodeids(cn, cldb):
1535 node = self.db.getnode(cn, nodeid, cldb)
1536 if node.has_key(self.db.RETIRED_FLAG):
1537 continue
1538 l.append(nodeid)
1539 finally:
1540 cldb.close()
1541 l.sort()
1542 return l
1544 def filter(self, search_matches, filterspec, sort=(None,None),
1545 group=(None,None), num_re = re.compile('^\d+$')):
1546 ''' Return a list of the ids of the active nodes in this class that
1547 match the 'filter' spec, sorted by the group spec and then the
1548 sort spec.
1550 "filterspec" is {propname: value(s)}
1551 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1552 and prop is a prop name or None
1553 "search_matches" is {nodeid: marker}
1555 The filter must match all properties specificed - but if the
1556 property value to match is a list, any one of the values in the
1557 list may match for that property to match.
1558 '''
1559 cn = self.classname
1561 # optimise filterspec
1562 l = []
1563 props = self.getprops()
1564 LINK = 0
1565 MULTILINK = 1
1566 STRING = 2
1567 OTHER = 6
1568 for k, v in filterspec.items():
1569 propclass = props[k]
1570 if isinstance(propclass, Link):
1571 if type(v) is not type([]):
1572 v = [v]
1573 # replace key values with node ids
1574 u = []
1575 link_class = self.db.classes[propclass.classname]
1576 for entry in v:
1577 if entry == '-1': entry = None
1578 elif not num_re.match(entry):
1579 try:
1580 entry = link_class.lookup(entry)
1581 except (TypeError,KeyError):
1582 raise ValueError, 'property "%s": %s not a %s'%(
1583 k, entry, self.properties[k].classname)
1584 u.append(entry)
1586 l.append((LINK, k, u))
1587 elif isinstance(propclass, Multilink):
1588 if type(v) is not type([]):
1589 v = [v]
1590 # replace key values with node ids
1591 u = []
1592 link_class = self.db.classes[propclass.classname]
1593 for entry in v:
1594 if not num_re.match(entry):
1595 try:
1596 entry = link_class.lookup(entry)
1597 except (TypeError,KeyError):
1598 raise ValueError, 'new property "%s": %s not a %s'%(
1599 k, entry, self.properties[k].classname)
1600 u.append(entry)
1601 l.append((MULTILINK, k, u))
1602 elif isinstance(propclass, String) and k != 'id':
1603 # simple glob searching
1604 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1605 v = v.replace('?', '.')
1606 v = v.replace('*', '.*?')
1607 l.append((STRING, k, re.compile(v, re.I)))
1608 elif isinstance(propclass, Boolean):
1609 if type(v) is type(''):
1610 bv = v.lower() in ('yes', 'true', 'on', '1')
1611 else:
1612 bv = v
1613 l.append((OTHER, k, bv))
1614 elif isinstance(propclass, Date):
1615 l.append((OTHER, k, date.Date(v)))
1616 elif isinstance(propclass, Interval):
1617 l.append((OTHER, k, date.Interval(v)))
1618 elif isinstance(propclass, Number):
1619 l.append((OTHER, k, int(v)))
1620 else:
1621 l.append((OTHER, k, v))
1622 filterspec = l
1624 # now, find all the nodes that are active and pass filtering
1625 l = []
1626 cldb = self.db.getclassdb(cn)
1627 try:
1628 # TODO: only full-scan once (use items())
1629 for nodeid in self.db.getnodeids(cn, cldb):
1630 node = self.db.getnode(cn, nodeid, cldb)
1631 if node.has_key(self.db.RETIRED_FLAG):
1632 continue
1633 # apply filter
1634 for t, k, v in filterspec:
1635 # handle the id prop
1636 if k == 'id' and v == nodeid:
1637 continue
1639 # make sure the node has the property
1640 if not node.has_key(k):
1641 # this node doesn't have this property, so reject it
1642 break
1644 # now apply the property filter
1645 if t == LINK:
1646 # link - if this node's property doesn't appear in the
1647 # filterspec's nodeid list, skip it
1648 if node[k] not in v:
1649 break
1650 elif t == MULTILINK:
1651 # multilink - if any of the nodeids required by the
1652 # filterspec aren't in this node's property, then skip
1653 # it
1654 have = node[k]
1655 for want in v:
1656 if want not in have:
1657 break
1658 else:
1659 continue
1660 break
1661 elif t == STRING:
1662 # RE search
1663 if node[k] is None or not v.search(node[k]):
1664 break
1665 elif t == OTHER:
1666 # straight value comparison for the other types
1667 if node[k] != v:
1668 break
1669 else:
1670 l.append((nodeid, node))
1671 finally:
1672 cldb.close()
1673 l.sort()
1675 # filter based on full text search
1676 if search_matches is not None:
1677 k = []
1678 for v in l:
1679 if search_matches.has_key(v[0]):
1680 k.append(v)
1681 l = k
1683 # now, sort the result
1684 def sortfun(a, b, sort=sort, group=group, properties=self.getprops(),
1685 db = self.db, cl=self):
1686 a_id, an = a
1687 b_id, bn = b
1688 # sort by group and then sort
1689 for dir, prop in group, sort:
1690 if dir is None or prop is None: continue
1692 # sorting is class-specific
1693 propclass = properties[prop]
1695 # handle the properties that might be "faked"
1696 # also, handle possible missing properties
1697 try:
1698 if not an.has_key(prop):
1699 an[prop] = cl.get(a_id, prop)
1700 av = an[prop]
1701 except KeyError:
1702 # the node doesn't have a value for this property
1703 if isinstance(propclass, Multilink): av = []
1704 else: av = ''
1705 try:
1706 if not bn.has_key(prop):
1707 bn[prop] = cl.get(b_id, prop)
1708 bv = bn[prop]
1709 except KeyError:
1710 # the node doesn't have a value for this property
1711 if isinstance(propclass, Multilink): bv = []
1712 else: bv = ''
1714 # String and Date values are sorted in the natural way
1715 if isinstance(propclass, String):
1716 # clean up the strings
1717 if av and av[0] in string.uppercase:
1718 av = av.lower()
1719 if bv and bv[0] in string.uppercase:
1720 bv = bv.lower()
1721 if (isinstance(propclass, String) or
1722 isinstance(propclass, Date)):
1723 # it might be a string that's really an integer
1724 try:
1725 av = int(av)
1726 bv = int(bv)
1727 except:
1728 pass
1729 if dir == '+':
1730 r = cmp(av, bv)
1731 if r != 0: return r
1732 elif dir == '-':
1733 r = cmp(bv, av)
1734 if r != 0: return r
1736 # Link properties are sorted according to the value of
1737 # the "order" property on the linked nodes if it is
1738 # present; or otherwise on the key string of the linked
1739 # nodes; or finally on the node ids.
1740 elif isinstance(propclass, Link):
1741 link = db.classes[propclass.classname]
1742 if av is None and bv is not None: return -1
1743 if av is not None and bv is None: return 1
1744 if av is None and bv is None: continue
1745 if link.getprops().has_key('order'):
1746 if dir == '+':
1747 r = cmp(link.get(av, 'order'),
1748 link.get(bv, 'order'))
1749 if r != 0: return r
1750 elif dir == '-':
1751 r = cmp(link.get(bv, 'order'),
1752 link.get(av, 'order'))
1753 if r != 0: return r
1754 elif link.getkey():
1755 key = link.getkey()
1756 if dir == '+':
1757 r = cmp(link.get(av, key), link.get(bv, key))
1758 if r != 0: return r
1759 elif dir == '-':
1760 r = cmp(link.get(bv, key), link.get(av, key))
1761 if r != 0: return r
1762 else:
1763 if dir == '+':
1764 r = cmp(av, bv)
1765 if r != 0: return r
1766 elif dir == '-':
1767 r = cmp(bv, av)
1768 if r != 0: return r
1770 # Multilink properties are sorted according to how many
1771 # links are present.
1772 elif isinstance(propclass, Multilink):
1773 r = cmp(len(av), len(bv))
1774 if r == 0:
1775 # Compare contents of multilink property if lenghts is
1776 # equal
1777 r = cmp ('.'.join(av), '.'.join(bv))
1778 if dir == '+':
1779 return r
1780 elif dir == '-':
1781 return -r
1782 elif isinstance(propclass, Number) or isinstance(propclass, Boolean):
1783 if dir == '+':
1784 r = cmp(av, bv)
1785 elif dir == '-':
1786 r = cmp(bv, av)
1788 # end for dir, prop in sort, group:
1789 # if all else fails, compare the ids
1790 return cmp(a[0], b[0])
1792 l.sort(sortfun)
1793 return [i[0] for i in l]
1795 def count(self):
1796 '''Get the number of nodes in this class.
1798 If the returned integer is 'numnodes', the ids of all the nodes
1799 in this class run from 1 to numnodes, and numnodes+1 will be the
1800 id of the next node to be created in this class.
1801 '''
1802 return self.db.countnodes(self.classname)
1804 # Manipulating properties:
1806 def getprops(self, protected=1):
1807 '''Return a dictionary mapping property names to property objects.
1808 If the "protected" flag is true, we include protected properties -
1809 those which may not be modified.
1811 In addition to the actual properties on the node, these
1812 methods provide the "creation" and "activity" properties. If the
1813 "protected" flag is true, we include protected properties - those
1814 which may not be modified.
1815 '''
1816 d = self.properties.copy()
1817 if protected:
1818 d['id'] = String()
1819 d['creation'] = hyperdb.Date()
1820 d['activity'] = hyperdb.Date()
1821 d['creator'] = hyperdb.Link('user')
1822 return d
1824 def addprop(self, **properties):
1825 '''Add properties to this class.
1827 The keyword arguments in 'properties' must map names to property
1828 objects, or a TypeError is raised. None of the keys in 'properties'
1829 may collide with the names of existing properties, or a ValueError
1830 is raised before any properties have been added.
1831 '''
1832 for key in properties.keys():
1833 if self.properties.has_key(key):
1834 raise ValueError, key
1835 self.properties.update(properties)
1837 def index(self, nodeid):
1838 '''Add (or refresh) the node to search indexes
1839 '''
1840 # find all the String properties that have indexme
1841 for prop, propclass in self.getprops().items():
1842 if isinstance(propclass, String) and propclass.indexme:
1843 try:
1844 value = str(self.get(nodeid, prop))
1845 except IndexError:
1846 # node no longer exists - entry should be removed
1847 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1848 else:
1849 # and index them under (classname, nodeid, property)
1850 self.db.indexer.add_text((self.classname, nodeid, prop),
1851 value)
1853 #
1854 # Detector interface
1855 #
1856 def audit(self, event, detector):
1857 '''Register a detector
1858 '''
1859 l = self.auditors[event]
1860 if detector not in l:
1861 self.auditors[event].append(detector)
1863 def fireAuditors(self, action, nodeid, newvalues):
1864 '''Fire all registered auditors.
1865 '''
1866 for audit in self.auditors[action]:
1867 audit(self.db, self, nodeid, newvalues)
1869 def react(self, event, detector):
1870 '''Register a detector
1871 '''
1872 l = self.reactors[event]
1873 if detector not in l:
1874 self.reactors[event].append(detector)
1876 def fireReactors(self, action, nodeid, oldvalues):
1877 '''Fire all registered reactors.
1878 '''
1879 for react in self.reactors[action]:
1880 react(self.db, self, nodeid, oldvalues)
1882 class FileClass(Class):
1883 '''This class defines a large chunk of data. To support this, it has a
1884 mandatory String property "content" which is typically saved off
1885 externally to the hyperdb.
1887 The default MIME type of this data is defined by the
1888 "default_mime_type" class attribute, which may be overridden by each
1889 node if the class defines a "type" String property.
1890 '''
1891 default_mime_type = 'text/plain'
1893 def create(self, **propvalues):
1894 ''' Snarf the "content" propvalue and store in a file
1895 '''
1896 # we need to fire the auditors now, or the content property won't
1897 # be in propvalues for the auditors to play with
1898 self.fireAuditors('create', None, propvalues)
1900 # now remove the content property so it's not stored in the db
1901 content = propvalues['content']
1902 del propvalues['content']
1904 # do the database create
1905 newid = Class.create_inner(self, **propvalues)
1907 # fire reactors
1908 self.fireReactors('create', newid, None)
1910 # store off the content as a file
1911 self.db.storefile(self.classname, newid, None, content)
1912 return newid
1914 def import_list(self, propnames, proplist):
1915 ''' Trap the "content" property...
1916 '''
1917 # dupe this list so we don't affect others
1918 propnames = propnames[:]
1920 # extract the "content" property from the proplist
1921 i = propnames.index('content')
1922 content = eval(proplist[i])
1923 del propnames[i]
1924 del proplist[i]
1926 # do the normal import
1927 newid = Class.import_list(self, propnames, proplist)
1929 # save off the "content" file
1930 self.db.storefile(self.classname, newid, None, content)
1931 return newid
1933 def get(self, nodeid, propname, default=_marker, cache=1):
1934 ''' trap the content propname and get it from the file
1935 '''
1936 poss_msg = 'Possibly an access right configuration problem.'
1937 if propname == 'content':
1938 try:
1939 return self.db.getfile(self.classname, nodeid, None)
1940 except IOError, (strerror):
1941 # XXX by catching this we donot see an error in the log.
1942 return 'ERROR reading file: %s%s\n%s\n%s'%(
1943 self.classname, nodeid, poss_msg, strerror)
1944 if default is not _marker:
1945 return Class.get(self, nodeid, propname, default, cache=cache)
1946 else:
1947 return Class.get(self, nodeid, propname, cache=cache)
1949 def getprops(self, protected=1):
1950 ''' In addition to the actual properties on the node, these methods
1951 provide the "content" property. If the "protected" flag is true,
1952 we include protected properties - those which may not be
1953 modified.
1954 '''
1955 d = Class.getprops(self, protected=protected).copy()
1956 d['content'] = hyperdb.String()
1957 return d
1959 def index(self, nodeid):
1960 ''' Index the node in the search index.
1962 We want to index the content in addition to the normal String
1963 property indexing.
1964 '''
1965 # perform normal indexing
1966 Class.index(self, nodeid)
1968 # get the content to index
1969 content = self.get(nodeid, 'content')
1971 # figure the mime type
1972 if self.properties.has_key('type'):
1973 mime_type = self.get(nodeid, 'type')
1974 else:
1975 mime_type = self.default_mime_type
1977 # and index!
1978 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
1979 mime_type)
1981 # deviation from spec - was called ItemClass
1982 class IssueClass(Class, roundupdb.IssueClass):
1983 # Overridden methods:
1984 def __init__(self, db, classname, **properties):
1985 '''The newly-created class automatically includes the "messages",
1986 "files", "nosy", and "superseder" properties. If the 'properties'
1987 dictionary attempts to specify any of these properties or a
1988 "creation" or "activity" property, a ValueError is raised.
1989 '''
1990 if not properties.has_key('title'):
1991 properties['title'] = hyperdb.String(indexme='yes')
1992 if not properties.has_key('messages'):
1993 properties['messages'] = hyperdb.Multilink("msg")
1994 if not properties.has_key('files'):
1995 properties['files'] = hyperdb.Multilink("file")
1996 if not properties.has_key('nosy'):
1997 # note: journalling is turned off as it really just wastes
1998 # space. this behaviour may be overridden in an instance
1999 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2000 if not properties.has_key('superseder'):
2001 properties['superseder'] = hyperdb.Multilink(classname)
2002 Class.__init__(self, db, classname, **properties)
2004 #