7091d19bd218995e1bd78a529f6c06b49bb207eb
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: back_anydbm.py,v 1.138 2004-03-18 01:58:45 richard Exp $
19 '''This module defines a backend that saves the hyperdatabase in a
20 database chosen by anydbm. It is guaranteed to always be available in python
21 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
22 serious bugs, and is not available)
23 '''
24 __docformat__ = 'restructuredtext'
26 try:
27 import anydbm, sys
28 # dumbdbm only works in python 2.1.2+
29 if sys.version_info < (2,1,2):
30 import dumbdbm
31 assert anydbm._defaultmod != dumbdbm
32 del dumbdbm
33 except AssertionError:
34 print "WARNING: you should upgrade to python 2.1.3"
36 import whichdb, os, marshal, re, weakref, string, copy
37 from roundup import hyperdb, date, password, roundupdb, security
38 from blobfiles import FileStorage
39 from sessions_dbm import Sessions, OneTimeKeys
40 from roundup.indexer import Indexer
41 from roundup.backends import locking
42 from roundup.hyperdb import String, Password, Date, Interval, Link, \
43 Multilink, DatabaseError, Boolean, Number, Node
44 from roundup.date import Range
46 #
47 # Now the database
48 #
49 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
50 '''A database for storing records containing flexible data types.
52 Transaction stuff TODO:
54 - check the timestamp of the class file and nuke the cache if it's
55 modified. Do some sort of conflict checking on the dirty stuff.
56 - perhaps detect write collisions (related to above)?
57 '''
58 def __init__(self, config, journaltag=None):
59 '''Open a hyperdatabase given a specifier to some storage.
61 The 'storagelocator' is obtained from config.DATABASE.
62 The meaning of 'storagelocator' depends on the particular
63 implementation of the hyperdatabase. It could be a file name,
64 a directory path, a socket descriptor for a connection to a
65 database over the network, etc.
67 The 'journaltag' is a token that will be attached to the journal
68 entries for any edits done on the database. If 'journaltag' is
69 None, the database is opened in read-only mode: the Class.create(),
70 Class.set(), Class.retire(), and Class.restore() methods are
71 disabled.
72 '''
73 self.config, self.journaltag = config, journaltag
74 self.dir = config.DATABASE
75 self.classes = {}
76 self.cache = {} # cache of nodes loaded or created
77 self.dirtynodes = {} # keep track of the dirty nodes by class
78 self.newnodes = {} # keep track of the new nodes by class
79 self.destroyednodes = {}# keep track of the destroyed nodes by class
80 self.transactions = []
81 self.indexer = Indexer(self.dir)
82 self.security = security.Security(self)
83 # ensure files are group readable and writable
84 os.umask(0002)
86 # lock it
87 lockfilenm = os.path.join(self.dir, 'lock')
88 self.lockfile = locking.acquire_lock(lockfilenm)
89 self.lockfile.write(str(os.getpid()))
90 self.lockfile.flush()
92 def post_init(self):
93 '''Called once the schema initialisation has finished.
94 '''
95 # reindex the db if necessary
96 if self.indexer.should_reindex():
97 self.reindex()
99 def refresh_database(self):
100 """Rebuild the database
101 """
102 self.reindex()
104 def getSessionManager(self):
105 return Sessions(self)
107 def getOTKManager(self):
108 return OneTimeKeys(self)
110 def reindex(self):
111 for klass in self.classes.values():
112 for nodeid in klass.list():
113 klass.index(nodeid)
114 self.indexer.save_index()
116 def __repr__(self):
117 return '<back_anydbm instance at %x>'%id(self)
119 #
120 # Classes
121 #
122 def __getattr__(self, classname):
123 '''A convenient way of calling self.getclass(classname).'''
124 if self.classes.has_key(classname):
125 if __debug__:
126 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
127 return self.classes[classname]
128 raise AttributeError, classname
130 def addclass(self, cl):
131 if __debug__:
132 print >>hyperdb.DEBUG, 'addclass', (self, cl)
133 cn = cl.classname
134 if self.classes.has_key(cn):
135 raise ValueError, cn
136 self.classes[cn] = cl
138 # add default Edit and View permissions
139 self.security.addPermission(name="Edit", klass=cn,
140 description="User is allowed to edit "+cn)
141 self.security.addPermission(name="View", klass=cn,
142 description="User is allowed to access "+cn)
144 def getclasses(self):
145 '''Return a list of the names of all existing classes.'''
146 if __debug__:
147 print >>hyperdb.DEBUG, 'getclasses', (self,)
148 l = self.classes.keys()
149 l.sort()
150 return l
152 def getclass(self, classname):
153 '''Get the Class object representing a particular class.
155 If 'classname' is not a valid class name, a KeyError is raised.
156 '''
157 if __debug__:
158 print >>hyperdb.DEBUG, 'getclass', (self, classname)
159 try:
160 return self.classes[classname]
161 except KeyError:
162 raise KeyError, 'There is no class called "%s"'%classname
164 #
165 # Class DBs
166 #
167 def clear(self):
168 '''Delete all database contents
169 '''
170 if __debug__:
171 print >>hyperdb.DEBUG, 'clear', (self,)
172 for cn in self.classes.keys():
173 for dummy in 'nodes', 'journals':
174 path = os.path.join(self.dir, 'journals.%s'%cn)
175 if os.path.exists(path):
176 os.remove(path)
177 elif os.path.exists(path+'.db'): # dbm appends .db
178 os.remove(path+'.db')
180 def getclassdb(self, classname, mode='r'):
181 ''' grab a connection to the class db that will be used for
182 multiple actions
183 '''
184 if __debug__:
185 print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode)
186 return self.opendb('nodes.%s'%classname, mode)
188 def determine_db_type(self, path):
189 ''' determine which DB wrote the class file
190 '''
191 db_type = ''
192 if os.path.exists(path):
193 db_type = whichdb.whichdb(path)
194 if not db_type:
195 raise DatabaseError, "Couldn't identify database type"
196 elif os.path.exists(path+'.db'):
197 # if the path ends in '.db', it's a dbm database, whether
198 # anydbm says it's dbhash or not!
199 db_type = 'dbm'
200 return db_type
202 def opendb(self, name, mode):
203 '''Low-level database opener that gets around anydbm/dbm
204 eccentricities.
205 '''
206 if __debug__:
207 print >>hyperdb.DEBUG, 'opendb', (self, name, mode)
209 # figure the class db type
210 path = os.path.join(os.getcwd(), self.dir, name)
211 db_type = self.determine_db_type(path)
213 # new database? let anydbm pick the best dbm
214 if not db_type:
215 if __debug__:
216 print >>hyperdb.DEBUG, "opendb anydbm.open(%r, 'c')"%path
217 return anydbm.open(path, 'c')
219 # open the database with the correct module
220 try:
221 dbm = __import__(db_type)
222 except ImportError:
223 raise DatabaseError, \
224 "Couldn't open database - the required module '%s'"\
225 " is not available"%db_type
226 if __debug__:
227 print >>hyperdb.DEBUG, "opendb %r.open(%r, %r)"%(db_type, path,
228 mode)
229 return dbm.open(path, mode)
231 #
232 # Node IDs
233 #
234 def newid(self, classname):
235 ''' Generate a new id for the given class
236 '''
237 # open the ids DB - create if if doesn't exist
238 db = self.opendb('_ids', 'c')
239 if db.has_key(classname):
240 newid = db[classname] = str(int(db[classname]) + 1)
241 else:
242 # the count() bit is transitional - older dbs won't start at 1
243 newid = str(self.getclass(classname).count()+1)
244 db[classname] = newid
245 db.close()
246 return newid
248 def setid(self, classname, setid):
249 ''' Set the id counter: used during import of database
250 '''
251 # open the ids DB - create if if doesn't exist
252 db = self.opendb('_ids', 'c')
253 db[classname] = str(setid)
254 db.close()
256 #
257 # Nodes
258 #
259 def addnode(self, classname, nodeid, node):
260 ''' add the specified node to its class's db
261 '''
262 if __debug__:
263 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
265 # we'll be supplied these props if we're doing an import
266 if not node.has_key('creator'):
267 # add in the "calculated" properties (dupe so we don't affect
268 # calling code's node assumptions)
269 node = node.copy()
270 node['creator'] = self.getuid()
271 node['actor'] = self.getuid()
272 node['creation'] = node['activity'] = date.Date()
274 self.newnodes.setdefault(classname, {})[nodeid] = 1
275 self.cache.setdefault(classname, {})[nodeid] = node
276 self.savenode(classname, nodeid, node)
278 def setnode(self, classname, nodeid, node):
279 ''' change the specified node
280 '''
281 if __debug__:
282 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
283 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
285 # update the activity time (dupe so we don't affect
286 # calling code's node assumptions)
287 node = node.copy()
288 node['activity'] = date.Date()
289 node['actor'] = self.getuid()
291 # can't set without having already loaded the node
292 self.cache[classname][nodeid] = node
293 self.savenode(classname, nodeid, node)
295 def savenode(self, classname, nodeid, node):
296 ''' perform the saving of data specified by the set/addnode
297 '''
298 if __debug__:
299 print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node)
300 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
302 def getnode(self, classname, nodeid, db=None, cache=1):
303 ''' get a node from the database
305 Note the "cache" parameter is not used, and exists purely for
306 backward compatibility!
307 '''
308 if __debug__:
309 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid, db)
311 # try the cache
312 cache_dict = self.cache.setdefault(classname, {})
313 if cache_dict.has_key(nodeid):
314 if __debug__:
315 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
316 nodeid)
317 return cache_dict[nodeid]
319 if __debug__:
320 print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid)
322 # get from the database and save in the cache
323 if db is None:
324 db = self.getclassdb(classname)
325 if not db.has_key(nodeid):
326 raise IndexError, "no such %s %s"%(classname, nodeid)
328 # check the uncommitted, destroyed nodes
329 if (self.destroyednodes.has_key(classname) and
330 self.destroyednodes[classname].has_key(nodeid)):
331 raise IndexError, "no such %s %s"%(classname, nodeid)
333 # decode
334 res = marshal.loads(db[nodeid])
336 # reverse the serialisation
337 res = self.unserialise(classname, res)
339 # store off in the cache dict
340 if cache:
341 cache_dict[nodeid] = res
343 return res
345 def destroynode(self, classname, nodeid):
346 '''Remove a node from the database. Called exclusively by the
347 destroy() method on Class.
348 '''
349 if __debug__:
350 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
352 # remove from cache and newnodes if it's there
353 if (self.cache.has_key(classname) and
354 self.cache[classname].has_key(nodeid)):
355 del self.cache[classname][nodeid]
356 if (self.newnodes.has_key(classname) and
357 self.newnodes[classname].has_key(nodeid)):
358 del self.newnodes[classname][nodeid]
360 # see if there's any obvious commit actions that we should get rid of
361 for entry in self.transactions[:]:
362 if entry[1][:2] == (classname, nodeid):
363 self.transactions.remove(entry)
365 # add to the destroyednodes map
366 self.destroyednodes.setdefault(classname, {})[nodeid] = 1
368 # add the destroy commit action
369 self.transactions.append((self.doDestroyNode, (classname, nodeid)))
371 def serialise(self, classname, node):
372 '''Copy the node contents, converting non-marshallable data into
373 marshallable data.
374 '''
375 if __debug__:
376 print >>hyperdb.DEBUG, 'serialise', classname, node
377 properties = self.getclass(classname).getprops()
378 d = {}
379 for k, v in node.items():
380 # if the property doesn't exist, or is the "retired" flag then
381 # it won't be in the properties dict
382 if not properties.has_key(k):
383 d[k] = v
384 continue
386 # get the property spec
387 prop = properties[k]
389 if isinstance(prop, Password) and v is not None:
390 d[k] = str(v)
391 elif isinstance(prop, Date) and v is not None:
392 d[k] = v.serialise()
393 elif isinstance(prop, Interval) and v is not None:
394 d[k] = v.serialise()
395 else:
396 d[k] = v
397 return d
399 def unserialise(self, classname, node):
400 '''Decode the marshalled node data
401 '''
402 if __debug__:
403 print >>hyperdb.DEBUG, 'unserialise', classname, node
404 properties = self.getclass(classname).getprops()
405 d = {}
406 for k, v in node.items():
407 # if the property doesn't exist, or is the "retired" flag then
408 # it won't be in the properties dict
409 if not properties.has_key(k):
410 d[k] = v
411 continue
413 # get the property spec
414 prop = properties[k]
416 if isinstance(prop, Date) and v is not None:
417 d[k] = date.Date(v)
418 elif isinstance(prop, Interval) and v is not None:
419 d[k] = date.Interval(v)
420 elif isinstance(prop, Password) and v is not None:
421 p = password.Password()
422 p.unpack(v)
423 d[k] = p
424 else:
425 d[k] = v
426 return d
428 def hasnode(self, classname, nodeid, db=None):
429 ''' determine if the database has a given node
430 '''
431 if __debug__:
432 print >>hyperdb.DEBUG, 'hasnode', (self, classname, nodeid, db)
434 # try the cache
435 cache = self.cache.setdefault(classname, {})
436 if cache.has_key(nodeid):
437 if __debug__:
438 print >>hyperdb.TRACE, 'has %s %s cached'%(classname, nodeid)
439 return 1
440 if __debug__:
441 print >>hyperdb.TRACE, 'has %s %s'%(classname, nodeid)
443 # not in the cache - check the database
444 if db is None:
445 db = self.getclassdb(classname)
446 res = db.has_key(nodeid)
447 return res
449 def countnodes(self, classname, db=None):
450 if __debug__:
451 print >>hyperdb.DEBUG, 'countnodes', (self, classname, db)
453 count = 0
455 # include the uncommitted nodes
456 if self.newnodes.has_key(classname):
457 count += len(self.newnodes[classname])
458 if self.destroyednodes.has_key(classname):
459 count -= len(self.destroyednodes[classname])
461 # and count those in the DB
462 if db is None:
463 db = self.getclassdb(classname)
464 count = count + len(db.keys())
465 return count
468 #
469 # Files - special node properties
470 # inherited from FileStorage
472 #
473 # Journal
474 #
475 def addjournal(self, classname, nodeid, action, params, creator=None,
476 creation=None):
477 ''' Journal the Action
478 'action' may be:
480 'create' or 'set' -- 'params' is a dictionary of property values
481 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
482 'retire' -- 'params' is None
483 '''
484 if __debug__:
485 print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid,
486 action, params, creator, creation)
487 self.transactions.append((self.doSaveJournal, (classname, nodeid,
488 action, params, creator, creation)))
490 def getjournal(self, classname, nodeid):
491 ''' get the journal for id
493 Raise IndexError if the node doesn't exist (as per history()'s
494 API)
495 '''
496 if __debug__:
497 print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid)
499 # our journal result
500 res = []
502 # add any journal entries for transactions not committed to the
503 # database
504 for method, args in self.transactions:
505 if method != self.doSaveJournal:
506 continue
507 (cache_classname, cache_nodeid, cache_action, cache_params,
508 cache_creator, cache_creation) = args
509 if cache_classname == classname and cache_nodeid == nodeid:
510 if not cache_creator:
511 cache_creator = self.getuid()
512 if not cache_creation:
513 cache_creation = date.Date()
514 res.append((cache_nodeid, cache_creation, cache_creator,
515 cache_action, cache_params))
517 # attempt to open the journal - in some rare cases, the journal may
518 # not exist
519 try:
520 db = self.opendb('journals.%s'%classname, 'r')
521 except anydbm.error, error:
522 if str(error) == "need 'c' or 'n' flag to open new db":
523 raise IndexError, 'no such %s %s'%(classname, nodeid)
524 elif error.args[0] != 2:
525 # this isn't a "not found" error, be alarmed!
526 raise
527 if res:
528 # we have unsaved journal entries, return them
529 return res
530 raise IndexError, 'no such %s %s'%(classname, nodeid)
531 try:
532 journal = marshal.loads(db[nodeid])
533 except KeyError:
534 db.close()
535 if res:
536 # we have some unsaved journal entries, be happy!
537 return res
538 raise IndexError, 'no such %s %s'%(classname, nodeid)
539 db.close()
541 # add all the saved journal entries for this node
542 for nodeid, date_stamp, user, action, params in journal:
543 res.append((nodeid, date.Date(date_stamp), user, action, params))
544 return res
546 def pack(self, pack_before):
547 ''' Delete all journal entries except "create" before 'pack_before'.
548 '''
549 if __debug__:
550 print >>hyperdb.DEBUG, 'packjournal', (self, pack_before)
552 pack_before = pack_before.serialise()
553 for classname in self.getclasses():
554 # get the journal db
555 db_name = 'journals.%s'%classname
556 path = os.path.join(os.getcwd(), self.dir, classname)
557 db_type = self.determine_db_type(path)
558 db = self.opendb(db_name, 'w')
560 for key in db.keys():
561 # get the journal for this db entry
562 journal = marshal.loads(db[key])
563 l = []
564 last_set_entry = None
565 for entry in journal:
566 # unpack the entry
567 (nodeid, date_stamp, self.journaltag, action,
568 params) = entry
569 # if the entry is after the pack date, _or_ the initial
570 # create entry, then it stays
571 if date_stamp > pack_before or action == 'create':
572 l.append(entry)
573 db[key] = marshal.dumps(l)
574 if db_type == 'gdbm':
575 db.reorganize()
576 db.close()
579 #
580 # Basic transaction support
581 #
582 def commit(self):
583 ''' Commit the current transactions.
584 '''
585 if __debug__:
586 print >>hyperdb.DEBUG, 'commit', (self,)
588 # keep a handle to all the database files opened
589 self.databases = {}
591 try:
592 # now, do all the transactions
593 reindex = {}
594 for method, args in self.transactions:
595 reindex[method(*args)] = 1
596 finally:
597 # make sure we close all the database files
598 for db in self.databases.values():
599 db.close()
600 del self.databases
602 # reindex the nodes that request it
603 for classname, nodeid in filter(None, reindex.keys()):
604 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
605 self.getclass(classname).index(nodeid)
607 # save the indexer state
608 self.indexer.save_index()
610 self.clearCache()
612 def clearCache(self):
613 # all transactions committed, back to normal
614 self.cache = {}
615 self.dirtynodes = {}
616 self.newnodes = {}
617 self.destroyednodes = {}
618 self.transactions = []
620 def getCachedClassDB(self, classname):
621 ''' get the class db, looking in our cache of databases for commit
622 '''
623 # get the database handle
624 db_name = 'nodes.%s'%classname
625 if not self.databases.has_key(db_name):
626 self.databases[db_name] = self.getclassdb(classname, 'c')
627 return self.databases[db_name]
629 def doSaveNode(self, classname, nodeid, node):
630 if __debug__:
631 print >>hyperdb.DEBUG, 'doSaveNode', (self, classname, nodeid,
632 node)
634 db = self.getCachedClassDB(classname)
636 # now save the marshalled data
637 db[nodeid] = marshal.dumps(self.serialise(classname, node))
639 # return the classname, nodeid so we reindex this content
640 return (classname, nodeid)
642 def getCachedJournalDB(self, classname):
643 ''' get the journal db, looking in our cache of databases for commit
644 '''
645 # get the database handle
646 db_name = 'journals.%s'%classname
647 if not self.databases.has_key(db_name):
648 self.databases[db_name] = self.opendb(db_name, 'c')
649 return self.databases[db_name]
651 def doSaveJournal(self, classname, nodeid, action, params, creator,
652 creation):
653 # serialise the parameters now if necessary
654 if isinstance(params, type({})):
655 if action in ('set', 'create'):
656 params = self.serialise(classname, params)
658 # handle supply of the special journalling parameters (usually
659 # supplied on importing an existing database)
660 if creator:
661 journaltag = creator
662 else:
663 journaltag = self.getuid()
664 if creation:
665 journaldate = creation.serialise()
666 else:
667 journaldate = date.Date().serialise()
669 # create the journal entry
670 entry = (nodeid, journaldate, journaltag, action, params)
672 if __debug__:
673 print >>hyperdb.DEBUG, 'doSaveJournal', entry
675 db = self.getCachedJournalDB(classname)
677 # now insert the journal entry
678 if db.has_key(nodeid):
679 # append to existing
680 s = db[nodeid]
681 l = marshal.loads(s)
682 l.append(entry)
683 else:
684 l = [entry]
686 db[nodeid] = marshal.dumps(l)
688 def doDestroyNode(self, classname, nodeid):
689 if __debug__:
690 print >>hyperdb.DEBUG, 'doDestroyNode', (self, classname, nodeid)
692 # delete from the class database
693 db = self.getCachedClassDB(classname)
694 if db.has_key(nodeid):
695 del db[nodeid]
697 # delete from the database
698 db = self.getCachedJournalDB(classname)
699 if db.has_key(nodeid):
700 del db[nodeid]
702 # return the classname, nodeid so we reindex this content
703 return (classname, nodeid)
705 def rollback(self):
706 ''' Reverse all actions from the current transaction.
707 '''
708 if __debug__:
709 print >>hyperdb.DEBUG, 'rollback', (self, )
710 for method, args in self.transactions:
711 # delete temporary files
712 if method == self.doStoreFile:
713 self.rollbackStoreFile(*args)
714 self.cache = {}
715 self.dirtynodes = {}
716 self.newnodes = {}
717 self.destroyednodes = {}
718 self.transactions = []
720 def close(self):
721 ''' Nothing to do
722 '''
723 if self.lockfile is not None:
724 locking.release_lock(self.lockfile)
725 if self.lockfile is not None:
726 self.lockfile.close()
727 self.lockfile = None
729 _marker = []
730 class Class(hyperdb.Class):
731 '''The handle to a particular class of nodes in a hyperdatabase.'''
733 def __init__(self, db, classname, **properties):
734 '''Create a new class with a given name and property specification.
736 'classname' must not collide with the name of an existing class,
737 or a ValueError is raised. The keyword arguments in 'properties'
738 must map names to property objects, or a TypeError is raised.
739 '''
740 for name in 'creation activity creator actor'.split():
741 if properties.has_key(name):
742 raise ValueError, '"creation", "activity", "creator" and '\
743 '"actor" are reserved'
745 self.classname = classname
746 self.properties = properties
747 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
748 self.key = ''
750 # should we journal changes (default yes)
751 self.do_journal = 1
753 # do the db-related init stuff
754 db.addclass(self)
756 self.auditors = {'create': [], 'set': [], 'retire': [], 'restore': []}
757 self.reactors = {'create': [], 'set': [], 'retire': [], 'restore': []}
759 def enableJournalling(self):
760 '''Turn journalling on for this class
761 '''
762 self.do_journal = 1
764 def disableJournalling(self):
765 '''Turn journalling off for this class
766 '''
767 self.do_journal = 0
769 # Editing nodes:
771 def create(self, **propvalues):
772 '''Create a new node of this class and return its id.
774 The keyword arguments in 'propvalues' map property names to values.
776 The values of arguments must be acceptable for the types of their
777 corresponding properties or a TypeError is raised.
779 If this class has a key property, it must be present and its value
780 must not collide with other key strings or a ValueError is raised.
782 Any other properties on this class that are missing from the
783 'propvalues' dictionary are set to None.
785 If an id in a link or multilink property does not refer to a valid
786 node, an IndexError is raised.
788 These operations trigger detectors and can be vetoed. Attempts
789 to modify the "creation" or "activity" properties cause a KeyError.
790 '''
791 self.fireAuditors('create', None, propvalues)
792 newid = self.create_inner(**propvalues)
793 self.fireReactors('create', newid, None)
794 return newid
796 def create_inner(self, **propvalues):
797 ''' Called by create, in-between the audit and react calls.
798 '''
799 if propvalues.has_key('id'):
800 raise KeyError, '"id" is reserved'
802 if self.db.journaltag is None:
803 raise DatabaseError, 'Database open read-only'
805 if propvalues.has_key('creation') or propvalues.has_key('activity'):
806 raise KeyError, '"creation" and "activity" are reserved'
807 # new node's id
808 newid = self.db.newid(self.classname)
810 # validate propvalues
811 num_re = re.compile('^\d+$')
812 for key, value in propvalues.items():
813 if key == self.key:
814 try:
815 self.lookup(value)
816 except KeyError:
817 pass
818 else:
819 raise ValueError, 'node with key "%s" exists'%value
821 # try to handle this property
822 try:
823 prop = self.properties[key]
824 except KeyError:
825 raise KeyError, '"%s" has no property "%s"'%(self.classname,
826 key)
828 if value is not None and isinstance(prop, Link):
829 if type(value) != type(''):
830 raise ValueError, 'link value must be String'
831 link_class = self.properties[key].classname
832 # if it isn't a number, it's a key
833 if not num_re.match(value):
834 try:
835 value = self.db.classes[link_class].lookup(value)
836 except (TypeError, KeyError):
837 raise IndexError, 'new property "%s": %s not a %s'%(
838 key, value, link_class)
839 elif not self.db.getclass(link_class).hasnode(value):
840 raise IndexError, '%s has no node %s'%(link_class, value)
842 # save off the value
843 propvalues[key] = value
845 # register the link with the newly linked node
846 if self.do_journal and self.properties[key].do_journal:
847 self.db.addjournal(link_class, value, 'link',
848 (self.classname, newid, key))
850 elif isinstance(prop, Multilink):
851 if type(value) != type([]):
852 raise TypeError, 'new property "%s" not a list of ids'%key
854 # clean up and validate the list of links
855 link_class = self.properties[key].classname
856 l = []
857 for entry in value:
858 if type(entry) != type(''):
859 raise ValueError, '"%s" multilink value (%r) '\
860 'must contain Strings'%(key, value)
861 # if it isn't a number, it's a key
862 if not num_re.match(entry):
863 try:
864 entry = self.db.classes[link_class].lookup(entry)
865 except (TypeError, KeyError):
866 raise IndexError, 'new property "%s": %s not a %s'%(
867 key, entry, self.properties[key].classname)
868 l.append(entry)
869 value = l
870 propvalues[key] = value
872 # handle additions
873 for nodeid in value:
874 if not self.db.getclass(link_class).hasnode(nodeid):
875 raise IndexError, '%s has no node %s'%(link_class,
876 nodeid)
877 # register the link with the newly linked node
878 if self.do_journal and self.properties[key].do_journal:
879 self.db.addjournal(link_class, nodeid, 'link',
880 (self.classname, newid, key))
882 elif isinstance(prop, String):
883 if type(value) != type('') and type(value) != type(u''):
884 raise TypeError, 'new property "%s" not a string'%key
886 elif isinstance(prop, Password):
887 if not isinstance(value, password.Password):
888 raise TypeError, 'new property "%s" not a Password'%key
890 elif isinstance(prop, Date):
891 if value is not None and not isinstance(value, date.Date):
892 raise TypeError, 'new property "%s" not a Date'%key
894 elif isinstance(prop, Interval):
895 if value is not None and not isinstance(value, date.Interval):
896 raise TypeError, 'new property "%s" not an Interval'%key
898 elif value is not None and isinstance(prop, Number):
899 try:
900 float(value)
901 except ValueError:
902 raise TypeError, 'new property "%s" not numeric'%key
904 elif value is not None and isinstance(prop, Boolean):
905 try:
906 int(value)
907 except ValueError:
908 raise TypeError, 'new property "%s" not boolean'%key
910 # make sure there's data where there needs to be
911 for key, prop in self.properties.items():
912 if propvalues.has_key(key):
913 continue
914 if key == self.key:
915 raise ValueError, 'key property "%s" is required'%key
916 if isinstance(prop, Multilink):
917 propvalues[key] = []
918 else:
919 propvalues[key] = None
921 # done
922 self.db.addnode(self.classname, newid, propvalues)
923 if self.do_journal:
924 self.db.addjournal(self.classname, newid, 'create', {})
926 return newid
928 def export_list(self, propnames, nodeid):
929 ''' Export a node - generate a list of CSV-able data in the order
930 specified by propnames for the given node.
931 '''
932 properties = self.getprops()
933 l = []
934 for prop in propnames:
935 proptype = properties[prop]
936 value = self.get(nodeid, prop)
937 # "marshal" data where needed
938 if value is None:
939 pass
940 elif isinstance(proptype, hyperdb.Date):
941 value = value.get_tuple()
942 elif isinstance(proptype, hyperdb.Interval):
943 value = value.get_tuple()
944 elif isinstance(proptype, hyperdb.Password):
945 value = str(value)
946 l.append(repr(value))
948 # append retired flag
949 l.append(repr(self.is_retired(nodeid)))
951 return l
953 def import_list(self, propnames, proplist):
954 ''' Import a node - all information including "id" is present and
955 should not be sanity checked. Triggers are not triggered. The
956 journal should be initialised using the "creator" and "created"
957 information.
959 Return the nodeid of the node imported.
960 '''
961 if self.db.journaltag is None:
962 raise DatabaseError, 'Database open read-only'
963 properties = self.getprops()
965 # make the new node's property map
966 d = {}
967 newid = None
968 for i in range(len(propnames)):
969 # Figure the property for this column
970 propname = propnames[i]
972 # Use eval to reverse the repr() used to output the CSV
973 value = eval(proplist[i])
975 # "unmarshal" where necessary
976 if propname == 'id':
977 newid = value
978 continue
979 elif propname == 'is retired':
980 # is the item retired?
981 if int(value):
982 d[self.db.RETIRED_FLAG] = 1
983 continue
984 elif value is None:
985 d[propname] = None
986 continue
988 prop = properties[propname]
989 if isinstance(prop, hyperdb.Date):
990 value = date.Date(value)
991 elif isinstance(prop, hyperdb.Interval):
992 value = date.Interval(value)
993 elif isinstance(prop, hyperdb.Password):
994 pwd = password.Password()
995 pwd.unpack(value)
996 value = pwd
997 d[propname] = value
999 # get a new id if necessary
1000 if newid is None:
1001 newid = self.db.newid(self.classname)
1003 # add the node and journal
1004 self.db.addnode(self.classname, newid, d)
1006 # extract the journalling stuff and nuke it
1007 if d.has_key('creator'):
1008 creator = d['creator']
1009 del d['creator']
1010 else:
1011 creator = None
1012 if d.has_key('creation'):
1013 creation = d['creation']
1014 del d['creation']
1015 else:
1016 creation = None
1017 if d.has_key('activity'):
1018 del d['activity']
1019 if d.has_key('actor'):
1020 del d['actor']
1021 self.db.addjournal(self.classname, newid, 'create', {}, creator,
1022 creation)
1023 return newid
1025 def get(self, nodeid, propname, default=_marker, cache=1):
1026 '''Get the value of a property on an existing node of this class.
1028 'nodeid' must be the id of an existing node of this class or an
1029 IndexError is raised. 'propname' must be the name of a property
1030 of this class or a KeyError is raised.
1032 'cache' exists for backward compatibility, and is not used.
1034 Attempts to get the "creation" or "activity" properties should
1035 do the right thing.
1036 '''
1037 if propname == 'id':
1038 return nodeid
1040 # get the node's dict
1041 d = self.db.getnode(self.classname, nodeid)
1043 # check for one of the special props
1044 if propname == 'creation':
1045 if d.has_key('creation'):
1046 return d['creation']
1047 if not self.do_journal:
1048 raise ValueError, 'Journalling is disabled for this class'
1049 journal = self.db.getjournal(self.classname, nodeid)
1050 if journal:
1051 return self.db.getjournal(self.classname, nodeid)[0][1]
1052 else:
1053 # on the strange chance that there's no journal
1054 return date.Date()
1055 if propname == 'activity':
1056 if d.has_key('activity'):
1057 return d['activity']
1058 if not self.do_journal:
1059 raise ValueError, 'Journalling is disabled for this class'
1060 journal = self.db.getjournal(self.classname, nodeid)
1061 if journal:
1062 return self.db.getjournal(self.classname, nodeid)[-1][1]
1063 else:
1064 # on the strange chance that there's no journal
1065 return date.Date()
1066 if propname == 'creator':
1067 if d.has_key('creator'):
1068 return d['creator']
1069 if not self.do_journal:
1070 raise ValueError, 'Journalling is disabled for this class'
1071 journal = self.db.getjournal(self.classname, nodeid)
1072 if journal:
1073 num_re = re.compile('^\d+$')
1074 value = journal[0][2]
1075 if num_re.match(value):
1076 return value
1077 else:
1078 # old-style "username" journal tag
1079 try:
1080 return self.db.user.lookup(value)
1081 except KeyError:
1082 # user's been retired, return admin
1083 return '1'
1084 else:
1085 return self.db.getuid()
1086 if propname == 'actor':
1087 if d.has_key('actor'):
1088 return d['actor']
1089 if not self.do_journal:
1090 raise ValueError, 'Journalling is disabled for this class'
1091 journal = self.db.getjournal(self.classname, nodeid)
1092 if journal:
1093 num_re = re.compile('^\d+$')
1094 value = journal[-1][2]
1095 if num_re.match(value):
1096 return value
1097 else:
1098 # old-style "username" journal tag
1099 try:
1100 return self.db.user.lookup(value)
1101 except KeyError:
1102 # user's been retired, return admin
1103 return '1'
1104 else:
1105 return self.db.getuid()
1107 # get the property (raises KeyErorr if invalid)
1108 prop = self.properties[propname]
1110 if not d.has_key(propname):
1111 if default is _marker:
1112 if isinstance(prop, Multilink):
1113 return []
1114 else:
1115 return None
1116 else:
1117 return default
1119 # return a dupe of the list so code doesn't get confused
1120 if isinstance(prop, Multilink):
1121 return d[propname][:]
1123 return d[propname]
1125 def set(self, nodeid, **propvalues):
1126 '''Modify a property on an existing node of this class.
1128 'nodeid' must be the id of an existing node of this class or an
1129 IndexError is raised.
1131 Each key in 'propvalues' must be the name of a property of this
1132 class or a KeyError is raised.
1134 All values in 'propvalues' must be acceptable types for their
1135 corresponding properties or a TypeError is raised.
1137 If the value of the key property is set, it must not collide with
1138 other key strings or a ValueError is raised.
1140 If the value of a Link or Multilink property contains an invalid
1141 node id, a ValueError is raised.
1143 These operations trigger detectors and can be vetoed. Attempts
1144 to modify the "creation" or "activity" properties cause a KeyError.
1145 '''
1146 if not propvalues:
1147 return propvalues
1149 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1150 raise KeyError, '"creation" and "activity" are reserved'
1152 if propvalues.has_key('id'):
1153 raise KeyError, '"id" is reserved'
1155 if self.db.journaltag is None:
1156 raise DatabaseError, 'Database open read-only'
1158 self.fireAuditors('set', nodeid, propvalues)
1159 # Take a copy of the node dict so that the subsequent set
1160 # operation doesn't modify the oldvalues structure.
1161 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1163 node = self.db.getnode(self.classname, nodeid)
1164 if node.has_key(self.db.RETIRED_FLAG):
1165 raise IndexError
1166 num_re = re.compile('^\d+$')
1168 # if the journal value is to be different, store it in here
1169 journalvalues = {}
1171 for propname, value in propvalues.items():
1172 # check to make sure we're not duplicating an existing key
1173 if propname == self.key and node[propname] != value:
1174 try:
1175 self.lookup(value)
1176 except KeyError:
1177 pass
1178 else:
1179 raise ValueError, 'node with key "%s" exists'%value
1181 # this will raise the KeyError if the property isn't valid
1182 # ... we don't use getprops() here because we only care about
1183 # the writeable properties.
1184 try:
1185 prop = self.properties[propname]
1186 except KeyError:
1187 raise KeyError, '"%s" has no property named "%s"'%(
1188 self.classname, propname)
1190 # if the value's the same as the existing value, no sense in
1191 # doing anything
1192 current = node.get(propname, None)
1193 if value == current:
1194 del propvalues[propname]
1195 continue
1196 journalvalues[propname] = current
1198 # do stuff based on the prop type
1199 if isinstance(prop, Link):
1200 link_class = prop.classname
1201 # if it isn't a number, it's a key
1202 if value is not None and not isinstance(value, type('')):
1203 raise ValueError, 'property "%s" link value be a string'%(
1204 propname)
1205 if isinstance(value, type('')) and not num_re.match(value):
1206 try:
1207 value = self.db.classes[link_class].lookup(value)
1208 except (TypeError, KeyError):
1209 raise IndexError, 'new property "%s": %s not a %s'%(
1210 propname, value, prop.classname)
1212 if (value is not None and
1213 not self.db.getclass(link_class).hasnode(value)):
1214 raise IndexError, '%s has no node %s'%(link_class, value)
1216 if self.do_journal and prop.do_journal:
1217 # register the unlink with the old linked node
1218 if node.has_key(propname) and node[propname] is not None:
1219 self.db.addjournal(link_class, node[propname], 'unlink',
1220 (self.classname, nodeid, propname))
1222 # register the link with the newly linked node
1223 if value is not None:
1224 self.db.addjournal(link_class, value, 'link',
1225 (self.classname, nodeid, propname))
1227 elif isinstance(prop, Multilink):
1228 if type(value) != type([]):
1229 raise TypeError, 'new property "%s" not a list of'\
1230 ' ids'%propname
1231 link_class = self.properties[propname].classname
1232 l = []
1233 for entry in value:
1234 # if it isn't a number, it's a key
1235 if type(entry) != type(''):
1236 raise ValueError, 'new property "%s" link value ' \
1237 'must be a string'%propname
1238 if not num_re.match(entry):
1239 try:
1240 entry = self.db.classes[link_class].lookup(entry)
1241 except (TypeError, KeyError):
1242 raise IndexError, 'new property "%s": %s not a %s'%(
1243 propname, entry,
1244 self.properties[propname].classname)
1245 l.append(entry)
1246 value = l
1247 propvalues[propname] = value
1249 # figure the journal entry for this property
1250 add = []
1251 remove = []
1253 # handle removals
1254 if node.has_key(propname):
1255 l = node[propname]
1256 else:
1257 l = []
1258 for id in l[:]:
1259 if id in value:
1260 continue
1261 # register the unlink with the old linked node
1262 if self.do_journal and self.properties[propname].do_journal:
1263 self.db.addjournal(link_class, id, 'unlink',
1264 (self.classname, nodeid, propname))
1265 l.remove(id)
1266 remove.append(id)
1268 # handle additions
1269 for id in value:
1270 if not self.db.getclass(link_class).hasnode(id):
1271 raise IndexError, '%s has no node %s'%(link_class, id)
1272 if id in l:
1273 continue
1274 # register the link with the newly linked node
1275 if self.do_journal and self.properties[propname].do_journal:
1276 self.db.addjournal(link_class, id, 'link',
1277 (self.classname, nodeid, propname))
1278 l.append(id)
1279 add.append(id)
1281 # figure the journal entry
1282 l = []
1283 if add:
1284 l.append(('+', add))
1285 if remove:
1286 l.append(('-', remove))
1287 if l:
1288 journalvalues[propname] = tuple(l)
1290 elif isinstance(prop, String):
1291 if value is not None and type(value) != type('') and type(value) != type(u''):
1292 raise TypeError, 'new property "%s" not a string'%propname
1294 elif isinstance(prop, Password):
1295 if not isinstance(value, password.Password):
1296 raise TypeError, 'new property "%s" not a Password'%propname
1297 propvalues[propname] = value
1299 elif value is not None and isinstance(prop, Date):
1300 if not isinstance(value, date.Date):
1301 raise TypeError, 'new property "%s" not a Date'% propname
1302 propvalues[propname] = value
1304 elif value is not None and isinstance(prop, Interval):
1305 if not isinstance(value, date.Interval):
1306 raise TypeError, 'new property "%s" not an '\
1307 'Interval'%propname
1308 propvalues[propname] = value
1310 elif value is not None and isinstance(prop, Number):
1311 try:
1312 float(value)
1313 except ValueError:
1314 raise TypeError, 'new property "%s" not numeric'%propname
1316 elif value is not None and isinstance(prop, Boolean):
1317 try:
1318 int(value)
1319 except ValueError:
1320 raise TypeError, 'new property "%s" not boolean'%propname
1322 node[propname] = value
1324 # nothing to do?
1325 if not propvalues:
1326 return propvalues
1328 # do the set, and journal it
1329 self.db.setnode(self.classname, nodeid, node)
1331 if self.do_journal:
1332 self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
1334 self.fireReactors('set', nodeid, oldvalues)
1336 return propvalues
1338 def retire(self, nodeid):
1339 '''Retire a node.
1341 The properties on the node remain available from the get() method,
1342 and the node's id is never reused.
1344 Retired nodes are not returned by the find(), list(), or lookup()
1345 methods, and other nodes may reuse the values of their key properties.
1347 These operations trigger detectors and can be vetoed. Attempts
1348 to modify the "creation" or "activity" properties cause a KeyError.
1349 '''
1350 if self.db.journaltag is None:
1351 raise DatabaseError, 'Database open read-only'
1353 self.fireAuditors('retire', nodeid, None)
1355 node = self.db.getnode(self.classname, nodeid)
1356 node[self.db.RETIRED_FLAG] = 1
1357 self.db.setnode(self.classname, nodeid, node)
1358 if self.do_journal:
1359 self.db.addjournal(self.classname, nodeid, 'retired', None)
1361 self.fireReactors('retire', nodeid, None)
1363 def restore(self, nodeid):
1364 '''Restpre a retired node.
1366 Make node available for all operations like it was before retirement.
1367 '''
1368 if self.db.journaltag is None:
1369 raise DatabaseError, 'Database open read-only'
1371 node = self.db.getnode(self.classname, nodeid)
1372 # check if key property was overrided
1373 key = self.getkey()
1374 try:
1375 id = self.lookup(node[key])
1376 except KeyError:
1377 pass
1378 else:
1379 raise KeyError, "Key property (%s) of retired node clashes with \
1380 existing one (%s)" % (key, node[key])
1381 # Now we can safely restore node
1382 self.fireAuditors('restore', nodeid, None)
1383 del node[self.db.RETIRED_FLAG]
1384 self.db.setnode(self.classname, nodeid, node)
1385 if self.do_journal:
1386 self.db.addjournal(self.classname, nodeid, 'restored', None)
1388 self.fireReactors('restore', nodeid, None)
1390 def is_retired(self, nodeid, cldb=None):
1391 '''Return true if the node is retired.
1392 '''
1393 node = self.db.getnode(self.classname, nodeid, cldb)
1394 if node.has_key(self.db.RETIRED_FLAG):
1395 return 1
1396 return 0
1398 def destroy(self, nodeid):
1399 '''Destroy a node.
1401 WARNING: this method should never be used except in extremely rare
1402 situations where there could never be links to the node being
1403 deleted
1405 WARNING: use retire() instead
1407 WARNING: the properties of this node will not be available ever again
1409 WARNING: really, use retire() instead
1411 Well, I think that's enough warnings. This method exists mostly to
1412 support the session storage of the cgi interface.
1413 '''
1414 if self.db.journaltag is None:
1415 raise DatabaseError, 'Database open read-only'
1416 self.db.destroynode(self.classname, nodeid)
1418 def history(self, nodeid):
1419 '''Retrieve the journal of edits on a particular node.
1421 'nodeid' must be the id of an existing node of this class or an
1422 IndexError is raised.
1424 The returned list contains tuples of the form
1426 (nodeid, date, tag, action, params)
1428 'date' is a Timestamp object specifying the time of the change and
1429 'tag' is the journaltag specified when the database was opened.
1430 '''
1431 if not self.do_journal:
1432 raise ValueError, 'Journalling is disabled for this class'
1433 return self.db.getjournal(self.classname, nodeid)
1435 # Locating nodes:
1436 def hasnode(self, nodeid):
1437 '''Determine if the given nodeid actually exists
1438 '''
1439 return self.db.hasnode(self.classname, nodeid)
1441 def setkey(self, propname):
1442 '''Select a String property of this class to be the key property.
1444 'propname' must be the name of a String property of this class or
1445 None, or a TypeError is raised. The values of the key property on
1446 all existing nodes must be unique or a ValueError is raised. If the
1447 property doesn't exist, KeyError is raised.
1448 '''
1449 prop = self.getprops()[propname]
1450 if not isinstance(prop, String):
1451 raise TypeError, 'key properties must be String'
1452 self.key = propname
1454 def getkey(self):
1455 '''Return the name of the key property for this class or None.'''
1456 return self.key
1458 def labelprop(self, default_to_id=0):
1459 '''Return the property name for a label for the given node.
1461 This method attempts to generate a consistent label for the node.
1462 It tries the following in order:
1464 1. key property
1465 2. "name" property
1466 3. "title" property
1467 4. first property from the sorted property name list
1468 '''
1469 k = self.getkey()
1470 if k:
1471 return k
1472 props = self.getprops()
1473 if props.has_key('name'):
1474 return 'name'
1475 elif props.has_key('title'):
1476 return 'title'
1477 if default_to_id:
1478 return 'id'
1479 props = props.keys()
1480 props.sort()
1481 return props[0]
1483 # TODO: set up a separate index db file for this? profile?
1484 def lookup(self, keyvalue):
1485 '''Locate a particular node by its key property and return its id.
1487 If this class has no key property, a TypeError is raised. If the
1488 'keyvalue' matches one of the values for the key property among
1489 the nodes in this class, the matching node's id is returned;
1490 otherwise a KeyError is raised.
1491 '''
1492 if not self.key:
1493 raise TypeError, 'No key property set for class %s'%self.classname
1494 cldb = self.db.getclassdb(self.classname)
1495 try:
1496 for nodeid in self.getnodeids(cldb):
1497 node = self.db.getnode(self.classname, nodeid, cldb)
1498 if node.has_key(self.db.RETIRED_FLAG):
1499 continue
1500 if node[self.key] == keyvalue:
1501 return nodeid
1502 finally:
1503 cldb.close()
1504 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1505 keyvalue, self.classname)
1507 # change from spec - allows multiple props to match
1508 def find(self, **propspec):
1509 '''Get the ids of items in this class which link to the given items.
1511 'propspec' consists of keyword args propname=itemid or
1512 propname={itemid:1, }
1513 'propname' must be the name of a property in this class, or a
1514 KeyError is raised. That property must be a Link or
1515 Multilink property, or a TypeError is raised.
1517 Any item in this class whose 'propname' property links to any of the
1518 itemids will be returned. Used by the full text indexing, which knows
1519 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1520 issues:
1522 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1523 '''
1524 propspec = propspec.items()
1525 for propname, itemids in propspec:
1526 # check the prop is OK
1527 prop = self.properties[propname]
1528 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1529 raise TypeError, "'%s' not a Link/Multilink property"%propname
1531 # ok, now do the find
1532 cldb = self.db.getclassdb(self.classname)
1533 l = []
1534 try:
1535 for id in self.getnodeids(db=cldb):
1536 item = self.db.getnode(self.classname, id, db=cldb)
1537 if item.has_key(self.db.RETIRED_FLAG):
1538 continue
1539 for propname, itemids in propspec:
1540 # can't test if the item doesn't have this property
1541 if not item.has_key(propname):
1542 continue
1543 if type(itemids) is not type({}):
1544 itemids = {itemids:1}
1546 # grab the property definition and its value on this item
1547 prop = self.properties[propname]
1548 value = item[propname]
1549 if isinstance(prop, Link) and itemids.has_key(value):
1550 l.append(id)
1551 break
1552 elif isinstance(prop, Multilink):
1553 hit = 0
1554 for v in value:
1555 if itemids.has_key(v):
1556 l.append(id)
1557 hit = 1
1558 break
1559 if hit:
1560 break
1561 finally:
1562 cldb.close()
1563 return l
1565 def stringFind(self, **requirements):
1566 '''Locate a particular node by matching a set of its String
1567 properties in a caseless search.
1569 If the property is not a String property, a TypeError is raised.
1571 The return is a list of the id of all nodes that match.
1572 '''
1573 for propname in requirements.keys():
1574 prop = self.properties[propname]
1575 if not isinstance(prop, String):
1576 raise TypeError, "'%s' not a String property"%propname
1577 requirements[propname] = requirements[propname].lower()
1578 l = []
1579 cldb = self.db.getclassdb(self.classname)
1580 try:
1581 for nodeid in self.getnodeids(cldb):
1582 node = self.db.getnode(self.classname, nodeid, cldb)
1583 if node.has_key(self.db.RETIRED_FLAG):
1584 continue
1585 for key, value in requirements.items():
1586 if not node.has_key(key):
1587 break
1588 if node[key] is None or node[key].lower() != value:
1589 break
1590 else:
1591 l.append(nodeid)
1592 finally:
1593 cldb.close()
1594 return l
1596 def list(self):
1597 ''' Return a list of the ids of the active nodes in this class.
1598 '''
1599 l = []
1600 cn = self.classname
1601 cldb = self.db.getclassdb(cn)
1602 try:
1603 for nodeid in self.getnodeids(cldb):
1604 node = self.db.getnode(cn, nodeid, cldb)
1605 if node.has_key(self.db.RETIRED_FLAG):
1606 continue
1607 l.append(nodeid)
1608 finally:
1609 cldb.close()
1610 l.sort()
1611 return l
1613 def getnodeids(self, db=None):
1614 ''' Return a list of ALL nodeids
1615 '''
1616 if __debug__:
1617 print >>hyperdb.DEBUG, 'getnodeids', (self, self.classname, db)
1619 res = []
1621 # start off with the new nodes
1622 if self.db.newnodes.has_key(self.classname):
1623 res += self.db.newnodes[self.classname].keys()
1625 if db is None:
1626 db = self.db.getclassdb(self.classname)
1627 res = res + db.keys()
1629 # remove the uncommitted, destroyed nodes
1630 if self.db.destroyednodes.has_key(self.classname):
1631 for nodeid in self.db.destroyednodes[self.classname].keys():
1632 if db.has_key(nodeid):
1633 res.remove(nodeid)
1635 return res
1637 def filter(self, search_matches, filterspec, sort=(None,None),
1638 group=(None,None), num_re = re.compile('^\d+$')):
1639 """Return a list of the ids of the active nodes in this class that
1640 match the 'filter' spec, sorted by the group spec and then the
1641 sort spec.
1643 "filterspec" is {propname: value(s)}
1645 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1646 and prop is a prop name or None
1648 "search_matches" is {nodeid: marker}
1650 The filter must match all properties specificed - but if the
1651 property value to match is a list, any one of the values in the
1652 list may match for that property to match. Unless the property
1653 is a Multilink, in which case the item's property list must
1654 match the filterspec list.
1655 """
1656 cn = self.classname
1658 # optimise filterspec
1659 l = []
1660 props = self.getprops()
1661 LINK = 0
1662 MULTILINK = 1
1663 STRING = 2
1664 DATE = 3
1665 INTERVAL = 4
1666 OTHER = 6
1668 timezone = self.db.getUserTimezone()
1669 for k, v in filterspec.items():
1670 propclass = props[k]
1671 if isinstance(propclass, Link):
1672 if type(v) is not type([]):
1673 v = [v]
1674 u = []
1675 for entry in v:
1676 # the value -1 is a special "not set" sentinel
1677 if entry == '-1':
1678 entry = None
1679 u.append(entry)
1680 l.append((LINK, k, u))
1681 elif isinstance(propclass, Multilink):
1682 # the value -1 is a special "not set" sentinel
1683 if v in ('-1', ['-1']):
1684 v = []
1685 elif type(v) is not type([]):
1686 v = [v]
1687 l.append((MULTILINK, k, v))
1688 elif isinstance(propclass, String) and k != 'id':
1689 if type(v) is not type([]):
1690 v = [v]
1691 m = []
1692 for v in v:
1693 # simple glob searching
1694 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1695 v = v.replace('?', '.')
1696 v = v.replace('*', '.*?')
1697 m.append(v)
1698 m = re.compile('(%s)'%('|'.join(m)), re.I)
1699 l.append((STRING, k, m))
1700 elif isinstance(propclass, Date):
1701 try:
1702 date_rng = Range(v, date.Date, offset=timezone)
1703 l.append((DATE, k, date_rng))
1704 except ValueError:
1705 # If range creation fails - ignore that search parameter
1706 pass
1707 elif isinstance(propclass, Interval):
1708 try:
1709 intv_rng = Range(v, date.Interval)
1710 l.append((INTERVAL, k, intv_rng))
1711 except ValueError:
1712 # If range creation fails - ignore that search parameter
1713 pass
1715 elif isinstance(propclass, Boolean):
1716 if type(v) is type(''):
1717 bv = v.lower() in ('yes', 'true', 'on', '1')
1718 else:
1719 bv = v
1720 l.append((OTHER, k, bv))
1721 elif isinstance(propclass, Number):
1722 l.append((OTHER, k, int(v)))
1723 else:
1724 l.append((OTHER, k, v))
1725 filterspec = l
1727 # now, find all the nodes that are active and pass filtering
1728 l = []
1729 cldb = self.db.getclassdb(cn)
1730 try:
1731 # TODO: only full-scan once (use items())
1732 for nodeid in self.getnodeids(cldb):
1733 node = self.db.getnode(cn, nodeid, cldb)
1734 if node.has_key(self.db.RETIRED_FLAG):
1735 continue
1736 # apply filter
1737 for t, k, v in filterspec:
1738 # handle the id prop
1739 if k == 'id' and v == nodeid:
1740 continue
1742 # make sure the node has the property
1743 if not node.has_key(k):
1744 # this node doesn't have this property, so reject it
1745 break
1747 # now apply the property filter
1748 if t == LINK:
1749 # link - if this node's property doesn't appear in the
1750 # filterspec's nodeid list, skip it
1751 if node[k] not in v:
1752 break
1753 elif t == MULTILINK:
1754 # multilink - if any of the nodeids required by the
1755 # filterspec aren't in this node's property, then skip
1756 # it
1757 have = node[k]
1758 # check for matching the absence of multilink values
1759 if not v and have:
1760 break
1762 # othewise, make sure this node has each of the
1763 # required values
1764 for want in v:
1765 if want not in have:
1766 break
1767 else:
1768 continue
1769 break
1770 elif t == STRING:
1771 if node[k] is None:
1772 break
1773 # RE search
1774 if not v.search(node[k]):
1775 break
1776 elif t == DATE or t == INTERVAL:
1777 if node[k] is None:
1778 break
1779 if v.to_value:
1780 if not (v.from_value <= node[k] and v.to_value >= node[k]):
1781 break
1782 else:
1783 if not (v.from_value <= node[k]):
1784 break
1785 elif t == OTHER:
1786 # straight value comparison for the other types
1787 if node[k] != v:
1788 break
1789 else:
1790 l.append((nodeid, node))
1791 finally:
1792 cldb.close()
1793 l.sort()
1795 # filter based on full text search
1796 if search_matches is not None:
1797 k = []
1798 for v in l:
1799 if search_matches.has_key(v[0]):
1800 k.append(v)
1801 l = k
1803 # now, sort the result
1804 def sortfun(a, b, sort=sort, group=group, properties=self.getprops(),
1805 db = self.db, cl=self):
1806 a_id, an = a
1807 b_id, bn = b
1808 # sort by group and then sort
1809 for dir, prop in group, sort:
1810 if dir is None or prop is None: continue
1812 # sorting is class-specific
1813 propclass = properties[prop]
1815 # handle the properties that might be "faked"
1816 # also, handle possible missing properties
1817 try:
1818 if not an.has_key(prop):
1819 an[prop] = cl.get(a_id, prop)
1820 av = an[prop]
1821 except KeyError:
1822 # the node doesn't have a value for this property
1823 if isinstance(propclass, Multilink): av = []
1824 else: av = ''
1825 try:
1826 if not bn.has_key(prop):
1827 bn[prop] = cl.get(b_id, prop)
1828 bv = bn[prop]
1829 except KeyError:
1830 # the node doesn't have a value for this property
1831 if isinstance(propclass, Multilink): bv = []
1832 else: bv = ''
1834 # String and Date values are sorted in the natural way
1835 if isinstance(propclass, String):
1836 # clean up the strings
1837 if av and av[0] in string.uppercase:
1838 av = av.lower()
1839 if bv and bv[0] in string.uppercase:
1840 bv = bv.lower()
1841 if (isinstance(propclass, String) or
1842 isinstance(propclass, Date)):
1843 # it might be a string that's really an integer
1844 try:
1845 av = int(av)
1846 bv = int(bv)
1847 except:
1848 pass
1849 if dir == '+':
1850 r = cmp(av, bv)
1851 if r != 0: return r
1852 elif dir == '-':
1853 r = cmp(bv, av)
1854 if r != 0: return r
1856 # Link properties are sorted according to the value of
1857 # the "order" property on the linked nodes if it is
1858 # present; or otherwise on the key string of the linked
1859 # nodes; or finally on the node ids.
1860 elif isinstance(propclass, Link):
1861 link = db.classes[propclass.classname]
1862 if av is None and bv is not None: return -1
1863 if av is not None and bv is None: return 1
1864 if av is None and bv is None: continue
1865 if link.getprops().has_key('order'):
1866 if dir == '+':
1867 r = cmp(link.get(av, 'order'),
1868 link.get(bv, 'order'))
1869 if r != 0: return r
1870 elif dir == '-':
1871 r = cmp(link.get(bv, 'order'),
1872 link.get(av, 'order'))
1873 if r != 0: return r
1874 elif link.getkey():
1875 key = link.getkey()
1876 if dir == '+':
1877 r = cmp(link.get(av, key), link.get(bv, key))
1878 if r != 0: return r
1879 elif dir == '-':
1880 r = cmp(link.get(bv, key), link.get(av, key))
1881 if r != 0: return r
1882 else:
1883 if dir == '+':
1884 r = cmp(av, bv)
1885 if r != 0: return r
1886 elif dir == '-':
1887 r = cmp(bv, av)
1888 if r != 0: return r
1890 else:
1891 # all other types just compare
1892 if dir == '+':
1893 r = cmp(av, bv)
1894 elif dir == '-':
1895 r = cmp(bv, av)
1896 if r != 0: return r
1898 # end for dir, prop in sort, group:
1899 # if all else fails, compare the ids
1900 return cmp(a[0], b[0])
1902 l.sort(sortfun)
1903 return [i[0] for i in l]
1905 def count(self):
1906 '''Get the number of nodes in this class.
1908 If the returned integer is 'numnodes', the ids of all the nodes
1909 in this class run from 1 to numnodes, and numnodes+1 will be the
1910 id of the next node to be created in this class.
1911 '''
1912 return self.db.countnodes(self.classname)
1914 # Manipulating properties:
1916 def getprops(self, protected=1):
1917 '''Return a dictionary mapping property names to property objects.
1918 If the "protected" flag is true, we include protected properties -
1919 those which may not be modified.
1921 In addition to the actual properties on the node, these
1922 methods provide the "creation" and "activity" properties. If the
1923 "protected" flag is true, we include protected properties - those
1924 which may not be modified.
1925 '''
1926 d = self.properties.copy()
1927 if protected:
1928 d['id'] = String()
1929 d['creation'] = hyperdb.Date()
1930 d['activity'] = hyperdb.Date()
1931 d['creator'] = hyperdb.Link('user')
1932 d['actor'] = hyperdb.Link('user')
1933 return d
1935 def addprop(self, **properties):
1936 '''Add properties to this class.
1938 The keyword arguments in 'properties' must map names to property
1939 objects, or a TypeError is raised. None of the keys in 'properties'
1940 may collide with the names of existing properties, or a ValueError
1941 is raised before any properties have been added.
1942 '''
1943 for key in properties.keys():
1944 if self.properties.has_key(key):
1945 raise ValueError, key
1946 self.properties.update(properties)
1948 def index(self, nodeid):
1949 '''Add (or refresh) the node to search indexes
1950 '''
1951 # find all the String properties that have indexme
1952 for prop, propclass in self.getprops().items():
1953 if isinstance(propclass, String) and propclass.indexme:
1954 try:
1955 value = str(self.get(nodeid, prop))
1956 except IndexError:
1957 # node no longer exists - entry should be removed
1958 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1959 else:
1960 # and index them under (classname, nodeid, property)
1961 self.db.indexer.add_text((self.classname, nodeid, prop),
1962 value)
1964 #
1965 # Detector interface
1966 #
1967 def audit(self, event, detector):
1968 '''Register a detector
1969 '''
1970 l = self.auditors[event]
1971 if detector not in l:
1972 self.auditors[event].append(detector)
1974 def fireAuditors(self, action, nodeid, newvalues):
1975 '''Fire all registered auditors.
1976 '''
1977 for audit in self.auditors[action]:
1978 audit(self.db, self, nodeid, newvalues)
1980 def react(self, event, detector):
1981 '''Register a detector
1982 '''
1983 l = self.reactors[event]
1984 if detector not in l:
1985 self.reactors[event].append(detector)
1987 def fireReactors(self, action, nodeid, oldvalues):
1988 '''Fire all registered reactors.
1989 '''
1990 for react in self.reactors[action]:
1991 react(self.db, self, nodeid, oldvalues)
1993 class FileClass(Class, hyperdb.FileClass):
1994 '''This class defines a large chunk of data. To support this, it has a
1995 mandatory String property "content" which is typically saved off
1996 externally to the hyperdb.
1998 The default MIME type of this data is defined by the
1999 "default_mime_type" class attribute, which may be overridden by each
2000 node if the class defines a "type" String property.
2001 '''
2002 default_mime_type = 'text/plain'
2004 def create(self, **propvalues):
2005 ''' Snarf the "content" propvalue and store in a file
2006 '''
2007 # we need to fire the auditors now, or the content property won't
2008 # be in propvalues for the auditors to play with
2009 self.fireAuditors('create', None, propvalues)
2011 # now remove the content property so it's not stored in the db
2012 content = propvalues['content']
2013 del propvalues['content']
2015 # do the database create
2016 newid = Class.create_inner(self, **propvalues)
2018 # fire reactors
2019 self.fireReactors('create', newid, None)
2021 # store off the content as a file
2022 self.db.storefile(self.classname, newid, None, content)
2023 return newid
2025 def import_list(self, propnames, proplist):
2026 ''' Trap the "content" property...
2027 '''
2028 # dupe this list so we don't affect others
2029 propnames = propnames[:]
2031 # extract the "content" property from the proplist
2032 i = propnames.index('content')
2033 content = eval(proplist[i])
2034 del propnames[i]
2035 del proplist[i]
2037 # do the normal import
2038 newid = Class.import_list(self, propnames, proplist)
2040 # save off the "content" file
2041 self.db.storefile(self.classname, newid, None, content)
2042 return newid
2044 def get(self, nodeid, propname, default=_marker, cache=1):
2045 ''' Trap the content propname and get it from the file
2047 'cache' exists for backwards compatibility, and is not used.
2048 '''
2049 poss_msg = 'Possibly an access right configuration problem.'
2050 if propname == 'content':
2051 try:
2052 return self.db.getfile(self.classname, nodeid, None)
2053 except IOError, (strerror):
2054 # XXX by catching this we donot see an error in the log.
2055 return 'ERROR reading file: %s%s\n%s\n%s'%(
2056 self.classname, nodeid, poss_msg, strerror)
2057 if default is not _marker:
2058 return Class.get(self, nodeid, propname, default)
2059 else:
2060 return Class.get(self, nodeid, propname)
2062 def getprops(self, protected=1):
2063 ''' In addition to the actual properties on the node, these methods
2064 provide the "content" property. If the "protected" flag is true,
2065 we include protected properties - those which may not be
2066 modified.
2067 '''
2068 d = Class.getprops(self, protected=protected).copy()
2069 d['content'] = hyperdb.String()
2070 return d
2072 def index(self, nodeid):
2073 ''' Index the node in the search index.
2075 We want to index the content in addition to the normal String
2076 property indexing.
2077 '''
2078 # perform normal indexing
2079 Class.index(self, nodeid)
2081 # get the content to index
2082 content = self.get(nodeid, 'content')
2084 # figure the mime type
2085 if self.properties.has_key('type'):
2086 mime_type = self.get(nodeid, 'type')
2087 else:
2088 mime_type = self.default_mime_type
2090 # and index!
2091 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
2092 mime_type)
2094 # deviation from spec - was called ItemClass
2095 class IssueClass(Class, roundupdb.IssueClass):
2096 # Overridden methods:
2097 def __init__(self, db, classname, **properties):
2098 '''The newly-created class automatically includes the "messages",
2099 "files", "nosy", and "superseder" properties. If the 'properties'
2100 dictionary attempts to specify any of these properties or a
2101 "creation" or "activity" property, a ValueError is raised.
2102 '''
2103 if not properties.has_key('title'):
2104 properties['title'] = hyperdb.String(indexme='yes')
2105 if not properties.has_key('messages'):
2106 properties['messages'] = hyperdb.Multilink("msg")
2107 if not properties.has_key('files'):
2108 properties['files'] = hyperdb.Multilink("file")
2109 if not properties.has_key('nosy'):
2110 # note: journalling is turned off as it really just wastes
2111 # space. this behaviour may be overridden in an instance
2112 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2113 if not properties.has_key('superseder'):
2114 properties['superseder'] = hyperdb.Multilink(classname)
2115 Class.__init__(self, db, classname, **properties)
2117 #