1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: back_anydbm.py,v 1.133 2003-12-05 09:47:46 richard Exp $
19 '''
20 This module defines a backend that saves the hyperdatabase in a database
21 chosen by anydbm. It is guaranteed to always be available in python
22 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
23 serious bugs, and is not available)
24 '''
26 try:
27 import anydbm, sys
28 # dumbdbm only works in python 2.1.2+
29 if sys.version_info < (2,1,2):
30 import dumbdbm
31 assert anydbm._defaultmod != dumbdbm
32 del dumbdbm
33 except AssertionError:
34 print "WARNING: you should upgrade to python 2.1.3"
36 import whichdb, os, marshal, re, weakref, string, copy
37 from roundup import hyperdb, date, password, roundupdb, security
38 from blobfiles import FileStorage
39 from sessions import Sessions, OneTimeKeys
40 from roundup.indexer import Indexer
41 from roundup.backends import locking
42 from roundup.hyperdb import String, Password, Date, Interval, Link, \
43 Multilink, DatabaseError, Boolean, Number, Node
44 from roundup.date import Range
46 #
47 # Now the database
48 #
49 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
50 '''A database for storing records containing flexible data types.
52 Transaction stuff TODO:
53 . check the timestamp of the class file and nuke the cache if it's
54 modified. Do some sort of conflict checking on the dirty stuff.
55 . perhaps detect write collisions (related to above)?
57 '''
58 def __init__(self, config, journaltag=None):
59 '''Open a hyperdatabase given a specifier to some storage.
61 The 'storagelocator' is obtained from config.DATABASE.
62 The meaning of 'storagelocator' depends on the particular
63 implementation of the hyperdatabase. It could be a file name,
64 a directory path, a socket descriptor for a connection to a
65 database over the network, etc.
67 The 'journaltag' is a token that will be attached to the journal
68 entries for any edits done on the database. If 'journaltag' is
69 None, the database is opened in read-only mode: the Class.create(),
70 Class.set(), Class.retire(), and Class.restore() methods are
71 disabled.
72 '''
73 self.config, self.journaltag = config, journaltag
74 self.dir = config.DATABASE
75 self.classes = {}
76 self.cache = {} # cache of nodes loaded or created
77 self.dirtynodes = {} # keep track of the dirty nodes by class
78 self.newnodes = {} # keep track of the new nodes by class
79 self.destroyednodes = {}# keep track of the destroyed nodes by class
80 self.transactions = []
81 self.indexer = Indexer(self.dir)
82 self.sessions = Sessions(self.config)
83 self.otks = OneTimeKeys(self.config)
84 self.security = security.Security(self)
85 # ensure files are group readable and writable
86 os.umask(0002)
88 # lock it
89 lockfilenm = os.path.join(self.dir, 'lock')
90 self.lockfile = locking.acquire_lock(lockfilenm)
91 self.lockfile.write(str(os.getpid()))
92 self.lockfile.flush()
94 def post_init(self):
95 ''' Called once the schema initialisation has finished.
96 '''
97 # reindex the db if necessary
98 if self.indexer.should_reindex():
99 self.reindex()
101 def refresh_database(self):
102 "Rebuild the database"
103 self.reindex()
105 def reindex(self):
106 for klass in self.classes.values():
107 for nodeid in klass.list():
108 klass.index(nodeid)
109 self.indexer.save_index()
111 def __repr__(self):
112 return '<back_anydbm instance at %x>'%id(self)
114 #
115 # Classes
116 #
117 def __getattr__(self, classname):
118 '''A convenient way of calling self.getclass(classname).'''
119 if self.classes.has_key(classname):
120 if __debug__:
121 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
122 return self.classes[classname]
123 raise AttributeError, classname
125 def addclass(self, cl):
126 if __debug__:
127 print >>hyperdb.DEBUG, 'addclass', (self, cl)
128 cn = cl.classname
129 if self.classes.has_key(cn):
130 raise ValueError, cn
131 self.classes[cn] = cl
133 def getclasses(self):
134 '''Return a list of the names of all existing classes.'''
135 if __debug__:
136 print >>hyperdb.DEBUG, 'getclasses', (self,)
137 l = self.classes.keys()
138 l.sort()
139 return l
141 def getclass(self, classname):
142 '''Get the Class object representing a particular class.
144 If 'classname' is not a valid class name, a KeyError is raised.
145 '''
146 if __debug__:
147 print >>hyperdb.DEBUG, 'getclass', (self, classname)
148 try:
149 return self.classes[classname]
150 except KeyError:
151 raise KeyError, 'There is no class called "%s"'%classname
153 #
154 # Class DBs
155 #
156 def clear(self):
157 '''Delete all database contents
158 '''
159 if __debug__:
160 print >>hyperdb.DEBUG, 'clear', (self,)
161 for cn in self.classes.keys():
162 for dummy in 'nodes', 'journals':
163 path = os.path.join(self.dir, 'journals.%s'%cn)
164 if os.path.exists(path):
165 os.remove(path)
166 elif os.path.exists(path+'.db'): # dbm appends .db
167 os.remove(path+'.db')
169 def getclassdb(self, classname, mode='r'):
170 ''' grab a connection to the class db that will be used for
171 multiple actions
172 '''
173 if __debug__:
174 print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode)
175 return self.opendb('nodes.%s'%classname, mode)
177 def determine_db_type(self, path):
178 ''' determine which DB wrote the class file
179 '''
180 db_type = ''
181 if os.path.exists(path):
182 db_type = whichdb.whichdb(path)
183 if not db_type:
184 raise DatabaseError, "Couldn't identify database type"
185 elif os.path.exists(path+'.db'):
186 # if the path ends in '.db', it's a dbm database, whether
187 # anydbm says it's dbhash or not!
188 db_type = 'dbm'
189 return db_type
191 def opendb(self, name, mode):
192 '''Low-level database opener that gets around anydbm/dbm
193 eccentricities.
194 '''
195 if __debug__:
196 print >>hyperdb.DEBUG, 'opendb', (self, name, mode)
198 # figure the class db type
199 path = os.path.join(os.getcwd(), self.dir, name)
200 db_type = self.determine_db_type(path)
202 # new database? let anydbm pick the best dbm
203 if not db_type:
204 if __debug__:
205 print >>hyperdb.DEBUG, "opendb anydbm.open(%r, 'c')"%path
206 return anydbm.open(path, 'c')
208 # open the database with the correct module
209 try:
210 dbm = __import__(db_type)
211 except ImportError:
212 raise DatabaseError, \
213 "Couldn't open database - the required module '%s'"\
214 " is not available"%db_type
215 if __debug__:
216 print >>hyperdb.DEBUG, "opendb %r.open(%r, %r)"%(db_type, path,
217 mode)
218 return dbm.open(path, mode)
220 #
221 # Node IDs
222 #
223 def newid(self, classname):
224 ''' Generate a new id for the given class
225 '''
226 # open the ids DB - create if if doesn't exist
227 db = self.opendb('_ids', 'c')
228 if db.has_key(classname):
229 newid = db[classname] = str(int(db[classname]) + 1)
230 else:
231 # the count() bit is transitional - older dbs won't start at 1
232 newid = str(self.getclass(classname).count()+1)
233 db[classname] = newid
234 db.close()
235 return newid
237 def setid(self, classname, setid):
238 ''' Set the id counter: used during import of database
239 '''
240 # open the ids DB - create if if doesn't exist
241 db = self.opendb('_ids', 'c')
242 db[classname] = str(setid)
243 db.close()
245 #
246 # Nodes
247 #
248 def addnode(self, classname, nodeid, node):
249 ''' add the specified node to its class's db
250 '''
251 if __debug__:
252 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
254 # we'll be supplied these props if we're doing an import
255 if not node.has_key('creator'):
256 # add in the "calculated" properties (dupe so we don't affect
257 # calling code's node assumptions)
258 node = node.copy()
259 node['creator'] = self.getuid()
260 node['creation'] = node['activity'] = date.Date()
262 self.newnodes.setdefault(classname, {})[nodeid] = 1
263 self.cache.setdefault(classname, {})[nodeid] = node
264 self.savenode(classname, nodeid, node)
266 def setnode(self, classname, nodeid, node):
267 ''' change the specified node
268 '''
269 if __debug__:
270 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
271 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
273 # update the activity time (dupe so we don't affect
274 # calling code's node assumptions)
275 node = node.copy()
276 node['activity'] = date.Date()
278 # can't set without having already loaded the node
279 self.cache[classname][nodeid] = node
280 self.savenode(classname, nodeid, node)
282 def savenode(self, classname, nodeid, node):
283 ''' perform the saving of data specified by the set/addnode
284 '''
285 if __debug__:
286 print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node)
287 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
289 def getnode(self, classname, nodeid, db=None, cache=1):
290 ''' get a node from the database
292 Note the "cache" parameter is not used, and exists purely for
293 backward compatibility!
294 '''
295 if __debug__:
296 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid, db)
298 # try the cache
299 cache_dict = self.cache.setdefault(classname, {})
300 if cache_dict.has_key(nodeid):
301 if __debug__:
302 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
303 nodeid)
304 return cache_dict[nodeid]
306 if __debug__:
307 print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid)
309 # get from the database and save in the cache
310 if db is None:
311 db = self.getclassdb(classname)
312 if not db.has_key(nodeid):
313 raise IndexError, "no such %s %s"%(classname, nodeid)
315 # check the uncommitted, destroyed nodes
316 if (self.destroyednodes.has_key(classname) and
317 self.destroyednodes[classname].has_key(nodeid)):
318 raise IndexError, "no such %s %s"%(classname, nodeid)
320 # decode
321 res = marshal.loads(db[nodeid])
323 # reverse the serialisation
324 res = self.unserialise(classname, res)
326 # store off in the cache dict
327 if cache:
328 cache_dict[nodeid] = res
330 return res
332 def destroynode(self, classname, nodeid):
333 '''Remove a node from the database. Called exclusively by the
334 destroy() method on Class.
335 '''
336 if __debug__:
337 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
339 # remove from cache and newnodes if it's there
340 if (self.cache.has_key(classname) and
341 self.cache[classname].has_key(nodeid)):
342 del self.cache[classname][nodeid]
343 if (self.newnodes.has_key(classname) and
344 self.newnodes[classname].has_key(nodeid)):
345 del self.newnodes[classname][nodeid]
347 # see if there's any obvious commit actions that we should get rid of
348 for entry in self.transactions[:]:
349 if entry[1][:2] == (classname, nodeid):
350 self.transactions.remove(entry)
352 # add to the destroyednodes map
353 self.destroyednodes.setdefault(classname, {})[nodeid] = 1
355 # add the destroy commit action
356 self.transactions.append((self.doDestroyNode, (classname, nodeid)))
358 def serialise(self, classname, node):
359 '''Copy the node contents, converting non-marshallable data into
360 marshallable data.
361 '''
362 if __debug__:
363 print >>hyperdb.DEBUG, 'serialise', classname, node
364 properties = self.getclass(classname).getprops()
365 d = {}
366 for k, v in node.items():
367 # if the property doesn't exist, or is the "retired" flag then
368 # it won't be in the properties dict
369 if not properties.has_key(k):
370 d[k] = v
371 continue
373 # get the property spec
374 prop = properties[k]
376 if isinstance(prop, Password) and v is not None:
377 d[k] = str(v)
378 elif isinstance(prop, Date) and v is not None:
379 d[k] = v.serialise()
380 elif isinstance(prop, Interval) and v is not None:
381 d[k] = v.serialise()
382 else:
383 d[k] = v
384 return d
386 def unserialise(self, classname, node):
387 '''Decode the marshalled node data
388 '''
389 if __debug__:
390 print >>hyperdb.DEBUG, 'unserialise', classname, node
391 properties = self.getclass(classname).getprops()
392 d = {}
393 for k, v in node.items():
394 # if the property doesn't exist, or is the "retired" flag then
395 # it won't be in the properties dict
396 if not properties.has_key(k):
397 d[k] = v
398 continue
400 # get the property spec
401 prop = properties[k]
403 if isinstance(prop, Date) and v is not None:
404 d[k] = date.Date(v)
405 elif isinstance(prop, Interval) and v is not None:
406 d[k] = date.Interval(v)
407 elif isinstance(prop, Password) and v is not None:
408 p = password.Password()
409 p.unpack(v)
410 d[k] = p
411 else:
412 d[k] = v
413 return d
415 def hasnode(self, classname, nodeid, db=None):
416 ''' determine if the database has a given node
417 '''
418 if __debug__:
419 print >>hyperdb.DEBUG, 'hasnode', (self, classname, nodeid, db)
421 # try the cache
422 cache = self.cache.setdefault(classname, {})
423 if cache.has_key(nodeid):
424 if __debug__:
425 print >>hyperdb.TRACE, 'has %s %s cached'%(classname, nodeid)
426 return 1
427 if __debug__:
428 print >>hyperdb.TRACE, 'has %s %s'%(classname, nodeid)
430 # not in the cache - check the database
431 if db is None:
432 db = self.getclassdb(classname)
433 res = db.has_key(nodeid)
434 return res
436 def countnodes(self, classname, db=None):
437 if __debug__:
438 print >>hyperdb.DEBUG, 'countnodes', (self, classname, db)
440 count = 0
442 # include the uncommitted nodes
443 if self.newnodes.has_key(classname):
444 count += len(self.newnodes[classname])
445 if self.destroyednodes.has_key(classname):
446 count -= len(self.destroyednodes[classname])
448 # and count those in the DB
449 if db is None:
450 db = self.getclassdb(classname)
451 count = count + len(db.keys())
452 return count
455 #
456 # Files - special node properties
457 # inherited from FileStorage
459 #
460 # Journal
461 #
462 def addjournal(self, classname, nodeid, action, params, creator=None,
463 creation=None):
464 ''' Journal the Action
465 'action' may be:
467 'create' or 'set' -- 'params' is a dictionary of property values
468 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
469 'retire' -- 'params' is None
470 '''
471 if __debug__:
472 print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid,
473 action, params, creator, creation)
474 self.transactions.append((self.doSaveJournal, (classname, nodeid,
475 action, params, creator, creation)))
477 def getjournal(self, classname, nodeid):
478 ''' get the journal for id
480 Raise IndexError if the node doesn't exist (as per history()'s
481 API)
482 '''
483 if __debug__:
484 print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid)
486 # our journal result
487 res = []
489 # add any journal entries for transactions not committed to the
490 # database
491 for method, args in self.transactions:
492 if method != self.doSaveJournal:
493 continue
494 (cache_classname, cache_nodeid, cache_action, cache_params,
495 cache_creator, cache_creation) = args
496 if cache_classname == classname and cache_nodeid == nodeid:
497 if not cache_creator:
498 cache_creator = self.getuid()
499 if not cache_creation:
500 cache_creation = date.Date()
501 res.append((cache_nodeid, cache_creation, cache_creator,
502 cache_action, cache_params))
504 # attempt to open the journal - in some rare cases, the journal may
505 # not exist
506 try:
507 db = self.opendb('journals.%s'%classname, 'r')
508 except anydbm.error, error:
509 if str(error) == "need 'c' or 'n' flag to open new db":
510 raise IndexError, 'no such %s %s'%(classname, nodeid)
511 elif error.args[0] != 2:
512 # this isn't a "not found" error, be alarmed!
513 raise
514 if res:
515 # we have unsaved journal entries, return them
516 return res
517 raise IndexError, 'no such %s %s'%(classname, nodeid)
518 try:
519 journal = marshal.loads(db[nodeid])
520 except KeyError:
521 db.close()
522 if res:
523 # we have some unsaved journal entries, be happy!
524 return res
525 raise IndexError, 'no such %s %s'%(classname, nodeid)
526 db.close()
528 # add all the saved journal entries for this node
529 for nodeid, date_stamp, user, action, params in journal:
530 res.append((nodeid, date.Date(date_stamp), user, action, params))
531 return res
533 def pack(self, pack_before):
534 ''' Delete all journal entries except "create" before 'pack_before'.
535 '''
536 if __debug__:
537 print >>hyperdb.DEBUG, 'packjournal', (self, pack_before)
539 pack_before = pack_before.serialise()
540 for classname in self.getclasses():
541 # get the journal db
542 db_name = 'journals.%s'%classname
543 path = os.path.join(os.getcwd(), self.dir, classname)
544 db_type = self.determine_db_type(path)
545 db = self.opendb(db_name, 'w')
547 for key in db.keys():
548 # get the journal for this db entry
549 journal = marshal.loads(db[key])
550 l = []
551 last_set_entry = None
552 for entry in journal:
553 # unpack the entry
554 (nodeid, date_stamp, self.journaltag, action,
555 params) = entry
556 # if the entry is after the pack date, _or_ the initial
557 # create entry, then it stays
558 if date_stamp > pack_before or action == 'create':
559 l.append(entry)
560 db[key] = marshal.dumps(l)
561 if db_type == 'gdbm':
562 db.reorganize()
563 db.close()
566 #
567 # Basic transaction support
568 #
569 def commit(self):
570 ''' Commit the current transactions.
571 '''
572 if __debug__:
573 print >>hyperdb.DEBUG, 'commit', (self,)
575 # keep a handle to all the database files opened
576 self.databases = {}
578 try:
579 # now, do all the transactions
580 reindex = {}
581 for method, args in self.transactions:
582 reindex[method(*args)] = 1
583 finally:
584 # make sure we close all the database files
585 for db in self.databases.values():
586 db.close()
587 del self.databases
589 # reindex the nodes that request it
590 for classname, nodeid in filter(None, reindex.keys()):
591 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
592 self.getclass(classname).index(nodeid)
594 # save the indexer state
595 self.indexer.save_index()
597 self.clearCache()
599 def clearCache(self):
600 # all transactions committed, back to normal
601 self.cache = {}
602 self.dirtynodes = {}
603 self.newnodes = {}
604 self.destroyednodes = {}
605 self.transactions = []
607 def getCachedClassDB(self, classname):
608 ''' get the class db, looking in our cache of databases for commit
609 '''
610 # get the database handle
611 db_name = 'nodes.%s'%classname
612 if not self.databases.has_key(db_name):
613 self.databases[db_name] = self.getclassdb(classname, 'c')
614 return self.databases[db_name]
616 def doSaveNode(self, classname, nodeid, node):
617 if __debug__:
618 print >>hyperdb.DEBUG, 'doSaveNode', (self, classname, nodeid,
619 node)
621 db = self.getCachedClassDB(classname)
623 # now save the marshalled data
624 db[nodeid] = marshal.dumps(self.serialise(classname, node))
626 # return the classname, nodeid so we reindex this content
627 return (classname, nodeid)
629 def getCachedJournalDB(self, classname):
630 ''' get the journal db, looking in our cache of databases for commit
631 '''
632 # get the database handle
633 db_name = 'journals.%s'%classname
634 if not self.databases.has_key(db_name):
635 self.databases[db_name] = self.opendb(db_name, 'c')
636 return self.databases[db_name]
638 def doSaveJournal(self, classname, nodeid, action, params, creator,
639 creation):
640 # serialise the parameters now if necessary
641 if isinstance(params, type({})):
642 if action in ('set', 'create'):
643 params = self.serialise(classname, params)
645 # handle supply of the special journalling parameters (usually
646 # supplied on importing an existing database)
647 if creator:
648 journaltag = creator
649 else:
650 journaltag = self.getuid()
651 if creation:
652 journaldate = creation.serialise()
653 else:
654 journaldate = date.Date().serialise()
656 # create the journal entry
657 entry = (nodeid, journaldate, journaltag, action, params)
659 if __debug__:
660 print >>hyperdb.DEBUG, 'doSaveJournal', entry
662 db = self.getCachedJournalDB(classname)
664 # now insert the journal entry
665 if db.has_key(nodeid):
666 # append to existing
667 s = db[nodeid]
668 l = marshal.loads(s)
669 l.append(entry)
670 else:
671 l = [entry]
673 db[nodeid] = marshal.dumps(l)
675 def doDestroyNode(self, classname, nodeid):
676 if __debug__:
677 print >>hyperdb.DEBUG, 'doDestroyNode', (self, classname, nodeid)
679 # delete from the class database
680 db = self.getCachedClassDB(classname)
681 if db.has_key(nodeid):
682 del db[nodeid]
684 # delete from the database
685 db = self.getCachedJournalDB(classname)
686 if db.has_key(nodeid):
687 del db[nodeid]
689 # return the classname, nodeid so we reindex this content
690 return (classname, nodeid)
692 def rollback(self):
693 ''' Reverse all actions from the current transaction.
694 '''
695 if __debug__:
696 print >>hyperdb.DEBUG, 'rollback', (self, )
697 for method, args in self.transactions:
698 # delete temporary files
699 if method == self.doStoreFile:
700 self.rollbackStoreFile(*args)
701 self.cache = {}
702 self.dirtynodes = {}
703 self.newnodes = {}
704 self.destroyednodes = {}
705 self.transactions = []
707 def close(self):
708 ''' Nothing to do
709 '''
710 if self.lockfile is not None:
711 locking.release_lock(self.lockfile)
712 if self.lockfile is not None:
713 self.lockfile.close()
714 self.lockfile = None
716 _marker = []
717 class Class(hyperdb.Class):
718 '''The handle to a particular class of nodes in a hyperdatabase.'''
720 def __init__(self, db, classname, **properties):
721 '''Create a new class with a given name and property specification.
723 'classname' must not collide with the name of an existing class,
724 or a ValueError is raised. The keyword arguments in 'properties'
725 must map names to property objects, or a TypeError is raised.
726 '''
727 if (properties.has_key('creation') or properties.has_key('activity')
728 or properties.has_key('creator')):
729 raise ValueError, '"creation", "activity" and "creator" are '\
730 'reserved'
732 self.classname = classname
733 self.properties = properties
734 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
735 self.key = ''
737 # should we journal changes (default yes)
738 self.do_journal = 1
740 # do the db-related init stuff
741 db.addclass(self)
743 self.auditors = {'create': [], 'set': [], 'retire': [], 'restore': []}
744 self.reactors = {'create': [], 'set': [], 'retire': [], 'restore': []}
746 def enableJournalling(self):
747 '''Turn journalling on for this class
748 '''
749 self.do_journal = 1
751 def disableJournalling(self):
752 '''Turn journalling off for this class
753 '''
754 self.do_journal = 0
756 # Editing nodes:
758 def create(self, **propvalues):
759 '''Create a new node of this class and return its id.
761 The keyword arguments in 'propvalues' map property names to values.
763 The values of arguments must be acceptable for the types of their
764 corresponding properties or a TypeError is raised.
766 If this class has a key property, it must be present and its value
767 must not collide with other key strings or a ValueError is raised.
769 Any other properties on this class that are missing from the
770 'propvalues' dictionary are set to None.
772 If an id in a link or multilink property does not refer to a valid
773 node, an IndexError is raised.
775 These operations trigger detectors and can be vetoed. Attempts
776 to modify the "creation" or "activity" properties cause a KeyError.
777 '''
778 self.fireAuditors('create', None, propvalues)
779 newid = self.create_inner(**propvalues)
780 self.fireReactors('create', newid, None)
781 return newid
783 def create_inner(self, **propvalues):
784 ''' Called by create, in-between the audit and react calls.
785 '''
786 if propvalues.has_key('id'):
787 raise KeyError, '"id" is reserved'
789 if self.db.journaltag is None:
790 raise DatabaseError, 'Database open read-only'
792 if propvalues.has_key('creation') or propvalues.has_key('activity'):
793 raise KeyError, '"creation" and "activity" are reserved'
794 # new node's id
795 newid = self.db.newid(self.classname)
797 # validate propvalues
798 num_re = re.compile('^\d+$')
799 for key, value in propvalues.items():
800 if key == self.key:
801 try:
802 self.lookup(value)
803 except KeyError:
804 pass
805 else:
806 raise ValueError, 'node with key "%s" exists'%value
808 # try to handle this property
809 try:
810 prop = self.properties[key]
811 except KeyError:
812 raise KeyError, '"%s" has no property "%s"'%(self.classname,
813 key)
815 if value is not None and isinstance(prop, Link):
816 if type(value) != type(''):
817 raise ValueError, 'link value must be String'
818 link_class = self.properties[key].classname
819 # if it isn't a number, it's a key
820 if not num_re.match(value):
821 try:
822 value = self.db.classes[link_class].lookup(value)
823 except (TypeError, KeyError):
824 raise IndexError, 'new property "%s": %s not a %s'%(
825 key, value, link_class)
826 elif not self.db.getclass(link_class).hasnode(value):
827 raise IndexError, '%s has no node %s'%(link_class, value)
829 # save off the value
830 propvalues[key] = value
832 # register the link with the newly linked node
833 if self.do_journal and self.properties[key].do_journal:
834 self.db.addjournal(link_class, value, 'link',
835 (self.classname, newid, key))
837 elif isinstance(prop, Multilink):
838 if type(value) != type([]):
839 raise TypeError, 'new property "%s" not a list of ids'%key
841 # clean up and validate the list of links
842 link_class = self.properties[key].classname
843 l = []
844 for entry in value:
845 if type(entry) != type(''):
846 raise ValueError, '"%s" multilink value (%r) '\
847 'must contain Strings'%(key, value)
848 # if it isn't a number, it's a key
849 if not num_re.match(entry):
850 try:
851 entry = self.db.classes[link_class].lookup(entry)
852 except (TypeError, KeyError):
853 raise IndexError, 'new property "%s": %s not a %s'%(
854 key, entry, self.properties[key].classname)
855 l.append(entry)
856 value = l
857 propvalues[key] = value
859 # handle additions
860 for nodeid in value:
861 if not self.db.getclass(link_class).hasnode(nodeid):
862 raise IndexError, '%s has no node %s'%(link_class,
863 nodeid)
864 # register the link with the newly linked node
865 if self.do_journal and self.properties[key].do_journal:
866 self.db.addjournal(link_class, nodeid, 'link',
867 (self.classname, newid, key))
869 elif isinstance(prop, String):
870 if type(value) != type('') and type(value) != type(u''):
871 raise TypeError, 'new property "%s" not a string'%key
873 elif isinstance(prop, Password):
874 if not isinstance(value, password.Password):
875 raise TypeError, 'new property "%s" not a Password'%key
877 elif isinstance(prop, Date):
878 if value is not None and not isinstance(value, date.Date):
879 raise TypeError, 'new property "%s" not a Date'%key
881 elif isinstance(prop, Interval):
882 if value is not None and not isinstance(value, date.Interval):
883 raise TypeError, 'new property "%s" not an Interval'%key
885 elif value is not None and isinstance(prop, Number):
886 try:
887 float(value)
888 except ValueError:
889 raise TypeError, 'new property "%s" not numeric'%key
891 elif value is not None and isinstance(prop, Boolean):
892 try:
893 int(value)
894 except ValueError:
895 raise TypeError, 'new property "%s" not boolean'%key
897 # make sure there's data where there needs to be
898 for key, prop in self.properties.items():
899 if propvalues.has_key(key):
900 continue
901 if key == self.key:
902 raise ValueError, 'key property "%s" is required'%key
903 if isinstance(prop, Multilink):
904 propvalues[key] = []
905 else:
906 propvalues[key] = None
908 # done
909 self.db.addnode(self.classname, newid, propvalues)
910 if self.do_journal:
911 self.db.addjournal(self.classname, newid, 'create', {})
913 return newid
915 def export_list(self, propnames, nodeid):
916 ''' Export a node - generate a list of CSV-able data in the order
917 specified by propnames for the given node.
918 '''
919 properties = self.getprops()
920 l = []
921 for prop in propnames:
922 proptype = properties[prop]
923 value = self.get(nodeid, prop)
924 # "marshal" data where needed
925 if value is None:
926 pass
927 elif isinstance(proptype, hyperdb.Date):
928 value = value.get_tuple()
929 elif isinstance(proptype, hyperdb.Interval):
930 value = value.get_tuple()
931 elif isinstance(proptype, hyperdb.Password):
932 value = str(value)
933 l.append(repr(value))
935 # append retired flag
936 l.append(repr(self.is_retired(nodeid)))
938 return l
940 def import_list(self, propnames, proplist):
941 ''' Import a node - all information including "id" is present and
942 should not be sanity checked. Triggers are not triggered. The
943 journal should be initialised using the "creator" and "created"
944 information.
946 Return the nodeid of the node imported.
947 '''
948 if self.db.journaltag is None:
949 raise DatabaseError, 'Database open read-only'
950 properties = self.getprops()
952 # make the new node's property map
953 d = {}
954 newid = None
955 for i in range(len(propnames)):
956 # Figure the property for this column
957 propname = propnames[i]
959 # Use eval to reverse the repr() used to output the CSV
960 value = eval(proplist[i])
962 # "unmarshal" where necessary
963 if propname == 'id':
964 newid = value
965 continue
966 elif propname == 'is retired':
967 # is the item retired?
968 if int(value):
969 d[self.db.RETIRED_FLAG] = 1
970 continue
971 elif value is None:
972 d[propname] = None
973 continue
975 prop = properties[propname]
976 if isinstance(prop, hyperdb.Date):
977 value = date.Date(value)
978 elif isinstance(prop, hyperdb.Interval):
979 value = date.Interval(value)
980 elif isinstance(prop, hyperdb.Password):
981 pwd = password.Password()
982 pwd.unpack(value)
983 value = pwd
984 d[propname] = value
986 # get a new id if necessary
987 if newid is None:
988 newid = self.db.newid(self.classname)
990 # add the node and journal
991 self.db.addnode(self.classname, newid, d)
993 # extract the journalling stuff and nuke it
994 if d.has_key('creator'):
995 creator = d['creator']
996 del d['creator']
997 else:
998 creator = None
999 if d.has_key('creation'):
1000 creation = d['creation']
1001 del d['creation']
1002 else:
1003 creation = None
1004 if d.has_key('activity'):
1005 del d['activity']
1006 self.db.addjournal(self.classname, newid, 'create', {}, creator,
1007 creation)
1008 return newid
1010 def get(self, nodeid, propname, default=_marker, cache=1):
1011 '''Get the value of a property on an existing node of this class.
1013 'nodeid' must be the id of an existing node of this class or an
1014 IndexError is raised. 'propname' must be the name of a property
1015 of this class or a KeyError is raised.
1017 'cache' exists for backward compatibility, and is not used.
1019 Attempts to get the "creation" or "activity" properties should
1020 do the right thing.
1021 '''
1022 if propname == 'id':
1023 return nodeid
1025 # get the node's dict
1026 d = self.db.getnode(self.classname, nodeid)
1028 # check for one of the special props
1029 if propname == 'creation':
1030 if d.has_key('creation'):
1031 return d['creation']
1032 if not self.do_journal:
1033 raise ValueError, 'Journalling is disabled for this class'
1034 journal = self.db.getjournal(self.classname, nodeid)
1035 if journal:
1036 return self.db.getjournal(self.classname, nodeid)[0][1]
1037 else:
1038 # on the strange chance that there's no journal
1039 return date.Date()
1040 if propname == 'activity':
1041 if d.has_key('activity'):
1042 return d['activity']
1043 if not self.do_journal:
1044 raise ValueError, 'Journalling is disabled for this class'
1045 journal = self.db.getjournal(self.classname, nodeid)
1046 if journal:
1047 return self.db.getjournal(self.classname, nodeid)[-1][1]
1048 else:
1049 # on the strange chance that there's no journal
1050 return date.Date()
1051 if propname == 'creator':
1052 if d.has_key('creator'):
1053 return d['creator']
1054 if not self.do_journal:
1055 raise ValueError, 'Journalling is disabled for this class'
1056 journal = self.db.getjournal(self.classname, nodeid)
1057 if journal:
1058 num_re = re.compile('^\d+$')
1059 value = self.db.getjournal(self.classname, nodeid)[0][2]
1060 if num_re.match(value):
1061 return value
1062 else:
1063 # old-style "username" journal tag
1064 try:
1065 return self.db.user.lookup(value)
1066 except KeyError:
1067 # user's been retired, return admin
1068 return '1'
1069 else:
1070 return self.db.getuid()
1072 # get the property (raises KeyErorr if invalid)
1073 prop = self.properties[propname]
1075 if not d.has_key(propname):
1076 if default is _marker:
1077 if isinstance(prop, Multilink):
1078 return []
1079 else:
1080 return None
1081 else:
1082 return default
1084 # return a dupe of the list so code doesn't get confused
1085 if isinstance(prop, Multilink):
1086 return d[propname][:]
1088 return d[propname]
1090 def set(self, nodeid, **propvalues):
1091 '''Modify a property on an existing node of this class.
1093 'nodeid' must be the id of an existing node of this class or an
1094 IndexError is raised.
1096 Each key in 'propvalues' must be the name of a property of this
1097 class or a KeyError is raised.
1099 All values in 'propvalues' must be acceptable types for their
1100 corresponding properties or a TypeError is raised.
1102 If the value of the key property is set, it must not collide with
1103 other key strings or a ValueError is raised.
1105 If the value of a Link or Multilink property contains an invalid
1106 node id, a ValueError is raised.
1108 These operations trigger detectors and can be vetoed. Attempts
1109 to modify the "creation" or "activity" properties cause a KeyError.
1110 '''
1111 if not propvalues:
1112 return propvalues
1114 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1115 raise KeyError, '"creation" and "activity" are reserved'
1117 if propvalues.has_key('id'):
1118 raise KeyError, '"id" is reserved'
1120 if self.db.journaltag is None:
1121 raise DatabaseError, 'Database open read-only'
1123 self.fireAuditors('set', nodeid, propvalues)
1124 # Take a copy of the node dict so that the subsequent set
1125 # operation doesn't modify the oldvalues structure.
1126 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1128 node = self.db.getnode(self.classname, nodeid)
1129 if node.has_key(self.db.RETIRED_FLAG):
1130 raise IndexError
1131 num_re = re.compile('^\d+$')
1133 # if the journal value is to be different, store it in here
1134 journalvalues = {}
1136 for propname, value in propvalues.items():
1137 # check to make sure we're not duplicating an existing key
1138 if propname == self.key and node[propname] != value:
1139 try:
1140 self.lookup(value)
1141 except KeyError:
1142 pass
1143 else:
1144 raise ValueError, 'node with key "%s" exists'%value
1146 # this will raise the KeyError if the property isn't valid
1147 # ... we don't use getprops() here because we only care about
1148 # the writeable properties.
1149 try:
1150 prop = self.properties[propname]
1151 except KeyError:
1152 raise KeyError, '"%s" has no property named "%s"'%(
1153 self.classname, propname)
1155 # if the value's the same as the existing value, no sense in
1156 # doing anything
1157 current = node.get(propname, None)
1158 if value == current:
1159 del propvalues[propname]
1160 continue
1161 journalvalues[propname] = current
1163 # do stuff based on the prop type
1164 if isinstance(prop, Link):
1165 link_class = prop.classname
1166 # if it isn't a number, it's a key
1167 if value is not None and not isinstance(value, type('')):
1168 raise ValueError, 'property "%s" link value be a string'%(
1169 propname)
1170 if isinstance(value, type('')) and not num_re.match(value):
1171 try:
1172 value = self.db.classes[link_class].lookup(value)
1173 except (TypeError, KeyError):
1174 raise IndexError, 'new property "%s": %s not a %s'%(
1175 propname, value, prop.classname)
1177 if (value is not None and
1178 not self.db.getclass(link_class).hasnode(value)):
1179 raise IndexError, '%s has no node %s'%(link_class, value)
1181 if self.do_journal and prop.do_journal:
1182 # register the unlink with the old linked node
1183 if node.has_key(propname) and node[propname] is not None:
1184 self.db.addjournal(link_class, node[propname], 'unlink',
1185 (self.classname, nodeid, propname))
1187 # register the link with the newly linked node
1188 if value is not None:
1189 self.db.addjournal(link_class, value, 'link',
1190 (self.classname, nodeid, propname))
1192 elif isinstance(prop, Multilink):
1193 if type(value) != type([]):
1194 raise TypeError, 'new property "%s" not a list of'\
1195 ' ids'%propname
1196 link_class = self.properties[propname].classname
1197 l = []
1198 for entry in value:
1199 # if it isn't a number, it's a key
1200 if type(entry) != type(''):
1201 raise ValueError, 'new property "%s" link value ' \
1202 'must be a string'%propname
1203 if not num_re.match(entry):
1204 try:
1205 entry = self.db.classes[link_class].lookup(entry)
1206 except (TypeError, KeyError):
1207 raise IndexError, 'new property "%s": %s not a %s'%(
1208 propname, entry,
1209 self.properties[propname].classname)
1210 l.append(entry)
1211 value = l
1212 propvalues[propname] = value
1214 # figure the journal entry for this property
1215 add = []
1216 remove = []
1218 # handle removals
1219 if node.has_key(propname):
1220 l = node[propname]
1221 else:
1222 l = []
1223 for id in l[:]:
1224 if id in value:
1225 continue
1226 # register the unlink with the old linked node
1227 if self.do_journal and self.properties[propname].do_journal:
1228 self.db.addjournal(link_class, id, 'unlink',
1229 (self.classname, nodeid, propname))
1230 l.remove(id)
1231 remove.append(id)
1233 # handle additions
1234 for id in value:
1235 if not self.db.getclass(link_class).hasnode(id):
1236 raise IndexError, '%s has no node %s'%(link_class, id)
1237 if id in l:
1238 continue
1239 # register the link with the newly linked node
1240 if self.do_journal and self.properties[propname].do_journal:
1241 self.db.addjournal(link_class, id, 'link',
1242 (self.classname, nodeid, propname))
1243 l.append(id)
1244 add.append(id)
1246 # figure the journal entry
1247 l = []
1248 if add:
1249 l.append(('+', add))
1250 if remove:
1251 l.append(('-', remove))
1252 if l:
1253 journalvalues[propname] = tuple(l)
1255 elif isinstance(prop, String):
1256 if value is not None and type(value) != type('') and type(value) != type(u''):
1257 raise TypeError, 'new property "%s" not a string'%propname
1259 elif isinstance(prop, Password):
1260 if not isinstance(value, password.Password):
1261 raise TypeError, 'new property "%s" not a Password'%propname
1262 propvalues[propname] = value
1264 elif value is not None and isinstance(prop, Date):
1265 if not isinstance(value, date.Date):
1266 raise TypeError, 'new property "%s" not a Date'% propname
1267 propvalues[propname] = value
1269 elif value is not None and isinstance(prop, Interval):
1270 if not isinstance(value, date.Interval):
1271 raise TypeError, 'new property "%s" not an '\
1272 'Interval'%propname
1273 propvalues[propname] = value
1275 elif value is not None and isinstance(prop, Number):
1276 try:
1277 float(value)
1278 except ValueError:
1279 raise TypeError, 'new property "%s" not numeric'%propname
1281 elif value is not None and isinstance(prop, Boolean):
1282 try:
1283 int(value)
1284 except ValueError:
1285 raise TypeError, 'new property "%s" not boolean'%propname
1287 node[propname] = value
1289 # nothing to do?
1290 if not propvalues:
1291 return propvalues
1293 # do the set, and journal it
1294 self.db.setnode(self.classname, nodeid, node)
1296 if self.do_journal:
1297 self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
1299 self.fireReactors('set', nodeid, oldvalues)
1301 return propvalues
1303 def retire(self, nodeid):
1304 '''Retire a node.
1306 The properties on the node remain available from the get() method,
1307 and the node's id is never reused.
1309 Retired nodes are not returned by the find(), list(), or lookup()
1310 methods, and other nodes may reuse the values of their key properties.
1312 These operations trigger detectors and can be vetoed. Attempts
1313 to modify the "creation" or "activity" properties cause a KeyError.
1314 '''
1315 if self.db.journaltag is None:
1316 raise DatabaseError, 'Database open read-only'
1318 self.fireAuditors('retire', nodeid, None)
1320 node = self.db.getnode(self.classname, nodeid)
1321 node[self.db.RETIRED_FLAG] = 1
1322 self.db.setnode(self.classname, nodeid, node)
1323 if self.do_journal:
1324 self.db.addjournal(self.classname, nodeid, 'retired', None)
1326 self.fireReactors('retire', nodeid, None)
1328 def restore(self, nodeid):
1329 '''Restpre a retired node.
1331 Make node available for all operations like it was before retirement.
1332 '''
1333 if self.db.journaltag is None:
1334 raise DatabaseError, 'Database open read-only'
1336 node = self.db.getnode(self.classname, nodeid)
1337 # check if key property was overrided
1338 key = self.getkey()
1339 try:
1340 id = self.lookup(node[key])
1341 except KeyError:
1342 pass
1343 else:
1344 raise KeyError, "Key property (%s) of retired node clashes with \
1345 existing one (%s)" % (key, node[key])
1346 # Now we can safely restore node
1347 self.fireAuditors('restore', nodeid, None)
1348 del node[self.db.RETIRED_FLAG]
1349 self.db.setnode(self.classname, nodeid, node)
1350 if self.do_journal:
1351 self.db.addjournal(self.classname, nodeid, 'restored', None)
1353 self.fireReactors('restore', nodeid, None)
1355 def is_retired(self, nodeid, cldb=None):
1356 '''Return true if the node is retired.
1357 '''
1358 node = self.db.getnode(self.classname, nodeid, cldb)
1359 if node.has_key(self.db.RETIRED_FLAG):
1360 return 1
1361 return 0
1363 def destroy(self, nodeid):
1364 '''Destroy a node.
1366 WARNING: this method should never be used except in extremely rare
1367 situations where there could never be links to the node being
1368 deleted
1369 WARNING: use retire() instead
1370 WARNING: the properties of this node will not be available ever again
1371 WARNING: really, use retire() instead
1373 Well, I think that's enough warnings. This method exists mostly to
1374 support the session storage of the cgi interface.
1375 '''
1376 if self.db.journaltag is None:
1377 raise DatabaseError, 'Database open read-only'
1378 self.db.destroynode(self.classname, nodeid)
1380 def history(self, nodeid):
1381 '''Retrieve the journal of edits on a particular node.
1383 'nodeid' must be the id of an existing node of this class or an
1384 IndexError is raised.
1386 The returned list contains tuples of the form
1388 (nodeid, date, tag, action, params)
1390 'date' is a Timestamp object specifying the time of the change and
1391 'tag' is the journaltag specified when the database was opened.
1392 '''
1393 if not self.do_journal:
1394 raise ValueError, 'Journalling is disabled for this class'
1395 return self.db.getjournal(self.classname, nodeid)
1397 # Locating nodes:
1398 def hasnode(self, nodeid):
1399 '''Determine if the given nodeid actually exists
1400 '''
1401 return self.db.hasnode(self.classname, nodeid)
1403 def setkey(self, propname):
1404 '''Select a String property of this class to be the key property.
1406 'propname' must be the name of a String property of this class or
1407 None, or a TypeError is raised. The values of the key property on
1408 all existing nodes must be unique or a ValueError is raised. If the
1409 property doesn't exist, KeyError is raised.
1410 '''
1411 prop = self.getprops()[propname]
1412 if not isinstance(prop, String):
1413 raise TypeError, 'key properties must be String'
1414 self.key = propname
1416 def getkey(self):
1417 '''Return the name of the key property for this class or None.'''
1418 return self.key
1420 def labelprop(self, default_to_id=0):
1421 ''' Return the property name for a label for the given node.
1423 This method attempts to generate a consistent label for the node.
1424 It tries the following in order:
1425 1. key property
1426 2. "name" property
1427 3. "title" property
1428 4. first property from the sorted property name list
1429 '''
1430 k = self.getkey()
1431 if k:
1432 return k
1433 props = self.getprops()
1434 if props.has_key('name'):
1435 return 'name'
1436 elif props.has_key('title'):
1437 return 'title'
1438 if default_to_id:
1439 return 'id'
1440 props = props.keys()
1441 props.sort()
1442 return props[0]
1444 # TODO: set up a separate index db file for this? profile?
1445 def lookup(self, keyvalue):
1446 '''Locate a particular node by its key property and return its id.
1448 If this class has no key property, a TypeError is raised. If the
1449 'keyvalue' matches one of the values for the key property among
1450 the nodes in this class, the matching node's id is returned;
1451 otherwise a KeyError is raised.
1452 '''
1453 if not self.key:
1454 raise TypeError, 'No key property set for class %s'%self.classname
1455 cldb = self.db.getclassdb(self.classname)
1456 try:
1457 for nodeid in self.getnodeids(cldb):
1458 node = self.db.getnode(self.classname, nodeid, cldb)
1459 if node.has_key(self.db.RETIRED_FLAG):
1460 continue
1461 if node[self.key] == keyvalue:
1462 return nodeid
1463 finally:
1464 cldb.close()
1465 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1466 keyvalue, self.classname)
1468 # change from spec - allows multiple props to match
1469 def find(self, **propspec):
1470 '''Get the ids of items in this class which link to the given items.
1472 'propspec' consists of keyword args propname=itemid or
1473 propname={itemid:1, }
1474 'propname' must be the name of a property in this class, or a
1475 KeyError is raised. That property must be a Link or
1476 Multilink property, or a TypeError is raised.
1478 Any item in this class whose 'propname' property links to any of the
1479 itemids will be returned. Used by the full text indexing, which knows
1480 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1481 issues:
1483 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1484 '''
1485 propspec = propspec.items()
1486 for propname, itemids in propspec:
1487 # check the prop is OK
1488 prop = self.properties[propname]
1489 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1490 raise TypeError, "'%s' not a Link/Multilink property"%propname
1492 # ok, now do the find
1493 cldb = self.db.getclassdb(self.classname)
1494 l = []
1495 try:
1496 for id in self.getnodeids(db=cldb):
1497 item = self.db.getnode(self.classname, id, db=cldb)
1498 if item.has_key(self.db.RETIRED_FLAG):
1499 continue
1500 for propname, itemids in propspec:
1501 # can't test if the item doesn't have this property
1502 if not item.has_key(propname):
1503 continue
1504 if type(itemids) is not type({}):
1505 itemids = {itemids:1}
1507 # grab the property definition and its value on this item
1508 prop = self.properties[propname]
1509 value = item[propname]
1510 if isinstance(prop, Link) and itemids.has_key(value):
1511 l.append(id)
1512 break
1513 elif isinstance(prop, Multilink):
1514 hit = 0
1515 for v in value:
1516 if itemids.has_key(v):
1517 l.append(id)
1518 hit = 1
1519 break
1520 if hit:
1521 break
1522 finally:
1523 cldb.close()
1524 return l
1526 def stringFind(self, **requirements):
1527 '''Locate a particular node by matching a set of its String
1528 properties in a caseless search.
1530 If the property is not a String property, a TypeError is raised.
1532 The return is a list of the id of all nodes that match.
1533 '''
1534 for propname in requirements.keys():
1535 prop = self.properties[propname]
1536 if not isinstance(prop, String):
1537 raise TypeError, "'%s' not a String property"%propname
1538 requirements[propname] = requirements[propname].lower()
1539 l = []
1540 cldb = self.db.getclassdb(self.classname)
1541 try:
1542 for nodeid in self.getnodeids(cldb):
1543 node = self.db.getnode(self.classname, nodeid, cldb)
1544 if node.has_key(self.db.RETIRED_FLAG):
1545 continue
1546 for key, value in requirements.items():
1547 if not node.has_key(key):
1548 break
1549 if node[key] is None or node[key].lower() != value:
1550 break
1551 else:
1552 l.append(nodeid)
1553 finally:
1554 cldb.close()
1555 return l
1557 def list(self):
1558 ''' Return a list of the ids of the active nodes in this class.
1559 '''
1560 l = []
1561 cn = self.classname
1562 cldb = self.db.getclassdb(cn)
1563 try:
1564 for nodeid in self.getnodeids(cldb):
1565 node = self.db.getnode(cn, nodeid, cldb)
1566 if node.has_key(self.db.RETIRED_FLAG):
1567 continue
1568 l.append(nodeid)
1569 finally:
1570 cldb.close()
1571 l.sort()
1572 return l
1574 def getnodeids(self, db=None):
1575 ''' Return a list of ALL nodeids
1576 '''
1577 if __debug__:
1578 print >>hyperdb.DEBUG, 'getnodeids', (self, self.classname, db)
1580 res = []
1582 # start off with the new nodes
1583 if self.db.newnodes.has_key(self.classname):
1584 res += self.db.newnodes[self.classname].keys()
1586 if db is None:
1587 db = self.db.getclassdb(self.classname)
1588 res = res + db.keys()
1590 # remove the uncommitted, destroyed nodes
1591 if self.db.destroyednodes.has_key(self.classname):
1592 for nodeid in self.db.destroyednodes[self.classname].keys():
1593 if db.has_key(nodeid):
1594 res.remove(nodeid)
1596 return res
1598 def filter(self, search_matches, filterspec, sort=(None,None),
1599 group=(None,None), num_re = re.compile('^\d+$')):
1600 ''' Return a list of the ids of the active nodes in this class that
1601 match the 'filter' spec, sorted by the group spec and then the
1602 sort spec.
1604 "filterspec" is {propname: value(s)}
1605 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1606 and prop is a prop name or None
1607 "search_matches" is {nodeid: marker}
1609 The filter must match all properties specificed - but if the
1610 property value to match is a list, any one of the values in the
1611 list may match for that property to match. Unless the property
1612 is a Multilink, in which case the item's property list must
1613 match the filterspec list.
1614 '''
1615 cn = self.classname
1617 # optimise filterspec
1618 l = []
1619 props = self.getprops()
1620 LINK = 0
1621 MULTILINK = 1
1622 STRING = 2
1623 DATE = 3
1624 INTERVAL = 4
1625 OTHER = 6
1627 timezone = self.db.getUserTimezone()
1628 for k, v in filterspec.items():
1629 propclass = props[k]
1630 if isinstance(propclass, Link):
1631 if type(v) is not type([]):
1632 v = [v]
1633 # replace key values with node ids
1634 u = []
1635 link_class = self.db.classes[propclass.classname]
1636 for entry in v:
1637 # the value -1 is a special "not set" sentinel
1638 if entry == '-1':
1639 entry = None
1640 elif not num_re.match(entry):
1641 try:
1642 entry = link_class.lookup(entry)
1643 except (TypeError,KeyError):
1644 raise ValueError, 'property "%s": %s not a %s'%(
1645 k, entry, self.properties[k].classname)
1646 u.append(entry)
1648 l.append((LINK, k, u))
1649 elif isinstance(propclass, Multilink):
1650 # the value -1 is a special "not set" sentinel
1651 if v in ('-1', ['-1']):
1652 v = []
1653 elif type(v) is not type([]):
1654 v = [v]
1656 # replace key values with node ids
1657 u = []
1658 link_class = self.db.classes[propclass.classname]
1659 for entry in v:
1660 if not num_re.match(entry):
1661 try:
1662 entry = link_class.lookup(entry)
1663 except (TypeError,KeyError):
1664 raise ValueError, 'new property "%s": %s not a %s'%(
1665 k, entry, self.properties[k].classname)
1666 u.append(entry)
1667 u.sort()
1668 l.append((MULTILINK, k, u))
1669 elif isinstance(propclass, String) and k != 'id':
1670 if type(v) is not type([]):
1671 v = [v]
1672 m = []
1673 for v in v:
1674 # simple glob searching
1675 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1676 v = v.replace('?', '.')
1677 v = v.replace('*', '.*?')
1678 m.append(v)
1679 m = re.compile('(%s)'%('|'.join(m)), re.I)
1680 l.append((STRING, k, m))
1681 elif isinstance(propclass, Date):
1682 try:
1683 date_rng = Range(v, date.Date, offset=timezone)
1684 l.append((DATE, k, date_rng))
1685 except ValueError:
1686 # If range creation fails - ignore that search parameter
1687 pass
1688 elif isinstance(propclass, Interval):
1689 try:
1690 intv_rng = Range(v, date.Interval)
1691 l.append((INTERVAL, k, intv_rng))
1692 except ValueError:
1693 # If range creation fails - ignore that search parameter
1694 pass
1696 elif isinstance(propclass, Boolean):
1697 if type(v) is type(''):
1698 bv = v.lower() in ('yes', 'true', 'on', '1')
1699 else:
1700 bv = v
1701 l.append((OTHER, k, bv))
1702 elif isinstance(propclass, Number):
1703 l.append((OTHER, k, int(v)))
1704 else:
1705 l.append((OTHER, k, v))
1706 filterspec = l
1708 # now, find all the nodes that are active and pass filtering
1709 l = []
1710 cldb = self.db.getclassdb(cn)
1711 try:
1712 # TODO: only full-scan once (use items())
1713 for nodeid in self.getnodeids(cldb):
1714 node = self.db.getnode(cn, nodeid, cldb)
1715 if node.has_key(self.db.RETIRED_FLAG):
1716 continue
1717 # apply filter
1718 for t, k, v in filterspec:
1719 # handle the id prop
1720 if k == 'id' and v == nodeid:
1721 continue
1723 # make sure the node has the property
1724 if not node.has_key(k):
1725 # this node doesn't have this property, so reject it
1726 break
1728 # now apply the property filter
1729 if t == LINK:
1730 # link - if this node's property doesn't appear in the
1731 # filterspec's nodeid list, skip it
1732 if node[k] not in v:
1733 break
1734 elif t == MULTILINK:
1735 # multilink - if any of the nodeids required by the
1736 # filterspec aren't in this node's property, then skip
1737 # it
1738 have = node[k]
1739 # check for matching the absence of multilink values
1740 if not v and have:
1741 break
1743 # othewise, make sure this node has each of the
1744 # required values
1745 for want in v:
1746 if want not in have:
1747 break
1748 else:
1749 continue
1750 break
1751 elif t == STRING:
1752 if node[k] is None:
1753 break
1754 # RE search
1755 if not v.search(node[k]):
1756 break
1757 elif t == DATE or t == INTERVAL:
1758 if node[k] is None:
1759 break
1760 if v.to_value:
1761 if not (v.from_value <= node[k] and v.to_value >= node[k]):
1762 break
1763 else:
1764 if not (v.from_value <= node[k]):
1765 break
1766 elif t == OTHER:
1767 # straight value comparison for the other types
1768 if node[k] != v:
1769 break
1770 else:
1771 l.append((nodeid, node))
1772 finally:
1773 cldb.close()
1774 l.sort()
1776 # filter based on full text search
1777 if search_matches is not None:
1778 k = []
1779 for v in l:
1780 if search_matches.has_key(v[0]):
1781 k.append(v)
1782 l = k
1784 # now, sort the result
1785 def sortfun(a, b, sort=sort, group=group, properties=self.getprops(),
1786 db = self.db, cl=self):
1787 a_id, an = a
1788 b_id, bn = b
1789 # sort by group and then sort
1790 for dir, prop in group, sort:
1791 if dir is None or prop is None: continue
1793 # sorting is class-specific
1794 propclass = properties[prop]
1796 # handle the properties that might be "faked"
1797 # also, handle possible missing properties
1798 try:
1799 if not an.has_key(prop):
1800 an[prop] = cl.get(a_id, prop)
1801 av = an[prop]
1802 except KeyError:
1803 # the node doesn't have a value for this property
1804 if isinstance(propclass, Multilink): av = []
1805 else: av = ''
1806 try:
1807 if not bn.has_key(prop):
1808 bn[prop] = cl.get(b_id, prop)
1809 bv = bn[prop]
1810 except KeyError:
1811 # the node doesn't have a value for this property
1812 if isinstance(propclass, Multilink): bv = []
1813 else: bv = ''
1815 # String and Date values are sorted in the natural way
1816 if isinstance(propclass, String):
1817 # clean up the strings
1818 if av and av[0] in string.uppercase:
1819 av = av.lower()
1820 if bv and bv[0] in string.uppercase:
1821 bv = bv.lower()
1822 if (isinstance(propclass, String) or
1823 isinstance(propclass, Date)):
1824 # it might be a string that's really an integer
1825 try:
1826 av = int(av)
1827 bv = int(bv)
1828 except:
1829 pass
1830 if dir == '+':
1831 r = cmp(av, bv)
1832 if r != 0: return r
1833 elif dir == '-':
1834 r = cmp(bv, av)
1835 if r != 0: return r
1837 # Link properties are sorted according to the value of
1838 # the "order" property on the linked nodes if it is
1839 # present; or otherwise on the key string of the linked
1840 # nodes; or finally on the node ids.
1841 elif isinstance(propclass, Link):
1842 link = db.classes[propclass.classname]
1843 if av is None and bv is not None: return -1
1844 if av is not None and bv is None: return 1
1845 if av is None and bv is None: continue
1846 if link.getprops().has_key('order'):
1847 if dir == '+':
1848 r = cmp(link.get(av, 'order'),
1849 link.get(bv, 'order'))
1850 if r != 0: return r
1851 elif dir == '-':
1852 r = cmp(link.get(bv, 'order'),
1853 link.get(av, 'order'))
1854 if r != 0: return r
1855 elif link.getkey():
1856 key = link.getkey()
1857 if dir == '+':
1858 r = cmp(link.get(av, key), link.get(bv, key))
1859 if r != 0: return r
1860 elif dir == '-':
1861 r = cmp(link.get(bv, key), link.get(av, key))
1862 if r != 0: return r
1863 else:
1864 if dir == '+':
1865 r = cmp(av, bv)
1866 if r != 0: return r
1867 elif dir == '-':
1868 r = cmp(bv, av)
1869 if r != 0: return r
1871 else:
1872 # all other types just compare
1873 if dir == '+':
1874 r = cmp(av, bv)
1875 elif dir == '-':
1876 r = cmp(bv, av)
1877 if r != 0: return r
1879 # end for dir, prop in sort, group:
1880 # if all else fails, compare the ids
1881 return cmp(a[0], b[0])
1883 l.sort(sortfun)
1884 return [i[0] for i in l]
1886 def count(self):
1887 '''Get the number of nodes in this class.
1889 If the returned integer is 'numnodes', the ids of all the nodes
1890 in this class run from 1 to numnodes, and numnodes+1 will be the
1891 id of the next node to be created in this class.
1892 '''
1893 return self.db.countnodes(self.classname)
1895 # Manipulating properties:
1897 def getprops(self, protected=1):
1898 '''Return a dictionary mapping property names to property objects.
1899 If the "protected" flag is true, we include protected properties -
1900 those which may not be modified.
1902 In addition to the actual properties on the node, these
1903 methods provide the "creation" and "activity" properties. If the
1904 "protected" flag is true, we include protected properties - those
1905 which may not be modified.
1906 '''
1907 d = self.properties.copy()
1908 if protected:
1909 d['id'] = String()
1910 d['creation'] = hyperdb.Date()
1911 d['activity'] = hyperdb.Date()
1912 d['creator'] = hyperdb.Link('user')
1913 return d
1915 def addprop(self, **properties):
1916 '''Add properties to this class.
1918 The keyword arguments in 'properties' must map names to property
1919 objects, or a TypeError is raised. None of the keys in 'properties'
1920 may collide with the names of existing properties, or a ValueError
1921 is raised before any properties have been added.
1922 '''
1923 for key in properties.keys():
1924 if self.properties.has_key(key):
1925 raise ValueError, key
1926 self.properties.update(properties)
1928 def index(self, nodeid):
1929 '''Add (or refresh) the node to search indexes
1930 '''
1931 # find all the String properties that have indexme
1932 for prop, propclass in self.getprops().items():
1933 if isinstance(propclass, String) and propclass.indexme:
1934 try:
1935 value = str(self.get(nodeid, prop))
1936 except IndexError:
1937 # node no longer exists - entry should be removed
1938 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1939 else:
1940 # and index them under (classname, nodeid, property)
1941 self.db.indexer.add_text((self.classname, nodeid, prop),
1942 value)
1944 #
1945 # Detector interface
1946 #
1947 def audit(self, event, detector):
1948 '''Register a detector
1949 '''
1950 l = self.auditors[event]
1951 if detector not in l:
1952 self.auditors[event].append(detector)
1954 def fireAuditors(self, action, nodeid, newvalues):
1955 '''Fire all registered auditors.
1956 '''
1957 for audit in self.auditors[action]:
1958 audit(self.db, self, nodeid, newvalues)
1960 def react(self, event, detector):
1961 '''Register a detector
1962 '''
1963 l = self.reactors[event]
1964 if detector not in l:
1965 self.reactors[event].append(detector)
1967 def fireReactors(self, action, nodeid, oldvalues):
1968 '''Fire all registered reactors.
1969 '''
1970 for react in self.reactors[action]:
1971 react(self.db, self, nodeid, oldvalues)
1973 class FileClass(Class, hyperdb.FileClass):
1974 '''This class defines a large chunk of data. To support this, it has a
1975 mandatory String property "content" which is typically saved off
1976 externally to the hyperdb.
1978 The default MIME type of this data is defined by the
1979 "default_mime_type" class attribute, which may be overridden by each
1980 node if the class defines a "type" String property.
1981 '''
1982 default_mime_type = 'text/plain'
1984 def create(self, **propvalues):
1985 ''' Snarf the "content" propvalue and store in a file
1986 '''
1987 # we need to fire the auditors now, or the content property won't
1988 # be in propvalues for the auditors to play with
1989 self.fireAuditors('create', None, propvalues)
1991 # now remove the content property so it's not stored in the db
1992 content = propvalues['content']
1993 del propvalues['content']
1995 # do the database create
1996 newid = Class.create_inner(self, **propvalues)
1998 # fire reactors
1999 self.fireReactors('create', newid, None)
2001 # store off the content as a file
2002 self.db.storefile(self.classname, newid, None, content)
2003 return newid
2005 def import_list(self, propnames, proplist):
2006 ''' Trap the "content" property...
2007 '''
2008 # dupe this list so we don't affect others
2009 propnames = propnames[:]
2011 # extract the "content" property from the proplist
2012 i = propnames.index('content')
2013 content = eval(proplist[i])
2014 del propnames[i]
2015 del proplist[i]
2017 # do the normal import
2018 newid = Class.import_list(self, propnames, proplist)
2020 # save off the "content" file
2021 self.db.storefile(self.classname, newid, None, content)
2022 return newid
2024 def get(self, nodeid, propname, default=_marker, cache=1):
2025 ''' Trap the content propname and get it from the file
2027 'cache' exists for backwards compatibility, and is not used.
2028 '''
2029 poss_msg = 'Possibly an access right configuration problem.'
2030 if propname == 'content':
2031 try:
2032 return self.db.getfile(self.classname, nodeid, None)
2033 except IOError, (strerror):
2034 # XXX by catching this we donot see an error in the log.
2035 return 'ERROR reading file: %s%s\n%s\n%s'%(
2036 self.classname, nodeid, poss_msg, strerror)
2037 if default is not _marker:
2038 return Class.get(self, nodeid, propname, default)
2039 else:
2040 return Class.get(self, nodeid, propname)
2042 def getprops(self, protected=1):
2043 ''' In addition to the actual properties on the node, these methods
2044 provide the "content" property. If the "protected" flag is true,
2045 we include protected properties - those which may not be
2046 modified.
2047 '''
2048 d = Class.getprops(self, protected=protected).copy()
2049 d['content'] = hyperdb.String()
2050 return d
2052 def index(self, nodeid):
2053 ''' Index the node in the search index.
2055 We want to index the content in addition to the normal String
2056 property indexing.
2057 '''
2058 # perform normal indexing
2059 Class.index(self, nodeid)
2061 # get the content to index
2062 content = self.get(nodeid, 'content')
2064 # figure the mime type
2065 if self.properties.has_key('type'):
2066 mime_type = self.get(nodeid, 'type')
2067 else:
2068 mime_type = self.default_mime_type
2070 # and index!
2071 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
2072 mime_type)
2074 # deviation from spec - was called ItemClass
2075 class IssueClass(Class, roundupdb.IssueClass):
2076 # Overridden methods:
2077 def __init__(self, db, classname, **properties):
2078 '''The newly-created class automatically includes the "messages",
2079 "files", "nosy", and "superseder" properties. If the 'properties'
2080 dictionary attempts to specify any of these properties or a
2081 "creation" or "activity" property, a ValueError is raised.
2082 '''
2083 if not properties.has_key('title'):
2084 properties['title'] = hyperdb.String(indexme='yes')
2085 if not properties.has_key('messages'):
2086 properties['messages'] = hyperdb.Multilink("msg")
2087 if not properties.has_key('files'):
2088 properties['files'] = hyperdb.Multilink("file")
2089 if not properties.has_key('nosy'):
2090 # note: journalling is turned off as it really just wastes
2091 # space. this behaviour may be overridden in an instance
2092 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2093 if not properties.has_key('superseder'):
2094 properties['superseder'] = hyperdb.Multilink(classname)
2095 Class.__init__(self, db, classname, **properties)
2097 #