c167c3cdd501a31fb5623b03ef5c6e619a47e9e7
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: back_anydbm.py,v 1.113 2003-03-17 22:03:04 kedder Exp $
19 '''
20 This module defines a backend that saves the hyperdatabase in a database
21 chosen by anydbm. It is guaranteed to always be available in python
22 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
23 serious bugs, and is not available)
24 '''
26 import whichdb, anydbm, os, marshal, re, weakref, string, copy
27 from roundup import hyperdb, date, password, roundupdb, security
28 from blobfiles import FileStorage
29 from sessions import Sessions, OneTimeKeys
30 from roundup.indexer import Indexer
31 from roundup.backends import locking
32 from roundup.hyperdb import String, Password, Date, Interval, Link, \
33 Multilink, DatabaseError, Boolean, Number, Node
34 from roundup.date import Range
36 #
37 # Now the database
38 #
39 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
40 '''A database for storing records containing flexible data types.
42 Transaction stuff TODO:
43 . check the timestamp of the class file and nuke the cache if it's
44 modified. Do some sort of conflict checking on the dirty stuff.
45 . perhaps detect write collisions (related to above)?
47 '''
48 def __init__(self, config, journaltag=None):
49 '''Open a hyperdatabase given a specifier to some storage.
51 The 'storagelocator' is obtained from config.DATABASE.
52 The meaning of 'storagelocator' depends on the particular
53 implementation of the hyperdatabase. It could be a file name,
54 a directory path, a socket descriptor for a connection to a
55 database over the network, etc.
57 The 'journaltag' is a token that will be attached to the journal
58 entries for any edits done on the database. If 'journaltag' is
59 None, the database is opened in read-only mode: the Class.create(),
60 Class.set(), Class.retire(), and Class.restore() methods are
61 disabled.
62 '''
63 self.config, self.journaltag = config, journaltag
64 self.dir = config.DATABASE
65 self.classes = {}
66 self.cache = {} # cache of nodes loaded or created
67 self.dirtynodes = {} # keep track of the dirty nodes by class
68 self.newnodes = {} # keep track of the new nodes by class
69 self.destroyednodes = {}# keep track of the destroyed nodes by class
70 self.transactions = []
71 self.indexer = Indexer(self.dir)
72 self.sessions = Sessions(self.config)
73 self.otks = OneTimeKeys(self.config)
74 self.security = security.Security(self)
75 # ensure files are group readable and writable
76 os.umask(0002)
78 # lock it
79 lockfilenm = os.path.join(self.dir, 'lock')
80 self.lockfile = locking.acquire_lock(lockfilenm)
81 self.lockfile.write(str(os.getpid()))
82 self.lockfile.flush()
84 def post_init(self):
85 ''' Called once the schema initialisation has finished.
86 '''
87 # reindex the db if necessary
88 if self.indexer.should_reindex():
89 self.reindex()
91 # figure the "curuserid"
92 if self.journaltag is None:
93 self.curuserid = None
94 elif self.journaltag == 'admin':
95 # admin user may not exist, but always has ID 1
96 self.curuserid = '1'
97 else:
98 self.curuserid = self.user.lookup(self.journaltag)
100 def reindex(self):
101 for klass in self.classes.values():
102 for nodeid in klass.list():
103 klass.index(nodeid)
104 self.indexer.save_index()
106 def __repr__(self):
107 return '<back_anydbm instance at %x>'%id(self)
109 #
110 # Classes
111 #
112 def __getattr__(self, classname):
113 '''A convenient way of calling self.getclass(classname).'''
114 if self.classes.has_key(classname):
115 if __debug__:
116 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
117 return self.classes[classname]
118 raise AttributeError, classname
120 def addclass(self, cl):
121 if __debug__:
122 print >>hyperdb.DEBUG, 'addclass', (self, cl)
123 cn = cl.classname
124 if self.classes.has_key(cn):
125 raise ValueError, cn
126 self.classes[cn] = cl
128 def getclasses(self):
129 '''Return a list of the names of all existing classes.'''
130 if __debug__:
131 print >>hyperdb.DEBUG, 'getclasses', (self,)
132 l = self.classes.keys()
133 l.sort()
134 return l
136 def getclass(self, classname):
137 '''Get the Class object representing a particular class.
139 If 'classname' is not a valid class name, a KeyError is raised.
140 '''
141 if __debug__:
142 print >>hyperdb.DEBUG, 'getclass', (self, classname)
143 try:
144 return self.classes[classname]
145 except KeyError:
146 raise KeyError, 'There is no class called "%s"'%classname
148 #
149 # Class DBs
150 #
151 def clear(self):
152 '''Delete all database contents
153 '''
154 if __debug__:
155 print >>hyperdb.DEBUG, 'clear', (self,)
156 for cn in self.classes.keys():
157 for dummy in 'nodes', 'journals':
158 path = os.path.join(self.dir, 'journals.%s'%cn)
159 if os.path.exists(path):
160 os.remove(path)
161 elif os.path.exists(path+'.db'): # dbm appends .db
162 os.remove(path+'.db')
164 def getclassdb(self, classname, mode='r'):
165 ''' grab a connection to the class db that will be used for
166 multiple actions
167 '''
168 if __debug__:
169 print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode)
170 return self.opendb('nodes.%s'%classname, mode)
172 def determine_db_type(self, path):
173 ''' determine which DB wrote the class file
174 '''
175 db_type = ''
176 if os.path.exists(path):
177 db_type = whichdb.whichdb(path)
178 if not db_type:
179 raise DatabaseError, "Couldn't identify database type"
180 elif os.path.exists(path+'.db'):
181 # if the path ends in '.db', it's a dbm database, whether
182 # anydbm says it's dbhash or not!
183 db_type = 'dbm'
184 return db_type
186 def opendb(self, name, mode):
187 '''Low-level database opener that gets around anydbm/dbm
188 eccentricities.
189 '''
190 if __debug__:
191 print >>hyperdb.DEBUG, 'opendb', (self, name, mode)
193 # figure the class db type
194 path = os.path.join(os.getcwd(), self.dir, name)
195 db_type = self.determine_db_type(path)
197 # new database? let anydbm pick the best dbm
198 if not db_type:
199 if __debug__:
200 print >>hyperdb.DEBUG, "opendb anydbm.open(%r, 'c')"%path
201 return anydbm.open(path, 'c')
203 # open the database with the correct module
204 try:
205 dbm = __import__(db_type)
206 except ImportError:
207 raise DatabaseError, \
208 "Couldn't open database - the required module '%s'"\
209 " is not available"%db_type
210 if __debug__:
211 print >>hyperdb.DEBUG, "opendb %r.open(%r, %r)"%(db_type, path,
212 mode)
213 return dbm.open(path, mode)
215 #
216 # Node IDs
217 #
218 def newid(self, classname):
219 ''' Generate a new id for the given class
220 '''
221 # open the ids DB - create if if doesn't exist
222 db = self.opendb('_ids', 'c')
223 if db.has_key(classname):
224 newid = db[classname] = str(int(db[classname]) + 1)
225 else:
226 # the count() bit is transitional - older dbs won't start at 1
227 newid = str(self.getclass(classname).count()+1)
228 db[classname] = newid
229 db.close()
230 return newid
232 def setid(self, classname, setid):
233 ''' Set the id counter: used during import of database
234 '''
235 # open the ids DB - create if if doesn't exist
236 db = self.opendb('_ids', 'c')
237 db[classname] = str(setid)
238 db.close()
240 #
241 # Nodes
242 #
243 def addnode(self, classname, nodeid, node):
244 ''' add the specified node to its class's db
245 '''
246 if __debug__:
247 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
249 # we'll be supplied these props if we're doing an import
250 if not node.has_key('creator'):
251 # add in the "calculated" properties (dupe so we don't affect
252 # calling code's node assumptions)
253 node = node.copy()
254 node['creator'] = self.curuserid
255 node['creation'] = node['activity'] = date.Date()
257 self.newnodes.setdefault(classname, {})[nodeid] = 1
258 self.cache.setdefault(classname, {})[nodeid] = node
259 self.savenode(classname, nodeid, node)
261 def setnode(self, classname, nodeid, node):
262 ''' change the specified node
263 '''
264 if __debug__:
265 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
266 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
268 # update the activity time (dupe so we don't affect
269 # calling code's node assumptions)
270 node = node.copy()
271 node['activity'] = date.Date()
273 # can't set without having already loaded the node
274 self.cache[classname][nodeid] = node
275 self.savenode(classname, nodeid, node)
277 def savenode(self, classname, nodeid, node):
278 ''' perform the saving of data specified by the set/addnode
279 '''
280 if __debug__:
281 print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node)
282 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
284 def getnode(self, classname, nodeid, db=None, cache=1):
285 ''' get a node from the database
286 '''
287 if __debug__:
288 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid, db)
289 if cache:
290 # try the cache
291 cache_dict = self.cache.setdefault(classname, {})
292 if cache_dict.has_key(nodeid):
293 if __debug__:
294 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
295 nodeid)
296 return cache_dict[nodeid]
298 if __debug__:
299 print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid)
301 # get from the database and save in the cache
302 if db is None:
303 db = self.getclassdb(classname)
304 if not db.has_key(nodeid):
305 # try the cache - might be a brand-new node
306 cache_dict = self.cache.setdefault(classname, {})
307 if cache_dict.has_key(nodeid):
308 if __debug__:
309 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
310 nodeid)
311 return cache_dict[nodeid]
312 raise IndexError, "no such %s %s"%(classname, nodeid)
314 # check the uncommitted, destroyed nodes
315 if (self.destroyednodes.has_key(classname) and
316 self.destroyednodes[classname].has_key(nodeid)):
317 raise IndexError, "no such %s %s"%(classname, nodeid)
319 # decode
320 res = marshal.loads(db[nodeid])
322 # reverse the serialisation
323 res = self.unserialise(classname, res)
325 # store off in the cache dict
326 if cache:
327 cache_dict[nodeid] = res
329 return res
331 def destroynode(self, classname, nodeid):
332 '''Remove a node from the database. Called exclusively by the
333 destroy() method on Class.
334 '''
335 if __debug__:
336 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
338 # remove from cache and newnodes if it's there
339 if (self.cache.has_key(classname) and
340 self.cache[classname].has_key(nodeid)):
341 del self.cache[classname][nodeid]
342 if (self.newnodes.has_key(classname) and
343 self.newnodes[classname].has_key(nodeid)):
344 del self.newnodes[classname][nodeid]
346 # see if there's any obvious commit actions that we should get rid of
347 for entry in self.transactions[:]:
348 if entry[1][:2] == (classname, nodeid):
349 self.transactions.remove(entry)
351 # add to the destroyednodes map
352 self.destroyednodes.setdefault(classname, {})[nodeid] = 1
354 # add the destroy commit action
355 self.transactions.append((self.doDestroyNode, (classname, nodeid)))
357 def serialise(self, classname, node):
358 '''Copy the node contents, converting non-marshallable data into
359 marshallable data.
360 '''
361 if __debug__:
362 print >>hyperdb.DEBUG, 'serialise', classname, node
363 properties = self.getclass(classname).getprops()
364 d = {}
365 for k, v in node.items():
366 # if the property doesn't exist, or is the "retired" flag then
367 # it won't be in the properties dict
368 if not properties.has_key(k):
369 d[k] = v
370 continue
372 # get the property spec
373 prop = properties[k]
375 if isinstance(prop, Password) and v is not None:
376 d[k] = str(v)
377 elif isinstance(prop, Date) and v is not None:
378 d[k] = v.serialise()
379 elif isinstance(prop, Interval) and v is not None:
380 d[k] = v.serialise()
381 else:
382 d[k] = v
383 return d
385 def unserialise(self, classname, node):
386 '''Decode the marshalled node data
387 '''
388 if __debug__:
389 print >>hyperdb.DEBUG, 'unserialise', classname, node
390 properties = self.getclass(classname).getprops()
391 d = {}
392 for k, v in node.items():
393 # if the property doesn't exist, or is the "retired" flag then
394 # it won't be in the properties dict
395 if not properties.has_key(k):
396 d[k] = v
397 continue
399 # get the property spec
400 prop = properties[k]
402 if isinstance(prop, Date) and v is not None:
403 d[k] = date.Date(v)
404 elif isinstance(prop, Interval) and v is not None:
405 d[k] = date.Interval(v)
406 elif isinstance(prop, Password) and v is not None:
407 p = password.Password()
408 p.unpack(v)
409 d[k] = p
410 else:
411 d[k] = v
412 return d
414 def hasnode(self, classname, nodeid, db=None):
415 ''' determine if the database has a given node
416 '''
417 if __debug__:
418 print >>hyperdb.DEBUG, 'hasnode', (self, classname, nodeid, db)
420 # try the cache
421 cache = self.cache.setdefault(classname, {})
422 if cache.has_key(nodeid):
423 if __debug__:
424 print >>hyperdb.TRACE, 'has %s %s cached'%(classname, nodeid)
425 return 1
426 if __debug__:
427 print >>hyperdb.TRACE, 'has %s %s'%(classname, nodeid)
429 # not in the cache - check the database
430 if db is None:
431 db = self.getclassdb(classname)
432 res = db.has_key(nodeid)
433 return res
435 def countnodes(self, classname, db=None):
436 if __debug__:
437 print >>hyperdb.DEBUG, 'countnodes', (self, classname, db)
439 count = 0
441 # include the uncommitted nodes
442 if self.newnodes.has_key(classname):
443 count += len(self.newnodes[classname])
444 if self.destroyednodes.has_key(classname):
445 count -= len(self.destroyednodes[classname])
447 # and count those in the DB
448 if db is None:
449 db = self.getclassdb(classname)
450 count = count + len(db.keys())
451 return count
454 #
455 # Files - special node properties
456 # inherited from FileStorage
458 #
459 # Journal
460 #
461 def addjournal(self, classname, nodeid, action, params, creator=None,
462 creation=None):
463 ''' Journal the Action
464 'action' may be:
466 'create' or 'set' -- 'params' is a dictionary of property values
467 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
468 'retire' -- 'params' is None
469 '''
470 if __debug__:
471 print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid,
472 action, params, creator, creation)
473 self.transactions.append((self.doSaveJournal, (classname, nodeid,
474 action, params, creator, creation)))
476 def getjournal(self, classname, nodeid):
477 ''' get the journal for id
479 Raise IndexError if the node doesn't exist (as per history()'s
480 API)
481 '''
482 if __debug__:
483 print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid)
484 # attempt to open the journal - in some rare cases, the journal may
485 # not exist
486 try:
487 db = self.opendb('journals.%s'%classname, 'r')
488 except anydbm.error, error:
489 if str(error) == "need 'c' or 'n' flag to open new db":
490 raise IndexError, 'no such %s %s'%(classname, nodeid)
491 elif error.args[0] != 2:
492 raise
493 raise IndexError, 'no such %s %s'%(classname, nodeid)
494 try:
495 journal = marshal.loads(db[nodeid])
496 except KeyError:
497 db.close()
498 raise IndexError, 'no such %s %s'%(classname, nodeid)
499 db.close()
500 res = []
501 for nodeid, date_stamp, user, action, params in journal:
502 res.append((nodeid, date.Date(date_stamp), user, action, params))
503 return res
505 def pack(self, pack_before):
506 ''' Delete all journal entries except "create" before 'pack_before'.
507 '''
508 if __debug__:
509 print >>hyperdb.DEBUG, 'packjournal', (self, pack_before)
511 pack_before = pack_before.serialise()
512 for classname in self.getclasses():
513 # get the journal db
514 db_name = 'journals.%s'%classname
515 path = os.path.join(os.getcwd(), self.dir, classname)
516 db_type = self.determine_db_type(path)
517 db = self.opendb(db_name, 'w')
519 for key in db.keys():
520 # get the journal for this db entry
521 journal = marshal.loads(db[key])
522 l = []
523 last_set_entry = None
524 for entry in journal:
525 # unpack the entry
526 (nodeid, date_stamp, self.journaltag, action,
527 params) = entry
528 # if the entry is after the pack date, _or_ the initial
529 # create entry, then it stays
530 if date_stamp > pack_before or action == 'create':
531 l.append(entry)
532 db[key] = marshal.dumps(l)
533 if db_type == 'gdbm':
534 db.reorganize()
535 db.close()
538 #
539 # Basic transaction support
540 #
541 def commit(self):
542 ''' Commit the current transactions.
543 '''
544 if __debug__:
545 print >>hyperdb.DEBUG, 'commit', (self,)
547 # keep a handle to all the database files opened
548 self.databases = {}
550 # now, do all the transactions
551 reindex = {}
552 for method, args in self.transactions:
553 reindex[method(*args)] = 1
555 # now close all the database files
556 for db in self.databases.values():
557 db.close()
558 del self.databases
560 # reindex the nodes that request it
561 for classname, nodeid in filter(None, reindex.keys()):
562 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
563 self.getclass(classname).index(nodeid)
565 # save the indexer state
566 self.indexer.save_index()
568 self.clearCache()
570 def clearCache(self):
571 # all transactions committed, back to normal
572 self.cache = {}
573 self.dirtynodes = {}
574 self.newnodes = {}
575 self.destroyednodes = {}
576 self.transactions = []
578 def getCachedClassDB(self, classname):
579 ''' get the class db, looking in our cache of databases for commit
580 '''
581 # get the database handle
582 db_name = 'nodes.%s'%classname
583 if not self.databases.has_key(db_name):
584 self.databases[db_name] = self.getclassdb(classname, 'c')
585 return self.databases[db_name]
587 def doSaveNode(self, classname, nodeid, node):
588 if __debug__:
589 print >>hyperdb.DEBUG, 'doSaveNode', (self, classname, nodeid,
590 node)
592 db = self.getCachedClassDB(classname)
594 # now save the marshalled data
595 db[nodeid] = marshal.dumps(self.serialise(classname, node))
597 # return the classname, nodeid so we reindex this content
598 return (classname, nodeid)
600 def getCachedJournalDB(self, classname):
601 ''' get the journal db, looking in our cache of databases for commit
602 '''
603 # get the database handle
604 db_name = 'journals.%s'%classname
605 if not self.databases.has_key(db_name):
606 self.databases[db_name] = self.opendb(db_name, 'c')
607 return self.databases[db_name]
609 def doSaveJournal(self, classname, nodeid, action, params, creator,
610 creation):
611 # serialise the parameters now if necessary
612 if isinstance(params, type({})):
613 if action in ('set', 'create'):
614 params = self.serialise(classname, params)
616 # handle supply of the special journalling parameters (usually
617 # supplied on importing an existing database)
618 if creator:
619 journaltag = creator
620 else:
621 journaltag = self.curuserid
622 if creation:
623 journaldate = creation.serialise()
624 else:
625 journaldate = date.Date().serialise()
627 # create the journal entry
628 entry = (nodeid, journaldate, journaltag, action, params)
630 if __debug__:
631 print >>hyperdb.DEBUG, 'doSaveJournal', entry
633 db = self.getCachedJournalDB(classname)
635 # now insert the journal entry
636 if db.has_key(nodeid):
637 # append to existing
638 s = db[nodeid]
639 l = marshal.loads(s)
640 l.append(entry)
641 else:
642 l = [entry]
644 db[nodeid] = marshal.dumps(l)
646 def doDestroyNode(self, classname, nodeid):
647 if __debug__:
648 print >>hyperdb.DEBUG, 'doDestroyNode', (self, classname, nodeid)
650 # delete from the class database
651 db = self.getCachedClassDB(classname)
652 if db.has_key(nodeid):
653 del db[nodeid]
655 # delete from the database
656 db = self.getCachedJournalDB(classname)
657 if db.has_key(nodeid):
658 del db[nodeid]
660 # return the classname, nodeid so we reindex this content
661 return (classname, nodeid)
663 def rollback(self):
664 ''' Reverse all actions from the current transaction.
665 '''
666 if __debug__:
667 print >>hyperdb.DEBUG, 'rollback', (self, )
668 for method, args in self.transactions:
669 # delete temporary files
670 if method == self.doStoreFile:
671 self.rollbackStoreFile(*args)
672 self.cache = {}
673 self.dirtynodes = {}
674 self.newnodes = {}
675 self.destroyednodes = {}
676 self.transactions = []
678 def close(self):
679 ''' Nothing to do
680 '''
681 if self.lockfile is not None:
682 locking.release_lock(self.lockfile)
683 if self.lockfile is not None:
684 self.lockfile.close()
685 self.lockfile = None
687 _marker = []
688 class Class(hyperdb.Class):
689 '''The handle to a particular class of nodes in a hyperdatabase.'''
691 def __init__(self, db, classname, **properties):
692 '''Create a new class with a given name and property specification.
694 'classname' must not collide with the name of an existing class,
695 or a ValueError is raised. The keyword arguments in 'properties'
696 must map names to property objects, or a TypeError is raised.
697 '''
698 if (properties.has_key('creation') or properties.has_key('activity')
699 or properties.has_key('creator')):
700 raise ValueError, '"creation", "activity" and "creator" are '\
701 'reserved'
703 self.classname = classname
704 self.properties = properties
705 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
706 self.key = ''
708 # should we journal changes (default yes)
709 self.do_journal = 1
711 # do the db-related init stuff
712 db.addclass(self)
714 self.auditors = {'create': [], 'set': [], 'retire': [], 'restore': []}
715 self.reactors = {'create': [], 'set': [], 'retire': [], 'restore': []}
717 def enableJournalling(self):
718 '''Turn journalling on for this class
719 '''
720 self.do_journal = 1
722 def disableJournalling(self):
723 '''Turn journalling off for this class
724 '''
725 self.do_journal = 0
727 # Editing nodes:
729 def create(self, **propvalues):
730 '''Create a new node of this class and return its id.
732 The keyword arguments in 'propvalues' map property names to values.
734 The values of arguments must be acceptable for the types of their
735 corresponding properties or a TypeError is raised.
737 If this class has a key property, it must be present and its value
738 must not collide with other key strings or a ValueError is raised.
740 Any other properties on this class that are missing from the
741 'propvalues' dictionary are set to None.
743 If an id in a link or multilink property does not refer to a valid
744 node, an IndexError is raised.
746 These operations trigger detectors and can be vetoed. Attempts
747 to modify the "creation" or "activity" properties cause a KeyError.
748 '''
749 self.fireAuditors('create', None, propvalues)
750 newid = self.create_inner(**propvalues)
751 self.fireReactors('create', newid, None)
752 return newid
754 def create_inner(self, **propvalues):
755 ''' Called by create, in-between the audit and react calls.
756 '''
757 if propvalues.has_key('id'):
758 raise KeyError, '"id" is reserved'
760 if self.db.journaltag is None:
761 raise DatabaseError, 'Database open read-only'
763 if propvalues.has_key('creation') or propvalues.has_key('activity'):
764 raise KeyError, '"creation" and "activity" are reserved'
765 # new node's id
766 newid = self.db.newid(self.classname)
768 # validate propvalues
769 num_re = re.compile('^\d+$')
770 for key, value in propvalues.items():
771 if key == self.key:
772 try:
773 self.lookup(value)
774 except KeyError:
775 pass
776 else:
777 raise ValueError, 'node with key "%s" exists'%value
779 # try to handle this property
780 try:
781 prop = self.properties[key]
782 except KeyError:
783 raise KeyError, '"%s" has no property "%s"'%(self.classname,
784 key)
786 if value is not None and isinstance(prop, Link):
787 if type(value) != type(''):
788 raise ValueError, 'link value must be String'
789 link_class = self.properties[key].classname
790 # if it isn't a number, it's a key
791 if not num_re.match(value):
792 try:
793 value = self.db.classes[link_class].lookup(value)
794 except (TypeError, KeyError):
795 raise IndexError, 'new property "%s": %s not a %s'%(
796 key, value, link_class)
797 elif not self.db.getclass(link_class).hasnode(value):
798 raise IndexError, '%s has no node %s'%(link_class, value)
800 # save off the value
801 propvalues[key] = value
803 # register the link with the newly linked node
804 if self.do_journal and self.properties[key].do_journal:
805 self.db.addjournal(link_class, value, 'link',
806 (self.classname, newid, key))
808 elif isinstance(prop, Multilink):
809 if type(value) != type([]):
810 raise TypeError, 'new property "%s" not a list of ids'%key
812 # clean up and validate the list of links
813 link_class = self.properties[key].classname
814 l = []
815 for entry in value:
816 if type(entry) != type(''):
817 raise ValueError, '"%s" multilink value (%r) '\
818 'must contain Strings'%(key, value)
819 # if it isn't a number, it's a key
820 if not num_re.match(entry):
821 try:
822 entry = self.db.classes[link_class].lookup(entry)
823 except (TypeError, KeyError):
824 raise IndexError, 'new property "%s": %s not a %s'%(
825 key, entry, self.properties[key].classname)
826 l.append(entry)
827 value = l
828 propvalues[key] = value
830 # handle additions
831 for nodeid in value:
832 if not self.db.getclass(link_class).hasnode(nodeid):
833 raise IndexError, '%s has no node %s'%(link_class,
834 nodeid)
835 # register the link with the newly linked node
836 if self.do_journal and self.properties[key].do_journal:
837 self.db.addjournal(link_class, nodeid, 'link',
838 (self.classname, newid, key))
840 elif isinstance(prop, String):
841 if type(value) != type('') and type(value) != type(u''):
842 raise TypeError, 'new property "%s" not a string'%key
844 elif isinstance(prop, Password):
845 if not isinstance(value, password.Password):
846 raise TypeError, 'new property "%s" not a Password'%key
848 elif isinstance(prop, Date):
849 if value is not None and not isinstance(value, date.Date):
850 raise TypeError, 'new property "%s" not a Date'%key
852 elif isinstance(prop, Interval):
853 if value is not None and not isinstance(value, date.Interval):
854 raise TypeError, 'new property "%s" not an Interval'%key
856 elif value is not None and isinstance(prop, Number):
857 try:
858 float(value)
859 except ValueError:
860 raise TypeError, 'new property "%s" not numeric'%key
862 elif value is not None and isinstance(prop, Boolean):
863 try:
864 int(value)
865 except ValueError:
866 raise TypeError, 'new property "%s" not boolean'%key
868 # make sure there's data where there needs to be
869 for key, prop in self.properties.items():
870 if propvalues.has_key(key):
871 continue
872 if key == self.key:
873 raise ValueError, 'key property "%s" is required'%key
874 if isinstance(prop, Multilink):
875 propvalues[key] = []
876 else:
877 propvalues[key] = None
879 # done
880 self.db.addnode(self.classname, newid, propvalues)
881 if self.do_journal:
882 self.db.addjournal(self.classname, newid, 'create', {})
884 return newid
886 def export_list(self, propnames, nodeid):
887 ''' Export a node - generate a list of CSV-able data in the order
888 specified by propnames for the given node.
889 '''
890 properties = self.getprops()
891 l = []
892 for prop in propnames:
893 proptype = properties[prop]
894 value = self.get(nodeid, prop)
895 # "marshal" data where needed
896 if value is None:
897 pass
898 elif isinstance(proptype, hyperdb.Date):
899 value = value.get_tuple()
900 elif isinstance(proptype, hyperdb.Interval):
901 value = value.get_tuple()
902 elif isinstance(proptype, hyperdb.Password):
903 value = str(value)
904 l.append(repr(value))
906 # append retired flag
907 l.append(self.is_retired(nodeid))
909 return l
911 def import_list(self, propnames, proplist):
912 ''' Import a node - all information including "id" is present and
913 should not be sanity checked. Triggers are not triggered. The
914 journal should be initialised using the "creator" and "created"
915 information.
917 Return the nodeid of the node imported.
918 '''
919 if self.db.journaltag is None:
920 raise DatabaseError, 'Database open read-only'
921 properties = self.getprops()
923 # make the new node's property map
924 d = {}
925 newid = None
926 for i in range(len(propnames)):
927 # Figure the property for this column
928 propname = propnames[i]
930 # Use eval to reverse the repr() used to output the CSV
931 value = eval(proplist[i])
933 # "unmarshal" where necessary
934 if propname == 'id':
935 newid = value
936 continue
937 elif propname == 'is retired':
938 # is the item retired?
939 if int(value):
940 d[self.db.RETIRED_FLAG] = 1
941 continue
942 elif value is None:
943 # don't set Nones
944 continue
946 prop = properties[propname]
947 if isinstance(prop, hyperdb.Date):
948 value = date.Date(value)
949 elif isinstance(prop, hyperdb.Interval):
950 value = date.Interval(value)
951 elif isinstance(prop, hyperdb.Password):
952 pwd = password.Password()
953 pwd.unpack(value)
954 value = pwd
955 d[propname] = value
957 # get a new id if necessary
958 if newid is None:
959 newid = self.db.newid(self.classname)
961 # add the node and journal
962 self.db.addnode(self.classname, newid, d)
964 # extract the journalling stuff and nuke it
965 if d.has_key('creator'):
966 creator = d['creator']
967 del d['creator']
968 else:
969 creator = None
970 if d.has_key('creation'):
971 creation = d['creation']
972 del d['creation']
973 else:
974 creation = None
975 if d.has_key('activity'):
976 del d['activity']
977 self.db.addjournal(self.classname, newid, 'create', {}, creator,
978 creation)
979 return newid
981 def get(self, nodeid, propname, default=_marker, cache=1):
982 '''Get the value of a property on an existing node of this class.
984 'nodeid' must be the id of an existing node of this class or an
985 IndexError is raised. 'propname' must be the name of a property
986 of this class or a KeyError is raised.
988 'cache' indicates whether the transaction cache should be queried
989 for the node. If the node has been modified and you need to
990 determine what its values prior to modification are, you need to
991 set cache=0.
993 Attempts to get the "creation" or "activity" properties should
994 do the right thing.
995 '''
996 if propname == 'id':
997 return nodeid
999 # get the node's dict
1000 d = self.db.getnode(self.classname, nodeid, cache=cache)
1002 # check for one of the special props
1003 if propname == 'creation':
1004 if d.has_key('creation'):
1005 return d['creation']
1006 if not self.do_journal:
1007 raise ValueError, 'Journalling is disabled for this class'
1008 journal = self.db.getjournal(self.classname, nodeid)
1009 if journal:
1010 return self.db.getjournal(self.classname, nodeid)[0][1]
1011 else:
1012 # on the strange chance that there's no journal
1013 return date.Date()
1014 if propname == 'activity':
1015 if d.has_key('activity'):
1016 return d['activity']
1017 if not self.do_journal:
1018 raise ValueError, 'Journalling is disabled for this class'
1019 journal = self.db.getjournal(self.classname, nodeid)
1020 if journal:
1021 return self.db.getjournal(self.classname, nodeid)[-1][1]
1022 else:
1023 # on the strange chance that there's no journal
1024 return date.Date()
1025 if propname == 'creator':
1026 if d.has_key('creator'):
1027 return d['creator']
1028 if not self.do_journal:
1029 raise ValueError, 'Journalling is disabled for this class'
1030 journal = self.db.getjournal(self.classname, nodeid)
1031 if journal:
1032 num_re = re.compile('^\d+$')
1033 value = self.db.getjournal(self.classname, nodeid)[0][2]
1034 if num_re.match(value):
1035 return value
1036 else:
1037 # old-style "username" journal tag
1038 try:
1039 return self.db.user.lookup(value)
1040 except KeyError:
1041 # user's been retired, return admin
1042 return '1'
1043 else:
1044 return self.db.curuserid
1046 # get the property (raises KeyErorr if invalid)
1047 prop = self.properties[propname]
1049 if not d.has_key(propname):
1050 if default is _marker:
1051 if isinstance(prop, Multilink):
1052 return []
1053 else:
1054 return None
1055 else:
1056 return default
1058 # return a dupe of the list so code doesn't get confused
1059 if isinstance(prop, Multilink):
1060 return d[propname][:]
1062 return d[propname]
1064 # not in spec
1065 def getnode(self, nodeid, cache=1):
1066 ''' Return a convenience wrapper for the node.
1068 'nodeid' must be the id of an existing node of this class or an
1069 IndexError is raised.
1071 'cache' indicates whether the transaction cache should be queried
1072 for the node. If the node has been modified and you need to
1073 determine what its values prior to modification are, you need to
1074 set cache=0.
1075 '''
1076 return Node(self, nodeid, cache=cache)
1078 def set(self, nodeid, **propvalues):
1079 '''Modify a property on an existing node of this class.
1081 'nodeid' must be the id of an existing node of this class or an
1082 IndexError is raised.
1084 Each key in 'propvalues' must be the name of a property of this
1085 class or a KeyError is raised.
1087 All values in 'propvalues' must be acceptable types for their
1088 corresponding properties or a TypeError is raised.
1090 If the value of the key property is set, it must not collide with
1091 other key strings or a ValueError is raised.
1093 If the value of a Link or Multilink property contains an invalid
1094 node id, a ValueError is raised.
1096 These operations trigger detectors and can be vetoed. Attempts
1097 to modify the "creation" or "activity" properties cause a KeyError.
1098 '''
1099 if not propvalues:
1100 return propvalues
1102 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1103 raise KeyError, '"creation" and "activity" are reserved'
1105 if propvalues.has_key('id'):
1106 raise KeyError, '"id" is reserved'
1108 if self.db.journaltag is None:
1109 raise DatabaseError, 'Database open read-only'
1111 self.fireAuditors('set', nodeid, propvalues)
1112 # Take a copy of the node dict so that the subsequent set
1113 # operation doesn't modify the oldvalues structure.
1114 try:
1115 # try not using the cache initially
1116 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid,
1117 cache=0))
1118 except IndexError:
1119 # this will be needed if somone does a create() and set()
1120 # with no intervening commit()
1121 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1123 node = self.db.getnode(self.classname, nodeid)
1124 if node.has_key(self.db.RETIRED_FLAG):
1125 raise IndexError
1126 num_re = re.compile('^\d+$')
1128 # if the journal value is to be different, store it in here
1129 journalvalues = {}
1131 for propname, value in propvalues.items():
1132 # check to make sure we're not duplicating an existing key
1133 if propname == self.key and node[propname] != value:
1134 try:
1135 self.lookup(value)
1136 except KeyError:
1137 pass
1138 else:
1139 raise ValueError, 'node with key "%s" exists'%value
1141 # this will raise the KeyError if the property isn't valid
1142 # ... we don't use getprops() here because we only care about
1143 # the writeable properties.
1144 try:
1145 prop = self.properties[propname]
1146 except KeyError:
1147 raise KeyError, '"%s" has no property named "%s"'%(
1148 self.classname, propname)
1150 # if the value's the same as the existing value, no sense in
1151 # doing anything
1152 current = node.get(propname, None)
1153 if value == current:
1154 del propvalues[propname]
1155 continue
1156 journalvalues[propname] = current
1158 # do stuff based on the prop type
1159 if isinstance(prop, Link):
1160 link_class = prop.classname
1161 # if it isn't a number, it's a key
1162 if value is not None and not isinstance(value, type('')):
1163 raise ValueError, 'property "%s" link value be a string'%(
1164 propname)
1165 if isinstance(value, type('')) and not num_re.match(value):
1166 try:
1167 value = self.db.classes[link_class].lookup(value)
1168 except (TypeError, KeyError):
1169 raise IndexError, 'new property "%s": %s not a %s'%(
1170 propname, value, prop.classname)
1172 if (value is not None and
1173 not self.db.getclass(link_class).hasnode(value)):
1174 raise IndexError, '%s has no node %s'%(link_class, value)
1176 if self.do_journal and prop.do_journal:
1177 # register the unlink with the old linked node
1178 if node.has_key(propname) and node[propname] is not None:
1179 self.db.addjournal(link_class, node[propname], 'unlink',
1180 (self.classname, nodeid, propname))
1182 # register the link with the newly linked node
1183 if value is not None:
1184 self.db.addjournal(link_class, value, 'link',
1185 (self.classname, nodeid, propname))
1187 elif isinstance(prop, Multilink):
1188 if type(value) != type([]):
1189 raise TypeError, 'new property "%s" not a list of'\
1190 ' ids'%propname
1191 link_class = self.properties[propname].classname
1192 l = []
1193 for entry in value:
1194 # if it isn't a number, it's a key
1195 if type(entry) != type(''):
1196 raise ValueError, 'new property "%s" link value ' \
1197 'must be a string'%propname
1198 if not num_re.match(entry):
1199 try:
1200 entry = self.db.classes[link_class].lookup(entry)
1201 except (TypeError, KeyError):
1202 raise IndexError, 'new property "%s": %s not a %s'%(
1203 propname, entry,
1204 self.properties[propname].classname)
1205 l.append(entry)
1206 value = l
1207 propvalues[propname] = value
1209 # figure the journal entry for this property
1210 add = []
1211 remove = []
1213 # handle removals
1214 if node.has_key(propname):
1215 l = node[propname]
1216 else:
1217 l = []
1218 for id in l[:]:
1219 if id in value:
1220 continue
1221 # register the unlink with the old linked node
1222 if self.do_journal and self.properties[propname].do_journal:
1223 self.db.addjournal(link_class, id, 'unlink',
1224 (self.classname, nodeid, propname))
1225 l.remove(id)
1226 remove.append(id)
1228 # handle additions
1229 for id in value:
1230 if not self.db.getclass(link_class).hasnode(id):
1231 raise IndexError, '%s has no node %s'%(link_class, id)
1232 if id in l:
1233 continue
1234 # register the link with the newly linked node
1235 if self.do_journal and self.properties[propname].do_journal:
1236 self.db.addjournal(link_class, id, 'link',
1237 (self.classname, nodeid, propname))
1238 l.append(id)
1239 add.append(id)
1241 # figure the journal entry
1242 l = []
1243 if add:
1244 l.append(('+', add))
1245 if remove:
1246 l.append(('-', remove))
1247 if l:
1248 journalvalues[propname] = tuple(l)
1250 elif isinstance(prop, String):
1251 if value is not None and type(value) != type('') and type(value) != type(u''):
1252 raise TypeError, 'new property "%s" not a string'%propname
1254 elif isinstance(prop, Password):
1255 if not isinstance(value, password.Password):
1256 raise TypeError, 'new property "%s" not a Password'%propname
1257 propvalues[propname] = value
1259 elif value is not None and isinstance(prop, Date):
1260 if not isinstance(value, date.Date):
1261 raise TypeError, 'new property "%s" not a Date'% propname
1262 propvalues[propname] = value
1264 elif value is not None and isinstance(prop, Interval):
1265 if not isinstance(value, date.Interval):
1266 raise TypeError, 'new property "%s" not an '\
1267 'Interval'%propname
1268 propvalues[propname] = value
1270 elif value is not None and isinstance(prop, Number):
1271 try:
1272 float(value)
1273 except ValueError:
1274 raise TypeError, 'new property "%s" not numeric'%propname
1276 elif value is not None and isinstance(prop, Boolean):
1277 try:
1278 int(value)
1279 except ValueError:
1280 raise TypeError, 'new property "%s" not boolean'%propname
1282 node[propname] = value
1284 # nothing to do?
1285 if not propvalues:
1286 return propvalues
1288 # do the set, and journal it
1289 self.db.setnode(self.classname, nodeid, node)
1291 if self.do_journal:
1292 self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
1294 self.fireReactors('set', nodeid, oldvalues)
1296 return propvalues
1298 def retire(self, nodeid):
1299 '''Retire a node.
1301 The properties on the node remain available from the get() method,
1302 and the node's id is never reused.
1304 Retired nodes are not returned by the find(), list(), or lookup()
1305 methods, and other nodes may reuse the values of their key properties.
1307 These operations trigger detectors and can be vetoed. Attempts
1308 to modify the "creation" or "activity" properties cause a KeyError.
1309 '''
1310 if self.db.journaltag is None:
1311 raise DatabaseError, 'Database open read-only'
1313 self.fireAuditors('retire', nodeid, None)
1315 node = self.db.getnode(self.classname, nodeid)
1316 node[self.db.RETIRED_FLAG] = 1
1317 self.db.setnode(self.classname, nodeid, node)
1318 if self.do_journal:
1319 self.db.addjournal(self.classname, nodeid, 'retired', None)
1321 self.fireReactors('retire', nodeid, None)
1323 def restore(self, nodeid):
1324 '''Restpre a retired node.
1326 Make node available for all operations like it was before retirement.
1327 '''
1328 if self.db.journaltag is None:
1329 raise DatabaseError, 'Database open read-only'
1331 node = self.db.getnode(self.classname, nodeid)
1332 # check if key property was overrided
1333 key = self.getkey()
1334 try:
1335 id = self.lookup(node[key])
1336 except KeyError:
1337 pass
1338 else:
1339 raise KeyError, "Key property (%s) of retired node clashes with \
1340 existing one (%s)" % (key, node[key])
1341 # Now we can safely restore node
1342 self.fireAuditors('restore', nodeid, None)
1343 del node[self.db.RETIRED_FLAG]
1344 self.db.setnode(self.classname, nodeid, node)
1345 if self.do_journal:
1346 self.db.addjournal(self.classname, nodeid, 'restored', None)
1348 self.fireReactors('restore', nodeid, None)
1350 def is_retired(self, nodeid, cldb=None):
1351 '''Return true if the node is retired.
1352 '''
1353 node = self.db.getnode(self.classname, nodeid, cldb)
1354 if node.has_key(self.db.RETIRED_FLAG):
1355 return 1
1356 return 0
1358 def destroy(self, nodeid):
1359 '''Destroy a node.
1361 WARNING: this method should never be used except in extremely rare
1362 situations where there could never be links to the node being
1363 deleted
1364 WARNING: use retire() instead
1365 WARNING: the properties of this node will not be available ever again
1366 WARNING: really, use retire() instead
1368 Well, I think that's enough warnings. This method exists mostly to
1369 support the session storage of the cgi interface.
1370 '''
1371 if self.db.journaltag is None:
1372 raise DatabaseError, 'Database open read-only'
1373 self.db.destroynode(self.classname, nodeid)
1375 def history(self, nodeid):
1376 '''Retrieve the journal of edits on a particular node.
1378 'nodeid' must be the id of an existing node of this class or an
1379 IndexError is raised.
1381 The returned list contains tuples of the form
1383 (nodeid, date, tag, action, params)
1385 'date' is a Timestamp object specifying the time of the change and
1386 'tag' is the journaltag specified when the database was opened.
1387 '''
1388 if not self.do_journal:
1389 raise ValueError, 'Journalling is disabled for this class'
1390 return self.db.getjournal(self.classname, nodeid)
1392 # Locating nodes:
1393 def hasnode(self, nodeid):
1394 '''Determine if the given nodeid actually exists
1395 '''
1396 return self.db.hasnode(self.classname, nodeid)
1398 def setkey(self, propname):
1399 '''Select a String property of this class to be the key property.
1401 'propname' must be the name of a String property of this class or
1402 None, or a TypeError is raised. The values of the key property on
1403 all existing nodes must be unique or a ValueError is raised. If the
1404 property doesn't exist, KeyError is raised.
1405 '''
1406 prop = self.getprops()[propname]
1407 if not isinstance(prop, String):
1408 raise TypeError, 'key properties must be String'
1409 self.key = propname
1411 def getkey(self):
1412 '''Return the name of the key property for this class or None.'''
1413 return self.key
1415 def labelprop(self, default_to_id=0):
1416 ''' Return the property name for a label for the given node.
1418 This method attempts to generate a consistent label for the node.
1419 It tries the following in order:
1420 1. key property
1421 2. "name" property
1422 3. "title" property
1423 4. first property from the sorted property name list
1424 '''
1425 k = self.getkey()
1426 if k:
1427 return k
1428 props = self.getprops()
1429 if props.has_key('name'):
1430 return 'name'
1431 elif props.has_key('title'):
1432 return 'title'
1433 if default_to_id:
1434 return 'id'
1435 props = props.keys()
1436 props.sort()
1437 return props[0]
1439 # TODO: set up a separate index db file for this? profile?
1440 def lookup(self, keyvalue):
1441 '''Locate a particular node by its key property and return its id.
1443 If this class has no key property, a TypeError is raised. If the
1444 'keyvalue' matches one of the values for the key property among
1445 the nodes in this class, the matching node's id is returned;
1446 otherwise a KeyError is raised.
1447 '''
1448 if not self.key:
1449 raise TypeError, 'No key property set for class %s'%self.classname
1450 cldb = self.db.getclassdb(self.classname)
1451 try:
1452 for nodeid in self.getnodeids(cldb):
1453 node = self.db.getnode(self.classname, nodeid, cldb)
1454 if node.has_key(self.db.RETIRED_FLAG):
1455 continue
1456 if node[self.key] == keyvalue:
1457 return nodeid
1458 finally:
1459 cldb.close()
1460 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1461 keyvalue, self.classname)
1463 # change from spec - allows multiple props to match
1464 def find(self, **propspec):
1465 '''Get the ids of nodes in this class which link to the given nodes.
1467 'propspec' consists of keyword args propname=nodeid or
1468 propname={nodeid:1, }
1469 'propname' must be the name of a property in this class, or a
1470 KeyError is raised. That property must be a Link or
1471 Multilink property, or a TypeError is raised.
1473 Any node in this class whose 'propname' property links to any of the
1474 nodeids will be returned. Used by the full text indexing, which knows
1475 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1476 issues:
1478 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1479 '''
1480 propspec = propspec.items()
1481 for propname, nodeids in propspec:
1482 # check the prop is OK
1483 prop = self.properties[propname]
1484 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1485 raise TypeError, "'%s' not a Link/Multilink property"%propname
1487 # ok, now do the find
1488 cldb = self.db.getclassdb(self.classname)
1489 l = []
1490 try:
1491 for id in self.getnodeids(db=cldb):
1492 node = self.db.getnode(self.classname, id, db=cldb)
1493 if node.has_key(self.db.RETIRED_FLAG):
1494 continue
1495 for propname, nodeids in propspec:
1496 # can't test if the node doesn't have this property
1497 if not node.has_key(propname):
1498 continue
1499 if type(nodeids) is type(''):
1500 nodeids = {nodeids:1}
1501 prop = self.properties[propname]
1502 value = node[propname]
1503 if isinstance(prop, Link) and nodeids.has_key(value):
1504 l.append(id)
1505 break
1506 elif isinstance(prop, Multilink):
1507 hit = 0
1508 for v in value:
1509 if nodeids.has_key(v):
1510 l.append(id)
1511 hit = 1
1512 break
1513 if hit:
1514 break
1515 finally:
1516 cldb.close()
1517 return l
1519 def stringFind(self, **requirements):
1520 '''Locate a particular node by matching a set of its String
1521 properties in a caseless search.
1523 If the property is not a String property, a TypeError is raised.
1525 The return is a list of the id of all nodes that match.
1526 '''
1527 for propname in requirements.keys():
1528 prop = self.properties[propname]
1529 if isinstance(not prop, String):
1530 raise TypeError, "'%s' not a String property"%propname
1531 requirements[propname] = requirements[propname].lower()
1532 l = []
1533 cldb = self.db.getclassdb(self.classname)
1534 try:
1535 for nodeid in self.getnodeids(cldb):
1536 node = self.db.getnode(self.classname, nodeid, cldb)
1537 if node.has_key(self.db.RETIRED_FLAG):
1538 continue
1539 for key, value in requirements.items():
1540 if not node.has_key(key):
1541 break
1542 if node[key] is None or node[key].lower() != value:
1543 break
1544 else:
1545 l.append(nodeid)
1546 finally:
1547 cldb.close()
1548 return l
1550 def list(self):
1551 ''' Return a list of the ids of the active nodes in this class.
1552 '''
1553 l = []
1554 cn = self.classname
1555 cldb = self.db.getclassdb(cn)
1556 try:
1557 for nodeid in self.getnodeids(cldb):
1558 node = self.db.getnode(cn, nodeid, cldb)
1559 if node.has_key(self.db.RETIRED_FLAG):
1560 continue
1561 l.append(nodeid)
1562 finally:
1563 cldb.close()
1564 l.sort()
1565 return l
1567 def getnodeids(self, db=None):
1568 ''' Return a list of ALL nodeids
1569 '''
1570 if __debug__:
1571 print >>hyperdb.DEBUG, 'getnodeids', (self, self.classname, db)
1573 res = []
1575 # start off with the new nodes
1576 if self.db.newnodes.has_key(self.classname):
1577 res += self.db.newnodes[self.classname].keys()
1579 if db is None:
1580 db = self.db.getclassdb(self.classname)
1581 res = res + db.keys()
1583 # remove the uncommitted, destroyed nodes
1584 if self.db.destroyednodes.has_key(self.classname):
1585 for nodeid in self.db.destroyednodes[self.classname].keys():
1586 if db.has_key(nodeid):
1587 res.remove(nodeid)
1589 return res
1591 def filter(self, search_matches, filterspec, sort=(None,None),
1592 group=(None,None), num_re = re.compile('^\d+$')):
1593 ''' Return a list of the ids of the active nodes in this class that
1594 match the 'filter' spec, sorted by the group spec and then the
1595 sort spec.
1597 "filterspec" is {propname: value(s)}
1598 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1599 and prop is a prop name or None
1600 "search_matches" is {nodeid: marker}
1602 The filter must match all properties specificed - but if the
1603 property value to match is a list, any one of the values in the
1604 list may match for that property to match.
1605 '''
1606 cn = self.classname
1608 # optimise filterspec
1609 l = []
1610 props = self.getprops()
1611 LINK = 0
1612 MULTILINK = 1
1613 STRING = 2
1614 DATE = 3
1615 OTHER = 6
1617 timezone = self.db.getUserTimezone()
1618 for k, v in filterspec.items():
1619 propclass = props[k]
1620 if isinstance(propclass, Link):
1621 if type(v) is not type([]):
1622 v = [v]
1623 # replace key values with node ids
1624 u = []
1625 link_class = self.db.classes[propclass.classname]
1626 for entry in v:
1627 if entry == '-1': entry = None
1628 elif not num_re.match(entry):
1629 try:
1630 entry = link_class.lookup(entry)
1631 except (TypeError,KeyError):
1632 raise ValueError, 'property "%s": %s not a %s'%(
1633 k, entry, self.properties[k].classname)
1634 u.append(entry)
1636 l.append((LINK, k, u))
1637 elif isinstance(propclass, Multilink):
1638 if type(v) is not type([]):
1639 v = [v]
1640 # replace key values with node ids
1641 u = []
1642 link_class = self.db.classes[propclass.classname]
1643 for entry in v:
1644 if not num_re.match(entry):
1645 try:
1646 entry = link_class.lookup(entry)
1647 except (TypeError,KeyError):
1648 raise ValueError, 'new property "%s": %s not a %s'%(
1649 k, entry, self.properties[k].classname)
1650 u.append(entry)
1651 l.append((MULTILINK, k, u))
1652 elif isinstance(propclass, String) and k != 'id':
1653 # simple glob searching
1654 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1655 v = v.replace('?', '.')
1656 v = v.replace('*', '.*?')
1657 l.append((STRING, k, re.compile(v, re.I)))
1658 elif isinstance(propclass, Date):
1659 try:
1660 date_rng = Range(v, date.Date, offset=timezone)
1661 l.append((DATE, k, date_rng))
1662 except ValueError:
1663 # If range creation fails - ignore that search parameter
1664 pass
1665 elif isinstance(propclass, Boolean):
1666 if type(v) is type(''):
1667 bv = v.lower() in ('yes', 'true', 'on', '1')
1668 else:
1669 bv = v
1670 l.append((OTHER, k, bv))
1671 # kedder: dates are filtered by ranges
1672 #elif isinstance(propclass, Date):
1673 # l.append((OTHER, k, date.Date(v)))
1674 elif isinstance(propclass, Interval):
1675 l.append((OTHER, k, date.Interval(v)))
1676 elif isinstance(propclass, Number):
1677 l.append((OTHER, k, int(v)))
1678 else:
1679 l.append((OTHER, k, v))
1680 filterspec = l
1682 # now, find all the nodes that are active and pass filtering
1683 l = []
1684 cldb = self.db.getclassdb(cn)
1685 try:
1686 # TODO: only full-scan once (use items())
1687 for nodeid in self.getnodeids(cldb):
1688 node = self.db.getnode(cn, nodeid, cldb)
1689 if node.has_key(self.db.RETIRED_FLAG):
1690 continue
1691 # apply filter
1692 for t, k, v in filterspec:
1693 # handle the id prop
1694 if k == 'id' and v == nodeid:
1695 continue
1697 # make sure the node has the property
1698 if not node.has_key(k):
1699 # this node doesn't have this property, so reject it
1700 break
1702 # now apply the property filter
1703 if t == LINK:
1704 # link - if this node's property doesn't appear in the
1705 # filterspec's nodeid list, skip it
1706 if node[k] not in v:
1707 break
1708 elif t == MULTILINK:
1709 # multilink - if any of the nodeids required by the
1710 # filterspec aren't in this node's property, then skip
1711 # it
1712 have = node[k]
1713 for want in v:
1714 if want not in have:
1715 break
1716 else:
1717 continue
1718 break
1719 elif t == STRING:
1720 # RE search
1721 if node[k] is None or not v.search(node[k]):
1722 break
1723 elif t == DATE:
1724 if node[k] is None: break
1725 if v.to_value:
1726 if not (v.from_value < node[k] and v.to_value > node[k]):
1727 break
1728 else:
1729 if not (v.from_value < node[k]):
1730 break
1731 elif t == OTHER:
1732 # straight value comparison for the other types
1733 if node[k] != v:
1734 break
1735 else:
1736 l.append((nodeid, node))
1737 finally:
1738 cldb.close()
1739 l.sort()
1741 # filter based on full text search
1742 if search_matches is not None:
1743 k = []
1744 for v in l:
1745 if search_matches.has_key(v[0]):
1746 k.append(v)
1747 l = k
1749 # now, sort the result
1750 def sortfun(a, b, sort=sort, group=group, properties=self.getprops(),
1751 db = self.db, cl=self):
1752 a_id, an = a
1753 b_id, bn = b
1754 # sort by group and then sort
1755 for dir, prop in group, sort:
1756 if dir is None or prop is None: continue
1758 # sorting is class-specific
1759 propclass = properties[prop]
1761 # handle the properties that might be "faked"
1762 # also, handle possible missing properties
1763 try:
1764 if not an.has_key(prop):
1765 an[prop] = cl.get(a_id, prop)
1766 av = an[prop]
1767 except KeyError:
1768 # the node doesn't have a value for this property
1769 if isinstance(propclass, Multilink): av = []
1770 else: av = ''
1771 try:
1772 if not bn.has_key(prop):
1773 bn[prop] = cl.get(b_id, prop)
1774 bv = bn[prop]
1775 except KeyError:
1776 # the node doesn't have a value for this property
1777 if isinstance(propclass, Multilink): bv = []
1778 else: bv = ''
1780 # String and Date values are sorted in the natural way
1781 if isinstance(propclass, String):
1782 # clean up the strings
1783 if av and av[0] in string.uppercase:
1784 av = av.lower()
1785 if bv and bv[0] in string.uppercase:
1786 bv = bv.lower()
1787 if (isinstance(propclass, String) or
1788 isinstance(propclass, Date)):
1789 # it might be a string that's really an integer
1790 try:
1791 av = int(av)
1792 bv = int(bv)
1793 except:
1794 pass
1795 if dir == '+':
1796 r = cmp(av, bv)
1797 if r != 0: return r
1798 elif dir == '-':
1799 r = cmp(bv, av)
1800 if r != 0: return r
1802 # Link properties are sorted according to the value of
1803 # the "order" property on the linked nodes if it is
1804 # present; or otherwise on the key string of the linked
1805 # nodes; or finally on the node ids.
1806 elif isinstance(propclass, Link):
1807 link = db.classes[propclass.classname]
1808 if av is None and bv is not None: return -1
1809 if av is not None and bv is None: return 1
1810 if av is None and bv is None: continue
1811 if link.getprops().has_key('order'):
1812 if dir == '+':
1813 r = cmp(link.get(av, 'order'),
1814 link.get(bv, 'order'))
1815 if r != 0: return r
1816 elif dir == '-':
1817 r = cmp(link.get(bv, 'order'),
1818 link.get(av, 'order'))
1819 if r != 0: return r
1820 elif link.getkey():
1821 key = link.getkey()
1822 if dir == '+':
1823 r = cmp(link.get(av, key), link.get(bv, key))
1824 if r != 0: return r
1825 elif dir == '-':
1826 r = cmp(link.get(bv, key), link.get(av, key))
1827 if r != 0: return r
1828 else:
1829 if dir == '+':
1830 r = cmp(av, bv)
1831 if r != 0: return r
1832 elif dir == '-':
1833 r = cmp(bv, av)
1834 if r != 0: return r
1836 # Multilink properties are sorted according to how many
1837 # links are present.
1838 elif isinstance(propclass, Multilink):
1839 r = cmp(len(av), len(bv))
1840 if r == 0:
1841 # Compare contents of multilink property if lenghts is
1842 # equal
1843 r = cmp ('.'.join(av), '.'.join(bv))
1844 if r:
1845 if dir == '+':
1846 return r
1847 else:
1848 return -r
1850 else:
1851 # all other types just compare
1852 if dir == '+':
1853 r = cmp(av, bv)
1854 elif dir == '-':
1855 r = cmp(bv, av)
1856 if r != 0: return r
1858 # end for dir, prop in sort, group:
1859 # if all else fails, compare the ids
1860 return cmp(a[0], b[0])
1862 l.sort(sortfun)
1863 return [i[0] for i in l]
1865 def count(self):
1866 '''Get the number of nodes in this class.
1868 If the returned integer is 'numnodes', the ids of all the nodes
1869 in this class run from 1 to numnodes, and numnodes+1 will be the
1870 id of the next node to be created in this class.
1871 '''
1872 return self.db.countnodes(self.classname)
1874 # Manipulating properties:
1876 def getprops(self, protected=1):
1877 '''Return a dictionary mapping property names to property objects.
1878 If the "protected" flag is true, we include protected properties -
1879 those which may not be modified.
1881 In addition to the actual properties on the node, these
1882 methods provide the "creation" and "activity" properties. If the
1883 "protected" flag is true, we include protected properties - those
1884 which may not be modified.
1885 '''
1886 d = self.properties.copy()
1887 if protected:
1888 d['id'] = String()
1889 d['creation'] = hyperdb.Date()
1890 d['activity'] = hyperdb.Date()
1891 d['creator'] = hyperdb.Link('user')
1892 return d
1894 def addprop(self, **properties):
1895 '''Add properties to this class.
1897 The keyword arguments in 'properties' must map names to property
1898 objects, or a TypeError is raised. None of the keys in 'properties'
1899 may collide with the names of existing properties, or a ValueError
1900 is raised before any properties have been added.
1901 '''
1902 for key in properties.keys():
1903 if self.properties.has_key(key):
1904 raise ValueError, key
1905 self.properties.update(properties)
1907 def index(self, nodeid):
1908 '''Add (or refresh) the node to search indexes
1909 '''
1910 # find all the String properties that have indexme
1911 for prop, propclass in self.getprops().items():
1912 if isinstance(propclass, String) and propclass.indexme:
1913 try:
1914 value = str(self.get(nodeid, prop))
1915 except IndexError:
1916 # node no longer exists - entry should be removed
1917 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1918 else:
1919 # and index them under (classname, nodeid, property)
1920 self.db.indexer.add_text((self.classname, nodeid, prop),
1921 value)
1923 #
1924 # Detector interface
1925 #
1926 def audit(self, event, detector):
1927 '''Register a detector
1928 '''
1929 l = self.auditors[event]
1930 if detector not in l:
1931 self.auditors[event].append(detector)
1933 def fireAuditors(self, action, nodeid, newvalues):
1934 '''Fire all registered auditors.
1935 '''
1936 for audit in self.auditors[action]:
1937 audit(self.db, self, nodeid, newvalues)
1939 def react(self, event, detector):
1940 '''Register a detector
1941 '''
1942 l = self.reactors[event]
1943 if detector not in l:
1944 self.reactors[event].append(detector)
1946 def fireReactors(self, action, nodeid, oldvalues):
1947 '''Fire all registered reactors.
1948 '''
1949 for react in self.reactors[action]:
1950 react(self.db, self, nodeid, oldvalues)
1952 class FileClass(Class, hyperdb.FileClass):
1953 '''This class defines a large chunk of data. To support this, it has a
1954 mandatory String property "content" which is typically saved off
1955 externally to the hyperdb.
1957 The default MIME type of this data is defined by the
1958 "default_mime_type" class attribute, which may be overridden by each
1959 node if the class defines a "type" String property.
1960 '''
1961 default_mime_type = 'text/plain'
1963 def create(self, **propvalues):
1964 ''' Snarf the "content" propvalue and store in a file
1965 '''
1966 # we need to fire the auditors now, or the content property won't
1967 # be in propvalues for the auditors to play with
1968 self.fireAuditors('create', None, propvalues)
1970 # now remove the content property so it's not stored in the db
1971 content = propvalues['content']
1972 del propvalues['content']
1974 # do the database create
1975 newid = Class.create_inner(self, **propvalues)
1977 # fire reactors
1978 self.fireReactors('create', newid, None)
1980 # store off the content as a file
1981 self.db.storefile(self.classname, newid, None, content)
1982 return newid
1984 def import_list(self, propnames, proplist):
1985 ''' Trap the "content" property...
1986 '''
1987 # dupe this list so we don't affect others
1988 propnames = propnames[:]
1990 # extract the "content" property from the proplist
1991 i = propnames.index('content')
1992 content = eval(proplist[i])
1993 del propnames[i]
1994 del proplist[i]
1996 # do the normal import
1997 newid = Class.import_list(self, propnames, proplist)
1999 # save off the "content" file
2000 self.db.storefile(self.classname, newid, None, content)
2001 return newid
2003 def get(self, nodeid, propname, default=_marker, cache=1):
2004 ''' trap the content propname and get it from the file
2005 '''
2006 poss_msg = 'Possibly an access right configuration problem.'
2007 if propname == 'content':
2008 try:
2009 return self.db.getfile(self.classname, nodeid, None)
2010 except IOError, (strerror):
2011 # XXX by catching this we donot see an error in the log.
2012 return 'ERROR reading file: %s%s\n%s\n%s'%(
2013 self.classname, nodeid, poss_msg, strerror)
2014 if default is not _marker:
2015 return Class.get(self, nodeid, propname, default, cache=cache)
2016 else:
2017 return Class.get(self, nodeid, propname, cache=cache)
2019 def getprops(self, protected=1):
2020 ''' In addition to the actual properties on the node, these methods
2021 provide the "content" property. If the "protected" flag is true,
2022 we include protected properties - those which may not be
2023 modified.
2024 '''
2025 d = Class.getprops(self, protected=protected).copy()
2026 d['content'] = hyperdb.String()
2027 return d
2029 def index(self, nodeid):
2030 ''' Index the node in the search index.
2032 We want to index the content in addition to the normal String
2033 property indexing.
2034 '''
2035 # perform normal indexing
2036 Class.index(self, nodeid)
2038 # get the content to index
2039 content = self.get(nodeid, 'content')
2041 # figure the mime type
2042 if self.properties.has_key('type'):
2043 mime_type = self.get(nodeid, 'type')
2044 else:
2045 mime_type = self.default_mime_type
2047 # and index!
2048 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
2049 mime_type)
2051 # deviation from spec - was called ItemClass
2052 class IssueClass(Class, roundupdb.IssueClass):
2053 # Overridden methods:
2054 def __init__(self, db, classname, **properties):
2055 '''The newly-created class automatically includes the "messages",
2056 "files", "nosy", and "superseder" properties. If the 'properties'
2057 dictionary attempts to specify any of these properties or a
2058 "creation" or "activity" property, a ValueError is raised.
2059 '''
2060 if not properties.has_key('title'):
2061 properties['title'] = hyperdb.String(indexme='yes')
2062 if not properties.has_key('messages'):
2063 properties['messages'] = hyperdb.Multilink("msg")
2064 if not properties.has_key('files'):
2065 properties['files'] = hyperdb.Multilink("file")
2066 if not properties.has_key('nosy'):
2067 # note: journalling is turned off as it really just wastes
2068 # space. this behaviour may be overridden in an instance
2069 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2070 if not properties.has_key('superseder'):
2071 properties['superseder'] = hyperdb.Multilink(classname)
2072 Class.__init__(self, db, classname, **properties)
2074 #