1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: back_anydbm.py,v 1.211 2008-08-07 05:53:14 richard Exp $
19 '''This module defines a backend that saves the hyperdatabase in a
20 database chosen by anydbm. It is guaranteed to always be available in python
21 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
22 serious bugs, and is not available)
23 '''
24 __docformat__ = 'restructuredtext'
26 try:
27 import anydbm, sys
28 # dumbdbm only works in python 2.1.2+
29 if sys.version_info < (2,1,2):
30 import dumbdbm
31 assert anydbm._defaultmod != dumbdbm
32 del dumbdbm
33 except AssertionError:
34 print "WARNING: you should upgrade to python 2.1.3"
36 import whichdb, os, marshal, re, weakref, string, copy, time, shutil, logging
38 from roundup import hyperdb, date, password, roundupdb, security, support
39 from roundup.support import reversed
40 from roundup.backends import locking
41 from roundup.i18n import _
43 from blobfiles import FileStorage
44 from sessions_dbm import Sessions, OneTimeKeys
46 try:
47 from indexer_xapian import Indexer
48 except ImportError:
49 from indexer_dbm import Indexer
51 def db_exists(config):
52 # check for the user db
53 for db in 'nodes.user nodes.user.db'.split():
54 if os.path.exists(os.path.join(config.DATABASE, db)):
55 return 1
56 return 0
58 def db_nuke(config):
59 shutil.rmtree(config.DATABASE)
61 #
62 # Now the database
63 #
64 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
65 '''A database for storing records containing flexible data types.
67 Transaction stuff TODO:
69 - check the timestamp of the class file and nuke the cache if it's
70 modified. Do some sort of conflict checking on the dirty stuff.
71 - perhaps detect write collisions (related to above)?
72 '''
73 def __init__(self, config, journaltag=None):
74 '''Open a hyperdatabase given a specifier to some storage.
76 The 'storagelocator' is obtained from config.DATABASE.
77 The meaning of 'storagelocator' depends on the particular
78 implementation of the hyperdatabase. It could be a file name,
79 a directory path, a socket descriptor for a connection to a
80 database over the network, etc.
82 The 'journaltag' is a token that will be attached to the journal
83 entries for any edits done on the database. If 'journaltag' is
84 None, the database is opened in read-only mode: the Class.create(),
85 Class.set(), Class.retire(), and Class.restore() methods are
86 disabled.
87 '''
88 FileStorage.__init__(self, config.UMASK)
89 self.config, self.journaltag = config, journaltag
90 self.dir = config.DATABASE
91 self.classes = {}
92 self.cache = {} # cache of nodes loaded or created
93 self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0,
94 'filtering': 0}
95 self.dirtynodes = {} # keep track of the dirty nodes by class
96 self.newnodes = {} # keep track of the new nodes by class
97 self.destroyednodes = {}# keep track of the destroyed nodes by class
98 self.transactions = []
99 self.indexer = Indexer(self)
100 self.security = security.Security(self)
101 os.umask(config.UMASK)
103 # lock it
104 lockfilenm = os.path.join(self.dir, 'lock')
105 self.lockfile = locking.acquire_lock(lockfilenm)
106 self.lockfile.write(str(os.getpid()))
107 self.lockfile.flush()
109 def post_init(self):
110 '''Called once the schema initialisation has finished.
111 '''
112 # reindex the db if necessary
113 if self.indexer.should_reindex():
114 self.reindex()
116 def refresh_database(self):
117 """Rebuild the database
118 """
119 self.reindex()
121 def getSessionManager(self):
122 return Sessions(self)
124 def getOTKManager(self):
125 return OneTimeKeys(self)
127 def reindex(self, classname=None, show_progress=False):
128 if classname:
129 classes = [self.getclass(classname)]
130 else:
131 classes = self.classes.values()
132 for klass in classes:
133 if show_progress:
134 for nodeid in support.Progress('Reindex %s'%klass.classname,
135 klass.list()):
136 klass.index(nodeid)
137 else:
138 for nodeid in klass.list():
139 klass.index(nodeid)
140 self.indexer.save_index()
142 def __repr__(self):
143 return '<back_anydbm instance at %x>'%id(self)
145 #
146 # Classes
147 #
148 def __getattr__(self, classname):
149 '''A convenient way of calling self.getclass(classname).'''
150 if self.classes.has_key(classname):
151 return self.classes[classname]
152 raise AttributeError, classname
154 def addclass(self, cl):
155 cn = cl.classname
156 if self.classes.has_key(cn):
157 raise ValueError, cn
158 self.classes[cn] = cl
160 # add default Edit and View permissions
161 self.security.addPermission(name="Create", klass=cn,
162 description="User is allowed to create "+cn)
163 self.security.addPermission(name="Edit", klass=cn,
164 description="User is allowed to edit "+cn)
165 self.security.addPermission(name="View", klass=cn,
166 description="User is allowed to access "+cn)
168 def getclasses(self):
169 '''Return a list of the names of all existing classes.'''
170 l = self.classes.keys()
171 l.sort()
172 return l
174 def getclass(self, classname):
175 '''Get the Class object representing a particular class.
177 If 'classname' is not a valid class name, a KeyError is raised.
178 '''
179 try:
180 return self.classes[classname]
181 except KeyError:
182 raise KeyError, 'There is no class called "%s"'%classname
184 #
185 # Class DBs
186 #
187 def clear(self):
188 '''Delete all database contents
189 '''
190 logging.getLogger('hyperdb').info('clear')
191 for cn in self.classes.keys():
192 for dummy in 'nodes', 'journals':
193 path = os.path.join(self.dir, 'journals.%s'%cn)
194 if os.path.exists(path):
195 os.remove(path)
196 elif os.path.exists(path+'.db'): # dbm appends .db
197 os.remove(path+'.db')
198 # reset id sequences
199 path = os.path.join(os.getcwd(), self.dir, '_ids')
200 if os.path.exists(path):
201 os.remove(path)
202 elif os.path.exists(path+'.db'): # dbm appends .db
203 os.remove(path+'.db')
205 def getclassdb(self, classname, mode='r'):
206 ''' grab a connection to the class db that will be used for
207 multiple actions
208 '''
209 return self.opendb('nodes.%s'%classname, mode)
211 def determine_db_type(self, path):
212 ''' determine which DB wrote the class file
213 '''
214 db_type = ''
215 if os.path.exists(path):
216 db_type = whichdb.whichdb(path)
217 if not db_type:
218 raise hyperdb.DatabaseError, \
219 _("Couldn't identify database type")
220 elif os.path.exists(path+'.db'):
221 # if the path ends in '.db', it's a dbm database, whether
222 # anydbm says it's dbhash or not!
223 db_type = 'dbm'
224 return db_type
226 def opendb(self, name, mode):
227 '''Low-level database opener that gets around anydbm/dbm
228 eccentricities.
229 '''
230 # figure the class db type
231 path = os.path.join(os.getcwd(), self.dir, name)
232 db_type = self.determine_db_type(path)
234 # new database? let anydbm pick the best dbm
235 if not db_type:
236 if __debug__:
237 logging.getLogger('hyperdb').debug("opendb anydbm.open(%r, 'c')"%path)
238 return anydbm.open(path, 'c')
240 # open the database with the correct module
241 try:
242 dbm = __import__(db_type)
243 except ImportError:
244 raise hyperdb.DatabaseError, \
245 _("Couldn't open database - the required module '%s'"\
246 " is not available")%db_type
247 if __debug__:
248 logging.getLogger('hyperdb').debug("opendb %r.open(%r, %r)"%(db_type, path,
249 mode))
250 return dbm.open(path, mode)
252 #
253 # Node IDs
254 #
255 def newid(self, classname):
256 ''' Generate a new id for the given class
257 '''
258 # open the ids DB - create if if doesn't exist
259 db = self.opendb('_ids', 'c')
260 if db.has_key(classname):
261 newid = db[classname] = str(int(db[classname]) + 1)
262 else:
263 # the count() bit is transitional - older dbs won't start at 1
264 newid = str(self.getclass(classname).count()+1)
265 db[classname] = newid
266 db.close()
267 return newid
269 def setid(self, classname, setid):
270 ''' Set the id counter: used during import of database
271 '''
272 # open the ids DB - create if if doesn't exist
273 db = self.opendb('_ids', 'c')
274 db[classname] = str(setid)
275 db.close()
277 #
278 # Nodes
279 #
280 def addnode(self, classname, nodeid, node):
281 ''' add the specified node to its class's db
282 '''
283 # we'll be supplied these props if we're doing an import
284 if not node.has_key('creator'):
285 # add in the "calculated" properties (dupe so we don't affect
286 # calling code's node assumptions)
287 node = node.copy()
288 node['creator'] = self.getuid()
289 node['actor'] = self.getuid()
290 node['creation'] = node['activity'] = date.Date()
292 self.newnodes.setdefault(classname, {})[nodeid] = 1
293 self.cache.setdefault(classname, {})[nodeid] = node
294 self.savenode(classname, nodeid, node)
296 def setnode(self, classname, nodeid, node):
297 ''' change the specified node
298 '''
299 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
301 # can't set without having already loaded the node
302 self.cache[classname][nodeid] = node
303 self.savenode(classname, nodeid, node)
305 def savenode(self, classname, nodeid, node):
306 ''' perform the saving of data specified by the set/addnode
307 '''
308 if __debug__:
309 logging.getLogger('hyperdb').debug('save %s%s %r'%(classname, nodeid, node))
310 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
312 def getnode(self, classname, nodeid, db=None, cache=1):
313 ''' get a node from the database
315 Note the "cache" parameter is not used, and exists purely for
316 backward compatibility!
317 '''
318 # try the cache
319 cache_dict = self.cache.setdefault(classname, {})
320 if cache_dict.has_key(nodeid):
321 if __debug__:
322 logging.getLogger('hyperdb').debug('get %s%s cached'%(classname, nodeid))
323 self.stats['cache_hits'] += 1
324 return cache_dict[nodeid]
326 if __debug__:
327 self.stats['cache_misses'] += 1
328 start_t = time.time()
329 logging.getLogger('hyperdb').debug('get %s%s'%(classname, nodeid))
331 # get from the database and save in the cache
332 if db is None:
333 db = self.getclassdb(classname)
334 if not db.has_key(nodeid):
335 raise IndexError, "no such %s %s"%(classname, nodeid)
337 # check the uncommitted, destroyed nodes
338 if (self.destroyednodes.has_key(classname) and
339 self.destroyednodes[classname].has_key(nodeid)):
340 raise IndexError, "no such %s %s"%(classname, nodeid)
342 # decode
343 res = marshal.loads(db[nodeid])
345 # reverse the serialisation
346 res = self.unserialise(classname, res)
348 # store off in the cache dict
349 if cache:
350 cache_dict[nodeid] = res
352 if __debug__:
353 self.stats['get_items'] += (time.time() - start_t)
355 return res
357 def destroynode(self, classname, nodeid):
358 '''Remove a node from the database. Called exclusively by the
359 destroy() method on Class.
360 '''
361 logging.getLogger('hyperdb').info('destroy %s%s'%(classname, nodeid))
363 # remove from cache and newnodes if it's there
364 if (self.cache.has_key(classname) and
365 self.cache[classname].has_key(nodeid)):
366 del self.cache[classname][nodeid]
367 if (self.newnodes.has_key(classname) and
368 self.newnodes[classname].has_key(nodeid)):
369 del self.newnodes[classname][nodeid]
371 # see if there's any obvious commit actions that we should get rid of
372 for entry in self.transactions[:]:
373 if entry[1][:2] == (classname, nodeid):
374 self.transactions.remove(entry)
376 # add to the destroyednodes map
377 self.destroyednodes.setdefault(classname, {})[nodeid] = 1
379 # add the destroy commit action
380 self.transactions.append((self.doDestroyNode, (classname, nodeid)))
381 self.transactions.append((FileStorage.destroy, (self, classname, nodeid)))
383 def serialise(self, classname, node):
384 '''Copy the node contents, converting non-marshallable data into
385 marshallable data.
386 '''
387 properties = self.getclass(classname).getprops()
388 d = {}
389 for k, v in node.items():
390 if k == self.RETIRED_FLAG:
391 d[k] = v
392 continue
394 # if the property doesn't exist then we really don't care
395 if not properties.has_key(k):
396 continue
398 # get the property spec
399 prop = properties[k]
401 if isinstance(prop, hyperdb.Password) and v is not None:
402 d[k] = str(v)
403 elif isinstance(prop, hyperdb.Date) and v is not None:
404 d[k] = v.serialise()
405 elif isinstance(prop, hyperdb.Interval) and v is not None:
406 d[k] = v.serialise()
407 else:
408 d[k] = v
409 return d
411 def unserialise(self, classname, node):
412 '''Decode the marshalled node data
413 '''
414 properties = self.getclass(classname).getprops()
415 d = {}
416 for k, v in node.items():
417 # if the property doesn't exist, or is the "retired" flag then
418 # it won't be in the properties dict
419 if not properties.has_key(k):
420 d[k] = v
421 continue
423 # get the property spec
424 prop = properties[k]
426 if isinstance(prop, hyperdb.Date) and v is not None:
427 d[k] = date.Date(v)
428 elif isinstance(prop, hyperdb.Interval) and v is not None:
429 d[k] = date.Interval(v)
430 elif isinstance(prop, hyperdb.Password) and v is not None:
431 p = password.Password()
432 p.unpack(v)
433 d[k] = p
434 else:
435 d[k] = v
436 return d
438 def hasnode(self, classname, nodeid, db=None):
439 ''' determine if the database has a given node
440 '''
441 # try the cache
442 cache = self.cache.setdefault(classname, {})
443 if cache.has_key(nodeid):
444 return 1
446 # not in the cache - check the database
447 if db is None:
448 db = self.getclassdb(classname)
449 res = db.has_key(nodeid)
450 return res
452 def countnodes(self, classname, db=None):
453 count = 0
455 # include the uncommitted nodes
456 if self.newnodes.has_key(classname):
457 count += len(self.newnodes[classname])
458 if self.destroyednodes.has_key(classname):
459 count -= len(self.destroyednodes[classname])
461 # and count those in the DB
462 if db is None:
463 db = self.getclassdb(classname)
464 count = count + len(db.keys())
465 return count
468 #
469 # Files - special node properties
470 # inherited from FileStorage
472 #
473 # Journal
474 #
475 def addjournal(self, classname, nodeid, action, params, creator=None,
476 creation=None):
477 ''' Journal the Action
478 'action' may be:
480 'create' or 'set' -- 'params' is a dictionary of property values
481 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
482 'retire' -- 'params' is None
484 'creator' -- the user performing the action, which defaults to
485 the current user.
486 '''
487 if __debug__:
488 logging.getLogger('hyperdb').debug('addjournal %s%s %s %r %s %r'%(classname,
489 nodeid, action, params, creator, creation))
490 if creator is None:
491 creator = self.getuid()
492 self.transactions.append((self.doSaveJournal, (classname, nodeid,
493 action, params, creator, creation)))
495 def setjournal(self, classname, nodeid, journal):
496 '''Set the journal to the "journal" list.'''
497 if __debug__:
498 logging.getLogger('hyperdb').debug('setjournal %s%s %r'%(classname,
499 nodeid, journal))
500 self.transactions.append((self.doSetJournal, (classname, nodeid,
501 journal)))
503 def getjournal(self, classname, nodeid):
504 ''' get the journal for id
506 Raise IndexError if the node doesn't exist (as per history()'s
507 API)
508 '''
509 # our journal result
510 res = []
512 # add any journal entries for transactions not committed to the
513 # database
514 for method, args in self.transactions:
515 if method != self.doSaveJournal:
516 continue
517 (cache_classname, cache_nodeid, cache_action, cache_params,
518 cache_creator, cache_creation) = args
519 if cache_classname == classname and cache_nodeid == nodeid:
520 if not cache_creator:
521 cache_creator = self.getuid()
522 if not cache_creation:
523 cache_creation = date.Date()
524 res.append((cache_nodeid, cache_creation, cache_creator,
525 cache_action, cache_params))
527 # attempt to open the journal - in some rare cases, the journal may
528 # not exist
529 try:
530 db = self.opendb('journals.%s'%classname, 'r')
531 except anydbm.error, error:
532 if str(error) == "need 'c' or 'n' flag to open new db":
533 raise IndexError, 'no such %s %s'%(classname, nodeid)
534 elif error.args[0] != 2:
535 # this isn't a "not found" error, be alarmed!
536 raise
537 if res:
538 # we have unsaved journal entries, return them
539 return res
540 raise IndexError, 'no such %s %s'%(classname, nodeid)
541 try:
542 journal = marshal.loads(db[nodeid])
543 except KeyError:
544 db.close()
545 if res:
546 # we have some unsaved journal entries, be happy!
547 return res
548 raise IndexError, 'no such %s %s'%(classname, nodeid)
549 db.close()
551 # add all the saved journal entries for this node
552 for nodeid, date_stamp, user, action, params in journal:
553 res.append((nodeid, date.Date(date_stamp), user, action, params))
554 return res
556 def pack(self, pack_before):
557 ''' Delete all journal entries except "create" before 'pack_before'.
558 '''
559 pack_before = pack_before.serialise()
560 for classname in self.getclasses():
561 packed = 0
562 # get the journal db
563 db_name = 'journals.%s'%classname
564 path = os.path.join(os.getcwd(), self.dir, classname)
565 db_type = self.determine_db_type(path)
566 db = self.opendb(db_name, 'w')
568 for key in db.keys():
569 # get the journal for this db entry
570 journal = marshal.loads(db[key])
571 l = []
572 last_set_entry = None
573 for entry in journal:
574 # unpack the entry
575 (nodeid, date_stamp, self.journaltag, action,
576 params) = entry
577 # if the entry is after the pack date, _or_ the initial
578 # create entry, then it stays
579 if date_stamp > pack_before or action == 'create':
580 l.append(entry)
581 else:
582 packed += 1
583 db[key] = marshal.dumps(l)
585 logging.getLogger('hyperdb').info('packed %d %s items'%(packed,
586 classname))
588 if db_type == 'gdbm':
589 db.reorganize()
590 db.close()
593 #
594 # Basic transaction support
595 #
596 def commit(self, fail_ok=False):
597 ''' Commit the current transactions.
599 Save all data changed since the database was opened or since the
600 last commit() or rollback().
602 fail_ok indicates that the commit is allowed to fail. This is used
603 in the web interface when committing cleaning of the session
604 database. We don't care if there's a concurrency issue there.
606 The only backend this seems to affect is postgres.
607 '''
608 logging.getLogger('hyperdb').info('commit %s transactions'%(
609 len(self.transactions)))
611 # keep a handle to all the database files opened
612 self.databases = {}
614 try:
615 # now, do all the transactions
616 reindex = {}
617 for method, args in self.transactions:
618 reindex[method(*args)] = 1
619 finally:
620 # make sure we close all the database files
621 for db in self.databases.values():
622 db.close()
623 del self.databases
625 # clear the transactions list now so the blobfile implementation
626 # doesn't think there's still pending file commits when it tries
627 # to access the file data
628 self.transactions = []
630 # reindex the nodes that request it
631 for classname, nodeid in filter(None, reindex.keys()):
632 self.getclass(classname).index(nodeid)
634 # save the indexer state
635 self.indexer.save_index()
637 self.clearCache()
639 def clearCache(self):
640 # all transactions committed, back to normal
641 self.cache = {}
642 self.dirtynodes = {}
643 self.newnodes = {}
644 self.destroyednodes = {}
645 self.transactions = []
647 def getCachedClassDB(self, classname):
648 ''' get the class db, looking in our cache of databases for commit
649 '''
650 # get the database handle
651 db_name = 'nodes.%s'%classname
652 if not self.databases.has_key(db_name):
653 self.databases[db_name] = self.getclassdb(classname, 'c')
654 return self.databases[db_name]
656 def doSaveNode(self, classname, nodeid, node):
657 db = self.getCachedClassDB(classname)
659 # now save the marshalled data
660 db[nodeid] = marshal.dumps(self.serialise(classname, node))
662 # return the classname, nodeid so we reindex this content
663 return (classname, nodeid)
665 def getCachedJournalDB(self, classname):
666 ''' get the journal db, looking in our cache of databases for commit
667 '''
668 # get the database handle
669 db_name = 'journals.%s'%classname
670 if not self.databases.has_key(db_name):
671 self.databases[db_name] = self.opendb(db_name, 'c')
672 return self.databases[db_name]
674 def doSaveJournal(self, classname, nodeid, action, params, creator,
675 creation):
676 # serialise the parameters now if necessary
677 if isinstance(params, type({})):
678 if action in ('set', 'create'):
679 params = self.serialise(classname, params)
681 # handle supply of the special journalling parameters (usually
682 # supplied on importing an existing database)
683 journaltag = creator
684 if creation:
685 journaldate = creation.serialise()
686 else:
687 journaldate = date.Date().serialise()
689 # create the journal entry
690 entry = (nodeid, journaldate, journaltag, action, params)
692 db = self.getCachedJournalDB(classname)
694 # now insert the journal entry
695 if db.has_key(nodeid):
696 # append to existing
697 s = db[nodeid]
698 l = marshal.loads(s)
699 l.append(entry)
700 else:
701 l = [entry]
703 db[nodeid] = marshal.dumps(l)
705 def doSetJournal(self, classname, nodeid, journal):
706 l = []
707 for nodeid, journaldate, journaltag, action, params in journal:
708 # serialise the parameters now if necessary
709 if isinstance(params, type({})):
710 if action in ('set', 'create'):
711 params = self.serialise(classname, params)
712 journaldate = journaldate.serialise()
713 l.append((nodeid, journaldate, journaltag, action, params))
714 db = self.getCachedJournalDB(classname)
715 db[nodeid] = marshal.dumps(l)
717 def doDestroyNode(self, classname, nodeid):
718 # delete from the class database
719 db = self.getCachedClassDB(classname)
720 if db.has_key(nodeid):
721 del db[nodeid]
723 # delete from the database
724 db = self.getCachedJournalDB(classname)
725 if db.has_key(nodeid):
726 del db[nodeid]
728 def rollback(self):
729 ''' Reverse all actions from the current transaction.
730 '''
731 logging.getLogger('hyperdb').info('rollback %s transactions'%(
732 len(self.transactions)))
734 for method, args in self.transactions:
735 # delete temporary files
736 if method == self.doStoreFile:
737 self.rollbackStoreFile(*args)
738 self.cache = {}
739 self.dirtynodes = {}
740 self.newnodes = {}
741 self.destroyednodes = {}
742 self.transactions = []
744 def close(self):
745 ''' Nothing to do
746 '''
747 if self.lockfile is not None:
748 locking.release_lock(self.lockfile)
749 self.lockfile.close()
750 self.lockfile = None
752 _marker = []
753 class Class(hyperdb.Class):
754 '''The handle to a particular class of nodes in a hyperdatabase.'''
756 def enableJournalling(self):
757 '''Turn journalling on for this class
758 '''
759 self.do_journal = 1
761 def disableJournalling(self):
762 '''Turn journalling off for this class
763 '''
764 self.do_journal = 0
766 # Editing nodes:
768 def create(self, **propvalues):
769 '''Create a new node of this class and return its id.
771 The keyword arguments in 'propvalues' map property names to values.
773 The values of arguments must be acceptable for the types of their
774 corresponding properties or a TypeError is raised.
776 If this class has a key property, it must be present and its value
777 must not collide with other key strings or a ValueError is raised.
779 Any other properties on this class that are missing from the
780 'propvalues' dictionary are set to None.
782 If an id in a link or multilink property does not refer to a valid
783 node, an IndexError is raised.
785 These operations trigger detectors and can be vetoed. Attempts
786 to modify the "creation" or "activity" properties cause a KeyError.
787 '''
788 self.fireAuditors('create', None, propvalues)
789 newid = self.create_inner(**propvalues)
790 self.fireReactors('create', newid, None)
791 return newid
793 def create_inner(self, **propvalues):
794 ''' Called by create, in-between the audit and react calls.
795 '''
796 if propvalues.has_key('id'):
797 raise KeyError, '"id" is reserved'
799 if self.db.journaltag is None:
800 raise hyperdb.DatabaseError, _('Database open read-only')
802 if propvalues.has_key('creation') or propvalues.has_key('activity'):
803 raise KeyError, '"creation" and "activity" are reserved'
804 # new node's id
805 newid = self.db.newid(self.classname)
807 # validate propvalues
808 num_re = re.compile('^\d+$')
809 for key, value in propvalues.items():
810 if key == self.key:
811 try:
812 self.lookup(value)
813 except KeyError:
814 pass
815 else:
816 raise ValueError, 'node with key "%s" exists'%value
818 # try to handle this property
819 try:
820 prop = self.properties[key]
821 except KeyError:
822 raise KeyError, '"%s" has no property "%s"'%(self.classname,
823 key)
825 if value is not None and isinstance(prop, hyperdb.Link):
826 if type(value) != type(''):
827 raise ValueError, 'link value must be String'
828 link_class = self.properties[key].classname
829 # if it isn't a number, it's a key
830 if not num_re.match(value):
831 try:
832 value = self.db.classes[link_class].lookup(value)
833 except (TypeError, KeyError):
834 raise IndexError, 'new property "%s": %s not a %s'%(
835 key, value, link_class)
836 elif not self.db.getclass(link_class).hasnode(value):
837 raise IndexError, '%s has no node %s'%(link_class, value)
839 # save off the value
840 propvalues[key] = value
842 # register the link with the newly linked node
843 if self.do_journal and self.properties[key].do_journal:
844 self.db.addjournal(link_class, value, 'link',
845 (self.classname, newid, key))
847 elif isinstance(prop, hyperdb.Multilink):
848 if value is None:
849 value = []
850 if not hasattr(value, '__iter__'):
851 raise TypeError, 'new property "%s" not an iterable of ids'%key
853 # clean up and validate the list of links
854 link_class = self.properties[key].classname
855 l = []
856 for entry in value:
857 if type(entry) != type(''):
858 raise ValueError, '"%s" multilink value (%r) '\
859 'must contain Strings'%(key, value)
860 # if it isn't a number, it's a key
861 if not num_re.match(entry):
862 try:
863 entry = self.db.classes[link_class].lookup(entry)
864 except (TypeError, KeyError):
865 raise IndexError, 'new property "%s": %s not a %s'%(
866 key, entry, self.properties[key].classname)
867 l.append(entry)
868 value = l
869 propvalues[key] = value
871 # handle additions
872 for nodeid in value:
873 if not self.db.getclass(link_class).hasnode(nodeid):
874 raise IndexError, '%s has no node %s'%(link_class,
875 nodeid)
876 # register the link with the newly linked node
877 if self.do_journal and self.properties[key].do_journal:
878 self.db.addjournal(link_class, nodeid, 'link',
879 (self.classname, newid, key))
881 elif isinstance(prop, hyperdb.String):
882 if type(value) != type('') and type(value) != type(u''):
883 raise TypeError, 'new property "%s" not a string'%key
884 if prop.indexme:
885 self.db.indexer.add_text((self.classname, newid, key),
886 value)
888 elif isinstance(prop, hyperdb.Password):
889 if not isinstance(value, password.Password):
890 raise TypeError, 'new property "%s" not a Password'%key
892 elif isinstance(prop, hyperdb.Date):
893 if value is not None and not isinstance(value, date.Date):
894 raise TypeError, 'new property "%s" not a Date'%key
896 elif isinstance(prop, hyperdb.Interval):
897 if value is not None and not isinstance(value, date.Interval):
898 raise TypeError, 'new property "%s" not an Interval'%key
900 elif value is not None and isinstance(prop, hyperdb.Number):
901 try:
902 float(value)
903 except ValueError:
904 raise TypeError, 'new property "%s" not numeric'%key
906 elif value is not None and isinstance(prop, hyperdb.Boolean):
907 try:
908 int(value)
909 except ValueError:
910 raise TypeError, 'new property "%s" not boolean'%key
912 # make sure there's data where there needs to be
913 for key, prop in self.properties.items():
914 if propvalues.has_key(key):
915 continue
916 if key == self.key:
917 raise ValueError, 'key property "%s" is required'%key
918 if isinstance(prop, hyperdb.Multilink):
919 propvalues[key] = []
921 # done
922 self.db.addnode(self.classname, newid, propvalues)
923 if self.do_journal:
924 self.db.addjournal(self.classname, newid, 'create', {})
926 return newid
928 def get(self, nodeid, propname, default=_marker, cache=1):
929 '''Get the value of a property on an existing node of this class.
931 'nodeid' must be the id of an existing node of this class or an
932 IndexError is raised. 'propname' must be the name of a property
933 of this class or a KeyError is raised.
935 'cache' exists for backward compatibility, and is not used.
937 Attempts to get the "creation" or "activity" properties should
938 do the right thing.
939 '''
940 if propname == 'id':
941 return nodeid
943 # get the node's dict
944 d = self.db.getnode(self.classname, nodeid)
946 # check for one of the special props
947 if propname == 'creation':
948 if d.has_key('creation'):
949 return d['creation']
950 if not self.do_journal:
951 raise ValueError, 'Journalling is disabled for this class'
952 journal = self.db.getjournal(self.classname, nodeid)
953 if journal:
954 return self.db.getjournal(self.classname, nodeid)[0][1]
955 else:
956 # on the strange chance that there's no journal
957 return date.Date()
958 if propname == 'activity':
959 if d.has_key('activity'):
960 return d['activity']
961 if not self.do_journal:
962 raise ValueError, 'Journalling is disabled for this class'
963 journal = self.db.getjournal(self.classname, nodeid)
964 if journal:
965 return self.db.getjournal(self.classname, nodeid)[-1][1]
966 else:
967 # on the strange chance that there's no journal
968 return date.Date()
969 if propname == 'creator':
970 if d.has_key('creator'):
971 return d['creator']
972 if not self.do_journal:
973 raise ValueError, 'Journalling is disabled for this class'
974 journal = self.db.getjournal(self.classname, nodeid)
975 if journal:
976 num_re = re.compile('^\d+$')
977 value = journal[0][2]
978 if num_re.match(value):
979 return value
980 else:
981 # old-style "username" journal tag
982 try:
983 return self.db.user.lookup(value)
984 except KeyError:
985 # user's been retired, return admin
986 return '1'
987 else:
988 return self.db.getuid()
989 if propname == 'actor':
990 if d.has_key('actor'):
991 return d['actor']
992 if not self.do_journal:
993 raise ValueError, 'Journalling is disabled for this class'
994 journal = self.db.getjournal(self.classname, nodeid)
995 if journal:
996 num_re = re.compile('^\d+$')
997 value = journal[-1][2]
998 if num_re.match(value):
999 return value
1000 else:
1001 # old-style "username" journal tag
1002 try:
1003 return self.db.user.lookup(value)
1004 except KeyError:
1005 # user's been retired, return admin
1006 return '1'
1007 else:
1008 return self.db.getuid()
1010 # get the property (raises KeyErorr if invalid)
1011 prop = self.properties[propname]
1013 if not d.has_key(propname):
1014 if default is _marker:
1015 if isinstance(prop, hyperdb.Multilink):
1016 return []
1017 else:
1018 return None
1019 else:
1020 return default
1022 # return a dupe of the list so code doesn't get confused
1023 if isinstance(prop, hyperdb.Multilink):
1024 return d[propname][:]
1026 return d[propname]
1028 def set(self, nodeid, **propvalues):
1029 '''Modify a property on an existing node of this class.
1031 'nodeid' must be the id of an existing node of this class or an
1032 IndexError is raised.
1034 Each key in 'propvalues' must be the name of a property of this
1035 class or a KeyError is raised.
1037 All values in 'propvalues' must be acceptable types for their
1038 corresponding properties or a TypeError is raised.
1040 If the value of the key property is set, it must not collide with
1041 other key strings or a ValueError is raised.
1043 If the value of a Link or Multilink property contains an invalid
1044 node id, a ValueError is raised.
1046 These operations trigger detectors and can be vetoed. Attempts
1047 to modify the "creation" or "activity" properties cause a KeyError.
1048 '''
1049 self.fireAuditors('set', nodeid, propvalues)
1050 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1051 for name,prop in self.getprops(protected=0).items():
1052 if oldvalues.has_key(name):
1053 continue
1054 if isinstance(prop, hyperdb.Multilink):
1055 oldvalues[name] = []
1056 else:
1057 oldvalues[name] = None
1058 propvalues = self.set_inner(nodeid, **propvalues)
1059 self.fireReactors('set', nodeid, oldvalues)
1060 return propvalues
1062 def set_inner(self, nodeid, **propvalues):
1063 ''' Called by set, in-between the audit and react calls.
1064 '''
1065 if not propvalues:
1066 return propvalues
1068 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1069 raise KeyError, '"creation" and "activity" are reserved'
1071 if propvalues.has_key('id'):
1072 raise KeyError, '"id" is reserved'
1074 if self.db.journaltag is None:
1075 raise hyperdb.DatabaseError, _('Database open read-only')
1077 node = self.db.getnode(self.classname, nodeid)
1078 if node.has_key(self.db.RETIRED_FLAG):
1079 raise IndexError
1080 num_re = re.compile('^\d+$')
1082 # if the journal value is to be different, store it in here
1083 journalvalues = {}
1085 for propname, value in propvalues.items():
1086 # check to make sure we're not duplicating an existing key
1087 if propname == self.key and node[propname] != value:
1088 try:
1089 self.lookup(value)
1090 except KeyError:
1091 pass
1092 else:
1093 raise ValueError, 'node with key "%s" exists'%value
1095 # this will raise the KeyError if the property isn't valid
1096 # ... we don't use getprops() here because we only care about
1097 # the writeable properties.
1098 try:
1099 prop = self.properties[propname]
1100 except KeyError:
1101 raise KeyError, '"%s" has no property named "%s"'%(
1102 self.classname, propname)
1104 # if the value's the same as the existing value, no sense in
1105 # doing anything
1106 current = node.get(propname, None)
1107 if value == current:
1108 del propvalues[propname]
1109 continue
1110 journalvalues[propname] = current
1112 # do stuff based on the prop type
1113 if isinstance(prop, hyperdb.Link):
1114 link_class = prop.classname
1115 # if it isn't a number, it's a key
1116 if value is not None and not isinstance(value, type('')):
1117 raise ValueError, 'property "%s" link value be a string'%(
1118 propname)
1119 if isinstance(value, type('')) and not num_re.match(value):
1120 try:
1121 value = self.db.classes[link_class].lookup(value)
1122 except (TypeError, KeyError):
1123 raise IndexError, 'new property "%s": %s not a %s'%(
1124 propname, value, prop.classname)
1126 if (value is not None and
1127 not self.db.getclass(link_class).hasnode(value)):
1128 raise IndexError, '%s has no node %s'%(link_class, value)
1130 if self.do_journal and prop.do_journal:
1131 # register the unlink with the old linked node
1132 if node.has_key(propname) and node[propname] is not None:
1133 self.db.addjournal(link_class, node[propname], 'unlink',
1134 (self.classname, nodeid, propname))
1136 # register the link with the newly linked node
1137 if value is not None:
1138 self.db.addjournal(link_class, value, 'link',
1139 (self.classname, nodeid, propname))
1141 elif isinstance(prop, hyperdb.Multilink):
1142 if value is None:
1143 value = []
1144 if not hasattr(value, '__iter__'):
1145 raise TypeError, 'new property "%s" not an iterable of'\
1146 ' ids'%propname
1147 link_class = self.properties[propname].classname
1148 l = []
1149 for entry in value:
1150 # if it isn't a number, it's a key
1151 if type(entry) != type(''):
1152 raise ValueError, 'new property "%s" link value ' \
1153 'must be a string'%propname
1154 if not num_re.match(entry):
1155 try:
1156 entry = self.db.classes[link_class].lookup(entry)
1157 except (TypeError, KeyError):
1158 raise IndexError, 'new property "%s": %s not a %s'%(
1159 propname, entry,
1160 self.properties[propname].classname)
1161 l.append(entry)
1162 value = l
1163 propvalues[propname] = value
1165 # figure the journal entry for this property
1166 add = []
1167 remove = []
1169 # handle removals
1170 if node.has_key(propname):
1171 l = node[propname]
1172 else:
1173 l = []
1174 for id in l[:]:
1175 if id in value:
1176 continue
1177 # register the unlink with the old linked node
1178 if self.do_journal and self.properties[propname].do_journal:
1179 self.db.addjournal(link_class, id, 'unlink',
1180 (self.classname, nodeid, propname))
1181 l.remove(id)
1182 remove.append(id)
1184 # handle additions
1185 for id in value:
1186 if not self.db.getclass(link_class).hasnode(id):
1187 raise IndexError, '%s has no node %s'%(link_class, id)
1188 if id in l:
1189 continue
1190 # register the link with the newly linked node
1191 if self.do_journal and self.properties[propname].do_journal:
1192 self.db.addjournal(link_class, id, 'link',
1193 (self.classname, nodeid, propname))
1194 l.append(id)
1195 add.append(id)
1197 # figure the journal entry
1198 l = []
1199 if add:
1200 l.append(('+', add))
1201 if remove:
1202 l.append(('-', remove))
1203 if l:
1204 journalvalues[propname] = tuple(l)
1206 elif isinstance(prop, hyperdb.String):
1207 if value is not None and type(value) != type('') and type(value) != type(u''):
1208 raise TypeError, 'new property "%s" not a string'%propname
1209 if prop.indexme:
1210 self.db.indexer.add_text((self.classname, nodeid, propname),
1211 value)
1213 elif isinstance(prop, hyperdb.Password):
1214 if not isinstance(value, password.Password):
1215 raise TypeError, 'new property "%s" not a Password'%propname
1216 propvalues[propname] = value
1218 elif value is not None and isinstance(prop, hyperdb.Date):
1219 if not isinstance(value, date.Date):
1220 raise TypeError, 'new property "%s" not a Date'% propname
1221 propvalues[propname] = value
1223 elif value is not None and isinstance(prop, hyperdb.Interval):
1224 if not isinstance(value, date.Interval):
1225 raise TypeError, 'new property "%s" not an '\
1226 'Interval'%propname
1227 propvalues[propname] = value
1229 elif value is not None and isinstance(prop, hyperdb.Number):
1230 try:
1231 float(value)
1232 except ValueError:
1233 raise TypeError, 'new property "%s" not numeric'%propname
1235 elif value is not None and isinstance(prop, hyperdb.Boolean):
1236 try:
1237 int(value)
1238 except ValueError:
1239 raise TypeError, 'new property "%s" not boolean'%propname
1241 node[propname] = value
1243 # nothing to do?
1244 if not propvalues:
1245 return propvalues
1247 # update the activity time
1248 node['activity'] = date.Date()
1249 node['actor'] = self.db.getuid()
1251 # do the set, and journal it
1252 self.db.setnode(self.classname, nodeid, node)
1254 if self.do_journal:
1255 self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
1257 return propvalues
1259 def retire(self, nodeid):
1260 '''Retire a node.
1262 The properties on the node remain available from the get() method,
1263 and the node's id is never reused.
1265 Retired nodes are not returned by the find(), list(), or lookup()
1266 methods, and other nodes may reuse the values of their key properties.
1268 These operations trigger detectors and can be vetoed. Attempts
1269 to modify the "creation" or "activity" properties cause a KeyError.
1270 '''
1271 if self.db.journaltag is None:
1272 raise hyperdb.DatabaseError, _('Database open read-only')
1274 self.fireAuditors('retire', nodeid, None)
1276 node = self.db.getnode(self.classname, nodeid)
1277 node[self.db.RETIRED_FLAG] = 1
1278 self.db.setnode(self.classname, nodeid, node)
1279 if self.do_journal:
1280 self.db.addjournal(self.classname, nodeid, 'retired', None)
1282 self.fireReactors('retire', nodeid, None)
1284 def restore(self, nodeid):
1285 '''Restpre a retired node.
1287 Make node available for all operations like it was before retirement.
1288 '''
1289 if self.db.journaltag is None:
1290 raise hyperdb.DatabaseError, _('Database open read-only')
1292 node = self.db.getnode(self.classname, nodeid)
1293 # check if key property was overrided
1294 key = self.getkey()
1295 try:
1296 id = self.lookup(node[key])
1297 except KeyError:
1298 pass
1299 else:
1300 raise KeyError, "Key property (%s) of retired node clashes with \
1301 existing one (%s)" % (key, node[key])
1302 # Now we can safely restore node
1303 self.fireAuditors('restore', nodeid, None)
1304 del node[self.db.RETIRED_FLAG]
1305 self.db.setnode(self.classname, nodeid, node)
1306 if self.do_journal:
1307 self.db.addjournal(self.classname, nodeid, 'restored', None)
1309 self.fireReactors('restore', nodeid, None)
1311 def is_retired(self, nodeid, cldb=None):
1312 '''Return true if the node is retired.
1313 '''
1314 node = self.db.getnode(self.classname, nodeid, cldb)
1315 if node.has_key(self.db.RETIRED_FLAG):
1316 return 1
1317 return 0
1319 def destroy(self, nodeid):
1320 '''Destroy a node.
1322 WARNING: this method should never be used except in extremely rare
1323 situations where there could never be links to the node being
1324 deleted
1326 WARNING: use retire() instead
1328 WARNING: the properties of this node will not be available ever again
1330 WARNING: really, use retire() instead
1332 Well, I think that's enough warnings. This method exists mostly to
1333 support the session storage of the cgi interface.
1334 '''
1335 if self.db.journaltag is None:
1336 raise hyperdb.DatabaseError, _('Database open read-only')
1337 self.db.destroynode(self.classname, nodeid)
1339 def history(self, nodeid):
1340 '''Retrieve the journal of edits on a particular node.
1342 'nodeid' must be the id of an existing node of this class or an
1343 IndexError is raised.
1345 The returned list contains tuples of the form
1347 (nodeid, date, tag, action, params)
1349 'date' is a Timestamp object specifying the time of the change and
1350 'tag' is the journaltag specified when the database was opened.
1351 '''
1352 if not self.do_journal:
1353 raise ValueError, 'Journalling is disabled for this class'
1354 return self.db.getjournal(self.classname, nodeid)
1356 # Locating nodes:
1357 def hasnode(self, nodeid):
1358 '''Determine if the given nodeid actually exists
1359 '''
1360 return self.db.hasnode(self.classname, nodeid)
1362 def setkey(self, propname):
1363 '''Select a String property of this class to be the key property.
1365 'propname' must be the name of a String property of this class or
1366 None, or a TypeError is raised. The values of the key property on
1367 all existing nodes must be unique or a ValueError is raised. If the
1368 property doesn't exist, KeyError is raised.
1369 '''
1370 prop = self.getprops()[propname]
1371 if not isinstance(prop, hyperdb.String):
1372 raise TypeError, 'key properties must be String'
1373 self.key = propname
1375 def getkey(self):
1376 '''Return the name of the key property for this class or None.'''
1377 return self.key
1379 # TODO: set up a separate index db file for this? profile?
1380 def lookup(self, keyvalue):
1381 '''Locate a particular node by its key property and return its id.
1383 If this class has no key property, a TypeError is raised. If the
1384 'keyvalue' matches one of the values for the key property among
1385 the nodes in this class, the matching node's id is returned;
1386 otherwise a KeyError is raised.
1387 '''
1388 if not self.key:
1389 raise TypeError, 'No key property set for class %s'%self.classname
1390 cldb = self.db.getclassdb(self.classname)
1391 try:
1392 for nodeid in self.getnodeids(cldb):
1393 node = self.db.getnode(self.classname, nodeid, cldb)
1394 if node.has_key(self.db.RETIRED_FLAG):
1395 continue
1396 if not node.has_key(self.key):
1397 continue
1398 if node[self.key] == keyvalue:
1399 return nodeid
1400 finally:
1401 cldb.close()
1402 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1403 keyvalue, self.classname)
1405 # change from spec - allows multiple props to match
1406 def find(self, **propspec):
1407 '''Get the ids of nodes in this class which link to the given nodes.
1409 'propspec' consists of keyword args propname=nodeid or
1410 propname={nodeid:1, }
1411 'propname' must be the name of a property in this class, or a
1412 KeyError is raised. That property must be a Link or
1413 Multilink property, or a TypeError is raised.
1415 Any node in this class whose 'propname' property links to any of
1416 the nodeids will be returned. Examples::
1418 db.issue.find(messages='1')
1419 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1420 '''
1421 propspec = propspec.items()
1422 for propname, itemids in propspec:
1423 # check the prop is OK
1424 prop = self.properties[propname]
1425 if not isinstance(prop, hyperdb.Link) and not isinstance(prop, hyperdb.Multilink):
1426 raise TypeError, "'%s' not a Link/Multilink property"%propname
1428 # ok, now do the find
1429 cldb = self.db.getclassdb(self.classname)
1430 l = []
1431 try:
1432 for id in self.getnodeids(db=cldb):
1433 item = self.db.getnode(self.classname, id, db=cldb)
1434 if item.has_key(self.db.RETIRED_FLAG):
1435 continue
1436 for propname, itemids in propspec:
1437 if type(itemids) is not type({}):
1438 itemids = {itemids:1}
1440 # special case if the item doesn't have this property
1441 if not item.has_key(propname):
1442 if itemids.has_key(None):
1443 l.append(id)
1444 break
1445 continue
1447 # grab the property definition and its value on this item
1448 prop = self.properties[propname]
1449 value = item[propname]
1450 if isinstance(prop, hyperdb.Link) and itemids.has_key(value):
1451 l.append(id)
1452 break
1453 elif isinstance(prop, hyperdb.Multilink):
1454 hit = 0
1455 for v in value:
1456 if itemids.has_key(v):
1457 l.append(id)
1458 hit = 1
1459 break
1460 if hit:
1461 break
1462 finally:
1463 cldb.close()
1464 return l
1466 def stringFind(self, **requirements):
1467 '''Locate a particular node by matching a set of its String
1468 properties in a caseless search.
1470 If the property is not a String property, a TypeError is raised.
1472 The return is a list of the id of all nodes that match.
1473 '''
1474 for propname in requirements.keys():
1475 prop = self.properties[propname]
1476 if not isinstance(prop, hyperdb.String):
1477 raise TypeError, "'%s' not a String property"%propname
1478 requirements[propname] = requirements[propname].lower()
1479 l = []
1480 cldb = self.db.getclassdb(self.classname)
1481 try:
1482 for nodeid in self.getnodeids(cldb):
1483 node = self.db.getnode(self.classname, nodeid, cldb)
1484 if node.has_key(self.db.RETIRED_FLAG):
1485 continue
1486 for key, value in requirements.items():
1487 if not node.has_key(key):
1488 break
1489 if node[key] is None or node[key].lower() != value:
1490 break
1491 else:
1492 l.append(nodeid)
1493 finally:
1494 cldb.close()
1495 return l
1497 def list(self):
1498 ''' Return a list of the ids of the active nodes in this class.
1499 '''
1500 l = []
1501 cn = self.classname
1502 cldb = self.db.getclassdb(cn)
1503 try:
1504 for nodeid in self.getnodeids(cldb):
1505 node = self.db.getnode(cn, nodeid, cldb)
1506 if node.has_key(self.db.RETIRED_FLAG):
1507 continue
1508 l.append(nodeid)
1509 finally:
1510 cldb.close()
1511 l.sort()
1512 return l
1514 def getnodeids(self, db=None, retired=None):
1515 ''' Return a list of ALL nodeids
1517 Set retired=None to get all nodes. Otherwise it'll get all the
1518 retired or non-retired nodes, depending on the flag.
1519 '''
1520 res = []
1522 # start off with the new nodes
1523 if self.db.newnodes.has_key(self.classname):
1524 res += self.db.newnodes[self.classname].keys()
1526 must_close = False
1527 if db is None:
1528 db = self.db.getclassdb(self.classname)
1529 must_close = True
1530 try:
1531 res = res + db.keys()
1533 # remove the uncommitted, destroyed nodes
1534 if self.db.destroyednodes.has_key(self.classname):
1535 for nodeid in self.db.destroyednodes[self.classname].keys():
1536 if db.has_key(nodeid):
1537 res.remove(nodeid)
1539 # check retired flag
1540 if retired is False or retired is True:
1541 l = []
1542 for nodeid in res:
1543 node = self.db.getnode(self.classname, nodeid, db)
1544 is_ret = node.has_key(self.db.RETIRED_FLAG)
1545 if retired == is_ret:
1546 l.append(nodeid)
1547 res = l
1548 finally:
1549 if must_close:
1550 db.close()
1551 return res
1553 def _filter(self, search_matches, filterspec, proptree,
1554 num_re = re.compile('^\d+$')):
1555 """Return a list of the ids of the active nodes in this class that
1556 match the 'filter' spec, sorted by the group spec and then the
1557 sort spec.
1559 "filterspec" is {propname: value(s)}
1561 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1562 and prop is a prop name or None
1564 "search_matches" is a sequence type or None
1566 The filter must match all properties specificed. If the property
1567 value to match is a list:
1569 1. String properties must match all elements in the list, and
1570 2. Other properties must match any of the elements in the list.
1571 """
1572 if __debug__:
1573 start_t = time.time()
1575 cn = self.classname
1577 # optimise filterspec
1578 l = []
1579 props = self.getprops()
1580 LINK = 'spec:link'
1581 MULTILINK = 'spec:multilink'
1582 STRING = 'spec:string'
1583 DATE = 'spec:date'
1584 INTERVAL = 'spec:interval'
1585 OTHER = 'spec:other'
1587 for k, v in filterspec.items():
1588 propclass = props[k]
1589 if isinstance(propclass, hyperdb.Link):
1590 if type(v) is not type([]):
1591 v = [v]
1592 u = []
1593 for entry in v:
1594 # the value -1 is a special "not set" sentinel
1595 if entry == '-1':
1596 entry = None
1597 u.append(entry)
1598 l.append((LINK, k, u))
1599 elif isinstance(propclass, hyperdb.Multilink):
1600 # the value -1 is a special "not set" sentinel
1601 if v in ('-1', ['-1']):
1602 v = []
1603 elif type(v) is not type([]):
1604 v = [v]
1605 l.append((MULTILINK, k, v))
1606 elif isinstance(propclass, hyperdb.String) and k != 'id':
1607 if type(v) is not type([]):
1608 v = [v]
1609 for v in v:
1610 # simple glob searching
1611 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1612 v = v.replace('?', '.')
1613 v = v.replace('*', '.*?')
1614 l.append((STRING, k, re.compile(v, re.I)))
1615 elif isinstance(propclass, hyperdb.Date):
1616 try:
1617 date_rng = propclass.range_from_raw(v, self.db)
1618 l.append((DATE, k, date_rng))
1619 except ValueError:
1620 # If range creation fails - ignore that search parameter
1621 pass
1622 elif isinstance(propclass, hyperdb.Interval):
1623 try:
1624 intv_rng = date.Range(v, date.Interval)
1625 l.append((INTERVAL, k, intv_rng))
1626 except ValueError:
1627 # If range creation fails - ignore that search parameter
1628 pass
1630 elif isinstance(propclass, hyperdb.Boolean):
1631 if type(v) != type([]):
1632 v = v.split(',')
1633 bv = []
1634 for val in v:
1635 if type(val) is type(''):
1636 bv.append(val.lower() in ('yes', 'true', 'on', '1'))
1637 else:
1638 bv.append(val)
1639 l.append((OTHER, k, bv))
1641 elif k == 'id':
1642 if type(v) != type([]):
1643 v = v.split(',')
1644 l.append((OTHER, k, [str(int(val)) for val in v]))
1646 elif isinstance(propclass, hyperdb.Number):
1647 if type(v) != type([]):
1648 v = v.split(',')
1649 l.append((OTHER, k, [float(val) for val in v]))
1651 filterspec = l
1653 # now, find all the nodes that are active and pass filtering
1654 matches = []
1655 cldb = self.db.getclassdb(cn)
1656 t = 0
1657 try:
1658 # TODO: only full-scan once (use items())
1659 for nodeid in self.getnodeids(cldb):
1660 node = self.db.getnode(cn, nodeid, cldb)
1661 if node.has_key(self.db.RETIRED_FLAG):
1662 continue
1663 # apply filter
1664 for t, k, v in filterspec:
1665 # handle the id prop
1666 if k == 'id':
1667 if nodeid not in v:
1668 break
1669 continue
1671 # get the node value
1672 nv = node.get(k, None)
1674 match = 0
1676 # now apply the property filter
1677 if t == LINK:
1678 # link - if this node's property doesn't appear in the
1679 # filterspec's nodeid list, skip it
1680 match = nv in v
1681 elif t == MULTILINK:
1682 # multilink - if any of the nodeids required by the
1683 # filterspec aren't in this node's property, then skip
1684 # it
1685 nv = node.get(k, [])
1687 # check for matching the absence of multilink values
1688 if not v:
1689 match = not nv
1690 else:
1691 # othewise, make sure this node has each of the
1692 # required values
1693 for want in v:
1694 if want in nv:
1695 match = 1
1696 break
1697 elif t == STRING:
1698 if nv is None:
1699 nv = ''
1700 # RE search
1701 match = v.search(nv)
1702 elif t == DATE or t == INTERVAL:
1703 if nv is None:
1704 match = v is None
1705 else:
1706 if v.to_value:
1707 if v.from_value <= nv and v.to_value >= nv:
1708 match = 1
1709 else:
1710 if v.from_value <= nv:
1711 match = 1
1712 elif t == OTHER:
1713 # straight value comparison for the other types
1714 match = nv in v
1715 if not match:
1716 break
1717 else:
1718 matches.append([nodeid, node])
1720 # filter based on full text search
1721 if search_matches is not None:
1722 k = []
1723 for v in matches:
1724 if v[0] in search_matches:
1725 k.append(v)
1726 matches = k
1728 # add sorting information to the proptree
1729 JPROPS = {'actor':1, 'activity':1, 'creator':1, 'creation':1}
1730 children = []
1731 if proptree:
1732 children = proptree.sortable_children()
1733 for pt in children:
1734 dir = pt.sort_direction
1735 prop = pt.name
1736 assert (dir and prop)
1737 propclass = props[prop]
1738 pt.sort_ids = []
1739 is_pointer = isinstance(propclass,(hyperdb.Link,
1740 hyperdb.Multilink))
1741 if not is_pointer:
1742 pt.sort_result = []
1743 try:
1744 # cache the opened link class db, if needed.
1745 lcldb = None
1746 # cache the linked class items too
1747 lcache = {}
1749 for entry in matches:
1750 itemid = entry[-2]
1751 item = entry[-1]
1752 # handle the properties that might be "faked"
1753 # also, handle possible missing properties
1754 try:
1755 v = item[prop]
1756 except KeyError:
1757 if JPROPS.has_key(prop):
1758 # force lookup of the special journal prop
1759 v = self.get(itemid, prop)
1760 else:
1761 # the node doesn't have a value for this
1762 # property
1763 v = None
1764 if isinstance(propclass, hyperdb.Multilink):
1765 v = []
1766 if prop == 'id':
1767 v = int (itemid)
1768 pt.sort_ids.append(v)
1769 if not is_pointer:
1770 pt.sort_result.append(v)
1771 continue
1773 # missing (None) values are always sorted first
1774 if v is None:
1775 pt.sort_ids.append(v)
1776 if not is_pointer:
1777 pt.sort_result.append(v)
1778 continue
1780 if isinstance(propclass, hyperdb.Link):
1781 lcn = propclass.classname
1782 link = self.db.classes[lcn]
1783 key = link.orderprop()
1784 child = pt.propdict[key]
1785 if key!='id':
1786 if not lcache.has_key(v):
1787 # open the link class db if it's not already
1788 if lcldb is None:
1789 lcldb = self.db.getclassdb(lcn)
1790 lcache[v] = self.db.getnode(lcn, v, lcldb)
1791 r = lcache[v][key]
1792 child.propdict[key].sort_ids.append(r)
1793 else:
1794 child.propdict[key].sort_ids.append(v)
1795 pt.sort_ids.append(v)
1796 if not is_pointer:
1797 r = propclass.sort_repr(pt.parent.cls, v, pt.name)
1798 pt.sort_result.append(r)
1799 finally:
1800 # if we opened the link class db, close it now
1801 if lcldb is not None:
1802 lcldb.close()
1803 del lcache
1804 finally:
1805 cldb.close()
1807 # pull the id out of the individual entries
1808 matches = [entry[-2] for entry in matches]
1809 if __debug__:
1810 self.db.stats['filtering'] += (time.time() - start_t)
1811 return matches
1813 def count(self):
1814 '''Get the number of nodes in this class.
1816 If the returned integer is 'numnodes', the ids of all the nodes
1817 in this class run from 1 to numnodes, and numnodes+1 will be the
1818 id of the next node to be created in this class.
1819 '''
1820 return self.db.countnodes(self.classname)
1822 # Manipulating properties:
1824 def getprops(self, protected=1):
1825 '''Return a dictionary mapping property names to property objects.
1826 If the "protected" flag is true, we include protected properties -
1827 those which may not be modified.
1829 In addition to the actual properties on the node, these
1830 methods provide the "creation" and "activity" properties. If the
1831 "protected" flag is true, we include protected properties - those
1832 which may not be modified.
1833 '''
1834 d = self.properties.copy()
1835 if protected:
1836 d['id'] = hyperdb.String()
1837 d['creation'] = hyperdb.Date()
1838 d['activity'] = hyperdb.Date()
1839 d['creator'] = hyperdb.Link('user')
1840 d['actor'] = hyperdb.Link('user')
1841 return d
1843 def addprop(self, **properties):
1844 '''Add properties to this class.
1846 The keyword arguments in 'properties' must map names to property
1847 objects, or a TypeError is raised. None of the keys in 'properties'
1848 may collide with the names of existing properties, or a ValueError
1849 is raised before any properties have been added.
1850 '''
1851 for key in properties.keys():
1852 if self.properties.has_key(key):
1853 raise ValueError, key
1854 self.properties.update(properties)
1856 def index(self, nodeid):
1857 ''' Add (or refresh) the node to search indexes '''
1858 # find all the String properties that have indexme
1859 for prop, propclass in self.getprops().items():
1860 if isinstance(propclass, hyperdb.String) and propclass.indexme:
1861 # index them under (classname, nodeid, property)
1862 try:
1863 value = str(self.get(nodeid, prop))
1864 except IndexError:
1865 # node has been destroyed
1866 continue
1867 self.db.indexer.add_text((self.classname, nodeid, prop), value)
1869 #
1870 # import / export support
1871 #
1872 def export_list(self, propnames, nodeid):
1873 ''' Export a node - generate a list of CSV-able data in the order
1874 specified by propnames for the given node.
1875 '''
1876 properties = self.getprops()
1877 l = []
1878 for prop in propnames:
1879 proptype = properties[prop]
1880 value = self.get(nodeid, prop)
1881 # "marshal" data where needed
1882 if value is None:
1883 pass
1884 elif isinstance(proptype, hyperdb.Date):
1885 value = value.get_tuple()
1886 elif isinstance(proptype, hyperdb.Interval):
1887 value = value.get_tuple()
1888 elif isinstance(proptype, hyperdb.Password):
1889 value = str(value)
1890 l.append(repr(value))
1892 # append retired flag
1893 l.append(repr(self.is_retired(nodeid)))
1895 return l
1897 def import_list(self, propnames, proplist):
1898 ''' Import a node - all information including "id" is present and
1899 should not be sanity checked. Triggers are not triggered. The
1900 journal should be initialised using the "creator" and "created"
1901 information.
1903 Return the nodeid of the node imported.
1904 '''
1905 if self.db.journaltag is None:
1906 raise hyperdb.DatabaseError, _('Database open read-only')
1907 properties = self.getprops()
1909 # make the new node's property map
1910 d = {}
1911 newid = None
1912 for i in range(len(propnames)):
1913 # Figure the property for this column
1914 propname = propnames[i]
1916 # Use eval to reverse the repr() used to output the CSV
1917 value = eval(proplist[i])
1919 # "unmarshal" where necessary
1920 if propname == 'id':
1921 newid = value
1922 continue
1923 elif propname == 'is retired':
1924 # is the item retired?
1925 if int(value):
1926 d[self.db.RETIRED_FLAG] = 1
1927 continue
1928 elif value is None:
1929 d[propname] = None
1930 continue
1932 prop = properties[propname]
1933 if isinstance(prop, hyperdb.Date):
1934 value = date.Date(value)
1935 elif isinstance(prop, hyperdb.Interval):
1936 value = date.Interval(value)
1937 elif isinstance(prop, hyperdb.Password):
1938 pwd = password.Password()
1939 pwd.unpack(value)
1940 value = pwd
1941 d[propname] = value
1943 # get a new id if necessary
1944 if newid is None:
1945 newid = self.db.newid(self.classname)
1947 # add the node and journal
1948 self.db.addnode(self.classname, newid, d)
1949 return newid
1951 def export_journals(self):
1952 '''Export a class's journal - generate a list of lists of
1953 CSV-able data:
1955 nodeid, date, user, action, params
1957 No heading here - the columns are fixed.
1958 '''
1959 properties = self.getprops()
1960 r = []
1961 for nodeid in self.getnodeids():
1962 for nodeid, date, user, action, params in self.history(nodeid):
1963 date = date.get_tuple()
1964 if action == 'set':
1965 export_data = {}
1966 for propname, value in params.items():
1967 if not properties.has_key(propname):
1968 # property no longer in the schema
1969 continue
1971 prop = properties[propname]
1972 # make sure the params are eval()'able
1973 if value is None:
1974 pass
1975 elif isinstance(prop, hyperdb.Date):
1976 # this is a hack - some dates are stored as strings
1977 if not isinstance(value, type('')):
1978 value = value.get_tuple()
1979 elif isinstance(prop, hyperdb.Interval):
1980 # hack too - some intervals are stored as strings
1981 if not isinstance(value, type('')):
1982 value = value.get_tuple()
1983 elif isinstance(prop, hyperdb.Password):
1984 value = str(value)
1985 export_data[propname] = value
1986 params = export_data
1987 l = [nodeid, date, user, action, params]
1988 r.append(map(repr, l))
1989 return r
1991 def import_journals(self, entries):
1992 '''Import a class's journal.
1994 Uses setjournal() to set the journal for each item.'''
1995 properties = self.getprops()
1996 d = {}
1997 for l in entries:
1998 l = map(eval, l)
1999 nodeid, jdate, user, action, params = l
2000 r = d.setdefault(nodeid, [])
2001 if action == 'set':
2002 for propname, value in params.items():
2003 prop = properties[propname]
2004 if value is None:
2005 pass
2006 elif isinstance(prop, hyperdb.Date):
2007 if type(value) == type(()):
2008 print _('WARNING: invalid date tuple %r')%(value,)
2009 value = date.Date( "2000-1-1" )
2010 value = date.Date(value)
2011 elif isinstance(prop, hyperdb.Interval):
2012 value = date.Interval(value)
2013 elif isinstance(prop, hyperdb.Password):
2014 pwd = password.Password()
2015 pwd.unpack(value)
2016 value = pwd
2017 params[propname] = value
2018 r.append((nodeid, date.Date(jdate), user, action, params))
2020 for nodeid, l in d.items():
2021 self.db.setjournal(self.classname, nodeid, l)
2023 class FileClass(hyperdb.FileClass, Class):
2024 '''This class defines a large chunk of data. To support this, it has a
2025 mandatory String property "content" which is typically saved off
2026 externally to the hyperdb.
2028 The default MIME type of this data is defined by the
2029 "default_mime_type" class attribute, which may be overridden by each
2030 node if the class defines a "type" String property.
2031 '''
2032 def __init__(self, db, classname, **properties):
2033 '''The newly-created class automatically includes the "content"
2034 and "type" properties.
2035 '''
2036 if not properties.has_key('content'):
2037 properties['content'] = hyperdb.String(indexme='yes')
2038 if not properties.has_key('type'):
2039 properties['type'] = hyperdb.String()
2040 Class.__init__(self, db, classname, **properties)
2042 def create(self, **propvalues):
2043 ''' Snarf the "content" propvalue and store in a file
2044 '''
2045 # we need to fire the auditors now, or the content property won't
2046 # be in propvalues for the auditors to play with
2047 self.fireAuditors('create', None, propvalues)
2049 # now remove the content property so it's not stored in the db
2050 content = propvalues['content']
2051 del propvalues['content']
2053 # make sure we have a MIME type
2054 mime_type = propvalues.get('type', self.default_mime_type)
2056 # do the database create
2057 newid = self.create_inner(**propvalues)
2059 # store off the content as a file
2060 self.db.storefile(self.classname, newid, None, content)
2062 # fire reactors
2063 self.fireReactors('create', newid, None)
2065 return newid
2067 def get(self, nodeid, propname, default=_marker, cache=1):
2068 ''' Trap the content propname and get it from the file
2070 'cache' exists for backwards compatibility, and is not used.
2071 '''
2072 poss_msg = 'Possibly an access right configuration problem.'
2073 if propname == 'content':
2074 try:
2075 return self.db.getfile(self.classname, nodeid, None)
2076 except IOError, (strerror):
2077 # XXX by catching this we don't see an error in the log.
2078 return 'ERROR reading file: %s%s\n%s\n%s'%(
2079 self.classname, nodeid, poss_msg, strerror)
2080 if default is not _marker:
2081 return Class.get(self, nodeid, propname, default)
2082 else:
2083 return Class.get(self, nodeid, propname)
2085 def set(self, itemid, **propvalues):
2086 ''' Snarf the "content" propvalue and update it in a file
2087 '''
2088 self.fireAuditors('set', itemid, propvalues)
2090 # create the oldvalues dict - fill in any missing values
2091 oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid))
2092 for name,prop in self.getprops(protected=0).items():
2093 if oldvalues.has_key(name):
2094 continue
2095 if isinstance(prop, hyperdb.Multilink):
2096 oldvalues[name] = []
2097 else:
2098 oldvalues[name] = None
2100 # now remove the content property so it's not stored in the db
2101 content = None
2102 if propvalues.has_key('content'):
2103 content = propvalues['content']
2104 del propvalues['content']
2106 # do the database update
2107 propvalues = self.set_inner(itemid, **propvalues)
2109 # do content?
2110 if content:
2111 # store and possibly index
2112 self.db.storefile(self.classname, itemid, None, content)
2113 if self.properties['content'].indexme:
2114 mime_type = self.get(itemid, 'type', self.default_mime_type)
2115 self.db.indexer.add_text((self.classname, itemid, 'content'),
2116 content, mime_type)
2117 propvalues['content'] = content
2119 # fire reactors
2120 self.fireReactors('set', itemid, oldvalues)
2121 return propvalues
2123 def index(self, nodeid):
2124 ''' Add (or refresh) the node to search indexes.
2126 Use the content-type property for the content property.
2127 '''
2128 # find all the String properties that have indexme
2129 for prop, propclass in self.getprops().items():
2130 if prop == 'content' and propclass.indexme:
2131 mime_type = self.get(nodeid, 'type', self.default_mime_type)
2132 self.db.indexer.add_text((self.classname, nodeid, 'content'),
2133 str(self.get(nodeid, 'content')), mime_type)
2134 elif isinstance(propclass, hyperdb.String) and propclass.indexme:
2135 # index them under (classname, nodeid, property)
2136 try:
2137 value = str(self.get(nodeid, prop))
2138 except IndexError:
2139 # node has been destroyed
2140 continue
2141 self.db.indexer.add_text((self.classname, nodeid, prop), value)
2143 # deviation from spec - was called ItemClass
2144 class IssueClass(Class, roundupdb.IssueClass):
2145 # Overridden methods:
2146 def __init__(self, db, classname, **properties):
2147 '''The newly-created class automatically includes the "messages",
2148 "files", "nosy", and "superseder" properties. If the 'properties'
2149 dictionary attempts to specify any of these properties or a
2150 "creation" or "activity" property, a ValueError is raised.
2151 '''
2152 if not properties.has_key('title'):
2153 properties['title'] = hyperdb.String(indexme='yes')
2154 if not properties.has_key('messages'):
2155 properties['messages'] = hyperdb.Multilink("msg")
2156 if not properties.has_key('files'):
2157 properties['files'] = hyperdb.Multilink("file")
2158 if not properties.has_key('nosy'):
2159 # note: journalling is turned off as it really just wastes
2160 # space. this behaviour may be overridden in an instance
2161 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2162 if not properties.has_key('superseder'):
2163 properties['superseder'] = hyperdb.Multilink(classname)
2164 Class.__init__(self, db, classname, **properties)
2166 # vim: set et sts=4 sw=4 :