d0eb1a9545150f64d20e59ca4c30206995c543f6
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 """This module defines a backend that saves the hyperdatabase in a
19 database chosen by anydbm. It is guaranteed to always be available in python
20 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
21 serious bugs, and is not available)
22 """
23 __docformat__ = 'restructuredtext'
25 try:
26 import anydbm, sys
27 # dumbdbm only works in python 2.1.2+
28 if sys.version_info < (2,1,2):
29 import dumbdbm
30 assert anydbm._defaultmod != dumbdbm
31 del dumbdbm
32 except AssertionError:
33 print "WARNING: you should upgrade to python 2.1.3"
35 import whichdb, os, marshal, re, weakref, string, copy, time, shutil, logging
37 from roundup import hyperdb, date, password, roundupdb, security, support
38 from roundup.support import reversed
39 from roundup.backends import locking
40 from roundup.i18n import _
42 from blobfiles import FileStorage
43 from sessions_dbm import Sessions, OneTimeKeys
45 try:
46 from indexer_xapian import Indexer
47 except ImportError:
48 from indexer_dbm import Indexer
50 def db_exists(config):
51 # check for the user db
52 for db in 'nodes.user nodes.user.db'.split():
53 if os.path.exists(os.path.join(config.DATABASE, db)):
54 return 1
55 return 0
57 def db_nuke(config):
58 shutil.rmtree(config.DATABASE)
60 #
61 # Now the database
62 #
63 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
64 """A database for storing records containing flexible data types.
66 Transaction stuff TODO:
68 - check the timestamp of the class file and nuke the cache if it's
69 modified. Do some sort of conflict checking on the dirty stuff.
70 - perhaps detect write collisions (related to above)?
71 """
72 def __init__(self, config, journaltag=None):
73 """Open a hyperdatabase given a specifier to some storage.
75 The 'storagelocator' is obtained from config.DATABASE.
76 The meaning of 'storagelocator' depends on the particular
77 implementation of the hyperdatabase. It could be a file name,
78 a directory path, a socket descriptor for a connection to a
79 database over the network, etc.
81 The 'journaltag' is a token that will be attached to the journal
82 entries for any edits done on the database. If 'journaltag' is
83 None, the database is opened in read-only mode: the Class.create(),
84 Class.set(), Class.retire(), and Class.restore() methods are
85 disabled.
86 """
87 FileStorage.__init__(self, config.UMASK)
88 self.config, self.journaltag = config, journaltag
89 self.dir = config.DATABASE
90 self.classes = {}
91 self.cache = {} # cache of nodes loaded or created
92 self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0,
93 'filtering': 0}
94 self.dirtynodes = {} # keep track of the dirty nodes by class
95 self.newnodes = {} # keep track of the new nodes by class
96 self.destroyednodes = {}# keep track of the destroyed nodes by class
97 self.transactions = []
98 self.indexer = Indexer(self)
99 self.security = security.Security(self)
100 os.umask(config.UMASK)
102 # lock it
103 lockfilenm = os.path.join(self.dir, 'lock')
104 self.lockfile = locking.acquire_lock(lockfilenm)
105 self.lockfile.write(str(os.getpid()))
106 self.lockfile.flush()
108 def post_init(self):
109 """Called once the schema initialisation has finished.
110 """
111 # reindex the db if necessary
112 if self.indexer.should_reindex():
113 self.reindex()
115 def refresh_database(self):
116 """Rebuild the database
117 """
118 self.reindex()
120 def getSessionManager(self):
121 return Sessions(self)
123 def getOTKManager(self):
124 return OneTimeKeys(self)
126 def reindex(self, classname=None, show_progress=False):
127 if classname:
128 classes = [self.getclass(classname)]
129 else:
130 classes = self.classes.values()
131 for klass in classes:
132 if show_progress:
133 for nodeid in support.Progress('Reindex %s'%klass.classname,
134 klass.list()):
135 klass.index(nodeid)
136 else:
137 for nodeid in klass.list():
138 klass.index(nodeid)
139 self.indexer.save_index()
141 def __repr__(self):
142 return '<back_anydbm instance at %x>'%id(self)
144 #
145 # Classes
146 #
147 def __getattr__(self, classname):
148 """A convenient way of calling self.getclass(classname)."""
149 if self.classes.has_key(classname):
150 return self.classes[classname]
151 raise AttributeError, classname
153 def addclass(self, cl):
154 cn = cl.classname
155 if self.classes.has_key(cn):
156 raise ValueError, cn
157 self.classes[cn] = cl
159 # add default Edit and View permissions
160 self.security.addPermission(name="Create", klass=cn,
161 description="User is allowed to create "+cn)
162 self.security.addPermission(name="Edit", klass=cn,
163 description="User is allowed to edit "+cn)
164 self.security.addPermission(name="View", klass=cn,
165 description="User is allowed to access "+cn)
167 def getclasses(self):
168 """Return a list of the names of all existing classes."""
169 l = self.classes.keys()
170 l.sort()
171 return l
173 def getclass(self, classname):
174 """Get the Class object representing a particular class.
176 If 'classname' is not a valid class name, a KeyError is raised.
177 """
178 try:
179 return self.classes[classname]
180 except KeyError:
181 raise KeyError, 'There is no class called "%s"'%classname
183 #
184 # Class DBs
185 #
186 def clear(self):
187 """Delete all database contents
188 """
189 logging.getLogger('hyperdb').info('clear')
190 for cn in self.classes.keys():
191 for dummy in 'nodes', 'journals':
192 path = os.path.join(self.dir, 'journals.%s'%cn)
193 if os.path.exists(path):
194 os.remove(path)
195 elif os.path.exists(path+'.db'): # dbm appends .db
196 os.remove(path+'.db')
197 # reset id sequences
198 path = os.path.join(os.getcwd(), self.dir, '_ids')
199 if os.path.exists(path):
200 os.remove(path)
201 elif os.path.exists(path+'.db'): # dbm appends .db
202 os.remove(path+'.db')
204 def getclassdb(self, classname, mode='r'):
205 """ grab a connection to the class db that will be used for
206 multiple actions
207 """
208 return self.opendb('nodes.%s'%classname, mode)
210 def determine_db_type(self, path):
211 """ determine which DB wrote the class file
212 """
213 db_type = ''
214 if os.path.exists(path):
215 db_type = whichdb.whichdb(path)
216 if not db_type:
217 raise hyperdb.DatabaseError, \
218 _("Couldn't identify database type")
219 elif os.path.exists(path+'.db'):
220 # if the path ends in '.db', it's a dbm database, whether
221 # anydbm says it's dbhash or not!
222 db_type = 'dbm'
223 return db_type
225 def opendb(self, name, mode):
226 """Low-level database opener that gets around anydbm/dbm
227 eccentricities.
228 """
229 # figure the class db type
230 path = os.path.join(os.getcwd(), self.dir, name)
231 db_type = self.determine_db_type(path)
233 # new database? let anydbm pick the best dbm
234 if not db_type:
235 if __debug__:
236 logging.getLogger('hyperdb').debug("opendb anydbm.open(%r, 'c')"%path)
237 return anydbm.open(path, 'c')
239 # open the database with the correct module
240 try:
241 dbm = __import__(db_type)
242 except ImportError:
243 raise hyperdb.DatabaseError, \
244 _("Couldn't open database - the required module '%s'"\
245 " is not available")%db_type
246 if __debug__:
247 logging.getLogger('hyperdb').debug("opendb %r.open(%r, %r)"%(db_type, path,
248 mode))
249 return dbm.open(path, mode)
251 #
252 # Node IDs
253 #
254 def newid(self, classname):
255 """ Generate a new id for the given class
256 """
257 # open the ids DB - create if if doesn't exist
258 db = self.opendb('_ids', 'c')
259 if db.has_key(classname):
260 newid = db[classname] = str(int(db[classname]) + 1)
261 else:
262 # the count() bit is transitional - older dbs won't start at 1
263 newid = str(self.getclass(classname).count()+1)
264 db[classname] = newid
265 db.close()
266 return newid
268 def setid(self, classname, setid):
269 """ Set the id counter: used during import of database
270 """
271 # open the ids DB - create if if doesn't exist
272 db = self.opendb('_ids', 'c')
273 db[classname] = str(setid)
274 db.close()
276 #
277 # Nodes
278 #
279 def addnode(self, classname, nodeid, node):
280 """ add the specified node to its class's db
281 """
282 # we'll be supplied these props if we're doing an import
283 if not node.has_key('creator'):
284 # add in the "calculated" properties (dupe so we don't affect
285 # calling code's node assumptions)
286 node = node.copy()
287 node['creator'] = self.getuid()
288 node['actor'] = self.getuid()
289 node['creation'] = node['activity'] = date.Date()
291 self.newnodes.setdefault(classname, {})[nodeid] = 1
292 self.cache.setdefault(classname, {})[nodeid] = node
293 self.savenode(classname, nodeid, node)
295 def setnode(self, classname, nodeid, node):
296 """ change the specified node
297 """
298 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
300 # can't set without having already loaded the node
301 self.cache[classname][nodeid] = node
302 self.savenode(classname, nodeid, node)
304 def savenode(self, classname, nodeid, node):
305 """ perform the saving of data specified by the set/addnode
306 """
307 if __debug__:
308 logging.getLogger('hyperdb').debug('save %s%s %r'%(classname, nodeid, node))
309 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
311 def getnode(self, classname, nodeid, db=None, cache=1):
312 """ get a node from the database
314 Note the "cache" parameter is not used, and exists purely for
315 backward compatibility!
316 """
317 # try the cache
318 cache_dict = self.cache.setdefault(classname, {})
319 if cache_dict.has_key(nodeid):
320 if __debug__:
321 logging.getLogger('hyperdb').debug('get %s%s cached'%(classname, nodeid))
322 self.stats['cache_hits'] += 1
323 return cache_dict[nodeid]
325 if __debug__:
326 self.stats['cache_misses'] += 1
327 start_t = time.time()
328 logging.getLogger('hyperdb').debug('get %s%s'%(classname, nodeid))
330 # get from the database and save in the cache
331 if db is None:
332 db = self.getclassdb(classname)
333 if not db.has_key(nodeid):
334 raise IndexError, "no such %s %s"%(classname, nodeid)
336 # check the uncommitted, destroyed nodes
337 if (self.destroyednodes.has_key(classname) and
338 self.destroyednodes[classname].has_key(nodeid)):
339 raise IndexError, "no such %s %s"%(classname, nodeid)
341 # decode
342 res = marshal.loads(db[nodeid])
344 # reverse the serialisation
345 res = self.unserialise(classname, res)
347 # store off in the cache dict
348 if cache:
349 cache_dict[nodeid] = res
351 if __debug__:
352 self.stats['get_items'] += (time.time() - start_t)
354 return res
356 def destroynode(self, classname, nodeid):
357 """Remove a node from the database. Called exclusively by the
358 destroy() method on Class.
359 """
360 logging.getLogger('hyperdb').info('destroy %s%s'%(classname, nodeid))
362 # remove from cache and newnodes if it's there
363 if (self.cache.has_key(classname) and
364 self.cache[classname].has_key(nodeid)):
365 del self.cache[classname][nodeid]
366 if (self.newnodes.has_key(classname) and
367 self.newnodes[classname].has_key(nodeid)):
368 del self.newnodes[classname][nodeid]
370 # see if there's any obvious commit actions that we should get rid of
371 for entry in self.transactions[:]:
372 if entry[1][:2] == (classname, nodeid):
373 self.transactions.remove(entry)
375 # add to the destroyednodes map
376 self.destroyednodes.setdefault(classname, {})[nodeid] = 1
378 # add the destroy commit action
379 self.transactions.append((self.doDestroyNode, (classname, nodeid)))
380 self.transactions.append((FileStorage.destroy, (self, classname, nodeid)))
382 def serialise(self, classname, node):
383 """Copy the node contents, converting non-marshallable data into
384 marshallable data.
385 """
386 properties = self.getclass(classname).getprops()
387 d = {}
388 for k, v in node.items():
389 if k == self.RETIRED_FLAG:
390 d[k] = v
391 continue
393 # if the property doesn't exist then we really don't care
394 if not properties.has_key(k):
395 continue
397 # get the property spec
398 prop = properties[k]
400 if isinstance(prop, hyperdb.Password) and v is not None:
401 d[k] = str(v)
402 elif isinstance(prop, hyperdb.Date) and v is not None:
403 d[k] = v.serialise()
404 elif isinstance(prop, hyperdb.Interval) and v is not None:
405 d[k] = v.serialise()
406 else:
407 d[k] = v
408 return d
410 def unserialise(self, classname, node):
411 """Decode the marshalled node data
412 """
413 properties = self.getclass(classname).getprops()
414 d = {}
415 for k, v in node.items():
416 # if the property doesn't exist, or is the "retired" flag then
417 # it won't be in the properties dict
418 if not properties.has_key(k):
419 d[k] = v
420 continue
422 # get the property spec
423 prop = properties[k]
425 if isinstance(prop, hyperdb.Date) and v is not None:
426 d[k] = date.Date(v)
427 elif isinstance(prop, hyperdb.Interval) and v is not None:
428 d[k] = date.Interval(v)
429 elif isinstance(prop, hyperdb.Password) and v is not None:
430 p = password.Password()
431 p.unpack(v)
432 d[k] = p
433 else:
434 d[k] = v
435 return d
437 def hasnode(self, classname, nodeid, db=None):
438 """ determine if the database has a given node
439 """
440 # try the cache
441 cache = self.cache.setdefault(classname, {})
442 if cache.has_key(nodeid):
443 return 1
445 # not in the cache - check the database
446 if db is None:
447 db = self.getclassdb(classname)
448 res = db.has_key(nodeid)
449 return res
451 def countnodes(self, classname, db=None):
452 count = 0
454 # include the uncommitted nodes
455 if self.newnodes.has_key(classname):
456 count += len(self.newnodes[classname])
457 if self.destroyednodes.has_key(classname):
458 count -= len(self.destroyednodes[classname])
460 # and count those in the DB
461 if db is None:
462 db = self.getclassdb(classname)
463 count = count + len(db.keys())
464 return count
467 #
468 # Files - special node properties
469 # inherited from FileStorage
471 #
472 # Journal
473 #
474 def addjournal(self, classname, nodeid, action, params, creator=None,
475 creation=None):
476 """ Journal the Action
477 'action' may be:
479 'create' or 'set' -- 'params' is a dictionary of property values
480 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
481 'retire' -- 'params' is None
483 'creator' -- the user performing the action, which defaults to
484 the current user.
485 """
486 if __debug__:
487 logging.getLogger('hyperdb').debug('addjournal %s%s %s %r %s %r'%(classname,
488 nodeid, action, params, creator, creation))
489 if creator is None:
490 creator = self.getuid()
491 self.transactions.append((self.doSaveJournal, (classname, nodeid,
492 action, params, creator, creation)))
494 def setjournal(self, classname, nodeid, journal):
495 """Set the journal to the "journal" list."""
496 if __debug__:
497 logging.getLogger('hyperdb').debug('setjournal %s%s %r'%(classname,
498 nodeid, journal))
499 self.transactions.append((self.doSetJournal, (classname, nodeid,
500 journal)))
502 def getjournal(self, classname, nodeid):
503 """ get the journal for id
505 Raise IndexError if the node doesn't exist (as per history()'s
506 API)
507 """
508 # our journal result
509 res = []
511 # add any journal entries for transactions not committed to the
512 # database
513 for method, args in self.transactions:
514 if method != self.doSaveJournal:
515 continue
516 (cache_classname, cache_nodeid, cache_action, cache_params,
517 cache_creator, cache_creation) = args
518 if cache_classname == classname and cache_nodeid == nodeid:
519 if not cache_creator:
520 cache_creator = self.getuid()
521 if not cache_creation:
522 cache_creation = date.Date()
523 res.append((cache_nodeid, cache_creation, cache_creator,
524 cache_action, cache_params))
526 # attempt to open the journal - in some rare cases, the journal may
527 # not exist
528 try:
529 db = self.opendb('journals.%s'%classname, 'r')
530 except anydbm.error, error:
531 if str(error) == "need 'c' or 'n' flag to open new db":
532 raise IndexError, 'no such %s %s'%(classname, nodeid)
533 elif error.args[0] != 2:
534 # this isn't a "not found" error, be alarmed!
535 raise
536 if res:
537 # we have unsaved journal entries, return them
538 return res
539 raise IndexError, 'no such %s %s'%(classname, nodeid)
540 try:
541 journal = marshal.loads(db[nodeid])
542 except KeyError:
543 db.close()
544 if res:
545 # we have some unsaved journal entries, be happy!
546 return res
547 raise IndexError, 'no such %s %s'%(classname, nodeid)
548 db.close()
550 # add all the saved journal entries for this node
551 for nodeid, date_stamp, user, action, params in journal:
552 res.append((nodeid, date.Date(date_stamp), user, action, params))
553 return res
555 def pack(self, pack_before):
556 """ Delete all journal entries except "create" before 'pack_before'.
557 """
558 pack_before = pack_before.serialise()
559 for classname in self.getclasses():
560 packed = 0
561 # get the journal db
562 db_name = 'journals.%s'%classname
563 path = os.path.join(os.getcwd(), self.dir, classname)
564 db_type = self.determine_db_type(path)
565 db = self.opendb(db_name, 'w')
567 for key in db.keys():
568 # get the journal for this db entry
569 journal = marshal.loads(db[key])
570 l = []
571 last_set_entry = None
572 for entry in journal:
573 # unpack the entry
574 (nodeid, date_stamp, self.journaltag, action,
575 params) = entry
576 # if the entry is after the pack date, _or_ the initial
577 # create entry, then it stays
578 if date_stamp > pack_before or action == 'create':
579 l.append(entry)
580 else:
581 packed += 1
582 db[key] = marshal.dumps(l)
584 logging.getLogger('hyperdb').info('packed %d %s items'%(packed,
585 classname))
587 if db_type == 'gdbm':
588 db.reorganize()
589 db.close()
592 #
593 # Basic transaction support
594 #
595 def commit(self, fail_ok=False):
596 """ Commit the current transactions.
598 Save all data changed since the database was opened or since the
599 last commit() or rollback().
601 fail_ok indicates that the commit is allowed to fail. This is used
602 in the web interface when committing cleaning of the session
603 database. We don't care if there's a concurrency issue there.
605 The only backend this seems to affect is postgres.
606 """
607 logging.getLogger('hyperdb').info('commit %s transactions'%(
608 len(self.transactions)))
610 # keep a handle to all the database files opened
611 self.databases = {}
613 try:
614 # now, do all the transactions
615 reindex = {}
616 for method, args in self.transactions:
617 reindex[method(*args)] = 1
618 finally:
619 # make sure we close all the database files
620 for db in self.databases.values():
621 db.close()
622 del self.databases
624 # clear the transactions list now so the blobfile implementation
625 # doesn't think there's still pending file commits when it tries
626 # to access the file data
627 self.transactions = []
629 # reindex the nodes that request it
630 for classname, nodeid in filter(None, reindex.keys()):
631 self.getclass(classname).index(nodeid)
633 # save the indexer state
634 self.indexer.save_index()
636 self.clearCache()
638 def clearCache(self):
639 # all transactions committed, back to normal
640 self.cache = {}
641 self.dirtynodes = {}
642 self.newnodes = {}
643 self.destroyednodes = {}
644 self.transactions = []
646 def getCachedClassDB(self, classname):
647 """ get the class db, looking in our cache of databases for commit
648 """
649 # get the database handle
650 db_name = 'nodes.%s'%classname
651 if not self.databases.has_key(db_name):
652 self.databases[db_name] = self.getclassdb(classname, 'c')
653 return self.databases[db_name]
655 def doSaveNode(self, classname, nodeid, node):
656 db = self.getCachedClassDB(classname)
658 # now save the marshalled data
659 db[nodeid] = marshal.dumps(self.serialise(classname, node))
661 # return the classname, nodeid so we reindex this content
662 return (classname, nodeid)
664 def getCachedJournalDB(self, classname):
665 """ get the journal db, looking in our cache of databases for commit
666 """
667 # get the database handle
668 db_name = 'journals.%s'%classname
669 if not self.databases.has_key(db_name):
670 self.databases[db_name] = self.opendb(db_name, 'c')
671 return self.databases[db_name]
673 def doSaveJournal(self, classname, nodeid, action, params, creator,
674 creation):
675 # serialise the parameters now if necessary
676 if isinstance(params, type({})):
677 if action in ('set', 'create'):
678 params = self.serialise(classname, params)
680 # handle supply of the special journalling parameters (usually
681 # supplied on importing an existing database)
682 journaltag = creator
683 if creation:
684 journaldate = creation.serialise()
685 else:
686 journaldate = date.Date().serialise()
688 # create the journal entry
689 entry = (nodeid, journaldate, journaltag, action, params)
691 db = self.getCachedJournalDB(classname)
693 # now insert the journal entry
694 if db.has_key(nodeid):
695 # append to existing
696 s = db[nodeid]
697 l = marshal.loads(s)
698 l.append(entry)
699 else:
700 l = [entry]
702 db[nodeid] = marshal.dumps(l)
704 def doSetJournal(self, classname, nodeid, journal):
705 l = []
706 for nodeid, journaldate, journaltag, action, params in journal:
707 # serialise the parameters now if necessary
708 if isinstance(params, type({})):
709 if action in ('set', 'create'):
710 params = self.serialise(classname, params)
711 journaldate = journaldate.serialise()
712 l.append((nodeid, journaldate, journaltag, action, params))
713 db = self.getCachedJournalDB(classname)
714 db[nodeid] = marshal.dumps(l)
716 def doDestroyNode(self, classname, nodeid):
717 # delete from the class database
718 db = self.getCachedClassDB(classname)
719 if db.has_key(nodeid):
720 del db[nodeid]
722 # delete from the database
723 db = self.getCachedJournalDB(classname)
724 if db.has_key(nodeid):
725 del db[nodeid]
727 def rollback(self):
728 """ Reverse all actions from the current transaction.
729 """
730 logging.getLogger('hyperdb').info('rollback %s transactions'%(
731 len(self.transactions)))
733 for method, args in self.transactions:
734 # delete temporary files
735 if method == self.doStoreFile:
736 self.rollbackStoreFile(*args)
737 self.cache = {}
738 self.dirtynodes = {}
739 self.newnodes = {}
740 self.destroyednodes = {}
741 self.transactions = []
743 def close(self):
744 """ Nothing to do
745 """
746 if self.lockfile is not None:
747 locking.release_lock(self.lockfile)
748 self.lockfile.close()
749 self.lockfile = None
751 _marker = []
752 class Class(hyperdb.Class):
753 """The handle to a particular class of nodes in a hyperdatabase."""
755 def enableJournalling(self):
756 """Turn journalling on for this class
757 """
758 self.do_journal = 1
760 def disableJournalling(self):
761 """Turn journalling off for this class
762 """
763 self.do_journal = 0
765 # Editing nodes:
767 def create(self, **propvalues):
768 """Create a new node of this class and return its id.
770 The keyword arguments in 'propvalues' map property names to values.
772 The values of arguments must be acceptable for the types of their
773 corresponding properties or a TypeError is raised.
775 If this class has a key property, it must be present and its value
776 must not collide with other key strings or a ValueError is raised.
778 Any other properties on this class that are missing from the
779 'propvalues' dictionary are set to None.
781 If an id in a link or multilink property does not refer to a valid
782 node, an IndexError is raised.
784 These operations trigger detectors and can be vetoed. Attempts
785 to modify the "creation" or "activity" properties cause a KeyError.
786 """
787 self.fireAuditors('create', None, propvalues)
788 newid = self.create_inner(**propvalues)
789 self.fireReactors('create', newid, None)
790 return newid
792 def create_inner(self, **propvalues):
793 """ Called by create, in-between the audit and react calls.
794 """
795 if propvalues.has_key('id'):
796 raise KeyError, '"id" is reserved'
798 if self.db.journaltag is None:
799 raise hyperdb.DatabaseError, _('Database open read-only')
801 if propvalues.has_key('creation') or propvalues.has_key('activity'):
802 raise KeyError, '"creation" and "activity" are reserved'
803 # new node's id
804 newid = self.db.newid(self.classname)
806 # validate propvalues
807 num_re = re.compile('^\d+$')
808 for key, value in propvalues.items():
809 if key == self.key:
810 try:
811 self.lookup(value)
812 except KeyError:
813 pass
814 else:
815 raise ValueError, 'node with key "%s" exists'%value
817 # try to handle this property
818 try:
819 prop = self.properties[key]
820 except KeyError:
821 raise KeyError, '"%s" has no property "%s"'%(self.classname,
822 key)
824 if value is not None and isinstance(prop, hyperdb.Link):
825 if type(value) != type(''):
826 raise ValueError, 'link value must be String'
827 link_class = self.properties[key].classname
828 # if it isn't a number, it's a key
829 if not num_re.match(value):
830 try:
831 value = self.db.classes[link_class].lookup(value)
832 except (TypeError, KeyError):
833 raise IndexError, 'new property "%s": %s not a %s'%(
834 key, value, link_class)
835 elif not self.db.getclass(link_class).hasnode(value):
836 raise IndexError, '%s has no node %s'%(link_class, value)
838 # save off the value
839 propvalues[key] = value
841 # register the link with the newly linked node
842 if self.do_journal and self.properties[key].do_journal:
843 self.db.addjournal(link_class, value, 'link',
844 (self.classname, newid, key))
846 elif isinstance(prop, hyperdb.Multilink):
847 if value is None:
848 value = []
849 if not hasattr(value, '__iter__'):
850 raise TypeError, 'new property "%s" not an iterable of ids'%key
852 # clean up and validate the list of links
853 link_class = self.properties[key].classname
854 l = []
855 for entry in value:
856 if type(entry) != type(''):
857 raise ValueError, '"%s" multilink value (%r) '\
858 'must contain Strings'%(key, value)
859 # if it isn't a number, it's a key
860 if not num_re.match(entry):
861 try:
862 entry = self.db.classes[link_class].lookup(entry)
863 except (TypeError, KeyError):
864 raise IndexError, 'new property "%s": %s not a %s'%(
865 key, entry, self.properties[key].classname)
866 l.append(entry)
867 value = l
868 propvalues[key] = value
870 # handle additions
871 for nodeid in value:
872 if not self.db.getclass(link_class).hasnode(nodeid):
873 raise IndexError, '%s has no node %s'%(link_class,
874 nodeid)
875 # register the link with the newly linked node
876 if self.do_journal and self.properties[key].do_journal:
877 self.db.addjournal(link_class, nodeid, 'link',
878 (self.classname, newid, key))
880 elif isinstance(prop, hyperdb.String):
881 if type(value) != type('') and type(value) != type(u''):
882 raise TypeError, 'new property "%s" not a string'%key
883 if prop.indexme:
884 self.db.indexer.add_text((self.classname, newid, key),
885 value)
887 elif isinstance(prop, hyperdb.Password):
888 if not isinstance(value, password.Password):
889 raise TypeError, 'new property "%s" not a Password'%key
891 elif isinstance(prop, hyperdb.Date):
892 if value is not None and not isinstance(value, date.Date):
893 raise TypeError, 'new property "%s" not a Date'%key
895 elif isinstance(prop, hyperdb.Interval):
896 if value is not None and not isinstance(value, date.Interval):
897 raise TypeError, 'new property "%s" not an Interval'%key
899 elif value is not None and isinstance(prop, hyperdb.Number):
900 try:
901 float(value)
902 except ValueError:
903 raise TypeError, 'new property "%s" not numeric'%key
905 elif value is not None and isinstance(prop, hyperdb.Boolean):
906 try:
907 int(value)
908 except ValueError:
909 raise TypeError, 'new property "%s" not boolean'%key
911 # make sure there's data where there needs to be
912 for key, prop in self.properties.items():
913 if propvalues.has_key(key):
914 continue
915 if key == self.key:
916 raise ValueError, 'key property "%s" is required'%key
917 if isinstance(prop, hyperdb.Multilink):
918 propvalues[key] = []
920 # done
921 self.db.addnode(self.classname, newid, propvalues)
922 if self.do_journal:
923 self.db.addjournal(self.classname, newid, 'create', {})
925 return newid
927 def get(self, nodeid, propname, default=_marker, cache=1):
928 """Get the value of a property on an existing node of this class.
930 'nodeid' must be the id of an existing node of this class or an
931 IndexError is raised. 'propname' must be the name of a property
932 of this class or a KeyError is raised.
934 'cache' exists for backward compatibility, and is not used.
936 Attempts to get the "creation" or "activity" properties should
937 do the right thing.
938 """
939 if propname == 'id':
940 return nodeid
942 # get the node's dict
943 d = self.db.getnode(self.classname, nodeid)
945 # check for one of the special props
946 if propname == 'creation':
947 if d.has_key('creation'):
948 return d['creation']
949 if not self.do_journal:
950 raise ValueError, 'Journalling is disabled for this class'
951 journal = self.db.getjournal(self.classname, nodeid)
952 if journal:
953 return journal[0][1]
954 else:
955 # on the strange chance that there's no journal
956 return date.Date()
957 if propname == 'activity':
958 if d.has_key('activity'):
959 return d['activity']
960 if not self.do_journal:
961 raise ValueError, 'Journalling is disabled for this class'
962 journal = self.db.getjournal(self.classname, nodeid)
963 if journal:
964 return self.db.getjournal(self.classname, nodeid)[-1][1]
965 else:
966 # on the strange chance that there's no journal
967 return date.Date()
968 if propname == 'creator':
969 if d.has_key('creator'):
970 return d['creator']
971 if not self.do_journal:
972 raise ValueError, 'Journalling is disabled for this class'
973 journal = self.db.getjournal(self.classname, nodeid)
974 if journal:
975 num_re = re.compile('^\d+$')
976 value = journal[0][2]
977 if num_re.match(value):
978 return value
979 else:
980 # old-style "username" journal tag
981 try:
982 return self.db.user.lookup(value)
983 except KeyError:
984 # user's been retired, return admin
985 return '1'
986 else:
987 return self.db.getuid()
988 if propname == 'actor':
989 if d.has_key('actor'):
990 return d['actor']
991 if not self.do_journal:
992 raise ValueError, 'Journalling is disabled for this class'
993 journal = self.db.getjournal(self.classname, nodeid)
994 if journal:
995 num_re = re.compile('^\d+$')
996 value = journal[-1][2]
997 if num_re.match(value):
998 return value
999 else:
1000 # old-style "username" journal tag
1001 try:
1002 return self.db.user.lookup(value)
1003 except KeyError:
1004 # user's been retired, return admin
1005 return '1'
1006 else:
1007 return self.db.getuid()
1009 # get the property (raises KeyErorr if invalid)
1010 prop = self.properties[propname]
1012 if not d.has_key(propname):
1013 if default is _marker:
1014 if isinstance(prop, hyperdb.Multilink):
1015 return []
1016 else:
1017 return None
1018 else:
1019 return default
1021 # return a dupe of the list so code doesn't get confused
1022 if isinstance(prop, hyperdb.Multilink):
1023 return d[propname][:]
1025 return d[propname]
1027 def set(self, nodeid, **propvalues):
1028 """Modify a property on an existing node of this class.
1030 'nodeid' must be the id of an existing node of this class or an
1031 IndexError is raised.
1033 Each key in 'propvalues' must be the name of a property of this
1034 class or a KeyError is raised.
1036 All values in 'propvalues' must be acceptable types for their
1037 corresponding properties or a TypeError is raised.
1039 If the value of the key property is set, it must not collide with
1040 other key strings or a ValueError is raised.
1042 If the value of a Link or Multilink property contains an invalid
1043 node id, a ValueError is raised.
1045 These operations trigger detectors and can be vetoed. Attempts
1046 to modify the "creation" or "activity" properties cause a KeyError.
1047 """
1048 self.fireAuditors('set', nodeid, propvalues)
1049 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1050 for name,prop in self.getprops(protected=0).items():
1051 if oldvalues.has_key(name):
1052 continue
1053 if isinstance(prop, hyperdb.Multilink):
1054 oldvalues[name] = []
1055 else:
1056 oldvalues[name] = None
1057 propvalues = self.set_inner(nodeid, **propvalues)
1058 self.fireReactors('set', nodeid, oldvalues)
1059 return propvalues
1061 def set_inner(self, nodeid, **propvalues):
1062 """ Called by set, in-between the audit and react calls.
1063 """
1064 if not propvalues:
1065 return propvalues
1067 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1068 raise KeyError, '"creation" and "activity" are reserved'
1070 if propvalues.has_key('id'):
1071 raise KeyError, '"id" is reserved'
1073 if self.db.journaltag is None:
1074 raise hyperdb.DatabaseError, _('Database open read-only')
1076 node = self.db.getnode(self.classname, nodeid)
1077 if node.has_key(self.db.RETIRED_FLAG):
1078 raise IndexError
1079 num_re = re.compile('^\d+$')
1081 # if the journal value is to be different, store it in here
1082 journalvalues = {}
1084 for propname, value in propvalues.items():
1085 # check to make sure we're not duplicating an existing key
1086 if propname == self.key and node[propname] != value:
1087 try:
1088 self.lookup(value)
1089 except KeyError:
1090 pass
1091 else:
1092 raise ValueError, 'node with key "%s" exists'%value
1094 # this will raise the KeyError if the property isn't valid
1095 # ... we don't use getprops() here because we only care about
1096 # the writeable properties.
1097 try:
1098 prop = self.properties[propname]
1099 except KeyError:
1100 raise KeyError, '"%s" has no property named "%s"'%(
1101 self.classname, propname)
1103 # if the value's the same as the existing value, no sense in
1104 # doing anything
1105 current = node.get(propname, None)
1106 if value == current:
1107 del propvalues[propname]
1108 continue
1109 journalvalues[propname] = current
1111 # do stuff based on the prop type
1112 if isinstance(prop, hyperdb.Link):
1113 link_class = prop.classname
1114 # if it isn't a number, it's a key
1115 if value is not None and not isinstance(value, type('')):
1116 raise ValueError, 'property "%s" link value be a string'%(
1117 propname)
1118 if isinstance(value, type('')) and not num_re.match(value):
1119 try:
1120 value = self.db.classes[link_class].lookup(value)
1121 except (TypeError, KeyError):
1122 raise IndexError, 'new property "%s": %s not a %s'%(
1123 propname, value, prop.classname)
1125 if (value is not None and
1126 not self.db.getclass(link_class).hasnode(value)):
1127 raise IndexError, '%s has no node %s'%(link_class, value)
1129 if self.do_journal and prop.do_journal:
1130 # register the unlink with the old linked node
1131 if node.has_key(propname) and node[propname] is not None:
1132 self.db.addjournal(link_class, node[propname], 'unlink',
1133 (self.classname, nodeid, propname))
1135 # register the link with the newly linked node
1136 if value is not None:
1137 self.db.addjournal(link_class, value, 'link',
1138 (self.classname, nodeid, propname))
1140 elif isinstance(prop, hyperdb.Multilink):
1141 if value is None:
1142 value = []
1143 if not hasattr(value, '__iter__'):
1144 raise TypeError, 'new property "%s" not an iterable of'\
1145 ' ids'%propname
1146 link_class = self.properties[propname].classname
1147 l = []
1148 for entry in value:
1149 # if it isn't a number, it's a key
1150 if type(entry) != type(''):
1151 raise ValueError, 'new property "%s" link value ' \
1152 'must be a string'%propname
1153 if not num_re.match(entry):
1154 try:
1155 entry = self.db.classes[link_class].lookup(entry)
1156 except (TypeError, KeyError):
1157 raise IndexError, 'new property "%s": %s not a %s'%(
1158 propname, entry,
1159 self.properties[propname].classname)
1160 l.append(entry)
1161 value = l
1162 propvalues[propname] = value
1164 # figure the journal entry for this property
1165 add = []
1166 remove = []
1168 # handle removals
1169 if node.has_key(propname):
1170 l = node[propname]
1171 else:
1172 l = []
1173 for id in l[:]:
1174 if id in value:
1175 continue
1176 # register the unlink with the old linked node
1177 if self.do_journal and self.properties[propname].do_journal:
1178 self.db.addjournal(link_class, id, 'unlink',
1179 (self.classname, nodeid, propname))
1180 l.remove(id)
1181 remove.append(id)
1183 # handle additions
1184 for id in value:
1185 if not self.db.getclass(link_class).hasnode(id):
1186 raise IndexError, '%s has no node %s'%(link_class, id)
1187 if id in l:
1188 continue
1189 # register the link with the newly linked node
1190 if self.do_journal and self.properties[propname].do_journal:
1191 self.db.addjournal(link_class, id, 'link',
1192 (self.classname, nodeid, propname))
1193 l.append(id)
1194 add.append(id)
1196 # figure the journal entry
1197 l = []
1198 if add:
1199 l.append(('+', add))
1200 if remove:
1201 l.append(('-', remove))
1202 if l:
1203 journalvalues[propname] = tuple(l)
1205 elif isinstance(prop, hyperdb.String):
1206 if value is not None and type(value) != type('') and type(value) != type(u''):
1207 raise TypeError, 'new property "%s" not a string'%propname
1208 if prop.indexme:
1209 self.db.indexer.add_text((self.classname, nodeid, propname),
1210 value)
1212 elif isinstance(prop, hyperdb.Password):
1213 if not isinstance(value, password.Password):
1214 raise TypeError, 'new property "%s" not a Password'%propname
1215 propvalues[propname] = value
1217 elif value is not None and isinstance(prop, hyperdb.Date):
1218 if not isinstance(value, date.Date):
1219 raise TypeError, 'new property "%s" not a Date'% propname
1220 propvalues[propname] = value
1222 elif value is not None and isinstance(prop, hyperdb.Interval):
1223 if not isinstance(value, date.Interval):
1224 raise TypeError, 'new property "%s" not an '\
1225 'Interval'%propname
1226 propvalues[propname] = value
1228 elif value is not None and isinstance(prop, hyperdb.Number):
1229 try:
1230 float(value)
1231 except ValueError:
1232 raise TypeError, 'new property "%s" not numeric'%propname
1234 elif value is not None and isinstance(prop, hyperdb.Boolean):
1235 try:
1236 int(value)
1237 except ValueError:
1238 raise TypeError, 'new property "%s" not boolean'%propname
1240 node[propname] = value
1242 # nothing to do?
1243 if not propvalues:
1244 return propvalues
1246 # update the activity time
1247 node['activity'] = date.Date()
1248 node['actor'] = self.db.getuid()
1250 # do the set, and journal it
1251 self.db.setnode(self.classname, nodeid, node)
1253 if self.do_journal:
1254 self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
1256 return propvalues
1258 def retire(self, nodeid):
1259 """Retire a node.
1261 The properties on the node remain available from the get() method,
1262 and the node's id is never reused.
1264 Retired nodes are not returned by the find(), list(), or lookup()
1265 methods, and other nodes may reuse the values of their key properties.
1267 These operations trigger detectors and can be vetoed. Attempts
1268 to modify the "creation" or "activity" properties cause a KeyError.
1269 """
1270 if self.db.journaltag is None:
1271 raise hyperdb.DatabaseError, _('Database open read-only')
1273 self.fireAuditors('retire', nodeid, None)
1275 node = self.db.getnode(self.classname, nodeid)
1276 node[self.db.RETIRED_FLAG] = 1
1277 self.db.setnode(self.classname, nodeid, node)
1278 if self.do_journal:
1279 self.db.addjournal(self.classname, nodeid, 'retired', None)
1281 self.fireReactors('retire', nodeid, None)
1283 def restore(self, nodeid):
1284 """Restpre a retired node.
1286 Make node available for all operations like it was before retirement.
1287 """
1288 if self.db.journaltag is None:
1289 raise hyperdb.DatabaseError, _('Database open read-only')
1291 node = self.db.getnode(self.classname, nodeid)
1292 # check if key property was overrided
1293 key = self.getkey()
1294 try:
1295 id = self.lookup(node[key])
1296 except KeyError:
1297 pass
1298 else:
1299 raise KeyError, "Key property (%s) of retired node clashes with \
1300 existing one (%s)" % (key, node[key])
1301 # Now we can safely restore node
1302 self.fireAuditors('restore', nodeid, None)
1303 del node[self.db.RETIRED_FLAG]
1304 self.db.setnode(self.classname, nodeid, node)
1305 if self.do_journal:
1306 self.db.addjournal(self.classname, nodeid, 'restored', None)
1308 self.fireReactors('restore', nodeid, None)
1310 def is_retired(self, nodeid, cldb=None):
1311 """Return true if the node is retired.
1312 """
1313 node = self.db.getnode(self.classname, nodeid, cldb)
1314 if node.has_key(self.db.RETIRED_FLAG):
1315 return 1
1316 return 0
1318 def destroy(self, nodeid):
1319 """Destroy a node.
1321 WARNING: this method should never be used except in extremely rare
1322 situations where there could never be links to the node being
1323 deleted
1325 WARNING: use retire() instead
1327 WARNING: the properties of this node will not be available ever again
1329 WARNING: really, use retire() instead
1331 Well, I think that's enough warnings. This method exists mostly to
1332 support the session storage of the cgi interface.
1333 """
1334 if self.db.journaltag is None:
1335 raise hyperdb.DatabaseError, _('Database open read-only')
1336 self.db.destroynode(self.classname, nodeid)
1338 def history(self, nodeid):
1339 """Retrieve the journal of edits on a particular node.
1341 'nodeid' must be the id of an existing node of this class or an
1342 IndexError is raised.
1344 The returned list contains tuples of the form
1346 (nodeid, date, tag, action, params)
1348 'date' is a Timestamp object specifying the time of the change and
1349 'tag' is the journaltag specified when the database was opened.
1350 """
1351 if not self.do_journal:
1352 raise ValueError, 'Journalling is disabled for this class'
1353 return self.db.getjournal(self.classname, nodeid)
1355 # Locating nodes:
1356 def hasnode(self, nodeid):
1357 """Determine if the given nodeid actually exists
1358 """
1359 return self.db.hasnode(self.classname, nodeid)
1361 def setkey(self, propname):
1362 """Select a String property of this class to be the key property.
1364 'propname' must be the name of a String property of this class or
1365 None, or a TypeError is raised. The values of the key property on
1366 all existing nodes must be unique or a ValueError is raised. If the
1367 property doesn't exist, KeyError is raised.
1368 """
1369 prop = self.getprops()[propname]
1370 if not isinstance(prop, hyperdb.String):
1371 raise TypeError, 'key properties must be String'
1372 self.key = propname
1374 def getkey(self):
1375 """Return the name of the key property for this class or None."""
1376 return self.key
1378 # TODO: set up a separate index db file for this? profile?
1379 def lookup(self, keyvalue):
1380 """Locate a particular node by its key property and return its id.
1382 If this class has no key property, a TypeError is raised. If the
1383 'keyvalue' matches one of the values for the key property among
1384 the nodes in this class, the matching node's id is returned;
1385 otherwise a KeyError is raised.
1386 """
1387 if not self.key:
1388 raise TypeError, 'No key property set for class %s'%self.classname
1389 cldb = self.db.getclassdb(self.classname)
1390 try:
1391 for nodeid in self.getnodeids(cldb):
1392 node = self.db.getnode(self.classname, nodeid, cldb)
1393 if node.has_key(self.db.RETIRED_FLAG):
1394 continue
1395 if not node.has_key(self.key):
1396 continue
1397 if node[self.key] == keyvalue:
1398 return nodeid
1399 finally:
1400 cldb.close()
1401 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1402 keyvalue, self.classname)
1404 # change from spec - allows multiple props to match
1405 def find(self, **propspec):
1406 """Get the ids of nodes in this class which link to the given nodes.
1408 'propspec' consists of keyword args propname=nodeid or
1409 propname={nodeid:1, }
1410 'propname' must be the name of a property in this class, or a
1411 KeyError is raised. That property must be a Link or
1412 Multilink property, or a TypeError is raised.
1414 Any node in this class whose 'propname' property links to any of
1415 the nodeids will be returned. Examples::
1417 db.issue.find(messages='1')
1418 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1419 """
1420 propspec = propspec.items()
1421 for propname, itemids in propspec:
1422 # check the prop is OK
1423 prop = self.properties[propname]
1424 if not isinstance(prop, hyperdb.Link) and not isinstance(prop, hyperdb.Multilink):
1425 raise TypeError, "'%s' not a Link/Multilink property"%propname
1427 # ok, now do the find
1428 cldb = self.db.getclassdb(self.classname)
1429 l = []
1430 try:
1431 for id in self.getnodeids(db=cldb):
1432 item = self.db.getnode(self.classname, id, db=cldb)
1433 if item.has_key(self.db.RETIRED_FLAG):
1434 continue
1435 for propname, itemids in propspec:
1436 if type(itemids) is not type({}):
1437 itemids = {itemids:1}
1439 # special case if the item doesn't have this property
1440 if not item.has_key(propname):
1441 if itemids.has_key(None):
1442 l.append(id)
1443 break
1444 continue
1446 # grab the property definition and its value on this item
1447 prop = self.properties[propname]
1448 value = item[propname]
1449 if isinstance(prop, hyperdb.Link) and itemids.has_key(value):
1450 l.append(id)
1451 break
1452 elif isinstance(prop, hyperdb.Multilink):
1453 hit = 0
1454 for v in value:
1455 if itemids.has_key(v):
1456 l.append(id)
1457 hit = 1
1458 break
1459 if hit:
1460 break
1461 finally:
1462 cldb.close()
1463 return l
1465 def stringFind(self, **requirements):
1466 """Locate a particular node by matching a set of its String
1467 properties in a caseless search.
1469 If the property is not a String property, a TypeError is raised.
1471 The return is a list of the id of all nodes that match.
1472 """
1473 for propname in requirements.keys():
1474 prop = self.properties[propname]
1475 if not isinstance(prop, hyperdb.String):
1476 raise TypeError, "'%s' not a String property"%propname
1477 requirements[propname] = requirements[propname].lower()
1478 l = []
1479 cldb = self.db.getclassdb(self.classname)
1480 try:
1481 for nodeid in self.getnodeids(cldb):
1482 node = self.db.getnode(self.classname, nodeid, cldb)
1483 if node.has_key(self.db.RETIRED_FLAG):
1484 continue
1485 for key, value in requirements.items():
1486 if not node.has_key(key):
1487 break
1488 if node[key] is None or node[key].lower() != value:
1489 break
1490 else:
1491 l.append(nodeid)
1492 finally:
1493 cldb.close()
1494 return l
1496 def list(self):
1497 """ Return a list of the ids of the active nodes in this class.
1498 """
1499 l = []
1500 cn = self.classname
1501 cldb = self.db.getclassdb(cn)
1502 try:
1503 for nodeid in self.getnodeids(cldb):
1504 node = self.db.getnode(cn, nodeid, cldb)
1505 if node.has_key(self.db.RETIRED_FLAG):
1506 continue
1507 l.append(nodeid)
1508 finally:
1509 cldb.close()
1510 l.sort()
1511 return l
1513 def getnodeids(self, db=None, retired=None):
1514 """ Return a list of ALL nodeids
1516 Set retired=None to get all nodes. Otherwise it'll get all the
1517 retired or non-retired nodes, depending on the flag.
1518 """
1519 res = []
1521 # start off with the new nodes
1522 if self.db.newnodes.has_key(self.classname):
1523 res += self.db.newnodes[self.classname].keys()
1525 must_close = False
1526 if db is None:
1527 db = self.db.getclassdb(self.classname)
1528 must_close = True
1529 try:
1530 res = res + db.keys()
1532 # remove the uncommitted, destroyed nodes
1533 if self.db.destroyednodes.has_key(self.classname):
1534 for nodeid in self.db.destroyednodes[self.classname].keys():
1535 if db.has_key(nodeid):
1536 res.remove(nodeid)
1538 # check retired flag
1539 if retired is False or retired is True:
1540 l = []
1541 for nodeid in res:
1542 node = self.db.getnode(self.classname, nodeid, db)
1543 is_ret = node.has_key(self.db.RETIRED_FLAG)
1544 if retired == is_ret:
1545 l.append(nodeid)
1546 res = l
1547 finally:
1548 if must_close:
1549 db.close()
1550 return res
1552 def _filter(self, search_matches, filterspec, proptree,
1553 num_re = re.compile('^\d+$')):
1554 """Return a list of the ids of the active nodes in this class that
1555 match the 'filter' spec, sorted by the group spec and then the
1556 sort spec.
1558 "filterspec" is {propname: value(s)}
1560 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1561 and prop is a prop name or None
1563 "search_matches" is a sequence type or None
1565 The filter must match all properties specificed. If the property
1566 value to match is a list:
1568 1. String properties must match all elements in the list, and
1569 2. Other properties must match any of the elements in the list.
1570 """
1571 if __debug__:
1572 start_t = time.time()
1574 cn = self.classname
1576 # optimise filterspec
1577 l = []
1578 props = self.getprops()
1579 LINK = 'spec:link'
1580 MULTILINK = 'spec:multilink'
1581 STRING = 'spec:string'
1582 DATE = 'spec:date'
1583 INTERVAL = 'spec:interval'
1584 OTHER = 'spec:other'
1586 for k, v in filterspec.items():
1587 propclass = props[k]
1588 if isinstance(propclass, hyperdb.Link):
1589 if type(v) is not type([]):
1590 v = [v]
1591 u = []
1592 for entry in v:
1593 # the value -1 is a special "not set" sentinel
1594 if entry == '-1':
1595 entry = None
1596 u.append(entry)
1597 l.append((LINK, k, u))
1598 elif isinstance(propclass, hyperdb.Multilink):
1599 # the value -1 is a special "not set" sentinel
1600 if v in ('-1', ['-1']):
1601 v = []
1602 elif type(v) is not type([]):
1603 v = [v]
1604 l.append((MULTILINK, k, v))
1605 elif isinstance(propclass, hyperdb.String) and k != 'id':
1606 if type(v) is not type([]):
1607 v = [v]
1608 for v in v:
1609 # simple glob searching
1610 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1611 v = v.replace('?', '.')
1612 v = v.replace('*', '.*?')
1613 l.append((STRING, k, re.compile(v, re.I)))
1614 elif isinstance(propclass, hyperdb.Date):
1615 try:
1616 date_rng = propclass.range_from_raw(v, self.db)
1617 l.append((DATE, k, date_rng))
1618 except ValueError:
1619 # If range creation fails - ignore that search parameter
1620 pass
1621 elif isinstance(propclass, hyperdb.Interval):
1622 try:
1623 intv_rng = date.Range(v, date.Interval)
1624 l.append((INTERVAL, k, intv_rng))
1625 except ValueError:
1626 # If range creation fails - ignore that search parameter
1627 pass
1629 elif isinstance(propclass, hyperdb.Boolean):
1630 if type(v) != type([]):
1631 v = v.split(',')
1632 bv = []
1633 for val in v:
1634 if type(val) is type(''):
1635 bv.append(val.lower() in ('yes', 'true', 'on', '1'))
1636 else:
1637 bv.append(val)
1638 l.append((OTHER, k, bv))
1640 elif k == 'id':
1641 if type(v) != type([]):
1642 v = v.split(',')
1643 l.append((OTHER, k, [str(int(val)) for val in v]))
1645 elif isinstance(propclass, hyperdb.Number):
1646 if type(v) != type([]):
1647 v = v.split(',')
1648 l.append((OTHER, k, [float(val) for val in v]))
1650 filterspec = l
1652 # now, find all the nodes that are active and pass filtering
1653 matches = []
1654 cldb = self.db.getclassdb(cn)
1655 t = 0
1656 try:
1657 # TODO: only full-scan once (use items())
1658 for nodeid in self.getnodeids(cldb):
1659 node = self.db.getnode(cn, nodeid, cldb)
1660 if node.has_key(self.db.RETIRED_FLAG):
1661 continue
1662 # apply filter
1663 for t, k, v in filterspec:
1664 # handle the id prop
1665 if k == 'id':
1666 if nodeid not in v:
1667 break
1668 continue
1670 # get the node value
1671 nv = node.get(k, None)
1673 match = 0
1675 # now apply the property filter
1676 if t == LINK:
1677 # link - if this node's property doesn't appear in the
1678 # filterspec's nodeid list, skip it
1679 match = nv in v
1680 elif t == MULTILINK:
1681 # multilink - if any of the nodeids required by the
1682 # filterspec aren't in this node's property, then skip
1683 # it
1684 nv = node.get(k, [])
1686 # check for matching the absence of multilink values
1687 if not v:
1688 match = not nv
1689 else:
1690 # othewise, make sure this node has each of the
1691 # required values
1692 for want in v:
1693 if want in nv:
1694 match = 1
1695 break
1696 elif t == STRING:
1697 if nv is None:
1698 nv = ''
1699 # RE search
1700 match = v.search(nv)
1701 elif t == DATE or t == INTERVAL:
1702 if nv is None:
1703 match = v is None
1704 else:
1705 if v.to_value:
1706 if v.from_value <= nv and v.to_value >= nv:
1707 match = 1
1708 else:
1709 if v.from_value <= nv:
1710 match = 1
1711 elif t == OTHER:
1712 # straight value comparison for the other types
1713 match = nv in v
1714 if not match:
1715 break
1716 else:
1717 matches.append([nodeid, node])
1719 # filter based on full text search
1720 if search_matches is not None:
1721 k = []
1722 for v in matches:
1723 if v[0] in search_matches:
1724 k.append(v)
1725 matches = k
1727 # add sorting information to the proptree
1728 JPROPS = {'actor':1, 'activity':1, 'creator':1, 'creation':1}
1729 children = []
1730 if proptree:
1731 children = proptree.sortable_children()
1732 for pt in children:
1733 dir = pt.sort_direction
1734 prop = pt.name
1735 assert (dir and prop)
1736 propclass = props[prop]
1737 pt.sort_ids = []
1738 is_pointer = isinstance(propclass,(hyperdb.Link,
1739 hyperdb.Multilink))
1740 if not is_pointer:
1741 pt.sort_result = []
1742 try:
1743 # cache the opened link class db, if needed.
1744 lcldb = None
1745 # cache the linked class items too
1746 lcache = {}
1748 for entry in matches:
1749 itemid = entry[-2]
1750 item = entry[-1]
1751 # handle the properties that might be "faked"
1752 # also, handle possible missing properties
1753 try:
1754 v = item[prop]
1755 except KeyError:
1756 if JPROPS.has_key(prop):
1757 # force lookup of the special journal prop
1758 v = self.get(itemid, prop)
1759 else:
1760 # the node doesn't have a value for this
1761 # property
1762 v = None
1763 if isinstance(propclass, hyperdb.Multilink):
1764 v = []
1765 if prop == 'id':
1766 v = int (itemid)
1767 pt.sort_ids.append(v)
1768 if not is_pointer:
1769 pt.sort_result.append(v)
1770 continue
1772 # missing (None) values are always sorted first
1773 if v is None:
1774 pt.sort_ids.append(v)
1775 if not is_pointer:
1776 pt.sort_result.append(v)
1777 continue
1779 if isinstance(propclass, hyperdb.Link):
1780 lcn = propclass.classname
1781 link = self.db.classes[lcn]
1782 key = link.orderprop()
1783 child = pt.propdict[key]
1784 if key!='id':
1785 if not lcache.has_key(v):
1786 # open the link class db if it's not already
1787 if lcldb is None:
1788 lcldb = self.db.getclassdb(lcn)
1789 lcache[v] = self.db.getnode(lcn, v, lcldb)
1790 r = lcache[v][key]
1791 child.propdict[key].sort_ids.append(r)
1792 else:
1793 child.propdict[key].sort_ids.append(v)
1794 pt.sort_ids.append(v)
1795 if not is_pointer:
1796 r = propclass.sort_repr(pt.parent.cls, v, pt.name)
1797 pt.sort_result.append(r)
1798 finally:
1799 # if we opened the link class db, close it now
1800 if lcldb is not None:
1801 lcldb.close()
1802 del lcache
1803 finally:
1804 cldb.close()
1806 # pull the id out of the individual entries
1807 matches = [entry[-2] for entry in matches]
1808 if __debug__:
1809 self.db.stats['filtering'] += (time.time() - start_t)
1810 return matches
1812 def count(self):
1813 """Get the number of nodes in this class.
1815 If the returned integer is 'numnodes', the ids of all the nodes
1816 in this class run from 1 to numnodes, and numnodes+1 will be the
1817 id of the next node to be created in this class.
1818 """
1819 return self.db.countnodes(self.classname)
1821 # Manipulating properties:
1823 def getprops(self, protected=1):
1824 """Return a dictionary mapping property names to property objects.
1825 If the "protected" flag is true, we include protected properties -
1826 those which may not be modified.
1828 In addition to the actual properties on the node, these
1829 methods provide the "creation" and "activity" properties. If the
1830 "protected" flag is true, we include protected properties - those
1831 which may not be modified.
1832 """
1833 d = self.properties.copy()
1834 if protected:
1835 d['id'] = hyperdb.String()
1836 d['creation'] = hyperdb.Date()
1837 d['activity'] = hyperdb.Date()
1838 d['creator'] = hyperdb.Link('user')
1839 d['actor'] = hyperdb.Link('user')
1840 return d
1842 def addprop(self, **properties):
1843 """Add properties to this class.
1845 The keyword arguments in 'properties' must map names to property
1846 objects, or a TypeError is raised. None of the keys in 'properties'
1847 may collide with the names of existing properties, or a ValueError
1848 is raised before any properties have been added.
1849 """
1850 for key in properties.keys():
1851 if self.properties.has_key(key):
1852 raise ValueError, key
1853 self.properties.update(properties)
1855 def index(self, nodeid):
1856 """ Add (or refresh) the node to search indexes """
1857 # find all the String properties that have indexme
1858 for prop, propclass in self.getprops().items():
1859 if isinstance(propclass, hyperdb.String) and propclass.indexme:
1860 # index them under (classname, nodeid, property)
1861 try:
1862 value = str(self.get(nodeid, prop))
1863 except IndexError:
1864 # node has been destroyed
1865 continue
1866 self.db.indexer.add_text((self.classname, nodeid, prop), value)
1868 #
1869 # import / export support
1870 #
1871 def export_list(self, propnames, nodeid):
1872 """ Export a node - generate a list of CSV-able data in the order
1873 specified by propnames for the given node.
1874 """
1875 properties = self.getprops()
1876 l = []
1877 for prop in propnames:
1878 proptype = properties[prop]
1879 value = self.get(nodeid, prop)
1880 # "marshal" data where needed
1881 if value is None:
1882 pass
1883 elif isinstance(proptype, hyperdb.Date):
1884 value = value.get_tuple()
1885 elif isinstance(proptype, hyperdb.Interval):
1886 value = value.get_tuple()
1887 elif isinstance(proptype, hyperdb.Password):
1888 value = str(value)
1889 l.append(repr(value))
1891 # append retired flag
1892 l.append(repr(self.is_retired(nodeid)))
1894 return l
1896 def import_list(self, propnames, proplist):
1897 """ Import a node - all information including "id" is present and
1898 should not be sanity checked. Triggers are not triggered. The
1899 journal should be initialised using the "creator" and "created"
1900 information.
1902 Return the nodeid of the node imported.
1903 """
1904 if self.db.journaltag is None:
1905 raise hyperdb.DatabaseError, _('Database open read-only')
1906 properties = self.getprops()
1908 # make the new node's property map
1909 d = {}
1910 newid = None
1911 for i in range(len(propnames)):
1912 # Figure the property for this column
1913 propname = propnames[i]
1915 # Use eval to reverse the repr() used to output the CSV
1916 value = eval(proplist[i])
1918 # "unmarshal" where necessary
1919 if propname == 'id':
1920 newid = value
1921 continue
1922 elif propname == 'is retired':
1923 # is the item retired?
1924 if int(value):
1925 d[self.db.RETIRED_FLAG] = 1
1926 continue
1927 elif value is None:
1928 d[propname] = None
1929 continue
1931 prop = properties[propname]
1932 if isinstance(prop, hyperdb.Date):
1933 value = date.Date(value)
1934 elif isinstance(prop, hyperdb.Interval):
1935 value = date.Interval(value)
1936 elif isinstance(prop, hyperdb.Password):
1937 pwd = password.Password()
1938 pwd.unpack(value)
1939 value = pwd
1940 d[propname] = value
1942 # get a new id if necessary
1943 if newid is None:
1944 newid = self.db.newid(self.classname)
1946 # add the node and journal
1947 self.db.addnode(self.classname, newid, d)
1948 return newid
1950 def export_journals(self):
1951 """Export a class's journal - generate a list of lists of
1952 CSV-able data:
1954 nodeid, date, user, action, params
1956 No heading here - the columns are fixed.
1957 """
1958 properties = self.getprops()
1959 r = []
1960 for nodeid in self.getnodeids():
1961 for nodeid, date, user, action, params in self.history(nodeid):
1962 date = date.get_tuple()
1963 if action == 'set':
1964 export_data = {}
1965 for propname, value in params.items():
1966 if not properties.has_key(propname):
1967 # property no longer in the schema
1968 continue
1970 prop = properties[propname]
1971 # make sure the params are eval()'able
1972 if value is None:
1973 pass
1974 elif isinstance(prop, hyperdb.Date):
1975 # this is a hack - some dates are stored as strings
1976 if not isinstance(value, type('')):
1977 value = value.get_tuple()
1978 elif isinstance(prop, hyperdb.Interval):
1979 # hack too - some intervals are stored as strings
1980 if not isinstance(value, type('')):
1981 value = value.get_tuple()
1982 elif isinstance(prop, hyperdb.Password):
1983 value = str(value)
1984 export_data[propname] = value
1985 params = export_data
1986 l = [nodeid, date, user, action, params]
1987 r.append(map(repr, l))
1988 return r
1990 def import_journals(self, entries):
1991 """Import a class's journal.
1993 Uses setjournal() to set the journal for each item."""
1994 properties = self.getprops()
1995 d = {}
1996 for l in entries:
1997 l = map(eval, l)
1998 nodeid, jdate, user, action, params = l
1999 r = d.setdefault(nodeid, [])
2000 if action == 'set':
2001 for propname, value in params.items():
2002 prop = properties[propname]
2003 if value is None:
2004 pass
2005 elif isinstance(prop, hyperdb.Date):
2006 if type(value) == type(()):
2007 print _('WARNING: invalid date tuple %r')%(value,)
2008 value = date.Date( "2000-1-1" )
2009 value = date.Date(value)
2010 elif isinstance(prop, hyperdb.Interval):
2011 value = date.Interval(value)
2012 elif isinstance(prop, hyperdb.Password):
2013 pwd = password.Password()
2014 pwd.unpack(value)
2015 value = pwd
2016 params[propname] = value
2017 r.append((nodeid, date.Date(jdate), user, action, params))
2019 for nodeid, l in d.items():
2020 self.db.setjournal(self.classname, nodeid, l)
2022 class FileClass(hyperdb.FileClass, Class):
2023 """This class defines a large chunk of data. To support this, it has a
2024 mandatory String property "content" which is typically saved off
2025 externally to the hyperdb.
2027 The default MIME type of this data is defined by the
2028 "default_mime_type" class attribute, which may be overridden by each
2029 node if the class defines a "type" String property.
2030 """
2031 def __init__(self, db, classname, **properties):
2032 """The newly-created class automatically includes the "content"
2033 and "type" properties.
2034 """
2035 if not properties.has_key('content'):
2036 properties['content'] = hyperdb.String(indexme='yes')
2037 if not properties.has_key('type'):
2038 properties['type'] = hyperdb.String()
2039 Class.__init__(self, db, classname, **properties)
2041 def create(self, **propvalues):
2042 """ Snarf the "content" propvalue and store in a file
2043 """
2044 # we need to fire the auditors now, or the content property won't
2045 # be in propvalues for the auditors to play with
2046 self.fireAuditors('create', None, propvalues)
2048 # now remove the content property so it's not stored in the db
2049 content = propvalues['content']
2050 del propvalues['content']
2052 # make sure we have a MIME type
2053 mime_type = propvalues.get('type', self.default_mime_type)
2055 # do the database create
2056 newid = self.create_inner(**propvalues)
2058 # store off the content as a file
2059 self.db.storefile(self.classname, newid, None, content)
2061 # fire reactors
2062 self.fireReactors('create', newid, None)
2064 return newid
2066 def get(self, nodeid, propname, default=_marker, cache=1):
2067 """ Trap the content propname and get it from the file
2069 'cache' exists for backwards compatibility, and is not used.
2070 """
2071 poss_msg = 'Possibly an access right configuration problem.'
2072 if propname == 'content':
2073 try:
2074 return self.db.getfile(self.classname, nodeid, None)
2075 except IOError, (strerror):
2076 # XXX by catching this we don't see an error in the log.
2077 return 'ERROR reading file: %s%s\n%s\n%s'%(
2078 self.classname, nodeid, poss_msg, strerror)
2079 if default is not _marker:
2080 return Class.get(self, nodeid, propname, default)
2081 else:
2082 return Class.get(self, nodeid, propname)
2084 def set(self, itemid, **propvalues):
2085 """ Snarf the "content" propvalue and update it in a file
2086 """
2087 self.fireAuditors('set', itemid, propvalues)
2089 # create the oldvalues dict - fill in any missing values
2090 oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid))
2091 for name,prop in self.getprops(protected=0).items():
2092 if oldvalues.has_key(name):
2093 continue
2094 if isinstance(prop, hyperdb.Multilink):
2095 oldvalues[name] = []
2096 else:
2097 oldvalues[name] = None
2099 # now remove the content property so it's not stored in the db
2100 content = None
2101 if propvalues.has_key('content'):
2102 content = propvalues['content']
2103 del propvalues['content']
2105 # do the database update
2106 propvalues = self.set_inner(itemid, **propvalues)
2108 # do content?
2109 if content:
2110 # store and possibly index
2111 self.db.storefile(self.classname, itemid, None, content)
2112 if self.properties['content'].indexme:
2113 mime_type = self.get(itemid, 'type', self.default_mime_type)
2114 self.db.indexer.add_text((self.classname, itemid, 'content'),
2115 content, mime_type)
2116 propvalues['content'] = content
2118 # fire reactors
2119 self.fireReactors('set', itemid, oldvalues)
2120 return propvalues
2122 def index(self, nodeid):
2123 """ Add (or refresh) the node to search indexes.
2125 Use the content-type property for the content property.
2126 """
2127 # find all the String properties that have indexme
2128 for prop, propclass in self.getprops().items():
2129 if prop == 'content' and propclass.indexme:
2130 mime_type = self.get(nodeid, 'type', self.default_mime_type)
2131 self.db.indexer.add_text((self.classname, nodeid, 'content'),
2132 str(self.get(nodeid, 'content')), mime_type)
2133 elif isinstance(propclass, hyperdb.String) and propclass.indexme:
2134 # index them under (classname, nodeid, property)
2135 try:
2136 value = str(self.get(nodeid, prop))
2137 except IndexError:
2138 # node has been destroyed
2139 continue
2140 self.db.indexer.add_text((self.classname, nodeid, prop), value)
2142 # deviation from spec - was called ItemClass
2143 class IssueClass(Class, roundupdb.IssueClass):
2144 # Overridden methods:
2145 def __init__(self, db, classname, **properties):
2146 """The newly-created class automatically includes the "messages",
2147 "files", "nosy", and "superseder" properties. If the 'properties'
2148 dictionary attempts to specify any of these properties or a
2149 "creation" or "activity" property, a ValueError is raised.
2150 """
2151 if not properties.has_key('title'):
2152 properties['title'] = hyperdb.String(indexme='yes')
2153 if not properties.has_key('messages'):
2154 properties['messages'] = hyperdb.Multilink("msg")
2155 if not properties.has_key('files'):
2156 properties['files'] = hyperdb.Multilink("file")
2157 if not properties.has_key('nosy'):
2158 # note: journalling is turned off as it really just wastes
2159 # space. this behaviour may be overridden in an instance
2160 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2161 if not properties.has_key('superseder'):
2162 properties['superseder'] = hyperdb.Multilink(classname)
2163 Class.__init__(self, db, classname, **properties)
2165 # vim: set et sts=4 sw=4 :