1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 """This module defines a backend that saves the hyperdatabase in a
19 database chosen by anydbm. It is guaranteed to always be available in python
20 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
21 serious bugs, and is not available)
22 """
23 __docformat__ = 'restructuredtext'
25 try:
26 import anydbm, sys
27 # dumbdbm only works in python 2.1.2+
28 if sys.version_info < (2,1,2):
29 import dumbdbm
30 assert anydbm._defaultmod != dumbdbm
31 del dumbdbm
32 except AssertionError:
33 print "WARNING: you should upgrade to python 2.1.3"
35 import whichdb, os, marshal, re, weakref, string, copy, time, shutil, logging
37 from roundup import hyperdb, date, password, roundupdb, security, support
38 from roundup.support import reversed
39 from roundup.backends import locking
40 from roundup.i18n import _
42 from blobfiles import FileStorage
43 from sessions_dbm import Sessions, OneTimeKeys
45 try:
46 from indexer_xapian import Indexer
47 except ImportError:
48 from indexer_dbm import Indexer
50 def db_exists(config):
51 # check for the user db
52 for db in 'nodes.user nodes.user.db'.split():
53 if os.path.exists(os.path.join(config.DATABASE, db)):
54 return 1
55 return 0
57 def db_nuke(config):
58 shutil.rmtree(config.DATABASE)
60 #
61 # Now the database
62 #
63 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
64 """A database for storing records containing flexible data types.
66 Transaction stuff TODO:
68 - check the timestamp of the class file and nuke the cache if it's
69 modified. Do some sort of conflict checking on the dirty stuff.
70 - perhaps detect write collisions (related to above)?
71 """
72 def __init__(self, config, journaltag=None):
73 """Open a hyperdatabase given a specifier to some storage.
75 The 'storagelocator' is obtained from config.DATABASE.
76 The meaning of 'storagelocator' depends on the particular
77 implementation of the hyperdatabase. It could be a file name,
78 a directory path, a socket descriptor for a connection to a
79 database over the network, etc.
81 The 'journaltag' is a token that will be attached to the journal
82 entries for any edits done on the database. If 'journaltag' is
83 None, the database is opened in read-only mode: the Class.create(),
84 Class.set(), Class.retire(), and Class.restore() methods are
85 disabled.
86 """
87 FileStorage.__init__(self, config.UMASK)
88 self.config, self.journaltag = config, journaltag
89 self.dir = config.DATABASE
90 self.classes = {}
91 self.cache = {} # cache of nodes loaded or created
92 self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0,
93 'filtering': 0}
94 self.dirtynodes = {} # keep track of the dirty nodes by class
95 self.newnodes = {} # keep track of the new nodes by class
96 self.destroyednodes = {}# keep track of the destroyed nodes by class
97 self.transactions = []
98 self.indexer = Indexer(self)
99 self.security = security.Security(self)
100 os.umask(config.UMASK)
102 # lock it
103 lockfilenm = os.path.join(self.dir, 'lock')
104 self.lockfile = locking.acquire_lock(lockfilenm)
105 self.lockfile.write(str(os.getpid()))
106 self.lockfile.flush()
108 def post_init(self):
109 """Called once the schema initialisation has finished.
110 """
111 # reindex the db if necessary
112 if self.indexer.should_reindex():
113 self.reindex()
115 def refresh_database(self):
116 """Rebuild the database
117 """
118 self.reindex()
120 def getSessionManager(self):
121 return Sessions(self)
123 def getOTKManager(self):
124 return OneTimeKeys(self)
126 def reindex(self, classname=None, show_progress=False):
127 if classname:
128 classes = [self.getclass(classname)]
129 else:
130 classes = self.classes.values()
131 for klass in classes:
132 if show_progress:
133 for nodeid in support.Progress('Reindex %s'%klass.classname,
134 klass.list()):
135 klass.index(nodeid)
136 else:
137 for nodeid in klass.list():
138 klass.index(nodeid)
139 self.indexer.save_index()
141 def __repr__(self):
142 return '<back_anydbm instance at %x>'%id(self)
144 #
145 # Classes
146 #
147 def __getattr__(self, classname):
148 """A convenient way of calling self.getclass(classname)."""
149 if self.classes.has_key(classname):
150 return self.classes[classname]
151 raise AttributeError, classname
153 def addclass(self, cl):
154 cn = cl.classname
155 if self.classes.has_key(cn):
156 raise ValueError, cn
157 self.classes[cn] = cl
159 # add default Edit and View permissions
160 self.security.addPermission(name="Create", klass=cn,
161 description="User is allowed to create "+cn)
162 self.security.addPermission(name="Edit", klass=cn,
163 description="User is allowed to edit "+cn)
164 self.security.addPermission(name="View", klass=cn,
165 description="User is allowed to access "+cn)
167 def getclasses(self):
168 """Return a list of the names of all existing classes."""
169 l = self.classes.keys()
170 l.sort()
171 return l
173 def getclass(self, classname):
174 """Get the Class object representing a particular class.
176 If 'classname' is not a valid class name, a KeyError is raised.
177 """
178 try:
179 return self.classes[classname]
180 except KeyError:
181 raise KeyError, 'There is no class called "%s"'%classname
183 #
184 # Class DBs
185 #
186 def clear(self):
187 """Delete all database contents
188 """
189 logging.getLogger('hyperdb').info('clear')
190 for cn in self.classes.keys():
191 for dummy in 'nodes', 'journals':
192 path = os.path.join(self.dir, 'journals.%s'%cn)
193 if os.path.exists(path):
194 os.remove(path)
195 elif os.path.exists(path+'.db'): # dbm appends .db
196 os.remove(path+'.db')
197 # reset id sequences
198 path = os.path.join(os.getcwd(), self.dir, '_ids')
199 if os.path.exists(path):
200 os.remove(path)
201 elif os.path.exists(path+'.db'): # dbm appends .db
202 os.remove(path+'.db')
204 def getclassdb(self, classname, mode='r'):
205 """ grab a connection to the class db that will be used for
206 multiple actions
207 """
208 return self.opendb('nodes.%s'%classname, mode)
210 def determine_db_type(self, path):
211 """ determine which DB wrote the class file
212 """
213 db_type = ''
214 if os.path.exists(path):
215 db_type = whichdb.whichdb(path)
216 if not db_type:
217 raise hyperdb.DatabaseError, \
218 _("Couldn't identify database type")
219 elif os.path.exists(path+'.db'):
220 # if the path ends in '.db', it's a dbm database, whether
221 # anydbm says it's dbhash or not!
222 db_type = 'dbm'
223 return db_type
225 def opendb(self, name, mode):
226 """Low-level database opener that gets around anydbm/dbm
227 eccentricities.
228 """
229 # figure the class db type
230 path = os.path.join(os.getcwd(), self.dir, name)
231 db_type = self.determine_db_type(path)
233 # new database? let anydbm pick the best dbm
234 if not db_type:
235 if __debug__:
236 logging.getLogger('hyperdb').debug("opendb anydbm.open(%r, 'c')"%path)
237 return anydbm.open(path, 'c')
239 # open the database with the correct module
240 try:
241 dbm = __import__(db_type)
242 except ImportError:
243 raise hyperdb.DatabaseError, \
244 _("Couldn't open database - the required module '%s'"\
245 " is not available")%db_type
246 if __debug__:
247 logging.getLogger('hyperdb').debug("opendb %r.open(%r, %r)"%(db_type, path,
248 mode))
249 return dbm.open(path, mode)
251 #
252 # Node IDs
253 #
254 def newid(self, classname):
255 """ Generate a new id for the given class
256 """
257 # open the ids DB - create if if doesn't exist
258 db = self.opendb('_ids', 'c')
259 if db.has_key(classname):
260 newid = db[classname] = str(int(db[classname]) + 1)
261 else:
262 # the count() bit is transitional - older dbs won't start at 1
263 newid = str(self.getclass(classname).count()+1)
264 db[classname] = newid
265 db.close()
266 return newid
268 def setid(self, classname, setid):
269 """ Set the id counter: used during import of database
270 """
271 # open the ids DB - create if if doesn't exist
272 db = self.opendb('_ids', 'c')
273 db[classname] = str(setid)
274 db.close()
276 #
277 # Nodes
278 #
279 def addnode(self, classname, nodeid, node):
280 """ add the specified node to its class's db
281 """
282 # we'll be supplied these props if we're doing an import
283 if not node.has_key('creator'):
284 # add in the "calculated" properties (dupe so we don't affect
285 # calling code's node assumptions)
286 node = node.copy()
287 node['creator'] = self.getuid()
288 node['actor'] = self.getuid()
289 node['creation'] = node['activity'] = date.Date()
291 self.newnodes.setdefault(classname, {})[nodeid] = 1
292 self.cache.setdefault(classname, {})[nodeid] = node
293 self.savenode(classname, nodeid, node)
295 def setnode(self, classname, nodeid, node):
296 """ change the specified node
297 """
298 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
300 # can't set without having already loaded the node
301 self.cache[classname][nodeid] = node
302 self.savenode(classname, nodeid, node)
304 def savenode(self, classname, nodeid, node):
305 """ perform the saving of data specified by the set/addnode
306 """
307 if __debug__:
308 logging.getLogger('hyperdb').debug('save %s%s %r'%(classname, nodeid, node))
309 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
311 def getnode(self, classname, nodeid, db=None, cache=1):
312 """ get a node from the database
314 Note the "cache" parameter is not used, and exists purely for
315 backward compatibility!
316 """
317 # try the cache
318 cache_dict = self.cache.setdefault(classname, {})
319 if cache_dict.has_key(nodeid):
320 if __debug__:
321 logging.getLogger('hyperdb').debug('get %s%s cached'%(classname, nodeid))
322 self.stats['cache_hits'] += 1
323 return cache_dict[nodeid]
325 if __debug__:
326 self.stats['cache_misses'] += 1
327 start_t = time.time()
328 logging.getLogger('hyperdb').debug('get %s%s'%(classname, nodeid))
330 # get from the database and save in the cache
331 if db is None:
332 db = self.getclassdb(classname)
333 if not db.has_key(nodeid):
334 raise IndexError, "no such %s %s"%(classname, nodeid)
336 # check the uncommitted, destroyed nodes
337 if (self.destroyednodes.has_key(classname) and
338 self.destroyednodes[classname].has_key(nodeid)):
339 raise IndexError, "no such %s %s"%(classname, nodeid)
341 # decode
342 res = marshal.loads(db[nodeid])
344 # reverse the serialisation
345 res = self.unserialise(classname, res)
347 # store off in the cache dict
348 if cache:
349 cache_dict[nodeid] = res
351 if __debug__:
352 self.stats['get_items'] += (time.time() - start_t)
354 return res
356 def destroynode(self, classname, nodeid):
357 """Remove a node from the database. Called exclusively by the
358 destroy() method on Class.
359 """
360 logging.getLogger('hyperdb').info('destroy %s%s'%(classname, nodeid))
362 # remove from cache and newnodes if it's there
363 if (self.cache.has_key(classname) and
364 self.cache[classname].has_key(nodeid)):
365 del self.cache[classname][nodeid]
366 if (self.newnodes.has_key(classname) and
367 self.newnodes[classname].has_key(nodeid)):
368 del self.newnodes[classname][nodeid]
370 # see if there's any obvious commit actions that we should get rid of
371 for entry in self.transactions[:]:
372 if entry[1][:2] == (classname, nodeid):
373 self.transactions.remove(entry)
375 # add to the destroyednodes map
376 self.destroyednodes.setdefault(classname, {})[nodeid] = 1
378 # add the destroy commit action
379 self.transactions.append((self.doDestroyNode, (classname, nodeid)))
380 self.transactions.append((FileStorage.destroy, (self, classname, nodeid)))
382 def serialise(self, classname, node):
383 """Copy the node contents, converting non-marshallable data into
384 marshallable data.
385 """
386 properties = self.getclass(classname).getprops()
387 d = {}
388 for k, v in node.items():
389 if k == self.RETIRED_FLAG:
390 d[k] = v
391 continue
393 # if the property doesn't exist then we really don't care
394 if not properties.has_key(k):
395 continue
397 # get the property spec
398 prop = properties[k]
400 if isinstance(prop, hyperdb.Password) and v is not None:
401 d[k] = str(v)
402 elif isinstance(prop, hyperdb.Date) and v is not None:
403 d[k] = v.serialise()
404 elif isinstance(prop, hyperdb.Interval) and v is not None:
405 d[k] = v.serialise()
406 else:
407 d[k] = v
408 return d
410 def unserialise(self, classname, node):
411 """Decode the marshalled node data
412 """
413 properties = self.getclass(classname).getprops()
414 d = {}
415 for k, v in node.items():
416 # if the property doesn't exist, or is the "retired" flag then
417 # it won't be in the properties dict
418 if not properties.has_key(k):
419 d[k] = v
420 continue
422 # get the property spec
423 prop = properties[k]
425 if isinstance(prop, hyperdb.Date) and v is not None:
426 d[k] = date.Date(v)
427 elif isinstance(prop, hyperdb.Interval) and v is not None:
428 d[k] = date.Interval(v)
429 elif isinstance(prop, hyperdb.Password) and v is not None:
430 p = password.Password()
431 p.unpack(v)
432 d[k] = p
433 else:
434 d[k] = v
435 return d
437 def hasnode(self, classname, nodeid, db=None):
438 """ determine if the database has a given node
439 """
440 # try the cache
441 cache = self.cache.setdefault(classname, {})
442 if cache.has_key(nodeid):
443 return 1
445 # not in the cache - check the database
446 if db is None:
447 db = self.getclassdb(classname)
448 res = db.has_key(nodeid)
449 return res
451 def countnodes(self, classname, db=None):
452 count = 0
454 # include the uncommitted nodes
455 if self.newnodes.has_key(classname):
456 count += len(self.newnodes[classname])
457 if self.destroyednodes.has_key(classname):
458 count -= len(self.destroyednodes[classname])
460 # and count those in the DB
461 if db is None:
462 db = self.getclassdb(classname)
463 count = count + len(db.keys())
464 return count
467 #
468 # Files - special node properties
469 # inherited from FileStorage
471 #
472 # Journal
473 #
474 def addjournal(self, classname, nodeid, action, params, creator=None,
475 creation=None):
476 """ Journal the Action
477 'action' may be:
479 'create' or 'set' -- 'params' is a dictionary of property values
480 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
481 'retire' -- 'params' is None
483 'creator' -- the user performing the action, which defaults to
484 the current user.
485 """
486 if __debug__:
487 logging.getLogger('hyperdb').debug('addjournal %s%s %s %r %s %r'%(classname,
488 nodeid, action, params, creator, creation))
489 if creator is None:
490 creator = self.getuid()
491 self.transactions.append((self.doSaveJournal, (classname, nodeid,
492 action, params, creator, creation)))
494 def setjournal(self, classname, nodeid, journal):
495 """Set the journal to the "journal" list."""
496 if __debug__:
497 logging.getLogger('hyperdb').debug('setjournal %s%s %r'%(classname,
498 nodeid, journal))
499 self.transactions.append((self.doSetJournal, (classname, nodeid,
500 journal)))
502 def getjournal(self, classname, nodeid):
503 """ get the journal for id
505 Raise IndexError if the node doesn't exist (as per history()'s
506 API)
507 """
508 # our journal result
509 res = []
511 # add any journal entries for transactions not committed to the
512 # database
513 for method, args in self.transactions:
514 if method != self.doSaveJournal:
515 continue
516 (cache_classname, cache_nodeid, cache_action, cache_params,
517 cache_creator, cache_creation) = args
518 if cache_classname == classname and cache_nodeid == nodeid:
519 if not cache_creator:
520 cache_creator = self.getuid()
521 if not cache_creation:
522 cache_creation = date.Date()
523 res.append((cache_nodeid, cache_creation, cache_creator,
524 cache_action, cache_params))
526 # attempt to open the journal - in some rare cases, the journal may
527 # not exist
528 try:
529 db = self.opendb('journals.%s'%classname, 'r')
530 except anydbm.error, error:
531 if str(error) == "need 'c' or 'n' flag to open new db":
532 raise IndexError, 'no such %s %s'%(classname, nodeid)
533 elif error.args[0] != 2:
534 # this isn't a "not found" error, be alarmed!
535 raise
536 if res:
537 # we have unsaved journal entries, return them
538 return res
539 raise IndexError, 'no such %s %s'%(classname, nodeid)
540 try:
541 journal = marshal.loads(db[nodeid])
542 except KeyError:
543 db.close()
544 if res:
545 # we have some unsaved journal entries, be happy!
546 return res
547 raise IndexError, 'no such %s %s'%(classname, nodeid)
548 db.close()
550 # add all the saved journal entries for this node
551 for nodeid, date_stamp, user, action, params in journal:
552 res.append((nodeid, date.Date(date_stamp), user, action, params))
553 return res
555 def pack(self, pack_before):
556 """ Delete all journal entries except "create" before 'pack_before'.
557 """
558 pack_before = pack_before.serialise()
559 for classname in self.getclasses():
560 packed = 0
561 # get the journal db
562 db_name = 'journals.%s'%classname
563 path = os.path.join(os.getcwd(), self.dir, classname)
564 db_type = self.determine_db_type(path)
565 db = self.opendb(db_name, 'w')
567 for key in db.keys():
568 # get the journal for this db entry
569 journal = marshal.loads(db[key])
570 l = []
571 last_set_entry = None
572 for entry in journal:
573 # unpack the entry
574 (nodeid, date_stamp, self.journaltag, action,
575 params) = entry
576 # if the entry is after the pack date, _or_ the initial
577 # create entry, then it stays
578 if date_stamp > pack_before or action == 'create':
579 l.append(entry)
580 else:
581 packed += 1
582 db[key] = marshal.dumps(l)
584 logging.getLogger('hyperdb').info('packed %d %s items'%(packed,
585 classname))
587 if db_type == 'gdbm':
588 db.reorganize()
589 db.close()
592 #
593 # Basic transaction support
594 #
595 def commit(self, fail_ok=False):
596 """ Commit the current transactions.
598 Save all data changed since the database was opened or since the
599 last commit() or rollback().
601 fail_ok indicates that the commit is allowed to fail. This is used
602 in the web interface when committing cleaning of the session
603 database. We don't care if there's a concurrency issue there.
605 The only backend this seems to affect is postgres.
606 """
607 logging.getLogger('hyperdb').info('commit %s transactions'%(
608 len(self.transactions)))
610 # keep a handle to all the database files opened
611 self.databases = {}
613 try:
614 # now, do all the transactions
615 reindex = {}
616 for method, args in self.transactions:
617 reindex[method(*args)] = 1
618 finally:
619 # make sure we close all the database files
620 for db in self.databases.values():
621 db.close()
622 del self.databases
624 # clear the transactions list now so the blobfile implementation
625 # doesn't think there's still pending file commits when it tries
626 # to access the file data
627 self.transactions = []
629 # reindex the nodes that request it
630 for classname, nodeid in filter(None, reindex.keys()):
631 self.getclass(classname).index(nodeid)
633 # save the indexer state
634 self.indexer.save_index()
636 self.clearCache()
638 def clearCache(self):
639 # all transactions committed, back to normal
640 self.cache = {}
641 self.dirtynodes = {}
642 self.newnodes = {}
643 self.destroyednodes = {}
644 self.transactions = []
646 def getCachedClassDB(self, classname):
647 """ get the class db, looking in our cache of databases for commit
648 """
649 # get the database handle
650 db_name = 'nodes.%s'%classname
651 if not self.databases.has_key(db_name):
652 self.databases[db_name] = self.getclassdb(classname, 'c')
653 return self.databases[db_name]
655 def doSaveNode(self, classname, nodeid, node):
656 db = self.getCachedClassDB(classname)
658 # now save the marshalled data
659 db[nodeid] = marshal.dumps(self.serialise(classname, node))
661 # return the classname, nodeid so we reindex this content
662 return (classname, nodeid)
664 def getCachedJournalDB(self, classname):
665 """ get the journal db, looking in our cache of databases for commit
666 """
667 # get the database handle
668 db_name = 'journals.%s'%classname
669 if not self.databases.has_key(db_name):
670 self.databases[db_name] = self.opendb(db_name, 'c')
671 return self.databases[db_name]
673 def doSaveJournal(self, classname, nodeid, action, params, creator,
674 creation):
675 # serialise the parameters now if necessary
676 if isinstance(params, type({})):
677 if action in ('set', 'create'):
678 params = self.serialise(classname, params)
680 # handle supply of the special journalling parameters (usually
681 # supplied on importing an existing database)
682 journaltag = creator
683 if creation:
684 journaldate = creation.serialise()
685 else:
686 journaldate = date.Date().serialise()
688 # create the journal entry
689 entry = (nodeid, journaldate, journaltag, action, params)
691 db = self.getCachedJournalDB(classname)
693 # now insert the journal entry
694 if db.has_key(nodeid):
695 # append to existing
696 s = db[nodeid]
697 l = marshal.loads(s)
698 l.append(entry)
699 else:
700 l = [entry]
702 db[nodeid] = marshal.dumps(l)
704 def doSetJournal(self, classname, nodeid, journal):
705 l = []
706 for nodeid, journaldate, journaltag, action, params in journal:
707 # serialise the parameters now if necessary
708 if isinstance(params, type({})):
709 if action in ('set', 'create'):
710 params = self.serialise(classname, params)
711 journaldate = journaldate.serialise()
712 l.append((nodeid, journaldate, journaltag, action, params))
713 db = self.getCachedJournalDB(classname)
714 db[nodeid] = marshal.dumps(l)
716 def doDestroyNode(self, classname, nodeid):
717 # delete from the class database
718 db = self.getCachedClassDB(classname)
719 if db.has_key(nodeid):
720 del db[nodeid]
722 # delete from the database
723 db = self.getCachedJournalDB(classname)
724 if db.has_key(nodeid):
725 del db[nodeid]
727 def rollback(self):
728 """ Reverse all actions from the current transaction.
729 """
730 logging.getLogger('hyperdb').info('rollback %s transactions'%(
731 len(self.transactions)))
733 for method, args in self.transactions:
734 # delete temporary files
735 if method == self.doStoreFile:
736 self.rollbackStoreFile(*args)
737 self.cache = {}
738 self.dirtynodes = {}
739 self.newnodes = {}
740 self.destroyednodes = {}
741 self.transactions = []
743 def close(self):
744 """ Nothing to do
745 """
746 if self.lockfile is not None:
747 locking.release_lock(self.lockfile)
748 self.lockfile.close()
749 self.lockfile = None
751 _marker = []
752 class Class(hyperdb.Class):
753 """The handle to a particular class of nodes in a hyperdatabase."""
755 def enableJournalling(self):
756 """Turn journalling on for this class
757 """
758 self.do_journal = 1
760 def disableJournalling(self):
761 """Turn journalling off for this class
762 """
763 self.do_journal = 0
765 # Editing nodes:
767 def create(self, **propvalues):
768 """Create a new node of this class and return its id.
770 The keyword arguments in 'propvalues' map property names to values.
772 The values of arguments must be acceptable for the types of their
773 corresponding properties or a TypeError is raised.
775 If this class has a key property, it must be present and its value
776 must not collide with other key strings or a ValueError is raised.
778 Any other properties on this class that are missing from the
779 'propvalues' dictionary are set to None.
781 If an id in a link or multilink property does not refer to a valid
782 node, an IndexError is raised.
784 These operations trigger detectors and can be vetoed. Attempts
785 to modify the "creation" or "activity" properties cause a KeyError.
786 """
787 if self.db.journaltag is None:
788 raise hyperdb.DatabaseError, _('Database open read-only')
789 self.fireAuditors('create', None, propvalues)
790 newid = self.create_inner(**propvalues)
791 self.fireReactors('create', newid, None)
792 return newid
794 def create_inner(self, **propvalues):
795 """ Called by create, in-between the audit and react calls.
796 """
797 if propvalues.has_key('id'):
798 raise KeyError, '"id" is reserved'
800 if self.db.journaltag is None:
801 raise hyperdb.DatabaseError, _('Database open read-only')
803 if propvalues.has_key('creation') or propvalues.has_key('activity'):
804 raise KeyError, '"creation" and "activity" are reserved'
805 # new node's id
806 newid = self.db.newid(self.classname)
808 # validate propvalues
809 num_re = re.compile('^\d+$')
810 for key, value in propvalues.items():
811 if key == self.key:
812 try:
813 self.lookup(value)
814 except KeyError:
815 pass
816 else:
817 raise ValueError, 'node with key "%s" exists'%value
819 # try to handle this property
820 try:
821 prop = self.properties[key]
822 except KeyError:
823 raise KeyError, '"%s" has no property "%s"'%(self.classname,
824 key)
826 if value is not None and isinstance(prop, hyperdb.Link):
827 if type(value) != type(''):
828 raise ValueError, 'link value must be String'
829 link_class = self.properties[key].classname
830 # if it isn't a number, it's a key
831 if not num_re.match(value):
832 try:
833 value = self.db.classes[link_class].lookup(value)
834 except (TypeError, KeyError):
835 raise IndexError, 'new property "%s": %s not a %s'%(
836 key, value, link_class)
837 elif not self.db.getclass(link_class).hasnode(value):
838 raise IndexError, '%s has no node %s'%(link_class, value)
840 # save off the value
841 propvalues[key] = value
843 # register the link with the newly linked node
844 if self.do_journal and self.properties[key].do_journal:
845 self.db.addjournal(link_class, value, 'link',
846 (self.classname, newid, key))
848 elif isinstance(prop, hyperdb.Multilink):
849 if value is None:
850 value = []
851 if not hasattr(value, '__iter__'):
852 raise TypeError, 'new property "%s" not an iterable of ids'%key
854 # clean up and validate the list of links
855 link_class = self.properties[key].classname
856 l = []
857 for entry in value:
858 if type(entry) != type(''):
859 raise ValueError, '"%s" multilink value (%r) '\
860 'must contain Strings'%(key, value)
861 # if it isn't a number, it's a key
862 if not num_re.match(entry):
863 try:
864 entry = self.db.classes[link_class].lookup(entry)
865 except (TypeError, KeyError):
866 raise IndexError, 'new property "%s": %s not a %s'%(
867 key, entry, self.properties[key].classname)
868 l.append(entry)
869 value = l
870 propvalues[key] = value
872 # handle additions
873 for nodeid in value:
874 if not self.db.getclass(link_class).hasnode(nodeid):
875 raise IndexError, '%s has no node %s'%(link_class,
876 nodeid)
877 # register the link with the newly linked node
878 if self.do_journal and self.properties[key].do_journal:
879 self.db.addjournal(link_class, nodeid, 'link',
880 (self.classname, newid, key))
882 elif isinstance(prop, hyperdb.String):
883 if type(value) != type('') and type(value) != type(u''):
884 raise TypeError, 'new property "%s" not a string'%key
885 if prop.indexme:
886 self.db.indexer.add_text((self.classname, newid, key),
887 value)
889 elif isinstance(prop, hyperdb.Password):
890 if not isinstance(value, password.Password):
891 raise TypeError, 'new property "%s" not a Password'%key
893 elif isinstance(prop, hyperdb.Date):
894 if value is not None and not isinstance(value, date.Date):
895 raise TypeError, 'new property "%s" not a Date'%key
897 elif isinstance(prop, hyperdb.Interval):
898 if value is not None and not isinstance(value, date.Interval):
899 raise TypeError, 'new property "%s" not an Interval'%key
901 elif value is not None and isinstance(prop, hyperdb.Number):
902 try:
903 float(value)
904 except ValueError:
905 raise TypeError, 'new property "%s" not numeric'%key
907 elif value is not None and isinstance(prop, hyperdb.Boolean):
908 try:
909 int(value)
910 except ValueError:
911 raise TypeError, 'new property "%s" not boolean'%key
913 # make sure there's data where there needs to be
914 for key, prop in self.properties.items():
915 if propvalues.has_key(key):
916 continue
917 if key == self.key:
918 raise ValueError, 'key property "%s" is required'%key
919 if isinstance(prop, hyperdb.Multilink):
920 propvalues[key] = []
922 # done
923 self.db.addnode(self.classname, newid, propvalues)
924 if self.do_journal:
925 self.db.addjournal(self.classname, newid, 'create', {})
927 return newid
929 def get(self, nodeid, propname, default=_marker, cache=1):
930 """Get the value of a property on an existing node of this class.
932 'nodeid' must be the id of an existing node of this class or an
933 IndexError is raised. 'propname' must be the name of a property
934 of this class or a KeyError is raised.
936 'cache' exists for backward compatibility, and is not used.
938 Attempts to get the "creation" or "activity" properties should
939 do the right thing.
940 """
941 if propname == 'id':
942 return nodeid
944 # get the node's dict
945 d = self.db.getnode(self.classname, nodeid)
947 # check for one of the special props
948 if propname == 'creation':
949 if d.has_key('creation'):
950 return d['creation']
951 if not self.do_journal:
952 raise ValueError, 'Journalling is disabled for this class'
953 journal = self.db.getjournal(self.classname, nodeid)
954 if journal:
955 return journal[0][1]
956 else:
957 # on the strange chance that there's no journal
958 return date.Date()
959 if propname == 'activity':
960 if d.has_key('activity'):
961 return d['activity']
962 if not self.do_journal:
963 raise ValueError, 'Journalling is disabled for this class'
964 journal = self.db.getjournal(self.classname, nodeid)
965 if journal:
966 return self.db.getjournal(self.classname, nodeid)[-1][1]
967 else:
968 # on the strange chance that there's no journal
969 return date.Date()
970 if propname == 'creator':
971 if d.has_key('creator'):
972 return d['creator']
973 if not self.do_journal:
974 raise ValueError, 'Journalling is disabled for this class'
975 journal = self.db.getjournal(self.classname, nodeid)
976 if journal:
977 num_re = re.compile('^\d+$')
978 value = journal[0][2]
979 if num_re.match(value):
980 return value
981 else:
982 # old-style "username" journal tag
983 try:
984 return self.db.user.lookup(value)
985 except KeyError:
986 # user's been retired, return admin
987 return '1'
988 else:
989 return self.db.getuid()
990 if propname == 'actor':
991 if d.has_key('actor'):
992 return d['actor']
993 if not self.do_journal:
994 raise ValueError, 'Journalling is disabled for this class'
995 journal = self.db.getjournal(self.classname, nodeid)
996 if journal:
997 num_re = re.compile('^\d+$')
998 value = journal[-1][2]
999 if num_re.match(value):
1000 return value
1001 else:
1002 # old-style "username" journal tag
1003 try:
1004 return self.db.user.lookup(value)
1005 except KeyError:
1006 # user's been retired, return admin
1007 return '1'
1008 else:
1009 return self.db.getuid()
1011 # get the property (raises KeyErorr if invalid)
1012 prop = self.properties[propname]
1014 if not d.has_key(propname):
1015 if default is _marker:
1016 if isinstance(prop, hyperdb.Multilink):
1017 return []
1018 else:
1019 return None
1020 else:
1021 return default
1023 # return a dupe of the list so code doesn't get confused
1024 if isinstance(prop, hyperdb.Multilink):
1025 return d[propname][:]
1027 return d[propname]
1029 def set(self, nodeid, **propvalues):
1030 """Modify a property on an existing node of this class.
1032 'nodeid' must be the id of an existing node of this class or an
1033 IndexError is raised.
1035 Each key in 'propvalues' must be the name of a property of this
1036 class or a KeyError is raised.
1038 All values in 'propvalues' must be acceptable types for their
1039 corresponding properties or a TypeError is raised.
1041 If the value of the key property is set, it must not collide with
1042 other key strings or a ValueError is raised.
1044 If the value of a Link or Multilink property contains an invalid
1045 node id, a ValueError is raised.
1047 These operations trigger detectors and can be vetoed. Attempts
1048 to modify the "creation" or "activity" properties cause a KeyError.
1049 """
1050 if self.db.journaltag is None:
1051 raise hyperdb.DatabaseError, _('Database open read-only')
1053 self.fireAuditors('set', nodeid, propvalues)
1054 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1055 for name,prop in self.getprops(protected=0).items():
1056 if oldvalues.has_key(name):
1057 continue
1058 if isinstance(prop, hyperdb.Multilink):
1059 oldvalues[name] = []
1060 else:
1061 oldvalues[name] = None
1062 propvalues = self.set_inner(nodeid, **propvalues)
1063 self.fireReactors('set', nodeid, oldvalues)
1064 return propvalues
1066 def set_inner(self, nodeid, **propvalues):
1067 """ Called by set, in-between the audit and react calls.
1068 """
1069 if not propvalues:
1070 return propvalues
1072 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1073 raise KeyError, '"creation" and "activity" are reserved'
1075 if propvalues.has_key('id'):
1076 raise KeyError, '"id" is reserved'
1078 if self.db.journaltag is None:
1079 raise hyperdb.DatabaseError, _('Database open read-only')
1081 node = self.db.getnode(self.classname, nodeid)
1082 if node.has_key(self.db.RETIRED_FLAG):
1083 raise IndexError
1084 num_re = re.compile('^\d+$')
1086 # if the journal value is to be different, store it in here
1087 journalvalues = {}
1089 for propname, value in propvalues.items():
1090 # check to make sure we're not duplicating an existing key
1091 if propname == self.key and node[propname] != value:
1092 try:
1093 self.lookup(value)
1094 except KeyError:
1095 pass
1096 else:
1097 raise ValueError, 'node with key "%s" exists'%value
1099 # this will raise the KeyError if the property isn't valid
1100 # ... we don't use getprops() here because we only care about
1101 # the writeable properties.
1102 try:
1103 prop = self.properties[propname]
1104 except KeyError:
1105 raise KeyError, '"%s" has no property named "%s"'%(
1106 self.classname, propname)
1108 # if the value's the same as the existing value, no sense in
1109 # doing anything
1110 current = node.get(propname, None)
1111 if value == current:
1112 del propvalues[propname]
1113 continue
1114 journalvalues[propname] = current
1116 # do stuff based on the prop type
1117 if isinstance(prop, hyperdb.Link):
1118 link_class = prop.classname
1119 # if it isn't a number, it's a key
1120 if value is not None and not isinstance(value, type('')):
1121 raise ValueError, 'property "%s" link value be a string'%(
1122 propname)
1123 if isinstance(value, type('')) and not num_re.match(value):
1124 try:
1125 value = self.db.classes[link_class].lookup(value)
1126 except (TypeError, KeyError):
1127 raise IndexError, 'new property "%s": %s not a %s'%(
1128 propname, value, prop.classname)
1130 if (value is not None and
1131 not self.db.getclass(link_class).hasnode(value)):
1132 raise IndexError, '%s has no node %s'%(link_class, value)
1134 if self.do_journal and prop.do_journal:
1135 # register the unlink with the old linked node
1136 if node.has_key(propname) and node[propname] is not None:
1137 self.db.addjournal(link_class, node[propname], 'unlink',
1138 (self.classname, nodeid, propname))
1140 # register the link with the newly linked node
1141 if value is not None:
1142 self.db.addjournal(link_class, value, 'link',
1143 (self.classname, nodeid, propname))
1145 elif isinstance(prop, hyperdb.Multilink):
1146 if value is None:
1147 value = []
1148 if not hasattr(value, '__iter__'):
1149 raise TypeError, 'new property "%s" not an iterable of'\
1150 ' ids'%propname
1151 link_class = self.properties[propname].classname
1152 l = []
1153 for entry in value:
1154 # if it isn't a number, it's a key
1155 if type(entry) != type(''):
1156 raise ValueError, 'new property "%s" link value ' \
1157 'must be a string'%propname
1158 if not num_re.match(entry):
1159 try:
1160 entry = self.db.classes[link_class].lookup(entry)
1161 except (TypeError, KeyError):
1162 raise IndexError, 'new property "%s": %s not a %s'%(
1163 propname, entry,
1164 self.properties[propname].classname)
1165 l.append(entry)
1166 value = l
1167 propvalues[propname] = value
1169 # figure the journal entry for this property
1170 add = []
1171 remove = []
1173 # handle removals
1174 if node.has_key(propname):
1175 l = node[propname]
1176 else:
1177 l = []
1178 for id in l[:]:
1179 if id in value:
1180 continue
1181 # register the unlink with the old linked node
1182 if self.do_journal and self.properties[propname].do_journal:
1183 self.db.addjournal(link_class, id, 'unlink',
1184 (self.classname, nodeid, propname))
1185 l.remove(id)
1186 remove.append(id)
1188 # handle additions
1189 for id in value:
1190 if not self.db.getclass(link_class).hasnode(id):
1191 raise IndexError, '%s has no node %s'%(link_class, id)
1192 if id in l:
1193 continue
1194 # register the link with the newly linked node
1195 if self.do_journal and self.properties[propname].do_journal:
1196 self.db.addjournal(link_class, id, 'link',
1197 (self.classname, nodeid, propname))
1198 l.append(id)
1199 add.append(id)
1201 # figure the journal entry
1202 l = []
1203 if add:
1204 l.append(('+', add))
1205 if remove:
1206 l.append(('-', remove))
1207 if l:
1208 journalvalues[propname] = tuple(l)
1210 elif isinstance(prop, hyperdb.String):
1211 if value is not None and type(value) != type('') and type(value) != type(u''):
1212 raise TypeError, 'new property "%s" not a string'%propname
1213 if prop.indexme:
1214 self.db.indexer.add_text((self.classname, nodeid, propname),
1215 value)
1217 elif isinstance(prop, hyperdb.Password):
1218 if not isinstance(value, password.Password):
1219 raise TypeError, 'new property "%s" not a Password'%propname
1220 propvalues[propname] = value
1222 elif value is not None and isinstance(prop, hyperdb.Date):
1223 if not isinstance(value, date.Date):
1224 raise TypeError, 'new property "%s" not a Date'% propname
1225 propvalues[propname] = value
1227 elif value is not None and isinstance(prop, hyperdb.Interval):
1228 if not isinstance(value, date.Interval):
1229 raise TypeError, 'new property "%s" not an '\
1230 'Interval'%propname
1231 propvalues[propname] = value
1233 elif value is not None and isinstance(prop, hyperdb.Number):
1234 try:
1235 float(value)
1236 except ValueError:
1237 raise TypeError, 'new property "%s" not numeric'%propname
1239 elif value is not None and isinstance(prop, hyperdb.Boolean):
1240 try:
1241 int(value)
1242 except ValueError:
1243 raise TypeError, 'new property "%s" not boolean'%propname
1245 node[propname] = value
1247 # nothing to do?
1248 if not propvalues:
1249 return propvalues
1251 # update the activity time
1252 node['activity'] = date.Date()
1253 node['actor'] = self.db.getuid()
1255 # do the set, and journal it
1256 self.db.setnode(self.classname, nodeid, node)
1258 if self.do_journal:
1259 self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
1261 return propvalues
1263 def retire(self, nodeid):
1264 """Retire a node.
1266 The properties on the node remain available from the get() method,
1267 and the node's id is never reused.
1269 Retired nodes are not returned by the find(), list(), or lookup()
1270 methods, and other nodes may reuse the values of their key properties.
1272 These operations trigger detectors and can be vetoed. Attempts
1273 to modify the "creation" or "activity" properties cause a KeyError.
1274 """
1275 if self.db.journaltag is None:
1276 raise hyperdb.DatabaseError, _('Database open read-only')
1278 self.fireAuditors('retire', nodeid, None)
1280 node = self.db.getnode(self.classname, nodeid)
1281 node[self.db.RETIRED_FLAG] = 1
1282 self.db.setnode(self.classname, nodeid, node)
1283 if self.do_journal:
1284 self.db.addjournal(self.classname, nodeid, 'retired', None)
1286 self.fireReactors('retire', nodeid, None)
1288 def restore(self, nodeid):
1289 """Restpre a retired node.
1291 Make node available for all operations like it was before retirement.
1292 """
1293 if self.db.journaltag is None:
1294 raise hyperdb.DatabaseError, _('Database open read-only')
1296 node = self.db.getnode(self.classname, nodeid)
1297 # check if key property was overrided
1298 key = self.getkey()
1299 try:
1300 id = self.lookup(node[key])
1301 except KeyError:
1302 pass
1303 else:
1304 raise KeyError, "Key property (%s) of retired node clashes with \
1305 existing one (%s)" % (key, node[key])
1306 # Now we can safely restore node
1307 self.fireAuditors('restore', nodeid, None)
1308 del node[self.db.RETIRED_FLAG]
1309 self.db.setnode(self.classname, nodeid, node)
1310 if self.do_journal:
1311 self.db.addjournal(self.classname, nodeid, 'restored', None)
1313 self.fireReactors('restore', nodeid, None)
1315 def is_retired(self, nodeid, cldb=None):
1316 """Return true if the node is retired.
1317 """
1318 node = self.db.getnode(self.classname, nodeid, cldb)
1319 if node.has_key(self.db.RETIRED_FLAG):
1320 return 1
1321 return 0
1323 def destroy(self, nodeid):
1324 """Destroy a node.
1326 WARNING: this method should never be used except in extremely rare
1327 situations where there could never be links to the node being
1328 deleted
1330 WARNING: use retire() instead
1332 WARNING: the properties of this node will not be available ever again
1334 WARNING: really, use retire() instead
1336 Well, I think that's enough warnings. This method exists mostly to
1337 support the session storage of the cgi interface.
1338 """
1339 if self.db.journaltag is None:
1340 raise hyperdb.DatabaseError, _('Database open read-only')
1341 self.db.destroynode(self.classname, nodeid)
1343 def history(self, nodeid):
1344 """Retrieve the journal of edits on a particular node.
1346 'nodeid' must be the id of an existing node of this class or an
1347 IndexError is raised.
1349 The returned list contains tuples of the form
1351 (nodeid, date, tag, action, params)
1353 'date' is a Timestamp object specifying the time of the change and
1354 'tag' is the journaltag specified when the database was opened.
1355 """
1356 if not self.do_journal:
1357 raise ValueError, 'Journalling is disabled for this class'
1358 return self.db.getjournal(self.classname, nodeid)
1360 # Locating nodes:
1361 def hasnode(self, nodeid):
1362 """Determine if the given nodeid actually exists
1363 """
1364 return self.db.hasnode(self.classname, nodeid)
1366 def setkey(self, propname):
1367 """Select a String property of this class to be the key property.
1369 'propname' must be the name of a String property of this class or
1370 None, or a TypeError is raised. The values of the key property on
1371 all existing nodes must be unique or a ValueError is raised. If the
1372 property doesn't exist, KeyError is raised.
1373 """
1374 prop = self.getprops()[propname]
1375 if not isinstance(prop, hyperdb.String):
1376 raise TypeError, 'key properties must be String'
1377 self.key = propname
1379 def getkey(self):
1380 """Return the name of the key property for this class or None."""
1381 return self.key
1383 # TODO: set up a separate index db file for this? profile?
1384 def lookup(self, keyvalue):
1385 """Locate a particular node by its key property and return its id.
1387 If this class has no key property, a TypeError is raised. If the
1388 'keyvalue' matches one of the values for the key property among
1389 the nodes in this class, the matching node's id is returned;
1390 otherwise a KeyError is raised.
1391 """
1392 if not self.key:
1393 raise TypeError, 'No key property set for class %s'%self.classname
1394 cldb = self.db.getclassdb(self.classname)
1395 try:
1396 for nodeid in self.getnodeids(cldb):
1397 node = self.db.getnode(self.classname, nodeid, cldb)
1398 if node.has_key(self.db.RETIRED_FLAG):
1399 continue
1400 if not node.has_key(self.key):
1401 continue
1402 if node[self.key] == keyvalue:
1403 return nodeid
1404 finally:
1405 cldb.close()
1406 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1407 keyvalue, self.classname)
1409 # change from spec - allows multiple props to match
1410 def find(self, **propspec):
1411 """Get the ids of nodes in this class which link to the given nodes.
1413 'propspec' consists of keyword args propname=nodeid or
1414 propname={nodeid:1, }
1415 'propname' must be the name of a property in this class, or a
1416 KeyError is raised. That property must be a Link or
1417 Multilink property, or a TypeError is raised.
1419 Any node in this class whose 'propname' property links to any of
1420 the nodeids will be returned. Examples::
1422 db.issue.find(messages='1')
1423 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1424 """
1425 propspec = propspec.items()
1426 for propname, itemids in propspec:
1427 # check the prop is OK
1428 prop = self.properties[propname]
1429 if not isinstance(prop, hyperdb.Link) and not isinstance(prop, hyperdb.Multilink):
1430 raise TypeError, "'%s' not a Link/Multilink property"%propname
1432 # ok, now do the find
1433 cldb = self.db.getclassdb(self.classname)
1434 l = []
1435 try:
1436 for id in self.getnodeids(db=cldb):
1437 item = self.db.getnode(self.classname, id, db=cldb)
1438 if item.has_key(self.db.RETIRED_FLAG):
1439 continue
1440 for propname, itemids in propspec:
1441 if type(itemids) is not type({}):
1442 itemids = {itemids:1}
1444 # special case if the item doesn't have this property
1445 if not item.has_key(propname):
1446 if itemids.has_key(None):
1447 l.append(id)
1448 break
1449 continue
1451 # grab the property definition and its value on this item
1452 prop = self.properties[propname]
1453 value = item[propname]
1454 if isinstance(prop, hyperdb.Link) and itemids.has_key(value):
1455 l.append(id)
1456 break
1457 elif isinstance(prop, hyperdb.Multilink):
1458 hit = 0
1459 for v in value:
1460 if itemids.has_key(v):
1461 l.append(id)
1462 hit = 1
1463 break
1464 if hit:
1465 break
1466 finally:
1467 cldb.close()
1468 return l
1470 def stringFind(self, **requirements):
1471 """Locate a particular node by matching a set of its String
1472 properties in a caseless search.
1474 If the property is not a String property, a TypeError is raised.
1476 The return is a list of the id of all nodes that match.
1477 """
1478 for propname in requirements.keys():
1479 prop = self.properties[propname]
1480 if not isinstance(prop, hyperdb.String):
1481 raise TypeError, "'%s' not a String property"%propname
1482 requirements[propname] = requirements[propname].lower()
1483 l = []
1484 cldb = self.db.getclassdb(self.classname)
1485 try:
1486 for nodeid in self.getnodeids(cldb):
1487 node = self.db.getnode(self.classname, nodeid, cldb)
1488 if node.has_key(self.db.RETIRED_FLAG):
1489 continue
1490 for key, value in requirements.items():
1491 if not node.has_key(key):
1492 break
1493 if node[key] is None or node[key].lower() != value:
1494 break
1495 else:
1496 l.append(nodeid)
1497 finally:
1498 cldb.close()
1499 return l
1501 def list(self):
1502 """ Return a list of the ids of the active nodes in this class.
1503 """
1504 l = []
1505 cn = self.classname
1506 cldb = self.db.getclassdb(cn)
1507 try:
1508 for nodeid in self.getnodeids(cldb):
1509 node = self.db.getnode(cn, nodeid, cldb)
1510 if node.has_key(self.db.RETIRED_FLAG):
1511 continue
1512 l.append(nodeid)
1513 finally:
1514 cldb.close()
1515 l.sort()
1516 return l
1518 def getnodeids(self, db=None, retired=None):
1519 """ Return a list of ALL nodeids
1521 Set retired=None to get all nodes. Otherwise it'll get all the
1522 retired or non-retired nodes, depending on the flag.
1523 """
1524 res = []
1526 # start off with the new nodes
1527 if self.db.newnodes.has_key(self.classname):
1528 res += self.db.newnodes[self.classname].keys()
1530 must_close = False
1531 if db is None:
1532 db = self.db.getclassdb(self.classname)
1533 must_close = True
1534 try:
1535 res = res + db.keys()
1537 # remove the uncommitted, destroyed nodes
1538 if self.db.destroyednodes.has_key(self.classname):
1539 for nodeid in self.db.destroyednodes[self.classname].keys():
1540 if db.has_key(nodeid):
1541 res.remove(nodeid)
1543 # check retired flag
1544 if retired is False or retired is True:
1545 l = []
1546 for nodeid in res:
1547 node = self.db.getnode(self.classname, nodeid, db)
1548 is_ret = node.has_key(self.db.RETIRED_FLAG)
1549 if retired == is_ret:
1550 l.append(nodeid)
1551 res = l
1552 finally:
1553 if must_close:
1554 db.close()
1555 return res
1557 def _filter(self, search_matches, filterspec, proptree,
1558 num_re = re.compile('^\d+$')):
1559 """Return a list of the ids of the active nodes in this class that
1560 match the 'filter' spec, sorted by the group spec and then the
1561 sort spec.
1563 "filterspec" is {propname: value(s)}
1565 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1566 and prop is a prop name or None
1568 "search_matches" is a sequence type or None
1570 The filter must match all properties specificed. If the property
1571 value to match is a list:
1573 1. String properties must match all elements in the list, and
1574 2. Other properties must match any of the elements in the list.
1575 """
1576 if __debug__:
1577 start_t = time.time()
1579 cn = self.classname
1581 # optimise filterspec
1582 l = []
1583 props = self.getprops()
1584 LINK = 'spec:link'
1585 MULTILINK = 'spec:multilink'
1586 STRING = 'spec:string'
1587 DATE = 'spec:date'
1588 INTERVAL = 'spec:interval'
1589 OTHER = 'spec:other'
1591 for k, v in filterspec.items():
1592 propclass = props[k]
1593 if isinstance(propclass, hyperdb.Link):
1594 if type(v) is not type([]):
1595 v = [v]
1596 u = []
1597 for entry in v:
1598 # the value -1 is a special "not set" sentinel
1599 if entry == '-1':
1600 entry = None
1601 u.append(entry)
1602 l.append((LINK, k, u))
1603 elif isinstance(propclass, hyperdb.Multilink):
1604 # the value -1 is a special "not set" sentinel
1605 if v in ('-1', ['-1']):
1606 v = []
1607 elif type(v) is not type([]):
1608 v = [v]
1609 l.append((MULTILINK, k, v))
1610 elif isinstance(propclass, hyperdb.String) and k != 'id':
1611 if type(v) is not type([]):
1612 v = [v]
1613 for v in v:
1614 # simple glob searching
1615 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1616 v = v.replace('?', '.')
1617 v = v.replace('*', '.*?')
1618 l.append((STRING, k, re.compile(v, re.I)))
1619 elif isinstance(propclass, hyperdb.Date):
1620 try:
1621 date_rng = propclass.range_from_raw(v, self.db)
1622 l.append((DATE, k, date_rng))
1623 except ValueError:
1624 # If range creation fails - ignore that search parameter
1625 pass
1626 elif isinstance(propclass, hyperdb.Interval):
1627 try:
1628 intv_rng = date.Range(v, date.Interval)
1629 l.append((INTERVAL, k, intv_rng))
1630 except ValueError:
1631 # If range creation fails - ignore that search parameter
1632 pass
1634 elif isinstance(propclass, hyperdb.Boolean):
1635 if type(v) != type([]):
1636 v = v.split(',')
1637 bv = []
1638 for val in v:
1639 if type(val) is type(''):
1640 bv.append(val.lower() in ('yes', 'true', 'on', '1'))
1641 else:
1642 bv.append(val)
1643 l.append((OTHER, k, bv))
1645 elif k == 'id':
1646 if type(v) != type([]):
1647 v = v.split(',')
1648 l.append((OTHER, k, [str(int(val)) for val in v]))
1650 elif isinstance(propclass, hyperdb.Number):
1651 if type(v) != type([]):
1652 v = v.split(',')
1653 l.append((OTHER, k, [float(val) for val in v]))
1655 filterspec = l
1657 # now, find all the nodes that are active and pass filtering
1658 matches = []
1659 cldb = self.db.getclassdb(cn)
1660 t = 0
1661 try:
1662 # TODO: only full-scan once (use items())
1663 for nodeid in self.getnodeids(cldb):
1664 node = self.db.getnode(cn, nodeid, cldb)
1665 if node.has_key(self.db.RETIRED_FLAG):
1666 continue
1667 # apply filter
1668 for t, k, v in filterspec:
1669 # handle the id prop
1670 if k == 'id':
1671 if nodeid not in v:
1672 break
1673 continue
1675 # get the node value
1676 nv = node.get(k, None)
1678 match = 0
1680 # now apply the property filter
1681 if t == LINK:
1682 # link - if this node's property doesn't appear in the
1683 # filterspec's nodeid list, skip it
1684 match = nv in v
1685 elif t == MULTILINK:
1686 # multilink - if any of the nodeids required by the
1687 # filterspec aren't in this node's property, then skip
1688 # it
1689 nv = node.get(k, [])
1691 # check for matching the absence of multilink values
1692 if not v:
1693 match = not nv
1694 else:
1695 # othewise, make sure this node has each of the
1696 # required values
1697 for want in v:
1698 if want in nv:
1699 match = 1
1700 break
1701 elif t == STRING:
1702 if nv is None:
1703 nv = ''
1704 # RE search
1705 match = v.search(nv)
1706 elif t == DATE or t == INTERVAL:
1707 if nv is None:
1708 match = v is None
1709 else:
1710 if v.to_value:
1711 if v.from_value <= nv and v.to_value >= nv:
1712 match = 1
1713 else:
1714 if v.from_value <= nv:
1715 match = 1
1716 elif t == OTHER:
1717 # straight value comparison for the other types
1718 match = nv in v
1719 if not match:
1720 break
1721 else:
1722 matches.append([nodeid, node])
1724 # filter based on full text search
1725 if search_matches is not None:
1726 k = []
1727 for v in matches:
1728 if v[0] in search_matches:
1729 k.append(v)
1730 matches = k
1732 # add sorting information to the proptree
1733 JPROPS = {'actor':1, 'activity':1, 'creator':1, 'creation':1}
1734 children = []
1735 if proptree:
1736 children = proptree.sortable_children()
1737 for pt in children:
1738 dir = pt.sort_direction
1739 prop = pt.name
1740 assert (dir and prop)
1741 propclass = props[prop]
1742 pt.sort_ids = []
1743 is_pointer = isinstance(propclass,(hyperdb.Link,
1744 hyperdb.Multilink))
1745 if not is_pointer:
1746 pt.sort_result = []
1747 try:
1748 # cache the opened link class db, if needed.
1749 lcldb = None
1750 # cache the linked class items too
1751 lcache = {}
1753 for entry in matches:
1754 itemid = entry[-2]
1755 item = entry[-1]
1756 # handle the properties that might be "faked"
1757 # also, handle possible missing properties
1758 try:
1759 v = item[prop]
1760 except KeyError:
1761 if JPROPS.has_key(prop):
1762 # force lookup of the special journal prop
1763 v = self.get(itemid, prop)
1764 else:
1765 # the node doesn't have a value for this
1766 # property
1767 v = None
1768 if isinstance(propclass, hyperdb.Multilink):
1769 v = []
1770 if prop == 'id':
1771 v = int (itemid)
1772 pt.sort_ids.append(v)
1773 if not is_pointer:
1774 pt.sort_result.append(v)
1775 continue
1777 # missing (None) values are always sorted first
1778 if v is None:
1779 pt.sort_ids.append(v)
1780 if not is_pointer:
1781 pt.sort_result.append(v)
1782 continue
1784 if isinstance(propclass, hyperdb.Link):
1785 lcn = propclass.classname
1786 link = self.db.classes[lcn]
1787 key = link.orderprop()
1788 child = pt.propdict[key]
1789 if key!='id':
1790 if not lcache.has_key(v):
1791 # open the link class db if it's not already
1792 if lcldb is None:
1793 lcldb = self.db.getclassdb(lcn)
1794 lcache[v] = self.db.getnode(lcn, v, lcldb)
1795 r = lcache[v][key]
1796 child.propdict[key].sort_ids.append(r)
1797 else:
1798 child.propdict[key].sort_ids.append(v)
1799 pt.sort_ids.append(v)
1800 if not is_pointer:
1801 r = propclass.sort_repr(pt.parent.cls, v, pt.name)
1802 pt.sort_result.append(r)
1803 finally:
1804 # if we opened the link class db, close it now
1805 if lcldb is not None:
1806 lcldb.close()
1807 del lcache
1808 finally:
1809 cldb.close()
1811 # pull the id out of the individual entries
1812 matches = [entry[-2] for entry in matches]
1813 if __debug__:
1814 self.db.stats['filtering'] += (time.time() - start_t)
1815 return matches
1817 def count(self):
1818 """Get the number of nodes in this class.
1820 If the returned integer is 'numnodes', the ids of all the nodes
1821 in this class run from 1 to numnodes, and numnodes+1 will be the
1822 id of the next node to be created in this class.
1823 """
1824 return self.db.countnodes(self.classname)
1826 # Manipulating properties:
1828 def getprops(self, protected=1):
1829 """Return a dictionary mapping property names to property objects.
1830 If the "protected" flag is true, we include protected properties -
1831 those which may not be modified.
1833 In addition to the actual properties on the node, these
1834 methods provide the "creation" and "activity" properties. If the
1835 "protected" flag is true, we include protected properties - those
1836 which may not be modified.
1837 """
1838 d = self.properties.copy()
1839 if protected:
1840 d['id'] = hyperdb.String()
1841 d['creation'] = hyperdb.Date()
1842 d['activity'] = hyperdb.Date()
1843 d['creator'] = hyperdb.Link('user')
1844 d['actor'] = hyperdb.Link('user')
1845 return d
1847 def addprop(self, **properties):
1848 """Add properties to this class.
1850 The keyword arguments in 'properties' must map names to property
1851 objects, or a TypeError is raised. None of the keys in 'properties'
1852 may collide with the names of existing properties, or a ValueError
1853 is raised before any properties have been added.
1854 """
1855 for key in properties.keys():
1856 if self.properties.has_key(key):
1857 raise ValueError, key
1858 self.properties.update(properties)
1860 def index(self, nodeid):
1861 """ Add (or refresh) the node to search indexes """
1862 # find all the String properties that have indexme
1863 for prop, propclass in self.getprops().items():
1864 if isinstance(propclass, hyperdb.String) and propclass.indexme:
1865 # index them under (classname, nodeid, property)
1866 try:
1867 value = str(self.get(nodeid, prop))
1868 except IndexError:
1869 # node has been destroyed
1870 continue
1871 self.db.indexer.add_text((self.classname, nodeid, prop), value)
1873 #
1874 # import / export support
1875 #
1876 def export_list(self, propnames, nodeid):
1877 """ Export a node - generate a list of CSV-able data in the order
1878 specified by propnames for the given node.
1879 """
1880 properties = self.getprops()
1881 l = []
1882 for prop in propnames:
1883 proptype = properties[prop]
1884 value = self.get(nodeid, prop)
1885 # "marshal" data where needed
1886 if value is None:
1887 pass
1888 elif isinstance(proptype, hyperdb.Date):
1889 value = value.get_tuple()
1890 elif isinstance(proptype, hyperdb.Interval):
1891 value = value.get_tuple()
1892 elif isinstance(proptype, hyperdb.Password):
1893 value = str(value)
1894 l.append(repr(value))
1896 # append retired flag
1897 l.append(repr(self.is_retired(nodeid)))
1899 return l
1901 def import_list(self, propnames, proplist):
1902 """ Import a node - all information including "id" is present and
1903 should not be sanity checked. Triggers are not triggered. The
1904 journal should be initialised using the "creator" and "created"
1905 information.
1907 Return the nodeid of the node imported.
1908 """
1909 if self.db.journaltag is None:
1910 raise hyperdb.DatabaseError, _('Database open read-only')
1911 properties = self.getprops()
1913 # make the new node's property map
1914 d = {}
1915 newid = None
1916 for i in range(len(propnames)):
1917 # Figure the property for this column
1918 propname = propnames[i]
1920 # Use eval to reverse the repr() used to output the CSV
1921 value = eval(proplist[i])
1923 # "unmarshal" where necessary
1924 if propname == 'id':
1925 newid = value
1926 continue
1927 elif propname == 'is retired':
1928 # is the item retired?
1929 if int(value):
1930 d[self.db.RETIRED_FLAG] = 1
1931 continue
1932 elif value is None:
1933 d[propname] = None
1934 continue
1936 prop = properties[propname]
1937 if isinstance(prop, hyperdb.Date):
1938 value = date.Date(value)
1939 elif isinstance(prop, hyperdb.Interval):
1940 value = date.Interval(value)
1941 elif isinstance(prop, hyperdb.Password):
1942 pwd = password.Password()
1943 pwd.unpack(value)
1944 value = pwd
1945 d[propname] = value
1947 # get a new id if necessary
1948 if newid is None:
1949 newid = self.db.newid(self.classname)
1951 # add the node and journal
1952 self.db.addnode(self.classname, newid, d)
1953 return newid
1955 def export_journals(self):
1956 """Export a class's journal - generate a list of lists of
1957 CSV-able data:
1959 nodeid, date, user, action, params
1961 No heading here - the columns are fixed.
1962 """
1963 properties = self.getprops()
1964 r = []
1965 for nodeid in self.getnodeids():
1966 for nodeid, date, user, action, params in self.history(nodeid):
1967 date = date.get_tuple()
1968 if action == 'set':
1969 export_data = {}
1970 for propname, value in params.items():
1971 if not properties.has_key(propname):
1972 # property no longer in the schema
1973 continue
1975 prop = properties[propname]
1976 # make sure the params are eval()'able
1977 if value is None:
1978 pass
1979 elif isinstance(prop, hyperdb.Date):
1980 # this is a hack - some dates are stored as strings
1981 if not isinstance(value, type('')):
1982 value = value.get_tuple()
1983 elif isinstance(prop, hyperdb.Interval):
1984 # hack too - some intervals are stored as strings
1985 if not isinstance(value, type('')):
1986 value = value.get_tuple()
1987 elif isinstance(prop, hyperdb.Password):
1988 value = str(value)
1989 export_data[propname] = value
1990 params = export_data
1991 l = [nodeid, date, user, action, params]
1992 r.append(map(repr, l))
1993 return r
1995 def import_journals(self, entries):
1996 """Import a class's journal.
1998 Uses setjournal() to set the journal for each item."""
1999 properties = self.getprops()
2000 d = {}
2001 for l in entries:
2002 l = map(eval, l)
2003 nodeid, jdate, user, action, params = l
2004 r = d.setdefault(nodeid, [])
2005 if action == 'set':
2006 for propname, value in params.items():
2007 prop = properties[propname]
2008 if value is None:
2009 pass
2010 elif isinstance(prop, hyperdb.Date):
2011 value = date.Date(value)
2012 elif isinstance(prop, hyperdb.Interval):
2013 value = date.Interval(value)
2014 elif isinstance(prop, hyperdb.Password):
2015 pwd = password.Password()
2016 pwd.unpack(value)
2017 value = pwd
2018 params[propname] = value
2019 r.append((nodeid, date.Date(jdate), user, action, params))
2021 for nodeid, l in d.items():
2022 self.db.setjournal(self.classname, nodeid, l)
2024 class FileClass(hyperdb.FileClass, Class):
2025 """This class defines a large chunk of data. To support this, it has a
2026 mandatory String property "content" which is typically saved off
2027 externally to the hyperdb.
2029 The default MIME type of this data is defined by the
2030 "default_mime_type" class attribute, which may be overridden by each
2031 node if the class defines a "type" String property.
2032 """
2033 def __init__(self, db, classname, **properties):
2034 """The newly-created class automatically includes the "content"
2035 and "type" properties.
2036 """
2037 if not properties.has_key('content'):
2038 properties['content'] = hyperdb.String(indexme='yes')
2039 if not properties.has_key('type'):
2040 properties['type'] = hyperdb.String()
2041 Class.__init__(self, db, classname, **properties)
2043 def create(self, **propvalues):
2044 """ Snarf the "content" propvalue and store in a file
2045 """
2046 # we need to fire the auditors now, or the content property won't
2047 # be in propvalues for the auditors to play with
2048 self.fireAuditors('create', None, propvalues)
2050 # now remove the content property so it's not stored in the db
2051 content = propvalues['content']
2052 del propvalues['content']
2054 # make sure we have a MIME type
2055 mime_type = propvalues.get('type', self.default_mime_type)
2057 # do the database create
2058 newid = self.create_inner(**propvalues)
2060 # store off the content as a file
2061 self.db.storefile(self.classname, newid, None, content)
2063 # fire reactors
2064 self.fireReactors('create', newid, None)
2066 return newid
2068 def get(self, nodeid, propname, default=_marker, cache=1):
2069 """ Trap the content propname and get it from the file
2071 'cache' exists for backwards compatibility, and is not used.
2072 """
2073 poss_msg = 'Possibly an access right configuration problem.'
2074 if propname == 'content':
2075 try:
2076 return self.db.getfile(self.classname, nodeid, None)
2077 except IOError, (strerror):
2078 # XXX by catching this we don't see an error in the log.
2079 return 'ERROR reading file: %s%s\n%s\n%s'%(
2080 self.classname, nodeid, poss_msg, strerror)
2081 if default is not _marker:
2082 return Class.get(self, nodeid, propname, default)
2083 else:
2084 return Class.get(self, nodeid, propname)
2086 def set(self, itemid, **propvalues):
2087 """ Snarf the "content" propvalue and update it in a file
2088 """
2089 self.fireAuditors('set', itemid, propvalues)
2091 # create the oldvalues dict - fill in any missing values
2092 oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid))
2093 for name,prop in self.getprops(protected=0).items():
2094 if oldvalues.has_key(name):
2095 continue
2096 if isinstance(prop, hyperdb.Multilink):
2097 oldvalues[name] = []
2098 else:
2099 oldvalues[name] = None
2101 # now remove the content property so it's not stored in the db
2102 content = None
2103 if propvalues.has_key('content'):
2104 content = propvalues['content']
2105 del propvalues['content']
2107 # do the database update
2108 propvalues = self.set_inner(itemid, **propvalues)
2110 # do content?
2111 if content:
2112 # store and possibly index
2113 self.db.storefile(self.classname, itemid, None, content)
2114 if self.properties['content'].indexme:
2115 mime_type = self.get(itemid, 'type', self.default_mime_type)
2116 self.db.indexer.add_text((self.classname, itemid, 'content'),
2117 content, mime_type)
2118 propvalues['content'] = content
2120 # fire reactors
2121 self.fireReactors('set', itemid, oldvalues)
2122 return propvalues
2124 def index(self, nodeid):
2125 """ Add (or refresh) the node to search indexes.
2127 Use the content-type property for the content property.
2128 """
2129 # find all the String properties that have indexme
2130 for prop, propclass in self.getprops().items():
2131 if prop == 'content' and propclass.indexme:
2132 mime_type = self.get(nodeid, 'type', self.default_mime_type)
2133 self.db.indexer.add_text((self.classname, nodeid, 'content'),
2134 str(self.get(nodeid, 'content')), mime_type)
2135 elif isinstance(propclass, hyperdb.String) and propclass.indexme:
2136 # index them under (classname, nodeid, property)
2137 try:
2138 value = str(self.get(nodeid, prop))
2139 except IndexError:
2140 # node has been destroyed
2141 continue
2142 self.db.indexer.add_text((self.classname, nodeid, prop), value)
2144 # deviation from spec - was called ItemClass
2145 class IssueClass(Class, roundupdb.IssueClass):
2146 # Overridden methods:
2147 def __init__(self, db, classname, **properties):
2148 """The newly-created class automatically includes the "messages",
2149 "files", "nosy", and "superseder" properties. If the 'properties'
2150 dictionary attempts to specify any of these properties or a
2151 "creation" or "activity" property, a ValueError is raised.
2152 """
2153 if not properties.has_key('title'):
2154 properties['title'] = hyperdb.String(indexme='yes')
2155 if not properties.has_key('messages'):
2156 properties['messages'] = hyperdb.Multilink("msg")
2157 if not properties.has_key('files'):
2158 properties['files'] = hyperdb.Multilink("file")
2159 if not properties.has_key('nosy'):
2160 # note: journalling is turned off as it really just wastes
2161 # space. this behaviour may be overridden in an instance
2162 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2163 if not properties.has_key('superseder'):
2164 properties['superseder'] = hyperdb.Multilink(classname)
2165 Class.__init__(self, db, classname, **properties)
2167 # vim: set et sts=4 sw=4 :