1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 """This module defines a backend that saves the hyperdatabase in a
19 database chosen by anydbm. It is guaranteed to always be available in python
20 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
21 serious bugs, and is not available)
22 """
23 __docformat__ = 'restructuredtext'
25 import os, marshal, re, weakref, string, copy, time, shutil, logging
27 from roundup.anypy.dbm_ import anydbm, whichdb, key_in
29 from roundup import hyperdb, date, password, roundupdb, security, support
30 from roundup.support import reversed
31 from roundup.backends import locking
32 from roundup.i18n import _
34 from roundup.backends.blobfiles import FileStorage
35 from roundup.backends.sessions_dbm import Sessions, OneTimeKeys
37 try:
38 from roundup.backends.indexer_xapian import Indexer
39 except ImportError:
40 from roundup.backends.indexer_dbm import Indexer
42 def db_exists(config):
43 # check for the user db
44 for db in 'nodes.user nodes.user.db'.split():
45 if os.path.exists(os.path.join(config.DATABASE, db)):
46 return 1
47 return 0
49 def db_nuke(config):
50 shutil.rmtree(config.DATABASE)
52 #
53 # Now the database
54 #
55 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
56 """A database for storing records containing flexible data types.
58 Transaction stuff TODO:
60 - check the timestamp of the class file and nuke the cache if it's
61 modified. Do some sort of conflict checking on the dirty stuff.
62 - perhaps detect write collisions (related to above)?
63 """
64 def __init__(self, config, journaltag=None):
65 """Open a hyperdatabase given a specifier to some storage.
67 The 'storagelocator' is obtained from config.DATABASE.
68 The meaning of 'storagelocator' depends on the particular
69 implementation of the hyperdatabase. It could be a file name,
70 a directory path, a socket descriptor for a connection to a
71 database over the network, etc.
73 The 'journaltag' is a token that will be attached to the journal
74 entries for any edits done on the database. If 'journaltag' is
75 None, the database is opened in read-only mode: the Class.create(),
76 Class.set(), Class.retire(), and Class.restore() methods are
77 disabled.
78 """
79 FileStorage.__init__(self, config.UMASK)
80 self.config, self.journaltag = config, journaltag
81 self.dir = config.DATABASE
82 self.classes = {}
83 self.cache = {} # cache of nodes loaded or created
84 self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0,
85 'filtering': 0}
86 self.dirtynodes = {} # keep track of the dirty nodes by class
87 self.newnodes = {} # keep track of the new nodes by class
88 self.destroyednodes = {}# keep track of the destroyed nodes by class
89 self.transactions = []
90 self.indexer = Indexer(self)
91 self.security = security.Security(self)
92 os.umask(config.UMASK)
94 # lock it
95 lockfilenm = os.path.join(self.dir, 'lock')
96 self.lockfile = locking.acquire_lock(lockfilenm)
97 self.lockfile.write(str(os.getpid()))
98 self.lockfile.flush()
100 def post_init(self):
101 """Called once the schema initialisation has finished.
102 """
103 # reindex the db if necessary
104 if self.indexer.should_reindex():
105 self.reindex()
107 def refresh_database(self):
108 """Rebuild the database
109 """
110 self.reindex()
112 def getSessionManager(self):
113 return Sessions(self)
115 def getOTKManager(self):
116 return OneTimeKeys(self)
118 def reindex(self, classname=None, show_progress=False):
119 if classname:
120 classes = [self.getclass(classname)]
121 else:
122 classes = self.classes.values()
123 for klass in classes:
124 if show_progress:
125 for nodeid in support.Progress('Reindex %s'%klass.classname,
126 klass.list()):
127 klass.index(nodeid)
128 else:
129 for nodeid in klass.list():
130 klass.index(nodeid)
131 self.indexer.save_index()
133 def __repr__(self):
134 return '<back_anydbm instance at %x>'%id(self)
136 #
137 # Classes
138 #
139 def __getattr__(self, classname):
140 """A convenient way of calling self.getclass(classname)."""
141 if classname in self.classes:
142 return self.classes[classname]
143 raise AttributeError, classname
145 def addclass(self, cl):
146 cn = cl.classname
147 if cn in self.classes:
148 raise ValueError, cn
149 self.classes[cn] = cl
151 # add default Edit and View permissions
152 self.security.addPermission(name="Create", klass=cn,
153 description="User is allowed to create "+cn)
154 self.security.addPermission(name="Edit", klass=cn,
155 description="User is allowed to edit "+cn)
156 self.security.addPermission(name="View", klass=cn,
157 description="User is allowed to access "+cn)
159 def getclasses(self):
160 """Return a list of the names of all existing classes."""
161 l = self.classes.keys()
162 l.sort()
163 return l
165 def getclass(self, classname):
166 """Get the Class object representing a particular class.
168 If 'classname' is not a valid class name, a KeyError is raised.
169 """
170 try:
171 return self.classes[classname]
172 except KeyError:
173 raise KeyError('There is no class called "%s"'%classname)
175 #
176 # Class DBs
177 #
178 def clear(self):
179 """Delete all database contents
180 """
181 logging.getLogger('roundup.hyperdb').info('clear')
182 for cn in self.classes:
183 for dummy in 'nodes', 'journals':
184 path = os.path.join(self.dir, 'journals.%s'%cn)
185 if os.path.exists(path):
186 os.remove(path)
187 elif os.path.exists(path+'.db'): # dbm appends .db
188 os.remove(path+'.db')
189 # reset id sequences
190 path = os.path.join(os.getcwd(), self.dir, '_ids')
191 if os.path.exists(path):
192 os.remove(path)
193 elif os.path.exists(path+'.db'): # dbm appends .db
194 os.remove(path+'.db')
196 def getclassdb(self, classname, mode='r'):
197 """ grab a connection to the class db that will be used for
198 multiple actions
199 """
200 return self.opendb('nodes.%s'%classname, mode)
202 def determine_db_type(self, path):
203 """ determine which DB wrote the class file
204 """
205 db_type = ''
206 if os.path.exists(path):
207 db_type = whichdb(path)
208 if not db_type:
209 raise hyperdb.DatabaseError(_("Couldn't identify database type"))
210 elif os.path.exists(path+'.db'):
211 # if the path ends in '.db', it's a dbm database, whether
212 # anydbm says it's dbhash or not!
213 db_type = 'dbm'
214 return db_type
216 def opendb(self, name, mode):
217 """Low-level database opener that gets around anydbm/dbm
218 eccentricities.
219 """
220 # figure the class db type
221 path = os.path.join(os.getcwd(), self.dir, name)
222 db_type = self.determine_db_type(path)
224 # new database? let anydbm pick the best dbm
225 # in Python 3+ the "dbm" ("anydbm" to us) module already uses the
226 # whichdb() function to do this
227 if not db_type or hasattr(anydbm, 'whichdb'):
228 if __debug__:
229 logging.getLogger('roundup.hyperdb').debug(
230 "opendb anydbm.open(%r, 'c')"%path)
231 return anydbm.open(path, 'c')
233 # in Python <3 it anydbm was a little dumb so manually open the
234 # database with the correct module
235 try:
236 dbm = __import__(db_type)
237 except ImportError:
238 raise hyperdb.DatabaseError(_("Couldn't open database - the "
239 "required module '%s' is not available")%db_type)
240 if __debug__:
241 logging.getLogger('roundup.hyperdb').debug(
242 "opendb %r.open(%r, %r)"%(db_type, path, mode))
243 return dbm.open(path, mode)
245 #
246 # Node IDs
247 #
248 def newid(self, classname):
249 """ Generate a new id for the given class
250 """
251 # open the ids DB - create if if doesn't exist
252 db = self.opendb('_ids', 'c')
253 if key_in(db, classname):
254 newid = db[classname] = str(int(db[classname]) + 1)
255 else:
256 # the count() bit is transitional - older dbs won't start at 1
257 newid = str(self.getclass(classname).count()+1)
258 db[classname] = newid
259 db.close()
260 return newid
262 def setid(self, classname, setid):
263 """ Set the id counter: used during import of database
264 """
265 # open the ids DB - create if if doesn't exist
266 db = self.opendb('_ids', 'c')
267 db[classname] = str(setid)
268 db.close()
270 #
271 # Nodes
272 #
273 def addnode(self, classname, nodeid, node):
274 """ add the specified node to its class's db
275 """
276 # we'll be supplied these props if we're doing an import
277 if 'creator' not in node:
278 # add in the "calculated" properties (dupe so we don't affect
279 # calling code's node assumptions)
280 node = node.copy()
281 node['creator'] = self.getuid()
282 node['actor'] = self.getuid()
283 node['creation'] = node['activity'] = date.Date()
285 self.newnodes.setdefault(classname, {})[nodeid] = 1
286 self.cache.setdefault(classname, {})[nodeid] = node
287 self.savenode(classname, nodeid, node)
289 def setnode(self, classname, nodeid, node):
290 """ change the specified node
291 """
292 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
294 # can't set without having already loaded the node
295 self.cache[classname][nodeid] = node
296 self.savenode(classname, nodeid, node)
298 def savenode(self, classname, nodeid, node):
299 """ perform the saving of data specified by the set/addnode
300 """
301 if __debug__:
302 logging.getLogger('roundup.hyperdb').debug(
303 'save %s%s %r'%(classname, nodeid, node))
304 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
306 def getnode(self, classname, nodeid, db=None, cache=1):
307 """ get a node from the database
309 Note the "cache" parameter is not used, and exists purely for
310 backward compatibility!
311 """
312 # try the cache
313 cache_dict = self.cache.setdefault(classname, {})
314 if nodeid in cache_dict:
315 if __debug__:
316 logging.getLogger('roundup.hyperdb').debug(
317 'get %s%s cached'%(classname, nodeid))
318 self.stats['cache_hits'] += 1
319 return cache_dict[nodeid]
321 if __debug__:
322 self.stats['cache_misses'] += 1
323 start_t = time.time()
324 logging.getLogger('roundup.hyperdb').debug(
325 'get %s%s'%(classname, nodeid))
327 # get from the database and save in the cache
328 if db is None:
329 db = self.getclassdb(classname)
330 if not key_in(db, nodeid):
331 raise IndexError("no such %s %s"%(classname, nodeid))
333 # check the uncommitted, destroyed nodes
334 if (classname in self.destroyednodes and
335 nodeid in self.destroyednodes[classname]):
336 raise IndexError("no such %s %s"%(classname, nodeid))
338 # decode
339 res = marshal.loads(db[nodeid])
341 # reverse the serialisation
342 res = self.unserialise(classname, res)
344 # store off in the cache dict
345 if cache:
346 cache_dict[nodeid] = res
348 if __debug__:
349 self.stats['get_items'] += (time.time() - start_t)
351 return res
353 def destroynode(self, classname, nodeid):
354 """Remove a node from the database. Called exclusively by the
355 destroy() method on Class.
356 """
357 logging.getLogger('roundup.hyperdb').info(
358 'destroy %s%s'%(classname, nodeid))
360 # remove from cache and newnodes if it's there
361 if (classname in self.cache and nodeid in self.cache[classname]):
362 del self.cache[classname][nodeid]
363 if (classname in self.newnodes and nodeid in self.newnodes[classname]):
364 del self.newnodes[classname][nodeid]
366 # see if there's any obvious commit actions that we should get rid of
367 for entry in self.transactions[:]:
368 if entry[1][:2] == (classname, nodeid):
369 self.transactions.remove(entry)
371 # add to the destroyednodes map
372 self.destroyednodes.setdefault(classname, {})[nodeid] = 1
374 # add the destroy commit action
375 self.transactions.append((self.doDestroyNode, (classname, nodeid)))
376 self.transactions.append((FileStorage.destroy, (self, classname, nodeid)))
378 def serialise(self, classname, node):
379 """Copy the node contents, converting non-marshallable data into
380 marshallable data.
381 """
382 properties = self.getclass(classname).getprops()
383 d = {}
384 for k, v in node.iteritems():
385 if k == self.RETIRED_FLAG:
386 d[k] = v
387 continue
389 # if the property doesn't exist then we really don't care
390 if k not in properties:
391 continue
393 # get the property spec
394 prop = properties[k]
396 if isinstance(prop, hyperdb.Password) and v is not None:
397 d[k] = str(v)
398 elif isinstance(prop, hyperdb.Date) and v is not None:
399 d[k] = v.serialise()
400 elif isinstance(prop, hyperdb.Interval) and v is not None:
401 d[k] = v.serialise()
402 else:
403 d[k] = v
404 return d
406 def unserialise(self, classname, node):
407 """Decode the marshalled node data
408 """
409 properties = self.getclass(classname).getprops()
410 d = {}
411 for k, v in node.iteritems():
412 # if the property doesn't exist, or is the "retired" flag then
413 # it won't be in the properties dict
414 if k not in properties:
415 d[k] = v
416 continue
418 # get the property spec
419 prop = properties[k]
421 if isinstance(prop, hyperdb.Date) and v is not None:
422 d[k] = date.Date(v)
423 elif isinstance(prop, hyperdb.Interval) and v is not None:
424 d[k] = date.Interval(v)
425 elif isinstance(prop, hyperdb.Password) and v is not None:
426 p = password.Password()
427 p.unpack(v)
428 d[k] = p
429 else:
430 d[k] = v
431 return d
433 def hasnode(self, classname, nodeid, db=None):
434 """ determine if the database has a given node
435 """
436 # try the cache
437 cache = self.cache.setdefault(classname, {})
438 if nodeid in cache:
439 return 1
441 # not in the cache - check the database
442 if db is None:
443 db = self.getclassdb(classname)
444 return key_in(db, nodeid)
446 def countnodes(self, classname, db=None):
447 count = 0
449 # include the uncommitted nodes
450 if classname in self.newnodes:
451 count += len(self.newnodes[classname])
452 if classname in self.destroyednodes:
453 count -= len(self.destroyednodes[classname])
455 # and count those in the DB
456 if db is None:
457 db = self.getclassdb(classname)
458 return count + len(db)
461 #
462 # Files - special node properties
463 # inherited from FileStorage
465 #
466 # Journal
467 #
468 def addjournal(self, classname, nodeid, action, params, creator=None,
469 creation=None):
470 """ Journal the Action
471 'action' may be:
473 'create' or 'set' -- 'params' is a dictionary of property values
474 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
475 'retire' -- 'params' is None
477 'creator' -- the user performing the action, which defaults to
478 the current user.
479 """
480 if __debug__:
481 logging.getLogger('roundup.hyperdb').debug(
482 'addjournal %s%s %s %r %s %r'%(classname,
483 nodeid, action, params, creator, creation))
484 if creator is None:
485 creator = self.getuid()
486 self.transactions.append((self.doSaveJournal, (classname, nodeid,
487 action, params, creator, creation)))
489 def setjournal(self, classname, nodeid, journal):
490 """Set the journal to the "journal" list."""
491 if __debug__:
492 logging.getLogger('roundup.hyperdb').debug(
493 'setjournal %s%s %r'%(classname, nodeid, journal))
494 self.transactions.append((self.doSetJournal, (classname, nodeid,
495 journal)))
497 def getjournal(self, classname, nodeid):
498 """ get the journal for id
500 Raise IndexError if the node doesn't exist (as per history()'s
501 API)
502 """
503 # our journal result
504 res = []
506 # add any journal entries for transactions not committed to the
507 # database
508 for method, args in self.transactions:
509 if method != self.doSaveJournal:
510 continue
511 (cache_classname, cache_nodeid, cache_action, cache_params,
512 cache_creator, cache_creation) = args
513 if cache_classname == classname and cache_nodeid == nodeid:
514 if not cache_creator:
515 cache_creator = self.getuid()
516 if not cache_creation:
517 cache_creation = date.Date()
518 res.append((cache_nodeid, cache_creation, cache_creator,
519 cache_action, cache_params))
521 # attempt to open the journal - in some rare cases, the journal may
522 # not exist
523 try:
524 db = self.opendb('journals.%s'%classname, 'r')
525 except anydbm.error, error:
526 if str(error) == "need 'c' or 'n' flag to open new db":
527 raise IndexError('no such %s %s'%(classname, nodeid))
528 elif error.args[0] != 2:
529 # this isn't a "not found" error, be alarmed!
530 raise
531 if res:
532 # we have unsaved journal entries, return them
533 return res
534 raise IndexError('no such %s %s'%(classname, nodeid))
535 try:
536 journal = marshal.loads(db[nodeid])
537 except KeyError:
538 db.close()
539 if res:
540 # we have some unsaved journal entries, be happy!
541 return res
542 raise IndexError('no such %s %s'%(classname, nodeid))
543 db.close()
545 # add all the saved journal entries for this node
546 for nodeid, date_stamp, user, action, params in journal:
547 res.append((nodeid, date.Date(date_stamp), user, action, params))
548 return res
550 def pack(self, pack_before):
551 """ Delete all journal entries except "create" before 'pack_before'.
552 """
553 pack_before = pack_before.serialise()
554 for classname in self.getclasses():
555 packed = 0
556 # get the journal db
557 db_name = 'journals.%s'%classname
558 path = os.path.join(os.getcwd(), self.dir, classname)
559 db_type = self.determine_db_type(path)
560 db = self.opendb(db_name, 'w')
562 for key in db.keys():
563 # get the journal for this db entry
564 journal = marshal.loads(db[key])
565 l = []
566 last_set_entry = None
567 for entry in journal:
568 # unpack the entry
569 (nodeid, date_stamp, self.journaltag, action,
570 params) = entry
571 # if the entry is after the pack date, _or_ the initial
572 # create entry, then it stays
573 if date_stamp > pack_before or action == 'create':
574 l.append(entry)
575 else:
576 packed += 1
577 db[key] = marshal.dumps(l)
579 logging.getLogger('roundup.hyperdb').info(
580 'packed %d %s items'%(packed, classname))
582 if db_type == 'gdbm':
583 db.reorganize()
584 db.close()
587 #
588 # Basic transaction support
589 #
590 def commit(self, fail_ok=False):
591 """ Commit the current transactions.
593 Save all data changed since the database was opened or since the
594 last commit() or rollback().
596 fail_ok indicates that the commit is allowed to fail. This is used
597 in the web interface when committing cleaning of the session
598 database. We don't care if there's a concurrency issue there.
600 The only backend this seems to affect is postgres.
601 """
602 logging.getLogger('roundup.hyperdb').info('commit %s transactions'%(
603 len(self.transactions)))
605 # keep a handle to all the database files opened
606 self.databases = {}
608 try:
609 # now, do all the transactions
610 reindex = {}
611 for method, args in self.transactions:
612 reindex[method(*args)] = 1
613 finally:
614 # make sure we close all the database files
615 for db in self.databases.itervalues():
616 db.close()
617 del self.databases
619 # clear the transactions list now so the blobfile implementation
620 # doesn't think there's still pending file commits when it tries
621 # to access the file data
622 self.transactions = []
624 # reindex the nodes that request it
625 for classname, nodeid in [k for k in reindex if k]:
626 self.getclass(classname).index(nodeid)
628 # save the indexer state
629 self.indexer.save_index()
631 self.clearCache()
633 def clearCache(self):
634 # all transactions committed, back to normal
635 self.cache = {}
636 self.dirtynodes = {}
637 self.newnodes = {}
638 self.destroyednodes = {}
639 self.transactions = []
641 def getCachedClassDB(self, classname):
642 """ get the class db, looking in our cache of databases for commit
643 """
644 # get the database handle
645 db_name = 'nodes.%s'%classname
646 if db_name not in self.databases:
647 self.databases[db_name] = self.getclassdb(classname, 'c')
648 return self.databases[db_name]
650 def doSaveNode(self, classname, nodeid, node):
651 db = self.getCachedClassDB(classname)
653 # now save the marshalled data
654 db[nodeid] = marshal.dumps(self.serialise(classname, node))
656 # return the classname, nodeid so we reindex this content
657 return (classname, nodeid)
659 def getCachedJournalDB(self, classname):
660 """ get the journal db, looking in our cache of databases for commit
661 """
662 # get the database handle
663 db_name = 'journals.%s'%classname
664 if db_name not in self.databases:
665 self.databases[db_name] = self.opendb(db_name, 'c')
666 return self.databases[db_name]
668 def doSaveJournal(self, classname, nodeid, action, params, creator,
669 creation):
670 # serialise the parameters now if necessary
671 if isinstance(params, type({})):
672 if action in ('set', 'create'):
673 params = self.serialise(classname, params)
675 # handle supply of the special journalling parameters (usually
676 # supplied on importing an existing database)
677 journaltag = creator
678 if creation:
679 journaldate = creation.serialise()
680 else:
681 journaldate = date.Date().serialise()
683 # create the journal entry
684 entry = (nodeid, journaldate, journaltag, action, params)
686 db = self.getCachedJournalDB(classname)
688 # now insert the journal entry
689 if key_in(db, nodeid):
690 # append to existing
691 s = db[nodeid]
692 l = marshal.loads(s)
693 l.append(entry)
694 else:
695 l = [entry]
697 db[nodeid] = marshal.dumps(l)
699 def doSetJournal(self, classname, nodeid, journal):
700 l = []
701 for nodeid, journaldate, journaltag, action, params in journal:
702 # serialise the parameters now if necessary
703 if isinstance(params, type({})):
704 if action in ('set', 'create'):
705 params = self.serialise(classname, params)
706 journaldate = journaldate.serialise()
707 l.append((nodeid, journaldate, journaltag, action, params))
708 db = self.getCachedJournalDB(classname)
709 db[nodeid] = marshal.dumps(l)
711 def doDestroyNode(self, classname, nodeid):
712 # delete from the class database
713 db = self.getCachedClassDB(classname)
714 if key_in(db, nodeid):
715 del db[nodeid]
717 # delete from the database
718 db = self.getCachedJournalDB(classname)
719 if key_in(db, nodeid):
720 del db[nodeid]
722 def rollback(self):
723 """ Reverse all actions from the current transaction.
724 """
725 logging.getLogger('roundup.hyperdb').info('rollback %s transactions'%(
726 len(self.transactions)))
728 for method, args in self.transactions:
729 # delete temporary files
730 if method == self.doStoreFile:
731 self.rollbackStoreFile(*args)
732 self.cache = {}
733 self.dirtynodes = {}
734 self.newnodes = {}
735 self.destroyednodes = {}
736 self.transactions = []
738 def close(self):
739 """ Nothing to do
740 """
741 if self.lockfile is not None:
742 locking.release_lock(self.lockfile)
743 self.lockfile.close()
744 self.lockfile = None
746 _marker = []
747 class Class(hyperdb.Class):
748 """The handle to a particular class of nodes in a hyperdatabase."""
750 def enableJournalling(self):
751 """Turn journalling on for this class
752 """
753 self.do_journal = 1
755 def disableJournalling(self):
756 """Turn journalling off for this class
757 """
758 self.do_journal = 0
760 # Editing nodes:
762 def create(self, **propvalues):
763 """Create a new node of this class and return its id.
765 The keyword arguments in 'propvalues' map property names to values.
767 The values of arguments must be acceptable for the types of their
768 corresponding properties or a TypeError is raised.
770 If this class has a key property, it must be present and its value
771 must not collide with other key strings or a ValueError is raised.
773 Any other properties on this class that are missing from the
774 'propvalues' dictionary are set to None.
776 If an id in a link or multilink property does not refer to a valid
777 node, an IndexError is raised.
779 These operations trigger detectors and can be vetoed. Attempts
780 to modify the "creation" or "activity" properties cause a KeyError.
781 """
782 if self.db.journaltag is None:
783 raise hyperdb.DatabaseError(_('Database open read-only'))
784 self.fireAuditors('create', None, propvalues)
785 newid = self.create_inner(**propvalues)
786 self.fireReactors('create', newid, None)
787 return newid
789 def create_inner(self, **propvalues):
790 """ Called by create, in-between the audit and react calls.
791 """
792 if 'id' in propvalues:
793 raise KeyError('"id" is reserved')
795 if self.db.journaltag is None:
796 raise hyperdb.DatabaseError(_('Database open read-only'))
798 if 'creation' in propvalues or 'activity' in propvalues:
799 raise KeyError('"creation" and "activity" are reserved')
800 # new node's id
801 newid = self.db.newid(self.classname)
803 # validate propvalues
804 num_re = re.compile('^\d+$')
805 for key, value in propvalues.iteritems():
806 if key == self.key:
807 try:
808 self.lookup(value)
809 except KeyError:
810 pass
811 else:
812 raise ValueError('node with key "%s" exists'%value)
814 # try to handle this property
815 try:
816 prop = self.properties[key]
817 except KeyError:
818 raise KeyError('"%s" has no property "%s"'%(self.classname,
819 key))
821 if value is not None and isinstance(prop, hyperdb.Link):
822 if type(value) != type(''):
823 raise ValueError('link value must be String')
824 link_class = self.properties[key].classname
825 # if it isn't a number, it's a key
826 if not num_re.match(value):
827 try:
828 value = self.db.classes[link_class].lookup(value)
829 except (TypeError, KeyError):
830 raise IndexError('new property "%s": %s not a %s'%(
831 key, value, link_class))
832 elif not self.db.getclass(link_class).hasnode(value):
833 raise IndexError('%s has no node %s'%(link_class,
834 value))
836 # save off the value
837 propvalues[key] = value
839 # register the link with the newly linked node
840 if self.do_journal and self.properties[key].do_journal:
841 self.db.addjournal(link_class, value, 'link',
842 (self.classname, newid, key))
844 elif isinstance(prop, hyperdb.Multilink):
845 if value is None:
846 value = []
847 if not hasattr(value, '__iter__'):
848 raise TypeError('new property "%s" not an iterable of ids'%key)
850 # clean up and validate the list of links
851 link_class = self.properties[key].classname
852 l = []
853 for entry in value:
854 if type(entry) != type(''):
855 raise ValueError('"%s" multilink value (%r) '\
856 'must contain Strings'%(key, value))
857 # if it isn't a number, it's a key
858 if not num_re.match(entry):
859 try:
860 entry = self.db.classes[link_class].lookup(entry)
861 except (TypeError, KeyError):
862 raise IndexError('new property "%s": %s not a %s'%(
863 key, entry, self.properties[key].classname))
864 l.append(entry)
865 value = l
866 propvalues[key] = value
868 # handle additions
869 for nodeid in value:
870 if not self.db.getclass(link_class).hasnode(nodeid):
871 raise IndexError('%s has no node %s'%(link_class,
872 nodeid))
873 # register the link with the newly linked node
874 if self.do_journal and self.properties[key].do_journal:
875 self.db.addjournal(link_class, nodeid, 'link',
876 (self.classname, newid, key))
878 elif isinstance(prop, hyperdb.String):
879 if type(value) != type('') and type(value) != type(u''):
880 raise TypeError('new property "%s" not a string'%key)
881 if prop.indexme:
882 self.db.indexer.add_text((self.classname, newid, key),
883 value)
885 elif isinstance(prop, hyperdb.Password):
886 if not isinstance(value, password.Password):
887 raise TypeError('new property "%s" not a Password'%key)
889 elif isinstance(prop, hyperdb.Date):
890 if value is not None and not isinstance(value, date.Date):
891 raise TypeError('new property "%s" not a Date'%key)
893 elif isinstance(prop, hyperdb.Interval):
894 if value is not None and not isinstance(value, date.Interval):
895 raise TypeError('new property "%s" not an Interval'%key)
897 elif value is not None and isinstance(prop, hyperdb.Number):
898 try:
899 float(value)
900 except ValueError:
901 raise TypeError('new property "%s" not numeric'%key)
903 elif value is not None and isinstance(prop, hyperdb.Boolean):
904 try:
905 int(value)
906 except ValueError:
907 raise TypeError('new property "%s" not boolean'%key)
909 # make sure there's data where there needs to be
910 for key, prop in self.properties.iteritems():
911 if key in propvalues:
912 continue
913 if key == self.key:
914 raise ValueError('key property "%s" is required'%key)
915 if isinstance(prop, hyperdb.Multilink):
916 propvalues[key] = []
918 # done
919 self.db.addnode(self.classname, newid, propvalues)
920 if self.do_journal:
921 self.db.addjournal(self.classname, newid, 'create', {})
923 return newid
925 def get(self, nodeid, propname, default=_marker, cache=1):
926 """Get the value of a property on an existing node of this class.
928 'nodeid' must be the id of an existing node of this class or an
929 IndexError is raised. 'propname' must be the name of a property
930 of this class or a KeyError is raised.
932 'cache' exists for backward compatibility, and is not used.
934 Attempts to get the "creation" or "activity" properties should
935 do the right thing.
936 """
937 if propname == 'id':
938 return nodeid
940 # get the node's dict
941 d = self.db.getnode(self.classname, nodeid)
943 # check for one of the special props
944 if propname == 'creation':
945 if 'creation' in d:
946 return d['creation']
947 if not self.do_journal:
948 raise ValueError('Journalling is disabled for this class')
949 journal = self.db.getjournal(self.classname, nodeid)
950 if journal:
951 return journal[0][1]
952 else:
953 # on the strange chance that there's no journal
954 return date.Date()
955 if propname == 'activity':
956 if 'activity' in d:
957 return d['activity']
958 if not self.do_journal:
959 raise ValueError('Journalling is disabled for this class')
960 journal = self.db.getjournal(self.classname, nodeid)
961 if journal:
962 return self.db.getjournal(self.classname, nodeid)[-1][1]
963 else:
964 # on the strange chance that there's no journal
965 return date.Date()
966 if propname == 'creator':
967 if 'creator' in d:
968 return d['creator']
969 if not self.do_journal:
970 raise ValueError('Journalling is disabled for this class')
971 journal = self.db.getjournal(self.classname, nodeid)
972 if journal:
973 num_re = re.compile('^\d+$')
974 value = journal[0][2]
975 if num_re.match(value):
976 return value
977 else:
978 # old-style "username" journal tag
979 try:
980 return self.db.user.lookup(value)
981 except KeyError:
982 # user's been retired, return admin
983 return '1'
984 else:
985 return self.db.getuid()
986 if propname == 'actor':
987 if 'actor' in d:
988 return d['actor']
989 if not self.do_journal:
990 raise ValueError('Journalling is disabled for this class')
991 journal = self.db.getjournal(self.classname, nodeid)
992 if journal:
993 num_re = re.compile('^\d+$')
994 value = journal[-1][2]
995 if num_re.match(value):
996 return value
997 else:
998 # old-style "username" journal tag
999 try:
1000 return self.db.user.lookup(value)
1001 except KeyError:
1002 # user's been retired, return admin
1003 return '1'
1004 else:
1005 return self.db.getuid()
1007 # get the property (raises KeyErorr if invalid)
1008 prop = self.properties[propname]
1010 if propname not in d:
1011 if default is _marker:
1012 if isinstance(prop, hyperdb.Multilink):
1013 return []
1014 else:
1015 return None
1016 else:
1017 return default
1019 # return a dupe of the list so code doesn't get confused
1020 if isinstance(prop, hyperdb.Multilink):
1021 return d[propname][:]
1023 return d[propname]
1025 def set(self, nodeid, **propvalues):
1026 """Modify a property on an existing node of this class.
1028 'nodeid' must be the id of an existing node of this class or an
1029 IndexError is raised.
1031 Each key in 'propvalues' must be the name of a property of this
1032 class or a KeyError is raised.
1034 All values in 'propvalues' must be acceptable types for their
1035 corresponding properties or a TypeError is raised.
1037 If the value of the key property is set, it must not collide with
1038 other key strings or a ValueError is raised.
1040 If the value of a Link or Multilink property contains an invalid
1041 node id, a ValueError is raised.
1043 These operations trigger detectors and can be vetoed. Attempts
1044 to modify the "creation" or "activity" properties cause a KeyError.
1045 """
1046 if self.db.journaltag is None:
1047 raise hyperdb.DatabaseError(_('Database open read-only'))
1049 self.fireAuditors('set', nodeid, propvalues)
1050 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1051 for name, prop in self.getprops(protected=0).iteritems():
1052 if name in oldvalues:
1053 continue
1054 if isinstance(prop, hyperdb.Multilink):
1055 oldvalues[name] = []
1056 else:
1057 oldvalues[name] = None
1058 propvalues = self.set_inner(nodeid, **propvalues)
1059 self.fireReactors('set', nodeid, oldvalues)
1060 return propvalues
1062 def set_inner(self, nodeid, **propvalues):
1063 """ Called by set, in-between the audit and react calls.
1064 """
1065 if not propvalues:
1066 return propvalues
1068 if 'creation' in propvalues or 'activity' in propvalues:
1069 raise KeyError, '"creation" and "activity" are reserved'
1071 if 'id' in propvalues:
1072 raise KeyError, '"id" is reserved'
1074 if self.db.journaltag is None:
1075 raise hyperdb.DatabaseError(_('Database open read-only'))
1077 node = self.db.getnode(self.classname, nodeid)
1078 if self.db.RETIRED_FLAG in node:
1079 raise IndexError
1080 num_re = re.compile('^\d+$')
1082 # if the journal value is to be different, store it in here
1083 journalvalues = {}
1085 # list() propvalues 'cos it might be modified by the loop
1086 for propname, value in list(propvalues.items()):
1087 # check to make sure we're not duplicating an existing key
1088 if propname == self.key and node[propname] != value:
1089 try:
1090 self.lookup(value)
1091 except KeyError:
1092 pass
1093 else:
1094 raise ValueError('node with key "%s" exists'%value)
1096 # this will raise the KeyError if the property isn't valid
1097 # ... we don't use getprops() here because we only care about
1098 # the writeable properties.
1099 try:
1100 prop = self.properties[propname]
1101 except KeyError:
1102 raise KeyError('"%s" has no property named "%s"'%(
1103 self.classname, propname))
1105 # if the value's the same as the existing value, no sense in
1106 # doing anything
1107 current = node.get(propname, None)
1108 if value == current:
1109 del propvalues[propname]
1110 continue
1111 journalvalues[propname] = current
1113 # do stuff based on the prop type
1114 if isinstance(prop, hyperdb.Link):
1115 link_class = prop.classname
1116 # if it isn't a number, it's a key
1117 if value is not None and not isinstance(value, type('')):
1118 raise ValueError('property "%s" link value be a string'%(
1119 propname))
1120 if isinstance(value, type('')) and not num_re.match(value):
1121 try:
1122 value = self.db.classes[link_class].lookup(value)
1123 except (TypeError, KeyError):
1124 raise IndexError('new property "%s": %s not a %s'%(
1125 propname, value, prop.classname))
1127 if (value is not None and
1128 not self.db.getclass(link_class).hasnode(value)):
1129 raise IndexError('%s has no node %s'%(link_class,
1130 value))
1132 if self.do_journal and prop.do_journal:
1133 # register the unlink with the old linked node
1134 if propname in node and node[propname] is not None:
1135 self.db.addjournal(link_class, node[propname], 'unlink',
1136 (self.classname, nodeid, propname))
1138 # register the link with the newly linked node
1139 if value is not None:
1140 self.db.addjournal(link_class, value, 'link',
1141 (self.classname, nodeid, propname))
1143 elif isinstance(prop, hyperdb.Multilink):
1144 if value is None:
1145 value = []
1146 if not hasattr(value, '__iter__'):
1147 raise TypeError('new property "%s" not an iterable of'
1148 ' ids'%propname)
1149 link_class = self.properties[propname].classname
1150 l = []
1151 for entry in value:
1152 # if it isn't a number, it's a key
1153 if type(entry) != type(''):
1154 raise ValueError('new property "%s" link value '
1155 'must be a string'%propname)
1156 if not num_re.match(entry):
1157 try:
1158 entry = self.db.classes[link_class].lookup(entry)
1159 except (TypeError, KeyError):
1160 raise IndexError('new property "%s": %s not a %s'%(
1161 propname, entry,
1162 self.properties[propname].classname))
1163 l.append(entry)
1164 value = l
1165 propvalues[propname] = value
1167 # figure the journal entry for this property
1168 add = []
1169 remove = []
1171 # handle removals
1172 if propname in node:
1173 l = node[propname]
1174 else:
1175 l = []
1176 for id in l[:]:
1177 if id in value:
1178 continue
1179 # register the unlink with the old linked node
1180 if self.do_journal and self.properties[propname].do_journal:
1181 self.db.addjournal(link_class, id, 'unlink',
1182 (self.classname, nodeid, propname))
1183 l.remove(id)
1184 remove.append(id)
1186 # handle additions
1187 for id in value:
1188 if not self.db.getclass(link_class).hasnode(id):
1189 raise IndexError('%s has no node %s'%(link_class,
1190 id))
1191 if id in l:
1192 continue
1193 # register the link with the newly linked node
1194 if self.do_journal and self.properties[propname].do_journal:
1195 self.db.addjournal(link_class, id, 'link',
1196 (self.classname, nodeid, propname))
1197 l.append(id)
1198 add.append(id)
1200 # figure the journal entry
1201 l = []
1202 if add:
1203 l.append(('+', add))
1204 if remove:
1205 l.append(('-', remove))
1206 if l:
1207 journalvalues[propname] = tuple(l)
1209 elif isinstance(prop, hyperdb.String):
1210 if value is not None and type(value) != type('') and type(value) != type(u''):
1211 raise TypeError('new property "%s" not a '
1212 'string'%propname)
1213 if prop.indexme:
1214 self.db.indexer.add_text((self.classname, nodeid, propname),
1215 value)
1217 elif isinstance(prop, hyperdb.Password):
1218 if not isinstance(value, password.Password):
1219 raise TypeError('new property "%s" not a '
1220 'Password'%propname)
1221 propvalues[propname] = value
1223 elif value is not None and isinstance(prop, hyperdb.Date):
1224 if not isinstance(value, date.Date):
1225 raise TypeError('new property "%s" not a '
1226 'Date'%propname)
1227 propvalues[propname] = value
1229 elif value is not None and isinstance(prop, hyperdb.Interval):
1230 if not isinstance(value, date.Interval):
1231 raise TypeError('new property "%s" not an '
1232 'Interval'%propname)
1233 propvalues[propname] = value
1235 elif value is not None and isinstance(prop, hyperdb.Number):
1236 try:
1237 float(value)
1238 except ValueError:
1239 raise TypeError('new property "%s" not '
1240 'numeric'%propname)
1242 elif value is not None and isinstance(prop, hyperdb.Boolean):
1243 try:
1244 int(value)
1245 except ValueError:
1246 raise TypeError('new property "%s" not '
1247 'boolean'%propname)
1249 node[propname] = value
1251 # nothing to do?
1252 if not propvalues:
1253 return propvalues
1255 # update the activity time
1256 node['activity'] = date.Date()
1257 node['actor'] = self.db.getuid()
1259 # do the set, and journal it
1260 self.db.setnode(self.classname, nodeid, node)
1262 if self.do_journal:
1263 self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
1265 return propvalues
1267 def retire(self, nodeid):
1268 """Retire a node.
1270 The properties on the node remain available from the get() method,
1271 and the node's id is never reused.
1273 Retired nodes are not returned by the find(), list(), or lookup()
1274 methods, and other nodes may reuse the values of their key properties.
1276 These operations trigger detectors and can be vetoed. Attempts
1277 to modify the "creation" or "activity" properties cause a KeyError.
1278 """
1279 if self.db.journaltag is None:
1280 raise hyperdb.DatabaseError(_('Database open read-only'))
1282 self.fireAuditors('retire', nodeid, None)
1284 node = self.db.getnode(self.classname, nodeid)
1285 node[self.db.RETIRED_FLAG] = 1
1286 self.db.setnode(self.classname, nodeid, node)
1287 if self.do_journal:
1288 self.db.addjournal(self.classname, nodeid, 'retired', None)
1290 self.fireReactors('retire', nodeid, None)
1292 def restore(self, nodeid):
1293 """Restpre a retired node.
1295 Make node available for all operations like it was before retirement.
1296 """
1297 if self.db.journaltag is None:
1298 raise hyperdb.DatabaseError(_('Database open read-only'))
1300 node = self.db.getnode(self.classname, nodeid)
1301 # check if key property was overrided
1302 key = self.getkey()
1303 try:
1304 id = self.lookup(node[key])
1305 except KeyError:
1306 pass
1307 else:
1308 raise KeyError("Key property (%s) of retired node clashes "
1309 "with existing one (%s)" % (key, node[key]))
1310 # Now we can safely restore node
1311 self.fireAuditors('restore', nodeid, None)
1312 del node[self.db.RETIRED_FLAG]
1313 self.db.setnode(self.classname, nodeid, node)
1314 if self.do_journal:
1315 self.db.addjournal(self.classname, nodeid, 'restored', None)
1317 self.fireReactors('restore', nodeid, None)
1319 def is_retired(self, nodeid, cldb=None):
1320 """Return true if the node is retired.
1321 """
1322 node = self.db.getnode(self.classname, nodeid, cldb)
1323 if self.db.RETIRED_FLAG in node:
1324 return 1
1325 return 0
1327 def destroy(self, nodeid):
1328 """Destroy a node.
1330 WARNING: this method should never be used except in extremely rare
1331 situations where there could never be links to the node being
1332 deleted
1334 WARNING: use retire() instead
1336 WARNING: the properties of this node will not be available ever again
1338 WARNING: really, use retire() instead
1340 Well, I think that's enough warnings. This method exists mostly to
1341 support the session storage of the cgi interface.
1342 """
1343 if self.db.journaltag is None:
1344 raise hyperdb.DatabaseError(_('Database open read-only'))
1345 self.db.destroynode(self.classname, nodeid)
1347 def history(self, nodeid):
1348 """Retrieve the journal of edits on a particular node.
1350 'nodeid' must be the id of an existing node of this class or an
1351 IndexError is raised.
1353 The returned list contains tuples of the form
1355 (nodeid, date, tag, action, params)
1357 'date' is a Timestamp object specifying the time of the change and
1358 'tag' is the journaltag specified when the database was opened.
1359 """
1360 if not self.do_journal:
1361 raise ValueError('Journalling is disabled for this class')
1362 return self.db.getjournal(self.classname, nodeid)
1364 # Locating nodes:
1365 def hasnode(self, nodeid):
1366 """Determine if the given nodeid actually exists
1367 """
1368 return self.db.hasnode(self.classname, nodeid)
1370 def setkey(self, propname):
1371 """Select a String property of this class to be the key property.
1373 'propname' must be the name of a String property of this class or
1374 None, or a TypeError is raised. The values of the key property on
1375 all existing nodes must be unique or a ValueError is raised. If the
1376 property doesn't exist, KeyError is raised.
1377 """
1378 prop = self.getprops()[propname]
1379 if not isinstance(prop, hyperdb.String):
1380 raise TypeError('key properties must be String')
1381 self.key = propname
1383 def getkey(self):
1384 """Return the name of the key property for this class or None."""
1385 return self.key
1387 # TODO: set up a separate index db file for this? profile?
1388 def lookup(self, keyvalue):
1389 """Locate a particular node by its key property and return its id.
1391 If this class has no key property, a TypeError is raised. If the
1392 'keyvalue' matches one of the values for the key property among
1393 the nodes in this class, the matching node's id is returned;
1394 otherwise a KeyError is raised.
1395 """
1396 if not self.key:
1397 raise TypeError('No key property set for '
1398 'class %s'%self.classname)
1399 cldb = self.db.getclassdb(self.classname)
1400 try:
1401 for nodeid in self.getnodeids(cldb):
1402 node = self.db.getnode(self.classname, nodeid, cldb)
1403 if self.db.RETIRED_FLAG in node:
1404 continue
1405 if self.key not in node:
1406 continue
1407 if node[self.key] == keyvalue:
1408 return nodeid
1409 finally:
1410 cldb.close()
1411 raise KeyError('No key (%s) value "%s" for "%s"'%(self.key,
1412 keyvalue, self.classname))
1414 # change from spec - allows multiple props to match
1415 def find(self, **propspec):
1416 """Get the ids of nodes in this class which link to the given nodes.
1418 'propspec' consists of keyword args propname=nodeid or
1419 propname={nodeid:1, }
1420 'propname' must be the name of a property in this class, or a
1421 KeyError is raised. That property must be a Link or
1422 Multilink property, or a TypeError is raised.
1424 Any node in this class whose 'propname' property links to any of
1425 the nodeids will be returned. Examples::
1427 db.issue.find(messages='1')
1428 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1429 """
1430 for propname, itemids in propspec.iteritems():
1431 # check the prop is OK
1432 prop = self.properties[propname]
1433 if not isinstance(prop, hyperdb.Link) and not isinstance(prop, hyperdb.Multilink):
1434 raise TypeError("'%s' not a Link/Multilink "
1435 "property"%propname)
1437 # ok, now do the find
1438 cldb = self.db.getclassdb(self.classname)
1439 l = []
1440 try:
1441 for id in self.getnodeids(db=cldb):
1442 item = self.db.getnode(self.classname, id, db=cldb)
1443 if self.db.RETIRED_FLAG in item:
1444 continue
1445 for propname, itemids in propspec.iteritems():
1446 if type(itemids) is not type({}):
1447 itemids = {itemids:1}
1449 # special case if the item doesn't have this property
1450 if propname not in item:
1451 if None in itemids:
1452 l.append(id)
1453 break
1454 continue
1456 # grab the property definition and its value on this item
1457 prop = self.properties[propname]
1458 value = item[propname]
1459 if isinstance(prop, hyperdb.Link) and value in itemids:
1460 l.append(id)
1461 break
1462 elif isinstance(prop, hyperdb.Multilink):
1463 hit = 0
1464 for v in value:
1465 if v in itemids:
1466 l.append(id)
1467 hit = 1
1468 break
1469 if hit:
1470 break
1471 finally:
1472 cldb.close()
1473 return l
1475 def stringFind(self, **requirements):
1476 """Locate a particular node by matching a set of its String
1477 properties in a caseless search.
1479 If the property is not a String property, a TypeError is raised.
1481 The return is a list of the id of all nodes that match.
1482 """
1483 for propname in requirements:
1484 prop = self.properties[propname]
1485 if not isinstance(prop, hyperdb.String):
1486 raise TypeError("'%s' not a String property"%propname)
1487 requirements[propname] = requirements[propname].lower()
1488 l = []
1489 cldb = self.db.getclassdb(self.classname)
1490 try:
1491 for nodeid in self.getnodeids(cldb):
1492 node = self.db.getnode(self.classname, nodeid, cldb)
1493 if self.db.RETIRED_FLAG in node:
1494 continue
1495 for key, value in requirements.iteritems():
1496 if key not in node:
1497 break
1498 if node[key] is None or node[key].lower() != value:
1499 break
1500 else:
1501 l.append(nodeid)
1502 finally:
1503 cldb.close()
1504 return l
1506 def list(self):
1507 """ Return a list of the ids of the active nodes in this class.
1508 """
1509 l = []
1510 cn = self.classname
1511 cldb = self.db.getclassdb(cn)
1512 try:
1513 for nodeid in self.getnodeids(cldb):
1514 node = self.db.getnode(cn, nodeid, cldb)
1515 if self.db.RETIRED_FLAG in node:
1516 continue
1517 l.append(nodeid)
1518 finally:
1519 cldb.close()
1520 l.sort()
1521 return l
1523 def getnodeids(self, db=None, retired=None):
1524 """ Return a list of ALL nodeids
1526 Set retired=None to get all nodes. Otherwise it'll get all the
1527 retired or non-retired nodes, depending on the flag.
1528 """
1529 res = []
1531 # start off with the new nodes
1532 if self.classname in self.db.newnodes:
1533 res.extend(self.db.newnodes[self.classname])
1535 must_close = False
1536 if db is None:
1537 db = self.db.getclassdb(self.classname)
1538 must_close = True
1539 try:
1540 res.extend(db.keys())
1542 # remove the uncommitted, destroyed nodes
1543 if self.classname in self.db.destroyednodes:
1544 for nodeid in self.db.destroyednodes[self.classname]:
1545 if key_in(db, nodeid):
1546 res.remove(nodeid)
1548 # check retired flag
1549 if retired is False or retired is True:
1550 l = []
1551 for nodeid in res:
1552 node = self.db.getnode(self.classname, nodeid, db)
1553 is_ret = self.db.RETIRED_FLAG in node
1554 if retired == is_ret:
1555 l.append(nodeid)
1556 res = l
1557 finally:
1558 if must_close:
1559 db.close()
1560 return res
1562 def _filter(self, search_matches, filterspec, proptree,
1563 num_re = re.compile('^\d+$')):
1564 """Return a list of the ids of the active nodes in this class that
1565 match the 'filter' spec, sorted by the group spec and then the
1566 sort spec.
1568 "filterspec" is {propname: value(s)}
1570 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1571 and prop is a prop name or None
1573 "search_matches" is a sequence type or None
1575 The filter must match all properties specificed. If the property
1576 value to match is a list:
1578 1. String properties must match all elements in the list, and
1579 2. Other properties must match any of the elements in the list.
1580 """
1581 if __debug__:
1582 start_t = time.time()
1584 cn = self.classname
1586 # optimise filterspec
1587 l = []
1588 props = self.getprops()
1589 LINK = 'spec:link'
1590 MULTILINK = 'spec:multilink'
1591 STRING = 'spec:string'
1592 DATE = 'spec:date'
1593 INTERVAL = 'spec:interval'
1594 OTHER = 'spec:other'
1596 for k, v in filterspec.iteritems():
1597 propclass = props[k]
1598 if isinstance(propclass, hyperdb.Link):
1599 if type(v) is not type([]):
1600 v = [v]
1601 u = []
1602 for entry in v:
1603 # the value -1 is a special "not set" sentinel
1604 if entry == '-1':
1605 entry = None
1606 u.append(entry)
1607 l.append((LINK, k, u))
1608 elif isinstance(propclass, hyperdb.Multilink):
1609 # the value -1 is a special "not set" sentinel
1610 if v in ('-1', ['-1']):
1611 v = []
1612 elif type(v) is not type([]):
1613 v = [v]
1614 l.append((MULTILINK, k, v))
1615 elif isinstance(propclass, hyperdb.String) and k != 'id':
1616 if type(v) is not type([]):
1617 v = [v]
1618 for v in v:
1619 # simple glob searching
1620 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1621 v = v.replace('?', '.')
1622 v = v.replace('*', '.*?')
1623 l.append((STRING, k, re.compile(v, re.I)))
1624 elif isinstance(propclass, hyperdb.Date):
1625 try:
1626 date_rng = propclass.range_from_raw(v, self.db)
1627 l.append((DATE, k, date_rng))
1628 except ValueError:
1629 # If range creation fails - ignore that search parameter
1630 pass
1631 elif isinstance(propclass, hyperdb.Interval):
1632 try:
1633 intv_rng = date.Range(v, date.Interval)
1634 l.append((INTERVAL, k, intv_rng))
1635 except ValueError:
1636 # If range creation fails - ignore that search parameter
1637 pass
1639 elif isinstance(propclass, hyperdb.Boolean):
1640 if type(v) == type(""):
1641 v = v.split(',')
1642 if type(v) != type([]):
1643 v = [v]
1644 bv = []
1645 for val in v:
1646 if type(val) is type(''):
1647 bv.append(propclass.from_raw (val))
1648 else:
1649 bv.append(val)
1650 l.append((OTHER, k, bv))
1652 elif k == 'id':
1653 if type(v) != type([]):
1654 v = v.split(',')
1655 l.append((OTHER, k, [str(int(val)) for val in v]))
1657 elif isinstance(propclass, hyperdb.Number):
1658 if type(v) != type([]):
1659 try :
1660 v = v.split(',')
1661 except AttributeError :
1662 v = [v]
1663 l.append((OTHER, k, [float(val) for val in v]))
1665 filterspec = l
1667 # now, find all the nodes that are active and pass filtering
1668 matches = []
1669 cldb = self.db.getclassdb(cn)
1670 t = 0
1671 try:
1672 # TODO: only full-scan once (use items())
1673 for nodeid in self.getnodeids(cldb):
1674 node = self.db.getnode(cn, nodeid, cldb)
1675 if self.db.RETIRED_FLAG in node:
1676 continue
1677 # apply filter
1678 for t, k, v in filterspec:
1679 # handle the id prop
1680 if k == 'id':
1681 if nodeid not in v:
1682 break
1683 continue
1685 # get the node value
1686 nv = node.get(k, None)
1688 match = 0
1690 # now apply the property filter
1691 if t == LINK:
1692 # link - if this node's property doesn't appear in the
1693 # filterspec's nodeid list, skip it
1694 match = nv in v
1695 elif t == MULTILINK:
1696 # multilink - if any of the nodeids required by the
1697 # filterspec aren't in this node's property, then skip
1698 # it
1699 nv = node.get(k, [])
1701 # check for matching the absence of multilink values
1702 if not v:
1703 match = not nv
1704 else:
1705 # othewise, make sure this node has each of the
1706 # required values
1707 for want in v:
1708 if want in nv:
1709 match = 1
1710 break
1711 elif t == STRING:
1712 if nv is None:
1713 nv = ''
1714 # RE search
1715 match = v.search(nv)
1716 elif t == DATE or t == INTERVAL:
1717 if nv is None:
1718 match = v is None
1719 else:
1720 if v.to_value:
1721 if v.from_value <= nv and v.to_value >= nv:
1722 match = 1
1723 else:
1724 if v.from_value <= nv:
1725 match = 1
1726 elif t == OTHER:
1727 # straight value comparison for the other types
1728 match = nv in v
1729 if not match:
1730 break
1731 else:
1732 matches.append([nodeid, node])
1734 # filter based on full text search
1735 if search_matches is not None:
1736 k = []
1737 for v in matches:
1738 if v[0] in search_matches:
1739 k.append(v)
1740 matches = k
1742 # add sorting information to the proptree
1743 JPROPS = {'actor':1, 'activity':1, 'creator':1, 'creation':1}
1744 children = []
1745 if proptree:
1746 children = proptree.sortable_children()
1747 for pt in children:
1748 dir = pt.sort_direction
1749 prop = pt.name
1750 assert (dir and prop)
1751 propclass = props[prop]
1752 pt.sort_ids = []
1753 is_pointer = isinstance(propclass,(hyperdb.Link,
1754 hyperdb.Multilink))
1755 if not is_pointer:
1756 pt.sort_result = []
1757 try:
1758 # cache the opened link class db, if needed.
1759 lcldb = None
1760 # cache the linked class items too
1761 lcache = {}
1763 for entry in matches:
1764 itemid = entry[-2]
1765 item = entry[-1]
1766 # handle the properties that might be "faked"
1767 # also, handle possible missing properties
1768 try:
1769 v = item[prop]
1770 except KeyError:
1771 if prop in JPROPS:
1772 # force lookup of the special journal prop
1773 v = self.get(itemid, prop)
1774 else:
1775 # the node doesn't have a value for this
1776 # property
1777 v = None
1778 if isinstance(propclass, hyperdb.Multilink):
1779 v = []
1780 if prop == 'id':
1781 v = int (itemid)
1782 pt.sort_ids.append(v)
1783 if not is_pointer:
1784 pt.sort_result.append(v)
1785 continue
1787 # missing (None) values are always sorted first
1788 if v is None:
1789 pt.sort_ids.append(v)
1790 if not is_pointer:
1791 pt.sort_result.append(v)
1792 continue
1794 if isinstance(propclass, hyperdb.Link):
1795 lcn = propclass.classname
1796 link = self.db.classes[lcn]
1797 key = link.orderprop()
1798 child = pt.propdict[key]
1799 if key!='id':
1800 if v not in lcache:
1801 # open the link class db if it's not already
1802 if lcldb is None:
1803 lcldb = self.db.getclassdb(lcn)
1804 lcache[v] = self.db.getnode(lcn, v, lcldb)
1805 r = lcache[v][key]
1806 child.propdict[key].sort_ids.append(r)
1807 else:
1808 child.propdict[key].sort_ids.append(v)
1809 pt.sort_ids.append(v)
1810 if not is_pointer:
1811 r = propclass.sort_repr(pt.parent.cls, v, pt.name)
1812 pt.sort_result.append(r)
1813 finally:
1814 # if we opened the link class db, close it now
1815 if lcldb is not None:
1816 lcldb.close()
1817 del lcache
1818 finally:
1819 cldb.close()
1821 # pull the id out of the individual entries
1822 matches = [entry[-2] for entry in matches]
1823 if __debug__:
1824 self.db.stats['filtering'] += (time.time() - start_t)
1825 return matches
1827 def count(self):
1828 """Get the number of nodes in this class.
1830 If the returned integer is 'numnodes', the ids of all the nodes
1831 in this class run from 1 to numnodes, and numnodes+1 will be the
1832 id of the next node to be created in this class.
1833 """
1834 return self.db.countnodes(self.classname)
1836 # Manipulating properties:
1838 def getprops(self, protected=1):
1839 """Return a dictionary mapping property names to property objects.
1840 If the "protected" flag is true, we include protected properties -
1841 those which may not be modified.
1843 In addition to the actual properties on the node, these
1844 methods provide the "creation" and "activity" properties. If the
1845 "protected" flag is true, we include protected properties - those
1846 which may not be modified.
1847 """
1848 d = self.properties.copy()
1849 if protected:
1850 d['id'] = hyperdb.String()
1851 d['creation'] = hyperdb.Date()
1852 d['activity'] = hyperdb.Date()
1853 d['creator'] = hyperdb.Link('user')
1854 d['actor'] = hyperdb.Link('user')
1855 return d
1857 def addprop(self, **properties):
1858 """Add properties to this class.
1860 The keyword arguments in 'properties' must map names to property
1861 objects, or a TypeError is raised. None of the keys in 'properties'
1862 may collide with the names of existing properties, or a ValueError
1863 is raised before any properties have been added.
1864 """
1865 for key in properties:
1866 if key in self.properties:
1867 raise ValueError(key)
1868 self.properties.update(properties)
1870 def index(self, nodeid):
1871 """ Add (or refresh) the node to search indexes """
1872 # find all the String properties that have indexme
1873 for prop, propclass in self.getprops().iteritems():
1874 if isinstance(propclass, hyperdb.String) and propclass.indexme:
1875 # index them under (classname, nodeid, property)
1876 try:
1877 value = str(self.get(nodeid, prop))
1878 except IndexError:
1879 # node has been destroyed
1880 continue
1881 self.db.indexer.add_text((self.classname, nodeid, prop), value)
1883 #
1884 # import / export support
1885 #
1886 def export_list(self, propnames, nodeid):
1887 """ Export a node - generate a list of CSV-able data in the order
1888 specified by propnames for the given node.
1889 """
1890 properties = self.getprops()
1891 l = []
1892 for prop in propnames:
1893 proptype = properties[prop]
1894 value = self.get(nodeid, prop)
1895 # "marshal" data where needed
1896 if value is None:
1897 pass
1898 elif isinstance(proptype, hyperdb.Date):
1899 value = value.get_tuple()
1900 elif isinstance(proptype, hyperdb.Interval):
1901 value = value.get_tuple()
1902 elif isinstance(proptype, hyperdb.Password):
1903 value = str(value)
1904 l.append(repr(value))
1906 # append retired flag
1907 l.append(repr(self.is_retired(nodeid)))
1909 return l
1911 def import_list(self, propnames, proplist):
1912 """ Import a node - all information including "id" is present and
1913 should not be sanity checked. Triggers are not triggered. The
1914 journal should be initialised using the "creator" and "created"
1915 information.
1917 Return the nodeid of the node imported.
1918 """
1919 if self.db.journaltag is None:
1920 raise hyperdb.DatabaseError(_('Database open read-only'))
1921 properties = self.getprops()
1923 # make the new node's property map
1924 d = {}
1925 newid = None
1926 for i in range(len(propnames)):
1927 # Figure the property for this column
1928 propname = propnames[i]
1930 # Use eval to reverse the repr() used to output the CSV
1931 value = eval(proplist[i])
1933 # "unmarshal" where necessary
1934 if propname == 'id':
1935 newid = value
1936 continue
1937 elif propname == 'is retired':
1938 # is the item retired?
1939 if int(value):
1940 d[self.db.RETIRED_FLAG] = 1
1941 continue
1942 elif value is None:
1943 d[propname] = None
1944 continue
1946 prop = properties[propname]
1947 if isinstance(prop, hyperdb.Date):
1948 value = date.Date(value)
1949 elif isinstance(prop, hyperdb.Interval):
1950 value = date.Interval(value)
1951 elif isinstance(prop, hyperdb.Password):
1952 pwd = password.Password()
1953 pwd.unpack(value)
1954 value = pwd
1955 d[propname] = value
1957 # get a new id if necessary
1958 if newid is None:
1959 newid = self.db.newid(self.classname)
1961 # add the node and journal
1962 self.db.addnode(self.classname, newid, d)
1963 return newid
1965 def export_journals(self):
1966 """Export a class's journal - generate a list of lists of
1967 CSV-able data:
1969 nodeid, date, user, action, params
1971 No heading here - the columns are fixed.
1972 """
1973 properties = self.getprops()
1974 r = []
1975 for nodeid in self.getnodeids():
1976 for nodeid, date, user, action, params in self.history(nodeid):
1977 date = date.get_tuple()
1978 if action == 'set':
1979 export_data = {}
1980 for propname, value in params.iteritems():
1981 if propname not in properties:
1982 # property no longer in the schema
1983 continue
1985 prop = properties[propname]
1986 # make sure the params are eval()'able
1987 if value is None:
1988 pass
1989 elif isinstance(prop, hyperdb.Date):
1990 # this is a hack - some dates are stored as strings
1991 if not isinstance(value, type('')):
1992 value = value.get_tuple()
1993 elif isinstance(prop, hyperdb.Interval):
1994 # hack too - some intervals are stored as strings
1995 if not isinstance(value, type('')):
1996 value = value.get_tuple()
1997 elif isinstance(prop, hyperdb.Password):
1998 value = str(value)
1999 export_data[propname] = value
2000 params = export_data
2001 r.append([repr(nodeid), repr(date), repr(user),
2002 repr(action), repr(params)])
2003 return r
2005 class FileClass(hyperdb.FileClass, Class):
2006 """This class defines a large chunk of data. To support this, it has a
2007 mandatory String property "content" which is typically saved off
2008 externally to the hyperdb.
2010 The default MIME type of this data is defined by the
2011 "default_mime_type" class attribute, which may be overridden by each
2012 node if the class defines a "type" String property.
2013 """
2014 def __init__(self, db, classname, **properties):
2015 """The newly-created class automatically includes the "content"
2016 and "type" properties.
2017 """
2018 if 'content' not in properties:
2019 properties['content'] = hyperdb.String(indexme='yes')
2020 if 'type' not in properties:
2021 properties['type'] = hyperdb.String()
2022 Class.__init__(self, db, classname, **properties)
2024 def create(self, **propvalues):
2025 """ Snarf the "content" propvalue and store in a file
2026 """
2027 # we need to fire the auditors now, or the content property won't
2028 # be in propvalues for the auditors to play with
2029 self.fireAuditors('create', None, propvalues)
2031 # now remove the content property so it's not stored in the db
2032 content = propvalues['content']
2033 del propvalues['content']
2035 # make sure we have a MIME type
2036 mime_type = propvalues.get('type', self.default_mime_type)
2038 # do the database create
2039 newid = self.create_inner(**propvalues)
2041 # store off the content as a file
2042 self.db.storefile(self.classname, newid, None, content)
2044 # fire reactors
2045 self.fireReactors('create', newid, None)
2047 return newid
2049 def get(self, nodeid, propname, default=_marker, cache=1):
2050 """ Trap the content propname and get it from the file
2052 'cache' exists for backwards compatibility, and is not used.
2053 """
2054 poss_msg = 'Possibly an access right configuration problem.'
2055 if propname == 'content':
2056 try:
2057 return self.db.getfile(self.classname, nodeid, None)
2058 except IOError, strerror:
2059 # XXX by catching this we don't see an error in the log.
2060 return 'ERROR reading file: %s%s\n%s\n%s'%(
2061 self.classname, nodeid, poss_msg, strerror)
2062 if default is not _marker:
2063 return Class.get(self, nodeid, propname, default)
2064 else:
2065 return Class.get(self, nodeid, propname)
2067 def set(self, itemid, **propvalues):
2068 """ Snarf the "content" propvalue and update it in a file
2069 """
2070 self.fireAuditors('set', itemid, propvalues)
2072 # create the oldvalues dict - fill in any missing values
2073 oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid))
2074 for name, prop in self.getprops(protected=0).iteritems():
2075 if name in oldvalues:
2076 continue
2077 if isinstance(prop, hyperdb.Multilink):
2078 oldvalues[name] = []
2079 else:
2080 oldvalues[name] = None
2082 # now remove the content property so it's not stored in the db
2083 content = None
2084 if 'content' in propvalues:
2085 content = propvalues['content']
2086 del propvalues['content']
2088 # do the database update
2089 propvalues = self.set_inner(itemid, **propvalues)
2091 # do content?
2092 if content:
2093 # store and possibly index
2094 self.db.storefile(self.classname, itemid, None, content)
2095 if self.properties['content'].indexme:
2096 mime_type = self.get(itemid, 'type', self.default_mime_type)
2097 self.db.indexer.add_text((self.classname, itemid, 'content'),
2098 content, mime_type)
2099 propvalues['content'] = content
2101 # fire reactors
2102 self.fireReactors('set', itemid, oldvalues)
2103 return propvalues
2105 def index(self, nodeid):
2106 """ Add (or refresh) the node to search indexes.
2108 Use the content-type property for the content property.
2109 """
2110 # find all the String properties that have indexme
2111 for prop, propclass in self.getprops().iteritems():
2112 if prop == 'content' and propclass.indexme:
2113 mime_type = self.get(nodeid, 'type', self.default_mime_type)
2114 self.db.indexer.add_text((self.classname, nodeid, 'content'),
2115 str(self.get(nodeid, 'content')), mime_type)
2116 elif isinstance(propclass, hyperdb.String) and propclass.indexme:
2117 # index them under (classname, nodeid, property)
2118 try:
2119 value = str(self.get(nodeid, prop))
2120 except IndexError:
2121 # node has been destroyed
2122 continue
2123 self.db.indexer.add_text((self.classname, nodeid, prop), value)
2125 # deviation from spec - was called ItemClass
2126 class IssueClass(Class, roundupdb.IssueClass):
2127 # Overridden methods:
2128 def __init__(self, db, classname, **properties):
2129 """The newly-created class automatically includes the "messages",
2130 "files", "nosy", and "superseder" properties. If the 'properties'
2131 dictionary attempts to specify any of these properties or a
2132 "creation" or "activity" property, a ValueError is raised.
2133 """
2134 if 'title' not in properties:
2135 properties['title'] = hyperdb.String(indexme='yes')
2136 if 'messages' not in properties:
2137 properties['messages'] = hyperdb.Multilink("msg")
2138 if 'files' not in properties:
2139 properties['files'] = hyperdb.Multilink("file")
2140 if 'nosy' not in properties:
2141 # note: journalling is turned off as it really just wastes
2142 # space. this behaviour may be overridden in an instance
2143 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2144 if 'superseder' not in properties:
2145 properties['superseder'] = hyperdb.Multilink(classname)
2146 Class.__init__(self, db, classname, **properties)
2148 # vim: set et sts=4 sw=4 :