1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 """This module defines a backend that saves the hyperdatabase in a
19 database chosen by anydbm. It is guaranteed to always be available in python
20 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
21 serious bugs, and is not available)
22 """
23 __docformat__ = 'restructuredtext'
25 import os, marshal, re, weakref, string, copy, time, shutil, logging
27 from roundup.anypy.dbm_ import anydbm, whichdb, key_in
29 from roundup import hyperdb, date, password, roundupdb, security, support
30 from roundup.support import reversed
31 from roundup.backends import locking
32 from roundup.i18n import _
34 from roundup.backends.blobfiles import FileStorage
35 from roundup.backends.sessions_dbm import Sessions, OneTimeKeys
37 try:
38 from roundup.backends.indexer_xapian import Indexer
39 except ImportError:
40 from roundup.backends.indexer_dbm import Indexer
42 def db_exists(config):
43 # check for the user db
44 for db in 'nodes.user nodes.user.db'.split():
45 if os.path.exists(os.path.join(config.DATABASE, db)):
46 return 1
47 return 0
49 def db_nuke(config):
50 shutil.rmtree(config.DATABASE)
52 #
53 # Now the database
54 #
55 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
56 """A database for storing records containing flexible data types.
58 Transaction stuff TODO:
60 - check the timestamp of the class file and nuke the cache if it's
61 modified. Do some sort of conflict checking on the dirty stuff.
62 - perhaps detect write collisions (related to above)?
63 """
64 def __init__(self, config, journaltag=None):
65 """Open a hyperdatabase given a specifier to some storage.
67 The 'storagelocator' is obtained from config.DATABASE.
68 The meaning of 'storagelocator' depends on the particular
69 implementation of the hyperdatabase. It could be a file name,
70 a directory path, a socket descriptor for a connection to a
71 database over the network, etc.
73 The 'journaltag' is a token that will be attached to the journal
74 entries for any edits done on the database. If 'journaltag' is
75 None, the database is opened in read-only mode: the Class.create(),
76 Class.set(), Class.retire(), and Class.restore() methods are
77 disabled.
78 """
79 FileStorage.__init__(self, config.UMASK)
80 self.config, self.journaltag = config, journaltag
81 self.dir = config.DATABASE
82 self.classes = {}
83 self.cache = {} # cache of nodes loaded or created
84 self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0,
85 'filtering': 0}
86 self.dirtynodes = {} # keep track of the dirty nodes by class
87 self.newnodes = {} # keep track of the new nodes by class
88 self.destroyednodes = {}# keep track of the destroyed nodes by class
89 self.transactions = []
90 self.indexer = Indexer(self)
91 self.security = security.Security(self)
92 os.umask(config.UMASK)
94 # lock it
95 lockfilenm = os.path.join(self.dir, 'lock')
96 self.lockfile = locking.acquire_lock(lockfilenm)
97 self.lockfile.write(str(os.getpid()))
98 self.lockfile.flush()
100 def post_init(self):
101 """Called once the schema initialisation has finished.
102 """
103 # reindex the db if necessary
104 if self.indexer.should_reindex():
105 self.reindex()
107 def refresh_database(self):
108 """Rebuild the database
109 """
110 self.reindex()
112 def getSessionManager(self):
113 return Sessions(self)
115 def getOTKManager(self):
116 return OneTimeKeys(self)
118 def reindex(self, classname=None, show_progress=False):
119 if classname:
120 classes = [self.getclass(classname)]
121 else:
122 classes = self.classes.values()
123 for klass in classes:
124 if show_progress:
125 for nodeid in support.Progress('Reindex %s'%klass.classname,
126 klass.list()):
127 klass.index(nodeid)
128 else:
129 for nodeid in klass.list():
130 klass.index(nodeid)
131 self.indexer.save_index()
133 def __repr__(self):
134 return '<back_anydbm instance at %x>'%id(self)
136 #
137 # Classes
138 #
139 def __getattr__(self, classname):
140 """A convenient way of calling self.getclass(classname)."""
141 if classname in self.classes:
142 return self.classes[classname]
143 raise AttributeError, classname
145 def addclass(self, cl):
146 cn = cl.classname
147 if cn in self.classes:
148 raise ValueError, cn
149 self.classes[cn] = cl
151 # add default Edit and View permissions
152 self.security.addPermission(name="Create", klass=cn,
153 description="User is allowed to create "+cn)
154 self.security.addPermission(name="Edit", klass=cn,
155 description="User is allowed to edit "+cn)
156 self.security.addPermission(name="View", klass=cn,
157 description="User is allowed to access "+cn)
159 def getclasses(self):
160 """Return a list of the names of all existing classes."""
161 return sorted(self.classes)
163 def getclass(self, classname):
164 """Get the Class object representing a particular class.
166 If 'classname' is not a valid class name, a KeyError is raised.
167 """
168 try:
169 return self.classes[classname]
170 except KeyError:
171 raise KeyError('There is no class called "%s"'%classname)
173 #
174 # Class DBs
175 #
176 def clear(self):
177 """Delete all database contents
178 """
179 logging.getLogger('roundup.hyperdb').info('clear')
180 for cn in self.classes:
181 for dummy in 'nodes', 'journals':
182 path = os.path.join(self.dir, 'journals.%s'%cn)
183 if os.path.exists(path):
184 os.remove(path)
185 elif os.path.exists(path+'.db'): # dbm appends .db
186 os.remove(path+'.db')
187 # reset id sequences
188 path = os.path.join(os.getcwd(), self.dir, '_ids')
189 if os.path.exists(path):
190 os.remove(path)
191 elif os.path.exists(path+'.db'): # dbm appends .db
192 os.remove(path+'.db')
194 def getclassdb(self, classname, mode='r'):
195 """ grab a connection to the class db that will be used for
196 multiple actions
197 """
198 return self.opendb('nodes.%s'%classname, mode)
200 def determine_db_type(self, path):
201 """ determine which DB wrote the class file
202 """
203 db_type = ''
204 if os.path.exists(path):
205 db_type = whichdb(path)
206 if not db_type:
207 raise hyperdb.DatabaseError(_("Couldn't identify database type"))
208 elif os.path.exists(path+'.db'):
209 # if the path ends in '.db', it's a dbm database, whether
210 # anydbm says it's dbhash or not!
211 db_type = 'dbm'
212 return db_type
214 def opendb(self, name, mode):
215 """Low-level database opener that gets around anydbm/dbm
216 eccentricities.
217 """
218 # figure the class db type
219 path = os.path.join(os.getcwd(), self.dir, name)
220 db_type = self.determine_db_type(path)
222 # new database? let anydbm pick the best dbm
223 # in Python 3+ the "dbm" ("anydbm" to us) module already uses the
224 # whichdb() function to do this
225 if not db_type or hasattr(anydbm, 'whichdb'):
226 if __debug__:
227 logging.getLogger('roundup.hyperdb').debug(
228 "opendb anydbm.open(%r, 'c')"%path)
229 return anydbm.open(path, 'c')
231 # in Python <3 it anydbm was a little dumb so manually open the
232 # database with the correct module
233 try:
234 dbm = __import__(db_type)
235 except ImportError:
236 raise hyperdb.DatabaseError(_("Couldn't open database - the "
237 "required module '%s' is not available")%db_type)
238 if __debug__:
239 logging.getLogger('roundup.hyperdb').debug(
240 "opendb %r.open(%r, %r)"%(db_type, path, mode))
241 return dbm.open(path, mode)
243 #
244 # Node IDs
245 #
246 def newid(self, classname):
247 """ Generate a new id for the given class
248 """
249 # open the ids DB - create if if doesn't exist
250 db = self.opendb('_ids', 'c')
251 if key_in(db, classname):
252 newid = db[classname] = str(int(db[classname]) + 1)
253 else:
254 # the count() bit is transitional - older dbs won't start at 1
255 newid = str(self.getclass(classname).count()+1)
256 db[classname] = newid
257 db.close()
258 return newid
260 def setid(self, classname, setid):
261 """ Set the id counter: used during import of database
262 """
263 # open the ids DB - create if if doesn't exist
264 db = self.opendb('_ids', 'c')
265 db[classname] = str(setid)
266 db.close()
268 #
269 # Nodes
270 #
271 def addnode(self, classname, nodeid, node):
272 """ add the specified node to its class's db
273 """
274 # we'll be supplied these props if we're doing an import
275 if 'creator' not in node:
276 # add in the "calculated" properties (dupe so we don't affect
277 # calling code's node assumptions)
278 node = node.copy()
279 node['creator'] = self.getuid()
280 node['actor'] = self.getuid()
281 node['creation'] = node['activity'] = date.Date()
283 self.newnodes.setdefault(classname, {})[nodeid] = 1
284 self.cache.setdefault(classname, {})[nodeid] = node
285 self.savenode(classname, nodeid, node)
287 def setnode(self, classname, nodeid, node):
288 """ change the specified node
289 """
290 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
292 # can't set without having already loaded the node
293 self.cache[classname][nodeid] = node
294 self.savenode(classname, nodeid, node)
296 def savenode(self, classname, nodeid, node):
297 """ perform the saving of data specified by the set/addnode
298 """
299 if __debug__:
300 logging.getLogger('roundup.hyperdb').debug(
301 'save %s%s %r'%(classname, nodeid, node))
302 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
304 def getnode(self, classname, nodeid, db=None, cache=1):
305 """ get a node from the database
307 Note the "cache" parameter is not used, and exists purely for
308 backward compatibility!
309 """
310 # try the cache
311 cache_dict = self.cache.setdefault(classname, {})
312 if nodeid in cache_dict:
313 if __debug__:
314 logging.getLogger('roundup.hyperdb').debug(
315 'get %s%s cached'%(classname, nodeid))
316 self.stats['cache_hits'] += 1
317 return cache_dict[nodeid]
319 if __debug__:
320 self.stats['cache_misses'] += 1
321 start_t = time.time()
322 logging.getLogger('roundup.hyperdb').debug(
323 'get %s%s'%(classname, nodeid))
325 # get from the database and save in the cache
326 if db is None:
327 db = self.getclassdb(classname)
328 if not key_in(db, nodeid):
329 raise IndexError("no such %s %s"%(classname, nodeid))
331 # check the uncommitted, destroyed nodes
332 if (classname in self.destroyednodes and
333 nodeid in self.destroyednodes[classname]):
334 raise IndexError("no such %s %s"%(classname, nodeid))
336 # decode
337 res = marshal.loads(db[nodeid])
339 # reverse the serialisation
340 res = self.unserialise(classname, res)
342 # store off in the cache dict
343 if cache:
344 cache_dict[nodeid] = res
346 if __debug__:
347 self.stats['get_items'] += (time.time() - start_t)
349 return res
351 def destroynode(self, classname, nodeid):
352 """Remove a node from the database. Called exclusively by the
353 destroy() method on Class.
354 """
355 logging.getLogger('roundup.hyperdb').info(
356 'destroy %s%s'%(classname, nodeid))
358 # remove from cache and newnodes if it's there
359 if (classname in self.cache and nodeid in self.cache[classname]):
360 del self.cache[classname][nodeid]
361 if (classname in self.newnodes and nodeid in self.newnodes[classname]):
362 del self.newnodes[classname][nodeid]
364 # see if there's any obvious commit actions that we should get rid of
365 for entry in self.transactions[:]:
366 if entry[1][:2] == (classname, nodeid):
367 self.transactions.remove(entry)
369 # add to the destroyednodes map
370 self.destroyednodes.setdefault(classname, {})[nodeid] = 1
372 # add the destroy commit action
373 self.transactions.append((self.doDestroyNode, (classname, nodeid)))
374 self.transactions.append((FileStorage.destroy, (self, classname, nodeid)))
376 def serialise(self, classname, node):
377 """Copy the node contents, converting non-marshallable data into
378 marshallable data.
379 """
380 properties = self.getclass(classname).getprops()
381 d = {}
382 for k, v in node.iteritems():
383 if k == self.RETIRED_FLAG:
384 d[k] = v
385 continue
387 # if the property doesn't exist then we really don't care
388 if k not in properties:
389 continue
391 # get the property spec
392 prop = properties[k]
394 if isinstance(prop, hyperdb.Password) and v is not None:
395 d[k] = str(v)
396 elif isinstance(prop, hyperdb.Date) and v is not None:
397 d[k] = v.serialise()
398 elif isinstance(prop, hyperdb.Interval) and v is not None:
399 d[k] = v.serialise()
400 else:
401 d[k] = v
402 return d
404 def unserialise(self, classname, node):
405 """Decode the marshalled node data
406 """
407 properties = self.getclass(classname).getprops()
408 d = {}
409 for k, v in node.iteritems():
410 # if the property doesn't exist, or is the "retired" flag then
411 # it won't be in the properties dict
412 if k not in properties:
413 d[k] = v
414 continue
416 # get the property spec
417 prop = properties[k]
419 if isinstance(prop, hyperdb.Date) and v is not None:
420 d[k] = date.Date(v)
421 elif isinstance(prop, hyperdb.Interval) and v is not None:
422 d[k] = date.Interval(v)
423 elif isinstance(prop, hyperdb.Password) and v is not None:
424 p = password.Password()
425 p.unpack(v)
426 d[k] = p
427 else:
428 d[k] = v
429 return d
431 def hasnode(self, classname, nodeid, db=None):
432 """ determine if the database has a given node
433 """
434 # try the cache
435 cache = self.cache.setdefault(classname, {})
436 if nodeid in cache:
437 return 1
439 # not in the cache - check the database
440 if db is None:
441 db = self.getclassdb(classname)
442 return key_in(db, nodeid)
444 def countnodes(self, classname, db=None):
445 count = 0
447 # include the uncommitted nodes
448 if classname in self.newnodes:
449 count += len(self.newnodes[classname])
450 if classname in self.destroyednodes:
451 count -= len(self.destroyednodes[classname])
453 # and count those in the DB
454 if db is None:
455 db = self.getclassdb(classname)
456 return count + len(db)
459 #
460 # Files - special node properties
461 # inherited from FileStorage
463 #
464 # Journal
465 #
466 def addjournal(self, classname, nodeid, action, params, creator=None,
467 creation=None):
468 """ Journal the Action
469 'action' may be:
471 'create' or 'set' -- 'params' is a dictionary of property values
472 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
473 'retire' -- 'params' is None
475 'creator' -- the user performing the action, which defaults to
476 the current user.
477 """
478 if __debug__:
479 logging.getLogger('roundup.hyperdb').debug(
480 'addjournal %s%s %s %r %s %r'%(classname,
481 nodeid, action, params, creator, creation))
482 if creator is None:
483 creator = self.getuid()
484 self.transactions.append((self.doSaveJournal, (classname, nodeid,
485 action, params, creator, creation)))
487 def setjournal(self, classname, nodeid, journal):
488 """Set the journal to the "journal" list."""
489 if __debug__:
490 logging.getLogger('roundup.hyperdb').debug(
491 'setjournal %s%s %r'%(classname, nodeid, journal))
492 self.transactions.append((self.doSetJournal, (classname, nodeid,
493 journal)))
495 def getjournal(self, classname, nodeid):
496 """ get the journal for id
498 Raise IndexError if the node doesn't exist (as per history()'s
499 API)
500 """
501 # our journal result
502 res = []
504 # add any journal entries for transactions not committed to the
505 # database
506 for method, args in self.transactions:
507 if method != self.doSaveJournal:
508 continue
509 (cache_classname, cache_nodeid, cache_action, cache_params,
510 cache_creator, cache_creation) = args
511 if cache_classname == classname and cache_nodeid == nodeid:
512 if not cache_creator:
513 cache_creator = self.getuid()
514 if not cache_creation:
515 cache_creation = date.Date()
516 res.append((cache_nodeid, cache_creation, cache_creator,
517 cache_action, cache_params))
519 # attempt to open the journal - in some rare cases, the journal may
520 # not exist
521 try:
522 db = self.opendb('journals.%s'%classname, 'r')
523 except anydbm.error, error:
524 if str(error) == "need 'c' or 'n' flag to open new db":
525 raise IndexError('no such %s %s'%(classname, nodeid))
526 elif error.args[0] != 2:
527 # this isn't a "not found" error, be alarmed!
528 raise
529 if res:
530 # we have unsaved journal entries, return them
531 return res
532 raise IndexError('no such %s %s'%(classname, nodeid))
533 try:
534 journal = marshal.loads(db[nodeid])
535 except KeyError:
536 db.close()
537 if res:
538 # we have some unsaved journal entries, be happy!
539 return res
540 raise IndexError('no such %s %s'%(classname, nodeid))
541 db.close()
543 # add all the saved journal entries for this node
544 for nodeid, date_stamp, user, action, params in journal:
545 res.append((nodeid, date.Date(date_stamp), user, action, params))
546 return res
548 def pack(self, pack_before):
549 """ Delete all journal entries except "create" before 'pack_before'.
550 """
551 pack_before = pack_before.serialise()
552 for classname in self.getclasses():
553 packed = 0
554 # get the journal db
555 db_name = 'journals.%s'%classname
556 path = os.path.join(os.getcwd(), self.dir, classname)
557 db_type = self.determine_db_type(path)
558 db = self.opendb(db_name, 'w')
560 for key in db.keys():
561 # get the journal for this db entry
562 journal = marshal.loads(db[key])
563 l = []
564 last_set_entry = None
565 for entry in journal:
566 # unpack the entry
567 (nodeid, date_stamp, self.journaltag, action,
568 params) = entry
569 # if the entry is after the pack date, _or_ the initial
570 # create entry, then it stays
571 if date_stamp > pack_before or action == 'create':
572 l.append(entry)
573 else:
574 packed += 1
575 db[key] = marshal.dumps(l)
577 logging.getLogger('roundup.hyperdb').info(
578 'packed %d %s items'%(packed, classname))
580 if db_type == 'gdbm':
581 db.reorganize()
582 db.close()
585 #
586 # Basic transaction support
587 #
588 def commit(self, fail_ok=False):
589 """ Commit the current transactions.
591 Save all data changed since the database was opened or since the
592 last commit() or rollback().
594 fail_ok indicates that the commit is allowed to fail. This is used
595 in the web interface when committing cleaning of the session
596 database. We don't care if there's a concurrency issue there.
598 The only backend this seems to affect is postgres.
599 """
600 logging.getLogger('roundup.hyperdb').info('commit %s transactions'%(
601 len(self.transactions)))
603 # keep a handle to all the database files opened
604 self.databases = {}
606 try:
607 # now, do all the transactions
608 reindex = {}
609 for method, args in self.transactions:
610 reindex[method(*args)] = 1
611 finally:
612 # make sure we close all the database files
613 for db in self.databases.itervalues():
614 db.close()
615 del self.databases
617 # clear the transactions list now so the blobfile implementation
618 # doesn't think there's still pending file commits when it tries
619 # to access the file data
620 self.transactions = []
622 # reindex the nodes that request it
623 for classname, nodeid in [k for k in reindex if k]:
624 self.getclass(classname).index(nodeid)
626 # save the indexer state
627 self.indexer.save_index()
629 self.clearCache()
631 def clearCache(self):
632 # all transactions committed, back to normal
633 self.cache = {}
634 self.dirtynodes = {}
635 self.newnodes = {}
636 self.destroyednodes = {}
637 self.transactions = []
639 def getCachedClassDB(self, classname):
640 """ get the class db, looking in our cache of databases for commit
641 """
642 # get the database handle
643 db_name = 'nodes.%s'%classname
644 if db_name not in self.databases:
645 self.databases[db_name] = self.getclassdb(classname, 'c')
646 return self.databases[db_name]
648 def doSaveNode(self, classname, nodeid, node):
649 db = self.getCachedClassDB(classname)
651 # now save the marshalled data
652 db[nodeid] = marshal.dumps(self.serialise(classname, node))
654 # return the classname, nodeid so we reindex this content
655 return (classname, nodeid)
657 def getCachedJournalDB(self, classname):
658 """ get the journal db, looking in our cache of databases for commit
659 """
660 # get the database handle
661 db_name = 'journals.%s'%classname
662 if db_name not in self.databases:
663 self.databases[db_name] = self.opendb(db_name, 'c')
664 return self.databases[db_name]
666 def doSaveJournal(self, classname, nodeid, action, params, creator,
667 creation):
668 # serialise the parameters now if necessary
669 if isinstance(params, type({})):
670 if action in ('set', 'create'):
671 params = self.serialise(classname, params)
673 # handle supply of the special journalling parameters (usually
674 # supplied on importing an existing database)
675 journaltag = creator
676 if creation:
677 journaldate = creation.serialise()
678 else:
679 journaldate = date.Date().serialise()
681 # create the journal entry
682 entry = (nodeid, journaldate, journaltag, action, params)
684 db = self.getCachedJournalDB(classname)
686 # now insert the journal entry
687 if key_in(db, nodeid):
688 # append to existing
689 s = db[nodeid]
690 l = marshal.loads(s)
691 l.append(entry)
692 else:
693 l = [entry]
695 db[nodeid] = marshal.dumps(l)
697 def doSetJournal(self, classname, nodeid, journal):
698 l = []
699 for nodeid, journaldate, journaltag, action, params in journal:
700 # serialise the parameters now if necessary
701 if isinstance(params, type({})):
702 if action in ('set', 'create'):
703 params = self.serialise(classname, params)
704 journaldate = journaldate.serialise()
705 l.append((nodeid, journaldate, journaltag, action, params))
706 db = self.getCachedJournalDB(classname)
707 db[nodeid] = marshal.dumps(l)
709 def doDestroyNode(self, classname, nodeid):
710 # delete from the class database
711 db = self.getCachedClassDB(classname)
712 if key_in(db, nodeid):
713 del db[nodeid]
715 # delete from the database
716 db = self.getCachedJournalDB(classname)
717 if key_in(db, nodeid):
718 del db[nodeid]
720 def rollback(self):
721 """ Reverse all actions from the current transaction.
722 """
723 logging.getLogger('roundup.hyperdb').info('rollback %s transactions'%(
724 len(self.transactions)))
726 for method, args in self.transactions:
727 # delete temporary files
728 if method == self.doStoreFile:
729 self.rollbackStoreFile(*args)
730 self.cache = {}
731 self.dirtynodes = {}
732 self.newnodes = {}
733 self.destroyednodes = {}
734 self.transactions = []
736 def close(self):
737 """ Nothing to do
738 """
739 if self.lockfile is not None:
740 locking.release_lock(self.lockfile)
741 self.lockfile.close()
742 self.lockfile = None
744 _marker = []
745 class Class(hyperdb.Class):
746 """The handle to a particular class of nodes in a hyperdatabase."""
748 def enableJournalling(self):
749 """Turn journalling on for this class
750 """
751 self.do_journal = 1
753 def disableJournalling(self):
754 """Turn journalling off for this class
755 """
756 self.do_journal = 0
758 # Editing nodes:
760 def create(self, **propvalues):
761 """Create a new node of this class and return its id.
763 The keyword arguments in 'propvalues' map property names to values.
765 The values of arguments must be acceptable for the types of their
766 corresponding properties or a TypeError is raised.
768 If this class has a key property, it must be present and its value
769 must not collide with other key strings or a ValueError is raised.
771 Any other properties on this class that are missing from the
772 'propvalues' dictionary are set to None.
774 If an id in a link or multilink property does not refer to a valid
775 node, an IndexError is raised.
777 These operations trigger detectors and can be vetoed. Attempts
778 to modify the "creation" or "activity" properties cause a KeyError.
779 """
780 if self.db.journaltag is None:
781 raise hyperdb.DatabaseError(_('Database open read-only'))
782 self.fireAuditors('create', None, propvalues)
783 newid = self.create_inner(**propvalues)
784 self.fireReactors('create', newid, None)
785 return newid
787 def create_inner(self, **propvalues):
788 """ Called by create, in-between the audit and react calls.
789 """
790 if 'id' in propvalues:
791 raise KeyError('"id" is reserved')
793 if self.db.journaltag is None:
794 raise hyperdb.DatabaseError(_('Database open read-only'))
796 if 'creation' in propvalues or 'activity' in propvalues:
797 raise KeyError('"creation" and "activity" are reserved')
798 # new node's id
799 newid = self.db.newid(self.classname)
801 # validate propvalues
802 num_re = re.compile('^\d+$')
803 for key, value in propvalues.iteritems():
804 if key == self.key:
805 try:
806 self.lookup(value)
807 except KeyError:
808 pass
809 else:
810 raise ValueError('node with key "%s" exists'%value)
812 # try to handle this property
813 try:
814 prop = self.properties[key]
815 except KeyError:
816 raise KeyError('"%s" has no property "%s"'%(self.classname,
817 key))
819 if value is not None and isinstance(prop, hyperdb.Link):
820 if type(value) != type(''):
821 raise ValueError('link value must be String')
822 link_class = self.properties[key].classname
823 # if it isn't a number, it's a key
824 if not num_re.match(value):
825 try:
826 value = self.db.classes[link_class].lookup(value)
827 except (TypeError, KeyError):
828 raise IndexError('new property "%s": %s not a %s'%(
829 key, value, link_class))
830 elif not self.db.getclass(link_class).hasnode(value):
831 raise IndexError('%s has no node %s'%(link_class,
832 value))
834 # save off the value
835 propvalues[key] = value
837 # register the link with the newly linked node
838 if self.do_journal and self.properties[key].do_journal:
839 self.db.addjournal(link_class, value, 'link',
840 (self.classname, newid, key))
842 elif isinstance(prop, hyperdb.Multilink):
843 if value is None:
844 value = []
845 if not hasattr(value, '__iter__'):
846 raise TypeError('new property "%s" not an iterable of ids'%key)
848 # clean up and validate the list of links
849 link_class = self.properties[key].classname
850 l = []
851 for entry in value:
852 if type(entry) != type(''):
853 raise ValueError('"%s" multilink value (%r) '\
854 'must contain Strings'%(key, value))
855 # if it isn't a number, it's a key
856 if not num_re.match(entry):
857 try:
858 entry = self.db.classes[link_class].lookup(entry)
859 except (TypeError, KeyError):
860 raise IndexError('new property "%s": %s not a %s'%(
861 key, entry, self.properties[key].classname))
862 l.append(entry)
863 value = l
864 propvalues[key] = value
866 # handle additions
867 for nodeid in value:
868 if not self.db.getclass(link_class).hasnode(nodeid):
869 raise IndexError('%s has no node %s'%(link_class,
870 nodeid))
871 # register the link with the newly linked node
872 if self.do_journal and self.properties[key].do_journal:
873 self.db.addjournal(link_class, nodeid, 'link',
874 (self.classname, newid, key))
876 elif isinstance(prop, hyperdb.String):
877 if type(value) != type('') and type(value) != type(u''):
878 raise TypeError('new property "%s" not a string'%key)
879 if prop.indexme:
880 self.db.indexer.add_text((self.classname, newid, key),
881 value)
883 elif isinstance(prop, hyperdb.Password):
884 if not isinstance(value, password.Password):
885 raise TypeError('new property "%s" not a Password'%key)
887 elif isinstance(prop, hyperdb.Date):
888 if value is not None and not isinstance(value, date.Date):
889 raise TypeError('new property "%s" not a Date'%key)
891 elif isinstance(prop, hyperdb.Interval):
892 if value is not None and not isinstance(value, date.Interval):
893 raise TypeError('new property "%s" not an Interval'%key)
895 elif value is not None and isinstance(prop, hyperdb.Number):
896 try:
897 float(value)
898 except ValueError:
899 raise TypeError('new property "%s" not numeric'%key)
901 elif value is not None and isinstance(prop, hyperdb.Boolean):
902 try:
903 int(value)
904 except ValueError:
905 raise TypeError('new property "%s" not boolean'%key)
907 # make sure there's data where there needs to be
908 for key, prop in self.properties.iteritems():
909 if key in propvalues:
910 continue
911 if key == self.key:
912 raise ValueError('key property "%s" is required'%key)
913 if isinstance(prop, hyperdb.Multilink):
914 propvalues[key] = []
916 # done
917 self.db.addnode(self.classname, newid, propvalues)
918 if self.do_journal:
919 self.db.addjournal(self.classname, newid, 'create', {})
921 return newid
923 def get(self, nodeid, propname, default=_marker, cache=1):
924 """Get the value of a property on an existing node of this class.
926 'nodeid' must be the id of an existing node of this class or an
927 IndexError is raised. 'propname' must be the name of a property
928 of this class or a KeyError is raised.
930 'cache' exists for backward compatibility, and is not used.
932 Attempts to get the "creation" or "activity" properties should
933 do the right thing.
934 """
935 if propname == 'id':
936 return nodeid
938 # get the node's dict
939 d = self.db.getnode(self.classname, nodeid)
941 # check for one of the special props
942 if propname == 'creation':
943 if 'creation' in d:
944 return d['creation']
945 if not self.do_journal:
946 raise ValueError('Journalling is disabled for this class')
947 journal = self.db.getjournal(self.classname, nodeid)
948 if journal:
949 return journal[0][1]
950 else:
951 # on the strange chance that there's no journal
952 return date.Date()
953 if propname == 'activity':
954 if 'activity' in d:
955 return d['activity']
956 if not self.do_journal:
957 raise ValueError('Journalling is disabled for this class')
958 journal = self.db.getjournal(self.classname, nodeid)
959 if journal:
960 return self.db.getjournal(self.classname, nodeid)[-1][1]
961 else:
962 # on the strange chance that there's no journal
963 return date.Date()
964 if propname == 'creator':
965 if 'creator' in d:
966 return d['creator']
967 if not self.do_journal:
968 raise ValueError('Journalling is disabled for this class')
969 journal = self.db.getjournal(self.classname, nodeid)
970 if journal:
971 num_re = re.compile('^\d+$')
972 value = journal[0][2]
973 if num_re.match(value):
974 return value
975 else:
976 # old-style "username" journal tag
977 try:
978 return self.db.user.lookup(value)
979 except KeyError:
980 # user's been retired, return admin
981 return '1'
982 else:
983 return self.db.getuid()
984 if propname == 'actor':
985 if 'actor' in d:
986 return d['actor']
987 if not self.do_journal:
988 raise ValueError('Journalling is disabled for this class')
989 journal = self.db.getjournal(self.classname, nodeid)
990 if journal:
991 num_re = re.compile('^\d+$')
992 value = journal[-1][2]
993 if num_re.match(value):
994 return value
995 else:
996 # old-style "username" journal tag
997 try:
998 return self.db.user.lookup(value)
999 except KeyError:
1000 # user's been retired, return admin
1001 return '1'
1002 else:
1003 return self.db.getuid()
1005 # get the property (raises KeyErorr if invalid)
1006 prop = self.properties[propname]
1008 if propname not in d:
1009 if default is _marker:
1010 if isinstance(prop, hyperdb.Multilink):
1011 return []
1012 else:
1013 return None
1014 else:
1015 return default
1017 # return a dupe of the list so code doesn't get confused
1018 if isinstance(prop, hyperdb.Multilink):
1019 return d[propname][:]
1021 return d[propname]
1023 def set(self, nodeid, **propvalues):
1024 """Modify a property on an existing node of this class.
1026 'nodeid' must be the id of an existing node of this class or an
1027 IndexError is raised.
1029 Each key in 'propvalues' must be the name of a property of this
1030 class or a KeyError is raised.
1032 All values in 'propvalues' must be acceptable types for their
1033 corresponding properties or a TypeError is raised.
1035 If the value of the key property is set, it must not collide with
1036 other key strings or a ValueError is raised.
1038 If the value of a Link or Multilink property contains an invalid
1039 node id, a ValueError is raised.
1041 These operations trigger detectors and can be vetoed. Attempts
1042 to modify the "creation" or "activity" properties cause a KeyError.
1043 """
1044 if self.db.journaltag is None:
1045 raise hyperdb.DatabaseError(_('Database open read-only'))
1047 self.fireAuditors('set', nodeid, propvalues)
1048 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1049 for name, prop in self.getprops(protected=0).iteritems():
1050 if name in oldvalues:
1051 continue
1052 if isinstance(prop, hyperdb.Multilink):
1053 oldvalues[name] = []
1054 else:
1055 oldvalues[name] = None
1056 propvalues = self.set_inner(nodeid, **propvalues)
1057 self.fireReactors('set', nodeid, oldvalues)
1058 return propvalues
1060 def set_inner(self, nodeid, **propvalues):
1061 """ Called by set, in-between the audit and react calls.
1062 """
1063 if not propvalues:
1064 return propvalues
1066 if 'creation' in propvalues or 'activity' in propvalues:
1067 raise KeyError, '"creation" and "activity" are reserved'
1069 if 'id' in propvalues:
1070 raise KeyError, '"id" is reserved'
1072 if self.db.journaltag is None:
1073 raise hyperdb.DatabaseError(_('Database open read-only'))
1075 node = self.db.getnode(self.classname, nodeid)
1076 if self.db.RETIRED_FLAG in node:
1077 raise IndexError
1078 num_re = re.compile('^\d+$')
1080 # if the journal value is to be different, store it in here
1081 journalvalues = {}
1083 # list() propvalues 'cos it might be modified by the loop
1084 for propname, value in list(propvalues.items()):
1085 # check to make sure we're not duplicating an existing key
1086 if propname == self.key and node[propname] != value:
1087 try:
1088 self.lookup(value)
1089 except KeyError:
1090 pass
1091 else:
1092 raise ValueError('node with key "%s" exists'%value)
1094 # this will raise the KeyError if the property isn't valid
1095 # ... we don't use getprops() here because we only care about
1096 # the writeable properties.
1097 try:
1098 prop = self.properties[propname]
1099 except KeyError:
1100 raise KeyError('"%s" has no property named "%s"'%(
1101 self.classname, propname))
1103 # if the value's the same as the existing value, no sense in
1104 # doing anything
1105 current = node.get(propname, None)
1106 if value == current:
1107 del propvalues[propname]
1108 continue
1109 journalvalues[propname] = current
1111 # do stuff based on the prop type
1112 if isinstance(prop, hyperdb.Link):
1113 link_class = prop.classname
1114 # if it isn't a number, it's a key
1115 if value is not None and not isinstance(value, type('')):
1116 raise ValueError('property "%s" link value be a string'%(
1117 propname))
1118 if isinstance(value, type('')) and not num_re.match(value):
1119 try:
1120 value = self.db.classes[link_class].lookup(value)
1121 except (TypeError, KeyError):
1122 raise IndexError('new property "%s": %s not a %s'%(
1123 propname, value, prop.classname))
1125 if (value is not None and
1126 not self.db.getclass(link_class).hasnode(value)):
1127 raise IndexError('%s has no node %s'%(link_class,
1128 value))
1130 if self.do_journal and prop.do_journal:
1131 # register the unlink with the old linked node
1132 if propname in node and node[propname] is not None:
1133 self.db.addjournal(link_class, node[propname], 'unlink',
1134 (self.classname, nodeid, propname))
1136 # register the link with the newly linked node
1137 if value is not None:
1138 self.db.addjournal(link_class, value, 'link',
1139 (self.classname, nodeid, propname))
1141 elif isinstance(prop, hyperdb.Multilink):
1142 if value is None:
1143 value = []
1144 if not hasattr(value, '__iter__'):
1145 raise TypeError('new property "%s" not an iterable of'
1146 ' ids'%propname)
1147 link_class = self.properties[propname].classname
1148 l = []
1149 for entry in value:
1150 # if it isn't a number, it's a key
1151 if type(entry) != type(''):
1152 raise ValueError('new property "%s" link value '
1153 'must be a string'%propname)
1154 if not num_re.match(entry):
1155 try:
1156 entry = self.db.classes[link_class].lookup(entry)
1157 except (TypeError, KeyError):
1158 raise IndexError('new property "%s": %s not a %s'%(
1159 propname, entry,
1160 self.properties[propname].classname))
1161 l.append(entry)
1162 value = l
1163 propvalues[propname] = value
1165 # figure the journal entry for this property
1166 add = []
1167 remove = []
1169 # handle removals
1170 if propname in node:
1171 l = node[propname]
1172 else:
1173 l = []
1174 for id in l[:]:
1175 if id in value:
1176 continue
1177 # register the unlink with the old linked node
1178 if self.do_journal and self.properties[propname].do_journal:
1179 self.db.addjournal(link_class, id, 'unlink',
1180 (self.classname, nodeid, propname))
1181 l.remove(id)
1182 remove.append(id)
1184 # handle additions
1185 for id in value:
1186 if not self.db.getclass(link_class).hasnode(id):
1187 raise IndexError('%s has no node %s'%(link_class,
1188 id))
1189 if id in l:
1190 continue
1191 # register the link with the newly linked node
1192 if self.do_journal and self.properties[propname].do_journal:
1193 self.db.addjournal(link_class, id, 'link',
1194 (self.classname, nodeid, propname))
1195 l.append(id)
1196 add.append(id)
1198 # figure the journal entry
1199 l = []
1200 if add:
1201 l.append(('+', add))
1202 if remove:
1203 l.append(('-', remove))
1204 if l:
1205 journalvalues[propname] = tuple(l)
1207 elif isinstance(prop, hyperdb.String):
1208 if value is not None and type(value) != type('') and type(value) != type(u''):
1209 raise TypeError('new property "%s" not a '
1210 'string'%propname)
1211 if prop.indexme:
1212 self.db.indexer.add_text((self.classname, nodeid, propname),
1213 value)
1215 elif isinstance(prop, hyperdb.Password):
1216 if not isinstance(value, password.Password):
1217 raise TypeError('new property "%s" not a '
1218 'Password'%propname)
1219 propvalues[propname] = value
1221 elif value is not None and isinstance(prop, hyperdb.Date):
1222 if not isinstance(value, date.Date):
1223 raise TypeError('new property "%s" not a '
1224 'Date'%propname)
1225 propvalues[propname] = value
1227 elif value is not None and isinstance(prop, hyperdb.Interval):
1228 if not isinstance(value, date.Interval):
1229 raise TypeError('new property "%s" not an '
1230 'Interval'%propname)
1231 propvalues[propname] = value
1233 elif value is not None and isinstance(prop, hyperdb.Number):
1234 try:
1235 float(value)
1236 except ValueError:
1237 raise TypeError('new property "%s" not '
1238 'numeric'%propname)
1240 elif value is not None and isinstance(prop, hyperdb.Boolean):
1241 try:
1242 int(value)
1243 except ValueError:
1244 raise TypeError('new property "%s" not '
1245 'boolean'%propname)
1247 node[propname] = value
1249 # nothing to do?
1250 if not propvalues:
1251 return propvalues
1253 # update the activity time
1254 node['activity'] = date.Date()
1255 node['actor'] = self.db.getuid()
1257 # do the set, and journal it
1258 self.db.setnode(self.classname, nodeid, node)
1260 if self.do_journal:
1261 self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
1263 return propvalues
1265 def retire(self, nodeid):
1266 """Retire a node.
1268 The properties on the node remain available from the get() method,
1269 and the node's id is never reused.
1271 Retired nodes are not returned by the find(), list(), or lookup()
1272 methods, and other nodes may reuse the values of their key properties.
1274 These operations trigger detectors and can be vetoed. Attempts
1275 to modify the "creation" or "activity" properties cause a KeyError.
1276 """
1277 if self.db.journaltag is None:
1278 raise hyperdb.DatabaseError(_('Database open read-only'))
1280 self.fireAuditors('retire', nodeid, None)
1282 node = self.db.getnode(self.classname, nodeid)
1283 node[self.db.RETIRED_FLAG] = 1
1284 self.db.setnode(self.classname, nodeid, node)
1285 if self.do_journal:
1286 self.db.addjournal(self.classname, nodeid, 'retired', None)
1288 self.fireReactors('retire', nodeid, None)
1290 def restore(self, nodeid):
1291 """Restpre a retired node.
1293 Make node available for all operations like it was before retirement.
1294 """
1295 if self.db.journaltag is None:
1296 raise hyperdb.DatabaseError(_('Database open read-only'))
1298 node = self.db.getnode(self.classname, nodeid)
1299 # check if key property was overrided
1300 key = self.getkey()
1301 try:
1302 id = self.lookup(node[key])
1303 except KeyError:
1304 pass
1305 else:
1306 raise KeyError("Key property (%s) of retired node clashes "
1307 "with existing one (%s)" % (key, node[key]))
1308 # Now we can safely restore node
1309 self.fireAuditors('restore', nodeid, None)
1310 del node[self.db.RETIRED_FLAG]
1311 self.db.setnode(self.classname, nodeid, node)
1312 if self.do_journal:
1313 self.db.addjournal(self.classname, nodeid, 'restored', None)
1315 self.fireReactors('restore', nodeid, None)
1317 def is_retired(self, nodeid, cldb=None):
1318 """Return true if the node is retired.
1319 """
1320 node = self.db.getnode(self.classname, nodeid, cldb)
1321 if self.db.RETIRED_FLAG in node:
1322 return 1
1323 return 0
1325 def destroy(self, nodeid):
1326 """Destroy a node.
1328 WARNING: this method should never be used except in extremely rare
1329 situations where there could never be links to the node being
1330 deleted
1332 WARNING: use retire() instead
1334 WARNING: the properties of this node will not be available ever again
1336 WARNING: really, use retire() instead
1338 Well, I think that's enough warnings. This method exists mostly to
1339 support the session storage of the cgi interface.
1340 """
1341 if self.db.journaltag is None:
1342 raise hyperdb.DatabaseError(_('Database open read-only'))
1343 self.db.destroynode(self.classname, nodeid)
1345 def history(self, nodeid):
1346 """Retrieve the journal of edits on a particular node.
1348 'nodeid' must be the id of an existing node of this class or an
1349 IndexError is raised.
1351 The returned list contains tuples of the form
1353 (nodeid, date, tag, action, params)
1355 'date' is a Timestamp object specifying the time of the change and
1356 'tag' is the journaltag specified when the database was opened.
1357 """
1358 if not self.do_journal:
1359 raise ValueError('Journalling is disabled for this class')
1360 return self.db.getjournal(self.classname, nodeid)
1362 # Locating nodes:
1363 def hasnode(self, nodeid):
1364 """Determine if the given nodeid actually exists
1365 """
1366 return self.db.hasnode(self.classname, nodeid)
1368 def setkey(self, propname):
1369 """Select a String property of this class to be the key property.
1371 'propname' must be the name of a String property of this class or
1372 None, or a TypeError is raised. The values of the key property on
1373 all existing nodes must be unique or a ValueError is raised. If the
1374 property doesn't exist, KeyError is raised.
1375 """
1376 prop = self.getprops()[propname]
1377 if not isinstance(prop, hyperdb.String):
1378 raise TypeError('key properties must be String')
1379 self.key = propname
1381 def getkey(self):
1382 """Return the name of the key property for this class or None."""
1383 return self.key
1385 # TODO: set up a separate index db file for this? profile?
1386 def lookup(self, keyvalue):
1387 """Locate a particular node by its key property and return its id.
1389 If this class has no key property, a TypeError is raised. If the
1390 'keyvalue' matches one of the values for the key property among
1391 the nodes in this class, the matching node's id is returned;
1392 otherwise a KeyError is raised.
1393 """
1394 if not self.key:
1395 raise TypeError('No key property set for '
1396 'class %s'%self.classname)
1397 cldb = self.db.getclassdb(self.classname)
1398 try:
1399 for nodeid in self.getnodeids(cldb):
1400 node = self.db.getnode(self.classname, nodeid, cldb)
1401 if self.db.RETIRED_FLAG in node:
1402 continue
1403 if self.key not in node:
1404 continue
1405 if node[self.key] == keyvalue:
1406 return nodeid
1407 finally:
1408 cldb.close()
1409 raise KeyError('No key (%s) value "%s" for "%s"'%(self.key,
1410 keyvalue, self.classname))
1412 # change from spec - allows multiple props to match
1413 def find(self, **propspec):
1414 """Get the ids of nodes in this class which link to the given nodes.
1416 'propspec' consists of keyword args propname=nodeid or
1417 propname={nodeid:1, }
1418 'propname' must be the name of a property in this class, or a
1419 KeyError is raised. That property must be a Link or
1420 Multilink property, or a TypeError is raised.
1422 Any node in this class whose 'propname' property links to any of
1423 the nodeids will be returned. Examples::
1425 db.issue.find(messages='1')
1426 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1427 """
1428 for propname, itemids in propspec.iteritems():
1429 # check the prop is OK
1430 prop = self.properties[propname]
1431 if not isinstance(prop, hyperdb.Link) and not isinstance(prop, hyperdb.Multilink):
1432 raise TypeError("'%s' not a Link/Multilink "
1433 "property"%propname)
1435 # ok, now do the find
1436 cldb = self.db.getclassdb(self.classname)
1437 l = []
1438 try:
1439 for id in self.getnodeids(db=cldb):
1440 item = self.db.getnode(self.classname, id, db=cldb)
1441 if self.db.RETIRED_FLAG in item:
1442 continue
1443 for propname, itemids in propspec.iteritems():
1444 if type(itemids) is not type({}):
1445 itemids = {itemids:1}
1447 # special case if the item doesn't have this property
1448 if propname not in item:
1449 if None in itemids:
1450 l.append(id)
1451 break
1452 continue
1454 # grab the property definition and its value on this item
1455 prop = self.properties[propname]
1456 value = item[propname]
1457 if isinstance(prop, hyperdb.Link) and value in itemids:
1458 l.append(id)
1459 break
1460 elif isinstance(prop, hyperdb.Multilink):
1461 hit = 0
1462 for v in value:
1463 if v in itemids:
1464 l.append(id)
1465 hit = 1
1466 break
1467 if hit:
1468 break
1469 finally:
1470 cldb.close()
1471 return l
1473 def stringFind(self, **requirements):
1474 """Locate a particular node by matching a set of its String
1475 properties in a caseless search.
1477 If the property is not a String property, a TypeError is raised.
1479 The return is a list of the id of all nodes that match.
1480 """
1481 for propname in requirements:
1482 prop = self.properties[propname]
1483 if not isinstance(prop, hyperdb.String):
1484 raise TypeError("'%s' not a String property"%propname)
1485 requirements[propname] = requirements[propname].lower()
1486 l = []
1487 cldb = self.db.getclassdb(self.classname)
1488 try:
1489 for nodeid in self.getnodeids(cldb):
1490 node = self.db.getnode(self.classname, nodeid, cldb)
1491 if self.db.RETIRED_FLAG in node:
1492 continue
1493 for key, value in requirements.iteritems():
1494 if key not in node:
1495 break
1496 if node[key] is None or node[key].lower() != value:
1497 break
1498 else:
1499 l.append(nodeid)
1500 finally:
1501 cldb.close()
1502 return l
1504 def list(self):
1505 """ Return a list of the ids of the active nodes in this class.
1506 """
1507 l = []
1508 cn = self.classname
1509 cldb = self.db.getclassdb(cn)
1510 try:
1511 for nodeid in self.getnodeids(cldb):
1512 node = self.db.getnode(cn, nodeid, cldb)
1513 if self.db.RETIRED_FLAG in node:
1514 continue
1515 l.append(nodeid)
1516 finally:
1517 cldb.close()
1518 l.sort()
1519 return l
1521 def getnodeids(self, db=None, retired=None):
1522 """ Return a list of ALL nodeids
1524 Set retired=None to get all nodes. Otherwise it'll get all the
1525 retired or non-retired nodes, depending on the flag.
1526 """
1527 res = []
1529 # start off with the new nodes
1530 if self.classname in self.db.newnodes:
1531 res.extend(self.db.newnodes[self.classname])
1533 must_close = False
1534 if db is None:
1535 db = self.db.getclassdb(self.classname)
1536 must_close = True
1537 try:
1538 res.extend(db.keys())
1540 # remove the uncommitted, destroyed nodes
1541 if self.classname in self.db.destroyednodes:
1542 for nodeid in self.db.destroyednodes[self.classname]:
1543 if key_in(db, nodeid):
1544 res.remove(nodeid)
1546 # check retired flag
1547 if retired is False or retired is True:
1548 l = []
1549 for nodeid in res:
1550 node = self.db.getnode(self.classname, nodeid, db)
1551 is_ret = self.db.RETIRED_FLAG in node
1552 if retired == is_ret:
1553 l.append(nodeid)
1554 res = l
1555 finally:
1556 if must_close:
1557 db.close()
1558 return res
1560 def _filter(self, search_matches, filterspec, proptree,
1561 num_re = re.compile('^\d+$')):
1562 """Return a list of the ids of the active nodes in this class that
1563 match the 'filter' spec, sorted by the group spec and then the
1564 sort spec.
1566 "filterspec" is {propname: value(s)}
1568 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1569 and prop is a prop name or None
1571 "search_matches" is a sequence type or None
1573 The filter must match all properties specificed. If the property
1574 value to match is a list:
1576 1. String properties must match all elements in the list, and
1577 2. Other properties must match any of the elements in the list.
1578 """
1579 if __debug__:
1580 start_t = time.time()
1582 cn = self.classname
1584 # optimise filterspec
1585 l = []
1586 props = self.getprops()
1587 LINK = 'spec:link'
1588 MULTILINK = 'spec:multilink'
1589 STRING = 'spec:string'
1590 DATE = 'spec:date'
1591 INTERVAL = 'spec:interval'
1592 OTHER = 'spec:other'
1594 for k, v in filterspec.iteritems():
1595 propclass = props[k]
1596 if isinstance(propclass, hyperdb.Link):
1597 if type(v) is not type([]):
1598 v = [v]
1599 u = []
1600 for entry in v:
1601 # the value -1 is a special "not set" sentinel
1602 if entry == '-1':
1603 entry = None
1604 u.append(entry)
1605 l.append((LINK, k, u))
1606 elif isinstance(propclass, hyperdb.Multilink):
1607 # the value -1 is a special "not set" sentinel
1608 if v in ('-1', ['-1']):
1609 v = []
1610 elif type(v) is not type([]):
1611 v = [v]
1612 l.append((MULTILINK, k, v))
1613 elif isinstance(propclass, hyperdb.String) and k != 'id':
1614 if type(v) is not type([]):
1615 v = [v]
1616 for v in v:
1617 # simple glob searching
1618 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1619 v = v.replace('?', '.')
1620 v = v.replace('*', '.*?')
1621 l.append((STRING, k, re.compile(v, re.I)))
1622 elif isinstance(propclass, hyperdb.Date):
1623 try:
1624 date_rng = propclass.range_from_raw(v, self.db)
1625 l.append((DATE, k, date_rng))
1626 except ValueError:
1627 # If range creation fails - ignore that search parameter
1628 pass
1629 elif isinstance(propclass, hyperdb.Interval):
1630 try:
1631 intv_rng = date.Range(v, date.Interval)
1632 l.append((INTERVAL, k, intv_rng))
1633 except ValueError:
1634 # If range creation fails - ignore that search parameter
1635 pass
1637 elif isinstance(propclass, hyperdb.Boolean):
1638 if type(v) == type(""):
1639 v = v.split(',')
1640 if type(v) != type([]):
1641 v = [v]
1642 bv = []
1643 for val in v:
1644 if type(val) is type(''):
1645 bv.append(propclass.from_raw (val))
1646 else:
1647 bv.append(val)
1648 l.append((OTHER, k, bv))
1650 elif k == 'id':
1651 if type(v) != type([]):
1652 v = v.split(',')
1653 l.append((OTHER, k, [str(int(val)) for val in v]))
1655 elif isinstance(propclass, hyperdb.Number):
1656 if type(v) != type([]):
1657 try :
1658 v = v.split(',')
1659 except AttributeError :
1660 v = [v]
1661 l.append((OTHER, k, [float(val) for val in v]))
1663 filterspec = l
1665 # now, find all the nodes that are active and pass filtering
1666 matches = []
1667 cldb = self.db.getclassdb(cn)
1668 t = 0
1669 try:
1670 # TODO: only full-scan once (use items())
1671 for nodeid in self.getnodeids(cldb):
1672 node = self.db.getnode(cn, nodeid, cldb)
1673 if self.db.RETIRED_FLAG in node:
1674 continue
1675 # apply filter
1676 for t, k, v in filterspec:
1677 # handle the id prop
1678 if k == 'id':
1679 if nodeid not in v:
1680 break
1681 continue
1683 # get the node value
1684 nv = node.get(k, None)
1686 match = 0
1688 # now apply the property filter
1689 if t == LINK:
1690 # link - if this node's property doesn't appear in the
1691 # filterspec's nodeid list, skip it
1692 match = nv in v
1693 elif t == MULTILINK:
1694 # multilink - if any of the nodeids required by the
1695 # filterspec aren't in this node's property, then skip
1696 # it
1697 nv = node.get(k, [])
1699 # check for matching the absence of multilink values
1700 if not v:
1701 match = not nv
1702 else:
1703 # othewise, make sure this node has each of the
1704 # required values
1705 for want in v:
1706 if want in nv:
1707 match = 1
1708 break
1709 elif t == STRING:
1710 if nv is None:
1711 nv = ''
1712 # RE search
1713 match = v.search(nv)
1714 elif t == DATE or t == INTERVAL:
1715 if nv is None:
1716 match = v is None
1717 else:
1718 if v.to_value:
1719 if v.from_value <= nv and v.to_value >= nv:
1720 match = 1
1721 else:
1722 if v.from_value <= nv:
1723 match = 1
1724 elif t == OTHER:
1725 # straight value comparison for the other types
1726 match = nv in v
1727 if not match:
1728 break
1729 else:
1730 matches.append([nodeid, node])
1732 # filter based on full text search
1733 if search_matches is not None:
1734 k = []
1735 for v in matches:
1736 if v[0] in search_matches:
1737 k.append(v)
1738 matches = k
1740 # add sorting information to the proptree
1741 JPROPS = {'actor':1, 'activity':1, 'creator':1, 'creation':1}
1742 children = []
1743 if proptree:
1744 children = proptree.sortable_children()
1745 for pt in children:
1746 dir = pt.sort_direction
1747 prop = pt.name
1748 assert (dir and prop)
1749 propclass = props[prop]
1750 pt.sort_ids = []
1751 is_pointer = isinstance(propclass,(hyperdb.Link,
1752 hyperdb.Multilink))
1753 if not is_pointer:
1754 pt.sort_result = []
1755 try:
1756 # cache the opened link class db, if needed.
1757 lcldb = None
1758 # cache the linked class items too
1759 lcache = {}
1761 for entry in matches:
1762 itemid = entry[-2]
1763 item = entry[-1]
1764 # handle the properties that might be "faked"
1765 # also, handle possible missing properties
1766 try:
1767 v = item[prop]
1768 except KeyError:
1769 if prop in JPROPS:
1770 # force lookup of the special journal prop
1771 v = self.get(itemid, prop)
1772 else:
1773 # the node doesn't have a value for this
1774 # property
1775 v = None
1776 if isinstance(propclass, hyperdb.Multilink):
1777 v = []
1778 if prop == 'id':
1779 v = int (itemid)
1780 pt.sort_ids.append(v)
1781 if not is_pointer:
1782 pt.sort_result.append(v)
1783 continue
1785 # missing (None) values are always sorted first
1786 if v is None:
1787 pt.sort_ids.append(v)
1788 if not is_pointer:
1789 pt.sort_result.append(v)
1790 continue
1792 if isinstance(propclass, hyperdb.Link):
1793 lcn = propclass.classname
1794 link = self.db.classes[lcn]
1795 key = link.orderprop()
1796 child = pt.propdict[key]
1797 if key!='id':
1798 if v not in lcache:
1799 # open the link class db if it's not already
1800 if lcldb is None:
1801 lcldb = self.db.getclassdb(lcn)
1802 lcache[v] = self.db.getnode(lcn, v, lcldb)
1803 r = lcache[v][key]
1804 child.propdict[key].sort_ids.append(r)
1805 else:
1806 child.propdict[key].sort_ids.append(v)
1807 pt.sort_ids.append(v)
1808 if not is_pointer:
1809 r = propclass.sort_repr(pt.parent.cls, v, pt.name)
1810 pt.sort_result.append(r)
1811 finally:
1812 # if we opened the link class db, close it now
1813 if lcldb is not None:
1814 lcldb.close()
1815 del lcache
1816 finally:
1817 cldb.close()
1819 # pull the id out of the individual entries
1820 matches = [entry[-2] for entry in matches]
1821 if __debug__:
1822 self.db.stats['filtering'] += (time.time() - start_t)
1823 return matches
1825 def count(self):
1826 """Get the number of nodes in this class.
1828 If the returned integer is 'numnodes', the ids of all the nodes
1829 in this class run from 1 to numnodes, and numnodes+1 will be the
1830 id of the next node to be created in this class.
1831 """
1832 return self.db.countnodes(self.classname)
1834 # Manipulating properties:
1836 def getprops(self, protected=1):
1837 """Return a dictionary mapping property names to property objects.
1838 If the "protected" flag is true, we include protected properties -
1839 those which may not be modified.
1841 In addition to the actual properties on the node, these
1842 methods provide the "creation" and "activity" properties. If the
1843 "protected" flag is true, we include protected properties - those
1844 which may not be modified.
1845 """
1846 d = self.properties.copy()
1847 if protected:
1848 d['id'] = hyperdb.String()
1849 d['creation'] = hyperdb.Date()
1850 d['activity'] = hyperdb.Date()
1851 d['creator'] = hyperdb.Link('user')
1852 d['actor'] = hyperdb.Link('user')
1853 return d
1855 def addprop(self, **properties):
1856 """Add properties to this class.
1858 The keyword arguments in 'properties' must map names to property
1859 objects, or a TypeError is raised. None of the keys in 'properties'
1860 may collide with the names of existing properties, or a ValueError
1861 is raised before any properties have been added.
1862 """
1863 for key in properties:
1864 if key in self.properties:
1865 raise ValueError(key)
1866 self.properties.update(properties)
1868 def index(self, nodeid):
1869 """ Add (or refresh) the node to search indexes """
1870 # find all the String properties that have indexme
1871 for prop, propclass in self.getprops().iteritems():
1872 if isinstance(propclass, hyperdb.String) and propclass.indexme:
1873 # index them under (classname, nodeid, property)
1874 try:
1875 value = str(self.get(nodeid, prop))
1876 except IndexError:
1877 # node has been destroyed
1878 continue
1879 self.db.indexer.add_text((self.classname, nodeid, prop), value)
1881 #
1882 # import / export support
1883 #
1884 def export_list(self, propnames, nodeid):
1885 """ Export a node - generate a list of CSV-able data in the order
1886 specified by propnames for the given node.
1887 """
1888 properties = self.getprops()
1889 l = []
1890 for prop in propnames:
1891 proptype = properties[prop]
1892 value = self.get(nodeid, prop)
1893 # "marshal" data where needed
1894 if value is None:
1895 pass
1896 elif isinstance(proptype, hyperdb.Date):
1897 value = value.get_tuple()
1898 elif isinstance(proptype, hyperdb.Interval):
1899 value = value.get_tuple()
1900 elif isinstance(proptype, hyperdb.Password):
1901 value = str(value)
1902 l.append(repr(value))
1904 # append retired flag
1905 l.append(repr(self.is_retired(nodeid)))
1907 return l
1909 def import_list(self, propnames, proplist):
1910 """ Import a node - all information including "id" is present and
1911 should not be sanity checked. Triggers are not triggered. The
1912 journal should be initialised using the "creator" and "created"
1913 information.
1915 Return the nodeid of the node imported.
1916 """
1917 if self.db.journaltag is None:
1918 raise hyperdb.DatabaseError(_('Database open read-only'))
1919 properties = self.getprops()
1921 # make the new node's property map
1922 d = {}
1923 newid = None
1924 for i in range(len(propnames)):
1925 # Figure the property for this column
1926 propname = propnames[i]
1928 # Use eval to reverse the repr() used to output the CSV
1929 value = eval(proplist[i])
1931 # "unmarshal" where necessary
1932 if propname == 'id':
1933 newid = value
1934 continue
1935 elif propname == 'is retired':
1936 # is the item retired?
1937 if int(value):
1938 d[self.db.RETIRED_FLAG] = 1
1939 continue
1940 elif value is None:
1941 d[propname] = None
1942 continue
1944 prop = properties[propname]
1945 if isinstance(prop, hyperdb.Date):
1946 value = date.Date(value)
1947 elif isinstance(prop, hyperdb.Interval):
1948 value = date.Interval(value)
1949 elif isinstance(prop, hyperdb.Password):
1950 pwd = password.Password()
1951 pwd.unpack(value)
1952 value = pwd
1953 d[propname] = value
1955 # get a new id if necessary
1956 if newid is None:
1957 newid = self.db.newid(self.classname)
1959 # add the node and journal
1960 self.db.addnode(self.classname, newid, d)
1961 return newid
1963 def export_journals(self):
1964 """Export a class's journal - generate a list of lists of
1965 CSV-able data:
1967 nodeid, date, user, action, params
1969 No heading here - the columns are fixed.
1970 """
1971 properties = self.getprops()
1972 r = []
1973 for nodeid in self.getnodeids():
1974 for nodeid, date, user, action, params in self.history(nodeid):
1975 date = date.get_tuple()
1976 if action == 'set':
1977 export_data = {}
1978 for propname, value in params.iteritems():
1979 if propname not in properties:
1980 # property no longer in the schema
1981 continue
1983 prop = properties[propname]
1984 # make sure the params are eval()'able
1985 if value is None:
1986 pass
1987 elif isinstance(prop, hyperdb.Date):
1988 # this is a hack - some dates are stored as strings
1989 if not isinstance(value, type('')):
1990 value = value.get_tuple()
1991 elif isinstance(prop, hyperdb.Interval):
1992 # hack too - some intervals are stored as strings
1993 if not isinstance(value, type('')):
1994 value = value.get_tuple()
1995 elif isinstance(prop, hyperdb.Password):
1996 value = str(value)
1997 export_data[propname] = value
1998 params = export_data
1999 r.append([repr(nodeid), repr(date), repr(user),
2000 repr(action), repr(params)])
2001 return r
2003 class FileClass(hyperdb.FileClass, Class):
2004 """This class defines a large chunk of data. To support this, it has a
2005 mandatory String property "content" which is typically saved off
2006 externally to the hyperdb.
2008 The default MIME type of this data is defined by the
2009 "default_mime_type" class attribute, which may be overridden by each
2010 node if the class defines a "type" String property.
2011 """
2012 def __init__(self, db, classname, **properties):
2013 """The newly-created class automatically includes the "content"
2014 and "type" properties.
2015 """
2016 if 'content' not in properties:
2017 properties['content'] = hyperdb.String(indexme='yes')
2018 if 'type' not in properties:
2019 properties['type'] = hyperdb.String()
2020 Class.__init__(self, db, classname, **properties)
2022 def create(self, **propvalues):
2023 """ Snarf the "content" propvalue and store in a file
2024 """
2025 # we need to fire the auditors now, or the content property won't
2026 # be in propvalues for the auditors to play with
2027 self.fireAuditors('create', None, propvalues)
2029 # now remove the content property so it's not stored in the db
2030 content = propvalues['content']
2031 del propvalues['content']
2033 # make sure we have a MIME type
2034 mime_type = propvalues.get('type', self.default_mime_type)
2036 # do the database create
2037 newid = self.create_inner(**propvalues)
2039 # store off the content as a file
2040 self.db.storefile(self.classname, newid, None, content)
2042 # fire reactors
2043 self.fireReactors('create', newid, None)
2045 return newid
2047 def get(self, nodeid, propname, default=_marker, cache=1):
2048 """ Trap the content propname and get it from the file
2050 'cache' exists for backwards compatibility, and is not used.
2051 """
2052 poss_msg = 'Possibly an access right configuration problem.'
2053 if propname == 'content':
2054 try:
2055 return self.db.getfile(self.classname, nodeid, None)
2056 except IOError, strerror:
2057 # XXX by catching this we don't see an error in the log.
2058 return 'ERROR reading file: %s%s\n%s\n%s'%(
2059 self.classname, nodeid, poss_msg, strerror)
2060 if default is not _marker:
2061 return Class.get(self, nodeid, propname, default)
2062 else:
2063 return Class.get(self, nodeid, propname)
2065 def set(self, itemid, **propvalues):
2066 """ Snarf the "content" propvalue and update it in a file
2067 """
2068 self.fireAuditors('set', itemid, propvalues)
2070 # create the oldvalues dict - fill in any missing values
2071 oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid))
2072 for name, prop in self.getprops(protected=0).iteritems():
2073 if name in oldvalues:
2074 continue
2075 if isinstance(prop, hyperdb.Multilink):
2076 oldvalues[name] = []
2077 else:
2078 oldvalues[name] = None
2080 # now remove the content property so it's not stored in the db
2081 content = None
2082 if 'content' in propvalues:
2083 content = propvalues['content']
2084 del propvalues['content']
2086 # do the database update
2087 propvalues = self.set_inner(itemid, **propvalues)
2089 # do content?
2090 if content:
2091 # store and possibly index
2092 self.db.storefile(self.classname, itemid, None, content)
2093 if self.properties['content'].indexme:
2094 mime_type = self.get(itemid, 'type', self.default_mime_type)
2095 self.db.indexer.add_text((self.classname, itemid, 'content'),
2096 content, mime_type)
2097 propvalues['content'] = content
2099 # fire reactors
2100 self.fireReactors('set', itemid, oldvalues)
2101 return propvalues
2103 def index(self, nodeid):
2104 """ Add (or refresh) the node to search indexes.
2106 Use the content-type property for the content property.
2107 """
2108 # find all the String properties that have indexme
2109 for prop, propclass in self.getprops().iteritems():
2110 if prop == 'content' and propclass.indexme:
2111 mime_type = self.get(nodeid, 'type', self.default_mime_type)
2112 self.db.indexer.add_text((self.classname, nodeid, 'content'),
2113 str(self.get(nodeid, 'content')), mime_type)
2114 elif isinstance(propclass, hyperdb.String) and propclass.indexme:
2115 # index them under (classname, nodeid, property)
2116 try:
2117 value = str(self.get(nodeid, prop))
2118 except IndexError:
2119 # node has been destroyed
2120 continue
2121 self.db.indexer.add_text((self.classname, nodeid, prop), value)
2123 # deviation from spec - was called ItemClass
2124 class IssueClass(Class, roundupdb.IssueClass):
2125 # Overridden methods:
2126 def __init__(self, db, classname, **properties):
2127 """The newly-created class automatically includes the "messages",
2128 "files", "nosy", and "superseder" properties. If the 'properties'
2129 dictionary attempts to specify any of these properties or a
2130 "creation" or "activity" property, a ValueError is raised.
2131 """
2132 if 'title' not in properties:
2133 properties['title'] = hyperdb.String(indexme='yes')
2134 if 'messages' not in properties:
2135 properties['messages'] = hyperdb.Multilink("msg")
2136 if 'files' not in properties:
2137 properties['files'] = hyperdb.Multilink("file")
2138 if 'nosy' not in properties:
2139 # note: journalling is turned off as it really just wastes
2140 # space. this behaviour may be overridden in an instance
2141 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2142 if 'superseder' not in properties:
2143 properties['superseder'] = hyperdb.Multilink(classname)
2144 Class.__init__(self, db, classname, **properties)
2146 # vim: set et sts=4 sw=4 :