77f16096dad48aa323ef4b496cc7052f7ffc20c2
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 """This module defines a backend that saves the hyperdatabase in a
19 database chosen by anydbm. It is guaranteed to always be available in python
20 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
21 serious bugs, and is not available)
22 """
23 __docformat__ = 'restructuredtext'
25 import os, marshal, re, weakref, string, copy, time, shutil, logging
27 from roundup.anypy.dbm_ import anydbm, whichdb, key_in
29 from roundup import hyperdb, date, password, roundupdb, security, support
30 from roundup.support import reversed
31 from roundup.backends import locking
32 from roundup.i18n import _
34 from roundup.backends.blobfiles import FileStorage
35 from roundup.backends.sessions_dbm import Sessions, OneTimeKeys
37 try:
38 from roundup.backends.indexer_xapian import Indexer
39 except ImportError:
40 from roundup.backends.indexer_dbm import Indexer
42 def db_exists(config):
43 # check for the user db
44 for db in 'nodes.user nodes.user.db'.split():
45 if os.path.exists(os.path.join(config.DATABASE, db)):
46 return 1
47 return 0
49 def db_nuke(config):
50 shutil.rmtree(config.DATABASE)
52 class Binary:
54 def __init__(self, x, y):
55 self.x = x
56 self.y = y
58 def visit(self, visitor):
59 self.x.visit(visitor)
60 self.y.visit(visitor)
62 class Unary:
64 def __init__(self, x):
65 self.x = x
67 def generate(self, atom):
68 return atom(self)
70 def visit(self, visitor):
71 self.x.visit(visitor)
73 class Equals(Unary):
75 def evaluate(self, v):
76 return self.x in v
78 def visit(self, visitor):
79 visitor(self)
81 class Not(Unary):
83 def evaluate(self, v):
84 return not self.x.evaluate(v)
86 def generate(self, atom):
87 return "NOT(%s)" % self.x.generate(atom)
89 class Or(Binary):
91 def evaluate(self, v):
92 return self.x.evaluate(v) or self.y.evaluate(v)
94 def generate(self, atom):
95 return "(%s)OR(%s)" % (
96 self.x.generate(atom),
97 self.y.generate(atom))
99 class And(Binary):
101 def evaluate(self, v):
102 return self.x.evaluate(v) and self.y.evaluate(v)
104 def generate(self, atom):
105 return "(%s)AND(%s)" % (
106 self.x.generate(atom),
107 self.y.generate(atom))
109 def compile_expression(opcodes):
111 stack = []
112 push, pop = stack.append, stack.pop
113 for opcode in opcodes:
114 if opcode == -2: push(Not(pop()))
115 elif opcode == -3: push(And(pop(), pop()))
116 elif opcode == -4: push(Or(pop(), pop()))
117 else: push(Equals(opcode))
119 return pop()
121 class Expression:
123 def __init__(self, v):
124 try:
125 opcodes = [int(x) for x in v]
126 if min(opcodes) >= -1: raise ValueError()
128 compiled = compile_expression(opcodes)
129 self.evaluate = lambda x: compiled.evaluate([int(y) for y in x])
130 except:
131 self.evaluate = lambda x: bool(set(x) & set(v))
133 #
134 # Now the database
135 #
136 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
137 """A database for storing records containing flexible data types.
139 Transaction stuff TODO:
141 - check the timestamp of the class file and nuke the cache if it's
142 modified. Do some sort of conflict checking on the dirty stuff.
143 - perhaps detect write collisions (related to above)?
144 """
145 def __init__(self, config, journaltag=None):
146 """Open a hyperdatabase given a specifier to some storage.
148 The 'storagelocator' is obtained from config.DATABASE.
149 The meaning of 'storagelocator' depends on the particular
150 implementation of the hyperdatabase. It could be a file name,
151 a directory path, a socket descriptor for a connection to a
152 database over the network, etc.
154 The 'journaltag' is a token that will be attached to the journal
155 entries for any edits done on the database. If 'journaltag' is
156 None, the database is opened in read-only mode: the Class.create(),
157 Class.set(), Class.retire(), and Class.restore() methods are
158 disabled.
159 """
160 FileStorage.__init__(self, config.UMASK)
161 self.config, self.journaltag = config, journaltag
162 self.dir = config.DATABASE
163 self.classes = {}
164 self.cache = {} # cache of nodes loaded or created
165 self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0,
166 'filtering': 0}
167 self.dirtynodes = {} # keep track of the dirty nodes by class
168 self.newnodes = {} # keep track of the new nodes by class
169 self.destroyednodes = {}# keep track of the destroyed nodes by class
170 self.transactions = []
171 self.indexer = Indexer(self)
172 self.security = security.Security(self)
173 os.umask(config.UMASK)
175 # lock it
176 lockfilenm = os.path.join(self.dir, 'lock')
177 self.lockfile = locking.acquire_lock(lockfilenm)
178 self.lockfile.write(str(os.getpid()))
179 self.lockfile.flush()
181 def post_init(self):
182 """Called once the schema initialisation has finished.
183 """
184 # reindex the db if necessary
185 if self.indexer.should_reindex():
186 self.reindex()
188 def refresh_database(self):
189 """Rebuild the database
190 """
191 self.reindex()
193 def getSessionManager(self):
194 return Sessions(self)
196 def getOTKManager(self):
197 return OneTimeKeys(self)
199 def reindex(self, classname=None, show_progress=False):
200 if classname:
201 classes = [self.getclass(classname)]
202 else:
203 classes = self.classes.values()
204 for klass in classes:
205 if show_progress:
206 for nodeid in support.Progress('Reindex %s'%klass.classname,
207 klass.list()):
208 klass.index(nodeid)
209 else:
210 for nodeid in klass.list():
211 klass.index(nodeid)
212 self.indexer.save_index()
214 def __repr__(self):
215 return '<back_anydbm instance at %x>'%id(self)
217 #
218 # Classes
219 #
220 def __getattr__(self, classname):
221 """A convenient way of calling self.getclass(classname)."""
222 if classname in self.classes:
223 return self.classes[classname]
224 raise AttributeError, classname
226 def addclass(self, cl):
227 cn = cl.classname
228 if cn in self.classes:
229 raise ValueError, cn
230 self.classes[cn] = cl
232 # add default Edit and View permissions
233 self.security.addPermission(name="Create", klass=cn,
234 description="User is allowed to create "+cn)
235 self.security.addPermission(name="Edit", klass=cn,
236 description="User is allowed to edit "+cn)
237 self.security.addPermission(name="View", klass=cn,
238 description="User is allowed to access "+cn)
239 self.security.addPermission(name="Retire", klass=cn,
240 description="User is allowed to retire "+cn)
242 def getclasses(self):
243 """Return a list of the names of all existing classes."""
244 l = self.classes.keys()
245 l.sort()
246 return l
248 def getclass(self, classname):
249 """Get the Class object representing a particular class.
251 If 'classname' is not a valid class name, a KeyError is raised.
252 """
253 try:
254 return self.classes[classname]
255 except KeyError:
256 raise KeyError('There is no class called "%s"'%classname)
258 #
259 # Class DBs
260 #
261 def clear(self):
262 """Delete all database contents
263 """
264 logging.getLogger('roundup.hyperdb').info('clear')
265 for cn in self.classes:
266 for dummy in 'nodes', 'journals':
267 path = os.path.join(self.dir, 'journals.%s'%cn)
268 if os.path.exists(path):
269 os.remove(path)
270 elif os.path.exists(path+'.db'): # dbm appends .db
271 os.remove(path+'.db')
272 # reset id sequences
273 path = os.path.join(os.getcwd(), self.dir, '_ids')
274 if os.path.exists(path):
275 os.remove(path)
276 elif os.path.exists(path+'.db'): # dbm appends .db
277 os.remove(path+'.db')
279 def getclassdb(self, classname, mode='r'):
280 """ grab a connection to the class db that will be used for
281 multiple actions
282 """
283 return self.opendb('nodes.%s'%classname, mode)
285 def determine_db_type(self, path):
286 """ determine which DB wrote the class file
287 """
288 db_type = ''
289 if os.path.exists(path):
290 db_type = whichdb(path)
291 if not db_type:
292 raise hyperdb.DatabaseError(_("Couldn't identify database type"))
293 elif os.path.exists(path+'.db'):
294 # if the path ends in '.db', it's a dbm database, whether
295 # anydbm says it's dbhash or not!
296 db_type = 'dbm'
297 return db_type
299 def opendb(self, name, mode):
300 """Low-level database opener that gets around anydbm/dbm
301 eccentricities.
302 """
303 # figure the class db type
304 path = os.path.join(os.getcwd(), self.dir, name)
305 db_type = self.determine_db_type(path)
307 # new database? let anydbm pick the best dbm
308 # in Python 3+ the "dbm" ("anydbm" to us) module already uses the
309 # whichdb() function to do this
310 if not db_type or hasattr(anydbm, 'whichdb'):
311 if __debug__:
312 logging.getLogger('roundup.hyperdb').debug(
313 "opendb anydbm.open(%r, 'c')"%path)
314 return anydbm.open(path, 'c')
316 # in Python <3 it anydbm was a little dumb so manually open the
317 # database with the correct module
318 try:
319 dbm = __import__(db_type)
320 except ImportError:
321 raise hyperdb.DatabaseError(_("Couldn't open database - the "
322 "required module '%s' is not available")%db_type)
323 if __debug__:
324 logging.getLogger('roundup.hyperdb').debug(
325 "opendb %r.open(%r, %r)"%(db_type, path, mode))
326 return dbm.open(path, mode)
328 #
329 # Node IDs
330 #
331 def newid(self, classname):
332 """ Generate a new id for the given class
333 """
334 # open the ids DB - create if if doesn't exist
335 db = self.opendb('_ids', 'c')
336 if key_in(db, classname):
337 newid = db[classname] = str(int(db[classname]) + 1)
338 else:
339 # the count() bit is transitional - older dbs won't start at 1
340 newid = str(self.getclass(classname).count()+1)
341 db[classname] = newid
342 db.close()
343 return newid
345 def setid(self, classname, setid):
346 """ Set the id counter: used during import of database
347 """
348 # open the ids DB - create if if doesn't exist
349 db = self.opendb('_ids', 'c')
350 db[classname] = str(setid)
351 db.close()
353 #
354 # Nodes
355 #
356 def addnode(self, classname, nodeid, node):
357 """ add the specified node to its class's db
358 """
359 # we'll be supplied these props if we're doing an import
360 if 'creator' not in node:
361 # add in the "calculated" properties (dupe so we don't affect
362 # calling code's node assumptions)
363 node = node.copy()
364 node['creator'] = self.getuid()
365 node['actor'] = self.getuid()
366 node['creation'] = node['activity'] = date.Date()
368 self.newnodes.setdefault(classname, {})[nodeid] = 1
369 self.cache.setdefault(classname, {})[nodeid] = node
370 self.savenode(classname, nodeid, node)
372 def setnode(self, classname, nodeid, node):
373 """ change the specified node
374 """
375 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
377 # can't set without having already loaded the node
378 self.cache[classname][nodeid] = node
379 self.savenode(classname, nodeid, node)
381 def savenode(self, classname, nodeid, node):
382 """ perform the saving of data specified by the set/addnode
383 """
384 if __debug__:
385 logging.getLogger('roundup.hyperdb').debug(
386 'save %s%s %r'%(classname, nodeid, node))
387 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
389 def getnode(self, classname, nodeid, db=None, cache=1):
390 """ get a node from the database
392 Note the "cache" parameter is not used, and exists purely for
393 backward compatibility!
394 """
395 # try the cache
396 cache_dict = self.cache.setdefault(classname, {})
397 if nodeid in cache_dict:
398 if __debug__:
399 logging.getLogger('roundup.hyperdb').debug(
400 'get %s%s cached'%(classname, nodeid))
401 self.stats['cache_hits'] += 1
402 return cache_dict[nodeid]
404 if __debug__:
405 self.stats['cache_misses'] += 1
406 start_t = time.time()
407 logging.getLogger('roundup.hyperdb').debug(
408 'get %s%s'%(classname, nodeid))
410 # get from the database and save in the cache
411 if db is None:
412 db = self.getclassdb(classname)
413 if not key_in(db, nodeid):
414 raise IndexError("no such %s %s"%(classname, nodeid))
416 # check the uncommitted, destroyed nodes
417 if (classname in self.destroyednodes and
418 nodeid in self.destroyednodes[classname]):
419 raise IndexError("no such %s %s"%(classname, nodeid))
421 # decode
422 res = marshal.loads(db[nodeid])
424 # reverse the serialisation
425 res = self.unserialise(classname, res)
427 # store off in the cache dict
428 if cache:
429 cache_dict[nodeid] = res
431 if __debug__:
432 self.stats['get_items'] += (time.time() - start_t)
434 return res
436 def destroynode(self, classname, nodeid):
437 """Remove a node from the database. Called exclusively by the
438 destroy() method on Class.
439 """
440 logging.getLogger('roundup.hyperdb').info(
441 'destroy %s%s'%(classname, nodeid))
443 # remove from cache and newnodes if it's there
444 if (classname in self.cache and nodeid in self.cache[classname]):
445 del self.cache[classname][nodeid]
446 if (classname in self.newnodes and nodeid in self.newnodes[classname]):
447 del self.newnodes[classname][nodeid]
449 # see if there's any obvious commit actions that we should get rid of
450 for entry in self.transactions[:]:
451 if entry[1][:2] == (classname, nodeid):
452 self.transactions.remove(entry)
454 # add to the destroyednodes map
455 self.destroyednodes.setdefault(classname, {})[nodeid] = 1
457 # add the destroy commit action
458 self.transactions.append((self.doDestroyNode, (classname, nodeid)))
459 self.transactions.append((FileStorage.destroy, (self, classname, nodeid)))
461 def serialise(self, classname, node):
462 """Copy the node contents, converting non-marshallable data into
463 marshallable data.
464 """
465 properties = self.getclass(classname).getprops()
466 d = {}
467 for k, v in node.iteritems():
468 if k == self.RETIRED_FLAG:
469 d[k] = v
470 continue
472 # if the property doesn't exist then we really don't care
473 if k not in properties:
474 continue
476 # get the property spec
477 prop = properties[k]
479 if isinstance(prop, hyperdb.Password) and v is not None:
480 d[k] = str(v)
481 elif isinstance(prop, hyperdb.Date) and v is not None:
482 d[k] = v.serialise()
483 elif isinstance(prop, hyperdb.Interval) and v is not None:
484 d[k] = v.serialise()
485 else:
486 d[k] = v
487 return d
489 def unserialise(self, classname, node):
490 """Decode the marshalled node data
491 """
492 properties = self.getclass(classname).getprops()
493 d = {}
494 for k, v in node.iteritems():
495 # if the property doesn't exist, or is the "retired" flag then
496 # it won't be in the properties dict
497 if k not in properties:
498 d[k] = v
499 continue
501 # get the property spec
502 prop = properties[k]
504 if isinstance(prop, hyperdb.Date) and v is not None:
505 d[k] = date.Date(v)
506 elif isinstance(prop, hyperdb.Interval) and v is not None:
507 d[k] = date.Interval(v)
508 elif isinstance(prop, hyperdb.Password) and v is not None:
509 d[k] = password.Password(encrypted=v)
510 else:
511 d[k] = v
512 return d
514 def hasnode(self, classname, nodeid, db=None):
515 """ determine if the database has a given node
516 """
517 # try the cache
518 cache = self.cache.setdefault(classname, {})
519 if nodeid in cache:
520 return 1
522 # not in the cache - check the database
523 if db is None:
524 db = self.getclassdb(classname)
525 return key_in(db, nodeid)
527 def countnodes(self, classname, db=None):
528 count = 0
530 # include the uncommitted nodes
531 if classname in self.newnodes:
532 count += len(self.newnodes[classname])
533 if classname in self.destroyednodes:
534 count -= len(self.destroyednodes[classname])
536 # and count those in the DB
537 if db is None:
538 db = self.getclassdb(classname)
539 return count + len(db)
542 #
543 # Files - special node properties
544 # inherited from FileStorage
546 #
547 # Journal
548 #
549 def addjournal(self, classname, nodeid, action, params, creator=None,
550 creation=None):
551 """ Journal the Action
552 'action' may be:
554 'create' or 'set' -- 'params' is a dictionary of property values
555 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
556 'retire' -- 'params' is None
558 'creator' -- the user performing the action, which defaults to
559 the current user.
560 """
561 if __debug__:
562 logging.getLogger('roundup.hyperdb').debug(
563 'addjournal %s%s %s %r %s %r'%(classname,
564 nodeid, action, params, creator, creation))
565 if creator is None:
566 creator = self.getuid()
567 self.transactions.append((self.doSaveJournal, (classname, nodeid,
568 action, params, creator, creation)))
570 def setjournal(self, classname, nodeid, journal):
571 """Set the journal to the "journal" list."""
572 if __debug__:
573 logging.getLogger('roundup.hyperdb').debug(
574 'setjournal %s%s %r'%(classname, nodeid, journal))
575 self.transactions.append((self.doSetJournal, (classname, nodeid,
576 journal)))
578 def getjournal(self, classname, nodeid):
579 """ get the journal for id
581 Raise IndexError if the node doesn't exist (as per history()'s
582 API)
583 """
584 # our journal result
585 res = []
587 # add any journal entries for transactions not committed to the
588 # database
589 for method, args in self.transactions:
590 if method != self.doSaveJournal:
591 continue
592 (cache_classname, cache_nodeid, cache_action, cache_params,
593 cache_creator, cache_creation) = args
594 if cache_classname == classname and cache_nodeid == nodeid:
595 if not cache_creator:
596 cache_creator = self.getuid()
597 if not cache_creation:
598 cache_creation = date.Date()
599 res.append((cache_nodeid, cache_creation, cache_creator,
600 cache_action, cache_params))
602 # attempt to open the journal - in some rare cases, the journal may
603 # not exist
604 try:
605 db = self.opendb('journals.%s'%classname, 'r')
606 except anydbm.error, error:
607 if str(error) == "need 'c' or 'n' flag to open new db":
608 raise IndexError('no such %s %s'%(classname, nodeid))
609 elif error.args[0] != 2:
610 # this isn't a "not found" error, be alarmed!
611 raise
612 if res:
613 # we have unsaved journal entries, return them
614 return res
615 raise IndexError('no such %s %s'%(classname, nodeid))
616 try:
617 journal = marshal.loads(db[nodeid])
618 except KeyError:
619 db.close()
620 if res:
621 # we have some unsaved journal entries, be happy!
622 return res
623 raise IndexError('no such %s %s'%(classname, nodeid))
624 db.close()
626 # add all the saved journal entries for this node
627 for nodeid, date_stamp, user, action, params in journal:
628 res.append((nodeid, date.Date(date_stamp), user, action, params))
629 return res
631 def pack(self, pack_before):
632 """ Delete all journal entries except "create" before 'pack_before'.
633 """
634 pack_before = pack_before.serialise()
635 for classname in self.getclasses():
636 packed = 0
637 # get the journal db
638 db_name = 'journals.%s'%classname
639 path = os.path.join(os.getcwd(), self.dir, classname)
640 db_type = self.determine_db_type(path)
641 db = self.opendb(db_name, 'w')
643 for key in db.keys():
644 # get the journal for this db entry
645 journal = marshal.loads(db[key])
646 l = []
647 last_set_entry = None
648 for entry in journal:
649 # unpack the entry
650 (nodeid, date_stamp, self.journaltag, action,
651 params) = entry
652 # if the entry is after the pack date, _or_ the initial
653 # create entry, then it stays
654 if date_stamp > pack_before or action == 'create':
655 l.append(entry)
656 else:
657 packed += 1
658 db[key] = marshal.dumps(l)
660 logging.getLogger('roundup.hyperdb').info(
661 'packed %d %s items'%(packed, classname))
663 if db_type == 'gdbm':
664 db.reorganize()
665 db.close()
668 #
669 # Basic transaction support
670 #
671 def commit(self, fail_ok=False):
672 """ Commit the current transactions.
674 Save all data changed since the database was opened or since the
675 last commit() or rollback().
677 fail_ok indicates that the commit is allowed to fail. This is used
678 in the web interface when committing cleaning of the session
679 database. We don't care if there's a concurrency issue there.
681 The only backend this seems to affect is postgres.
682 """
683 logging.getLogger('roundup.hyperdb').info('commit %s transactions'%(
684 len(self.transactions)))
686 # keep a handle to all the database files opened
687 self.databases = {}
689 try:
690 # now, do all the transactions
691 reindex = {}
692 for method, args in self.transactions:
693 reindex[method(*args)] = 1
694 finally:
695 # make sure we close all the database files
696 for db in self.databases.itervalues():
697 db.close()
698 del self.databases
700 # clear the transactions list now so the blobfile implementation
701 # doesn't think there's still pending file commits when it tries
702 # to access the file data
703 self.transactions = []
705 # reindex the nodes that request it
706 for classname, nodeid in [k for k in reindex if k]:
707 self.getclass(classname).index(nodeid)
709 # save the indexer state
710 self.indexer.save_index()
712 self.clearCache()
714 def clearCache(self):
715 # all transactions committed, back to normal
716 self.cache = {}
717 self.dirtynodes = {}
718 self.newnodes = {}
719 self.destroyednodes = {}
720 self.transactions = []
722 def getCachedClassDB(self, classname):
723 """ get the class db, looking in our cache of databases for commit
724 """
725 # get the database handle
726 db_name = 'nodes.%s'%classname
727 if db_name not in self.databases:
728 self.databases[db_name] = self.getclassdb(classname, 'c')
729 return self.databases[db_name]
731 def doSaveNode(self, classname, nodeid, node):
732 db = self.getCachedClassDB(classname)
734 # now save the marshalled data
735 db[nodeid] = marshal.dumps(self.serialise(classname, node))
737 # return the classname, nodeid so we reindex this content
738 return (classname, nodeid)
740 def getCachedJournalDB(self, classname):
741 """ get the journal db, looking in our cache of databases for commit
742 """
743 # get the database handle
744 db_name = 'journals.%s'%classname
745 if db_name not in self.databases:
746 self.databases[db_name] = self.opendb(db_name, 'c')
747 return self.databases[db_name]
749 def doSaveJournal(self, classname, nodeid, action, params, creator,
750 creation):
751 # serialise the parameters now if necessary
752 if isinstance(params, type({})):
753 if action in ('set', 'create'):
754 params = self.serialise(classname, params)
756 # handle supply of the special journalling parameters (usually
757 # supplied on importing an existing database)
758 journaltag = creator
759 if creation:
760 journaldate = creation.serialise()
761 else:
762 journaldate = date.Date().serialise()
764 # create the journal entry
765 entry = (nodeid, journaldate, journaltag, action, params)
767 db = self.getCachedJournalDB(classname)
769 # now insert the journal entry
770 if key_in(db, nodeid):
771 # append to existing
772 s = db[nodeid]
773 l = marshal.loads(s)
774 l.append(entry)
775 else:
776 l = [entry]
778 db[nodeid] = marshal.dumps(l)
780 def doSetJournal(self, classname, nodeid, journal):
781 l = []
782 for nodeid, journaldate, journaltag, action, params in journal:
783 # serialise the parameters now if necessary
784 if isinstance(params, type({})):
785 if action in ('set', 'create'):
786 params = self.serialise(classname, params)
787 journaldate = journaldate.serialise()
788 l.append((nodeid, journaldate, journaltag, action, params))
789 db = self.getCachedJournalDB(classname)
790 db[nodeid] = marshal.dumps(l)
792 def doDestroyNode(self, classname, nodeid):
793 # delete from the class database
794 db = self.getCachedClassDB(classname)
795 if key_in(db, nodeid):
796 del db[nodeid]
798 # delete from the database
799 db = self.getCachedJournalDB(classname)
800 if key_in(db, nodeid):
801 del db[nodeid]
803 def rollback(self):
804 """ Reverse all actions from the current transaction.
805 """
806 logging.getLogger('roundup.hyperdb').info('rollback %s transactions'%(
807 len(self.transactions)))
809 for method, args in self.transactions:
810 # delete temporary files
811 if method == self.doStoreFile:
812 self.rollbackStoreFile(*args)
813 self.cache = {}
814 self.dirtynodes = {}
815 self.newnodes = {}
816 self.destroyednodes = {}
817 self.transactions = []
819 def close(self):
820 """ Nothing to do
821 """
822 if self.lockfile is not None:
823 locking.release_lock(self.lockfile)
824 self.lockfile.close()
825 self.lockfile = None
827 _marker = []
828 class Class(hyperdb.Class):
829 """The handle to a particular class of nodes in a hyperdatabase."""
831 def enableJournalling(self):
832 """Turn journalling on for this class
833 """
834 self.do_journal = 1
836 def disableJournalling(self):
837 """Turn journalling off for this class
838 """
839 self.do_journal = 0
841 # Editing nodes:
843 def create(self, **propvalues):
844 """Create a new node of this class and return its id.
846 The keyword arguments in 'propvalues' map property names to values.
848 The values of arguments must be acceptable for the types of their
849 corresponding properties or a TypeError is raised.
851 If this class has a key property, it must be present and its value
852 must not collide with other key strings or a ValueError is raised.
854 Any other properties on this class that are missing from the
855 'propvalues' dictionary are set to None.
857 If an id in a link or multilink property does not refer to a valid
858 node, an IndexError is raised.
860 These operations trigger detectors and can be vetoed. Attempts
861 to modify the "creation" or "activity" properties cause a KeyError.
862 """
863 if self.db.journaltag is None:
864 raise hyperdb.DatabaseError(_('Database open read-only'))
865 self.fireAuditors('create', None, propvalues)
866 newid = self.create_inner(**propvalues)
867 self.fireReactors('create', newid, None)
868 return newid
870 def create_inner(self, **propvalues):
871 """ Called by create, in-between the audit and react calls.
872 """
873 if 'id' in propvalues:
874 raise KeyError('"id" is reserved')
876 if self.db.journaltag is None:
877 raise hyperdb.DatabaseError(_('Database open read-only'))
879 if 'creation' in propvalues or 'activity' in propvalues:
880 raise KeyError('"creation" and "activity" are reserved')
881 # new node's id
882 newid = self.db.newid(self.classname)
884 # validate propvalues
885 num_re = re.compile('^\d+$')
886 for key, value in propvalues.iteritems():
887 if key == self.key:
888 try:
889 self.lookup(value)
890 except KeyError:
891 pass
892 else:
893 raise ValueError('node with key "%s" exists'%value)
895 # try to handle this property
896 try:
897 prop = self.properties[key]
898 except KeyError:
899 raise KeyError('"%s" has no property "%s"'%(self.classname,
900 key))
902 if value is not None and isinstance(prop, hyperdb.Link):
903 if type(value) != type(''):
904 raise ValueError('link value must be String')
905 link_class = self.properties[key].classname
906 # if it isn't a number, it's a key
907 if not num_re.match(value):
908 try:
909 value = self.db.classes[link_class].lookup(value)
910 except (TypeError, KeyError):
911 raise IndexError('new property "%s": %s not a %s'%(
912 key, value, link_class))
913 elif not self.db.getclass(link_class).hasnode(value):
914 raise IndexError('%s has no node %s'%(link_class,
915 value))
917 # save off the value
918 propvalues[key] = value
920 # register the link with the newly linked node
921 if self.do_journal and self.properties[key].do_journal:
922 self.db.addjournal(link_class, value, 'link',
923 (self.classname, newid, key))
925 elif isinstance(prop, hyperdb.Multilink):
926 if value is None:
927 value = []
928 if not hasattr(value, '__iter__'):
929 raise TypeError('new property "%s" not an iterable of ids'%key)
931 # clean up and validate the list of links
932 link_class = self.properties[key].classname
933 l = []
934 for entry in value:
935 if type(entry) != type(''):
936 raise ValueError('"%s" multilink value (%r) '\
937 'must contain Strings'%(key, value))
938 # if it isn't a number, it's a key
939 if not num_re.match(entry):
940 try:
941 entry = self.db.classes[link_class].lookup(entry)
942 except (TypeError, KeyError):
943 raise IndexError('new property "%s": %s not a %s'%(
944 key, entry, self.properties[key].classname))
945 l.append(entry)
946 value = l
947 propvalues[key] = value
949 # handle additions
950 for nodeid in value:
951 if not self.db.getclass(link_class).hasnode(nodeid):
952 raise IndexError('%s has no node %s'%(link_class,
953 nodeid))
954 # register the link with the newly linked node
955 if self.do_journal and self.properties[key].do_journal:
956 self.db.addjournal(link_class, nodeid, 'link',
957 (self.classname, newid, key))
959 elif isinstance(prop, hyperdb.String):
960 if type(value) != type('') and type(value) != type(u''):
961 raise TypeError('new property "%s" not a string'%key)
962 if prop.indexme:
963 self.db.indexer.add_text((self.classname, newid, key),
964 value)
966 elif isinstance(prop, hyperdb.Password):
967 if not isinstance(value, password.Password):
968 raise TypeError('new property "%s" not a Password'%key)
970 elif isinstance(prop, hyperdb.Date):
971 if value is not None and not isinstance(value, date.Date):
972 raise TypeError('new property "%s" not a Date'%key)
974 elif isinstance(prop, hyperdb.Interval):
975 if value is not None and not isinstance(value, date.Interval):
976 raise TypeError('new property "%s" not an Interval'%key)
978 elif value is not None and isinstance(prop, hyperdb.Number):
979 try:
980 float(value)
981 except ValueError:
982 raise TypeError('new property "%s" not numeric'%key)
984 elif value is not None and isinstance(prop, hyperdb.Boolean):
985 try:
986 int(value)
987 except ValueError:
988 raise TypeError('new property "%s" not boolean'%key)
990 # make sure there's data where there needs to be
991 for key, prop in self.properties.iteritems():
992 if key in propvalues:
993 continue
994 if key == self.key:
995 raise ValueError('key property "%s" is required'%key)
996 if isinstance(prop, hyperdb.Multilink):
997 propvalues[key] = []
999 # done
1000 self.db.addnode(self.classname, newid, propvalues)
1001 if self.do_journal:
1002 self.db.addjournal(self.classname, newid, 'create', {})
1004 return newid
1006 def get(self, nodeid, propname, default=_marker, cache=1):
1007 """Get the value of a property on an existing node of this class.
1009 'nodeid' must be the id of an existing node of this class or an
1010 IndexError is raised. 'propname' must be the name of a property
1011 of this class or a KeyError is raised.
1013 'cache' exists for backward compatibility, and is not used.
1015 Attempts to get the "creation" or "activity" properties should
1016 do the right thing.
1017 """
1018 if propname == 'id':
1019 return nodeid
1021 # get the node's dict
1022 d = self.db.getnode(self.classname, nodeid)
1024 # check for one of the special props
1025 if propname == 'creation':
1026 if 'creation' in d:
1027 return d['creation']
1028 if not self.do_journal:
1029 raise ValueError('Journalling is disabled for this class')
1030 journal = self.db.getjournal(self.classname, nodeid)
1031 if journal:
1032 return journal[0][1]
1033 else:
1034 # on the strange chance that there's no journal
1035 return date.Date()
1036 if propname == 'activity':
1037 if 'activity' in d:
1038 return d['activity']
1039 if not self.do_journal:
1040 raise ValueError('Journalling is disabled for this class')
1041 journal = self.db.getjournal(self.classname, nodeid)
1042 if journal:
1043 return self.db.getjournal(self.classname, nodeid)[-1][1]
1044 else:
1045 # on the strange chance that there's no journal
1046 return date.Date()
1047 if propname == 'creator':
1048 if 'creator' in d:
1049 return d['creator']
1050 if not self.do_journal:
1051 raise ValueError('Journalling is disabled for this class')
1052 journal = self.db.getjournal(self.classname, nodeid)
1053 if journal:
1054 num_re = re.compile('^\d+$')
1055 value = journal[0][2]
1056 if num_re.match(value):
1057 return value
1058 else:
1059 # old-style "username" journal tag
1060 try:
1061 return self.db.user.lookup(value)
1062 except KeyError:
1063 # user's been retired, return admin
1064 return '1'
1065 else:
1066 return self.db.getuid()
1067 if propname == 'actor':
1068 if 'actor' in d:
1069 return d['actor']
1070 if not self.do_journal:
1071 raise ValueError('Journalling is disabled for this class')
1072 journal = self.db.getjournal(self.classname, nodeid)
1073 if journal:
1074 num_re = re.compile('^\d+$')
1075 value = journal[-1][2]
1076 if num_re.match(value):
1077 return value
1078 else:
1079 # old-style "username" journal tag
1080 try:
1081 return self.db.user.lookup(value)
1082 except KeyError:
1083 # user's been retired, return admin
1084 return '1'
1085 else:
1086 return self.db.getuid()
1088 # get the property (raises KeyErorr if invalid)
1089 prop = self.properties[propname]
1091 if propname not in d:
1092 if default is _marker:
1093 if isinstance(prop, hyperdb.Multilink):
1094 return []
1095 else:
1096 return None
1097 else:
1098 return default
1100 # return a dupe of the list so code doesn't get confused
1101 if isinstance(prop, hyperdb.Multilink):
1102 return d[propname][:]
1104 return d[propname]
1106 def set(self, nodeid, **propvalues):
1107 """Modify a property on an existing node of this class.
1109 'nodeid' must be the id of an existing node of this class or an
1110 IndexError is raised.
1112 Each key in 'propvalues' must be the name of a property of this
1113 class or a KeyError is raised.
1115 All values in 'propvalues' must be acceptable types for their
1116 corresponding properties or a TypeError is raised.
1118 If the value of the key property is set, it must not collide with
1119 other key strings or a ValueError is raised.
1121 If the value of a Link or Multilink property contains an invalid
1122 node id, a ValueError is raised.
1124 These operations trigger detectors and can be vetoed. Attempts
1125 to modify the "creation" or "activity" properties cause a KeyError.
1126 """
1127 if self.db.journaltag is None:
1128 raise hyperdb.DatabaseError(_('Database open read-only'))
1130 self.fireAuditors('set', nodeid, propvalues)
1131 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1132 for name, prop in self.getprops(protected=0).iteritems():
1133 if name in oldvalues:
1134 continue
1135 if isinstance(prop, hyperdb.Multilink):
1136 oldvalues[name] = []
1137 else:
1138 oldvalues[name] = None
1139 propvalues = self.set_inner(nodeid, **propvalues)
1140 self.fireReactors('set', nodeid, oldvalues)
1141 return propvalues
1143 def set_inner(self, nodeid, **propvalues):
1144 """ Called by set, in-between the audit and react calls.
1145 """
1146 if not propvalues:
1147 return propvalues
1149 if 'creation' in propvalues or 'activity' in propvalues:
1150 raise KeyError, '"creation" and "activity" are reserved'
1152 if 'id' in propvalues:
1153 raise KeyError, '"id" is reserved'
1155 if self.db.journaltag is None:
1156 raise hyperdb.DatabaseError(_('Database open read-only'))
1158 node = self.db.getnode(self.classname, nodeid)
1159 if self.db.RETIRED_FLAG in node:
1160 raise IndexError
1161 num_re = re.compile('^\d+$')
1163 # if the journal value is to be different, store it in here
1164 journalvalues = {}
1166 # list() propvalues 'cos it might be modified by the loop
1167 for propname, value in list(propvalues.items()):
1168 # check to make sure we're not duplicating an existing key
1169 if propname == self.key and node[propname] != value:
1170 try:
1171 self.lookup(value)
1172 except KeyError:
1173 pass
1174 else:
1175 raise ValueError('node with key "%s" exists'%value)
1177 # this will raise the KeyError if the property isn't valid
1178 # ... we don't use getprops() here because we only care about
1179 # the writeable properties.
1180 try:
1181 prop = self.properties[propname]
1182 except KeyError:
1183 raise KeyError('"%s" has no property named "%s"'%(
1184 self.classname, propname))
1186 # if the value's the same as the existing value, no sense in
1187 # doing anything
1188 current = node.get(propname, None)
1189 if value == current:
1190 del propvalues[propname]
1191 continue
1192 journalvalues[propname] = current
1194 # do stuff based on the prop type
1195 if isinstance(prop, hyperdb.Link):
1196 link_class = prop.classname
1197 # if it isn't a number, it's a key
1198 if value is not None and not isinstance(value, type('')):
1199 raise ValueError('property "%s" link value be a string'%(
1200 propname))
1201 if isinstance(value, type('')) and not num_re.match(value):
1202 try:
1203 value = self.db.classes[link_class].lookup(value)
1204 except (TypeError, KeyError):
1205 raise IndexError('new property "%s": %s not a %s'%(
1206 propname, value, prop.classname))
1208 if (value is not None and
1209 not self.db.getclass(link_class).hasnode(value)):
1210 raise IndexError('%s has no node %s'%(link_class,
1211 value))
1213 if self.do_journal and prop.do_journal:
1214 # register the unlink with the old linked node
1215 if propname in node and node[propname] is not None:
1216 self.db.addjournal(link_class, node[propname], 'unlink',
1217 (self.classname, nodeid, propname))
1219 # register the link with the newly linked node
1220 if value is not None:
1221 self.db.addjournal(link_class, value, 'link',
1222 (self.classname, nodeid, propname))
1224 elif isinstance(prop, hyperdb.Multilink):
1225 if value is None:
1226 value = []
1227 if not hasattr(value, '__iter__'):
1228 raise TypeError('new property "%s" not an iterable of'
1229 ' ids'%propname)
1230 link_class = self.properties[propname].classname
1231 l = []
1232 for entry in value:
1233 # if it isn't a number, it's a key
1234 if type(entry) != type(''):
1235 raise ValueError('new property "%s" link value '
1236 'must be a string'%propname)
1237 if not num_re.match(entry):
1238 try:
1239 entry = self.db.classes[link_class].lookup(entry)
1240 except (TypeError, KeyError):
1241 raise IndexError('new property "%s": %s not a %s'%(
1242 propname, entry,
1243 self.properties[propname].classname))
1244 l.append(entry)
1245 value = l
1246 propvalues[propname] = value
1248 # figure the journal entry for this property
1249 add = []
1250 remove = []
1252 # handle removals
1253 if propname in node:
1254 l = node[propname]
1255 else:
1256 l = []
1257 for id in l[:]:
1258 if id in value:
1259 continue
1260 # register the unlink with the old linked node
1261 if self.do_journal and self.properties[propname].do_journal:
1262 self.db.addjournal(link_class, id, 'unlink',
1263 (self.classname, nodeid, propname))
1264 l.remove(id)
1265 remove.append(id)
1267 # handle additions
1268 for id in value:
1269 if not self.db.getclass(link_class).hasnode(id):
1270 raise IndexError('%s has no node %s'%(link_class,
1271 id))
1272 if id in l:
1273 continue
1274 # register the link with the newly linked node
1275 if self.do_journal and self.properties[propname].do_journal:
1276 self.db.addjournal(link_class, id, 'link',
1277 (self.classname, nodeid, propname))
1278 l.append(id)
1279 add.append(id)
1281 # figure the journal entry
1282 l = []
1283 if add:
1284 l.append(('+', add))
1285 if remove:
1286 l.append(('-', remove))
1287 if l:
1288 journalvalues[propname] = tuple(l)
1290 elif isinstance(prop, hyperdb.String):
1291 if value is not None and type(value) != type('') and type(value) != type(u''):
1292 raise TypeError('new property "%s" not a '
1293 'string'%propname)
1294 if prop.indexme:
1295 self.db.indexer.add_text((self.classname, nodeid, propname),
1296 value)
1298 elif isinstance(prop, hyperdb.Password):
1299 if not isinstance(value, password.Password):
1300 raise TypeError('new property "%s" not a '
1301 'Password'%propname)
1302 propvalues[propname] = value
1303 journalvalues[propname] = \
1304 current and password.JournalPassword(current)
1306 elif value is not None and isinstance(prop, hyperdb.Date):
1307 if not isinstance(value, date.Date):
1308 raise TypeError('new property "%s" not a '
1309 'Date'%propname)
1310 propvalues[propname] = value
1312 elif value is not None and isinstance(prop, hyperdb.Interval):
1313 if not isinstance(value, date.Interval):
1314 raise TypeError('new property "%s" not an '
1315 'Interval'%propname)
1316 propvalues[propname] = value
1318 elif value is not None and isinstance(prop, hyperdb.Number):
1319 try:
1320 float(value)
1321 except ValueError:
1322 raise TypeError('new property "%s" not '
1323 'numeric'%propname)
1325 elif value is not None and isinstance(prop, hyperdb.Boolean):
1326 try:
1327 int(value)
1328 except ValueError:
1329 raise TypeError('new property "%s" not '
1330 'boolean'%propname)
1332 node[propname] = value
1334 # nothing to do?
1335 if not propvalues:
1336 return propvalues
1338 # update the activity time
1339 node['activity'] = date.Date()
1340 node['actor'] = self.db.getuid()
1342 # do the set, and journal it
1343 self.db.setnode(self.classname, nodeid, node)
1345 if self.do_journal:
1346 self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
1348 return propvalues
1350 def retire(self, nodeid):
1351 """Retire a node.
1353 The properties on the node remain available from the get() method,
1354 and the node's id is never reused.
1356 Retired nodes are not returned by the find(), list(), or lookup()
1357 methods, and other nodes may reuse the values of their key properties.
1359 These operations trigger detectors and can be vetoed. Attempts
1360 to modify the "creation" or "activity" properties cause a KeyError.
1361 """
1362 if self.db.journaltag is None:
1363 raise hyperdb.DatabaseError(_('Database open read-only'))
1365 self.fireAuditors('retire', nodeid, None)
1367 node = self.db.getnode(self.classname, nodeid)
1368 node[self.db.RETIRED_FLAG] = 1
1369 self.db.setnode(self.classname, nodeid, node)
1370 if self.do_journal:
1371 self.db.addjournal(self.classname, nodeid, 'retired', None)
1373 self.fireReactors('retire', nodeid, None)
1375 def restore(self, nodeid):
1376 """Restpre a retired node.
1378 Make node available for all operations like it was before retirement.
1379 """
1380 if self.db.journaltag is None:
1381 raise hyperdb.DatabaseError(_('Database open read-only'))
1383 node = self.db.getnode(self.classname, nodeid)
1384 # check if key property was overrided
1385 key = self.getkey()
1386 try:
1387 id = self.lookup(node[key])
1388 except KeyError:
1389 pass
1390 else:
1391 raise KeyError("Key property (%s) of retired node clashes "
1392 "with existing one (%s)" % (key, node[key]))
1393 # Now we can safely restore node
1394 self.fireAuditors('restore', nodeid, None)
1395 del node[self.db.RETIRED_FLAG]
1396 self.db.setnode(self.classname, nodeid, node)
1397 if self.do_journal:
1398 self.db.addjournal(self.classname, nodeid, 'restored', None)
1400 self.fireReactors('restore', nodeid, None)
1402 def is_retired(self, nodeid, cldb=None):
1403 """Return true if the node is retired.
1404 """
1405 node = self.db.getnode(self.classname, nodeid, cldb)
1406 if self.db.RETIRED_FLAG in node:
1407 return 1
1408 return 0
1410 def destroy(self, nodeid):
1411 """Destroy a node.
1413 WARNING: this method should never be used except in extremely rare
1414 situations where there could never be links to the node being
1415 deleted
1417 WARNING: use retire() instead
1419 WARNING: the properties of this node will not be available ever again
1421 WARNING: really, use retire() instead
1423 Well, I think that's enough warnings. This method exists mostly to
1424 support the session storage of the cgi interface.
1425 """
1426 if self.db.journaltag is None:
1427 raise hyperdb.DatabaseError(_('Database open read-only'))
1428 self.db.destroynode(self.classname, nodeid)
1430 # Locating nodes:
1431 def hasnode(self, nodeid):
1432 """Determine if the given nodeid actually exists
1433 """
1434 return self.db.hasnode(self.classname, nodeid)
1436 def setkey(self, propname):
1437 """Select a String property of this class to be the key property.
1439 'propname' must be the name of a String property of this class or
1440 None, or a TypeError is raised. The values of the key property on
1441 all existing nodes must be unique or a ValueError is raised. If the
1442 property doesn't exist, KeyError is raised.
1443 """
1444 prop = self.getprops()[propname]
1445 if not isinstance(prop, hyperdb.String):
1446 raise TypeError('key properties must be String')
1447 self.key = propname
1449 def getkey(self):
1450 """Return the name of the key property for this class or None."""
1451 return self.key
1453 # TODO: set up a separate index db file for this? profile?
1454 def lookup(self, keyvalue):
1455 """Locate a particular node by its key property and return its id.
1457 If this class has no key property, a TypeError is raised. If the
1458 'keyvalue' matches one of the values for the key property among
1459 the nodes in this class, the matching node's id is returned;
1460 otherwise a KeyError is raised.
1461 """
1462 if not self.key:
1463 raise TypeError('No key property set for '
1464 'class %s'%self.classname)
1465 cldb = self.db.getclassdb(self.classname)
1466 try:
1467 for nodeid in self.getnodeids(cldb):
1468 node = self.db.getnode(self.classname, nodeid, cldb)
1469 if self.db.RETIRED_FLAG in node:
1470 continue
1471 if self.key not in node:
1472 continue
1473 if node[self.key] == keyvalue:
1474 return nodeid
1475 finally:
1476 cldb.close()
1477 raise KeyError('No key (%s) value "%s" for "%s"'%(self.key,
1478 keyvalue, self.classname))
1480 # change from spec - allows multiple props to match
1481 def find(self, **propspec):
1482 """Get the ids of nodes in this class which link to the given nodes.
1484 'propspec' consists of keyword args propname=nodeid or
1485 propname={nodeid:1, }
1486 'propname' must be the name of a property in this class, or a
1487 KeyError is raised. That property must be a Link or
1488 Multilink property, or a TypeError is raised.
1490 Any node in this class whose 'propname' property links to any of
1491 the nodeids will be returned. Examples::
1493 db.issue.find(messages='1')
1494 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1495 """
1496 for propname, itemids in propspec.iteritems():
1497 # check the prop is OK
1498 prop = self.properties[propname]
1499 if not isinstance(prop, hyperdb.Link) and not isinstance(prop, hyperdb.Multilink):
1500 raise TypeError("'%s' not a Link/Multilink "
1501 "property"%propname)
1503 # ok, now do the find
1504 cldb = self.db.getclassdb(self.classname)
1505 l = []
1506 try:
1507 for id in self.getnodeids(db=cldb):
1508 item = self.db.getnode(self.classname, id, db=cldb)
1509 if self.db.RETIRED_FLAG in item:
1510 continue
1511 for propname, itemids in propspec.iteritems():
1512 if type(itemids) is not type({}):
1513 itemids = {itemids:1}
1515 # special case if the item doesn't have this property
1516 if propname not in item:
1517 if None in itemids:
1518 l.append(id)
1519 break
1520 continue
1522 # grab the property definition and its value on this item
1523 prop = self.properties[propname]
1524 value = item[propname]
1525 if isinstance(prop, hyperdb.Link) and value in itemids:
1526 l.append(id)
1527 break
1528 elif isinstance(prop, hyperdb.Multilink):
1529 hit = 0
1530 for v in value:
1531 if v in itemids:
1532 l.append(id)
1533 hit = 1
1534 break
1535 if hit:
1536 break
1537 finally:
1538 cldb.close()
1539 return l
1541 def stringFind(self, **requirements):
1542 """Locate a particular node by matching a set of its String
1543 properties in a caseless search.
1545 If the property is not a String property, a TypeError is raised.
1547 The return is a list of the id of all nodes that match.
1548 """
1549 for propname in requirements:
1550 prop = self.properties[propname]
1551 if not isinstance(prop, hyperdb.String):
1552 raise TypeError("'%s' not a String property"%propname)
1553 requirements[propname] = requirements[propname].lower()
1554 l = []
1555 cldb = self.db.getclassdb(self.classname)
1556 try:
1557 for nodeid in self.getnodeids(cldb):
1558 node = self.db.getnode(self.classname, nodeid, cldb)
1559 if self.db.RETIRED_FLAG in node:
1560 continue
1561 for key, value in requirements.iteritems():
1562 if key not in node:
1563 break
1564 if node[key] is None or node[key].lower() != value:
1565 break
1566 else:
1567 l.append(nodeid)
1568 finally:
1569 cldb.close()
1570 return l
1572 def list(self):
1573 """ Return a list of the ids of the active nodes in this class.
1574 """
1575 l = []
1576 cn = self.classname
1577 cldb = self.db.getclassdb(cn)
1578 try:
1579 for nodeid in self.getnodeids(cldb):
1580 node = self.db.getnode(cn, nodeid, cldb)
1581 if self.db.RETIRED_FLAG in node:
1582 continue
1583 l.append(nodeid)
1584 finally:
1585 cldb.close()
1586 l.sort()
1587 return l
1589 def getnodeids(self, db=None, retired=None):
1590 """ Return a list of ALL nodeids
1592 Set retired=None to get all nodes. Otherwise it'll get all the
1593 retired or non-retired nodes, depending on the flag.
1594 """
1595 res = []
1597 # start off with the new nodes
1598 if self.classname in self.db.newnodes:
1599 res.extend(self.db.newnodes[self.classname])
1601 must_close = False
1602 if db is None:
1603 db = self.db.getclassdb(self.classname)
1604 must_close = True
1605 try:
1606 res.extend(db.keys())
1608 # remove the uncommitted, destroyed nodes
1609 if self.classname in self.db.destroyednodes:
1610 for nodeid in self.db.destroyednodes[self.classname]:
1611 if key_in(db, nodeid):
1612 res.remove(nodeid)
1614 # check retired flag
1615 if retired is False or retired is True:
1616 l = []
1617 for nodeid in res:
1618 node = self.db.getnode(self.classname, nodeid, db)
1619 is_ret = self.db.RETIRED_FLAG in node
1620 if retired == is_ret:
1621 l.append(nodeid)
1622 res = l
1623 finally:
1624 if must_close:
1625 db.close()
1626 return res
1628 def _filter(self, search_matches, filterspec, proptree,
1629 num_re = re.compile('^\d+$')):
1630 """Return a list of the ids of the active nodes in this class that
1631 match the 'filter' spec, sorted by the group spec and then the
1632 sort spec.
1634 "filterspec" is {propname: value(s)}
1636 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1637 and prop is a prop name or None
1639 "search_matches" is a sequence type or None
1641 The filter must match all properties specificed. If the property
1642 value to match is a list:
1644 1. String properties must match all elements in the list, and
1645 2. Other properties must match any of the elements in the list.
1646 """
1647 if __debug__:
1648 start_t = time.time()
1650 cn = self.classname
1652 # optimise filterspec
1653 l = []
1654 props = self.getprops()
1655 LINK = 'spec:link'
1656 MULTILINK = 'spec:multilink'
1657 STRING = 'spec:string'
1658 DATE = 'spec:date'
1659 INTERVAL = 'spec:interval'
1660 OTHER = 'spec:other'
1662 for k, v in filterspec.iteritems():
1663 propclass = props[k]
1664 if isinstance(propclass, hyperdb.Link):
1665 if type(v) is not type([]):
1666 v = [v]
1667 u = []
1668 for entry in v:
1669 # the value -1 is a special "not set" sentinel
1670 if entry == '-1':
1671 entry = None
1672 u.append(entry)
1673 l.append((LINK, k, u))
1674 elif isinstance(propclass, hyperdb.Multilink):
1675 # the value -1 is a special "not set" sentinel
1676 if v in ('-1', ['-1']):
1677 v = []
1678 elif type(v) is not type([]):
1679 v = [v]
1680 l.append((MULTILINK, k, v))
1681 elif isinstance(propclass, hyperdb.String) and k != 'id':
1682 if type(v) is not type([]):
1683 v = [v]
1684 for v in v:
1685 # simple glob searching
1686 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1687 v = v.replace('?', '.')
1688 v = v.replace('*', '.*?')
1689 l.append((STRING, k, re.compile(v, re.I)))
1690 elif isinstance(propclass, hyperdb.Date):
1691 try:
1692 date_rng = propclass.range_from_raw(v, self.db)
1693 l.append((DATE, k, date_rng))
1694 except ValueError:
1695 # If range creation fails - ignore that search parameter
1696 pass
1697 elif isinstance(propclass, hyperdb.Interval):
1698 try:
1699 intv_rng = date.Range(v, date.Interval)
1700 l.append((INTERVAL, k, intv_rng))
1701 except ValueError:
1702 # If range creation fails - ignore that search parameter
1703 pass
1705 elif isinstance(propclass, hyperdb.Boolean):
1706 if type(v) == type(""):
1707 v = v.split(',')
1708 if type(v) != type([]):
1709 v = [v]
1710 bv = []
1711 for val in v:
1712 if type(val) is type(''):
1713 bv.append(propclass.from_raw (val))
1714 else:
1715 bv.append(val)
1716 l.append((OTHER, k, bv))
1718 elif k == 'id':
1719 if type(v) != type([]):
1720 v = v.split(',')
1721 l.append((OTHER, k, [str(int(val)) for val in v]))
1723 elif isinstance(propclass, hyperdb.Number):
1724 if type(v) != type([]):
1725 try :
1726 v = v.split(',')
1727 except AttributeError :
1728 v = [v]
1729 l.append((OTHER, k, [float(val) for val in v]))
1731 filterspec = l
1733 # now, find all the nodes that are active and pass filtering
1734 matches = []
1735 cldb = self.db.getclassdb(cn)
1736 t = 0
1737 try:
1738 # TODO: only full-scan once (use items())
1739 for nodeid in self.getnodeids(cldb):
1740 node = self.db.getnode(cn, nodeid, cldb)
1741 if self.db.RETIRED_FLAG in node:
1742 continue
1743 # apply filter
1744 for t, k, v in filterspec:
1745 # handle the id prop
1746 if k == 'id':
1747 if nodeid not in v:
1748 break
1749 continue
1751 # get the node value
1752 nv = node.get(k, None)
1754 match = 0
1756 # now apply the property filter
1757 if t == LINK:
1758 # link - if this node's property doesn't appear in the
1759 # filterspec's nodeid list, skip it
1760 match = nv in v
1761 elif t == MULTILINK:
1762 # multilink - if any of the nodeids required by the
1763 # filterspec aren't in this node's property, then skip
1764 # it
1765 nv = node.get(k, [])
1767 # check for matching the absence of multilink values
1768 if not v:
1769 match = not nv
1770 else:
1771 # otherwise, make sure this node has each of the
1772 # required values
1773 expr = Expression(v)
1774 if expr.evaluate(nv): match = 1
1775 elif t == STRING:
1776 if nv is None:
1777 nv = ''
1778 # RE search
1779 match = v.search(nv)
1780 elif t == DATE or t == INTERVAL:
1781 if nv is None:
1782 match = v is None
1783 else:
1784 if v.to_value:
1785 if v.from_value <= nv and v.to_value >= nv:
1786 match = 1
1787 else:
1788 if v.from_value <= nv:
1789 match = 1
1790 elif t == OTHER:
1791 # straight value comparison for the other types
1792 match = nv in v
1793 if not match:
1794 break
1795 else:
1796 matches.append([nodeid, node])
1798 # filter based on full text search
1799 if search_matches is not None:
1800 k = []
1801 for v in matches:
1802 if v[0] in search_matches:
1803 k.append(v)
1804 matches = k
1806 # add sorting information to the proptree
1807 JPROPS = {'actor':1, 'activity':1, 'creator':1, 'creation':1}
1808 children = []
1809 if proptree:
1810 children = proptree.sortable_children()
1811 for pt in children:
1812 dir = pt.sort_direction
1813 prop = pt.name
1814 assert (dir and prop)
1815 propclass = props[prop]
1816 pt.sort_ids = []
1817 is_pointer = isinstance(propclass,(hyperdb.Link,
1818 hyperdb.Multilink))
1819 if not is_pointer:
1820 pt.sort_result = []
1821 try:
1822 # cache the opened link class db, if needed.
1823 lcldb = None
1824 # cache the linked class items too
1825 lcache = {}
1827 for entry in matches:
1828 itemid = entry[-2]
1829 item = entry[-1]
1830 # handle the properties that might be "faked"
1831 # also, handle possible missing properties
1832 try:
1833 v = item[prop]
1834 except KeyError:
1835 if prop in JPROPS:
1836 # force lookup of the special journal prop
1837 v = self.get(itemid, prop)
1838 else:
1839 # the node doesn't have a value for this
1840 # property
1841 v = None
1842 if isinstance(propclass, hyperdb.Multilink):
1843 v = []
1844 if prop == 'id':
1845 v = int (itemid)
1846 pt.sort_ids.append(v)
1847 if not is_pointer:
1848 pt.sort_result.append(v)
1849 continue
1851 # missing (None) values are always sorted first
1852 if v is None:
1853 pt.sort_ids.append(v)
1854 if not is_pointer:
1855 pt.sort_result.append(v)
1856 continue
1858 if isinstance(propclass, hyperdb.Link):
1859 lcn = propclass.classname
1860 link = self.db.classes[lcn]
1861 key = link.orderprop()
1862 child = pt.propdict[key]
1863 if key!='id':
1864 if v not in lcache:
1865 # open the link class db if it's not already
1866 if lcldb is None:
1867 lcldb = self.db.getclassdb(lcn)
1868 lcache[v] = self.db.getnode(lcn, v, lcldb)
1869 r = lcache[v][key]
1870 child.propdict[key].sort_ids.append(r)
1871 else:
1872 child.propdict[key].sort_ids.append(v)
1873 pt.sort_ids.append(v)
1874 if not is_pointer:
1875 r = propclass.sort_repr(pt.parent.cls, v, pt.name)
1876 pt.sort_result.append(r)
1877 finally:
1878 # if we opened the link class db, close it now
1879 if lcldb is not None:
1880 lcldb.close()
1881 del lcache
1882 finally:
1883 cldb.close()
1885 # pull the id out of the individual entries
1886 matches = [entry[-2] for entry in matches]
1887 if __debug__:
1888 self.db.stats['filtering'] += (time.time() - start_t)
1889 return matches
1891 def count(self):
1892 """Get the number of nodes in this class.
1894 If the returned integer is 'numnodes', the ids of all the nodes
1895 in this class run from 1 to numnodes, and numnodes+1 will be the
1896 id of the next node to be created in this class.
1897 """
1898 return self.db.countnodes(self.classname)
1900 # Manipulating properties:
1902 def getprops(self, protected=1):
1903 """Return a dictionary mapping property names to property objects.
1904 If the "protected" flag is true, we include protected properties -
1905 those which may not be modified.
1907 In addition to the actual properties on the node, these
1908 methods provide the "creation" and "activity" properties. If the
1909 "protected" flag is true, we include protected properties - those
1910 which may not be modified.
1911 """
1912 d = self.properties.copy()
1913 if protected:
1914 d['id'] = hyperdb.String()
1915 d['creation'] = hyperdb.Date()
1916 d['activity'] = hyperdb.Date()
1917 d['creator'] = hyperdb.Link('user')
1918 d['actor'] = hyperdb.Link('user')
1919 return d
1921 def addprop(self, **properties):
1922 """Add properties to this class.
1924 The keyword arguments in 'properties' must map names to property
1925 objects, or a TypeError is raised. None of the keys in 'properties'
1926 may collide with the names of existing properties, or a ValueError
1927 is raised before any properties have been added.
1928 """
1929 for key in properties:
1930 if key in self.properties:
1931 raise ValueError(key)
1932 self.properties.update(properties)
1934 def index(self, nodeid):
1935 """ Add (or refresh) the node to search indexes """
1936 # find all the String properties that have indexme
1937 for prop, propclass in self.getprops().iteritems():
1938 if isinstance(propclass, hyperdb.String) and propclass.indexme:
1939 # index them under (classname, nodeid, property)
1940 try:
1941 value = str(self.get(nodeid, prop))
1942 except IndexError:
1943 # node has been destroyed
1944 continue
1945 self.db.indexer.add_text((self.classname, nodeid, prop), value)
1947 #
1948 # import / export support
1949 #
1950 def export_list(self, propnames, nodeid):
1951 """ Export a node - generate a list of CSV-able data in the order
1952 specified by propnames for the given node.
1953 """
1954 properties = self.getprops()
1955 l = []
1956 for prop in propnames:
1957 proptype = properties[prop]
1958 value = self.get(nodeid, prop)
1959 # "marshal" data where needed
1960 if value is None:
1961 pass
1962 elif isinstance(proptype, hyperdb.Date):
1963 value = value.get_tuple()
1964 elif isinstance(proptype, hyperdb.Interval):
1965 value = value.get_tuple()
1966 elif isinstance(proptype, hyperdb.Password):
1967 value = str(value)
1968 l.append(repr(value))
1970 # append retired flag
1971 l.append(repr(self.is_retired(nodeid)))
1973 return l
1975 def import_list(self, propnames, proplist):
1976 """ Import a node - all information including "id" is present and
1977 should not be sanity checked. Triggers are not triggered. The
1978 journal should be initialised using the "creator" and "created"
1979 information.
1981 Return the nodeid of the node imported.
1982 """
1983 if self.db.journaltag is None:
1984 raise hyperdb.DatabaseError(_('Database open read-only'))
1985 properties = self.getprops()
1987 # make the new node's property map
1988 d = {}
1989 newid = None
1990 for i in range(len(propnames)):
1991 # Figure the property for this column
1992 propname = propnames[i]
1994 # Use eval to reverse the repr() used to output the CSV
1995 value = eval(proplist[i])
1997 # "unmarshal" where necessary
1998 if propname == 'id':
1999 newid = value
2000 continue
2001 elif propname == 'is retired':
2002 # is the item retired?
2003 if int(value):
2004 d[self.db.RETIRED_FLAG] = 1
2005 continue
2006 elif value is None:
2007 d[propname] = None
2008 continue
2010 prop = properties[propname]
2011 if isinstance(prop, hyperdb.Date):
2012 value = date.Date(value)
2013 elif isinstance(prop, hyperdb.Interval):
2014 value = date.Interval(value)
2015 elif isinstance(prop, hyperdb.Password):
2016 value = password.Password(encrypted=value)
2017 d[propname] = value
2019 # get a new id if necessary
2020 if newid is None:
2021 newid = self.db.newid(self.classname)
2023 # add the node and journal
2024 self.db.addnode(self.classname, newid, d)
2025 return newid
2027 def export_journals(self):
2028 """Export a class's journal - generate a list of lists of
2029 CSV-able data:
2031 nodeid, date, user, action, params
2033 No heading here - the columns are fixed.
2034 """
2035 properties = self.getprops()
2036 r = []
2037 for nodeid in self.getnodeids():
2038 for nodeid, date, user, action, params in self.history(nodeid):
2039 date = date.get_tuple()
2040 if action == 'set':
2041 export_data = {}
2042 for propname, value in params.iteritems():
2043 if propname not in properties:
2044 # property no longer in the schema
2045 continue
2047 prop = properties[propname]
2048 # make sure the params are eval()'able
2049 if value is None:
2050 pass
2051 elif isinstance(prop, hyperdb.Date):
2052 # this is a hack - some dates are stored as strings
2053 if not isinstance(value, type('')):
2054 value = value.get_tuple()
2055 elif isinstance(prop, hyperdb.Interval):
2056 # hack too - some intervals are stored as strings
2057 if not isinstance(value, type('')):
2058 value = value.get_tuple()
2059 elif isinstance(prop, hyperdb.Password):
2060 value = str(value)
2061 export_data[propname] = value
2062 params = export_data
2063 r.append([repr(nodeid), repr(date), repr(user),
2064 repr(action), repr(params)])
2065 return r
2067 class FileClass(hyperdb.FileClass, Class):
2068 """This class defines a large chunk of data. To support this, it has a
2069 mandatory String property "content" which is typically saved off
2070 externally to the hyperdb.
2072 The default MIME type of this data is defined by the
2073 "default_mime_type" class attribute, which may be overridden by each
2074 node if the class defines a "type" String property.
2075 """
2076 def __init__(self, db, classname, **properties):
2077 """The newly-created class automatically includes the "content"
2078 and "type" properties.
2079 """
2080 if 'content' not in properties:
2081 properties['content'] = hyperdb.String(indexme='yes')
2082 if 'type' not in properties:
2083 properties['type'] = hyperdb.String()
2084 Class.__init__(self, db, classname, **properties)
2086 def create(self, **propvalues):
2087 """ Snarf the "content" propvalue and store in a file
2088 """
2089 # we need to fire the auditors now, or the content property won't
2090 # be in propvalues for the auditors to play with
2091 self.fireAuditors('create', None, propvalues)
2093 # now remove the content property so it's not stored in the db
2094 content = propvalues['content']
2095 del propvalues['content']
2097 # make sure we have a MIME type
2098 mime_type = propvalues.get('type', self.default_mime_type)
2100 # do the database create
2101 newid = self.create_inner(**propvalues)
2103 # store off the content as a file
2104 self.db.storefile(self.classname, newid, None, content)
2106 # fire reactors
2107 self.fireReactors('create', newid, None)
2109 return newid
2111 def get(self, nodeid, propname, default=_marker, cache=1):
2112 """ Trap the content propname and get it from the file
2114 'cache' exists for backwards compatibility, and is not used.
2115 """
2116 poss_msg = 'Possibly an access right configuration problem.'
2117 if propname == 'content':
2118 try:
2119 return self.db.getfile(self.classname, nodeid, None)
2120 except IOError, strerror:
2121 # XXX by catching this we don't see an error in the log.
2122 return 'ERROR reading file: %s%s\n%s\n%s'%(
2123 self.classname, nodeid, poss_msg, strerror)
2124 if default is not _marker:
2125 return Class.get(self, nodeid, propname, default)
2126 else:
2127 return Class.get(self, nodeid, propname)
2129 def set(self, itemid, **propvalues):
2130 """ Snarf the "content" propvalue and update it in a file
2131 """
2132 self.fireAuditors('set', itemid, propvalues)
2134 # create the oldvalues dict - fill in any missing values
2135 oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid))
2136 for name, prop in self.getprops(protected=0).iteritems():
2137 if name in oldvalues:
2138 continue
2139 if isinstance(prop, hyperdb.Multilink):
2140 oldvalues[name] = []
2141 else:
2142 oldvalues[name] = None
2144 # now remove the content property so it's not stored in the db
2145 content = None
2146 if 'content' in propvalues:
2147 content = propvalues['content']
2148 del propvalues['content']
2150 # do the database update
2151 propvalues = self.set_inner(itemid, **propvalues)
2153 # do content?
2154 if content:
2155 # store and possibly index
2156 self.db.storefile(self.classname, itemid, None, content)
2157 if self.properties['content'].indexme:
2158 mime_type = self.get(itemid, 'type', self.default_mime_type)
2159 self.db.indexer.add_text((self.classname, itemid, 'content'),
2160 content, mime_type)
2161 propvalues['content'] = content
2163 # fire reactors
2164 self.fireReactors('set', itemid, oldvalues)
2165 return propvalues
2167 def index(self, nodeid):
2168 """ Add (or refresh) the node to search indexes.
2170 Use the content-type property for the content property.
2171 """
2172 # find all the String properties that have indexme
2173 for prop, propclass in self.getprops().iteritems():
2174 if prop == 'content' and propclass.indexme:
2175 mime_type = self.get(nodeid, 'type', self.default_mime_type)
2176 self.db.indexer.add_text((self.classname, nodeid, 'content'),
2177 str(self.get(nodeid, 'content')), mime_type)
2178 elif isinstance(propclass, hyperdb.String) and propclass.indexme:
2179 # index them under (classname, nodeid, property)
2180 try:
2181 value = str(self.get(nodeid, prop))
2182 except IndexError:
2183 # node has been destroyed
2184 continue
2185 self.db.indexer.add_text((self.classname, nodeid, prop), value)
2187 # deviation from spec - was called ItemClass
2188 class IssueClass(Class, roundupdb.IssueClass):
2189 # Overridden methods:
2190 def __init__(self, db, classname, **properties):
2191 """The newly-created class automatically includes the "messages",
2192 "files", "nosy", and "superseder" properties. If the 'properties'
2193 dictionary attempts to specify any of these properties or a
2194 "creation" or "activity" property, a ValueError is raised.
2195 """
2196 if 'title' not in properties:
2197 properties['title'] = hyperdb.String(indexme='yes')
2198 if 'messages' not in properties:
2199 properties['messages'] = hyperdb.Multilink("msg")
2200 if 'files' not in properties:
2201 properties['files'] = hyperdb.Multilink("file")
2202 if 'nosy' not in properties:
2203 # note: journalling is turned off as it really just wastes
2204 # space. this behaviour may be overridden in an instance
2205 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2206 if 'superseder' not in properties:
2207 properties['superseder'] = hyperdb.Multilink(classname)
2208 Class.__init__(self, db, classname, **properties)
2210 # vim: set et sts=4 sw=4 :