1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 """This module defines a backend that saves the hyperdatabase in a
19 database chosen by anydbm. It is guaranteed to always be available in python
20 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
21 serious bugs, and is not available)
22 """
23 __docformat__ = 'restructuredtext'
25 import os, marshal, re, weakref, string, copy, time, shutil, logging
27 from roundup.anypy.dbm_ import anydbm, whichdb, key_in
29 from roundup import hyperdb, date, password, roundupdb, security, support
30 from roundup.support import reversed
31 from roundup.backends import locking
32 from roundup.i18n import _
34 from roundup.backends.blobfiles import FileStorage
35 from roundup.backends.sessions_dbm import Sessions, OneTimeKeys
37 try:
38 from roundup.backends.indexer_xapian import Indexer
39 except ImportError:
40 from roundup.backends.indexer_dbm import Indexer
42 def db_exists(config):
43 # check for the user db
44 for db in 'nodes.user nodes.user.db'.split():
45 if os.path.exists(os.path.join(config.DATABASE, db)):
46 return 1
47 return 0
49 def db_nuke(config):
50 shutil.rmtree(config.DATABASE)
52 class Binary:
54 def __init__(self, x, y):
55 self.x = x
56 self.y = y
58 def visit(self, visitor):
59 self.x.visit(visitor)
60 self.y.visit(visitor)
62 class Unary:
64 def __init__(self, x):
65 self.x = x
67 def generate(self, atom):
68 return atom(self)
70 def visit(self, visitor):
71 self.x.visit(visitor)
73 class Equals(Unary):
75 def evaluate(self, v):
76 return self.x in v
78 def visit(self, visitor):
79 visitor(self)
81 class Not(Unary):
83 def evaluate(self, v):
84 return not self.x.evaluate(v)
86 def generate(self, atom):
87 return "NOT(%s)" % self.x.generate(atom)
89 class Or(Binary):
91 def evaluate(self, v):
92 return self.x.evaluate(v) or self.y.evaluate(v)
94 def generate(self, atom):
95 return "(%s)OR(%s)" % (
96 self.x.generate(atom),
97 self.y.generate(atom))
99 class And(Binary):
101 def evaluate(self, v):
102 return self.x.evaluate(v) and self.y.evaluate(v)
104 def generate(self, atom):
105 return "(%s)AND(%s)" % (
106 self.x.generate(atom),
107 self.y.generate(atom))
109 def compile_expression(opcodes):
111 stack = []
112 push, pop = stack.append, stack.pop
113 for opcode in opcodes:
114 if opcode == -2: push(Not(pop()))
115 elif opcode == -3: push(And(pop(), pop()))
116 elif opcode == -4: push(Or(pop(), pop()))
117 else: push(Equals(opcode))
119 return pop()
121 class Expression:
123 def __init__(self, v):
124 try:
125 opcodes = [int(x) for x in v]
126 if min(opcodes) >= -1: raise ValueError()
128 compiled = compile_expression(opcodes)
129 self.evaluate = lambda x: compiled.evaluate([int(y) for y in x])
130 except:
131 self.evaluate = lambda x: bool(set(x) & set(v))
133 #
134 # Now the database
135 #
136 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
137 """A database for storing records containing flexible data types.
139 Transaction stuff TODO:
141 - check the timestamp of the class file and nuke the cache if it's
142 modified. Do some sort of conflict checking on the dirty stuff.
143 - perhaps detect write collisions (related to above)?
144 """
145 def __init__(self, config, journaltag=None):
146 """Open a hyperdatabase given a specifier to some storage.
148 The 'storagelocator' is obtained from config.DATABASE.
149 The meaning of 'storagelocator' depends on the particular
150 implementation of the hyperdatabase. It could be a file name,
151 a directory path, a socket descriptor for a connection to a
152 database over the network, etc.
154 The 'journaltag' is a token that will be attached to the journal
155 entries for any edits done on the database. If 'journaltag' is
156 None, the database is opened in read-only mode: the Class.create(),
157 Class.set(), Class.retire(), and Class.restore() methods are
158 disabled.
159 """
160 FileStorage.__init__(self, config.UMASK)
161 self.config, self.journaltag = config, journaltag
162 self.dir = config.DATABASE
163 self.classes = {}
164 self.cache = {} # cache of nodes loaded or created
165 self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0,
166 'filtering': 0}
167 self.dirtynodes = {} # keep track of the dirty nodes by class
168 self.newnodes = {} # keep track of the new nodes by class
169 self.destroyednodes = {}# keep track of the destroyed nodes by class
170 self.transactions = []
171 self.indexer = Indexer(self)
172 self.security = security.Security(self)
173 os.umask(config.UMASK)
175 # lock it
176 lockfilenm = os.path.join(self.dir, 'lock')
177 self.lockfile = locking.acquire_lock(lockfilenm)
178 self.lockfile.write(str(os.getpid()))
179 self.lockfile.flush()
181 def post_init(self):
182 """Called once the schema initialisation has finished.
183 """
184 # reindex the db if necessary
185 if self.indexer.should_reindex():
186 self.reindex()
188 def refresh_database(self):
189 """Rebuild the database
190 """
191 self.reindex()
193 def getSessionManager(self):
194 return Sessions(self)
196 def getOTKManager(self):
197 return OneTimeKeys(self)
199 def reindex(self, classname=None, show_progress=False):
200 if classname:
201 classes = [self.getclass(classname)]
202 else:
203 classes = self.classes.values()
204 for klass in classes:
205 if show_progress:
206 for nodeid in support.Progress('Reindex %s'%klass.classname,
207 klass.list()):
208 klass.index(nodeid)
209 else:
210 for nodeid in klass.list():
211 klass.index(nodeid)
212 self.indexer.save_index()
214 def __repr__(self):
215 return '<back_anydbm instance at %x>'%id(self)
217 #
218 # Classes
219 #
220 def __getattr__(self, classname):
221 """A convenient way of calling self.getclass(classname)."""
222 if classname in self.classes:
223 return self.classes[classname]
224 raise AttributeError, classname
226 def addclass(self, cl):
227 cn = cl.classname
228 if cn in self.classes:
229 raise ValueError, cn
230 self.classes[cn] = cl
232 # add default Edit and View permissions
233 self.security.addPermission(name="Create", klass=cn,
234 description="User is allowed to create "+cn)
235 self.security.addPermission(name="Edit", klass=cn,
236 description="User is allowed to edit "+cn)
237 self.security.addPermission(name="View", klass=cn,
238 description="User is allowed to access "+cn)
239 self.security.addPermission(name="Retire", klass=cn,
240 description="User is allowed to retire "+cn)
242 def getclasses(self):
243 """Return a list of the names of all existing classes."""
244 l = self.classes.keys()
245 l.sort()
246 return l
248 def getclass(self, classname):
249 """Get the Class object representing a particular class.
251 If 'classname' is not a valid class name, a KeyError is raised.
252 """
253 try:
254 return self.classes[classname]
255 except KeyError:
256 raise KeyError('There is no class called "%s"'%classname)
258 #
259 # Class DBs
260 #
261 def clear(self):
262 """Delete all database contents
263 """
264 logging.getLogger('roundup.hyperdb').info('clear')
265 for cn in self.classes:
266 for dummy in 'nodes', 'journals':
267 path = os.path.join(self.dir, 'journals.%s'%cn)
268 if os.path.exists(path):
269 os.remove(path)
270 elif os.path.exists(path+'.db'): # dbm appends .db
271 os.remove(path+'.db')
272 # reset id sequences
273 path = os.path.join(os.getcwd(), self.dir, '_ids')
274 if os.path.exists(path):
275 os.remove(path)
276 elif os.path.exists(path+'.db'): # dbm appends .db
277 os.remove(path+'.db')
279 def getclassdb(self, classname, mode='r'):
280 """ grab a connection to the class db that will be used for
281 multiple actions
282 """
283 return self.opendb('nodes.%s'%classname, mode)
285 def determine_db_type(self, path):
286 """ determine which DB wrote the class file
287 """
288 db_type = ''
289 if os.path.exists(path):
290 db_type = whichdb(path)
291 if not db_type:
292 raise hyperdb.DatabaseError(_("Couldn't identify database type"))
293 elif os.path.exists(path+'.db'):
294 # if the path ends in '.db', it's a dbm database, whether
295 # anydbm says it's dbhash or not!
296 db_type = 'dbm'
297 return db_type
299 def opendb(self, name, mode):
300 """Low-level database opener that gets around anydbm/dbm
301 eccentricities.
302 """
303 # figure the class db type
304 path = os.path.join(os.getcwd(), self.dir, name)
305 db_type = self.determine_db_type(path)
307 # new database? let anydbm pick the best dbm
308 # in Python 3+ the "dbm" ("anydbm" to us) module already uses the
309 # whichdb() function to do this
310 if not db_type or hasattr(anydbm, 'whichdb'):
311 if __debug__:
312 logging.getLogger('roundup.hyperdb').debug(
313 "opendb anydbm.open(%r, 'c')"%path)
314 return anydbm.open(path, 'c')
316 # in Python <3 it anydbm was a little dumb so manually open the
317 # database with the correct module
318 try:
319 dbm = __import__(db_type)
320 except ImportError:
321 raise hyperdb.DatabaseError(_("Couldn't open database - the "
322 "required module '%s' is not available")%db_type)
323 if __debug__:
324 logging.getLogger('roundup.hyperdb').debug(
325 "opendb %r.open(%r, %r)"%(db_type, path, mode))
326 return dbm.open(path, mode)
328 #
329 # Node IDs
330 #
331 def newid(self, classname):
332 """ Generate a new id for the given class
333 """
334 # open the ids DB - create if if doesn't exist
335 db = self.opendb('_ids', 'c')
336 if key_in(db, classname):
337 newid = db[classname] = str(int(db[classname]) + 1)
338 else:
339 # the count() bit is transitional - older dbs won't start at 1
340 newid = str(self.getclass(classname).count()+1)
341 db[classname] = newid
342 db.close()
343 return newid
345 def setid(self, classname, setid):
346 """ Set the id counter: used during import of database
347 """
348 # open the ids DB - create if if doesn't exist
349 db = self.opendb('_ids', 'c')
350 db[classname] = str(setid)
351 db.close()
353 #
354 # Nodes
355 #
356 def addnode(self, classname, nodeid, node):
357 """ add the specified node to its class's db
358 """
359 # we'll be supplied these props if we're doing an import
360 if 'creator' not in node:
361 # add in the "calculated" properties (dupe so we don't affect
362 # calling code's node assumptions)
363 node = node.copy()
364 node['creator'] = self.getuid()
365 node['actor'] = self.getuid()
366 node['creation'] = node['activity'] = date.Date()
368 self.newnodes.setdefault(classname, {})[nodeid] = 1
369 self.cache.setdefault(classname, {})[nodeid] = node
370 self.savenode(classname, nodeid, node)
372 def setnode(self, classname, nodeid, node):
373 """ change the specified node
374 """
375 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
377 # can't set without having already loaded the node
378 self.cache[classname][nodeid] = node
379 self.savenode(classname, nodeid, node)
381 def savenode(self, classname, nodeid, node):
382 """ perform the saving of data specified by the set/addnode
383 """
384 if __debug__:
385 logging.getLogger('roundup.hyperdb').debug(
386 'save %s%s %r'%(classname, nodeid, node))
387 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
389 def getnode(self, classname, nodeid, db=None, cache=1):
390 """ get a node from the database
392 Note the "cache" parameter is not used, and exists purely for
393 backward compatibility!
394 """
395 # try the cache
396 cache_dict = self.cache.setdefault(classname, {})
397 if nodeid in cache_dict:
398 if __debug__:
399 logging.getLogger('roundup.hyperdb').debug(
400 'get %s%s cached'%(classname, nodeid))
401 self.stats['cache_hits'] += 1
402 return cache_dict[nodeid]
404 if __debug__:
405 self.stats['cache_misses'] += 1
406 start_t = time.time()
407 logging.getLogger('roundup.hyperdb').debug(
408 'get %s%s'%(classname, nodeid))
410 # get from the database and save in the cache
411 if db is None:
412 db = self.getclassdb(classname)
413 if not key_in(db, nodeid):
414 raise IndexError("no such %s %s"%(classname, nodeid))
416 # check the uncommitted, destroyed nodes
417 if (classname in self.destroyednodes and
418 nodeid in self.destroyednodes[classname]):
419 raise IndexError("no such %s %s"%(classname, nodeid))
421 # decode
422 res = marshal.loads(db[nodeid])
424 # reverse the serialisation
425 res = self.unserialise(classname, res)
427 # store off in the cache dict
428 if cache:
429 cache_dict[nodeid] = res
431 if __debug__:
432 self.stats['get_items'] += (time.time() - start_t)
434 return res
436 def destroynode(self, classname, nodeid):
437 """Remove a node from the database. Called exclusively by the
438 destroy() method on Class.
439 """
440 logging.getLogger('roundup.hyperdb').info(
441 'destroy %s%s'%(classname, nodeid))
443 # remove from cache and newnodes if it's there
444 if (classname in self.cache and nodeid in self.cache[classname]):
445 del self.cache[classname][nodeid]
446 if (classname in self.newnodes and nodeid in self.newnodes[classname]):
447 del self.newnodes[classname][nodeid]
449 # see if there's any obvious commit actions that we should get rid of
450 for entry in self.transactions[:]:
451 if entry[1][:2] == (classname, nodeid):
452 self.transactions.remove(entry)
454 # add to the destroyednodes map
455 self.destroyednodes.setdefault(classname, {})[nodeid] = 1
457 # add the destroy commit action
458 self.transactions.append((self.doDestroyNode, (classname, nodeid)))
459 self.transactions.append((FileStorage.destroy, (self, classname, nodeid)))
461 def serialise(self, classname, node):
462 """Copy the node contents, converting non-marshallable data into
463 marshallable data.
464 """
465 properties = self.getclass(classname).getprops()
466 d = {}
467 for k, v in node.iteritems():
468 if k == self.RETIRED_FLAG:
469 d[k] = v
470 continue
472 # if the property doesn't exist then we really don't care
473 if k not in properties:
474 continue
476 # get the property spec
477 prop = properties[k]
479 if isinstance(prop, hyperdb.Password) and v is not None:
480 d[k] = str(v)
481 elif isinstance(prop, hyperdb.Date) and v is not None:
482 d[k] = v.serialise()
483 elif isinstance(prop, hyperdb.Interval) and v is not None:
484 d[k] = v.serialise()
485 else:
486 d[k] = v
487 return d
489 def unserialise(self, classname, node):
490 """Decode the marshalled node data
491 """
492 properties = self.getclass(classname).getprops()
493 d = {}
494 for k, v in node.iteritems():
495 # if the property doesn't exist, or is the "retired" flag then
496 # it won't be in the properties dict
497 if k not in properties:
498 d[k] = v
499 continue
501 # get the property spec
502 prop = properties[k]
504 if isinstance(prop, hyperdb.Date) and v is not None:
505 d[k] = date.Date(v)
506 elif isinstance(prop, hyperdb.Interval) and v is not None:
507 d[k] = date.Interval(v)
508 elif isinstance(prop, hyperdb.Password) and v is not None:
509 d[k] = password.Password(encrypted=v)
510 else:
511 d[k] = v
512 return d
514 def hasnode(self, classname, nodeid, db=None):
515 """ determine if the database has a given node
516 """
517 # try the cache
518 cache = self.cache.setdefault(classname, {})
519 if nodeid in cache:
520 return 1
522 # not in the cache - check the database
523 if db is None:
524 db = self.getclassdb(classname)
525 return key_in(db, nodeid)
527 def countnodes(self, classname, db=None):
528 count = 0
530 # include the uncommitted nodes
531 if classname in self.newnodes:
532 count += len(self.newnodes[classname])
533 if classname in self.destroyednodes:
534 count -= len(self.destroyednodes[classname])
536 # and count those in the DB
537 if db is None:
538 db = self.getclassdb(classname)
539 return count + len(db)
542 #
543 # Files - special node properties
544 # inherited from FileStorage
546 #
547 # Journal
548 #
549 def addjournal(self, classname, nodeid, action, params, creator=None,
550 creation=None):
551 """ Journal the Action
552 'action' may be:
554 'create' or 'set' -- 'params' is a dictionary of property values
555 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
556 'retire' -- 'params' is None
558 'creator' -- the user performing the action, which defaults to
559 the current user.
560 """
561 if __debug__:
562 logging.getLogger('roundup.hyperdb').debug(
563 'addjournal %s%s %s %r %s %r'%(classname,
564 nodeid, action, params, creator, creation))
565 if creator is None:
566 creator = self.getuid()
567 self.transactions.append((self.doSaveJournal, (classname, nodeid,
568 action, params, creator, creation)))
570 def setjournal(self, classname, nodeid, journal):
571 """Set the journal to the "journal" list."""
572 if __debug__:
573 logging.getLogger('roundup.hyperdb').debug(
574 'setjournal %s%s %r'%(classname, nodeid, journal))
575 self.transactions.append((self.doSetJournal, (classname, nodeid,
576 journal)))
578 def fix_journal(self, classname, journal):
579 """ fix password entries to correct type """
580 pwprops = {}
581 for pn, prop in self.getclass(classname).properties.iteritems():
582 if isinstance(prop, hyperdb.Password):
583 pwprops [pn] = 1
584 if not pwprops:
585 return journal
586 for j in journal:
587 if j[3] == 'set':
588 for k, v in j[4].items():
589 if k in pwprops and j[4][k]:
590 j[4][k] = password.JournalPassword(j[4][k])
591 return journal
593 def getjournal(self, classname, nodeid):
594 """ get the journal for id
596 Raise IndexError if the node doesn't exist (as per history()'s
597 API)
598 """
599 # our journal result
600 res = []
602 # add any journal entries for transactions not committed to the
603 # database
604 for method, args in self.transactions:
605 if method != self.doSaveJournal:
606 continue
607 (cache_classname, cache_nodeid, cache_action, cache_params,
608 cache_creator, cache_creation) = args
609 if cache_classname == classname and cache_nodeid == nodeid:
610 if not cache_creator:
611 cache_creator = self.getuid()
612 if not cache_creation:
613 cache_creation = date.Date()
614 res.append((cache_nodeid, cache_creation, cache_creator,
615 cache_action, cache_params))
617 # attempt to open the journal - in some rare cases, the journal may
618 # not exist
619 try:
620 db = self.opendb('journals.%s'%classname, 'r')
621 except anydbm.error, error:
622 if str(error) == "need 'c' or 'n' flag to open new db":
623 raise IndexError('no such %s %s'%(classname, nodeid))
624 elif error.args[0] != 2:
625 # this isn't a "not found" error, be alarmed!
626 raise
627 if res:
628 # we have unsaved journal entries, return them
629 return self.fix_journal (classname, res)
630 raise IndexError('no such %s %s'%(classname, nodeid))
631 try:
632 journal = marshal.loads(db[nodeid])
633 except KeyError:
634 db.close()
635 if res:
636 # we have some unsaved journal entries, be happy!
637 return self.fix_journal (classname, res)
638 raise IndexError('no such %s %s'%(classname, nodeid))
639 db.close()
641 # add all the saved journal entries for this node
642 for nodeid, date_stamp, user, action, params in journal:
643 res.append((nodeid, date.Date(date_stamp), user, action, params))
644 return self.fix_journal (classname, res)
646 def pack(self, pack_before):
647 """ Delete all journal entries except "create" before 'pack_before'.
648 """
649 pack_before = pack_before.serialise()
650 for classname in self.getclasses():
651 packed = 0
652 # get the journal db
653 db_name = 'journals.%s'%classname
654 path = os.path.join(os.getcwd(), self.dir, classname)
655 db_type = self.determine_db_type(path)
656 db = self.opendb(db_name, 'w')
658 for key in db.keys():
659 # get the journal for this db entry
660 journal = marshal.loads(db[key])
661 l = []
662 last_set_entry = None
663 for entry in journal:
664 # unpack the entry
665 (nodeid, date_stamp, self.journaltag, action,
666 params) = entry
667 # if the entry is after the pack date, _or_ the initial
668 # create entry, then it stays
669 if date_stamp > pack_before or action == 'create':
670 l.append(entry)
671 else:
672 packed += 1
673 db[key] = marshal.dumps(l)
675 logging.getLogger('roundup.hyperdb').info(
676 'packed %d %s items'%(packed, classname))
678 if db_type == 'gdbm':
679 db.reorganize()
680 db.close()
683 #
684 # Basic transaction support
685 #
686 def commit(self, fail_ok=False):
687 """ Commit the current transactions.
689 Save all data changed since the database was opened or since the
690 last commit() or rollback().
692 fail_ok indicates that the commit is allowed to fail. This is used
693 in the web interface when committing cleaning of the session
694 database. We don't care if there's a concurrency issue there.
696 The only backend this seems to affect is postgres.
697 """
698 logging.getLogger('roundup.hyperdb').info('commit %s transactions'%(
699 len(self.transactions)))
701 # keep a handle to all the database files opened
702 self.databases = {}
704 try:
705 # now, do all the transactions
706 reindex = {}
707 for method, args in self.transactions:
708 reindex[method(*args)] = 1
709 finally:
710 # make sure we close all the database files
711 for db in self.databases.itervalues():
712 db.close()
713 del self.databases
715 # clear the transactions list now so the blobfile implementation
716 # doesn't think there's still pending file commits when it tries
717 # to access the file data
718 self.transactions = []
720 # reindex the nodes that request it
721 for classname, nodeid in [k for k in reindex if k]:
722 self.getclass(classname).index(nodeid)
724 # save the indexer state
725 self.indexer.save_index()
727 self.clearCache()
729 def clearCache(self):
730 # all transactions committed, back to normal
731 self.cache = {}
732 self.dirtynodes = {}
733 self.newnodes = {}
734 self.destroyednodes = {}
735 self.transactions = []
737 def getCachedClassDB(self, classname):
738 """ get the class db, looking in our cache of databases for commit
739 """
740 # get the database handle
741 db_name = 'nodes.%s'%classname
742 if db_name not in self.databases:
743 self.databases[db_name] = self.getclassdb(classname, 'c')
744 return self.databases[db_name]
746 def doSaveNode(self, classname, nodeid, node):
747 db = self.getCachedClassDB(classname)
749 # now save the marshalled data
750 db[nodeid] = marshal.dumps(self.serialise(classname, node))
752 # return the classname, nodeid so we reindex this content
753 return (classname, nodeid)
755 def getCachedJournalDB(self, classname):
756 """ get the journal db, looking in our cache of databases for commit
757 """
758 # get the database handle
759 db_name = 'journals.%s'%classname
760 if db_name not in self.databases:
761 self.databases[db_name] = self.opendb(db_name, 'c')
762 return self.databases[db_name]
764 def doSaveJournal(self, classname, nodeid, action, params, creator,
765 creation):
766 # serialise the parameters now if necessary
767 if isinstance(params, type({})):
768 if action in ('set', 'create'):
769 params = self.serialise(classname, params)
771 # handle supply of the special journalling parameters (usually
772 # supplied on importing an existing database)
773 journaltag = creator
774 if creation:
775 journaldate = creation.serialise()
776 else:
777 journaldate = date.Date().serialise()
779 # create the journal entry
780 entry = (nodeid, journaldate, journaltag, action, params)
782 db = self.getCachedJournalDB(classname)
784 # now insert the journal entry
785 if key_in(db, nodeid):
786 # append to existing
787 s = db[nodeid]
788 l = marshal.loads(s)
789 l.append(entry)
790 else:
791 l = [entry]
793 db[nodeid] = marshal.dumps(l)
795 def doSetJournal(self, classname, nodeid, journal):
796 l = []
797 for nodeid, journaldate, journaltag, action, params in journal:
798 # serialise the parameters now if necessary
799 if isinstance(params, type({})):
800 if action in ('set', 'create'):
801 params = self.serialise(classname, params)
802 journaldate = journaldate.serialise()
803 l.append((nodeid, journaldate, journaltag, action, params))
804 db = self.getCachedJournalDB(classname)
805 db[nodeid] = marshal.dumps(l)
807 def doDestroyNode(self, classname, nodeid):
808 # delete from the class database
809 db = self.getCachedClassDB(classname)
810 if key_in(db, nodeid):
811 del db[nodeid]
813 # delete from the database
814 db = self.getCachedJournalDB(classname)
815 if key_in(db, nodeid):
816 del db[nodeid]
818 def rollback(self):
819 """ Reverse all actions from the current transaction.
820 """
821 logging.getLogger('roundup.hyperdb').info('rollback %s transactions'%(
822 len(self.transactions)))
824 for method, args in self.transactions:
825 # delete temporary files
826 if method == self.doStoreFile:
827 self.rollbackStoreFile(*args)
828 self.cache = {}
829 self.dirtynodes = {}
830 self.newnodes = {}
831 self.destroyednodes = {}
832 self.transactions = []
834 def close(self):
835 """ Nothing to do
836 """
837 if self.lockfile is not None:
838 locking.release_lock(self.lockfile)
839 self.lockfile.close()
840 self.lockfile = None
842 _marker = []
843 class Class(hyperdb.Class):
844 """The handle to a particular class of nodes in a hyperdatabase."""
846 def enableJournalling(self):
847 """Turn journalling on for this class
848 """
849 self.do_journal = 1
851 def disableJournalling(self):
852 """Turn journalling off for this class
853 """
854 self.do_journal = 0
856 # Editing nodes:
858 def create(self, **propvalues):
859 """Create a new node of this class and return its id.
861 The keyword arguments in 'propvalues' map property names to values.
863 The values of arguments must be acceptable for the types of their
864 corresponding properties or a TypeError is raised.
866 If this class has a key property, it must be present and its value
867 must not collide with other key strings or a ValueError is raised.
869 Any other properties on this class that are missing from the
870 'propvalues' dictionary are set to None.
872 If an id in a link or multilink property does not refer to a valid
873 node, an IndexError is raised.
875 These operations trigger detectors and can be vetoed. Attempts
876 to modify the "creation" or "activity" properties cause a KeyError.
877 """
878 if self.db.journaltag is None:
879 raise hyperdb.DatabaseError(_('Database open read-only'))
880 self.fireAuditors('create', None, propvalues)
881 newid = self.create_inner(**propvalues)
882 self.fireReactors('create', newid, None)
883 return newid
885 def create_inner(self, **propvalues):
886 """ Called by create, in-between the audit and react calls.
887 """
888 if 'id' in propvalues:
889 raise KeyError('"id" is reserved')
891 if self.db.journaltag is None:
892 raise hyperdb.DatabaseError(_('Database open read-only'))
894 if 'creation' in propvalues or 'activity' in propvalues:
895 raise KeyError('"creation" and "activity" are reserved')
896 # new node's id
897 newid = self.db.newid(self.classname)
899 # validate propvalues
900 num_re = re.compile('^\d+$')
901 for key, value in propvalues.iteritems():
902 if key == self.key:
903 try:
904 self.lookup(value)
905 except KeyError:
906 pass
907 else:
908 raise ValueError('node with key "%s" exists'%value)
910 # try to handle this property
911 try:
912 prop = self.properties[key]
913 except KeyError:
914 raise KeyError('"%s" has no property "%s"'%(self.classname,
915 key))
917 if value is not None and isinstance(prop, hyperdb.Link):
918 if type(value) != type(''):
919 raise ValueError('link value must be String')
920 link_class = self.properties[key].classname
921 # if it isn't a number, it's a key
922 if not num_re.match(value):
923 try:
924 value = self.db.classes[link_class].lookup(value)
925 except (TypeError, KeyError):
926 raise IndexError('new property "%s": %s not a %s'%(
927 key, value, link_class))
928 elif not self.db.getclass(link_class).hasnode(value):
929 raise IndexError('%s has no node %s'%(link_class,
930 value))
932 # save off the value
933 propvalues[key] = value
935 # register the link with the newly linked node
936 if self.do_journal and self.properties[key].do_journal:
937 self.db.addjournal(link_class, value, 'link',
938 (self.classname, newid, key))
940 elif isinstance(prop, hyperdb.Multilink):
941 if value is None:
942 value = []
943 if not hasattr(value, '__iter__'):
944 raise TypeError('new property "%s" not an iterable of ids'%key)
946 # clean up and validate the list of links
947 link_class = self.properties[key].classname
948 l = []
949 for entry in value:
950 if type(entry) != type(''):
951 raise ValueError('"%s" multilink value (%r) '\
952 'must contain Strings'%(key, value))
953 # if it isn't a number, it's a key
954 if not num_re.match(entry):
955 try:
956 entry = self.db.classes[link_class].lookup(entry)
957 except (TypeError, KeyError):
958 raise IndexError('new property "%s": %s not a %s'%(
959 key, entry, self.properties[key].classname))
960 l.append(entry)
961 value = l
962 propvalues[key] = value
964 # handle additions
965 for nodeid in value:
966 if not self.db.getclass(link_class).hasnode(nodeid):
967 raise IndexError('%s has no node %s'%(link_class,
968 nodeid))
969 # register the link with the newly linked node
970 if self.do_journal and self.properties[key].do_journal:
971 self.db.addjournal(link_class, nodeid, 'link',
972 (self.classname, newid, key))
974 elif isinstance(prop, hyperdb.String):
975 if type(value) != type('') and type(value) != type(u''):
976 raise TypeError('new property "%s" not a string'%key)
977 if prop.indexme:
978 self.db.indexer.add_text((self.classname, newid, key),
979 value)
981 elif isinstance(prop, hyperdb.Password):
982 if not isinstance(value, password.Password):
983 raise TypeError('new property "%s" not a Password'%key)
985 elif isinstance(prop, hyperdb.Date):
986 if value is not None and not isinstance(value, date.Date):
987 raise TypeError('new property "%s" not a Date'%key)
989 elif isinstance(prop, hyperdb.Interval):
990 if value is not None and not isinstance(value, date.Interval):
991 raise TypeError('new property "%s" not an Interval'%key)
993 elif value is not None and isinstance(prop, hyperdb.Number):
994 try:
995 float(value)
996 except ValueError:
997 raise TypeError('new property "%s" not numeric'%key)
999 elif value is not None and isinstance(prop, hyperdb.Boolean):
1000 try:
1001 int(value)
1002 except ValueError:
1003 raise TypeError('new property "%s" not boolean'%key)
1005 # make sure there's data where there needs to be
1006 for key, prop in self.properties.iteritems():
1007 if key in propvalues:
1008 continue
1009 if key == self.key:
1010 raise ValueError('key property "%s" is required'%key)
1011 if isinstance(prop, hyperdb.Multilink):
1012 propvalues[key] = []
1014 # done
1015 self.db.addnode(self.classname, newid, propvalues)
1016 if self.do_journal:
1017 self.db.addjournal(self.classname, newid, 'create', {})
1019 return newid
1021 def get(self, nodeid, propname, default=_marker, cache=1):
1022 """Get the value of a property on an existing node of this class.
1024 'nodeid' must be the id of an existing node of this class or an
1025 IndexError is raised. 'propname' must be the name of a property
1026 of this class or a KeyError is raised.
1028 'cache' exists for backward compatibility, and is not used.
1030 Attempts to get the "creation" or "activity" properties should
1031 do the right thing.
1032 """
1033 if propname == 'id':
1034 return nodeid
1036 # get the node's dict
1037 d = self.db.getnode(self.classname, nodeid)
1039 # check for one of the special props
1040 if propname == 'creation':
1041 if 'creation' in d:
1042 return d['creation']
1043 if not self.do_journal:
1044 raise ValueError('Journalling is disabled for this class')
1045 journal = self.db.getjournal(self.classname, nodeid)
1046 if journal:
1047 return journal[0][1]
1048 else:
1049 # on the strange chance that there's no journal
1050 return date.Date()
1051 if propname == 'activity':
1052 if 'activity' in d:
1053 return d['activity']
1054 if not self.do_journal:
1055 raise ValueError('Journalling is disabled for this class')
1056 journal = self.db.getjournal(self.classname, nodeid)
1057 if journal:
1058 return self.db.getjournal(self.classname, nodeid)[-1][1]
1059 else:
1060 # on the strange chance that there's no journal
1061 return date.Date()
1062 if propname == 'creator':
1063 if 'creator' in d:
1064 return d['creator']
1065 if not self.do_journal:
1066 raise ValueError('Journalling is disabled for this class')
1067 journal = self.db.getjournal(self.classname, nodeid)
1068 if journal:
1069 num_re = re.compile('^\d+$')
1070 value = journal[0][2]
1071 if num_re.match(value):
1072 return value
1073 else:
1074 # old-style "username" journal tag
1075 try:
1076 return self.db.user.lookup(value)
1077 except KeyError:
1078 # user's been retired, return admin
1079 return '1'
1080 else:
1081 return self.db.getuid()
1082 if propname == 'actor':
1083 if 'actor' in d:
1084 return d['actor']
1085 if not self.do_journal:
1086 raise ValueError('Journalling is disabled for this class')
1087 journal = self.db.getjournal(self.classname, nodeid)
1088 if journal:
1089 num_re = re.compile('^\d+$')
1090 value = journal[-1][2]
1091 if num_re.match(value):
1092 return value
1093 else:
1094 # old-style "username" journal tag
1095 try:
1096 return self.db.user.lookup(value)
1097 except KeyError:
1098 # user's been retired, return admin
1099 return '1'
1100 else:
1101 return self.db.getuid()
1103 # get the property (raises KeyErorr if invalid)
1104 prop = self.properties[propname]
1106 if propname not in d:
1107 if default is _marker:
1108 if isinstance(prop, hyperdb.Multilink):
1109 return []
1110 else:
1111 return None
1112 else:
1113 return default
1115 # return a dupe of the list so code doesn't get confused
1116 if isinstance(prop, hyperdb.Multilink):
1117 return d[propname][:]
1119 return d[propname]
1121 def set(self, nodeid, **propvalues):
1122 """Modify a property on an existing node of this class.
1124 'nodeid' must be the id of an existing node of this class or an
1125 IndexError is raised.
1127 Each key in 'propvalues' must be the name of a property of this
1128 class or a KeyError is raised.
1130 All values in 'propvalues' must be acceptable types for their
1131 corresponding properties or a TypeError is raised.
1133 If the value of the key property is set, it must not collide with
1134 other key strings or a ValueError is raised.
1136 If the value of a Link or Multilink property contains an invalid
1137 node id, a ValueError is raised.
1139 These operations trigger detectors and can be vetoed. Attempts
1140 to modify the "creation" or "activity" properties cause a KeyError.
1141 """
1142 if self.db.journaltag is None:
1143 raise hyperdb.DatabaseError(_('Database open read-only'))
1145 self.fireAuditors('set', nodeid, propvalues)
1146 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1147 for name, prop in self.getprops(protected=0).iteritems():
1148 if name in oldvalues:
1149 continue
1150 if isinstance(prop, hyperdb.Multilink):
1151 oldvalues[name] = []
1152 else:
1153 oldvalues[name] = None
1154 propvalues = self.set_inner(nodeid, **propvalues)
1155 self.fireReactors('set', nodeid, oldvalues)
1156 return propvalues
1158 def set_inner(self, nodeid, **propvalues):
1159 """ Called by set, in-between the audit and react calls.
1160 """
1161 if not propvalues:
1162 return propvalues
1164 if 'creation' in propvalues or 'activity' in propvalues:
1165 raise KeyError, '"creation" and "activity" are reserved'
1167 if 'id' in propvalues:
1168 raise KeyError, '"id" is reserved'
1170 if self.db.journaltag is None:
1171 raise hyperdb.DatabaseError(_('Database open read-only'))
1173 node = self.db.getnode(self.classname, nodeid)
1174 if self.db.RETIRED_FLAG in node:
1175 raise IndexError
1176 num_re = re.compile('^\d+$')
1178 # if the journal value is to be different, store it in here
1179 journalvalues = {}
1181 # list() propvalues 'cos it might be modified by the loop
1182 for propname, value in list(propvalues.items()):
1183 # check to make sure we're not duplicating an existing key
1184 if propname == self.key and node[propname] != value:
1185 try:
1186 self.lookup(value)
1187 except KeyError:
1188 pass
1189 else:
1190 raise ValueError('node with key "%s" exists'%value)
1192 # this will raise the KeyError if the property isn't valid
1193 # ... we don't use getprops() here because we only care about
1194 # the writeable properties.
1195 try:
1196 prop = self.properties[propname]
1197 except KeyError:
1198 raise KeyError('"%s" has no property named "%s"'%(
1199 self.classname, propname))
1201 # if the value's the same as the existing value, no sense in
1202 # doing anything
1203 current = node.get(propname, None)
1204 if value == current:
1205 del propvalues[propname]
1206 continue
1207 journalvalues[propname] = current
1209 # do stuff based on the prop type
1210 if isinstance(prop, hyperdb.Link):
1211 link_class = prop.classname
1212 # if it isn't a number, it's a key
1213 if value is not None and not isinstance(value, type('')):
1214 raise ValueError('property "%s" link value be a string'%(
1215 propname))
1216 if isinstance(value, type('')) and not num_re.match(value):
1217 try:
1218 value = self.db.classes[link_class].lookup(value)
1219 except (TypeError, KeyError):
1220 raise IndexError('new property "%s": %s not a %s'%(
1221 propname, value, prop.classname))
1223 if (value is not None and
1224 not self.db.getclass(link_class).hasnode(value)):
1225 raise IndexError('%s has no node %s'%(link_class,
1226 value))
1228 if self.do_journal and prop.do_journal:
1229 # register the unlink with the old linked node
1230 if propname in node and node[propname] is not None:
1231 self.db.addjournal(link_class, node[propname], 'unlink',
1232 (self.classname, nodeid, propname))
1234 # register the link with the newly linked node
1235 if value is not None:
1236 self.db.addjournal(link_class, value, 'link',
1237 (self.classname, nodeid, propname))
1239 elif isinstance(prop, hyperdb.Multilink):
1240 if value is None:
1241 value = []
1242 if not hasattr(value, '__iter__'):
1243 raise TypeError('new property "%s" not an iterable of'
1244 ' ids'%propname)
1245 link_class = self.properties[propname].classname
1246 l = []
1247 for entry in value:
1248 # if it isn't a number, it's a key
1249 if type(entry) != type(''):
1250 raise ValueError('new property "%s" link value '
1251 'must be a string'%propname)
1252 if not num_re.match(entry):
1253 try:
1254 entry = self.db.classes[link_class].lookup(entry)
1255 except (TypeError, KeyError):
1256 raise IndexError('new property "%s": %s not a %s'%(
1257 propname, entry,
1258 self.properties[propname].classname))
1259 l.append(entry)
1260 value = l
1261 propvalues[propname] = value
1263 # figure the journal entry for this property
1264 add = []
1265 remove = []
1267 # handle removals
1268 if propname in node:
1269 l = node[propname]
1270 else:
1271 l = []
1272 for id in l[:]:
1273 if id in value:
1274 continue
1275 # register the unlink with the old linked node
1276 if self.do_journal and self.properties[propname].do_journal:
1277 self.db.addjournal(link_class, id, 'unlink',
1278 (self.classname, nodeid, propname))
1279 l.remove(id)
1280 remove.append(id)
1282 # handle additions
1283 for id in value:
1284 if not self.db.getclass(link_class).hasnode(id):
1285 raise IndexError('%s has no node %s'%(link_class,
1286 id))
1287 if id in l:
1288 continue
1289 # register the link with the newly linked node
1290 if self.do_journal and self.properties[propname].do_journal:
1291 self.db.addjournal(link_class, id, 'link',
1292 (self.classname, nodeid, propname))
1293 l.append(id)
1294 add.append(id)
1296 # figure the journal entry
1297 l = []
1298 if add:
1299 l.append(('+', add))
1300 if remove:
1301 l.append(('-', remove))
1302 if l:
1303 journalvalues[propname] = tuple(l)
1305 elif isinstance(prop, hyperdb.String):
1306 if value is not None and type(value) != type('') and type(value) != type(u''):
1307 raise TypeError('new property "%s" not a '
1308 'string'%propname)
1309 if prop.indexme:
1310 self.db.indexer.add_text((self.classname, nodeid, propname),
1311 value)
1313 elif isinstance(prop, hyperdb.Password):
1314 if not isinstance(value, password.Password):
1315 raise TypeError('new property "%s" not a '
1316 'Password'%propname)
1317 propvalues[propname] = value
1318 journalvalues[propname] = \
1319 current and password.JournalPassword(current)
1321 elif value is not None and isinstance(prop, hyperdb.Date):
1322 if not isinstance(value, date.Date):
1323 raise TypeError('new property "%s" not a '
1324 'Date'%propname)
1325 propvalues[propname] = value
1327 elif value is not None and isinstance(prop, hyperdb.Interval):
1328 if not isinstance(value, date.Interval):
1329 raise TypeError('new property "%s" not an '
1330 'Interval'%propname)
1331 propvalues[propname] = value
1333 elif value is not None and isinstance(prop, hyperdb.Number):
1334 try:
1335 float(value)
1336 except ValueError:
1337 raise TypeError('new property "%s" not '
1338 'numeric'%propname)
1340 elif value is not None and isinstance(prop, hyperdb.Boolean):
1341 try:
1342 int(value)
1343 except ValueError:
1344 raise TypeError('new property "%s" not '
1345 'boolean'%propname)
1347 node[propname] = value
1349 # nothing to do?
1350 if not propvalues:
1351 return propvalues
1353 # update the activity time
1354 node['activity'] = date.Date()
1355 node['actor'] = self.db.getuid()
1357 # do the set, and journal it
1358 self.db.setnode(self.classname, nodeid, node)
1360 if self.do_journal:
1361 self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
1363 return propvalues
1365 def retire(self, nodeid):
1366 """Retire a node.
1368 The properties on the node remain available from the get() method,
1369 and the node's id is never reused.
1371 Retired nodes are not returned by the find(), list(), or lookup()
1372 methods, and other nodes may reuse the values of their key properties.
1374 These operations trigger detectors and can be vetoed. Attempts
1375 to modify the "creation" or "activity" properties cause a KeyError.
1376 """
1377 if self.db.journaltag is None:
1378 raise hyperdb.DatabaseError(_('Database open read-only'))
1380 self.fireAuditors('retire', nodeid, None)
1382 node = self.db.getnode(self.classname, nodeid)
1383 node[self.db.RETIRED_FLAG] = 1
1384 self.db.setnode(self.classname, nodeid, node)
1385 if self.do_journal:
1386 self.db.addjournal(self.classname, nodeid, 'retired', None)
1388 self.fireReactors('retire', nodeid, None)
1390 def restore(self, nodeid):
1391 """Restpre a retired node.
1393 Make node available for all operations like it was before retirement.
1394 """
1395 if self.db.journaltag is None:
1396 raise hyperdb.DatabaseError(_('Database open read-only'))
1398 node = self.db.getnode(self.classname, nodeid)
1399 # check if key property was overrided
1400 key = self.getkey()
1401 try:
1402 id = self.lookup(node[key])
1403 except KeyError:
1404 pass
1405 else:
1406 raise KeyError("Key property (%s) of retired node clashes "
1407 "with existing one (%s)" % (key, node[key]))
1408 # Now we can safely restore node
1409 self.fireAuditors('restore', nodeid, None)
1410 del node[self.db.RETIRED_FLAG]
1411 self.db.setnode(self.classname, nodeid, node)
1412 if self.do_journal:
1413 self.db.addjournal(self.classname, nodeid, 'restored', None)
1415 self.fireReactors('restore', nodeid, None)
1417 def is_retired(self, nodeid, cldb=None):
1418 """Return true if the node is retired.
1419 """
1420 node = self.db.getnode(self.classname, nodeid, cldb)
1421 if self.db.RETIRED_FLAG in node:
1422 return 1
1423 return 0
1425 def destroy(self, nodeid):
1426 """Destroy a node.
1428 WARNING: this method should never be used except in extremely rare
1429 situations where there could never be links to the node being
1430 deleted
1432 WARNING: use retire() instead
1434 WARNING: the properties of this node will not be available ever again
1436 WARNING: really, use retire() instead
1438 Well, I think that's enough warnings. This method exists mostly to
1439 support the session storage of the cgi interface.
1440 """
1441 if self.db.journaltag is None:
1442 raise hyperdb.DatabaseError(_('Database open read-only'))
1443 self.db.destroynode(self.classname, nodeid)
1445 # Locating nodes:
1446 def hasnode(self, nodeid):
1447 """Determine if the given nodeid actually exists
1448 """
1449 return self.db.hasnode(self.classname, nodeid)
1451 def setkey(self, propname):
1452 """Select a String property of this class to be the key property.
1454 'propname' must be the name of a String property of this class or
1455 None, or a TypeError is raised. The values of the key property on
1456 all existing nodes must be unique or a ValueError is raised. If the
1457 property doesn't exist, KeyError is raised.
1458 """
1459 prop = self.getprops()[propname]
1460 if not isinstance(prop, hyperdb.String):
1461 raise TypeError('key properties must be String')
1462 self.key = propname
1464 def getkey(self):
1465 """Return the name of the key property for this class or None."""
1466 return self.key
1468 # TODO: set up a separate index db file for this? profile?
1469 def lookup(self, keyvalue):
1470 """Locate a particular node by its key property and return its id.
1472 If this class has no key property, a TypeError is raised. If the
1473 'keyvalue' matches one of the values for the key property among
1474 the nodes in this class, the matching node's id is returned;
1475 otherwise a KeyError is raised.
1476 """
1477 if not self.key:
1478 raise TypeError('No key property set for '
1479 'class %s'%self.classname)
1480 cldb = self.db.getclassdb(self.classname)
1481 try:
1482 for nodeid in self.getnodeids(cldb):
1483 node = self.db.getnode(self.classname, nodeid, cldb)
1484 if self.db.RETIRED_FLAG in node:
1485 continue
1486 if self.key not in node:
1487 continue
1488 if node[self.key] == keyvalue:
1489 return nodeid
1490 finally:
1491 cldb.close()
1492 raise KeyError('No key (%s) value "%s" for "%s"'%(self.key,
1493 keyvalue, self.classname))
1495 # change from spec - allows multiple props to match
1496 def find(self, **propspec):
1497 """Get the ids of nodes in this class which link to the given nodes.
1499 'propspec' consists of keyword args propname=nodeid or
1500 propname={nodeid:1, }
1501 'propname' must be the name of a property in this class, or a
1502 KeyError is raised. That property must be a Link or
1503 Multilink property, or a TypeError is raised.
1505 Any node in this class whose 'propname' property links to any of
1506 the nodeids will be returned. Examples::
1508 db.issue.find(messages='1')
1509 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1510 """
1511 for propname, itemids in propspec.iteritems():
1512 # check the prop is OK
1513 prop = self.properties[propname]
1514 if not isinstance(prop, hyperdb.Link) and not isinstance(prop, hyperdb.Multilink):
1515 raise TypeError("'%s' not a Link/Multilink "
1516 "property"%propname)
1518 # ok, now do the find
1519 cldb = self.db.getclassdb(self.classname)
1520 l = []
1521 try:
1522 for id in self.getnodeids(db=cldb):
1523 item = self.db.getnode(self.classname, id, db=cldb)
1524 if self.db.RETIRED_FLAG in item:
1525 continue
1526 for propname, itemids in propspec.iteritems():
1527 if type(itemids) is not type({}):
1528 itemids = {itemids:1}
1530 # special case if the item doesn't have this property
1531 if propname not in item:
1532 if None in itemids:
1533 l.append(id)
1534 break
1535 continue
1537 # grab the property definition and its value on this item
1538 prop = self.properties[propname]
1539 value = item[propname]
1540 if isinstance(prop, hyperdb.Link) and value in itemids:
1541 l.append(id)
1542 break
1543 elif isinstance(prop, hyperdb.Multilink):
1544 hit = 0
1545 for v in value:
1546 if v in itemids:
1547 l.append(id)
1548 hit = 1
1549 break
1550 if hit:
1551 break
1552 finally:
1553 cldb.close()
1554 return l
1556 def stringFind(self, **requirements):
1557 """Locate a particular node by matching a set of its String
1558 properties in a caseless search.
1560 If the property is not a String property, a TypeError is raised.
1562 The return is a list of the id of all nodes that match.
1563 """
1564 for propname in requirements:
1565 prop = self.properties[propname]
1566 if not isinstance(prop, hyperdb.String):
1567 raise TypeError("'%s' not a String property"%propname)
1568 requirements[propname] = requirements[propname].lower()
1569 l = []
1570 cldb = self.db.getclassdb(self.classname)
1571 try:
1572 for nodeid in self.getnodeids(cldb):
1573 node = self.db.getnode(self.classname, nodeid, cldb)
1574 if self.db.RETIRED_FLAG in node:
1575 continue
1576 for key, value in requirements.iteritems():
1577 if key not in node:
1578 break
1579 if node[key] is None or node[key].lower() != value:
1580 break
1581 else:
1582 l.append(nodeid)
1583 finally:
1584 cldb.close()
1585 return l
1587 def list(self):
1588 """ Return a list of the ids of the active nodes in this class.
1589 """
1590 l = []
1591 cn = self.classname
1592 cldb = self.db.getclassdb(cn)
1593 try:
1594 for nodeid in self.getnodeids(cldb):
1595 node = self.db.getnode(cn, nodeid, cldb)
1596 if self.db.RETIRED_FLAG in node:
1597 continue
1598 l.append(nodeid)
1599 finally:
1600 cldb.close()
1601 l.sort()
1602 return l
1604 def getnodeids(self, db=None, retired=None):
1605 """ Return a list of ALL nodeids
1607 Set retired=None to get all nodes. Otherwise it'll get all the
1608 retired or non-retired nodes, depending on the flag.
1609 """
1610 res = []
1612 # start off with the new nodes
1613 if self.classname in self.db.newnodes:
1614 res.extend(self.db.newnodes[self.classname])
1616 must_close = False
1617 if db is None:
1618 db = self.db.getclassdb(self.classname)
1619 must_close = True
1620 try:
1621 res.extend(db.keys())
1623 # remove the uncommitted, destroyed nodes
1624 if self.classname in self.db.destroyednodes:
1625 for nodeid in self.db.destroyednodes[self.classname]:
1626 if key_in(db, nodeid):
1627 res.remove(nodeid)
1629 # check retired flag
1630 if retired is False or retired is True:
1631 l = []
1632 for nodeid in res:
1633 node = self.db.getnode(self.classname, nodeid, db)
1634 is_ret = self.db.RETIRED_FLAG in node
1635 if retired == is_ret:
1636 l.append(nodeid)
1637 res = l
1638 finally:
1639 if must_close:
1640 db.close()
1641 return res
1643 def _filter(self, search_matches, filterspec, proptree,
1644 num_re = re.compile('^\d+$')):
1645 """Return a list of the ids of the active nodes in this class that
1646 match the 'filter' spec, sorted by the group spec and then the
1647 sort spec.
1649 "filterspec" is {propname: value(s)}
1651 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1652 and prop is a prop name or None
1654 "search_matches" is a sequence type or None
1656 The filter must match all properties specificed. If the property
1657 value to match is a list:
1659 1. String properties must match all elements in the list, and
1660 2. Other properties must match any of the elements in the list.
1661 """
1662 if __debug__:
1663 start_t = time.time()
1665 cn = self.classname
1667 # optimise filterspec
1668 l = []
1669 props = self.getprops()
1670 LINK = 'spec:link'
1671 MULTILINK = 'spec:multilink'
1672 STRING = 'spec:string'
1673 DATE = 'spec:date'
1674 INTERVAL = 'spec:interval'
1675 OTHER = 'spec:other'
1677 for k, v in filterspec.iteritems():
1678 propclass = props[k]
1679 if isinstance(propclass, hyperdb.Link):
1680 if type(v) is not type([]):
1681 v = [v]
1682 u = []
1683 for entry in v:
1684 # the value -1 is a special "not set" sentinel
1685 if entry == '-1':
1686 entry = None
1687 u.append(entry)
1688 l.append((LINK, k, u))
1689 elif isinstance(propclass, hyperdb.Multilink):
1690 # the value -1 is a special "not set" sentinel
1691 if v in ('-1', ['-1']):
1692 v = []
1693 elif type(v) is not type([]):
1694 v = [v]
1695 l.append((MULTILINK, k, v))
1696 elif isinstance(propclass, hyperdb.String) and k != 'id':
1697 if type(v) is not type([]):
1698 v = [v]
1699 for v in v:
1700 # simple glob searching
1701 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1702 v = v.replace('?', '.')
1703 v = v.replace('*', '.*?')
1704 l.append((STRING, k, re.compile(v, re.I)))
1705 elif isinstance(propclass, hyperdb.Date):
1706 try:
1707 date_rng = propclass.range_from_raw(v, self.db)
1708 l.append((DATE, k, date_rng))
1709 except ValueError:
1710 # If range creation fails - ignore that search parameter
1711 pass
1712 elif isinstance(propclass, hyperdb.Interval):
1713 try:
1714 intv_rng = date.Range(v, date.Interval)
1715 l.append((INTERVAL, k, intv_rng))
1716 except ValueError:
1717 # If range creation fails - ignore that search parameter
1718 pass
1720 elif isinstance(propclass, hyperdb.Boolean):
1721 if type(v) == type(""):
1722 v = v.split(',')
1723 if type(v) != type([]):
1724 v = [v]
1725 bv = []
1726 for val in v:
1727 if type(val) is type(''):
1728 bv.append(propclass.from_raw (val))
1729 else:
1730 bv.append(val)
1731 l.append((OTHER, k, bv))
1733 elif k == 'id':
1734 if type(v) != type([]):
1735 v = v.split(',')
1736 l.append((OTHER, k, [str(int(val)) for val in v]))
1738 elif isinstance(propclass, hyperdb.Number):
1739 if type(v) != type([]):
1740 try :
1741 v = v.split(',')
1742 except AttributeError :
1743 v = [v]
1744 l.append((OTHER, k, [float(val) for val in v]))
1746 filterspec = l
1748 # now, find all the nodes that are active and pass filtering
1749 matches = []
1750 cldb = self.db.getclassdb(cn)
1751 t = 0
1752 try:
1753 # TODO: only full-scan once (use items())
1754 for nodeid in self.getnodeids(cldb):
1755 node = self.db.getnode(cn, nodeid, cldb)
1756 if self.db.RETIRED_FLAG in node:
1757 continue
1758 # apply filter
1759 for t, k, v in filterspec:
1760 # handle the id prop
1761 if k == 'id':
1762 if nodeid not in v:
1763 break
1764 continue
1766 # get the node value
1767 nv = node.get(k, None)
1769 match = 0
1771 # now apply the property filter
1772 if t == LINK:
1773 # link - if this node's property doesn't appear in the
1774 # filterspec's nodeid list, skip it
1775 match = nv in v
1776 elif t == MULTILINK:
1777 # multilink - if any of the nodeids required by the
1778 # filterspec aren't in this node's property, then skip
1779 # it
1780 nv = node.get(k, [])
1782 # check for matching the absence of multilink values
1783 if not v:
1784 match = not nv
1785 else:
1786 # otherwise, make sure this node has each of the
1787 # required values
1788 expr = Expression(v)
1789 if expr.evaluate(nv): match = 1
1790 elif t == STRING:
1791 if nv is None:
1792 nv = ''
1793 # RE search
1794 match = v.search(nv)
1795 elif t == DATE or t == INTERVAL:
1796 if nv is None:
1797 match = v is None
1798 else:
1799 if v.to_value:
1800 if v.from_value <= nv and v.to_value >= nv:
1801 match = 1
1802 else:
1803 if v.from_value <= nv:
1804 match = 1
1805 elif t == OTHER:
1806 # straight value comparison for the other types
1807 match = nv in v
1808 if not match:
1809 break
1810 else:
1811 matches.append([nodeid, node])
1813 # filter based on full text search
1814 if search_matches is not None:
1815 k = []
1816 for v in matches:
1817 if v[0] in search_matches:
1818 k.append(v)
1819 matches = k
1821 # add sorting information to the proptree
1822 JPROPS = {'actor':1, 'activity':1, 'creator':1, 'creation':1}
1823 children = []
1824 if proptree:
1825 children = proptree.sortable_children()
1826 for pt in children:
1827 dir = pt.sort_direction
1828 prop = pt.name
1829 assert (dir and prop)
1830 propclass = props[prop]
1831 pt.sort_ids = []
1832 is_pointer = isinstance(propclass,(hyperdb.Link,
1833 hyperdb.Multilink))
1834 if not is_pointer:
1835 pt.sort_result = []
1836 try:
1837 # cache the opened link class db, if needed.
1838 lcldb = None
1839 # cache the linked class items too
1840 lcache = {}
1842 for entry in matches:
1843 itemid = entry[-2]
1844 item = entry[-1]
1845 # handle the properties that might be "faked"
1846 # also, handle possible missing properties
1847 try:
1848 v = item[prop]
1849 except KeyError:
1850 if prop in JPROPS:
1851 # force lookup of the special journal prop
1852 v = self.get(itemid, prop)
1853 else:
1854 # the node doesn't have a value for this
1855 # property
1856 v = None
1857 if isinstance(propclass, hyperdb.Multilink):
1858 v = []
1859 if prop == 'id':
1860 v = int (itemid)
1861 pt.sort_ids.append(v)
1862 if not is_pointer:
1863 pt.sort_result.append(v)
1864 continue
1866 # missing (None) values are always sorted first
1867 if v is None:
1868 pt.sort_ids.append(v)
1869 if not is_pointer:
1870 pt.sort_result.append(v)
1871 continue
1873 if isinstance(propclass, hyperdb.Link):
1874 lcn = propclass.classname
1875 link = self.db.classes[lcn]
1876 key = link.orderprop()
1877 child = pt.propdict[key]
1878 if key!='id':
1879 if v not in lcache:
1880 # open the link class db if it's not already
1881 if lcldb is None:
1882 lcldb = self.db.getclassdb(lcn)
1883 lcache[v] = self.db.getnode(lcn, v, lcldb)
1884 r = lcache[v][key]
1885 child.propdict[key].sort_ids.append(r)
1886 else:
1887 child.propdict[key].sort_ids.append(v)
1888 pt.sort_ids.append(v)
1889 if not is_pointer:
1890 r = propclass.sort_repr(pt.parent.cls, v, pt.name)
1891 pt.sort_result.append(r)
1892 finally:
1893 # if we opened the link class db, close it now
1894 if lcldb is not None:
1895 lcldb.close()
1896 del lcache
1897 finally:
1898 cldb.close()
1900 # pull the id out of the individual entries
1901 matches = [entry[-2] for entry in matches]
1902 if __debug__:
1903 self.db.stats['filtering'] += (time.time() - start_t)
1904 return matches
1906 def count(self):
1907 """Get the number of nodes in this class.
1909 If the returned integer is 'numnodes', the ids of all the nodes
1910 in this class run from 1 to numnodes, and numnodes+1 will be the
1911 id of the next node to be created in this class.
1912 """
1913 return self.db.countnodes(self.classname)
1915 # Manipulating properties:
1917 def getprops(self, protected=1):
1918 """Return a dictionary mapping property names to property objects.
1919 If the "protected" flag is true, we include protected properties -
1920 those which may not be modified.
1922 In addition to the actual properties on the node, these
1923 methods provide the "creation" and "activity" properties. If the
1924 "protected" flag is true, we include protected properties - those
1925 which may not be modified.
1926 """
1927 d = self.properties.copy()
1928 if protected:
1929 d['id'] = hyperdb.String()
1930 d['creation'] = hyperdb.Date()
1931 d['activity'] = hyperdb.Date()
1932 d['creator'] = hyperdb.Link('user')
1933 d['actor'] = hyperdb.Link('user')
1934 return d
1936 def addprop(self, **properties):
1937 """Add properties to this class.
1939 The keyword arguments in 'properties' must map names to property
1940 objects, or a TypeError is raised. None of the keys in 'properties'
1941 may collide with the names of existing properties, or a ValueError
1942 is raised before any properties have been added.
1943 """
1944 for key in properties:
1945 if key in self.properties:
1946 raise ValueError(key)
1947 self.properties.update(properties)
1949 def index(self, nodeid):
1950 """ Add (or refresh) the node to search indexes """
1951 # find all the String properties that have indexme
1952 for prop, propclass in self.getprops().iteritems():
1953 if isinstance(propclass, hyperdb.String) and propclass.indexme:
1954 # index them under (classname, nodeid, property)
1955 try:
1956 value = str(self.get(nodeid, prop))
1957 except IndexError:
1958 # node has been destroyed
1959 continue
1960 self.db.indexer.add_text((self.classname, nodeid, prop), value)
1962 #
1963 # import / export support
1964 #
1965 def export_list(self, propnames, nodeid):
1966 """ Export a node - generate a list of CSV-able data in the order
1967 specified by propnames for the given node.
1968 """
1969 properties = self.getprops()
1970 l = []
1971 for prop in propnames:
1972 proptype = properties[prop]
1973 value = self.get(nodeid, prop)
1974 # "marshal" data where needed
1975 if value is None:
1976 pass
1977 elif isinstance(proptype, hyperdb.Date):
1978 value = value.get_tuple()
1979 elif isinstance(proptype, hyperdb.Interval):
1980 value = value.get_tuple()
1981 elif isinstance(proptype, hyperdb.Password):
1982 value = str(value)
1983 l.append(repr(value))
1985 # append retired flag
1986 l.append(repr(self.is_retired(nodeid)))
1988 return l
1990 def import_list(self, propnames, proplist):
1991 """ Import a node - all information including "id" is present and
1992 should not be sanity checked. Triggers are not triggered. The
1993 journal should be initialised using the "creator" and "created"
1994 information.
1996 Return the nodeid of the node imported.
1997 """
1998 if self.db.journaltag is None:
1999 raise hyperdb.DatabaseError(_('Database open read-only'))
2000 properties = self.getprops()
2002 # make the new node's property map
2003 d = {}
2004 newid = None
2005 for i in range(len(propnames)):
2006 # Figure the property for this column
2007 propname = propnames[i]
2009 # Use eval to reverse the repr() used to output the CSV
2010 value = eval(proplist[i])
2012 # "unmarshal" where necessary
2013 if propname == 'id':
2014 newid = value
2015 continue
2016 elif propname == 'is retired':
2017 # is the item retired?
2018 if int(value):
2019 d[self.db.RETIRED_FLAG] = 1
2020 continue
2021 elif value is None:
2022 d[propname] = None
2023 continue
2025 prop = properties[propname]
2026 if isinstance(prop, hyperdb.Date):
2027 value = date.Date(value)
2028 elif isinstance(prop, hyperdb.Interval):
2029 value = date.Interval(value)
2030 elif isinstance(prop, hyperdb.Password):
2031 value = password.Password(encrypted=value)
2032 d[propname] = value
2034 # get a new id if necessary
2035 if newid is None:
2036 newid = self.db.newid(self.classname)
2038 # add the node and journal
2039 self.db.addnode(self.classname, newid, d)
2040 return newid
2042 def export_journals(self):
2043 """Export a class's journal - generate a list of lists of
2044 CSV-able data:
2046 nodeid, date, user, action, params
2048 No heading here - the columns are fixed.
2049 """
2050 properties = self.getprops()
2051 r = []
2052 for nodeid in self.getnodeids():
2053 for nodeid, date, user, action, params in self.history(nodeid):
2054 date = date.get_tuple()
2055 if action == 'set':
2056 export_data = {}
2057 for propname, value in params.iteritems():
2058 if propname not in properties:
2059 # property no longer in the schema
2060 continue
2062 prop = properties[propname]
2063 # make sure the params are eval()'able
2064 if value is None:
2065 pass
2066 elif isinstance(prop, hyperdb.Date):
2067 # this is a hack - some dates are stored as strings
2068 if not isinstance(value, type('')):
2069 value = value.get_tuple()
2070 elif isinstance(prop, hyperdb.Interval):
2071 # hack too - some intervals are stored as strings
2072 if not isinstance(value, type('')):
2073 value = value.get_tuple()
2074 elif isinstance(prop, hyperdb.Password):
2075 value = str(value)
2076 export_data[propname] = value
2077 params = export_data
2078 r.append([repr(nodeid), repr(date), repr(user),
2079 repr(action), repr(params)])
2080 return r
2082 class FileClass(hyperdb.FileClass, Class):
2083 """This class defines a large chunk of data. To support this, it has a
2084 mandatory String property "content" which is typically saved off
2085 externally to the hyperdb.
2087 The default MIME type of this data is defined by the
2088 "default_mime_type" class attribute, which may be overridden by each
2089 node if the class defines a "type" String property.
2090 """
2091 def __init__(self, db, classname, **properties):
2092 """The newly-created class automatically includes the "content"
2093 and "type" properties.
2094 """
2095 if 'content' not in properties:
2096 properties['content'] = hyperdb.String(indexme='yes')
2097 if 'type' not in properties:
2098 properties['type'] = hyperdb.String()
2099 Class.__init__(self, db, classname, **properties)
2101 def create(self, **propvalues):
2102 """ Snarf the "content" propvalue and store in a file
2103 """
2104 # we need to fire the auditors now, or the content property won't
2105 # be in propvalues for the auditors to play with
2106 self.fireAuditors('create', None, propvalues)
2108 # now remove the content property so it's not stored in the db
2109 content = propvalues['content']
2110 del propvalues['content']
2112 # make sure we have a MIME type
2113 mime_type = propvalues.get('type', self.default_mime_type)
2115 # do the database create
2116 newid = self.create_inner(**propvalues)
2118 # store off the content as a file
2119 self.db.storefile(self.classname, newid, None, content)
2121 # fire reactors
2122 self.fireReactors('create', newid, None)
2124 return newid
2126 def get(self, nodeid, propname, default=_marker, cache=1):
2127 """ Trap the content propname and get it from the file
2129 'cache' exists for backwards compatibility, and is not used.
2130 """
2131 poss_msg = 'Possibly an access right configuration problem.'
2132 if propname == 'content':
2133 try:
2134 return self.db.getfile(self.classname, nodeid, None)
2135 except IOError, strerror:
2136 # XXX by catching this we don't see an error in the log.
2137 return 'ERROR reading file: %s%s\n%s\n%s'%(
2138 self.classname, nodeid, poss_msg, strerror)
2139 if default is not _marker:
2140 return Class.get(self, nodeid, propname, default)
2141 else:
2142 return Class.get(self, nodeid, propname)
2144 def set(self, itemid, **propvalues):
2145 """ Snarf the "content" propvalue and update it in a file
2146 """
2147 self.fireAuditors('set', itemid, propvalues)
2149 # create the oldvalues dict - fill in any missing values
2150 oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid))
2151 for name, prop in self.getprops(protected=0).iteritems():
2152 if name in oldvalues:
2153 continue
2154 if isinstance(prop, hyperdb.Multilink):
2155 oldvalues[name] = []
2156 else:
2157 oldvalues[name] = None
2159 # now remove the content property so it's not stored in the db
2160 content = None
2161 if 'content' in propvalues:
2162 content = propvalues['content']
2163 del propvalues['content']
2165 # do the database update
2166 propvalues = self.set_inner(itemid, **propvalues)
2168 # do content?
2169 if content:
2170 # store and possibly index
2171 self.db.storefile(self.classname, itemid, None, content)
2172 if self.properties['content'].indexme:
2173 mime_type = self.get(itemid, 'type', self.default_mime_type)
2174 self.db.indexer.add_text((self.classname, itemid, 'content'),
2175 content, mime_type)
2176 propvalues['content'] = content
2178 # fire reactors
2179 self.fireReactors('set', itemid, oldvalues)
2180 return propvalues
2182 def index(self, nodeid):
2183 """ Add (or refresh) the node to search indexes.
2185 Use the content-type property for the content property.
2186 """
2187 # find all the String properties that have indexme
2188 for prop, propclass in self.getprops().iteritems():
2189 if prop == 'content' and propclass.indexme:
2190 mime_type = self.get(nodeid, 'type', self.default_mime_type)
2191 self.db.indexer.add_text((self.classname, nodeid, 'content'),
2192 str(self.get(nodeid, 'content')), mime_type)
2193 elif isinstance(propclass, hyperdb.String) and propclass.indexme:
2194 # index them under (classname, nodeid, property)
2195 try:
2196 value = str(self.get(nodeid, prop))
2197 except IndexError:
2198 # node has been destroyed
2199 continue
2200 self.db.indexer.add_text((self.classname, nodeid, prop), value)
2202 # deviation from spec - was called ItemClass
2203 class IssueClass(Class, roundupdb.IssueClass):
2204 # Overridden methods:
2205 def __init__(self, db, classname, **properties):
2206 """The newly-created class automatically includes the "messages",
2207 "files", "nosy", and "superseder" properties. If the 'properties'
2208 dictionary attempts to specify any of these properties or a
2209 "creation" or "activity" property, a ValueError is raised.
2210 """
2211 if 'title' not in properties:
2212 properties['title'] = hyperdb.String(indexme='yes')
2213 if 'messages' not in properties:
2214 properties['messages'] = hyperdb.Multilink("msg")
2215 if 'files' not in properties:
2216 properties['files'] = hyperdb.Multilink("file")
2217 if 'nosy' not in properties:
2218 # note: journalling is turned off as it really just wastes
2219 # space. this behaviour may be overridden in an instance
2220 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2221 if 'superseder' not in properties:
2222 properties['superseder'] = hyperdb.Multilink(classname)
2223 Class.__init__(self, db, classname, **properties)
2225 # vim: set et sts=4 sw=4 :