1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 """This module defines a backend that saves the hyperdatabase in a
19 database chosen by anydbm. It is guaranteed to always be available in python
20 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
21 serious bugs, and is not available)
22 """
23 __docformat__ = 'restructuredtext'
25 import os, marshal, re, weakref, string, copy, time, shutil, logging
27 from roundup.anypy.dbm_ import anydbm, whichdb, key_in
29 from roundup import hyperdb, date, password, roundupdb, security, support
30 from roundup.support import reversed
31 from roundup.backends import locking
32 from roundup.i18n import _
34 from roundup.backends.blobfiles import FileStorage
35 from roundup.backends.sessions_dbm import Sessions, OneTimeKeys
37 try:
38 from roundup.backends.indexer_xapian import Indexer
39 except ImportError:
40 from roundup.backends.indexer_dbm import Indexer
42 def db_exists(config):
43 # check for the user db
44 for db in 'nodes.user nodes.user.db'.split():
45 if os.path.exists(os.path.join(config.DATABASE, db)):
46 return 1
47 return 0
49 def db_nuke(config):
50 shutil.rmtree(config.DATABASE)
52 class Binary:
54 def __init__(self, x, y):
55 self.x = x
56 self.y = y
58 def visit(self, visitor):
59 self.x.visit(visitor)
60 self.y.visit(visitor)
62 class Unary:
64 def __init__(self, x):
65 self.x = x
67 def generate(self, atom):
68 return atom(self)
70 def visit(self, visitor):
71 self.x.visit(visitor)
73 class Equals(Unary):
75 def evaluate(self, v):
76 return self.x in v
78 def visit(self, visitor):
79 visitor(self)
81 class Not(Unary):
83 def evaluate(self, v):
84 return not self.x.evaluate(v)
86 def generate(self, atom):
87 return "NOT(%s)" % self.x.generate(atom)
89 class Or(Binary):
91 def evaluate(self, v):
92 return self.x.evaluate(v) or self.y.evaluate(v)
94 def generate(self, atom):
95 return "(%s)OR(%s)" % (
96 self.x.generate(atom),
97 self.y.generate(atom))
99 class And(Binary):
101 def evaluate(self, v):
102 return self.x.evaluate(v) and self.y.evaluate(v)
104 def generate(self, atom):
105 return "(%s)AND(%s)" % (
106 self.x.generate(atom),
107 self.y.generate(atom))
109 def compile_expression(opcodes):
111 stack = []
112 push, pop = stack.append, stack.pop
113 for opcode in opcodes:
114 if opcode == -2: push(Not(pop()))
115 elif opcode == -3: push(And(pop(), pop()))
116 elif opcode == -4: push(Or(pop(), pop()))
117 else: push(Equals(opcode))
119 return pop()
121 class Expression:
123 def __init__(self, v):
124 try:
125 opcodes = [int(x) for x in v]
126 if min(opcodes) >= -1: raise ValueError()
128 compiled = compile_expression(opcodes)
129 self.evaluate = lambda x: compiled.evaluate([int(y) for y in x])
130 except:
131 self.evaluate = lambda x: bool(set(x) & set(v))
133 #
134 # Now the database
135 #
136 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
137 """A database for storing records containing flexible data types.
139 Transaction stuff TODO:
141 - check the timestamp of the class file and nuke the cache if it's
142 modified. Do some sort of conflict checking on the dirty stuff.
143 - perhaps detect write collisions (related to above)?
144 """
145 def __init__(self, config, journaltag=None):
146 """Open a hyperdatabase given a specifier to some storage.
148 The 'storagelocator' is obtained from config.DATABASE.
149 The meaning of 'storagelocator' depends on the particular
150 implementation of the hyperdatabase. It could be a file name,
151 a directory path, a socket descriptor for a connection to a
152 database over the network, etc.
154 The 'journaltag' is a token that will be attached to the journal
155 entries for any edits done on the database. If 'journaltag' is
156 None, the database is opened in read-only mode: the Class.create(),
157 Class.set(), Class.retire(), and Class.restore() methods are
158 disabled.
159 """
160 FileStorage.__init__(self, config.UMASK)
161 self.config, self.journaltag = config, journaltag
162 self.dir = config.DATABASE
163 self.classes = {}
164 self.cache = {} # cache of nodes loaded or created
165 self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0,
166 'filtering': 0}
167 self.dirtynodes = {} # keep track of the dirty nodes by class
168 self.newnodes = {} # keep track of the new nodes by class
169 self.destroyednodes = {}# keep track of the destroyed nodes by class
170 self.transactions = []
171 self.indexer = Indexer(self)
172 self.security = security.Security(self)
173 os.umask(config.UMASK)
175 # lock it
176 lockfilenm = os.path.join(self.dir, 'lock')
177 self.lockfile = locking.acquire_lock(lockfilenm)
178 self.lockfile.write(str(os.getpid()))
179 self.lockfile.flush()
181 def post_init(self):
182 """Called once the schema initialisation has finished.
183 """
184 # reindex the db if necessary
185 if self.indexer.should_reindex():
186 self.reindex()
188 def refresh_database(self):
189 """Rebuild the database
190 """
191 self.reindex()
193 def getSessionManager(self):
194 return Sessions(self)
196 def getOTKManager(self):
197 return OneTimeKeys(self)
199 def reindex(self, classname=None, show_progress=False):
200 if classname:
201 classes = [self.getclass(classname)]
202 else:
203 classes = self.classes.values()
204 for klass in classes:
205 if show_progress:
206 for nodeid in support.Progress('Reindex %s'%klass.classname,
207 klass.list()):
208 klass.index(nodeid)
209 else:
210 for nodeid in klass.list():
211 klass.index(nodeid)
212 self.indexer.save_index()
214 def __repr__(self):
215 return '<back_anydbm instance at %x>'%id(self)
217 #
218 # Classes
219 #
220 def __getattr__(self, classname):
221 """A convenient way of calling self.getclass(classname)."""
222 if classname in self.classes:
223 return self.classes[classname]
224 raise AttributeError, classname
226 def addclass(self, cl):
227 cn = cl.classname
228 if cn in self.classes:
229 raise ValueError, cn
230 self.classes[cn] = cl
232 # add default Edit and View permissions
233 self.security.addPermission(name="Create", klass=cn,
234 description="User is allowed to create "+cn)
235 self.security.addPermission(name="Edit", klass=cn,
236 description="User is allowed to edit "+cn)
237 self.security.addPermission(name="View", klass=cn,
238 description="User is allowed to access "+cn)
240 def getclasses(self):
241 """Return a list of the names of all existing classes."""
242 l = self.classes.keys()
243 l.sort()
244 return l
246 def getclass(self, classname):
247 """Get the Class object representing a particular class.
249 If 'classname' is not a valid class name, a KeyError is raised.
250 """
251 try:
252 return self.classes[classname]
253 except KeyError:
254 raise KeyError('There is no class called "%s"'%classname)
256 #
257 # Class DBs
258 #
259 def clear(self):
260 """Delete all database contents
261 """
262 logging.getLogger('roundup.hyperdb').info('clear')
263 for cn in self.classes:
264 for dummy in 'nodes', 'journals':
265 path = os.path.join(self.dir, 'journals.%s'%cn)
266 if os.path.exists(path):
267 os.remove(path)
268 elif os.path.exists(path+'.db'): # dbm appends .db
269 os.remove(path+'.db')
270 # reset id sequences
271 path = os.path.join(os.getcwd(), self.dir, '_ids')
272 if os.path.exists(path):
273 os.remove(path)
274 elif os.path.exists(path+'.db'): # dbm appends .db
275 os.remove(path+'.db')
277 def getclassdb(self, classname, mode='r'):
278 """ grab a connection to the class db that will be used for
279 multiple actions
280 """
281 return self.opendb('nodes.%s'%classname, mode)
283 def determine_db_type(self, path):
284 """ determine which DB wrote the class file
285 """
286 db_type = ''
287 if os.path.exists(path):
288 db_type = whichdb(path)
289 if not db_type:
290 raise hyperdb.DatabaseError(_("Couldn't identify database type"))
291 elif os.path.exists(path+'.db'):
292 # if the path ends in '.db', it's a dbm database, whether
293 # anydbm says it's dbhash or not!
294 db_type = 'dbm'
295 return db_type
297 def opendb(self, name, mode):
298 """Low-level database opener that gets around anydbm/dbm
299 eccentricities.
300 """
301 # figure the class db type
302 path = os.path.join(os.getcwd(), self.dir, name)
303 db_type = self.determine_db_type(path)
305 # new database? let anydbm pick the best dbm
306 # in Python 3+ the "dbm" ("anydbm" to us) module already uses the
307 # whichdb() function to do this
308 if not db_type or hasattr(anydbm, 'whichdb'):
309 if __debug__:
310 logging.getLogger('roundup.hyperdb').debug(
311 "opendb anydbm.open(%r, 'c')"%path)
312 return anydbm.open(path, 'c')
314 # in Python <3 it anydbm was a little dumb so manually open the
315 # database with the correct module
316 try:
317 dbm = __import__(db_type)
318 except ImportError:
319 raise hyperdb.DatabaseError(_("Couldn't open database - the "
320 "required module '%s' is not available")%db_type)
321 if __debug__:
322 logging.getLogger('roundup.hyperdb').debug(
323 "opendb %r.open(%r, %r)"%(db_type, path, mode))
324 return dbm.open(path, mode)
326 #
327 # Node IDs
328 #
329 def newid(self, classname):
330 """ Generate a new id for the given class
331 """
332 # open the ids DB - create if if doesn't exist
333 db = self.opendb('_ids', 'c')
334 if key_in(db, classname):
335 newid = db[classname] = str(int(db[classname]) + 1)
336 else:
337 # the count() bit is transitional - older dbs won't start at 1
338 newid = str(self.getclass(classname).count()+1)
339 db[classname] = newid
340 db.close()
341 return newid
343 def setid(self, classname, setid):
344 """ Set the id counter: used during import of database
345 """
346 # open the ids DB - create if if doesn't exist
347 db = self.opendb('_ids', 'c')
348 db[classname] = str(setid)
349 db.close()
351 #
352 # Nodes
353 #
354 def addnode(self, classname, nodeid, node):
355 """ add the specified node to its class's db
356 """
357 # we'll be supplied these props if we're doing an import
358 if 'creator' not in node:
359 # add in the "calculated" properties (dupe so we don't affect
360 # calling code's node assumptions)
361 node = node.copy()
362 node['creator'] = self.getuid()
363 node['actor'] = self.getuid()
364 node['creation'] = node['activity'] = date.Date()
366 self.newnodes.setdefault(classname, {})[nodeid] = 1
367 self.cache.setdefault(classname, {})[nodeid] = node
368 self.savenode(classname, nodeid, node)
370 def setnode(self, classname, nodeid, node):
371 """ change the specified node
372 """
373 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
375 # can't set without having already loaded the node
376 self.cache[classname][nodeid] = node
377 self.savenode(classname, nodeid, node)
379 def savenode(self, classname, nodeid, node):
380 """ perform the saving of data specified by the set/addnode
381 """
382 if __debug__:
383 logging.getLogger('roundup.hyperdb').debug(
384 'save %s%s %r'%(classname, nodeid, node))
385 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
387 def getnode(self, classname, nodeid, db=None, cache=1):
388 """ get a node from the database
390 Note the "cache" parameter is not used, and exists purely for
391 backward compatibility!
392 """
393 # try the cache
394 cache_dict = self.cache.setdefault(classname, {})
395 if nodeid in cache_dict:
396 if __debug__:
397 logging.getLogger('roundup.hyperdb').debug(
398 'get %s%s cached'%(classname, nodeid))
399 self.stats['cache_hits'] += 1
400 return cache_dict[nodeid]
402 if __debug__:
403 self.stats['cache_misses'] += 1
404 start_t = time.time()
405 logging.getLogger('roundup.hyperdb').debug(
406 'get %s%s'%(classname, nodeid))
408 # get from the database and save in the cache
409 if db is None:
410 db = self.getclassdb(classname)
411 if not key_in(db, nodeid):
412 raise IndexError("no such %s %s"%(classname, nodeid))
414 # check the uncommitted, destroyed nodes
415 if (classname in self.destroyednodes and
416 nodeid in self.destroyednodes[classname]):
417 raise IndexError("no such %s %s"%(classname, nodeid))
419 # decode
420 res = marshal.loads(db[nodeid])
422 # reverse the serialisation
423 res = self.unserialise(classname, res)
425 # store off in the cache dict
426 if cache:
427 cache_dict[nodeid] = res
429 if __debug__:
430 self.stats['get_items'] += (time.time() - start_t)
432 return res
434 def destroynode(self, classname, nodeid):
435 """Remove a node from the database. Called exclusively by the
436 destroy() method on Class.
437 """
438 logging.getLogger('roundup.hyperdb').info(
439 'destroy %s%s'%(classname, nodeid))
441 # remove from cache and newnodes if it's there
442 if (classname in self.cache and nodeid in self.cache[classname]):
443 del self.cache[classname][nodeid]
444 if (classname in self.newnodes and nodeid in self.newnodes[classname]):
445 del self.newnodes[classname][nodeid]
447 # see if there's any obvious commit actions that we should get rid of
448 for entry in self.transactions[:]:
449 if entry[1][:2] == (classname, nodeid):
450 self.transactions.remove(entry)
452 # add to the destroyednodes map
453 self.destroyednodes.setdefault(classname, {})[nodeid] = 1
455 # add the destroy commit action
456 self.transactions.append((self.doDestroyNode, (classname, nodeid)))
457 self.transactions.append((FileStorage.destroy, (self, classname, nodeid)))
459 def serialise(self, classname, node):
460 """Copy the node contents, converting non-marshallable data into
461 marshallable data.
462 """
463 properties = self.getclass(classname).getprops()
464 d = {}
465 for k, v in node.iteritems():
466 if k == self.RETIRED_FLAG:
467 d[k] = v
468 continue
470 # if the property doesn't exist then we really don't care
471 if k not in properties:
472 continue
474 # get the property spec
475 prop = properties[k]
477 if isinstance(prop, hyperdb.Password) and v is not None:
478 d[k] = str(v)
479 elif isinstance(prop, hyperdb.Date) and v is not None:
480 d[k] = v.serialise()
481 elif isinstance(prop, hyperdb.Interval) and v is not None:
482 d[k] = v.serialise()
483 else:
484 d[k] = v
485 return d
487 def unserialise(self, classname, node):
488 """Decode the marshalled node data
489 """
490 properties = self.getclass(classname).getprops()
491 d = {}
492 for k, v in node.iteritems():
493 # if the property doesn't exist, or is the "retired" flag then
494 # it won't be in the properties dict
495 if k not in properties:
496 d[k] = v
497 continue
499 # get the property spec
500 prop = properties[k]
502 if isinstance(prop, hyperdb.Date) and v is not None:
503 d[k] = date.Date(v)
504 elif isinstance(prop, hyperdb.Interval) and v is not None:
505 d[k] = date.Interval(v)
506 elif isinstance(prop, hyperdb.Password) and v is not None:
507 d[k] = password.Password(encrypted=v)
508 else:
509 d[k] = v
510 return d
512 def hasnode(self, classname, nodeid, db=None):
513 """ determine if the database has a given node
514 """
515 # try the cache
516 cache = self.cache.setdefault(classname, {})
517 if nodeid in cache:
518 return 1
520 # not in the cache - check the database
521 if db is None:
522 db = self.getclassdb(classname)
523 return key_in(db, nodeid)
525 def countnodes(self, classname, db=None):
526 count = 0
528 # include the uncommitted nodes
529 if classname in self.newnodes:
530 count += len(self.newnodes[classname])
531 if classname in self.destroyednodes:
532 count -= len(self.destroyednodes[classname])
534 # and count those in the DB
535 if db is None:
536 db = self.getclassdb(classname)
537 return count + len(db)
540 #
541 # Files - special node properties
542 # inherited from FileStorage
544 #
545 # Journal
546 #
547 def addjournal(self, classname, nodeid, action, params, creator=None,
548 creation=None):
549 """ Journal the Action
550 'action' may be:
552 'create' or 'set' -- 'params' is a dictionary of property values
553 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
554 'retire' -- 'params' is None
556 'creator' -- the user performing the action, which defaults to
557 the current user.
558 """
559 if __debug__:
560 logging.getLogger('roundup.hyperdb').debug(
561 'addjournal %s%s %s %r %s %r'%(classname,
562 nodeid, action, params, creator, creation))
563 if creator is None:
564 creator = self.getuid()
565 self.transactions.append((self.doSaveJournal, (classname, nodeid,
566 action, params, creator, creation)))
568 def setjournal(self, classname, nodeid, journal):
569 """Set the journal to the "journal" list."""
570 if __debug__:
571 logging.getLogger('roundup.hyperdb').debug(
572 'setjournal %s%s %r'%(classname, nodeid, journal))
573 self.transactions.append((self.doSetJournal, (classname, nodeid,
574 journal)))
576 def getjournal(self, classname, nodeid):
577 """ get the journal for id
579 Raise IndexError if the node doesn't exist (as per history()'s
580 API)
581 """
582 # our journal result
583 res = []
585 # add any journal entries for transactions not committed to the
586 # database
587 for method, args in self.transactions:
588 if method != self.doSaveJournal:
589 continue
590 (cache_classname, cache_nodeid, cache_action, cache_params,
591 cache_creator, cache_creation) = args
592 if cache_classname == classname and cache_nodeid == nodeid:
593 if not cache_creator:
594 cache_creator = self.getuid()
595 if not cache_creation:
596 cache_creation = date.Date()
597 res.append((cache_nodeid, cache_creation, cache_creator,
598 cache_action, cache_params))
600 # attempt to open the journal - in some rare cases, the journal may
601 # not exist
602 try:
603 db = self.opendb('journals.%s'%classname, 'r')
604 except anydbm.error, error:
605 if str(error) == "need 'c' or 'n' flag to open new db":
606 raise IndexError('no such %s %s'%(classname, nodeid))
607 elif error.args[0] != 2:
608 # this isn't a "not found" error, be alarmed!
609 raise
610 if res:
611 # we have unsaved journal entries, return them
612 return res
613 raise IndexError('no such %s %s'%(classname, nodeid))
614 try:
615 journal = marshal.loads(db[nodeid])
616 except KeyError:
617 db.close()
618 if res:
619 # we have some unsaved journal entries, be happy!
620 return res
621 raise IndexError('no such %s %s'%(classname, nodeid))
622 db.close()
624 # add all the saved journal entries for this node
625 for nodeid, date_stamp, user, action, params in journal:
626 res.append((nodeid, date.Date(date_stamp), user, action, params))
627 return res
629 def pack(self, pack_before):
630 """ Delete all journal entries except "create" before 'pack_before'.
631 """
632 pack_before = pack_before.serialise()
633 for classname in self.getclasses():
634 packed = 0
635 # get the journal db
636 db_name = 'journals.%s'%classname
637 path = os.path.join(os.getcwd(), self.dir, classname)
638 db_type = self.determine_db_type(path)
639 db = self.opendb(db_name, 'w')
641 for key in db.keys():
642 # get the journal for this db entry
643 journal = marshal.loads(db[key])
644 l = []
645 last_set_entry = None
646 for entry in journal:
647 # unpack the entry
648 (nodeid, date_stamp, self.journaltag, action,
649 params) = entry
650 # if the entry is after the pack date, _or_ the initial
651 # create entry, then it stays
652 if date_stamp > pack_before or action == 'create':
653 l.append(entry)
654 else:
655 packed += 1
656 db[key] = marshal.dumps(l)
658 logging.getLogger('roundup.hyperdb').info(
659 'packed %d %s items'%(packed, classname))
661 if db_type == 'gdbm':
662 db.reorganize()
663 db.close()
666 #
667 # Basic transaction support
668 #
669 def commit(self, fail_ok=False):
670 """ Commit the current transactions.
672 Save all data changed since the database was opened or since the
673 last commit() or rollback().
675 fail_ok indicates that the commit is allowed to fail. This is used
676 in the web interface when committing cleaning of the session
677 database. We don't care if there's a concurrency issue there.
679 The only backend this seems to affect is postgres.
680 """
681 logging.getLogger('roundup.hyperdb').info('commit %s transactions'%(
682 len(self.transactions)))
684 # keep a handle to all the database files opened
685 self.databases = {}
687 try:
688 # now, do all the transactions
689 reindex = {}
690 for method, args in self.transactions:
691 reindex[method(*args)] = 1
692 finally:
693 # make sure we close all the database files
694 for db in self.databases.itervalues():
695 db.close()
696 del self.databases
698 # clear the transactions list now so the blobfile implementation
699 # doesn't think there's still pending file commits when it tries
700 # to access the file data
701 self.transactions = []
703 # reindex the nodes that request it
704 for classname, nodeid in [k for k in reindex if k]:
705 self.getclass(classname).index(nodeid)
707 # save the indexer state
708 self.indexer.save_index()
710 self.clearCache()
712 def clearCache(self):
713 # all transactions committed, back to normal
714 self.cache = {}
715 self.dirtynodes = {}
716 self.newnodes = {}
717 self.destroyednodes = {}
718 self.transactions = []
720 def getCachedClassDB(self, classname):
721 """ get the class db, looking in our cache of databases for commit
722 """
723 # get the database handle
724 db_name = 'nodes.%s'%classname
725 if db_name not in self.databases:
726 self.databases[db_name] = self.getclassdb(classname, 'c')
727 return self.databases[db_name]
729 def doSaveNode(self, classname, nodeid, node):
730 db = self.getCachedClassDB(classname)
732 # now save the marshalled data
733 db[nodeid] = marshal.dumps(self.serialise(classname, node))
735 # return the classname, nodeid so we reindex this content
736 return (classname, nodeid)
738 def getCachedJournalDB(self, classname):
739 """ get the journal db, looking in our cache of databases for commit
740 """
741 # get the database handle
742 db_name = 'journals.%s'%classname
743 if db_name not in self.databases:
744 self.databases[db_name] = self.opendb(db_name, 'c')
745 return self.databases[db_name]
747 def doSaveJournal(self, classname, nodeid, action, params, creator,
748 creation):
749 # serialise the parameters now if necessary
750 if isinstance(params, type({})):
751 if action in ('set', 'create'):
752 params = self.serialise(classname, params)
754 # handle supply of the special journalling parameters (usually
755 # supplied on importing an existing database)
756 journaltag = creator
757 if creation:
758 journaldate = creation.serialise()
759 else:
760 journaldate = date.Date().serialise()
762 # create the journal entry
763 entry = (nodeid, journaldate, journaltag, action, params)
765 db = self.getCachedJournalDB(classname)
767 # now insert the journal entry
768 if key_in(db, nodeid):
769 # append to existing
770 s = db[nodeid]
771 l = marshal.loads(s)
772 l.append(entry)
773 else:
774 l = [entry]
776 db[nodeid] = marshal.dumps(l)
778 def doSetJournal(self, classname, nodeid, journal):
779 l = []
780 for nodeid, journaldate, journaltag, action, params in journal:
781 # serialise the parameters now if necessary
782 if isinstance(params, type({})):
783 if action in ('set', 'create'):
784 params = self.serialise(classname, params)
785 journaldate = journaldate.serialise()
786 l.append((nodeid, journaldate, journaltag, action, params))
787 db = self.getCachedJournalDB(classname)
788 db[nodeid] = marshal.dumps(l)
790 def doDestroyNode(self, classname, nodeid):
791 # delete from the class database
792 db = self.getCachedClassDB(classname)
793 if key_in(db, nodeid):
794 del db[nodeid]
796 # delete from the database
797 db = self.getCachedJournalDB(classname)
798 if key_in(db, nodeid):
799 del db[nodeid]
801 def rollback(self):
802 """ Reverse all actions from the current transaction.
803 """
804 logging.getLogger('roundup.hyperdb').info('rollback %s transactions'%(
805 len(self.transactions)))
807 for method, args in self.transactions:
808 # delete temporary files
809 if method == self.doStoreFile:
810 self.rollbackStoreFile(*args)
811 self.cache = {}
812 self.dirtynodes = {}
813 self.newnodes = {}
814 self.destroyednodes = {}
815 self.transactions = []
817 def close(self):
818 """ Nothing to do
819 """
820 if self.lockfile is not None:
821 locking.release_lock(self.lockfile)
822 self.lockfile.close()
823 self.lockfile = None
825 _marker = []
826 class Class(hyperdb.Class):
827 """The handle to a particular class of nodes in a hyperdatabase."""
829 def enableJournalling(self):
830 """Turn journalling on for this class
831 """
832 self.do_journal = 1
834 def disableJournalling(self):
835 """Turn journalling off for this class
836 """
837 self.do_journal = 0
839 # Editing nodes:
841 def create(self, **propvalues):
842 """Create a new node of this class and return its id.
844 The keyword arguments in 'propvalues' map property names to values.
846 The values of arguments must be acceptable for the types of their
847 corresponding properties or a TypeError is raised.
849 If this class has a key property, it must be present and its value
850 must not collide with other key strings or a ValueError is raised.
852 Any other properties on this class that are missing from the
853 'propvalues' dictionary are set to None.
855 If an id in a link or multilink property does not refer to a valid
856 node, an IndexError is raised.
858 These operations trigger detectors and can be vetoed. Attempts
859 to modify the "creation" or "activity" properties cause a KeyError.
860 """
861 if self.db.journaltag is None:
862 raise hyperdb.DatabaseError(_('Database open read-only'))
863 self.fireAuditors('create', None, propvalues)
864 newid = self.create_inner(**propvalues)
865 self.fireReactors('create', newid, None)
866 return newid
868 def create_inner(self, **propvalues):
869 """ Called by create, in-between the audit and react calls.
870 """
871 if 'id' in propvalues:
872 raise KeyError('"id" is reserved')
874 if self.db.journaltag is None:
875 raise hyperdb.DatabaseError(_('Database open read-only'))
877 if 'creation' in propvalues or 'activity' in propvalues:
878 raise KeyError('"creation" and "activity" are reserved')
879 # new node's id
880 newid = self.db.newid(self.classname)
882 # validate propvalues
883 num_re = re.compile('^\d+$')
884 for key, value in propvalues.iteritems():
885 if key == self.key:
886 try:
887 self.lookup(value)
888 except KeyError:
889 pass
890 else:
891 raise ValueError('node with key "%s" exists'%value)
893 # try to handle this property
894 try:
895 prop = self.properties[key]
896 except KeyError:
897 raise KeyError('"%s" has no property "%s"'%(self.classname,
898 key))
900 if value is not None and isinstance(prop, hyperdb.Link):
901 if type(value) != type(''):
902 raise ValueError('link value must be String')
903 link_class = self.properties[key].classname
904 # if it isn't a number, it's a key
905 if not num_re.match(value):
906 try:
907 value = self.db.classes[link_class].lookup(value)
908 except (TypeError, KeyError):
909 raise IndexError('new property "%s": %s not a %s'%(
910 key, value, link_class))
911 elif not self.db.getclass(link_class).hasnode(value):
912 raise IndexError('%s has no node %s'%(link_class,
913 value))
915 # save off the value
916 propvalues[key] = value
918 # register the link with the newly linked node
919 if self.do_journal and self.properties[key].do_journal:
920 self.db.addjournal(link_class, value, 'link',
921 (self.classname, newid, key))
923 elif isinstance(prop, hyperdb.Multilink):
924 if value is None:
925 value = []
926 if not hasattr(value, '__iter__'):
927 raise TypeError('new property "%s" not an iterable of ids'%key)
929 # clean up and validate the list of links
930 link_class = self.properties[key].classname
931 l = []
932 for entry in value:
933 if type(entry) != type(''):
934 raise ValueError('"%s" multilink value (%r) '\
935 'must contain Strings'%(key, value))
936 # if it isn't a number, it's a key
937 if not num_re.match(entry):
938 try:
939 entry = self.db.classes[link_class].lookup(entry)
940 except (TypeError, KeyError):
941 raise IndexError('new property "%s": %s not a %s'%(
942 key, entry, self.properties[key].classname))
943 l.append(entry)
944 value = l
945 propvalues[key] = value
947 # handle additions
948 for nodeid in value:
949 if not self.db.getclass(link_class).hasnode(nodeid):
950 raise IndexError('%s has no node %s'%(link_class,
951 nodeid))
952 # register the link with the newly linked node
953 if self.do_journal and self.properties[key].do_journal:
954 self.db.addjournal(link_class, nodeid, 'link',
955 (self.classname, newid, key))
957 elif isinstance(prop, hyperdb.String):
958 if type(value) != type('') and type(value) != type(u''):
959 raise TypeError('new property "%s" not a string'%key)
960 if prop.indexme:
961 self.db.indexer.add_text((self.classname, newid, key),
962 value)
964 elif isinstance(prop, hyperdb.Password):
965 if not isinstance(value, password.Password):
966 raise TypeError('new property "%s" not a Password'%key)
968 elif isinstance(prop, hyperdb.Date):
969 if value is not None and not isinstance(value, date.Date):
970 raise TypeError('new property "%s" not a Date'%key)
972 elif isinstance(prop, hyperdb.Interval):
973 if value is not None and not isinstance(value, date.Interval):
974 raise TypeError('new property "%s" not an Interval'%key)
976 elif value is not None and isinstance(prop, hyperdb.Number):
977 try:
978 float(value)
979 except ValueError:
980 raise TypeError('new property "%s" not numeric'%key)
982 elif value is not None and isinstance(prop, hyperdb.Boolean):
983 try:
984 int(value)
985 except ValueError:
986 raise TypeError('new property "%s" not boolean'%key)
988 # make sure there's data where there needs to be
989 for key, prop in self.properties.iteritems():
990 if key in propvalues:
991 continue
992 if key == self.key:
993 raise ValueError('key property "%s" is required'%key)
994 if isinstance(prop, hyperdb.Multilink):
995 propvalues[key] = []
997 # done
998 self.db.addnode(self.classname, newid, propvalues)
999 if self.do_journal:
1000 self.db.addjournal(self.classname, newid, 'create', {})
1002 return newid
1004 def get(self, nodeid, propname, default=_marker, cache=1):
1005 """Get the value of a property on an existing node of this class.
1007 'nodeid' must be the id of an existing node of this class or an
1008 IndexError is raised. 'propname' must be the name of a property
1009 of this class or a KeyError is raised.
1011 'cache' exists for backward compatibility, and is not used.
1013 Attempts to get the "creation" or "activity" properties should
1014 do the right thing.
1015 """
1016 if propname == 'id':
1017 return nodeid
1019 # get the node's dict
1020 d = self.db.getnode(self.classname, nodeid)
1022 # check for one of the special props
1023 if propname == 'creation':
1024 if 'creation' in d:
1025 return d['creation']
1026 if not self.do_journal:
1027 raise ValueError('Journalling is disabled for this class')
1028 journal = self.db.getjournal(self.classname, nodeid)
1029 if journal:
1030 return journal[0][1]
1031 else:
1032 # on the strange chance that there's no journal
1033 return date.Date()
1034 if propname == 'activity':
1035 if 'activity' in d:
1036 return d['activity']
1037 if not self.do_journal:
1038 raise ValueError('Journalling is disabled for this class')
1039 journal = self.db.getjournal(self.classname, nodeid)
1040 if journal:
1041 return self.db.getjournal(self.classname, nodeid)[-1][1]
1042 else:
1043 # on the strange chance that there's no journal
1044 return date.Date()
1045 if propname == 'creator':
1046 if 'creator' in d:
1047 return d['creator']
1048 if not self.do_journal:
1049 raise ValueError('Journalling is disabled for this class')
1050 journal = self.db.getjournal(self.classname, nodeid)
1051 if journal:
1052 num_re = re.compile('^\d+$')
1053 value = journal[0][2]
1054 if num_re.match(value):
1055 return value
1056 else:
1057 # old-style "username" journal tag
1058 try:
1059 return self.db.user.lookup(value)
1060 except KeyError:
1061 # user's been retired, return admin
1062 return '1'
1063 else:
1064 return self.db.getuid()
1065 if propname == 'actor':
1066 if 'actor' in d:
1067 return d['actor']
1068 if not self.do_journal:
1069 raise ValueError('Journalling is disabled for this class')
1070 journal = self.db.getjournal(self.classname, nodeid)
1071 if journal:
1072 num_re = re.compile('^\d+$')
1073 value = journal[-1][2]
1074 if num_re.match(value):
1075 return value
1076 else:
1077 # old-style "username" journal tag
1078 try:
1079 return self.db.user.lookup(value)
1080 except KeyError:
1081 # user's been retired, return admin
1082 return '1'
1083 else:
1084 return self.db.getuid()
1086 # get the property (raises KeyErorr if invalid)
1087 prop = self.properties[propname]
1089 if propname not in d:
1090 if default is _marker:
1091 if isinstance(prop, hyperdb.Multilink):
1092 return []
1093 else:
1094 return None
1095 else:
1096 return default
1098 # return a dupe of the list so code doesn't get confused
1099 if isinstance(prop, hyperdb.Multilink):
1100 return d[propname][:]
1102 return d[propname]
1104 def set(self, nodeid, **propvalues):
1105 """Modify a property on an existing node of this class.
1107 'nodeid' must be the id of an existing node of this class or an
1108 IndexError is raised.
1110 Each key in 'propvalues' must be the name of a property of this
1111 class or a KeyError is raised.
1113 All values in 'propvalues' must be acceptable types for their
1114 corresponding properties or a TypeError is raised.
1116 If the value of the key property is set, it must not collide with
1117 other key strings or a ValueError is raised.
1119 If the value of a Link or Multilink property contains an invalid
1120 node id, a ValueError is raised.
1122 These operations trigger detectors and can be vetoed. Attempts
1123 to modify the "creation" or "activity" properties cause a KeyError.
1124 """
1125 if self.db.journaltag is None:
1126 raise hyperdb.DatabaseError(_('Database open read-only'))
1128 self.fireAuditors('set', nodeid, propvalues)
1129 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1130 for name, prop in self.getprops(protected=0).iteritems():
1131 if name in oldvalues:
1132 continue
1133 if isinstance(prop, hyperdb.Multilink):
1134 oldvalues[name] = []
1135 else:
1136 oldvalues[name] = None
1137 propvalues = self.set_inner(nodeid, **propvalues)
1138 self.fireReactors('set', nodeid, oldvalues)
1139 return propvalues
1141 def set_inner(self, nodeid, **propvalues):
1142 """ Called by set, in-between the audit and react calls.
1143 """
1144 if not propvalues:
1145 return propvalues
1147 if 'creation' in propvalues or 'activity' in propvalues:
1148 raise KeyError, '"creation" and "activity" are reserved'
1150 if 'id' in propvalues:
1151 raise KeyError, '"id" is reserved'
1153 if self.db.journaltag is None:
1154 raise hyperdb.DatabaseError(_('Database open read-only'))
1156 node = self.db.getnode(self.classname, nodeid)
1157 if self.db.RETIRED_FLAG in node:
1158 raise IndexError
1159 num_re = re.compile('^\d+$')
1161 # if the journal value is to be different, store it in here
1162 journalvalues = {}
1164 # list() propvalues 'cos it might be modified by the loop
1165 for propname, value in list(propvalues.items()):
1166 # check to make sure we're not duplicating an existing key
1167 if propname == self.key and node[propname] != value:
1168 try:
1169 self.lookup(value)
1170 except KeyError:
1171 pass
1172 else:
1173 raise ValueError('node with key "%s" exists'%value)
1175 # this will raise the KeyError if the property isn't valid
1176 # ... we don't use getprops() here because we only care about
1177 # the writeable properties.
1178 try:
1179 prop = self.properties[propname]
1180 except KeyError:
1181 raise KeyError('"%s" has no property named "%s"'%(
1182 self.classname, propname))
1184 # if the value's the same as the existing value, no sense in
1185 # doing anything
1186 current = node.get(propname, None)
1187 if value == current:
1188 del propvalues[propname]
1189 continue
1190 journalvalues[propname] = current
1192 # do stuff based on the prop type
1193 if isinstance(prop, hyperdb.Link):
1194 link_class = prop.classname
1195 # if it isn't a number, it's a key
1196 if value is not None and not isinstance(value, type('')):
1197 raise ValueError('property "%s" link value be a string'%(
1198 propname))
1199 if isinstance(value, type('')) and not num_re.match(value):
1200 try:
1201 value = self.db.classes[link_class].lookup(value)
1202 except (TypeError, KeyError):
1203 raise IndexError('new property "%s": %s not a %s'%(
1204 propname, value, prop.classname))
1206 if (value is not None and
1207 not self.db.getclass(link_class).hasnode(value)):
1208 raise IndexError('%s has no node %s'%(link_class,
1209 value))
1211 if self.do_journal and prop.do_journal:
1212 # register the unlink with the old linked node
1213 if propname in node and node[propname] is not None:
1214 self.db.addjournal(link_class, node[propname], 'unlink',
1215 (self.classname, nodeid, propname))
1217 # register the link with the newly linked node
1218 if value is not None:
1219 self.db.addjournal(link_class, value, 'link',
1220 (self.classname, nodeid, propname))
1222 elif isinstance(prop, hyperdb.Multilink):
1223 if value is None:
1224 value = []
1225 if not hasattr(value, '__iter__'):
1226 raise TypeError('new property "%s" not an iterable of'
1227 ' ids'%propname)
1228 link_class = self.properties[propname].classname
1229 l = []
1230 for entry in value:
1231 # if it isn't a number, it's a key
1232 if type(entry) != type(''):
1233 raise ValueError('new property "%s" link value '
1234 'must be a string'%propname)
1235 if not num_re.match(entry):
1236 try:
1237 entry = self.db.classes[link_class].lookup(entry)
1238 except (TypeError, KeyError):
1239 raise IndexError('new property "%s": %s not a %s'%(
1240 propname, entry,
1241 self.properties[propname].classname))
1242 l.append(entry)
1243 value = l
1244 propvalues[propname] = value
1246 # figure the journal entry for this property
1247 add = []
1248 remove = []
1250 # handle removals
1251 if propname in node:
1252 l = node[propname]
1253 else:
1254 l = []
1255 for id in l[:]:
1256 if id in value:
1257 continue
1258 # register the unlink with the old linked node
1259 if self.do_journal and self.properties[propname].do_journal:
1260 self.db.addjournal(link_class, id, 'unlink',
1261 (self.classname, nodeid, propname))
1262 l.remove(id)
1263 remove.append(id)
1265 # handle additions
1266 for id in value:
1267 if not self.db.getclass(link_class).hasnode(id):
1268 raise IndexError('%s has no node %s'%(link_class,
1269 id))
1270 if id in l:
1271 continue
1272 # register the link with the newly linked node
1273 if self.do_journal and self.properties[propname].do_journal:
1274 self.db.addjournal(link_class, id, 'link',
1275 (self.classname, nodeid, propname))
1276 l.append(id)
1277 add.append(id)
1279 # figure the journal entry
1280 l = []
1281 if add:
1282 l.append(('+', add))
1283 if remove:
1284 l.append(('-', remove))
1285 if l:
1286 journalvalues[propname] = tuple(l)
1288 elif isinstance(prop, hyperdb.String):
1289 if value is not None and type(value) != type('') and type(value) != type(u''):
1290 raise TypeError('new property "%s" not a '
1291 'string'%propname)
1292 if prop.indexme:
1293 self.db.indexer.add_text((self.classname, nodeid, propname),
1294 value)
1296 elif isinstance(prop, hyperdb.Password):
1297 if not isinstance(value, password.Password):
1298 raise TypeError('new property "%s" not a '
1299 'Password'%propname)
1300 propvalues[propname] = value
1301 journalvalues[propname] = \
1302 current and password.JournalPassword(current)
1304 elif value is not None and isinstance(prop, hyperdb.Date):
1305 if not isinstance(value, date.Date):
1306 raise TypeError('new property "%s" not a '
1307 'Date'%propname)
1308 propvalues[propname] = value
1310 elif value is not None and isinstance(prop, hyperdb.Interval):
1311 if not isinstance(value, date.Interval):
1312 raise TypeError('new property "%s" not an '
1313 'Interval'%propname)
1314 propvalues[propname] = value
1316 elif value is not None and isinstance(prop, hyperdb.Number):
1317 try:
1318 float(value)
1319 except ValueError:
1320 raise TypeError('new property "%s" not '
1321 'numeric'%propname)
1323 elif value is not None and isinstance(prop, hyperdb.Boolean):
1324 try:
1325 int(value)
1326 except ValueError:
1327 raise TypeError('new property "%s" not '
1328 'boolean'%propname)
1330 node[propname] = value
1332 # nothing to do?
1333 if not propvalues:
1334 return propvalues
1336 # update the activity time
1337 node['activity'] = date.Date()
1338 node['actor'] = self.db.getuid()
1340 # do the set, and journal it
1341 self.db.setnode(self.classname, nodeid, node)
1343 if self.do_journal:
1344 self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
1346 return propvalues
1348 def retire(self, nodeid):
1349 """Retire a node.
1351 The properties on the node remain available from the get() method,
1352 and the node's id is never reused.
1354 Retired nodes are not returned by the find(), list(), or lookup()
1355 methods, and other nodes may reuse the values of their key properties.
1357 These operations trigger detectors and can be vetoed. Attempts
1358 to modify the "creation" or "activity" properties cause a KeyError.
1359 """
1360 if self.db.journaltag is None:
1361 raise hyperdb.DatabaseError(_('Database open read-only'))
1363 self.fireAuditors('retire', nodeid, None)
1365 node = self.db.getnode(self.classname, nodeid)
1366 node[self.db.RETIRED_FLAG] = 1
1367 self.db.setnode(self.classname, nodeid, node)
1368 if self.do_journal:
1369 self.db.addjournal(self.classname, nodeid, 'retired', None)
1371 self.fireReactors('retire', nodeid, None)
1373 def restore(self, nodeid):
1374 """Restpre a retired node.
1376 Make node available for all operations like it was before retirement.
1377 """
1378 if self.db.journaltag is None:
1379 raise hyperdb.DatabaseError(_('Database open read-only'))
1381 node = self.db.getnode(self.classname, nodeid)
1382 # check if key property was overrided
1383 key = self.getkey()
1384 try:
1385 id = self.lookup(node[key])
1386 except KeyError:
1387 pass
1388 else:
1389 raise KeyError("Key property (%s) of retired node clashes "
1390 "with existing one (%s)" % (key, node[key]))
1391 # Now we can safely restore node
1392 self.fireAuditors('restore', nodeid, None)
1393 del node[self.db.RETIRED_FLAG]
1394 self.db.setnode(self.classname, nodeid, node)
1395 if self.do_journal:
1396 self.db.addjournal(self.classname, nodeid, 'restored', None)
1398 self.fireReactors('restore', nodeid, None)
1400 def is_retired(self, nodeid, cldb=None):
1401 """Return true if the node is retired.
1402 """
1403 node = self.db.getnode(self.classname, nodeid, cldb)
1404 if self.db.RETIRED_FLAG in node:
1405 return 1
1406 return 0
1408 def destroy(self, nodeid):
1409 """Destroy a node.
1411 WARNING: this method should never be used except in extremely rare
1412 situations where there could never be links to the node being
1413 deleted
1415 WARNING: use retire() instead
1417 WARNING: the properties of this node will not be available ever again
1419 WARNING: really, use retire() instead
1421 Well, I think that's enough warnings. This method exists mostly to
1422 support the session storage of the cgi interface.
1423 """
1424 if self.db.journaltag is None:
1425 raise hyperdb.DatabaseError(_('Database open read-only'))
1426 self.db.destroynode(self.classname, nodeid)
1428 # Locating nodes:
1429 def hasnode(self, nodeid):
1430 """Determine if the given nodeid actually exists
1431 """
1432 return self.db.hasnode(self.classname, nodeid)
1434 def setkey(self, propname):
1435 """Select a String property of this class to be the key property.
1437 'propname' must be the name of a String property of this class or
1438 None, or a TypeError is raised. The values of the key property on
1439 all existing nodes must be unique or a ValueError is raised. If the
1440 property doesn't exist, KeyError is raised.
1441 """
1442 prop = self.getprops()[propname]
1443 if not isinstance(prop, hyperdb.String):
1444 raise TypeError('key properties must be String')
1445 self.key = propname
1447 def getkey(self):
1448 """Return the name of the key property for this class or None."""
1449 return self.key
1451 # TODO: set up a separate index db file for this? profile?
1452 def lookup(self, keyvalue):
1453 """Locate a particular node by its key property and return its id.
1455 If this class has no key property, a TypeError is raised. If the
1456 'keyvalue' matches one of the values for the key property among
1457 the nodes in this class, the matching node's id is returned;
1458 otherwise a KeyError is raised.
1459 """
1460 if not self.key:
1461 raise TypeError('No key property set for '
1462 'class %s'%self.classname)
1463 cldb = self.db.getclassdb(self.classname)
1464 try:
1465 for nodeid in self.getnodeids(cldb):
1466 node = self.db.getnode(self.classname, nodeid, cldb)
1467 if self.db.RETIRED_FLAG in node:
1468 continue
1469 if self.key not in node:
1470 continue
1471 if node[self.key] == keyvalue:
1472 return nodeid
1473 finally:
1474 cldb.close()
1475 raise KeyError('No key (%s) value "%s" for "%s"'%(self.key,
1476 keyvalue, self.classname))
1478 # change from spec - allows multiple props to match
1479 def find(self, **propspec):
1480 """Get the ids of nodes in this class which link to the given nodes.
1482 'propspec' consists of keyword args propname=nodeid or
1483 propname={nodeid:1, }
1484 'propname' must be the name of a property in this class, or a
1485 KeyError is raised. That property must be a Link or
1486 Multilink property, or a TypeError is raised.
1488 Any node in this class whose 'propname' property links to any of
1489 the nodeids will be returned. Examples::
1491 db.issue.find(messages='1')
1492 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1493 """
1494 for propname, itemids in propspec.iteritems():
1495 # check the prop is OK
1496 prop = self.properties[propname]
1497 if not isinstance(prop, hyperdb.Link) and not isinstance(prop, hyperdb.Multilink):
1498 raise TypeError("'%s' not a Link/Multilink "
1499 "property"%propname)
1501 # ok, now do the find
1502 cldb = self.db.getclassdb(self.classname)
1503 l = []
1504 try:
1505 for id in self.getnodeids(db=cldb):
1506 item = self.db.getnode(self.classname, id, db=cldb)
1507 if self.db.RETIRED_FLAG in item:
1508 continue
1509 for propname, itemids in propspec.iteritems():
1510 if type(itemids) is not type({}):
1511 itemids = {itemids:1}
1513 # special case if the item doesn't have this property
1514 if propname not in item:
1515 if None in itemids:
1516 l.append(id)
1517 break
1518 continue
1520 # grab the property definition and its value on this item
1521 prop = self.properties[propname]
1522 value = item[propname]
1523 if isinstance(prop, hyperdb.Link) and value in itemids:
1524 l.append(id)
1525 break
1526 elif isinstance(prop, hyperdb.Multilink):
1527 hit = 0
1528 for v in value:
1529 if v in itemids:
1530 l.append(id)
1531 hit = 1
1532 break
1533 if hit:
1534 break
1535 finally:
1536 cldb.close()
1537 return l
1539 def stringFind(self, **requirements):
1540 """Locate a particular node by matching a set of its String
1541 properties in a caseless search.
1543 If the property is not a String property, a TypeError is raised.
1545 The return is a list of the id of all nodes that match.
1546 """
1547 for propname in requirements:
1548 prop = self.properties[propname]
1549 if not isinstance(prop, hyperdb.String):
1550 raise TypeError("'%s' not a String property"%propname)
1551 requirements[propname] = requirements[propname].lower()
1552 l = []
1553 cldb = self.db.getclassdb(self.classname)
1554 try:
1555 for nodeid in self.getnodeids(cldb):
1556 node = self.db.getnode(self.classname, nodeid, cldb)
1557 if self.db.RETIRED_FLAG in node:
1558 continue
1559 for key, value in requirements.iteritems():
1560 if key not in node:
1561 break
1562 if node[key] is None or node[key].lower() != value:
1563 break
1564 else:
1565 l.append(nodeid)
1566 finally:
1567 cldb.close()
1568 return l
1570 def list(self):
1571 """ Return a list of the ids of the active nodes in this class.
1572 """
1573 l = []
1574 cn = self.classname
1575 cldb = self.db.getclassdb(cn)
1576 try:
1577 for nodeid in self.getnodeids(cldb):
1578 node = self.db.getnode(cn, nodeid, cldb)
1579 if self.db.RETIRED_FLAG in node:
1580 continue
1581 l.append(nodeid)
1582 finally:
1583 cldb.close()
1584 l.sort()
1585 return l
1587 def getnodeids(self, db=None, retired=None):
1588 """ Return a list of ALL nodeids
1590 Set retired=None to get all nodes. Otherwise it'll get all the
1591 retired or non-retired nodes, depending on the flag.
1592 """
1593 res = []
1595 # start off with the new nodes
1596 if self.classname in self.db.newnodes:
1597 res.extend(self.db.newnodes[self.classname])
1599 must_close = False
1600 if db is None:
1601 db = self.db.getclassdb(self.classname)
1602 must_close = True
1603 try:
1604 res.extend(db.keys())
1606 # remove the uncommitted, destroyed nodes
1607 if self.classname in self.db.destroyednodes:
1608 for nodeid in self.db.destroyednodes[self.classname]:
1609 if key_in(db, nodeid):
1610 res.remove(nodeid)
1612 # check retired flag
1613 if retired is False or retired is True:
1614 l = []
1615 for nodeid in res:
1616 node = self.db.getnode(self.classname, nodeid, db)
1617 is_ret = self.db.RETIRED_FLAG in node
1618 if retired == is_ret:
1619 l.append(nodeid)
1620 res = l
1621 finally:
1622 if must_close:
1623 db.close()
1624 return res
1626 def _filter(self, search_matches, filterspec, proptree,
1627 num_re = re.compile('^\d+$')):
1628 """Return a list of the ids of the active nodes in this class that
1629 match the 'filter' spec, sorted by the group spec and then the
1630 sort spec.
1632 "filterspec" is {propname: value(s)}
1634 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1635 and prop is a prop name or None
1637 "search_matches" is a sequence type or None
1639 The filter must match all properties specificed. If the property
1640 value to match is a list:
1642 1. String properties must match all elements in the list, and
1643 2. Other properties must match any of the elements in the list.
1644 """
1645 if __debug__:
1646 start_t = time.time()
1648 cn = self.classname
1650 # optimise filterspec
1651 l = []
1652 props = self.getprops()
1653 LINK = 'spec:link'
1654 MULTILINK = 'spec:multilink'
1655 STRING = 'spec:string'
1656 DATE = 'spec:date'
1657 INTERVAL = 'spec:interval'
1658 OTHER = 'spec:other'
1660 for k, v in filterspec.iteritems():
1661 propclass = props[k]
1662 if isinstance(propclass, hyperdb.Link):
1663 if type(v) is not type([]):
1664 v = [v]
1665 u = []
1666 for entry in v:
1667 # the value -1 is a special "not set" sentinel
1668 if entry == '-1':
1669 entry = None
1670 u.append(entry)
1671 l.append((LINK, k, u))
1672 elif isinstance(propclass, hyperdb.Multilink):
1673 # the value -1 is a special "not set" sentinel
1674 if v in ('-1', ['-1']):
1675 v = []
1676 elif type(v) is not type([]):
1677 v = [v]
1678 l.append((MULTILINK, k, v))
1679 elif isinstance(propclass, hyperdb.String) and k != 'id':
1680 if type(v) is not type([]):
1681 v = [v]
1682 for v in v:
1683 # simple glob searching
1684 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1685 v = v.replace('?', '.')
1686 v = v.replace('*', '.*?')
1687 l.append((STRING, k, re.compile(v, re.I)))
1688 elif isinstance(propclass, hyperdb.Date):
1689 try:
1690 date_rng = propclass.range_from_raw(v, self.db)
1691 l.append((DATE, k, date_rng))
1692 except ValueError:
1693 # If range creation fails - ignore that search parameter
1694 pass
1695 elif isinstance(propclass, hyperdb.Interval):
1696 try:
1697 intv_rng = date.Range(v, date.Interval)
1698 l.append((INTERVAL, k, intv_rng))
1699 except ValueError:
1700 # If range creation fails - ignore that search parameter
1701 pass
1703 elif isinstance(propclass, hyperdb.Boolean):
1704 if type(v) == type(""):
1705 v = v.split(',')
1706 if type(v) != type([]):
1707 v = [v]
1708 bv = []
1709 for val in v:
1710 if type(val) is type(''):
1711 bv.append(propclass.from_raw (val))
1712 else:
1713 bv.append(val)
1714 l.append((OTHER, k, bv))
1716 elif k == 'id':
1717 if type(v) != type([]):
1718 v = v.split(',')
1719 l.append((OTHER, k, [str(int(val)) for val in v]))
1721 elif isinstance(propclass, hyperdb.Number):
1722 if type(v) != type([]):
1723 try :
1724 v = v.split(',')
1725 except AttributeError :
1726 v = [v]
1727 l.append((OTHER, k, [float(val) for val in v]))
1729 filterspec = l
1731 # now, find all the nodes that are active and pass filtering
1732 matches = []
1733 cldb = self.db.getclassdb(cn)
1734 t = 0
1735 try:
1736 # TODO: only full-scan once (use items())
1737 for nodeid in self.getnodeids(cldb):
1738 node = self.db.getnode(cn, nodeid, cldb)
1739 if self.db.RETIRED_FLAG in node:
1740 continue
1741 # apply filter
1742 for t, k, v in filterspec:
1743 # handle the id prop
1744 if k == 'id':
1745 if nodeid not in v:
1746 break
1747 continue
1749 # get the node value
1750 nv = node.get(k, None)
1752 match = 0
1754 # now apply the property filter
1755 if t == LINK:
1756 # link - if this node's property doesn't appear in the
1757 # filterspec's nodeid list, skip it
1758 match = nv in v
1759 elif t == MULTILINK:
1760 # multilink - if any of the nodeids required by the
1761 # filterspec aren't in this node's property, then skip
1762 # it
1763 nv = node.get(k, [])
1765 # check for matching the absence of multilink values
1766 if not v:
1767 match = not nv
1768 else:
1769 # otherwise, make sure this node has each of the
1770 # required values
1771 expr = Expression(v)
1772 if expr.evaluate(nv): match = 1
1773 elif t == STRING:
1774 if nv is None:
1775 nv = ''
1776 # RE search
1777 match = v.search(nv)
1778 elif t == DATE or t == INTERVAL:
1779 if nv is None:
1780 match = v is None
1781 else:
1782 if v.to_value:
1783 if v.from_value <= nv and v.to_value >= nv:
1784 match = 1
1785 else:
1786 if v.from_value <= nv:
1787 match = 1
1788 elif t == OTHER:
1789 # straight value comparison for the other types
1790 match = nv in v
1791 if not match:
1792 break
1793 else:
1794 matches.append([nodeid, node])
1796 # filter based on full text search
1797 if search_matches is not None:
1798 k = []
1799 for v in matches:
1800 if v[0] in search_matches:
1801 k.append(v)
1802 matches = k
1804 # add sorting information to the proptree
1805 JPROPS = {'actor':1, 'activity':1, 'creator':1, 'creation':1}
1806 children = []
1807 if proptree:
1808 children = proptree.sortable_children()
1809 for pt in children:
1810 dir = pt.sort_direction
1811 prop = pt.name
1812 assert (dir and prop)
1813 propclass = props[prop]
1814 pt.sort_ids = []
1815 is_pointer = isinstance(propclass,(hyperdb.Link,
1816 hyperdb.Multilink))
1817 if not is_pointer:
1818 pt.sort_result = []
1819 try:
1820 # cache the opened link class db, if needed.
1821 lcldb = None
1822 # cache the linked class items too
1823 lcache = {}
1825 for entry in matches:
1826 itemid = entry[-2]
1827 item = entry[-1]
1828 # handle the properties that might be "faked"
1829 # also, handle possible missing properties
1830 try:
1831 v = item[prop]
1832 except KeyError:
1833 if prop in JPROPS:
1834 # force lookup of the special journal prop
1835 v = self.get(itemid, prop)
1836 else:
1837 # the node doesn't have a value for this
1838 # property
1839 v = None
1840 if isinstance(propclass, hyperdb.Multilink):
1841 v = []
1842 if prop == 'id':
1843 v = int (itemid)
1844 pt.sort_ids.append(v)
1845 if not is_pointer:
1846 pt.sort_result.append(v)
1847 continue
1849 # missing (None) values are always sorted first
1850 if v is None:
1851 pt.sort_ids.append(v)
1852 if not is_pointer:
1853 pt.sort_result.append(v)
1854 continue
1856 if isinstance(propclass, hyperdb.Link):
1857 lcn = propclass.classname
1858 link = self.db.classes[lcn]
1859 key = link.orderprop()
1860 child = pt.propdict[key]
1861 if key!='id':
1862 if v not in lcache:
1863 # open the link class db if it's not already
1864 if lcldb is None:
1865 lcldb = self.db.getclassdb(lcn)
1866 lcache[v] = self.db.getnode(lcn, v, lcldb)
1867 r = lcache[v][key]
1868 child.propdict[key].sort_ids.append(r)
1869 else:
1870 child.propdict[key].sort_ids.append(v)
1871 pt.sort_ids.append(v)
1872 if not is_pointer:
1873 r = propclass.sort_repr(pt.parent.cls, v, pt.name)
1874 pt.sort_result.append(r)
1875 finally:
1876 # if we opened the link class db, close it now
1877 if lcldb is not None:
1878 lcldb.close()
1879 del lcache
1880 finally:
1881 cldb.close()
1883 # pull the id out of the individual entries
1884 matches = [entry[-2] for entry in matches]
1885 if __debug__:
1886 self.db.stats['filtering'] += (time.time() - start_t)
1887 return matches
1889 def count(self):
1890 """Get the number of nodes in this class.
1892 If the returned integer is 'numnodes', the ids of all the nodes
1893 in this class run from 1 to numnodes, and numnodes+1 will be the
1894 id of the next node to be created in this class.
1895 """
1896 return self.db.countnodes(self.classname)
1898 # Manipulating properties:
1900 def getprops(self, protected=1):
1901 """Return a dictionary mapping property names to property objects.
1902 If the "protected" flag is true, we include protected properties -
1903 those which may not be modified.
1905 In addition to the actual properties on the node, these
1906 methods provide the "creation" and "activity" properties. If the
1907 "protected" flag is true, we include protected properties - those
1908 which may not be modified.
1909 """
1910 d = self.properties.copy()
1911 if protected:
1912 d['id'] = hyperdb.String()
1913 d['creation'] = hyperdb.Date()
1914 d['activity'] = hyperdb.Date()
1915 d['creator'] = hyperdb.Link('user')
1916 d['actor'] = hyperdb.Link('user')
1917 return d
1919 def addprop(self, **properties):
1920 """Add properties to this class.
1922 The keyword arguments in 'properties' must map names to property
1923 objects, or a TypeError is raised. None of the keys in 'properties'
1924 may collide with the names of existing properties, or a ValueError
1925 is raised before any properties have been added.
1926 """
1927 for key in properties:
1928 if key in self.properties:
1929 raise ValueError(key)
1930 self.properties.update(properties)
1932 def index(self, nodeid):
1933 """ Add (or refresh) the node to search indexes """
1934 # find all the String properties that have indexme
1935 for prop, propclass in self.getprops().iteritems():
1936 if isinstance(propclass, hyperdb.String) and propclass.indexme:
1937 # index them under (classname, nodeid, property)
1938 try:
1939 value = str(self.get(nodeid, prop))
1940 except IndexError:
1941 # node has been destroyed
1942 continue
1943 self.db.indexer.add_text((self.classname, nodeid, prop), value)
1945 #
1946 # import / export support
1947 #
1948 def export_list(self, propnames, nodeid):
1949 """ Export a node - generate a list of CSV-able data in the order
1950 specified by propnames for the given node.
1951 """
1952 properties = self.getprops()
1953 l = []
1954 for prop in propnames:
1955 proptype = properties[prop]
1956 value = self.get(nodeid, prop)
1957 # "marshal" data where needed
1958 if value is None:
1959 pass
1960 elif isinstance(proptype, hyperdb.Date):
1961 value = value.get_tuple()
1962 elif isinstance(proptype, hyperdb.Interval):
1963 value = value.get_tuple()
1964 elif isinstance(proptype, hyperdb.Password):
1965 value = str(value)
1966 l.append(repr(value))
1968 # append retired flag
1969 l.append(repr(self.is_retired(nodeid)))
1971 return l
1973 def import_list(self, propnames, proplist):
1974 """ Import a node - all information including "id" is present and
1975 should not be sanity checked. Triggers are not triggered. The
1976 journal should be initialised using the "creator" and "created"
1977 information.
1979 Return the nodeid of the node imported.
1980 """
1981 if self.db.journaltag is None:
1982 raise hyperdb.DatabaseError(_('Database open read-only'))
1983 properties = self.getprops()
1985 # make the new node's property map
1986 d = {}
1987 newid = None
1988 for i in range(len(propnames)):
1989 # Figure the property for this column
1990 propname = propnames[i]
1992 # Use eval to reverse the repr() used to output the CSV
1993 value = eval(proplist[i])
1995 # "unmarshal" where necessary
1996 if propname == 'id':
1997 newid = value
1998 continue
1999 elif propname == 'is retired':
2000 # is the item retired?
2001 if int(value):
2002 d[self.db.RETIRED_FLAG] = 1
2003 continue
2004 elif value is None:
2005 d[propname] = None
2006 continue
2008 prop = properties[propname]
2009 if isinstance(prop, hyperdb.Date):
2010 value = date.Date(value)
2011 elif isinstance(prop, hyperdb.Interval):
2012 value = date.Interval(value)
2013 elif isinstance(prop, hyperdb.Password):
2014 value = password.Password(encrypted=value)
2015 d[propname] = value
2017 # get a new id if necessary
2018 if newid is None:
2019 newid = self.db.newid(self.classname)
2021 # add the node and journal
2022 self.db.addnode(self.classname, newid, d)
2023 return newid
2025 def export_journals(self):
2026 """Export a class's journal - generate a list of lists of
2027 CSV-able data:
2029 nodeid, date, user, action, params
2031 No heading here - the columns are fixed.
2032 """
2033 properties = self.getprops()
2034 r = []
2035 for nodeid in self.getnodeids():
2036 for nodeid, date, user, action, params in self.history(nodeid):
2037 date = date.get_tuple()
2038 if action == 'set':
2039 export_data = {}
2040 for propname, value in params.iteritems():
2041 if propname not in properties:
2042 # property no longer in the schema
2043 continue
2045 prop = properties[propname]
2046 # make sure the params are eval()'able
2047 if value is None:
2048 pass
2049 elif isinstance(prop, hyperdb.Date):
2050 # this is a hack - some dates are stored as strings
2051 if not isinstance(value, type('')):
2052 value = value.get_tuple()
2053 elif isinstance(prop, hyperdb.Interval):
2054 # hack too - some intervals are stored as strings
2055 if not isinstance(value, type('')):
2056 value = value.get_tuple()
2057 elif isinstance(prop, hyperdb.Password):
2058 value = str(value)
2059 export_data[propname] = value
2060 params = export_data
2061 r.append([repr(nodeid), repr(date), repr(user),
2062 repr(action), repr(params)])
2063 return r
2065 class FileClass(hyperdb.FileClass, Class):
2066 """This class defines a large chunk of data. To support this, it has a
2067 mandatory String property "content" which is typically saved off
2068 externally to the hyperdb.
2070 The default MIME type of this data is defined by the
2071 "default_mime_type" class attribute, which may be overridden by each
2072 node if the class defines a "type" String property.
2073 """
2074 def __init__(self, db, classname, **properties):
2075 """The newly-created class automatically includes the "content"
2076 and "type" properties.
2077 """
2078 if 'content' not in properties:
2079 properties['content'] = hyperdb.String(indexme='yes')
2080 if 'type' not in properties:
2081 properties['type'] = hyperdb.String()
2082 Class.__init__(self, db, classname, **properties)
2084 def create(self, **propvalues):
2085 """ Snarf the "content" propvalue and store in a file
2086 """
2087 # we need to fire the auditors now, or the content property won't
2088 # be in propvalues for the auditors to play with
2089 self.fireAuditors('create', None, propvalues)
2091 # now remove the content property so it's not stored in the db
2092 content = propvalues['content']
2093 del propvalues['content']
2095 # make sure we have a MIME type
2096 mime_type = propvalues.get('type', self.default_mime_type)
2098 # do the database create
2099 newid = self.create_inner(**propvalues)
2101 # store off the content as a file
2102 self.db.storefile(self.classname, newid, None, content)
2104 # fire reactors
2105 self.fireReactors('create', newid, None)
2107 return newid
2109 def get(self, nodeid, propname, default=_marker, cache=1):
2110 """ Trap the content propname and get it from the file
2112 'cache' exists for backwards compatibility, and is not used.
2113 """
2114 poss_msg = 'Possibly an access right configuration problem.'
2115 if propname == 'content':
2116 try:
2117 return self.db.getfile(self.classname, nodeid, None)
2118 except IOError, strerror:
2119 # XXX by catching this we don't see an error in the log.
2120 return 'ERROR reading file: %s%s\n%s\n%s'%(
2121 self.classname, nodeid, poss_msg, strerror)
2122 if default is not _marker:
2123 return Class.get(self, nodeid, propname, default)
2124 else:
2125 return Class.get(self, nodeid, propname)
2127 def set(self, itemid, **propvalues):
2128 """ Snarf the "content" propvalue and update it in a file
2129 """
2130 self.fireAuditors('set', itemid, propvalues)
2132 # create the oldvalues dict - fill in any missing values
2133 oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid))
2134 for name, prop in self.getprops(protected=0).iteritems():
2135 if name in oldvalues:
2136 continue
2137 if isinstance(prop, hyperdb.Multilink):
2138 oldvalues[name] = []
2139 else:
2140 oldvalues[name] = None
2142 # now remove the content property so it's not stored in the db
2143 content = None
2144 if 'content' in propvalues:
2145 content = propvalues['content']
2146 del propvalues['content']
2148 # do the database update
2149 propvalues = self.set_inner(itemid, **propvalues)
2151 # do content?
2152 if content:
2153 # store and possibly index
2154 self.db.storefile(self.classname, itemid, None, content)
2155 if self.properties['content'].indexme:
2156 mime_type = self.get(itemid, 'type', self.default_mime_type)
2157 self.db.indexer.add_text((self.classname, itemid, 'content'),
2158 content, mime_type)
2159 propvalues['content'] = content
2161 # fire reactors
2162 self.fireReactors('set', itemid, oldvalues)
2163 return propvalues
2165 def index(self, nodeid):
2166 """ Add (or refresh) the node to search indexes.
2168 Use the content-type property for the content property.
2169 """
2170 # find all the String properties that have indexme
2171 for prop, propclass in self.getprops().iteritems():
2172 if prop == 'content' and propclass.indexme:
2173 mime_type = self.get(nodeid, 'type', self.default_mime_type)
2174 self.db.indexer.add_text((self.classname, nodeid, 'content'),
2175 str(self.get(nodeid, 'content')), mime_type)
2176 elif isinstance(propclass, hyperdb.String) and propclass.indexme:
2177 # index them under (classname, nodeid, property)
2178 try:
2179 value = str(self.get(nodeid, prop))
2180 except IndexError:
2181 # node has been destroyed
2182 continue
2183 self.db.indexer.add_text((self.classname, nodeid, prop), value)
2185 # deviation from spec - was called ItemClass
2186 class IssueClass(Class, roundupdb.IssueClass):
2187 # Overridden methods:
2188 def __init__(self, db, classname, **properties):
2189 """The newly-created class automatically includes the "messages",
2190 "files", "nosy", and "superseder" properties. If the 'properties'
2191 dictionary attempts to specify any of these properties or a
2192 "creation" or "activity" property, a ValueError is raised.
2193 """
2194 if 'title' not in properties:
2195 properties['title'] = hyperdb.String(indexme='yes')
2196 if 'messages' not in properties:
2197 properties['messages'] = hyperdb.Multilink("msg")
2198 if 'files' not in properties:
2199 properties['files'] = hyperdb.Multilink("file")
2200 if 'nosy' not in properties:
2201 # note: journalling is turned off as it really just wastes
2202 # space. this behaviour may be overridden in an instance
2203 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2204 if 'superseder' not in properties:
2205 properties['superseder'] = hyperdb.Multilink(classname)
2206 Class.__init__(self, db, classname, **properties)
2208 # vim: set et sts=4 sw=4 :