f8ba81399c6a232a9c9642f22260facdff304d5c
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 """This module defines a backend that saves the hyperdatabase in a
19 database chosen by anydbm. It is guaranteed to always be available in python
20 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
21 serious bugs, and is not available)
22 """
23 __docformat__ = 'restructuredtext'
25 import os, marshal, re, weakref, string, copy, time, shutil, logging
27 from roundup.anypy.dbm_ import anydbm, whichdb, key_in
29 from roundup import hyperdb, date, password, roundupdb, security, support
30 from roundup.support import reversed
31 from roundup.backends import locking
32 from roundup.i18n import _
34 from roundup.backends.blobfiles import FileStorage
35 from roundup.backends.sessions_dbm import Sessions, OneTimeKeys
37 try:
38 from roundup.backends.indexer_xapian import Indexer
39 except ImportError:
40 from roundup.backends.indexer_dbm import Indexer
42 def db_exists(config):
43 # check for the user db
44 for db in 'nodes.user nodes.user.db'.split():
45 if os.path.exists(os.path.join(config.DATABASE, db)):
46 return 1
47 return 0
49 def db_nuke(config):
50 shutil.rmtree(config.DATABASE)
52 class Binary:
54 def __init__(self, x, y):
55 self.x = x
56 self.y = y
58 def visit(self, visitor):
59 self.x.visit(visitor)
60 self.y.visit(visitor)
62 class Unary:
64 def __init__(self, x):
65 self.x = x
67 def generate(self, atom):
68 return atom(self)
70 def visit(self, visitor):
71 self.x.visit(visitor)
73 class Equals(Unary):
75 def evaluate(self, v):
76 return self.x in v
78 def visit(self, visitor):
79 visitor(self)
81 class Not(Unary):
83 def evaluate(self, v):
84 return not self.x.evaluate(v)
86 def generate(self, atom):
87 return "NOT(%s)" % self.x.generate(atom)
89 class Or(Binary):
91 def evaluate(self, v):
92 return self.x.evaluate(v) or self.y.evaluate(v)
94 def generate(self, atom):
95 return "(%s)OR(%s)" % (
96 self.x.generate(atom),
97 self.y.generate(atom))
99 class And(Binary):
101 def evaluate(self, v):
102 return self.x.evaluate(v) and self.y.evaluate(v)
104 def generate(self, atom):
105 return "(%s)AND(%s)" % (
106 self.x.generate(atom),
107 self.y.generate(atom))
109 def compile_expression(opcodes):
111 stack = []
112 push, pop = stack.append, stack.pop
113 for opcode in opcodes:
114 if opcode == -2: push(Not(pop()))
115 elif opcode == -3: push(And(pop(), pop()))
116 elif opcode == -4: push(Or(pop(), pop()))
117 else: push(Equals(opcode))
119 return pop()
121 class Expression:
123 def __init__(self, v):
124 try:
125 opcodes = [int(x) for x in v]
126 if min(opcodes) >= -1: raise ValueError()
128 compiled = compile_expression(opcodes)
129 self.evaluate = lambda x: compiled.evaluate([int(y) for y in x])
130 except:
131 self.evaluate = lambda x: bool(set(x) & set(v))
133 #
134 # Now the database
135 #
136 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
137 """A database for storing records containing flexible data types.
139 Transaction stuff TODO:
141 - check the timestamp of the class file and nuke the cache if it's
142 modified. Do some sort of conflict checking on the dirty stuff.
143 - perhaps detect write collisions (related to above)?
144 """
145 def __init__(self, config, journaltag=None):
146 """Open a hyperdatabase given a specifier to some storage.
148 The 'storagelocator' is obtained from config.DATABASE.
149 The meaning of 'storagelocator' depends on the particular
150 implementation of the hyperdatabase. It could be a file name,
151 a directory path, a socket descriptor for a connection to a
152 database over the network, etc.
154 The 'journaltag' is a token that will be attached to the journal
155 entries for any edits done on the database. If 'journaltag' is
156 None, the database is opened in read-only mode: the Class.create(),
157 Class.set(), Class.retire(), and Class.restore() methods are
158 disabled.
159 """
160 FileStorage.__init__(self, config.UMASK)
161 self.config, self.journaltag = config, journaltag
162 self.dir = config.DATABASE
163 self.classes = {}
164 self.cache = {} # cache of nodes loaded or created
165 self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0,
166 'filtering': 0}
167 self.dirtynodes = {} # keep track of the dirty nodes by class
168 self.newnodes = {} # keep track of the new nodes by class
169 self.destroyednodes = {}# keep track of the destroyed nodes by class
170 self.transactions = []
171 self.indexer = Indexer(self)
172 self.security = security.Security(self)
173 os.umask(config.UMASK)
175 # lock it
176 lockfilenm = os.path.join(self.dir, 'lock')
177 self.lockfile = locking.acquire_lock(lockfilenm)
178 self.lockfile.write(str(os.getpid()))
179 self.lockfile.flush()
181 def post_init(self):
182 """Called once the schema initialisation has finished.
183 """
184 # reindex the db if necessary
185 if self.indexer.should_reindex():
186 self.reindex()
188 def refresh_database(self):
189 """Rebuild the database
190 """
191 self.reindex()
193 def getSessionManager(self):
194 return Sessions(self)
196 def getOTKManager(self):
197 return OneTimeKeys(self)
199 def reindex(self, classname=None, show_progress=False):
200 if classname:
201 classes = [self.getclass(classname)]
202 else:
203 classes = self.classes.values()
204 for klass in classes:
205 if show_progress:
206 for nodeid in support.Progress('Reindex %s'%klass.classname,
207 klass.list()):
208 klass.index(nodeid)
209 else:
210 for nodeid in klass.list():
211 klass.index(nodeid)
212 self.indexer.save_index()
214 def __repr__(self):
215 return '<back_anydbm instance at %x>'%id(self)
217 #
218 # Classes
219 #
220 def __getattr__(self, classname):
221 """A convenient way of calling self.getclass(classname)."""
222 if classname in self.classes:
223 return self.classes[classname]
224 raise AttributeError, classname
226 def addclass(self, cl):
227 cn = cl.classname
228 if cn in self.classes:
229 raise ValueError, cn
230 self.classes[cn] = cl
232 # add default Edit and View permissions
233 self.security.addPermission(name="Create", klass=cn,
234 description="User is allowed to create "+cn)
235 self.security.addPermission(name="Edit", klass=cn,
236 description="User is allowed to edit "+cn)
237 self.security.addPermission(name="View", klass=cn,
238 description="User is allowed to access "+cn)
240 def getclasses(self):
241 """Return a list of the names of all existing classes."""
242 l = self.classes.keys()
243 l.sort()
244 return l
246 def getclass(self, classname):
247 """Get the Class object representing a particular class.
249 If 'classname' is not a valid class name, a KeyError is raised.
250 """
251 try:
252 return self.classes[classname]
253 except KeyError:
254 raise KeyError('There is no class called "%s"'%classname)
256 #
257 # Class DBs
258 #
259 def clear(self):
260 """Delete all database contents
261 """
262 logging.getLogger('roundup.hyperdb').info('clear')
263 for cn in self.classes:
264 for dummy in 'nodes', 'journals':
265 path = os.path.join(self.dir, 'journals.%s'%cn)
266 if os.path.exists(path):
267 os.remove(path)
268 elif os.path.exists(path+'.db'): # dbm appends .db
269 os.remove(path+'.db')
270 # reset id sequences
271 path = os.path.join(os.getcwd(), self.dir, '_ids')
272 if os.path.exists(path):
273 os.remove(path)
274 elif os.path.exists(path+'.db'): # dbm appends .db
275 os.remove(path+'.db')
277 def getclassdb(self, classname, mode='r'):
278 """ grab a connection to the class db that will be used for
279 multiple actions
280 """
281 return self.opendb('nodes.%s'%classname, mode)
283 def determine_db_type(self, path):
284 """ determine which DB wrote the class file
285 """
286 db_type = ''
287 if os.path.exists(path):
288 db_type = whichdb(path)
289 if not db_type:
290 raise hyperdb.DatabaseError(_("Couldn't identify database type"))
291 elif os.path.exists(path+'.db'):
292 # if the path ends in '.db', it's a dbm database, whether
293 # anydbm says it's dbhash or not!
294 db_type = 'dbm'
295 return db_type
297 def opendb(self, name, mode):
298 """Low-level database opener that gets around anydbm/dbm
299 eccentricities.
300 """
301 # figure the class db type
302 path = os.path.join(os.getcwd(), self.dir, name)
303 db_type = self.determine_db_type(path)
305 # new database? let anydbm pick the best dbm
306 # in Python 3+ the "dbm" ("anydbm" to us) module already uses the
307 # whichdb() function to do this
308 if not db_type or hasattr(anydbm, 'whichdb'):
309 if __debug__:
310 logging.getLogger('roundup.hyperdb').debug(
311 "opendb anydbm.open(%r, 'c')"%path)
312 return anydbm.open(path, 'c')
314 # in Python <3 it anydbm was a little dumb so manually open the
315 # database with the correct module
316 try:
317 dbm = __import__(db_type)
318 except ImportError:
319 raise hyperdb.DatabaseError(_("Couldn't open database - the "
320 "required module '%s' is not available")%db_type)
321 if __debug__:
322 logging.getLogger('roundup.hyperdb').debug(
323 "opendb %r.open(%r, %r)"%(db_type, path, mode))
324 return dbm.open(path, mode)
326 #
327 # Node IDs
328 #
329 def newid(self, classname):
330 """ Generate a new id for the given class
331 """
332 # open the ids DB - create if if doesn't exist
333 db = self.opendb('_ids', 'c')
334 if key_in(db, classname):
335 newid = db[classname] = str(int(db[classname]) + 1)
336 else:
337 # the count() bit is transitional - older dbs won't start at 1
338 newid = str(self.getclass(classname).count()+1)
339 db[classname] = newid
340 db.close()
341 return newid
343 def setid(self, classname, setid):
344 """ Set the id counter: used during import of database
345 """
346 # open the ids DB - create if if doesn't exist
347 db = self.opendb('_ids', 'c')
348 db[classname] = str(setid)
349 db.close()
351 #
352 # Nodes
353 #
354 def addnode(self, classname, nodeid, node):
355 """ add the specified node to its class's db
356 """
357 # we'll be supplied these props if we're doing an import
358 if 'creator' not in node:
359 # add in the "calculated" properties (dupe so we don't affect
360 # calling code's node assumptions)
361 node = node.copy()
362 node['creator'] = self.getuid()
363 node['actor'] = self.getuid()
364 node['creation'] = node['activity'] = date.Date()
366 self.newnodes.setdefault(classname, {})[nodeid] = 1
367 self.cache.setdefault(classname, {})[nodeid] = node
368 self.savenode(classname, nodeid, node)
370 def setnode(self, classname, nodeid, node):
371 """ change the specified node
372 """
373 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
375 # can't set without having already loaded the node
376 self.cache[classname][nodeid] = node
377 self.savenode(classname, nodeid, node)
379 def savenode(self, classname, nodeid, node):
380 """ perform the saving of data specified by the set/addnode
381 """
382 if __debug__:
383 logging.getLogger('roundup.hyperdb').debug(
384 'save %s%s %r'%(classname, nodeid, node))
385 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
387 def getnode(self, classname, nodeid, db=None, cache=1):
388 """ get a node from the database
390 Note the "cache" parameter is not used, and exists purely for
391 backward compatibility!
392 """
393 # try the cache
394 cache_dict = self.cache.setdefault(classname, {})
395 if nodeid in cache_dict:
396 if __debug__:
397 logging.getLogger('roundup.hyperdb').debug(
398 'get %s%s cached'%(classname, nodeid))
399 self.stats['cache_hits'] += 1
400 return cache_dict[nodeid]
402 if __debug__:
403 self.stats['cache_misses'] += 1
404 start_t = time.time()
405 logging.getLogger('roundup.hyperdb').debug(
406 'get %s%s'%(classname, nodeid))
408 # get from the database and save in the cache
409 if db is None:
410 db = self.getclassdb(classname)
411 if not key_in(db, nodeid):
412 raise IndexError("no such %s %s"%(classname, nodeid))
414 # check the uncommitted, destroyed nodes
415 if (classname in self.destroyednodes and
416 nodeid in self.destroyednodes[classname]):
417 raise IndexError("no such %s %s"%(classname, nodeid))
419 # decode
420 res = marshal.loads(db[nodeid])
422 # reverse the serialisation
423 res = self.unserialise(classname, res)
425 # store off in the cache dict
426 if cache:
427 cache_dict[nodeid] = res
429 if __debug__:
430 self.stats['get_items'] += (time.time() - start_t)
432 return res
434 def destroynode(self, classname, nodeid):
435 """Remove a node from the database. Called exclusively by the
436 destroy() method on Class.
437 """
438 logging.getLogger('roundup.hyperdb').info(
439 'destroy %s%s'%(classname, nodeid))
441 # remove from cache and newnodes if it's there
442 if (classname in self.cache and nodeid in self.cache[classname]):
443 del self.cache[classname][nodeid]
444 if (classname in self.newnodes and nodeid in self.newnodes[classname]):
445 del self.newnodes[classname][nodeid]
447 # see if there's any obvious commit actions that we should get rid of
448 for entry in self.transactions[:]:
449 if entry[1][:2] == (classname, nodeid):
450 self.transactions.remove(entry)
452 # add to the destroyednodes map
453 self.destroyednodes.setdefault(classname, {})[nodeid] = 1
455 # add the destroy commit action
456 self.transactions.append((self.doDestroyNode, (classname, nodeid)))
457 self.transactions.append((FileStorage.destroy, (self, classname, nodeid)))
459 def serialise(self, classname, node):
460 """Copy the node contents, converting non-marshallable data into
461 marshallable data.
462 """
463 properties = self.getclass(classname).getprops()
464 d = {}
465 for k, v in node.iteritems():
466 if k == self.RETIRED_FLAG:
467 d[k] = v
468 continue
470 # if the property doesn't exist then we really don't care
471 if k not in properties:
472 continue
474 # get the property spec
475 prop = properties[k]
477 if isinstance(prop, hyperdb.Password) and v is not None:
478 d[k] = str(v)
479 elif isinstance(prop, hyperdb.Date) and v is not None:
480 d[k] = v.serialise()
481 elif isinstance(prop, hyperdb.Interval) and v is not None:
482 d[k] = v.serialise()
483 else:
484 d[k] = v
485 return d
487 def unserialise(self, classname, node):
488 """Decode the marshalled node data
489 """
490 properties = self.getclass(classname).getprops()
491 d = {}
492 for k, v in node.iteritems():
493 # if the property doesn't exist, or is the "retired" flag then
494 # it won't be in the properties dict
495 if k not in properties:
496 d[k] = v
497 continue
499 # get the property spec
500 prop = properties[k]
502 if isinstance(prop, hyperdb.Date) and v is not None:
503 d[k] = date.Date(v)
504 elif isinstance(prop, hyperdb.Interval) and v is not None:
505 d[k] = date.Interval(v)
506 elif isinstance(prop, hyperdb.Password) and v is not None:
507 d[k] = password.Password(encrypted=v)
508 else:
509 d[k] = v
510 return d
512 def hasnode(self, classname, nodeid, db=None):
513 """ determine if the database has a given node
514 """
515 # try the cache
516 cache = self.cache.setdefault(classname, {})
517 if nodeid in cache:
518 return 1
520 # not in the cache - check the database
521 if db is None:
522 db = self.getclassdb(classname)
523 return key_in(db, nodeid)
525 def countnodes(self, classname, db=None):
526 count = 0
528 # include the uncommitted nodes
529 if classname in self.newnodes:
530 count += len(self.newnodes[classname])
531 if classname in self.destroyednodes:
532 count -= len(self.destroyednodes[classname])
534 # and count those in the DB
535 if db is None:
536 db = self.getclassdb(classname)
537 return count + len(db)
540 #
541 # Files - special node properties
542 # inherited from FileStorage
544 #
545 # Journal
546 #
547 def addjournal(self, classname, nodeid, action, params, creator=None,
548 creation=None):
549 """ Journal the Action
550 'action' may be:
552 'create' or 'set' -- 'params' is a dictionary of property values
553 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
554 'retire' -- 'params' is None
556 'creator' -- the user performing the action, which defaults to
557 the current user.
558 """
559 if __debug__:
560 logging.getLogger('roundup.hyperdb').debug(
561 'addjournal %s%s %s %r %s %r'%(classname,
562 nodeid, action, params, creator, creation))
563 if creator is None:
564 creator = self.getuid()
565 self.transactions.append((self.doSaveJournal, (classname, nodeid,
566 action, params, creator, creation)))
568 def setjournal(self, classname, nodeid, journal):
569 """Set the journal to the "journal" list."""
570 if __debug__:
571 logging.getLogger('roundup.hyperdb').debug(
572 'setjournal %s%s %r'%(classname, nodeid, journal))
573 self.transactions.append((self.doSetJournal, (classname, nodeid,
574 journal)))
576 def getjournal(self, classname, nodeid):
577 """ get the journal for id
579 Raise IndexError if the node doesn't exist (as per history()'s
580 API)
581 """
582 # our journal result
583 res = []
585 # add any journal entries for transactions not committed to the
586 # database
587 for method, args in self.transactions:
588 if method != self.doSaveJournal:
589 continue
590 (cache_classname, cache_nodeid, cache_action, cache_params,
591 cache_creator, cache_creation) = args
592 if cache_classname == classname and cache_nodeid == nodeid:
593 if not cache_creator:
594 cache_creator = self.getuid()
595 if not cache_creation:
596 cache_creation = date.Date()
597 res.append((cache_nodeid, cache_creation, cache_creator,
598 cache_action, cache_params))
600 # attempt to open the journal - in some rare cases, the journal may
601 # not exist
602 try:
603 db = self.opendb('journals.%s'%classname, 'r')
604 except anydbm.error, error:
605 if str(error) == "need 'c' or 'n' flag to open new db":
606 raise IndexError('no such %s %s'%(classname, nodeid))
607 elif error.args[0] != 2:
608 # this isn't a "not found" error, be alarmed!
609 raise
610 if res:
611 # we have unsaved journal entries, return them
612 return res
613 raise IndexError('no such %s %s'%(classname, nodeid))
614 try:
615 journal = marshal.loads(db[nodeid])
616 except KeyError:
617 db.close()
618 if res:
619 # we have some unsaved journal entries, be happy!
620 return res
621 raise IndexError('no such %s %s'%(classname, nodeid))
622 db.close()
624 # add all the saved journal entries for this node
625 for nodeid, date_stamp, user, action, params in journal:
626 res.append((nodeid, date.Date(date_stamp), user, action, params))
627 return res
629 def pack(self, pack_before):
630 """ Delete all journal entries except "create" before 'pack_before'.
631 """
632 pack_before = pack_before.serialise()
633 for classname in self.getclasses():
634 packed = 0
635 # get the journal db
636 db_name = 'journals.%s'%classname
637 path = os.path.join(os.getcwd(), self.dir, classname)
638 db_type = self.determine_db_type(path)
639 db = self.opendb(db_name, 'w')
641 for key in db.keys():
642 # get the journal for this db entry
643 journal = marshal.loads(db[key])
644 l = []
645 last_set_entry = None
646 for entry in journal:
647 # unpack the entry
648 (nodeid, date_stamp, self.journaltag, action,
649 params) = entry
650 # if the entry is after the pack date, _or_ the initial
651 # create entry, then it stays
652 if date_stamp > pack_before or action == 'create':
653 l.append(entry)
654 else:
655 packed += 1
656 db[key] = marshal.dumps(l)
658 logging.getLogger('roundup.hyperdb').info(
659 'packed %d %s items'%(packed, classname))
661 if db_type == 'gdbm':
662 db.reorganize()
663 db.close()
666 #
667 # Basic transaction support
668 #
669 def commit(self, fail_ok=False):
670 """ Commit the current transactions.
672 Save all data changed since the database was opened or since the
673 last commit() or rollback().
675 fail_ok indicates that the commit is allowed to fail. This is used
676 in the web interface when committing cleaning of the session
677 database. We don't care if there's a concurrency issue there.
679 The only backend this seems to affect is postgres.
680 """
681 logging.getLogger('roundup.hyperdb').info('commit %s transactions'%(
682 len(self.transactions)))
684 # keep a handle to all the database files opened
685 self.databases = {}
687 try:
688 # now, do all the transactions
689 reindex = {}
690 for method, args in self.transactions:
691 reindex[method(*args)] = 1
692 finally:
693 # make sure we close all the database files
694 for db in self.databases.itervalues():
695 db.close()
696 del self.databases
698 # clear the transactions list now so the blobfile implementation
699 # doesn't think there's still pending file commits when it tries
700 # to access the file data
701 self.transactions = []
703 # reindex the nodes that request it
704 for classname, nodeid in [k for k in reindex if k]:
705 self.getclass(classname).index(nodeid)
707 # save the indexer state
708 self.indexer.save_index()
710 self.clearCache()
712 def clearCache(self):
713 # all transactions committed, back to normal
714 self.cache = {}
715 self.dirtynodes = {}
716 self.newnodes = {}
717 self.destroyednodes = {}
718 self.transactions = []
720 def getCachedClassDB(self, classname):
721 """ get the class db, looking in our cache of databases for commit
722 """
723 # get the database handle
724 db_name = 'nodes.%s'%classname
725 if db_name not in self.databases:
726 self.databases[db_name] = self.getclassdb(classname, 'c')
727 return self.databases[db_name]
729 def doSaveNode(self, classname, nodeid, node):
730 db = self.getCachedClassDB(classname)
732 # now save the marshalled data
733 db[nodeid] = marshal.dumps(self.serialise(classname, node))
735 # return the classname, nodeid so we reindex this content
736 return (classname, nodeid)
738 def getCachedJournalDB(self, classname):
739 """ get the journal db, looking in our cache of databases for commit
740 """
741 # get the database handle
742 db_name = 'journals.%s'%classname
743 if db_name not in self.databases:
744 self.databases[db_name] = self.opendb(db_name, 'c')
745 return self.databases[db_name]
747 def doSaveJournal(self, classname, nodeid, action, params, creator,
748 creation):
749 # serialise the parameters now if necessary
750 if isinstance(params, type({})):
751 if action in ('set', 'create'):
752 params = self.serialise(classname, params)
754 # handle supply of the special journalling parameters (usually
755 # supplied on importing an existing database)
756 journaltag = creator
757 if creation:
758 journaldate = creation.serialise()
759 else:
760 journaldate = date.Date().serialise()
762 # create the journal entry
763 entry = (nodeid, journaldate, journaltag, action, params)
765 db = self.getCachedJournalDB(classname)
767 # now insert the journal entry
768 if key_in(db, nodeid):
769 # append to existing
770 s = db[nodeid]
771 l = marshal.loads(s)
772 l.append(entry)
773 else:
774 l = [entry]
776 db[nodeid] = marshal.dumps(l)
778 def doSetJournal(self, classname, nodeid, journal):
779 l = []
780 for nodeid, journaldate, journaltag, action, params in journal:
781 # serialise the parameters now if necessary
782 if isinstance(params, type({})):
783 if action in ('set', 'create'):
784 params = self.serialise(classname, params)
785 journaldate = journaldate.serialise()
786 l.append((nodeid, journaldate, journaltag, action, params))
787 db = self.getCachedJournalDB(classname)
788 db[nodeid] = marshal.dumps(l)
790 def doDestroyNode(self, classname, nodeid):
791 # delete from the class database
792 db = self.getCachedClassDB(classname)
793 if key_in(db, nodeid):
794 del db[nodeid]
796 # delete from the database
797 db = self.getCachedJournalDB(classname)
798 if key_in(db, nodeid):
799 del db[nodeid]
801 def rollback(self):
802 """ Reverse all actions from the current transaction.
803 """
804 logging.getLogger('roundup.hyperdb').info('rollback %s transactions'%(
805 len(self.transactions)))
807 for method, args in self.transactions:
808 # delete temporary files
809 if method == self.doStoreFile:
810 self.rollbackStoreFile(*args)
811 self.cache = {}
812 self.dirtynodes = {}
813 self.newnodes = {}
814 self.destroyednodes = {}
815 self.transactions = []
817 def close(self):
818 """ Nothing to do
819 """
820 if self.lockfile is not None:
821 locking.release_lock(self.lockfile)
822 self.lockfile.close()
823 self.lockfile = None
825 _marker = []
826 class Class(hyperdb.Class):
827 """The handle to a particular class of nodes in a hyperdatabase."""
829 def enableJournalling(self):
830 """Turn journalling on for this class
831 """
832 self.do_journal = 1
834 def disableJournalling(self):
835 """Turn journalling off for this class
836 """
837 self.do_journal = 0
839 # Editing nodes:
841 def create(self, **propvalues):
842 """Create a new node of this class and return its id.
844 The keyword arguments in 'propvalues' map property names to values.
846 The values of arguments must be acceptable for the types of their
847 corresponding properties or a TypeError is raised.
849 If this class has a key property, it must be present and its value
850 must not collide with other key strings or a ValueError is raised.
852 Any other properties on this class that are missing from the
853 'propvalues' dictionary are set to None.
855 If an id in a link or multilink property does not refer to a valid
856 node, an IndexError is raised.
858 These operations trigger detectors and can be vetoed. Attempts
859 to modify the "creation" or "activity" properties cause a KeyError.
860 """
861 if self.db.journaltag is None:
862 raise hyperdb.DatabaseError(_('Database open read-only'))
863 self.fireAuditors('create', None, propvalues)
864 newid = self.create_inner(**propvalues)
865 self.fireReactors('create', newid, None)
866 return newid
868 def create_inner(self, **propvalues):
869 """ Called by create, in-between the audit and react calls.
870 """
871 if 'id' in propvalues:
872 raise KeyError('"id" is reserved')
874 if self.db.journaltag is None:
875 raise hyperdb.DatabaseError(_('Database open read-only'))
877 if 'creation' in propvalues or 'activity' in propvalues:
878 raise KeyError('"creation" and "activity" are reserved')
879 # new node's id
880 newid = self.db.newid(self.classname)
882 # validate propvalues
883 num_re = re.compile('^\d+$')
884 for key, value in propvalues.iteritems():
885 if key == self.key:
886 try:
887 self.lookup(value)
888 except KeyError:
889 pass
890 else:
891 raise ValueError('node with key "%s" exists'%value)
893 # try to handle this property
894 try:
895 prop = self.properties[key]
896 except KeyError:
897 raise KeyError('"%s" has no property "%s"'%(self.classname,
898 key))
900 if value is not None and isinstance(prop, hyperdb.Link):
901 if type(value) != type(''):
902 raise ValueError('link value must be String')
903 link_class = self.properties[key].classname
904 # if it isn't a number, it's a key
905 if not num_re.match(value):
906 try:
907 value = self.db.classes[link_class].lookup(value)
908 except (TypeError, KeyError):
909 raise IndexError('new property "%s": %s not a %s'%(
910 key, value, link_class))
911 elif not self.db.getclass(link_class).hasnode(value):
912 raise IndexError('%s has no node %s'%(link_class,
913 value))
915 # save off the value
916 propvalues[key] = value
918 # register the link with the newly linked node
919 if self.do_journal and self.properties[key].do_journal:
920 self.db.addjournal(link_class, value, 'link',
921 (self.classname, newid, key))
923 elif isinstance(prop, hyperdb.Multilink):
924 if value is None:
925 value = []
926 if not hasattr(value, '__iter__'):
927 raise TypeError('new property "%s" not an iterable of ids'%key)
929 # clean up and validate the list of links
930 link_class = self.properties[key].classname
931 l = []
932 for entry in value:
933 if type(entry) != type(''):
934 raise ValueError('"%s" multilink value (%r) '\
935 'must contain Strings'%(key, value))
936 # if it isn't a number, it's a key
937 if not num_re.match(entry):
938 try:
939 entry = self.db.classes[link_class].lookup(entry)
940 except (TypeError, KeyError):
941 raise IndexError('new property "%s": %s not a %s'%(
942 key, entry, self.properties[key].classname))
943 l.append(entry)
944 value = l
945 propvalues[key] = value
947 # handle additions
948 for nodeid in value:
949 if not self.db.getclass(link_class).hasnode(nodeid):
950 raise IndexError('%s has no node %s'%(link_class,
951 nodeid))
952 # register the link with the newly linked node
953 if self.do_journal and self.properties[key].do_journal:
954 self.db.addjournal(link_class, nodeid, 'link',
955 (self.classname, newid, key))
957 elif isinstance(prop, hyperdb.String):
958 if type(value) != type('') and type(value) != type(u''):
959 raise TypeError('new property "%s" not a string'%key)
960 if prop.indexme:
961 self.db.indexer.add_text((self.classname, newid, key),
962 value)
964 elif isinstance(prop, hyperdb.Password):
965 if not isinstance(value, password.Password):
966 raise TypeError('new property "%s" not a Password'%key)
968 elif isinstance(prop, hyperdb.Date):
969 if value is not None and not isinstance(value, date.Date):
970 raise TypeError('new property "%s" not a Date'%key)
972 elif isinstance(prop, hyperdb.Interval):
973 if value is not None and not isinstance(value, date.Interval):
974 raise TypeError('new property "%s" not an Interval'%key)
976 elif value is not None and isinstance(prop, hyperdb.Number):
977 try:
978 float(value)
979 except ValueError:
980 raise TypeError('new property "%s" not numeric'%key)
982 elif value is not None and isinstance(prop, hyperdb.Boolean):
983 try:
984 int(value)
985 except ValueError:
986 raise TypeError('new property "%s" not boolean'%key)
988 # make sure there's data where there needs to be
989 for key, prop in self.properties.iteritems():
990 if key in propvalues:
991 continue
992 if key == self.key:
993 raise ValueError('key property "%s" is required'%key)
994 if isinstance(prop, hyperdb.Multilink):
995 propvalues[key] = []
997 # done
998 self.db.addnode(self.classname, newid, propvalues)
999 if self.do_journal:
1000 self.db.addjournal(self.classname, newid, 'create', {})
1002 return newid
1004 def get(self, nodeid, propname, default=_marker, cache=1):
1005 """Get the value of a property on an existing node of this class.
1007 'nodeid' must be the id of an existing node of this class or an
1008 IndexError is raised. 'propname' must be the name of a property
1009 of this class or a KeyError is raised.
1011 'cache' exists for backward compatibility, and is not used.
1013 Attempts to get the "creation" or "activity" properties should
1014 do the right thing.
1015 """
1016 if propname == 'id':
1017 return nodeid
1019 # get the node's dict
1020 d = self.db.getnode(self.classname, nodeid)
1022 # check for one of the special props
1023 if propname == 'creation':
1024 if 'creation' in d:
1025 return d['creation']
1026 if not self.do_journal:
1027 raise ValueError('Journalling is disabled for this class')
1028 journal = self.db.getjournal(self.classname, nodeid)
1029 if journal:
1030 return journal[0][1]
1031 else:
1032 # on the strange chance that there's no journal
1033 return date.Date()
1034 if propname == 'activity':
1035 if 'activity' in d:
1036 return d['activity']
1037 if not self.do_journal:
1038 raise ValueError('Journalling is disabled for this class')
1039 journal = self.db.getjournal(self.classname, nodeid)
1040 if journal:
1041 return self.db.getjournal(self.classname, nodeid)[-1][1]
1042 else:
1043 # on the strange chance that there's no journal
1044 return date.Date()
1045 if propname == 'creator':
1046 if 'creator' in d:
1047 return d['creator']
1048 if not self.do_journal:
1049 raise ValueError('Journalling is disabled for this class')
1050 journal = self.db.getjournal(self.classname, nodeid)
1051 if journal:
1052 num_re = re.compile('^\d+$')
1053 value = journal[0][2]
1054 if num_re.match(value):
1055 return value
1056 else:
1057 # old-style "username" journal tag
1058 try:
1059 return self.db.user.lookup(value)
1060 except KeyError:
1061 # user's been retired, return admin
1062 return '1'
1063 else:
1064 return self.db.getuid()
1065 if propname == 'actor':
1066 if 'actor' in d:
1067 return d['actor']
1068 if not self.do_journal:
1069 raise ValueError('Journalling is disabled for this class')
1070 journal = self.db.getjournal(self.classname, nodeid)
1071 if journal:
1072 num_re = re.compile('^\d+$')
1073 value = journal[-1][2]
1074 if num_re.match(value):
1075 return value
1076 else:
1077 # old-style "username" journal tag
1078 try:
1079 return self.db.user.lookup(value)
1080 except KeyError:
1081 # user's been retired, return admin
1082 return '1'
1083 else:
1084 return self.db.getuid()
1086 # get the property (raises KeyErorr if invalid)
1087 prop = self.properties[propname]
1089 if propname not in d:
1090 if default is _marker:
1091 if isinstance(prop, hyperdb.Multilink):
1092 return []
1093 else:
1094 return None
1095 else:
1096 return default
1098 # return a dupe of the list so code doesn't get confused
1099 if isinstance(prop, hyperdb.Multilink):
1100 return d[propname][:]
1102 return d[propname]
1104 def set(self, nodeid, **propvalues):
1105 """Modify a property on an existing node of this class.
1107 'nodeid' must be the id of an existing node of this class or an
1108 IndexError is raised.
1110 Each key in 'propvalues' must be the name of a property of this
1111 class or a KeyError is raised.
1113 All values in 'propvalues' must be acceptable types for their
1114 corresponding properties or a TypeError is raised.
1116 If the value of the key property is set, it must not collide with
1117 other key strings or a ValueError is raised.
1119 If the value of a Link or Multilink property contains an invalid
1120 node id, a ValueError is raised.
1122 These operations trigger detectors and can be vetoed. Attempts
1123 to modify the "creation" or "activity" properties cause a KeyError.
1124 """
1125 if self.db.journaltag is None:
1126 raise hyperdb.DatabaseError(_('Database open read-only'))
1128 self.fireAuditors('set', nodeid, propvalues)
1129 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1130 for name, prop in self.getprops(protected=0).iteritems():
1131 if name in oldvalues:
1132 continue
1133 if isinstance(prop, hyperdb.Multilink):
1134 oldvalues[name] = []
1135 else:
1136 oldvalues[name] = None
1137 propvalues = self.set_inner(nodeid, **propvalues)
1138 self.fireReactors('set', nodeid, oldvalues)
1139 return propvalues
1141 def set_inner(self, nodeid, **propvalues):
1142 """ Called by set, in-between the audit and react calls.
1143 """
1144 if not propvalues:
1145 return propvalues
1147 if 'creation' in propvalues or 'activity' in propvalues:
1148 raise KeyError, '"creation" and "activity" are reserved'
1150 if 'id' in propvalues:
1151 raise KeyError, '"id" is reserved'
1153 if self.db.journaltag is None:
1154 raise hyperdb.DatabaseError(_('Database open read-only'))
1156 node = self.db.getnode(self.classname, nodeid)
1157 if self.db.RETIRED_FLAG in node:
1158 raise IndexError
1159 num_re = re.compile('^\d+$')
1161 # if the journal value is to be different, store it in here
1162 journalvalues = {}
1164 # list() propvalues 'cos it might be modified by the loop
1165 for propname, value in list(propvalues.items()):
1166 # check to make sure we're not duplicating an existing key
1167 if propname == self.key and node[propname] != value:
1168 try:
1169 self.lookup(value)
1170 except KeyError:
1171 pass
1172 else:
1173 raise ValueError('node with key "%s" exists'%value)
1175 # this will raise the KeyError if the property isn't valid
1176 # ... we don't use getprops() here because we only care about
1177 # the writeable properties.
1178 try:
1179 prop = self.properties[propname]
1180 except KeyError:
1181 raise KeyError('"%s" has no property named "%s"'%(
1182 self.classname, propname))
1184 # if the value's the same as the existing value, no sense in
1185 # doing anything
1186 current = node.get(propname, None)
1187 if value == current:
1188 del propvalues[propname]
1189 continue
1190 journalvalues[propname] = current
1192 # do stuff based on the prop type
1193 if isinstance(prop, hyperdb.Link):
1194 link_class = prop.classname
1195 # if it isn't a number, it's a key
1196 if value is not None and not isinstance(value, type('')):
1197 raise ValueError('property "%s" link value be a string'%(
1198 propname))
1199 if isinstance(value, type('')) and not num_re.match(value):
1200 try:
1201 value = self.db.classes[link_class].lookup(value)
1202 except (TypeError, KeyError):
1203 raise IndexError('new property "%s": %s not a %s'%(
1204 propname, value, prop.classname))
1206 if (value is not None and
1207 not self.db.getclass(link_class).hasnode(value)):
1208 raise IndexError('%s has no node %s'%(link_class,
1209 value))
1211 if self.do_journal and prop.do_journal:
1212 # register the unlink with the old linked node
1213 if propname in node and node[propname] is not None:
1214 self.db.addjournal(link_class, node[propname], 'unlink',
1215 (self.classname, nodeid, propname))
1217 # register the link with the newly linked node
1218 if value is not None:
1219 self.db.addjournal(link_class, value, 'link',
1220 (self.classname, nodeid, propname))
1222 elif isinstance(prop, hyperdb.Multilink):
1223 if value is None:
1224 value = []
1225 if not hasattr(value, '__iter__'):
1226 raise TypeError('new property "%s" not an iterable of'
1227 ' ids'%propname)
1228 link_class = self.properties[propname].classname
1229 l = []
1230 for entry in value:
1231 # if it isn't a number, it's a key
1232 if type(entry) != type(''):
1233 raise ValueError('new property "%s" link value '
1234 'must be a string'%propname)
1235 if not num_re.match(entry):
1236 try:
1237 entry = self.db.classes[link_class].lookup(entry)
1238 except (TypeError, KeyError):
1239 raise IndexError('new property "%s": %s not a %s'%(
1240 propname, entry,
1241 self.properties[propname].classname))
1242 l.append(entry)
1243 value = l
1244 propvalues[propname] = value
1246 # figure the journal entry for this property
1247 add = []
1248 remove = []
1250 # handle removals
1251 if propname in node:
1252 l = node[propname]
1253 else:
1254 l = []
1255 for id in l[:]:
1256 if id in value:
1257 continue
1258 # register the unlink with the old linked node
1259 if self.do_journal and self.properties[propname].do_journal:
1260 self.db.addjournal(link_class, id, 'unlink',
1261 (self.classname, nodeid, propname))
1262 l.remove(id)
1263 remove.append(id)
1265 # handle additions
1266 for id in value:
1267 if not self.db.getclass(link_class).hasnode(id):
1268 raise IndexError('%s has no node %s'%(link_class,
1269 id))
1270 if id in l:
1271 continue
1272 # register the link with the newly linked node
1273 if self.do_journal and self.properties[propname].do_journal:
1274 self.db.addjournal(link_class, id, 'link',
1275 (self.classname, nodeid, propname))
1276 l.append(id)
1277 add.append(id)
1279 # figure the journal entry
1280 l = []
1281 if add:
1282 l.append(('+', add))
1283 if remove:
1284 l.append(('-', remove))
1285 if l:
1286 journalvalues[propname] = tuple(l)
1288 elif isinstance(prop, hyperdb.String):
1289 if value is not None and type(value) != type('') and type(value) != type(u''):
1290 raise TypeError('new property "%s" not a '
1291 'string'%propname)
1292 if prop.indexme:
1293 self.db.indexer.add_text((self.classname, nodeid, propname),
1294 value)
1296 elif isinstance(prop, hyperdb.Password):
1297 if not isinstance(value, password.Password):
1298 raise TypeError('new property "%s" not a '
1299 'Password'%propname)
1300 propvalues[propname] = value
1302 elif value is not None and isinstance(prop, hyperdb.Date):
1303 if not isinstance(value, date.Date):
1304 raise TypeError('new property "%s" not a '
1305 'Date'%propname)
1306 propvalues[propname] = value
1308 elif value is not None and isinstance(prop, hyperdb.Interval):
1309 if not isinstance(value, date.Interval):
1310 raise TypeError('new property "%s" not an '
1311 'Interval'%propname)
1312 propvalues[propname] = value
1314 elif value is not None and isinstance(prop, hyperdb.Number):
1315 try:
1316 float(value)
1317 except ValueError:
1318 raise TypeError('new property "%s" not '
1319 'numeric'%propname)
1321 elif value is not None and isinstance(prop, hyperdb.Boolean):
1322 try:
1323 int(value)
1324 except ValueError:
1325 raise TypeError('new property "%s" not '
1326 'boolean'%propname)
1328 node[propname] = value
1330 # nothing to do?
1331 if not propvalues:
1332 return propvalues
1334 # update the activity time
1335 node['activity'] = date.Date()
1336 node['actor'] = self.db.getuid()
1338 # do the set, and journal it
1339 self.db.setnode(self.classname, nodeid, node)
1341 if self.do_journal:
1342 self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
1344 return propvalues
1346 def retire(self, nodeid):
1347 """Retire a node.
1349 The properties on the node remain available from the get() method,
1350 and the node's id is never reused.
1352 Retired nodes are not returned by the find(), list(), or lookup()
1353 methods, and other nodes may reuse the values of their key properties.
1355 These operations trigger detectors and can be vetoed. Attempts
1356 to modify the "creation" or "activity" properties cause a KeyError.
1357 """
1358 if self.db.journaltag is None:
1359 raise hyperdb.DatabaseError(_('Database open read-only'))
1361 self.fireAuditors('retire', nodeid, None)
1363 node = self.db.getnode(self.classname, nodeid)
1364 node[self.db.RETIRED_FLAG] = 1
1365 self.db.setnode(self.classname, nodeid, node)
1366 if self.do_journal:
1367 self.db.addjournal(self.classname, nodeid, 'retired', None)
1369 self.fireReactors('retire', nodeid, None)
1371 def restore(self, nodeid):
1372 """Restpre a retired node.
1374 Make node available for all operations like it was before retirement.
1375 """
1376 if self.db.journaltag is None:
1377 raise hyperdb.DatabaseError(_('Database open read-only'))
1379 node = self.db.getnode(self.classname, nodeid)
1380 # check if key property was overrided
1381 key = self.getkey()
1382 try:
1383 id = self.lookup(node[key])
1384 except KeyError:
1385 pass
1386 else:
1387 raise KeyError("Key property (%s) of retired node clashes "
1388 "with existing one (%s)" % (key, node[key]))
1389 # Now we can safely restore node
1390 self.fireAuditors('restore', nodeid, None)
1391 del node[self.db.RETIRED_FLAG]
1392 self.db.setnode(self.classname, nodeid, node)
1393 if self.do_journal:
1394 self.db.addjournal(self.classname, nodeid, 'restored', None)
1396 self.fireReactors('restore', nodeid, None)
1398 def is_retired(self, nodeid, cldb=None):
1399 """Return true if the node is retired.
1400 """
1401 node = self.db.getnode(self.classname, nodeid, cldb)
1402 if self.db.RETIRED_FLAG in node:
1403 return 1
1404 return 0
1406 def destroy(self, nodeid):
1407 """Destroy a node.
1409 WARNING: this method should never be used except in extremely rare
1410 situations where there could never be links to the node being
1411 deleted
1413 WARNING: use retire() instead
1415 WARNING: the properties of this node will not be available ever again
1417 WARNING: really, use retire() instead
1419 Well, I think that's enough warnings. This method exists mostly to
1420 support the session storage of the cgi interface.
1421 """
1422 if self.db.journaltag is None:
1423 raise hyperdb.DatabaseError(_('Database open read-only'))
1424 self.db.destroynode(self.classname, nodeid)
1426 def history(self, nodeid):
1427 """Retrieve the journal of edits on a particular node.
1429 'nodeid' must be the id of an existing node of this class or an
1430 IndexError is raised.
1432 The returned list contains tuples of the form
1434 (nodeid, date, tag, action, params)
1436 'date' is a Timestamp object specifying the time of the change and
1437 'tag' is the journaltag specified when the database was opened.
1438 """
1439 if not self.do_journal:
1440 raise ValueError('Journalling is disabled for this class')
1441 return self.db.getjournal(self.classname, nodeid)
1443 # Locating nodes:
1444 def hasnode(self, nodeid):
1445 """Determine if the given nodeid actually exists
1446 """
1447 return self.db.hasnode(self.classname, nodeid)
1449 def setkey(self, propname):
1450 """Select a String property of this class to be the key property.
1452 'propname' must be the name of a String property of this class or
1453 None, or a TypeError is raised. The values of the key property on
1454 all existing nodes must be unique or a ValueError is raised. If the
1455 property doesn't exist, KeyError is raised.
1456 """
1457 prop = self.getprops()[propname]
1458 if not isinstance(prop, hyperdb.String):
1459 raise TypeError('key properties must be String')
1460 self.key = propname
1462 def getkey(self):
1463 """Return the name of the key property for this class or None."""
1464 return self.key
1466 # TODO: set up a separate index db file for this? profile?
1467 def lookup(self, keyvalue):
1468 """Locate a particular node by its key property and return its id.
1470 If this class has no key property, a TypeError is raised. If the
1471 'keyvalue' matches one of the values for the key property among
1472 the nodes in this class, the matching node's id is returned;
1473 otherwise a KeyError is raised.
1474 """
1475 if not self.key:
1476 raise TypeError('No key property set for '
1477 'class %s'%self.classname)
1478 cldb = self.db.getclassdb(self.classname)
1479 try:
1480 for nodeid in self.getnodeids(cldb):
1481 node = self.db.getnode(self.classname, nodeid, cldb)
1482 if self.db.RETIRED_FLAG in node:
1483 continue
1484 if self.key not in node:
1485 continue
1486 if node[self.key] == keyvalue:
1487 return nodeid
1488 finally:
1489 cldb.close()
1490 raise KeyError('No key (%s) value "%s" for "%s"'%(self.key,
1491 keyvalue, self.classname))
1493 # change from spec - allows multiple props to match
1494 def find(self, **propspec):
1495 """Get the ids of nodes in this class which link to the given nodes.
1497 'propspec' consists of keyword args propname=nodeid or
1498 propname={nodeid:1, }
1499 'propname' must be the name of a property in this class, or a
1500 KeyError is raised. That property must be a Link or
1501 Multilink property, or a TypeError is raised.
1503 Any node in this class whose 'propname' property links to any of
1504 the nodeids will be returned. Examples::
1506 db.issue.find(messages='1')
1507 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1508 """
1509 for propname, itemids in propspec.iteritems():
1510 # check the prop is OK
1511 prop = self.properties[propname]
1512 if not isinstance(prop, hyperdb.Link) and not isinstance(prop, hyperdb.Multilink):
1513 raise TypeError("'%s' not a Link/Multilink "
1514 "property"%propname)
1516 # ok, now do the find
1517 cldb = self.db.getclassdb(self.classname)
1518 l = []
1519 try:
1520 for id in self.getnodeids(db=cldb):
1521 item = self.db.getnode(self.classname, id, db=cldb)
1522 if self.db.RETIRED_FLAG in item:
1523 continue
1524 for propname, itemids in propspec.iteritems():
1525 if type(itemids) is not type({}):
1526 itemids = {itemids:1}
1528 # special case if the item doesn't have this property
1529 if propname not in item:
1530 if None in itemids:
1531 l.append(id)
1532 break
1533 continue
1535 # grab the property definition and its value on this item
1536 prop = self.properties[propname]
1537 value = item[propname]
1538 if isinstance(prop, hyperdb.Link) and value in itemids:
1539 l.append(id)
1540 break
1541 elif isinstance(prop, hyperdb.Multilink):
1542 hit = 0
1543 for v in value:
1544 if v in itemids:
1545 l.append(id)
1546 hit = 1
1547 break
1548 if hit:
1549 break
1550 finally:
1551 cldb.close()
1552 return l
1554 def stringFind(self, **requirements):
1555 """Locate a particular node by matching a set of its String
1556 properties in a caseless search.
1558 If the property is not a String property, a TypeError is raised.
1560 The return is a list of the id of all nodes that match.
1561 """
1562 for propname in requirements:
1563 prop = self.properties[propname]
1564 if not isinstance(prop, hyperdb.String):
1565 raise TypeError("'%s' not a String property"%propname)
1566 requirements[propname] = requirements[propname].lower()
1567 l = []
1568 cldb = self.db.getclassdb(self.classname)
1569 try:
1570 for nodeid in self.getnodeids(cldb):
1571 node = self.db.getnode(self.classname, nodeid, cldb)
1572 if self.db.RETIRED_FLAG in node:
1573 continue
1574 for key, value in requirements.iteritems():
1575 if key not in node:
1576 break
1577 if node[key] is None or node[key].lower() != value:
1578 break
1579 else:
1580 l.append(nodeid)
1581 finally:
1582 cldb.close()
1583 return l
1585 def list(self):
1586 """ Return a list of the ids of the active nodes in this class.
1587 """
1588 l = []
1589 cn = self.classname
1590 cldb = self.db.getclassdb(cn)
1591 try:
1592 for nodeid in self.getnodeids(cldb):
1593 node = self.db.getnode(cn, nodeid, cldb)
1594 if self.db.RETIRED_FLAG in node:
1595 continue
1596 l.append(nodeid)
1597 finally:
1598 cldb.close()
1599 l.sort()
1600 return l
1602 def getnodeids(self, db=None, retired=None):
1603 """ Return a list of ALL nodeids
1605 Set retired=None to get all nodes. Otherwise it'll get all the
1606 retired or non-retired nodes, depending on the flag.
1607 """
1608 res = []
1610 # start off with the new nodes
1611 if self.classname in self.db.newnodes:
1612 res.extend(self.db.newnodes[self.classname])
1614 must_close = False
1615 if db is None:
1616 db = self.db.getclassdb(self.classname)
1617 must_close = True
1618 try:
1619 res.extend(db.keys())
1621 # remove the uncommitted, destroyed nodes
1622 if self.classname in self.db.destroyednodes:
1623 for nodeid in self.db.destroyednodes[self.classname]:
1624 if key_in(db, nodeid):
1625 res.remove(nodeid)
1627 # check retired flag
1628 if retired is False or retired is True:
1629 l = []
1630 for nodeid in res:
1631 node = self.db.getnode(self.classname, nodeid, db)
1632 is_ret = self.db.RETIRED_FLAG in node
1633 if retired == is_ret:
1634 l.append(nodeid)
1635 res = l
1636 finally:
1637 if must_close:
1638 db.close()
1639 return res
1641 def _filter(self, search_matches, filterspec, proptree,
1642 num_re = re.compile('^\d+$')):
1643 """Return a list of the ids of the active nodes in this class that
1644 match the 'filter' spec, sorted by the group spec and then the
1645 sort spec.
1647 "filterspec" is {propname: value(s)}
1649 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1650 and prop is a prop name or None
1652 "search_matches" is a sequence type or None
1654 The filter must match all properties specificed. If the property
1655 value to match is a list:
1657 1. String properties must match all elements in the list, and
1658 2. Other properties must match any of the elements in the list.
1659 """
1660 if __debug__:
1661 start_t = time.time()
1663 cn = self.classname
1665 # optimise filterspec
1666 l = []
1667 props = self.getprops()
1668 LINK = 'spec:link'
1669 MULTILINK = 'spec:multilink'
1670 STRING = 'spec:string'
1671 DATE = 'spec:date'
1672 INTERVAL = 'spec:interval'
1673 OTHER = 'spec:other'
1675 for k, v in filterspec.iteritems():
1676 propclass = props[k]
1677 if isinstance(propclass, hyperdb.Link):
1678 if type(v) is not type([]):
1679 v = [v]
1680 u = []
1681 for entry in v:
1682 # the value -1 is a special "not set" sentinel
1683 if entry == '-1':
1684 entry = None
1685 u.append(entry)
1686 l.append((LINK, k, u))
1687 elif isinstance(propclass, hyperdb.Multilink):
1688 # the value -1 is a special "not set" sentinel
1689 if v in ('-1', ['-1']):
1690 v = []
1691 elif type(v) is not type([]):
1692 v = [v]
1693 l.append((MULTILINK, k, v))
1694 elif isinstance(propclass, hyperdb.String) and k != 'id':
1695 if type(v) is not type([]):
1696 v = [v]
1697 for v in v:
1698 # simple glob searching
1699 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1700 v = v.replace('?', '.')
1701 v = v.replace('*', '.*?')
1702 l.append((STRING, k, re.compile(v, re.I)))
1703 elif isinstance(propclass, hyperdb.Date):
1704 try:
1705 date_rng = propclass.range_from_raw(v, self.db)
1706 l.append((DATE, k, date_rng))
1707 except ValueError:
1708 # If range creation fails - ignore that search parameter
1709 pass
1710 elif isinstance(propclass, hyperdb.Interval):
1711 try:
1712 intv_rng = date.Range(v, date.Interval)
1713 l.append((INTERVAL, k, intv_rng))
1714 except ValueError:
1715 # If range creation fails - ignore that search parameter
1716 pass
1718 elif isinstance(propclass, hyperdb.Boolean):
1719 if type(v) == type(""):
1720 v = v.split(',')
1721 if type(v) != type([]):
1722 v = [v]
1723 bv = []
1724 for val in v:
1725 if type(val) is type(''):
1726 bv.append(propclass.from_raw (val))
1727 else:
1728 bv.append(val)
1729 l.append((OTHER, k, bv))
1731 elif k == 'id':
1732 if type(v) != type([]):
1733 v = v.split(',')
1734 l.append((OTHER, k, [str(int(val)) for val in v]))
1736 elif isinstance(propclass, hyperdb.Number):
1737 if type(v) != type([]):
1738 try :
1739 v = v.split(',')
1740 except AttributeError :
1741 v = [v]
1742 l.append((OTHER, k, [float(val) for val in v]))
1744 filterspec = l
1746 # now, find all the nodes that are active and pass filtering
1747 matches = []
1748 cldb = self.db.getclassdb(cn)
1749 t = 0
1750 try:
1751 # TODO: only full-scan once (use items())
1752 for nodeid in self.getnodeids(cldb):
1753 node = self.db.getnode(cn, nodeid, cldb)
1754 if self.db.RETIRED_FLAG in node:
1755 continue
1756 # apply filter
1757 for t, k, v in filterspec:
1758 # handle the id prop
1759 if k == 'id':
1760 if nodeid not in v:
1761 break
1762 continue
1764 # get the node value
1765 nv = node.get(k, None)
1767 match = 0
1769 # now apply the property filter
1770 if t == LINK:
1771 # link - if this node's property doesn't appear in the
1772 # filterspec's nodeid list, skip it
1773 match = nv in v
1774 elif t == MULTILINK:
1775 # multilink - if any of the nodeids required by the
1776 # filterspec aren't in this node's property, then skip
1777 # it
1778 nv = node.get(k, [])
1780 # check for matching the absence of multilink values
1781 if not v:
1782 match = not nv
1783 else:
1784 # otherwise, make sure this node has each of the
1785 # required values
1786 expr = Expression(v)
1787 if expr.evaluate(nv): match = 1
1788 elif t == STRING:
1789 if nv is None:
1790 nv = ''
1791 # RE search
1792 match = v.search(nv)
1793 elif t == DATE or t == INTERVAL:
1794 if nv is None:
1795 match = v is None
1796 else:
1797 if v.to_value:
1798 if v.from_value <= nv and v.to_value >= nv:
1799 match = 1
1800 else:
1801 if v.from_value <= nv:
1802 match = 1
1803 elif t == OTHER:
1804 # straight value comparison for the other types
1805 match = nv in v
1806 if not match:
1807 break
1808 else:
1809 matches.append([nodeid, node])
1811 # filter based on full text search
1812 if search_matches is not None:
1813 k = []
1814 for v in matches:
1815 if v[0] in search_matches:
1816 k.append(v)
1817 matches = k
1819 # add sorting information to the proptree
1820 JPROPS = {'actor':1, 'activity':1, 'creator':1, 'creation':1}
1821 children = []
1822 if proptree:
1823 children = proptree.sortable_children()
1824 for pt in children:
1825 dir = pt.sort_direction
1826 prop = pt.name
1827 assert (dir and prop)
1828 propclass = props[prop]
1829 pt.sort_ids = []
1830 is_pointer = isinstance(propclass,(hyperdb.Link,
1831 hyperdb.Multilink))
1832 if not is_pointer:
1833 pt.sort_result = []
1834 try:
1835 # cache the opened link class db, if needed.
1836 lcldb = None
1837 # cache the linked class items too
1838 lcache = {}
1840 for entry in matches:
1841 itemid = entry[-2]
1842 item = entry[-1]
1843 # handle the properties that might be "faked"
1844 # also, handle possible missing properties
1845 try:
1846 v = item[prop]
1847 except KeyError:
1848 if prop in JPROPS:
1849 # force lookup of the special journal prop
1850 v = self.get(itemid, prop)
1851 else:
1852 # the node doesn't have a value for this
1853 # property
1854 v = None
1855 if isinstance(propclass, hyperdb.Multilink):
1856 v = []
1857 if prop == 'id':
1858 v = int (itemid)
1859 pt.sort_ids.append(v)
1860 if not is_pointer:
1861 pt.sort_result.append(v)
1862 continue
1864 # missing (None) values are always sorted first
1865 if v is None:
1866 pt.sort_ids.append(v)
1867 if not is_pointer:
1868 pt.sort_result.append(v)
1869 continue
1871 if isinstance(propclass, hyperdb.Link):
1872 lcn = propclass.classname
1873 link = self.db.classes[lcn]
1874 key = link.orderprop()
1875 child = pt.propdict[key]
1876 if key!='id':
1877 if v not in lcache:
1878 # open the link class db if it's not already
1879 if lcldb is None:
1880 lcldb = self.db.getclassdb(lcn)
1881 lcache[v] = self.db.getnode(lcn, v, lcldb)
1882 r = lcache[v][key]
1883 child.propdict[key].sort_ids.append(r)
1884 else:
1885 child.propdict[key].sort_ids.append(v)
1886 pt.sort_ids.append(v)
1887 if not is_pointer:
1888 r = propclass.sort_repr(pt.parent.cls, v, pt.name)
1889 pt.sort_result.append(r)
1890 finally:
1891 # if we opened the link class db, close it now
1892 if lcldb is not None:
1893 lcldb.close()
1894 del lcache
1895 finally:
1896 cldb.close()
1898 # pull the id out of the individual entries
1899 matches = [entry[-2] for entry in matches]
1900 if __debug__:
1901 self.db.stats['filtering'] += (time.time() - start_t)
1902 return matches
1904 def count(self):
1905 """Get the number of nodes in this class.
1907 If the returned integer is 'numnodes', the ids of all the nodes
1908 in this class run from 1 to numnodes, and numnodes+1 will be the
1909 id of the next node to be created in this class.
1910 """
1911 return self.db.countnodes(self.classname)
1913 # Manipulating properties:
1915 def getprops(self, protected=1):
1916 """Return a dictionary mapping property names to property objects.
1917 If the "protected" flag is true, we include protected properties -
1918 those which may not be modified.
1920 In addition to the actual properties on the node, these
1921 methods provide the "creation" and "activity" properties. If the
1922 "protected" flag is true, we include protected properties - those
1923 which may not be modified.
1924 """
1925 d = self.properties.copy()
1926 if protected:
1927 d['id'] = hyperdb.String()
1928 d['creation'] = hyperdb.Date()
1929 d['activity'] = hyperdb.Date()
1930 d['creator'] = hyperdb.Link('user')
1931 d['actor'] = hyperdb.Link('user')
1932 return d
1934 def addprop(self, **properties):
1935 """Add properties to this class.
1937 The keyword arguments in 'properties' must map names to property
1938 objects, or a TypeError is raised. None of the keys in 'properties'
1939 may collide with the names of existing properties, or a ValueError
1940 is raised before any properties have been added.
1941 """
1942 for key in properties:
1943 if key in self.properties:
1944 raise ValueError(key)
1945 self.properties.update(properties)
1947 def index(self, nodeid):
1948 """ Add (or refresh) the node to search indexes """
1949 # find all the String properties that have indexme
1950 for prop, propclass in self.getprops().iteritems():
1951 if isinstance(propclass, hyperdb.String) and propclass.indexme:
1952 # index them under (classname, nodeid, property)
1953 try:
1954 value = str(self.get(nodeid, prop))
1955 except IndexError:
1956 # node has been destroyed
1957 continue
1958 self.db.indexer.add_text((self.classname, nodeid, prop), value)
1960 #
1961 # import / export support
1962 #
1963 def export_list(self, propnames, nodeid):
1964 """ Export a node - generate a list of CSV-able data in the order
1965 specified by propnames for the given node.
1966 """
1967 properties = self.getprops()
1968 l = []
1969 for prop in propnames:
1970 proptype = properties[prop]
1971 value = self.get(nodeid, prop)
1972 # "marshal" data where needed
1973 if value is None:
1974 pass
1975 elif isinstance(proptype, hyperdb.Date):
1976 value = value.get_tuple()
1977 elif isinstance(proptype, hyperdb.Interval):
1978 value = value.get_tuple()
1979 elif isinstance(proptype, hyperdb.Password):
1980 value = str(value)
1981 l.append(repr(value))
1983 # append retired flag
1984 l.append(repr(self.is_retired(nodeid)))
1986 return l
1988 def import_list(self, propnames, proplist):
1989 """ Import a node - all information including "id" is present and
1990 should not be sanity checked. Triggers are not triggered. The
1991 journal should be initialised using the "creator" and "created"
1992 information.
1994 Return the nodeid of the node imported.
1995 """
1996 if self.db.journaltag is None:
1997 raise hyperdb.DatabaseError(_('Database open read-only'))
1998 properties = self.getprops()
2000 # make the new node's property map
2001 d = {}
2002 newid = None
2003 for i in range(len(propnames)):
2004 # Figure the property for this column
2005 propname = propnames[i]
2007 # Use eval to reverse the repr() used to output the CSV
2008 value = eval(proplist[i])
2010 # "unmarshal" where necessary
2011 if propname == 'id':
2012 newid = value
2013 continue
2014 elif propname == 'is retired':
2015 # is the item retired?
2016 if int(value):
2017 d[self.db.RETIRED_FLAG] = 1
2018 continue
2019 elif value is None:
2020 d[propname] = None
2021 continue
2023 prop = properties[propname]
2024 if isinstance(prop, hyperdb.Date):
2025 value = date.Date(value)
2026 elif isinstance(prop, hyperdb.Interval):
2027 value = date.Interval(value)
2028 elif isinstance(prop, hyperdb.Password):
2029 value = password.Password(encrypted=value)
2030 d[propname] = value
2032 # get a new id if necessary
2033 if newid is None:
2034 newid = self.db.newid(self.classname)
2036 # add the node and journal
2037 self.db.addnode(self.classname, newid, d)
2038 return newid
2040 def export_journals(self):
2041 """Export a class's journal - generate a list of lists of
2042 CSV-able data:
2044 nodeid, date, user, action, params
2046 No heading here - the columns are fixed.
2047 """
2048 properties = self.getprops()
2049 r = []
2050 for nodeid in self.getnodeids():
2051 for nodeid, date, user, action, params in self.history(nodeid):
2052 date = date.get_tuple()
2053 if action == 'set':
2054 export_data = {}
2055 for propname, value in params.iteritems():
2056 if propname not in properties:
2057 # property no longer in the schema
2058 continue
2060 prop = properties[propname]
2061 # make sure the params are eval()'able
2062 if value is None:
2063 pass
2064 elif isinstance(prop, hyperdb.Date):
2065 # this is a hack - some dates are stored as strings
2066 if not isinstance(value, type('')):
2067 value = value.get_tuple()
2068 elif isinstance(prop, hyperdb.Interval):
2069 # hack too - some intervals are stored as strings
2070 if not isinstance(value, type('')):
2071 value = value.get_tuple()
2072 elif isinstance(prop, hyperdb.Password):
2073 value = str(value)
2074 export_data[propname] = value
2075 params = export_data
2076 r.append([repr(nodeid), repr(date), repr(user),
2077 repr(action), repr(params)])
2078 return r
2080 class FileClass(hyperdb.FileClass, Class):
2081 """This class defines a large chunk of data. To support this, it has a
2082 mandatory String property "content" which is typically saved off
2083 externally to the hyperdb.
2085 The default MIME type of this data is defined by the
2086 "default_mime_type" class attribute, which may be overridden by each
2087 node if the class defines a "type" String property.
2088 """
2089 def __init__(self, db, classname, **properties):
2090 """The newly-created class automatically includes the "content"
2091 and "type" properties.
2092 """
2093 if 'content' not in properties:
2094 properties['content'] = hyperdb.String(indexme='yes')
2095 if 'type' not in properties:
2096 properties['type'] = hyperdb.String()
2097 Class.__init__(self, db, classname, **properties)
2099 def create(self, **propvalues):
2100 """ Snarf the "content" propvalue and store in a file
2101 """
2102 # we need to fire the auditors now, or the content property won't
2103 # be in propvalues for the auditors to play with
2104 self.fireAuditors('create', None, propvalues)
2106 # now remove the content property so it's not stored in the db
2107 content = propvalues['content']
2108 del propvalues['content']
2110 # make sure we have a MIME type
2111 mime_type = propvalues.get('type', self.default_mime_type)
2113 # do the database create
2114 newid = self.create_inner(**propvalues)
2116 # store off the content as a file
2117 self.db.storefile(self.classname, newid, None, content)
2119 # fire reactors
2120 self.fireReactors('create', newid, None)
2122 return newid
2124 def get(self, nodeid, propname, default=_marker, cache=1):
2125 """ Trap the content propname and get it from the file
2127 'cache' exists for backwards compatibility, and is not used.
2128 """
2129 poss_msg = 'Possibly an access right configuration problem.'
2130 if propname == 'content':
2131 try:
2132 return self.db.getfile(self.classname, nodeid, None)
2133 except IOError, strerror:
2134 # XXX by catching this we don't see an error in the log.
2135 return 'ERROR reading file: %s%s\n%s\n%s'%(
2136 self.classname, nodeid, poss_msg, strerror)
2137 if default is not _marker:
2138 return Class.get(self, nodeid, propname, default)
2139 else:
2140 return Class.get(self, nodeid, propname)
2142 def set(self, itemid, **propvalues):
2143 """ Snarf the "content" propvalue and update it in a file
2144 """
2145 self.fireAuditors('set', itemid, propvalues)
2147 # create the oldvalues dict - fill in any missing values
2148 oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid))
2149 for name, prop in self.getprops(protected=0).iteritems():
2150 if name in oldvalues:
2151 continue
2152 if isinstance(prop, hyperdb.Multilink):
2153 oldvalues[name] = []
2154 else:
2155 oldvalues[name] = None
2157 # now remove the content property so it's not stored in the db
2158 content = None
2159 if 'content' in propvalues:
2160 content = propvalues['content']
2161 del propvalues['content']
2163 # do the database update
2164 propvalues = self.set_inner(itemid, **propvalues)
2166 # do content?
2167 if content:
2168 # store and possibly index
2169 self.db.storefile(self.classname, itemid, None, content)
2170 if self.properties['content'].indexme:
2171 mime_type = self.get(itemid, 'type', self.default_mime_type)
2172 self.db.indexer.add_text((self.classname, itemid, 'content'),
2173 content, mime_type)
2174 propvalues['content'] = content
2176 # fire reactors
2177 self.fireReactors('set', itemid, oldvalues)
2178 return propvalues
2180 def index(self, nodeid):
2181 """ Add (or refresh) the node to search indexes.
2183 Use the content-type property for the content property.
2184 """
2185 # find all the String properties that have indexme
2186 for prop, propclass in self.getprops().iteritems():
2187 if prop == 'content' and propclass.indexme:
2188 mime_type = self.get(nodeid, 'type', self.default_mime_type)
2189 self.db.indexer.add_text((self.classname, nodeid, 'content'),
2190 str(self.get(nodeid, 'content')), mime_type)
2191 elif isinstance(propclass, hyperdb.String) and propclass.indexme:
2192 # index them under (classname, nodeid, property)
2193 try:
2194 value = str(self.get(nodeid, prop))
2195 except IndexError:
2196 # node has been destroyed
2197 continue
2198 self.db.indexer.add_text((self.classname, nodeid, prop), value)
2200 # deviation from spec - was called ItemClass
2201 class IssueClass(Class, roundupdb.IssueClass):
2202 # Overridden methods:
2203 def __init__(self, db, classname, **properties):
2204 """The newly-created class automatically includes the "messages",
2205 "files", "nosy", and "superseder" properties. If the 'properties'
2206 dictionary attempts to specify any of these properties or a
2207 "creation" or "activity" property, a ValueError is raised.
2208 """
2209 if 'title' not in properties:
2210 properties['title'] = hyperdb.String(indexme='yes')
2211 if 'messages' not in properties:
2212 properties['messages'] = hyperdb.Multilink("msg")
2213 if 'files' not in properties:
2214 properties['files'] = hyperdb.Multilink("file")
2215 if 'nosy' not in properties:
2216 # note: journalling is turned off as it really just wastes
2217 # space. this behaviour may be overridden in an instance
2218 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2219 if 'superseder' not in properties:
2220 properties['superseder'] = hyperdb.Multilink(classname)
2221 Class.__init__(self, db, classname, **properties)
2223 # vim: set et sts=4 sw=4 :