6524c388c5765bd2a74bdba97d05fff61a47173e
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: back_anydbm.py,v 1.46 2002-07-14 06:06:34 richard Exp $
19 '''
20 This module defines a backend that saves the hyperdatabase in a database
21 chosen by anydbm. It is guaranteed to always be available in python
22 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
23 serious bugs, and is not available)
24 '''
26 import whichdb, anydbm, os, marshal, re, weakref, string, copy
27 from roundup import hyperdb, date, password, roundupdb
28 from blobfiles import FileStorage
29 from roundup.indexer import Indexer
30 from locking import acquire_lock, release_lock
31 from roundup.hyperdb import String, Password, Date, Interval, Link, \
32 Multilink, DatabaseError
34 #
35 # Now the database
36 #
37 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
38 """A database for storing records containing flexible data types.
40 Transaction stuff TODO:
41 . check the timestamp of the class file and nuke the cache if it's
42 modified. Do some sort of conflict checking on the dirty stuff.
43 . perhaps detect write collisions (related to above)?
45 """
46 def __init__(self, config, journaltag=None):
47 """Open a hyperdatabase given a specifier to some storage.
49 The 'storagelocator' is obtained from config.DATABASE.
50 The meaning of 'storagelocator' depends on the particular
51 implementation of the hyperdatabase. It could be a file name,
52 a directory path, a socket descriptor for a connection to a
53 database over the network, etc.
55 The 'journaltag' is a token that will be attached to the journal
56 entries for any edits done on the database. If 'journaltag' is
57 None, the database is opened in read-only mode: the Class.create(),
58 Class.set(), and Class.retire() methods are disabled.
59 """
60 self.config, self.journaltag = config, journaltag
61 self.dir = config.DATABASE
62 self.classes = {}
63 self.cache = {} # cache of nodes loaded or created
64 self.dirtynodes = {} # keep track of the dirty nodes by class
65 self.newnodes = {} # keep track of the new nodes by class
66 self.transactions = []
67 self.indexer = Indexer(self.dir)
68 # ensure files are group readable and writable
69 os.umask(0002)
71 def post_init(self):
72 """Called once the schema initialisation has finished."""
73 # reindex the db if necessary
74 if self.indexer.should_reindex():
75 self.reindex()
77 def reindex(self):
78 for klass in self.classes.values():
79 for nodeid in klass.list():
80 klass.index(nodeid)
81 self.indexer.save_index()
83 def __repr__(self):
84 return '<back_anydbm instance at %x>'%id(self)
86 #
87 # Classes
88 #
89 def __getattr__(self, classname):
90 """A convenient way of calling self.getclass(classname)."""
91 if self.classes.has_key(classname):
92 if __debug__:
93 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
94 return self.classes[classname]
95 raise AttributeError, classname
97 def addclass(self, cl):
98 if __debug__:
99 print >>hyperdb.DEBUG, 'addclass', (self, cl)
100 cn = cl.classname
101 if self.classes.has_key(cn):
102 raise ValueError, cn
103 self.classes[cn] = cl
105 def getclasses(self):
106 """Return a list of the names of all existing classes."""
107 if __debug__:
108 print >>hyperdb.DEBUG, 'getclasses', (self,)
109 l = self.classes.keys()
110 l.sort()
111 return l
113 def getclass(self, classname):
114 """Get the Class object representing a particular class.
116 If 'classname' is not a valid class name, a KeyError is raised.
117 """
118 if __debug__:
119 print >>hyperdb.DEBUG, 'getclass', (self, classname)
120 return self.classes[classname]
122 #
123 # Class DBs
124 #
125 def clear(self):
126 '''Delete all database contents
127 '''
128 if __debug__:
129 print >>hyperdb.DEBUG, 'clear', (self,)
130 for cn in self.classes.keys():
131 for dummy in 'nodes', 'journals':
132 path = os.path.join(self.dir, 'journals.%s'%cn)
133 if os.path.exists(path):
134 os.remove(path)
135 elif os.path.exists(path+'.db'): # dbm appends .db
136 os.remove(path+'.db')
138 def getclassdb(self, classname, mode='r'):
139 ''' grab a connection to the class db that will be used for
140 multiple actions
141 '''
142 if __debug__:
143 print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode)
144 return self._opendb('nodes.%s'%classname, mode)
146 def determine_db_type(self, path):
147 ''' determine which DB wrote the class file
148 '''
149 db_type = ''
150 if os.path.exists(path):
151 db_type = whichdb.whichdb(path)
152 if not db_type:
153 raise hyperdb.DatabaseError, "Couldn't identify database type"
154 elif os.path.exists(path+'.db'):
155 # if the path ends in '.db', it's a dbm database, whether
156 # anydbm says it's dbhash or not!
157 db_type = 'dbm'
158 return db_type
160 def _opendb(self, name, mode):
161 '''Low-level database opener that gets around anydbm/dbm
162 eccentricities.
163 '''
164 if __debug__:
165 print >>hyperdb.DEBUG, '_opendb', (self, name, mode)
167 # figure the class db type
168 path = os.path.join(os.getcwd(), self.dir, name)
169 db_type = self.determine_db_type(path)
171 # new database? let anydbm pick the best dbm
172 if not db_type:
173 if __debug__:
174 print >>hyperdb.DEBUG, "_opendb anydbm.open(%r, 'n')"%path
175 return anydbm.open(path, 'n')
177 # open the database with the correct module
178 try:
179 dbm = __import__(db_type)
180 except ImportError:
181 raise hyperdb.DatabaseError, \
182 "Couldn't open database - the required module '%s'"\
183 " is not available"%db_type
184 if __debug__:
185 print >>hyperdb.DEBUG, "_opendb %r.open(%r, %r)"%(db_type, path,
186 mode)
187 return dbm.open(path, mode)
189 def _lockdb(self, name):
190 ''' Lock a database file
191 '''
192 path = os.path.join(os.getcwd(), self.dir, '%s.lock'%name)
193 return acquire_lock(path)
195 #
196 # Node IDs
197 #
198 def newid(self, classname):
199 ''' Generate a new id for the given class
200 '''
201 # open the ids DB - create if if doesn't exist
202 lock = self._lockdb('_ids')
203 db = self._opendb('_ids', 'c')
204 if db.has_key(classname):
205 newid = db[classname] = str(int(db[classname]) + 1)
206 else:
207 # the count() bit is transitional - older dbs won't start at 1
208 newid = str(self.getclass(classname).count()+1)
209 db[classname] = newid
210 db.close()
211 release_lock(lock)
212 return newid
214 #
215 # Nodes
216 #
217 def addnode(self, classname, nodeid, node):
218 ''' add the specified node to its class's db
219 '''
220 if __debug__:
221 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
222 self.newnodes.setdefault(classname, {})[nodeid] = 1
223 self.cache.setdefault(classname, {})[nodeid] = node
224 self.savenode(classname, nodeid, node)
226 def setnode(self, classname, nodeid, node):
227 ''' change the specified node
228 '''
229 if __debug__:
230 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
231 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
233 # can't set without having already loaded the node
234 self.cache[classname][nodeid] = node
235 self.savenode(classname, nodeid, node)
237 def savenode(self, classname, nodeid, node):
238 ''' perform the saving of data specified by the set/addnode
239 '''
240 if __debug__:
241 print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node)
242 self.transactions.append((self._doSaveNode, (classname, nodeid, node)))
244 def getnode(self, classname, nodeid, db=None, cache=1):
245 ''' get a node from the database
246 '''
247 if __debug__:
248 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid, db)
249 if cache:
250 # try the cache
251 cache_dict = self.cache.setdefault(classname, {})
252 if cache_dict.has_key(nodeid):
253 if __debug__:
254 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
255 nodeid)
256 return cache_dict[nodeid]
258 if __debug__:
259 print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid)
261 # get from the database and save in the cache
262 if db is None:
263 db = self.getclassdb(classname)
264 if not db.has_key(nodeid):
265 raise IndexError, "no such %s %s"%(classname, nodeid)
267 # decode
268 res = marshal.loads(db[nodeid])
270 # reverse the serialisation
271 res = self.unserialise(classname, res)
273 # store off in the cache dict
274 if cache:
275 cache_dict[nodeid] = res
277 return res
279 def serialise(self, classname, node):
280 '''Copy the node contents, converting non-marshallable data into
281 marshallable data.
282 '''
283 if __debug__:
284 print >>hyperdb.DEBUG, 'serialise', classname, node
285 properties = self.getclass(classname).getprops()
286 d = {}
287 for k, v in node.items():
288 # if the property doesn't exist, or is the "retired" flag then
289 # it won't be in the properties dict
290 if not properties.has_key(k):
291 d[k] = v
292 continue
294 # get the property spec
295 prop = properties[k]
297 if isinstance(prop, Password):
298 d[k] = str(v)
299 elif isinstance(prop, Date) and v is not None:
300 d[k] = v.get_tuple()
301 elif isinstance(prop, Interval) and v is not None:
302 d[k] = v.get_tuple()
303 else:
304 d[k] = v
305 return d
307 def unserialise(self, classname, node):
308 '''Decode the marshalled node data
309 '''
310 if __debug__:
311 print >>hyperdb.DEBUG, 'unserialise', classname, node
312 properties = self.getclass(classname).getprops()
313 d = {}
314 for k, v in node.items():
315 # if the property doesn't exist, or is the "retired" flag then
316 # it won't be in the properties dict
317 if not properties.has_key(k):
318 d[k] = v
319 continue
321 # get the property spec
322 prop = properties[k]
324 if isinstance(prop, Date) and v is not None:
325 d[k] = date.Date(v)
326 elif isinstance(prop, Interval) and v is not None:
327 d[k] = date.Interval(v)
328 elif isinstance(prop, Password):
329 p = password.Password()
330 p.unpack(v)
331 d[k] = p
332 else:
333 d[k] = v
334 return d
336 def hasnode(self, classname, nodeid, db=None):
337 ''' determine if the database has a given node
338 '''
339 if __debug__:
340 print >>hyperdb.DEBUG, 'hasnode', (self, classname, nodeid, db)
342 # try the cache
343 cache = self.cache.setdefault(classname, {})
344 if cache.has_key(nodeid):
345 if __debug__:
346 print >>hyperdb.TRACE, 'has %s %s cached'%(classname, nodeid)
347 return 1
348 if __debug__:
349 print >>hyperdb.TRACE, 'has %s %s'%(classname, nodeid)
351 # not in the cache - check the database
352 if db is None:
353 db = self.getclassdb(classname)
354 res = db.has_key(nodeid)
355 return res
357 def countnodes(self, classname, db=None):
358 if __debug__:
359 print >>hyperdb.DEBUG, 'countnodes', (self, classname, db)
360 # include the new nodes not saved to the DB yet
361 count = len(self.newnodes.get(classname, {}))
363 # and count those in the DB
364 if db is None:
365 db = self.getclassdb(classname)
366 count = count + len(db.keys())
367 return count
369 def getnodeids(self, classname, db=None):
370 if __debug__:
371 print >>hyperdb.DEBUG, 'getnodeids', (self, classname, db)
372 # start off with the new nodes
373 res = self.newnodes.get(classname, {}).keys()
375 if db is None:
376 db = self.getclassdb(classname)
377 res = res + db.keys()
378 return res
381 #
382 # Files - special node properties
383 # inherited from FileStorage
385 #
386 # Journal
387 #
388 def addjournal(self, classname, nodeid, action, params):
389 ''' Journal the Action
390 'action' may be:
392 'create' or 'set' -- 'params' is a dictionary of property values
393 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
394 'retire' -- 'params' is None
395 '''
396 if __debug__:
397 print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid,
398 action, params)
399 self.transactions.append((self._doSaveJournal, (classname, nodeid,
400 action, params)))
402 def getjournal(self, classname, nodeid):
403 ''' get the journal for id
404 '''
405 if __debug__:
406 print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid)
407 # attempt to open the journal - in some rare cases, the journal may
408 # not exist
409 try:
410 db = self._opendb('journals.%s'%classname, 'r')
411 except anydbm.error, error:
412 if str(error) == "need 'c' or 'n' flag to open new db": return []
413 elif error.args[0] != 2: raise
414 return []
415 try:
416 journal = marshal.loads(db[nodeid])
417 except KeyError:
418 db.close()
419 raise KeyError, 'no such %s %s'%(classname, nodeid)
420 db.close()
421 res = []
422 for entry in journal:
423 (nodeid, date_stamp, user, action, params) = entry
424 date_obj = date.Date(date_stamp)
425 res.append((nodeid, date_obj, user, action, params))
426 return res
428 def pack(self, pack_before):
429 ''' delete all journal entries before 'pack_before' '''
430 if __debug__:
431 print >>hyperdb.DEBUG, 'packjournal', (self, pack_before)
433 pack_before = pack_before.get_tuple()
435 classes = self.getclasses()
437 # figure the class db type
439 for classname in classes:
440 db_name = 'journals.%s'%classname
441 path = os.path.join(os.getcwd(), self.dir, classname)
442 db_type = self.determine_db_type(path)
443 db = self._opendb(db_name, 'w')
445 for key in db.keys():
446 journal = marshal.loads(db[key])
447 l = []
448 last_set_entry = None
449 for entry in journal:
450 (nodeid, date_stamp, self.journaltag, action,
451 params) = entry
452 if date_stamp > pack_before or action == 'create':
453 l.append(entry)
454 elif action == 'set':
455 # grab the last set entry to keep information on
456 # activity
457 last_set_entry = entry
458 if last_set_entry:
459 date_stamp = last_set_entry[1]
460 # if the last set entry was made after the pack date
461 # then it is already in the list
462 if date_stamp < pack_before:
463 l.append(last_set_entry)
464 db[key] = marshal.dumps(l)
465 if db_type == 'gdbm':
466 db.reorganize()
467 db.close()
470 #
471 # Basic transaction support
472 #
473 def commit(self):
474 ''' Commit the current transactions.
475 '''
476 if __debug__:
477 print >>hyperdb.DEBUG, 'commit', (self,)
478 # TODO: lock the DB
480 # keep a handle to all the database files opened
481 self.databases = {}
483 # now, do all the transactions
484 reindex = {}
485 for method, args in self.transactions:
486 reindex[method(*args)] = 1
488 # now close all the database files
489 for db in self.databases.values():
490 db.close()
491 del self.databases
492 # TODO: unlock the DB
494 # reindex the nodes that request it
495 for classname, nodeid in filter(None, reindex.keys()):
496 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
497 self.getclass(classname).index(nodeid)
499 # save the indexer state
500 self.indexer.save_index()
502 # all transactions committed, back to normal
503 self.cache = {}
504 self.dirtynodes = {}
505 self.newnodes = {}
506 self.transactions = []
508 def _doSaveNode(self, classname, nodeid, node):
509 if __debug__:
510 print >>hyperdb.DEBUG, '_doSaveNode', (self, classname, nodeid,
511 node)
513 # get the database handle
514 db_name = 'nodes.%s'%classname
515 if self.databases.has_key(db_name):
516 db = self.databases[db_name]
517 else:
518 db = self.databases[db_name] = self.getclassdb(classname, 'c')
520 # now save the marshalled data
521 db[nodeid] = marshal.dumps(self.serialise(classname, node))
523 # return the classname, nodeid so we reindex this content
524 return (classname, nodeid)
526 def _doSaveJournal(self, classname, nodeid, action, params):
527 # serialise first
528 if action in ('set', 'create'):
529 params = self.serialise(classname, params)
531 # create the journal entry
532 entry = (nodeid, date.Date().get_tuple(), self.journaltag, action,
533 params)
535 if __debug__:
536 print >>hyperdb.DEBUG, '_doSaveJournal', entry
538 # get the database handle
539 db_name = 'journals.%s'%classname
540 if self.databases.has_key(db_name):
541 db = self.databases[db_name]
542 else:
543 db = self.databases[db_name] = self._opendb(db_name, 'c')
545 # now insert the journal entry
546 if db.has_key(nodeid):
547 # append to existing
548 s = db[nodeid]
549 l = marshal.loads(s)
550 l.append(entry)
551 else:
552 l = [entry]
554 db[nodeid] = marshal.dumps(l)
556 def rollback(self):
557 ''' Reverse all actions from the current transaction.
558 '''
559 if __debug__:
560 print >>hyperdb.DEBUG, 'rollback', (self, )
561 for method, args in self.transactions:
562 # delete temporary files
563 if method == self._doStoreFile:
564 self._rollbackStoreFile(*args)
565 self.cache = {}
566 self.dirtynodes = {}
567 self.newnodes = {}
568 self.transactions = []
570 _marker = []
571 class Class(hyperdb.Class):
572 """The handle to a particular class of nodes in a hyperdatabase."""
574 def __init__(self, db, classname, **properties):
575 """Create a new class with a given name and property specification.
577 'classname' must not collide with the name of an existing class,
578 or a ValueError is raised. The keyword arguments in 'properties'
579 must map names to property objects, or a TypeError is raised.
580 """
581 if (properties.has_key('creation') or properties.has_key('activity')
582 or properties.has_key('creator')):
583 raise ValueError, '"creation", "activity" and "creator" are '\
584 'reserved'
586 self.classname = classname
587 self.properties = properties
588 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
589 self.key = ''
591 # should we journal changes (default yes)
592 self.do_journal = 1
594 # do the db-related init stuff
595 db.addclass(self)
597 self.auditors = {'create': [], 'set': [], 'retire': []}
598 self.reactors = {'create': [], 'set': [], 'retire': []}
600 def enableJournalling(self):
601 '''Turn journalling on for this class
602 '''
603 self.do_journal = 1
605 def disableJournalling(self):
606 '''Turn journalling off for this class
607 '''
608 self.do_journal = 0
610 # Editing nodes:
612 def create(self, **propvalues):
613 """Create a new node of this class and return its id.
615 The keyword arguments in 'propvalues' map property names to values.
617 The values of arguments must be acceptable for the types of their
618 corresponding properties or a TypeError is raised.
620 If this class has a key property, it must be present and its value
621 must not collide with other key strings or a ValueError is raised.
623 Any other properties on this class that are missing from the
624 'propvalues' dictionary are set to None.
626 If an id in a link or multilink property does not refer to a valid
627 node, an IndexError is raised.
629 These operations trigger detectors and can be vetoed. Attempts
630 to modify the "creation" or "activity" properties cause a KeyError.
631 """
632 if propvalues.has_key('id'):
633 raise KeyError, '"id" is reserved'
635 if self.db.journaltag is None:
636 raise DatabaseError, 'Database open read-only'
638 if propvalues.has_key('creation') or propvalues.has_key('activity'):
639 raise KeyError, '"creation" and "activity" are reserved'
641 self.fireAuditors('create', None, propvalues)
643 # new node's id
644 newid = self.db.newid(self.classname)
646 # validate propvalues
647 num_re = re.compile('^\d+$')
648 for key, value in propvalues.items():
649 if key == self.key:
650 try:
651 self.lookup(value)
652 except KeyError:
653 pass
654 else:
655 raise ValueError, 'node with key "%s" exists'%value
657 # try to handle this property
658 try:
659 prop = self.properties[key]
660 except KeyError:
661 raise KeyError, '"%s" has no property "%s"'%(self.classname,
662 key)
664 if isinstance(prop, Link):
665 if type(value) != type(''):
666 raise ValueError, 'link value must be String'
667 link_class = self.properties[key].classname
668 # if it isn't a number, it's a key
669 if not num_re.match(value):
670 try:
671 value = self.db.classes[link_class].lookup(value)
672 except (TypeError, KeyError):
673 raise IndexError, 'new property "%s": %s not a %s'%(
674 key, value, link_class)
675 elif not self.db.hasnode(link_class, value):
676 raise IndexError, '%s has no node %s'%(link_class, value)
678 # save off the value
679 propvalues[key] = value
681 # register the link with the newly linked node
682 if self.do_journal and self.properties[key].do_journal:
683 self.db.addjournal(link_class, value, 'link',
684 (self.classname, newid, key))
686 elif isinstance(prop, Multilink):
687 if type(value) != type([]):
688 raise TypeError, 'new property "%s" not a list of ids'%key
690 # clean up and validate the list of links
691 link_class = self.properties[key].classname
692 l = []
693 for entry in value:
694 if type(entry) != type(''):
695 raise ValueError, '"%s" link value (%s) must be '\
696 'String'%(key, value)
697 # if it isn't a number, it's a key
698 if not num_re.match(entry):
699 try:
700 entry = self.db.classes[link_class].lookup(entry)
701 except (TypeError, KeyError):
702 raise IndexError, 'new property "%s": %s not a %s'%(
703 key, entry, self.properties[key].classname)
704 l.append(entry)
705 value = l
706 propvalues[key] = value
708 # handle additions
709 for id in value:
710 if not self.db.hasnode(link_class, id):
711 raise IndexError, '%s has no node %s'%(link_class, id)
712 # register the link with the newly linked node
713 if self.do_journal and self.properties[key].do_journal:
714 self.db.addjournal(link_class, id, 'link',
715 (self.classname, newid, key))
717 elif isinstance(prop, String):
718 if type(value) != type(''):
719 raise TypeError, 'new property "%s" not a string'%key
721 elif isinstance(prop, Password):
722 if not isinstance(value, password.Password):
723 raise TypeError, 'new property "%s" not a Password'%key
725 elif isinstance(prop, Date):
726 if value is not None and not isinstance(value, date.Date):
727 raise TypeError, 'new property "%s" not a Date'%key
729 elif isinstance(prop, Interval):
730 if value is not None and not isinstance(value, date.Interval):
731 raise TypeError, 'new property "%s" not an Interval'%key
733 # make sure there's data where there needs to be
734 for key, prop in self.properties.items():
735 if propvalues.has_key(key):
736 continue
737 if key == self.key:
738 raise ValueError, 'key property "%s" is required'%key
739 if isinstance(prop, Multilink):
740 propvalues[key] = []
741 else:
742 propvalues[key] = None
744 # done
745 self.db.addnode(self.classname, newid, propvalues)
746 if self.do_journal:
747 self.db.addjournal(self.classname, newid, 'create', propvalues)
749 self.fireReactors('create', newid, None)
751 return newid
753 def get(self, nodeid, propname, default=_marker, cache=1):
754 """Get the value of a property on an existing node of this class.
756 'nodeid' must be the id of an existing node of this class or an
757 IndexError is raised. 'propname' must be the name of a property
758 of this class or a KeyError is raised.
760 'cache' indicates whether the transaction cache should be queried
761 for the node. If the node has been modified and you need to
762 determine what its values prior to modification are, you need to
763 set cache=0.
765 Attempts to get the "creation" or "activity" properties should
766 do the right thing.
767 """
768 if propname == 'id':
769 return nodeid
771 if propname == 'creation':
772 if not self.do_journal:
773 raise ValueError, 'Journalling is disabled for this class'
774 journal = self.db.getjournal(self.classname, nodeid)
775 if journal:
776 return self.db.getjournal(self.classname, nodeid)[0][1]
777 else:
778 # on the strange chance that there's no journal
779 return date.Date()
780 if propname == 'activity':
781 if not self.do_journal:
782 raise ValueError, 'Journalling is disabled for this class'
783 journal = self.db.getjournal(self.classname, nodeid)
784 if journal:
785 return self.db.getjournal(self.classname, nodeid)[-1][1]
786 else:
787 # on the strange chance that there's no journal
788 return date.Date()
789 if propname == 'creator':
790 if not self.do_journal:
791 raise ValueError, 'Journalling is disabled for this class'
792 journal = self.db.getjournal(self.classname, nodeid)
793 if journal:
794 name = self.db.getjournal(self.classname, nodeid)[0][2]
795 else:
796 return None
797 return self.db.user.lookup(name)
799 # get the property (raises KeyErorr if invalid)
800 prop = self.properties[propname]
802 # get the node's dict
803 d = self.db.getnode(self.classname, nodeid, cache=cache)
805 if not d.has_key(propname):
806 if default is _marker:
807 if isinstance(prop, Multilink):
808 return []
809 else:
810 return None
811 else:
812 return default
814 return d[propname]
816 # XXX not in spec
817 def getnode(self, nodeid, cache=1):
818 ''' Return a convenience wrapper for the node.
820 'nodeid' must be the id of an existing node of this class or an
821 IndexError is raised.
823 'cache' indicates whether the transaction cache should be queried
824 for the node. If the node has been modified and you need to
825 determine what its values prior to modification are, you need to
826 set cache=0.
827 '''
828 return Node(self, nodeid, cache=cache)
830 def set(self, nodeid, **propvalues):
831 """Modify a property on an existing node of this class.
833 'nodeid' must be the id of an existing node of this class or an
834 IndexError is raised.
836 Each key in 'propvalues' must be the name of a property of this
837 class or a KeyError is raised.
839 All values in 'propvalues' must be acceptable types for their
840 corresponding properties or a TypeError is raised.
842 If the value of the key property is set, it must not collide with
843 other key strings or a ValueError is raised.
845 If the value of a Link or Multilink property contains an invalid
846 node id, a ValueError is raised.
848 These operations trigger detectors and can be vetoed. Attempts
849 to modify the "creation" or "activity" properties cause a KeyError.
850 """
851 if not propvalues:
852 return
854 if propvalues.has_key('creation') or propvalues.has_key('activity'):
855 raise KeyError, '"creation" and "activity" are reserved'
857 if propvalues.has_key('id'):
858 raise KeyError, '"id" is reserved'
860 if self.db.journaltag is None:
861 raise DatabaseError, 'Database open read-only'
863 self.fireAuditors('set', nodeid, propvalues)
864 # Take a copy of the node dict so that the subsequent set
865 # operation doesn't modify the oldvalues structure.
866 try:
867 # try not using the cache initially
868 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid,
869 cache=0))
870 except IndexError:
871 # this will be needed if somone does a create() and set()
872 # with no intervening commit()
873 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
875 node = self.db.getnode(self.classname, nodeid)
876 if node.has_key(self.db.RETIRED_FLAG):
877 raise IndexError
878 num_re = re.compile('^\d+$')
879 for key, value in propvalues.items():
880 # check to make sure we're not duplicating an existing key
881 if key == self.key and node[key] != value:
882 try:
883 self.lookup(value)
884 except KeyError:
885 pass
886 else:
887 raise ValueError, 'node with key "%s" exists'%value
889 # this will raise the KeyError if the property isn't valid
890 # ... we don't use getprops() here because we only care about
891 # the writeable properties.
892 prop = self.properties[key]
894 # if the value's the same as the existing value, no sense in
895 # doing anything
896 if node.has_key(key) and value == node[key]:
897 del propvalues[key]
898 continue
900 # do stuff based on the prop type
901 if isinstance(prop, Link):
902 link_class = self.properties[key].classname
903 # if it isn't a number, it's a key
904 if type(value) != type(''):
905 raise ValueError, 'link value must be String'
906 if not num_re.match(value):
907 try:
908 value = self.db.classes[link_class].lookup(value)
909 except (TypeError, KeyError):
910 raise IndexError, 'new property "%s": %s not a %s'%(
911 key, value, self.properties[key].classname)
913 if not self.db.hasnode(link_class, value):
914 raise IndexError, '%s has no node %s'%(link_class, value)
916 if self.do_journal and self.properties[key].do_journal:
917 # register the unlink with the old linked node
918 if node[key] is not None:
919 self.db.addjournal(link_class, node[key], 'unlink',
920 (self.classname, nodeid, key))
922 # register the link with the newly linked node
923 if value is not None:
924 self.db.addjournal(link_class, value, 'link',
925 (self.classname, nodeid, key))
927 elif isinstance(prop, Multilink):
928 if type(value) != type([]):
929 raise TypeError, 'new property "%s" not a list of ids'%key
930 link_class = self.properties[key].classname
931 l = []
932 for entry in value:
933 # if it isn't a number, it's a key
934 if type(entry) != type(''):
935 raise ValueError, 'new property "%s" link value ' \
936 'must be a string'%key
937 if not num_re.match(entry):
938 try:
939 entry = self.db.classes[link_class].lookup(entry)
940 except (TypeError, KeyError):
941 raise IndexError, 'new property "%s": %s not a %s'%(
942 key, entry, self.properties[key].classname)
943 l.append(entry)
944 value = l
945 propvalues[key] = value
947 # handle removals
948 if node.has_key(key):
949 l = node[key]
950 else:
951 l = []
952 for id in l[:]:
953 if id in value:
954 continue
955 # register the unlink with the old linked node
956 if self.do_journal and self.properties[key].do_journal:
957 self.db.addjournal(link_class, id, 'unlink',
958 (self.classname, nodeid, key))
959 l.remove(id)
961 # handle additions
962 for id in value:
963 if not self.db.hasnode(link_class, id):
964 raise IndexError, '%s has no node %s'%(
965 link_class, id)
966 if id in l:
967 continue
968 # register the link with the newly linked node
969 if self.do_journal and self.properties[key].do_journal:
970 self.db.addjournal(link_class, id, 'link',
971 (self.classname, nodeid, key))
972 l.append(id)
974 elif isinstance(prop, String):
975 if value is not None and type(value) != type(''):
976 raise TypeError, 'new property "%s" not a string'%key
978 elif isinstance(prop, Password):
979 if not isinstance(value, password.Password):
980 raise TypeError, 'new property "%s" not a Password'% key
981 propvalues[key] = value
983 elif value is not None and isinstance(prop, Date):
984 if not isinstance(value, date.Date):
985 raise TypeError, 'new property "%s" not a Date'% key
986 propvalues[key] = value
988 elif value is not None and isinstance(prop, Interval):
989 if not isinstance(value, date.Interval):
990 raise TypeError, 'new property "%s" not an Interval'% key
991 propvalues[key] = value
993 node[key] = value
995 # nothing to do?
996 if not propvalues:
997 return
999 # do the set, and journal it
1000 self.db.setnode(self.classname, nodeid, node)
1001 if self.do_journal:
1002 self.db.addjournal(self.classname, nodeid, 'set', propvalues)
1004 self.fireReactors('set', nodeid, oldvalues)
1006 def retire(self, nodeid):
1007 """Retire a node.
1009 The properties on the node remain available from the get() method,
1010 and the node's id is never reused.
1012 Retired nodes are not returned by the find(), list(), or lookup()
1013 methods, and other nodes may reuse the values of their key properties.
1015 These operations trigger detectors and can be vetoed. Attempts
1016 to modify the "creation" or "activity" properties cause a KeyError.
1017 """
1018 if self.db.journaltag is None:
1019 raise DatabaseError, 'Database open read-only'
1021 self.fireAuditors('retire', nodeid, None)
1023 node = self.db.getnode(self.classname, nodeid)
1024 node[self.db.RETIRED_FLAG] = 1
1025 self.db.setnode(self.classname, nodeid, node)
1026 if self.do_journal:
1027 self.db.addjournal(self.classname, nodeid, 'retired', None)
1029 self.fireReactors('retire', nodeid, None)
1031 def history(self, nodeid):
1032 """Retrieve the journal of edits on a particular node.
1034 'nodeid' must be the id of an existing node of this class or an
1035 IndexError is raised.
1037 The returned list contains tuples of the form
1039 (date, tag, action, params)
1041 'date' is a Timestamp object specifying the time of the change and
1042 'tag' is the journaltag specified when the database was opened.
1043 """
1044 if not self.do_journal:
1045 raise ValueError, 'Journalling is disabled for this class'
1046 return self.db.getjournal(self.classname, nodeid)
1048 # Locating nodes:
1049 def hasnode(self, nodeid):
1050 '''Determine if the given nodeid actually exists
1051 '''
1052 return self.db.hasnode(self.classname, nodeid)
1054 def setkey(self, propname):
1055 """Select a String property of this class to be the key property.
1057 'propname' must be the name of a String property of this class or
1058 None, or a TypeError is raised. The values of the key property on
1059 all existing nodes must be unique or a ValueError is raised. If the
1060 property doesn't exist, KeyError is raised.
1061 """
1062 prop = self.getprops()[propname]
1063 if not isinstance(prop, String):
1064 raise TypeError, 'key properties must be String'
1065 self.key = propname
1067 def getkey(self):
1068 """Return the name of the key property for this class or None."""
1069 return self.key
1071 def labelprop(self, default_to_id=0):
1072 ''' Return the property name for a label for the given node.
1074 This method attempts to generate a consistent label for the node.
1075 It tries the following in order:
1076 1. key property
1077 2. "name" property
1078 3. "title" property
1079 4. first property from the sorted property name list
1080 '''
1081 k = self.getkey()
1082 if k:
1083 return k
1084 props = self.getprops()
1085 if props.has_key('name'):
1086 return 'name'
1087 elif props.has_key('title'):
1088 return 'title'
1089 if default_to_id:
1090 return 'id'
1091 props = props.keys()
1092 props.sort()
1093 return props[0]
1095 # TODO: set up a separate index db file for this? profile?
1096 def lookup(self, keyvalue):
1097 """Locate a particular node by its key property and return its id.
1099 If this class has no key property, a TypeError is raised. If the
1100 'keyvalue' matches one of the values for the key property among
1101 the nodes in this class, the matching node's id is returned;
1102 otherwise a KeyError is raised.
1103 """
1104 cldb = self.db.getclassdb(self.classname)
1105 try:
1106 for nodeid in self.db.getnodeids(self.classname, cldb):
1107 node = self.db.getnode(self.classname, nodeid, cldb)
1108 if node.has_key(self.db.RETIRED_FLAG):
1109 continue
1110 if node[self.key] == keyvalue:
1111 cldb.close()
1112 return nodeid
1113 finally:
1114 cldb.close()
1115 raise KeyError, keyvalue
1117 # XXX: change from spec - allows multiple props to match
1118 def find(self, **propspec):
1119 """Get the ids of nodes in this class which link to the given nodes.
1121 'propspec' consists of keyword args propname={nodeid:1,}
1122 'propname' must be the name of a property in this class, or a
1123 KeyError is raised. That property must be a Link or Multilink
1124 property, or a TypeError is raised.
1126 Any node in this class whose 'propname' property links to any of the
1127 nodeids will be returned. Used by the full text indexing, which knows
1128 that "foo" occurs in msg1, msg3 and file7, so we have hits on these issues:
1129 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1130 """
1131 propspec = propspec.items()
1132 for propname, nodeids in propspec:
1133 # check the prop is OK
1134 prop = self.properties[propname]
1135 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1136 raise TypeError, "'%s' not a Link/Multilink property"%propname
1137 #XXX edit is expensive and of questionable use
1138 #for nodeid in nodeids:
1139 # if not self.db.hasnode(prop.classname, nodeid):
1140 # raise ValueError, '%s has no node %s'%(prop.classname, nodeid)
1142 # ok, now do the find
1143 cldb = self.db.getclassdb(self.classname)
1144 l = []
1145 try:
1146 for id in self.db.getnodeids(self.classname, db=cldb):
1147 node = self.db.getnode(self.classname, id, db=cldb)
1148 if node.has_key(self.db.RETIRED_FLAG):
1149 continue
1150 for propname, nodeids in propspec:
1151 # can't test if the node doesn't have this property
1152 if not node.has_key(propname):
1153 continue
1154 if type(nodeids) is type(''):
1155 nodeids = {nodeids:1}
1156 prop = self.properties[propname]
1157 value = node[propname]
1158 if isinstance(prop, Link) and nodeids.has_key(value):
1159 l.append(id)
1160 break
1161 elif isinstance(prop, Multilink):
1162 hit = 0
1163 for v in value:
1164 if nodeids.has_key(v):
1165 l.append(id)
1166 hit = 1
1167 break
1168 if hit:
1169 break
1170 finally:
1171 cldb.close()
1172 return l
1174 def stringFind(self, **requirements):
1175 """Locate a particular node by matching a set of its String
1176 properties in a caseless search.
1178 If the property is not a String property, a TypeError is raised.
1180 The return is a list of the id of all nodes that match.
1181 """
1182 for propname in requirements.keys():
1183 prop = self.properties[propname]
1184 if isinstance(not prop, String):
1185 raise TypeError, "'%s' not a String property"%propname
1186 requirements[propname] = requirements[propname].lower()
1187 l = []
1188 cldb = self.db.getclassdb(self.classname)
1189 try:
1190 for nodeid in self.db.getnodeids(self.classname, cldb):
1191 node = self.db.getnode(self.classname, nodeid, cldb)
1192 if node.has_key(self.db.RETIRED_FLAG):
1193 continue
1194 for key, value in requirements.items():
1195 if node[key] and node[key].lower() != value:
1196 break
1197 else:
1198 l.append(nodeid)
1199 finally:
1200 cldb.close()
1201 return l
1203 def list(self):
1204 """Return a list of the ids of the active nodes in this class."""
1205 l = []
1206 cn = self.classname
1207 cldb = self.db.getclassdb(cn)
1208 try:
1209 for nodeid in self.db.getnodeids(cn, cldb):
1210 node = self.db.getnode(cn, nodeid, cldb)
1211 if node.has_key(self.db.RETIRED_FLAG):
1212 continue
1213 l.append(nodeid)
1214 finally:
1215 cldb.close()
1216 l.sort()
1217 return l
1219 # XXX not in spec
1220 def filter(self, search_matches, filterspec, sort, group,
1221 num_re = re.compile('^\d+$')):
1222 ''' Return a list of the ids of the active nodes in this class that
1223 match the 'filter' spec, sorted by the group spec and then the
1224 sort spec
1225 '''
1226 cn = self.classname
1228 # optimise filterspec
1229 l = []
1230 props = self.getprops()
1231 for k, v in filterspec.items():
1232 propclass = props[k]
1233 if isinstance(propclass, Link):
1234 if type(v) is not type([]):
1235 v = [v]
1236 # replace key values with node ids
1237 u = []
1238 link_class = self.db.classes[propclass.classname]
1239 for entry in v:
1240 if entry == '-1': entry = None
1241 elif not num_re.match(entry):
1242 try:
1243 entry = link_class.lookup(entry)
1244 except (TypeError,KeyError):
1245 raise ValueError, 'property "%s": %s not a %s'%(
1246 k, entry, self.properties[k].classname)
1247 u.append(entry)
1249 l.append((0, k, u))
1250 elif isinstance(propclass, Multilink):
1251 if type(v) is not type([]):
1252 v = [v]
1253 # replace key values with node ids
1254 u = []
1255 link_class = self.db.classes[propclass.classname]
1256 for entry in v:
1257 if not num_re.match(entry):
1258 try:
1259 entry = link_class.lookup(entry)
1260 except (TypeError,KeyError):
1261 raise ValueError, 'new property "%s": %s not a %s'%(
1262 k, entry, self.properties[k].classname)
1263 u.append(entry)
1264 l.append((1, k, u))
1265 elif isinstance(propclass, String):
1266 # simple glob searching
1267 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1268 v = v.replace('?', '.')
1269 v = v.replace('*', '.*?')
1270 l.append((2, k, re.compile(v, re.I)))
1271 else:
1272 l.append((6, k, v))
1273 filterspec = l
1275 # now, find all the nodes that are active and pass filtering
1276 l = []
1277 cldb = self.db.getclassdb(cn)
1278 try:
1279 for nodeid in self.db.getnodeids(cn, cldb):
1280 node = self.db.getnode(cn, nodeid, cldb)
1281 if node.has_key(self.db.RETIRED_FLAG):
1282 continue
1283 # apply filter
1284 for t, k, v in filterspec:
1285 # this node doesn't have this property, so reject it
1286 if not node.has_key(k): break
1288 if t == 0 and node[k] not in v:
1289 # link - if this node'd property doesn't appear in the
1290 # filterspec's nodeid list, skip it
1291 break
1292 elif t == 1:
1293 # multilink - if any of the nodeids required by the
1294 # filterspec aren't in this node's property, then skip
1295 # it
1296 for value in v:
1297 if value not in node[k]:
1298 break
1299 else:
1300 continue
1301 break
1302 elif t == 2 and (node[k] is None or not v.search(node[k])):
1303 # RE search
1304 break
1305 elif t == 6 and node[k] != v:
1306 # straight value comparison for the other types
1307 break
1308 else:
1309 l.append((nodeid, node))
1310 finally:
1311 cldb.close()
1312 l.sort()
1314 # filter based on full text search
1315 if search_matches is not None:
1316 k = []
1317 l_debug = []
1318 for v in l:
1319 l_debug.append(v[0])
1320 if search_matches.has_key(v[0]):
1321 k.append(v)
1322 l = k
1324 # optimise sort
1325 m = []
1326 for entry in sort:
1327 if entry[0] != '-':
1328 m.append(('+', entry))
1329 else:
1330 m.append((entry[0], entry[1:]))
1331 sort = m
1333 # optimise group
1334 m = []
1335 for entry in group:
1336 if entry[0] != '-':
1337 m.append(('+', entry))
1338 else:
1339 m.append((entry[0], entry[1:]))
1340 group = m
1341 # now, sort the result
1342 def sortfun(a, b, sort=sort, group=group, properties=self.getprops(),
1343 db = self.db, cl=self):
1344 a_id, an = a
1345 b_id, bn = b
1346 # sort by group and then sort
1347 for list in group, sort:
1348 for dir, prop in list:
1349 # sorting is class-specific
1350 propclass = properties[prop]
1352 # handle the properties that might be "faked"
1353 # also, handle possible missing properties
1354 try:
1355 if not an.has_key(prop):
1356 an[prop] = cl.get(a_id, prop)
1357 av = an[prop]
1358 except KeyError:
1359 # the node doesn't have a value for this property
1360 if isinstance(propclass, Multilink): av = []
1361 else: av = ''
1362 try:
1363 if not bn.has_key(prop):
1364 bn[prop] = cl.get(b_id, prop)
1365 bv = bn[prop]
1366 except KeyError:
1367 # the node doesn't have a value for this property
1368 if isinstance(propclass, Multilink): bv = []
1369 else: bv = ''
1371 # String and Date values are sorted in the natural way
1372 if isinstance(propclass, String):
1373 # clean up the strings
1374 if av and av[0] in string.uppercase:
1375 av = an[prop] = av.lower()
1376 if bv and bv[0] in string.uppercase:
1377 bv = bn[prop] = bv.lower()
1378 if (isinstance(propclass, String) or
1379 isinstance(propclass, Date)):
1380 # it might be a string that's really an integer
1381 try:
1382 av = int(av)
1383 bv = int(bv)
1384 except:
1385 pass
1386 if dir == '+':
1387 r = cmp(av, bv)
1388 if r != 0: return r
1389 elif dir == '-':
1390 r = cmp(bv, av)
1391 if r != 0: return r
1393 # Link properties are sorted according to the value of
1394 # the "order" property on the linked nodes if it is
1395 # present; or otherwise on the key string of the linked
1396 # nodes; or finally on the node ids.
1397 elif isinstance(propclass, Link):
1398 link = db.classes[propclass.classname]
1399 if av is None and bv is not None: return -1
1400 if av is not None and bv is None: return 1
1401 if av is None and bv is None: continue
1402 if link.getprops().has_key('order'):
1403 if dir == '+':
1404 r = cmp(link.get(av, 'order'),
1405 link.get(bv, 'order'))
1406 if r != 0: return r
1407 elif dir == '-':
1408 r = cmp(link.get(bv, 'order'),
1409 link.get(av, 'order'))
1410 if r != 0: return r
1411 elif link.getkey():
1412 key = link.getkey()
1413 if dir == '+':
1414 r = cmp(link.get(av, key), link.get(bv, key))
1415 if r != 0: return r
1416 elif dir == '-':
1417 r = cmp(link.get(bv, key), link.get(av, key))
1418 if r != 0: return r
1419 else:
1420 if dir == '+':
1421 r = cmp(av, bv)
1422 if r != 0: return r
1423 elif dir == '-':
1424 r = cmp(bv, av)
1425 if r != 0: return r
1427 # Multilink properties are sorted according to how many
1428 # links are present.
1429 elif isinstance(propclass, Multilink):
1430 if dir == '+':
1431 r = cmp(len(av), len(bv))
1432 if r != 0: return r
1433 elif dir == '-':
1434 r = cmp(len(bv), len(av))
1435 if r != 0: return r
1436 # end for dir, prop in list:
1437 # end for list in sort, group:
1438 # if all else fails, compare the ids
1439 return cmp(a[0], b[0])
1441 l.sort(sortfun)
1442 return [i[0] for i in l]
1444 def count(self):
1445 """Get the number of nodes in this class.
1447 If the returned integer is 'numnodes', the ids of all the nodes
1448 in this class run from 1 to numnodes, and numnodes+1 will be the
1449 id of the next node to be created in this class.
1450 """
1451 return self.db.countnodes(self.classname)
1453 # Manipulating properties:
1455 def getprops(self, protected=1):
1456 """Return a dictionary mapping property names to property objects.
1457 If the "protected" flag is true, we include protected properties -
1458 those which may not be modified.
1460 In addition to the actual properties on the node, these
1461 methods provide the "creation" and "activity" properties. If the
1462 "protected" flag is true, we include protected properties - those
1463 which may not be modified.
1464 """
1465 d = self.properties.copy()
1466 if protected:
1467 d['id'] = String()
1468 d['creation'] = hyperdb.Date()
1469 d['activity'] = hyperdb.Date()
1470 d['creator'] = hyperdb.Link("user")
1471 return d
1473 def addprop(self, **properties):
1474 """Add properties to this class.
1476 The keyword arguments in 'properties' must map names to property
1477 objects, or a TypeError is raised. None of the keys in 'properties'
1478 may collide with the names of existing properties, or a ValueError
1479 is raised before any properties have been added.
1480 """
1481 for key in properties.keys():
1482 if self.properties.has_key(key):
1483 raise ValueError, key
1484 self.properties.update(properties)
1486 def index(self, nodeid):
1487 '''Add (or refresh) the node to search indexes
1488 '''
1489 # find all the String properties that have indexme
1490 for prop, propclass in self.getprops().items():
1491 if isinstance(propclass, String) and propclass.indexme:
1492 # and index them under (classname, nodeid, property)
1493 self.db.indexer.add_text((self.classname, nodeid, prop),
1494 str(self.get(nodeid, prop)))
1496 #
1497 # Detector interface
1498 #
1499 def audit(self, event, detector):
1500 """Register a detector
1501 """
1502 l = self.auditors[event]
1503 if detector not in l:
1504 self.auditors[event].append(detector)
1506 def fireAuditors(self, action, nodeid, newvalues):
1507 """Fire all registered auditors.
1508 """
1509 for audit in self.auditors[action]:
1510 audit(self.db, self, nodeid, newvalues)
1512 def react(self, event, detector):
1513 """Register a detector
1514 """
1515 l = self.reactors[event]
1516 if detector not in l:
1517 self.reactors[event].append(detector)
1519 def fireReactors(self, action, nodeid, oldvalues):
1520 """Fire all registered reactors.
1521 """
1522 for react in self.reactors[action]:
1523 react(self.db, self, nodeid, oldvalues)
1525 class FileClass(Class):
1526 '''This class defines a large chunk of data. To support this, it has a
1527 mandatory String property "content" which is typically saved off
1528 externally to the hyperdb.
1530 The default MIME type of this data is defined by the
1531 "default_mime_type" class attribute, which may be overridden by each
1532 node if the class defines a "type" String property.
1533 '''
1534 default_mime_type = 'text/plain'
1536 def create(self, **propvalues):
1537 ''' snaffle the file propvalue and store in a file
1538 '''
1539 content = propvalues['content']
1540 del propvalues['content']
1541 newid = Class.create(self, **propvalues)
1542 self.db.storefile(self.classname, newid, None, content)
1543 return newid
1545 def get(self, nodeid, propname, default=_marker, cache=1):
1546 ''' trap the content propname and get it from the file
1547 '''
1549 poss_msg = 'Possibly a access right configuration problem.'
1550 if propname == 'content':
1551 try:
1552 return self.db.getfile(self.classname, nodeid, None)
1553 except IOError, (strerror):
1554 # BUG: by catching this we donot see an error in the log.
1555 return 'ERROR reading file: %s%s\n%s\n%s'%(
1556 self.classname, nodeid, poss_msg, strerror)
1557 if default is not _marker:
1558 return Class.get(self, nodeid, propname, default, cache=cache)
1559 else:
1560 return Class.get(self, nodeid, propname, cache=cache)
1562 def getprops(self, protected=1):
1563 ''' In addition to the actual properties on the node, these methods
1564 provide the "content" property. If the "protected" flag is true,
1565 we include protected properties - those which may not be
1566 modified.
1567 '''
1568 d = Class.getprops(self, protected=protected).copy()
1569 if protected:
1570 d['content'] = hyperdb.String()
1571 return d
1573 def index(self, nodeid):
1574 ''' Index the node in the search index.
1576 We want to index the content in addition to the normal String
1577 property indexing.
1578 '''
1579 # perform normal indexing
1580 Class.index(self, nodeid)
1582 # get the content to index
1583 content = self.get(nodeid, 'content')
1585 # figure the mime type
1586 if self.properties.has_key('type'):
1587 mime_type = self.get(nodeid, 'type')
1588 else:
1589 mime_type = self.default_mime_type
1591 # and index!
1592 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
1593 mime_type)
1595 # XXX deviation from spec - was called ItemClass
1596 class IssueClass(Class, roundupdb.IssueClass):
1597 # Overridden methods:
1598 def __init__(self, db, classname, **properties):
1599 """The newly-created class automatically includes the "messages",
1600 "files", "nosy", and "superseder" properties. If the 'properties'
1601 dictionary attempts to specify any of these properties or a
1602 "creation" or "activity" property, a ValueError is raised.
1603 """
1604 if not properties.has_key('title'):
1605 properties['title'] = hyperdb.String(indexme='yes')
1606 if not properties.has_key('messages'):
1607 properties['messages'] = hyperdb.Multilink("msg")
1608 if not properties.has_key('files'):
1609 properties['files'] = hyperdb.Multilink("file")
1610 if not properties.has_key('nosy'):
1611 properties['nosy'] = hyperdb.Multilink("user")
1612 if not properties.has_key('superseder'):
1613 properties['superseder'] = hyperdb.Multilink(classname)
1614 Class.__init__(self, db, classname, **properties)
1616 #
1617 #$Log: not supported by cvs2svn $
1618 #Revision 1.45 2002/07/14 04:03:14 richard
1619 #Implemented a switch to disable journalling for a Class. CGI session
1620 #database now uses it.
1621 #
1622 #Revision 1.44 2002/07/14 02:05:53 richard
1623 #. all storage-specific code (ie. backend) is now implemented by the backends
1624 #
1625 #Revision 1.43 2002/07/10 06:30:30 richard
1626 #...except of course it's nice to use valid Python syntax
1627 #
1628 #Revision 1.42 2002/07/10 06:21:38 richard
1629 #Be extra safe
1630 #
1631 #Revision 1.41 2002/07/10 00:21:45 richard
1632 #explicit database closing
1633 #
1634 #Revision 1.40 2002/07/09 04:19:09 richard
1635 #Added reindex command to roundup-admin.
1636 #Fixed reindex on first access.
1637 #Also fixed reindexing of entries that change.
1638 #
1639 #Revision 1.39 2002/07/09 03:02:52 richard
1640 #More indexer work:
1641 #- all String properties may now be indexed too. Currently there's a bit of
1642 # "issue" specific code in the actual searching which needs to be
1643 # addressed. In a nutshell:
1644 # + pass 'indexme="yes"' as a String() property initialisation arg, eg:
1645 # file = FileClass(db, "file", name=String(), type=String(),
1646 # comment=String(indexme="yes"))
1647 # + the comment will then be indexed and be searchable, with the results
1648 # related back to the issue that the file is linked to
1649 #- as a result of this work, the FileClass has a default MIME type that may
1650 # be overridden in a subclass, or by the use of a "type" property as is
1651 # done in the default templates.
1652 #- the regeneration of the indexes (if necessary) is done once the schema is
1653 # set up in the dbinit.
1654 #
1655 #Revision 1.38 2002/07/08 06:58:15 richard
1656 #cleaned up the indexer code:
1657 # - it splits more words out (much simpler, faster splitter)
1658 # - removed code we'll never use (roundup.roundup_indexer has the full
1659 # implementation, and replaces roundup.indexer)
1660 # - only index text/plain and rfc822/message (ideas for other text formats to
1661 # index are welcome)
1662 # - added simple unit test for indexer. Needs more tests for regression.
1663 #
1664 #Revision 1.37 2002/06/20 23:52:35 richard
1665 #More informative error message
1666 #
1667 #Revision 1.36 2002/06/19 03:07:19 richard
1668 #Moved the file storage commit into blobfiles where it belongs.
1669 #
1670 #Revision 1.35 2002/05/25 07:16:24 rochecompaan
1671 #Merged search_indexing-branch with HEAD
1672 #
1673 #Revision 1.34 2002/05/15 06:21:21 richard
1674 # . node caching now works, and gives a small boost in performance
1675 #
1676 #As a part of this, I cleaned up the DEBUG output and implemented TRACE
1677 #output (HYPERDBTRACE='file to trace to') with checkpoints at the start of
1678 #CGI requests. Run roundup with python -O to skip all the DEBUG/TRACE stuff
1679 #(using if __debug__ which is compiled out with -O)
1680 #
1681 #Revision 1.33 2002/04/24 10:38:26 rochecompaan
1682 #All database files are now created group readable and writable.
1683 #
1684 #Revision 1.32 2002/04/15 23:25:15 richard
1685 #. node ids are now generated from a lockable store - no more race conditions
1686 #
1687 #We're using the portalocker code by Jonathan Feinberg that was contributed
1688 #to the ASPN Python cookbook. This gives us locking across Unix and Windows.
1689 #
1690 #Revision 1.31 2002/04/03 05:54:31 richard
1691 #Fixed serialisation problem by moving the serialisation step out of the
1692 #hyperdb.Class (get, set) into the hyperdb.Database.
1693 #
1694 #Also fixed htmltemplate after the showid changes I made yesterday.
1695 #
1696 #Unit tests for all of the above written.
1697 #
1698 #Revision 1.30.2.1 2002/04/03 11:55:57 rochecompaan
1699 # . Added feature #526730 - search for messages capability
1700 #
1701 #Revision 1.30 2002/02/27 03:40:59 richard
1702 #Ran it through pychecker, made fixes
1703 #
1704 #Revision 1.29 2002/02/25 14:34:31 grubert
1705 # . use blobfiles in back_anydbm which is used in back_bsddb.
1706 # change test_db as dirlist does not work for subdirectories.
1707 # ATTENTION: blobfiles now creates subdirectories for files.
1708 #
1709 #Revision 1.28 2002/02/16 09:14:17 richard
1710 # . #514854 ] History: "User" is always ticket creator
1711 #
1712 #Revision 1.27 2002/01/22 07:21:13 richard
1713 #. fixed back_bsddb so it passed the journal tests
1714 #
1715 #... it didn't seem happy using the back_anydbm _open method, which is odd.
1716 #Yet another occurrance of whichdb not being able to recognise older bsddb
1717 #databases. Yadda yadda. Made the HYPERDBDEBUG stuff more sane in the
1718 #process.
1719 #
1720 #Revision 1.26 2002/01/22 05:18:38 rochecompaan
1721 #last_set_entry was referenced before assignment
1722 #
1723 #Revision 1.25 2002/01/22 05:06:08 rochecompaan
1724 #We need to keep the last 'set' entry in the journal to preserve
1725 #information on 'activity' for nodes.
1726 #
1727 #Revision 1.24 2002/01/21 16:33:20 rochecompaan
1728 #You can now use the roundup-admin tool to pack the database
1729 #
1730 #Revision 1.23 2002/01/18 04:32:04 richard
1731 #Rollback was breaking because a message hadn't actually been written to the file. Needs
1732 #more investigation.
1733 #
1734 #Revision 1.22 2002/01/14 02:20:15 richard
1735 # . changed all config accesses so they access either the instance or the
1736 # config attriubute on the db. This means that all config is obtained from
1737 # instance_config instead of the mish-mash of classes. This will make
1738 # switching to a ConfigParser setup easier too, I hope.
1739 #
1740 #At a minimum, this makes migration a _little_ easier (a lot easier in the
1741 #0.5.0 switch, I hope!)
1742 #
1743 #Revision 1.21 2002/01/02 02:31:38 richard
1744 #Sorry for the huge checkin message - I was only intending to implement #496356
1745 #but I found a number of places where things had been broken by transactions:
1746 # . modified ROUNDUPDBSENDMAILDEBUG to be SENDMAILDEBUG and hold a filename
1747 # for _all_ roundup-generated smtp messages to be sent to.
1748 # . the transaction cache had broken the roundupdb.Class set() reactors
1749 # . newly-created author users in the mailgw weren't being committed to the db
1750 #
1751 #Stuff that made it into CHANGES.txt (ie. the stuff I was actually working
1752 #on when I found that stuff :):
1753 # . #496356 ] Use threading in messages
1754 # . detectors were being registered multiple times
1755 # . added tests for mailgw
1756 # . much better attaching of erroneous messages in the mail gateway
1757 #
1758 #Revision 1.20 2001/12/18 15:30:34 rochecompaan
1759 #Fixed bugs:
1760 # . Fixed file creation and retrieval in same transaction in anydbm
1761 # backend
1762 # . Cgi interface now renders new issue after issue creation
1763 # . Could not set issue status to resolved through cgi interface
1764 # . Mail gateway was changing status back to 'chatting' if status was
1765 # omitted as an argument
1766 #
1767 #Revision 1.19 2001/12/17 03:52:48 richard
1768 #Implemented file store rollback. As a bonus, the hyperdb is now capable of
1769 #storing more than one file per node - if a property name is supplied,
1770 #the file is called designator.property.
1771 #I decided not to migrate the existing files stored over to the new naming
1772 #scheme - the FileClass just doesn't specify the property name.
1773 #
1774 #Revision 1.18 2001/12/16 10:53:38 richard
1775 #take a copy of the node dict so that the subsequent set
1776 #operation doesn't modify the oldvalues structure
1777 #
1778 #Revision 1.17 2001/12/14 23:42:57 richard
1779 #yuck, a gdbm instance tests false :(
1780 #I've left the debugging code in - it should be removed one day if we're ever
1781 #_really_ anal about performace :)
1782 #
1783 #Revision 1.16 2001/12/12 03:23:14 richard
1784 #Cor blimey this anydbm/whichdb stuff is yecchy. Turns out that whichdb
1785 #incorrectly identifies a dbm file as a dbhash file on my system. This has
1786 #been submitted to the python bug tracker as issue #491888:
1787 #https://sourceforge.net/tracker/index.php?func=detail&aid=491888&group_id=5470&atid=105470
1788 #
1789 #Revision 1.15 2001/12/12 02:30:51 richard
1790 #I fixed the problems with people whose anydbm was using the dbm module at the
1791 #backend. It turns out the dbm module modifies the file name to append ".db"
1792 #and my check to determine if we're opening an existing or new db just
1793 #tested os.path.exists() on the filename. Well, no longer! We now perform a
1794 #much better check _and_ cope with the anydbm implementation module changing
1795 #too!
1796 #I also fixed the backends __init__ so only ImportError is squashed.
1797 #
1798 #Revision 1.14 2001/12/10 22:20:01 richard
1799 #Enabled transaction support in the bsddb backend. It uses the anydbm code
1800 #where possible, only replacing methods where the db is opened (it uses the
1801 #btree opener specifically.)
1802 #Also cleaned up some change note generation.
1803 #Made the backends package work with pydoc too.
1804 #
1805 #Revision 1.13 2001/12/02 05:06:16 richard
1806 #. We now use weakrefs in the Classes to keep the database reference, so
1807 # the close() method on the database is no longer needed.
1808 # I bumped the minimum python requirement up to 2.1 accordingly.
1809 #. #487480 ] roundup-server
1810 #. #487476 ] INSTALL.txt
1811 #
1812 #I also cleaned up the change message / post-edit stuff in the cgi client.
1813 #There's now a clearly marked "TODO: append the change note" where I believe
1814 #the change note should be added there. The "changes" list will obviously
1815 #have to be modified to be a dict of the changes, or somesuch.
1816 #
1817 #More testing needed.
1818 #
1819 #Revision 1.12 2001/12/01 07:17:50 richard
1820 #. We now have basic transaction support! Information is only written to
1821 # the database when the commit() method is called. Only the anydbm
1822 # backend is modified in this way - neither of the bsddb backends have been.
1823 # The mail, admin and cgi interfaces all use commit (except the admin tool
1824 # doesn't have a commit command, so interactive users can't commit...)
1825 #. Fixed login/registration forwarding the user to the right page (or not,
1826 # on a failure)
1827 #
1828 #Revision 1.11 2001/11/21 02:34:18 richard
1829 #Added a target version field to the extended issue schema
1830 #
1831 #Revision 1.10 2001/10/09 23:58:10 richard
1832 #Moved the data stringification up into the hyperdb.Class class' get, set
1833 #and create methods. This means that the data is also stringified for the
1834 #journal call, and removes duplication of code from the backends. The
1835 #backend code now only sees strings.
1836 #
1837 #Revision 1.9 2001/10/09 07:25:59 richard
1838 #Added the Password property type. See "pydoc roundup.password" for
1839 #implementation details. Have updated some of the documentation too.
1840 #
1841 #Revision 1.8 2001/09/29 13:27:00 richard
1842 #CGI interfaces now spit up a top-level index of all the instances they can
1843 #serve.
1844 #
1845 #Revision 1.7 2001/08/12 06:32:36 richard
1846 #using isinstance(blah, Foo) now instead of isFooType
1847 #
1848 #Revision 1.6 2001/08/07 00:24:42 richard
1849 #stupid typo
1850 #
1851 #Revision 1.5 2001/08/07 00:15:51 richard
1852 #Added the copyright/license notice to (nearly) all files at request of
1853 #Bizar Software.
1854 #
1855 #Revision 1.4 2001/07/30 01:41:36 richard
1856 #Makes schema changes mucho easier.
1857 #
1858 #Revision 1.3 2001/07/25 01:23:07 richard
1859 #Added the Roundup spec to the new documentation directory.
1860 #
1861 #Revision 1.2 2001/07/23 08:20:44 richard
1862 #Moved over to using marshal in the bsddb and anydbm backends.
1863 #roundup-admin now has a "freshen" command that'll load/save all nodes (not
1864 # retired - mod hyperdb.Class.list() so it lists retired nodes)
1865 #
1866 #