![[tokkee]](http://tokkee.org/images/avatar.png){kind=avatar}
128453ff67575d67f11b1ded6dfd1912d8d3759a
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: back_anydbm.py,v 1.53 2002-07-25 07:14:06 richard Exp $
19 '''
20 This module defines a backend that saves the hyperdatabase in a database
21 chosen by anydbm. It is guaranteed to always be available in python
22 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
23 serious bugs, and is not available)
24 '''
26 import whichdb, anydbm, os, marshal, re, weakref, string, copy
27 from roundup import hyperdb, date, password, roundupdb, security
28 from blobfiles import FileStorage
29 from roundup.indexer import Indexer
30 from locking import acquire_lock, release_lock
31 from roundup.hyperdb import String, Password, Date, Interval, Link, \
32 Multilink, DatabaseError, Boolean, Number
34 #
35 # Now the database
36 #
37 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
38 """A database for storing records containing flexible data types.
40 Transaction stuff TODO:
41 . check the timestamp of the class file and nuke the cache if it's
42 modified. Do some sort of conflict checking on the dirty stuff.
43 . perhaps detect write collisions (related to above)?
45 """
46 def __init__(self, config, journaltag=None):
47 """Open a hyperdatabase given a specifier to some storage.
49 The 'storagelocator' is obtained from config.DATABASE.
50 The meaning of 'storagelocator' depends on the particular
51 implementation of the hyperdatabase. It could be a file name,
52 a directory path, a socket descriptor for a connection to a
53 database over the network, etc.
55 The 'journaltag' is a token that will be attached to the journal
56 entries for any edits done on the database. If 'journaltag' is
57 None, the database is opened in read-only mode: the Class.create(),
58 Class.set(), and Class.retire() methods are disabled.
59 """
60 self.config, self.journaltag = config, journaltag
61 self.dir = config.DATABASE
62 self.classes = {}
63 self.cache = {} # cache of nodes loaded or created
64 self.dirtynodes = {} # keep track of the dirty nodes by class
65 self.newnodes = {} # keep track of the new nodes by class
66 self.destroyednodes = {}# keep track of the destroyed nodes by class
67 self.transactions = []
68 self.indexer = Indexer(self.dir)
69 self.security = security.Security(self)
70 # ensure files are group readable and writable
71 os.umask(0002)
73 def post_init(self):
74 """Called once the schema initialisation has finished."""
75 # reindex the db if necessary
76 if self.indexer.should_reindex():
77 self.reindex()
79 def reindex(self):
80 for klass in self.classes.values():
81 for nodeid in klass.list():
82 klass.index(nodeid)
83 self.indexer.save_index()
85 def __repr__(self):
86 return '<back_anydbm instance at %x>'%id(self)
88 #
89 # Classes
90 #
91 def __getattr__(self, classname):
92 """A convenient way of calling self.getclass(classname)."""
93 if self.classes.has_key(classname):
94 if __debug__:
95 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
96 return self.classes[classname]
97 raise AttributeError, classname
99 def addclass(self, cl):
100 if __debug__:
101 print >>hyperdb.DEBUG, 'addclass', (self, cl)
102 cn = cl.classname
103 if self.classes.has_key(cn):
104 raise ValueError, cn
105 self.classes[cn] = cl
107 def getclasses(self):
108 """Return a list of the names of all existing classes."""
109 if __debug__:
110 print >>hyperdb.DEBUG, 'getclasses', (self,)
111 l = self.classes.keys()
112 l.sort()
113 return l
115 def getclass(self, classname):
116 """Get the Class object representing a particular class.
118 If 'classname' is not a valid class name, a KeyError is raised.
119 """
120 if __debug__:
121 print >>hyperdb.DEBUG, 'getclass', (self, classname)
122 return self.classes[classname]
124 #
125 # Class DBs
126 #
127 def clear(self):
128 '''Delete all database contents
129 '''
130 if __debug__:
131 print >>hyperdb.DEBUG, 'clear', (self,)
132 for cn in self.classes.keys():
133 for dummy in 'nodes', 'journals':
134 path = os.path.join(self.dir, 'journals.%s'%cn)
135 if os.path.exists(path):
136 os.remove(path)
137 elif os.path.exists(path+'.db'): # dbm appends .db
138 os.remove(path+'.db')
140 def getclassdb(self, classname, mode='r'):
141 ''' grab a connection to the class db that will be used for
142 multiple actions
143 '''
144 if __debug__:
145 print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode)
146 return self.opendb('nodes.%s'%classname, mode)
148 def determine_db_type(self, path):
149 ''' determine which DB wrote the class file
150 '''
151 db_type = ''
152 if os.path.exists(path):
153 db_type = whichdb.whichdb(path)
154 if not db_type:
155 raise hyperdb.DatabaseError, "Couldn't identify database type"
156 elif os.path.exists(path+'.db'):
157 # if the path ends in '.db', it's a dbm database, whether
158 # anydbm says it's dbhash or not!
159 db_type = 'dbm'
160 return db_type
162 def opendb(self, name, mode):
163 '''Low-level database opener that gets around anydbm/dbm
164 eccentricities.
165 '''
166 if __debug__:
167 print >>hyperdb.DEBUG, 'opendb', (self, name, mode)
169 # figure the class db type
170 path = os.path.join(os.getcwd(), self.dir, name)
171 db_type = self.determine_db_type(path)
173 # new database? let anydbm pick the best dbm
174 if not db_type:
175 if __debug__:
176 print >>hyperdb.DEBUG, "opendb anydbm.open(%r, 'n')"%path
177 return anydbm.open(path, 'n')
179 # open the database with the correct module
180 try:
181 dbm = __import__(db_type)
182 except ImportError:
183 raise hyperdb.DatabaseError, \
184 "Couldn't open database - the required module '%s'"\
185 " is not available"%db_type
186 if __debug__:
187 print >>hyperdb.DEBUG, "opendb %r.open(%r, %r)"%(db_type, path,
188 mode)
189 return dbm.open(path, mode)
191 def lockdb(self, name):
192 ''' Lock a database file
193 '''
194 path = os.path.join(os.getcwd(), self.dir, '%s.lock'%name)
195 return acquire_lock(path)
197 #
198 # Node IDs
199 #
200 def newid(self, classname):
201 ''' Generate a new id for the given class
202 '''
203 # open the ids DB - create if if doesn't exist
204 lock = self.lockdb('_ids')
205 db = self.opendb('_ids', 'c')
206 if db.has_key(classname):
207 newid = db[classname] = str(int(db[classname]) + 1)
208 else:
209 # the count() bit is transitional - older dbs won't start at 1
210 newid = str(self.getclass(classname).count()+1)
211 db[classname] = newid
212 db.close()
213 release_lock(lock)
214 return newid
216 #
217 # Nodes
218 #
219 def addnode(self, classname, nodeid, node):
220 ''' add the specified node to its class's db
221 '''
222 if __debug__:
223 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
224 self.newnodes.setdefault(classname, {})[nodeid] = 1
225 self.cache.setdefault(classname, {})[nodeid] = node
226 self.savenode(classname, nodeid, node)
228 def setnode(self, classname, nodeid, node):
229 ''' change the specified node
230 '''
231 if __debug__:
232 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
233 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
235 # can't set without having already loaded the node
236 self.cache[classname][nodeid] = node
237 self.savenode(classname, nodeid, node)
239 def savenode(self, classname, nodeid, node):
240 ''' perform the saving of data specified by the set/addnode
241 '''
242 if __debug__:
243 print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node)
244 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
246 def getnode(self, classname, nodeid, db=None, cache=1):
247 ''' get a node from the database
248 '''
249 if __debug__:
250 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid, db)
251 if cache:
252 # try the cache
253 cache_dict = self.cache.setdefault(classname, {})
254 if cache_dict.has_key(nodeid):
255 if __debug__:
256 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
257 nodeid)
258 return cache_dict[nodeid]
260 if __debug__:
261 print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid)
263 # get from the database and save in the cache
264 if db is None:
265 db = self.getclassdb(classname)
266 if not db.has_key(nodeid):
267 raise IndexError, "no such %s %s"%(classname, nodeid)
269 # check the uncommitted, destroyed nodes
270 if (self.destroyednodes.has_key(classname) and
271 self.destroyednodes[classname].has_key(nodeid)):
272 raise IndexError, "no such %s %s"%(classname, nodeid)
274 # decode
275 res = marshal.loads(db[nodeid])
277 # reverse the serialisation
278 res = self.unserialise(classname, res)
280 # store off in the cache dict
281 if cache:
282 cache_dict[nodeid] = res
284 return res
286 def destroynode(self, classname, nodeid):
287 '''Remove a node from the database. Called exclusively by the
288 destroy() method on Class.
289 '''
290 if __debug__:
291 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
293 # remove from cache and newnodes if it's there
294 if (self.cache.has_key(classname) and
295 self.cache[classname].has_key(nodeid)):
296 del self.cache[classname][nodeid]
297 if (self.newnodes.has_key(classname) and
298 self.newnodes[classname].has_key(nodeid)):
299 del self.newnodes[classname][nodeid]
301 # see if there's any obvious commit actions that we should get rid of
302 for entry in self.transactions[:]:
303 if entry[1][:2] == (classname, nodeid):
304 self.transactions.remove(entry)
306 # add to the destroyednodes map
307 self.destroyednodes.setdefault(classname, {})[nodeid] = 1
309 # add the destroy commit action
310 self.transactions.append((self.doDestroyNode, (classname, nodeid)))
312 def serialise(self, classname, node):
313 '''Copy the node contents, converting non-marshallable data into
314 marshallable data.
315 '''
316 if __debug__:
317 print >>hyperdb.DEBUG, 'serialise', classname, node
318 properties = self.getclass(classname).getprops()
319 d = {}
320 for k, v in node.items():
321 # if the property doesn't exist, or is the "retired" flag then
322 # it won't be in the properties dict
323 if not properties.has_key(k):
324 d[k] = v
325 continue
327 # get the property spec
328 prop = properties[k]
330 if isinstance(prop, Password):
331 d[k] = str(v)
332 elif isinstance(prop, Date) and v is not None:
333 d[k] = v.get_tuple()
334 elif isinstance(prop, Interval) and v is not None:
335 d[k] = v.get_tuple()
336 else:
337 d[k] = v
338 return d
340 def unserialise(self, classname, node):
341 '''Decode the marshalled node data
342 '''
343 if __debug__:
344 print >>hyperdb.DEBUG, 'unserialise', classname, node
345 properties = self.getclass(classname).getprops()
346 d = {}
347 for k, v in node.items():
348 # if the property doesn't exist, or is the "retired" flag then
349 # it won't be in the properties dict
350 if not properties.has_key(k):
351 d[k] = v
352 continue
354 # get the property spec
355 prop = properties[k]
357 if isinstance(prop, Date) and v is not None:
358 d[k] = date.Date(v)
359 elif isinstance(prop, Interval) and v is not None:
360 d[k] = date.Interval(v)
361 elif isinstance(prop, Password):
362 p = password.Password()
363 p.unpack(v)
364 d[k] = p
365 else:
366 d[k] = v
367 return d
369 def hasnode(self, classname, nodeid, db=None):
370 ''' determine if the database has a given node
371 '''
372 if __debug__:
373 print >>hyperdb.DEBUG, 'hasnode', (self, classname, nodeid, db)
375 # try the cache
376 cache = self.cache.setdefault(classname, {})
377 if cache.has_key(nodeid):
378 if __debug__:
379 print >>hyperdb.TRACE, 'has %s %s cached'%(classname, nodeid)
380 return 1
381 if __debug__:
382 print >>hyperdb.TRACE, 'has %s %s'%(classname, nodeid)
384 # not in the cache - check the database
385 if db is None:
386 db = self.getclassdb(classname)
387 res = db.has_key(nodeid)
388 return res
390 def countnodes(self, classname, db=None):
391 if __debug__:
392 print >>hyperdb.DEBUG, 'countnodes', (self, classname, db)
394 count = 0
396 # include the uncommitted nodes
397 if self.newnodes.has_key(classname):
398 count += len(self.newnodes[classname])
399 if self.destroyednodes.has_key(classname):
400 count -= len(self.destroyednodes[classname])
402 # and count those in the DB
403 if db is None:
404 db = self.getclassdb(classname)
405 count = count + len(db.keys())
406 return count
408 def getnodeids(self, classname, db=None):
409 if __debug__:
410 print >>hyperdb.DEBUG, 'getnodeids', (self, classname, db)
412 res = []
414 # start off with the new nodes
415 if self.newnodes.has_key(classname):
416 res += self.newnodes[classname].keys()
418 if db is None:
419 db = self.getclassdb(classname)
420 res = res + db.keys()
422 # remove the uncommitted, destroyed nodes
423 if self.destroyednodes.has_key(classname):
424 for nodeid in self.destroyednodes[classname].keys():
425 if db.has_key(nodeid):
426 res.remove(nodeid)
428 return res
431 #
432 # Files - special node properties
433 # inherited from FileStorage
435 #
436 # Journal
437 #
438 def addjournal(self, classname, nodeid, action, params):
439 ''' Journal the Action
440 'action' may be:
442 'create' or 'set' -- 'params' is a dictionary of property values
443 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
444 'retire' -- 'params' is None
445 '''
446 if __debug__:
447 print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid,
448 action, params)
449 self.transactions.append((self.doSaveJournal, (classname, nodeid,
450 action, params)))
452 def getjournal(self, classname, nodeid):
453 ''' get the journal for id
455 Raise IndexError if the node doesn't exist (as per history()'s
456 API)
457 '''
458 if __debug__:
459 print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid)
460 # attempt to open the journal - in some rare cases, the journal may
461 # not exist
462 try:
463 db = self.opendb('journals.%s'%classname, 'r')
464 except anydbm.error, error:
465 if str(error) == "need 'c' or 'n' flag to open new db":
466 raise IndexError, 'no such %s %s'%(classname, nodeid)
467 elif error.args[0] != 2:
468 raise
469 raise IndexError, 'no such %s %s'%(classname, nodeid)
470 try:
471 journal = marshal.loads(db[nodeid])
472 except KeyError:
473 db.close()
474 raise IndexError, 'no such %s %s'%(classname, nodeid)
475 db.close()
476 res = []
477 for nodeid, date_stamp, user, action, params in journal:
478 res.append((nodeid, date.Date(date_stamp), user, action, params))
479 return res
481 def pack(self, pack_before):
482 ''' delete all journal entries before 'pack_before' '''
483 if __debug__:
484 print >>hyperdb.DEBUG, 'packjournal', (self, pack_before)
486 pack_before = pack_before.get_tuple()
488 classes = self.getclasses()
490 # figure the class db type
492 for classname in classes:
493 db_name = 'journals.%s'%classname
494 path = os.path.join(os.getcwd(), self.dir, classname)
495 db_type = self.determine_db_type(path)
496 db = self.opendb(db_name, 'w')
498 for key in db.keys():
499 journal = marshal.loads(db[key])
500 l = []
501 last_set_entry = None
502 for entry in journal:
503 (nodeid, date_stamp, self.journaltag, action,
504 params) = entry
505 if date_stamp > pack_before or action == 'create':
506 l.append(entry)
507 elif action == 'set':
508 # grab the last set entry to keep information on
509 # activity
510 last_set_entry = entry
511 if last_set_entry:
512 date_stamp = last_set_entry[1]
513 # if the last set entry was made after the pack date
514 # then it is already in the list
515 if date_stamp < pack_before:
516 l.append(last_set_entry)
517 db[key] = marshal.dumps(l)
518 if db_type == 'gdbm':
519 db.reorganize()
520 db.close()
523 #
524 # Basic transaction support
525 #
526 def commit(self):
527 ''' Commit the current transactions.
528 '''
529 if __debug__:
530 print >>hyperdb.DEBUG, 'commit', (self,)
531 # TODO: lock the DB
533 # keep a handle to all the database files opened
534 self.databases = {}
536 # now, do all the transactions
537 reindex = {}
538 for method, args in self.transactions:
539 reindex[method(*args)] = 1
541 # now close all the database files
542 for db in self.databases.values():
543 db.close()
544 del self.databases
545 # TODO: unlock the DB
547 # reindex the nodes that request it
548 for classname, nodeid in filter(None, reindex.keys()):
549 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
550 self.getclass(classname).index(nodeid)
552 # save the indexer state
553 self.indexer.save_index()
555 # all transactions committed, back to normal
556 self.cache = {}
557 self.dirtynodes = {}
558 self.newnodes = {}
559 self.destroyednodes = {}
560 self.transactions = []
562 def getCachedClassDB(self, classname):
563 ''' get the class db, looking in our cache of databases for commit
564 '''
565 # get the database handle
566 db_name = 'nodes.%s'%classname
567 if not self.databases.has_key(db_name):
568 self.databases[db_name] = self.getclassdb(classname, 'c')
569 return self.databases[db_name]
571 def doSaveNode(self, classname, nodeid, node):
572 if __debug__:
573 print >>hyperdb.DEBUG, 'doSaveNode', (self, classname, nodeid,
574 node)
576 db = self.getCachedClassDB(classname)
578 # now save the marshalled data
579 db[nodeid] = marshal.dumps(self.serialise(classname, node))
581 # return the classname, nodeid so we reindex this content
582 return (classname, nodeid)
584 def getCachedJournalDB(self, classname):
585 ''' get the journal db, looking in our cache of databases for commit
586 '''
587 # get the database handle
588 db_name = 'journals.%s'%classname
589 if not self.databases.has_key(db_name):
590 self.databases[db_name] = self.opendb(db_name, 'c')
591 return self.databases[db_name]
593 def doSaveJournal(self, classname, nodeid, action, params):
594 # serialise first
595 if action in ('set', 'create'):
596 params = self.serialise(classname, params)
598 # create the journal entry
599 entry = (nodeid, date.Date().get_tuple(), self.journaltag, action,
600 params)
602 if __debug__:
603 print >>hyperdb.DEBUG, 'doSaveJournal', entry
605 db = self.getCachedJournalDB(classname)
607 # now insert the journal entry
608 if db.has_key(nodeid):
609 # append to existing
610 s = db[nodeid]
611 l = marshal.loads(s)
612 l.append(entry)
613 else:
614 l = [entry]
616 db[nodeid] = marshal.dumps(l)
618 def doDestroyNode(self, classname, nodeid):
619 if __debug__:
620 print >>hyperdb.DEBUG, 'doDestroyNode', (self, classname, nodeid)
622 # delete from the class database
623 db = self.getCachedClassDB(classname)
624 if db.has_key(nodeid):
625 del db[nodeid]
627 # delete from the database
628 db = self.getCachedJournalDB(classname)
629 if db.has_key(nodeid):
630 del db[nodeid]
632 # return the classname, nodeid so we reindex this content
633 return (classname, nodeid)
635 def rollback(self):
636 ''' Reverse all actions from the current transaction.
637 '''
638 if __debug__:
639 print >>hyperdb.DEBUG, 'rollback', (self, )
640 for method, args in self.transactions:
641 # delete temporary files
642 if method == self.doStoreFile:
643 self.rollbackStoreFile(*args)
644 self.cache = {}
645 self.dirtynodes = {}
646 self.newnodes = {}
647 self.destroyednodes = {}
648 self.transactions = []
650 _marker = []
651 class Class(hyperdb.Class):
652 """The handle to a particular class of nodes in a hyperdatabase."""
654 def __init__(self, db, classname, **properties):
655 """Create a new class with a given name and property specification.
657 'classname' must not collide with the name of an existing class,
658 or a ValueError is raised. The keyword arguments in 'properties'
659 must map names to property objects, or a TypeError is raised.
660 """
661 if (properties.has_key('creation') or properties.has_key('activity')
662 or properties.has_key('creator')):
663 raise ValueError, '"creation", "activity" and "creator" are '\
664 'reserved'
666 self.classname = classname
667 self.properties = properties
668 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
669 self.key = ''
671 # should we journal changes (default yes)
672 self.do_journal = 1
674 # do the db-related init stuff
675 db.addclass(self)
677 self.auditors = {'create': [], 'set': [], 'retire': []}
678 self.reactors = {'create': [], 'set': [], 'retire': []}
680 def enableJournalling(self):
681 '''Turn journalling on for this class
682 '''
683 self.do_journal = 1
685 def disableJournalling(self):
686 '''Turn journalling off for this class
687 '''
688 self.do_journal = 0
690 # Editing nodes:
692 def create(self, **propvalues):
693 """Create a new node of this class and return its id.
695 The keyword arguments in 'propvalues' map property names to values.
697 The values of arguments must be acceptable for the types of their
698 corresponding properties or a TypeError is raised.
700 If this class has a key property, it must be present and its value
701 must not collide with other key strings or a ValueError is raised.
703 Any other properties on this class that are missing from the
704 'propvalues' dictionary are set to None.
706 If an id in a link or multilink property does not refer to a valid
707 node, an IndexError is raised.
709 These operations trigger detectors and can be vetoed. Attempts
710 to modify the "creation" or "activity" properties cause a KeyError.
711 """
712 if propvalues.has_key('id'):
713 raise KeyError, '"id" is reserved'
715 if self.db.journaltag is None:
716 raise DatabaseError, 'Database open read-only'
718 if propvalues.has_key('creation') or propvalues.has_key('activity'):
719 raise KeyError, '"creation" and "activity" are reserved'
721 self.fireAuditors('create', None, propvalues)
723 # new node's id
724 newid = self.db.newid(self.classname)
726 # validate propvalues
727 num_re = re.compile('^\d+$')
728 for key, value in propvalues.items():
729 if key == self.key:
730 try:
731 self.lookup(value)
732 except KeyError:
733 pass
734 else:
735 raise ValueError, 'node with key "%s" exists'%value
737 # try to handle this property
738 try:
739 prop = self.properties[key]
740 except KeyError:
741 raise KeyError, '"%s" has no property "%s"'%(self.classname,
742 key)
744 if isinstance(prop, Link):
745 if type(value) != type(''):
746 raise ValueError, 'link value must be String'
747 link_class = self.properties[key].classname
748 # if it isn't a number, it's a key
749 if not num_re.match(value):
750 try:
751 value = self.db.classes[link_class].lookup(value)
752 except (TypeError, KeyError):
753 raise IndexError, 'new property "%s": %s not a %s'%(
754 key, value, link_class)
755 elif not self.db.getclass(link_class).hasnode(value):
756 raise IndexError, '%s has no node %s'%(link_class, value)
758 # save off the value
759 propvalues[key] = value
761 # register the link with the newly linked node
762 if self.do_journal and self.properties[key].do_journal:
763 self.db.addjournal(link_class, value, 'link',
764 (self.classname, newid, key))
766 elif isinstance(prop, Multilink):
767 if type(value) != type([]):
768 raise TypeError, 'new property "%s" not a list of ids'%key
770 # clean up and validate the list of links
771 link_class = self.properties[key].classname
772 l = []
773 for entry in value:
774 if type(entry) != type(''):
775 raise ValueError, '"%s" link value (%s) must be '\
776 'String'%(key, value)
777 # if it isn't a number, it's a key
778 if not num_re.match(entry):
779 try:
780 entry = self.db.classes[link_class].lookup(entry)
781 except (TypeError, KeyError):
782 raise IndexError, 'new property "%s": %s not a %s'%(
783 key, entry, self.properties[key].classname)
784 l.append(entry)
785 value = l
786 propvalues[key] = value
788 # handle additions
789 for nodeid in value:
790 if not self.db.getclass(link_class).hasnode(nodeid):
791 raise IndexError, '%s has no node %s'%(link_class,
792 nodeid)
793 # register the link with the newly linked node
794 if self.do_journal and self.properties[key].do_journal:
795 self.db.addjournal(link_class, nodeid, 'link',
796 (self.classname, newid, key))
798 elif isinstance(prop, String):
799 if type(value) != type(''):
800 raise TypeError, 'new property "%s" not a string'%key
802 elif isinstance(prop, Password):
803 if not isinstance(value, password.Password):
804 raise TypeError, 'new property "%s" not a Password'%key
806 elif isinstance(prop, Date):
807 if value is not None and not isinstance(value, date.Date):
808 raise TypeError, 'new property "%s" not a Date'%key
810 elif isinstance(prop, Interval):
811 if value is not None and not isinstance(value, date.Interval):
812 raise TypeError, 'new property "%s" not an Interval'%key
814 elif value is not None and isinstance(prop, Number):
815 try:
816 float(value)
817 except ValueError:
818 raise TypeError, 'new property "%s" not numeric'%key
820 elif value is not None and isinstance(prop, Boolean):
821 try:
822 int(value)
823 except ValueError:
824 raise TypeError, 'new property "%s" not boolean'%key
826 # make sure there's data where there needs to be
827 for key, prop in self.properties.items():
828 if propvalues.has_key(key):
829 continue
830 if key == self.key:
831 raise ValueError, 'key property "%s" is required'%key
832 if isinstance(prop, Multilink):
833 propvalues[key] = []
834 else:
835 propvalues[key] = None
837 # done
838 self.db.addnode(self.classname, newid, propvalues)
839 if self.do_journal:
840 self.db.addjournal(self.classname, newid, 'create', propvalues)
842 self.fireReactors('create', newid, None)
844 return newid
846 def get(self, nodeid, propname, default=_marker, cache=1):
847 """Get the value of a property on an existing node of this class.
849 'nodeid' must be the id of an existing node of this class or an
850 IndexError is raised. 'propname' must be the name of a property
851 of this class or a KeyError is raised.
853 'cache' indicates whether the transaction cache should be queried
854 for the node. If the node has been modified and you need to
855 determine what its values prior to modification are, you need to
856 set cache=0.
858 Attempts to get the "creation" or "activity" properties should
859 do the right thing.
860 """
861 if propname == 'id':
862 return nodeid
864 if propname == 'creation':
865 if not self.do_journal:
866 raise ValueError, 'Journalling is disabled for this class'
867 journal = self.db.getjournal(self.classname, nodeid)
868 if journal:
869 return self.db.getjournal(self.classname, nodeid)[0][1]
870 else:
871 # on the strange chance that there's no journal
872 return date.Date()
873 if propname == 'activity':
874 if not self.do_journal:
875 raise ValueError, 'Journalling is disabled for this class'
876 journal = self.db.getjournal(self.classname, nodeid)
877 if journal:
878 return self.db.getjournal(self.classname, nodeid)[-1][1]
879 else:
880 # on the strange chance that there's no journal
881 return date.Date()
882 if propname == 'creator':
883 if not self.do_journal:
884 raise ValueError, 'Journalling is disabled for this class'
885 journal = self.db.getjournal(self.classname, nodeid)
886 if journal:
887 name = self.db.getjournal(self.classname, nodeid)[0][2]
888 else:
889 return None
890 return self.db.user.lookup(name)
892 # get the property (raises KeyErorr if invalid)
893 prop = self.properties[propname]
895 # get the node's dict
896 d = self.db.getnode(self.classname, nodeid, cache=cache)
898 if not d.has_key(propname):
899 if default is _marker:
900 if isinstance(prop, Multilink):
901 return []
902 else:
903 return None
904 else:
905 return default
907 return d[propname]
909 # XXX not in spec
910 def getnode(self, nodeid, cache=1):
911 ''' Return a convenience wrapper for the node.
913 'nodeid' must be the id of an existing node of this class or an
914 IndexError is raised.
916 'cache' indicates whether the transaction cache should be queried
917 for the node. If the node has been modified and you need to
918 determine what its values prior to modification are, you need to
919 set cache=0.
920 '''
921 return Node(self, nodeid, cache=cache)
923 def set(self, nodeid, **propvalues):
924 """Modify a property on an existing node of this class.
926 'nodeid' must be the id of an existing node of this class or an
927 IndexError is raised.
929 Each key in 'propvalues' must be the name of a property of this
930 class or a KeyError is raised.
932 All values in 'propvalues' must be acceptable types for their
933 corresponding properties or a TypeError is raised.
935 If the value of the key property is set, it must not collide with
936 other key strings or a ValueError is raised.
938 If the value of a Link or Multilink property contains an invalid
939 node id, a ValueError is raised.
941 These operations trigger detectors and can be vetoed. Attempts
942 to modify the "creation" or "activity" properties cause a KeyError.
943 """
944 if not propvalues:
945 return
947 if propvalues.has_key('creation') or propvalues.has_key('activity'):
948 raise KeyError, '"creation" and "activity" are reserved'
950 if propvalues.has_key('id'):
951 raise KeyError, '"id" is reserved'
953 if self.db.journaltag is None:
954 raise DatabaseError, 'Database open read-only'
956 self.fireAuditors('set', nodeid, propvalues)
957 # Take a copy of the node dict so that the subsequent set
958 # operation doesn't modify the oldvalues structure.
959 try:
960 # try not using the cache initially
961 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid,
962 cache=0))
963 except IndexError:
964 # this will be needed if somone does a create() and set()
965 # with no intervening commit()
966 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
968 node = self.db.getnode(self.classname, nodeid)
969 if node.has_key(self.db.RETIRED_FLAG):
970 raise IndexError
971 num_re = re.compile('^\d+$')
973 # if the journal value is to be different, store it in here
974 journalvalues = {}
976 for propname, value in propvalues.items():
977 # check to make sure we're not duplicating an existing key
978 if propname == self.key and node[propname] != value:
979 try:
980 self.lookup(value)
981 except KeyError:
982 pass
983 else:
984 raise ValueError, 'node with key "%s" exists'%value
986 # this will raise the KeyError if the property isn't valid
987 # ... we don't use getprops() here because we only care about
988 # the writeable properties.
989 prop = self.properties[propname]
991 # if the value's the same as the existing value, no sense in
992 # doing anything
993 if node.has_key(propname) and value == node[propname]:
994 del propvalues[propname]
995 continue
997 # do stuff based on the prop type
998 if isinstance(prop, Link):
999 link_class = self.properties[propname].classname
1000 # if it isn't a number, it's a key
1001 if type(value) != type(''):
1002 raise ValueError, 'link value must be String'
1003 if not num_re.match(value):
1004 try:
1005 value = self.db.classes[link_class].lookup(value)
1006 except (TypeError, KeyError):
1007 raise IndexError, 'new property "%s": %s not a %s'%(
1008 propname, value, self.properties[propname].classname)
1010 if not self.db.getclass(link_class).hasnode(value):
1011 raise IndexError, '%s has no node %s'%(link_class, value)
1013 if self.do_journal and self.properties[propname].do_journal:
1014 # register the unlink with the old linked node
1015 if node[propname] is not None:
1016 self.db.addjournal(link_class, node[propname], 'unlink',
1017 (self.classname, nodeid, propname))
1019 # register the link with the newly linked node
1020 if value is not None:
1021 self.db.addjournal(link_class, value, 'link',
1022 (self.classname, nodeid, propname))
1024 elif isinstance(prop, Multilink):
1025 if type(value) != type([]):
1026 raise TypeError, 'new property "%s" not a list of'\
1027 ' ids'%propname
1028 link_class = self.properties[propname].classname
1029 l = []
1030 for entry in value:
1031 # if it isn't a number, it's a key
1032 if type(entry) != type(''):
1033 raise ValueError, 'new property "%s" link value ' \
1034 'must be a string'%propname
1035 if not num_re.match(entry):
1036 try:
1037 entry = self.db.classes[link_class].lookup(entry)
1038 except (TypeError, KeyError):
1039 raise IndexError, 'new property "%s": %s not a %s'%(
1040 propname, entry,
1041 self.properties[propname].classname)
1042 l.append(entry)
1043 value = l
1044 propvalues[propname] = value
1046 # figure the journal entry for this property
1047 add = []
1048 remove = []
1050 # handle removals
1051 if node.has_key(propname):
1052 l = node[propname]
1053 else:
1054 l = []
1055 for id in l[:]:
1056 if id in value:
1057 continue
1058 # register the unlink with the old linked node
1059 if self.do_journal and self.properties[propname].do_journal:
1060 self.db.addjournal(link_class, id, 'unlink',
1061 (self.classname, nodeid, propname))
1062 l.remove(id)
1063 remove.append(id)
1065 # handle additions
1066 for id in value:
1067 if not self.db.getclass(link_class).hasnode(id):
1068 raise IndexError, '%s has no node %s'%(link_class, id)
1069 if id in l:
1070 continue
1071 # register the link with the newly linked node
1072 if self.do_journal and self.properties[propname].do_journal:
1073 self.db.addjournal(link_class, id, 'link',
1074 (self.classname, nodeid, propname))
1075 l.append(id)
1076 add.append(id)
1078 # figure the journal entry
1079 l = []
1080 if add:
1081 l.append(('add', add))
1082 if remove:
1083 l.append(('remove', remove))
1084 if l:
1085 journalvalues[propname] = tuple(l)
1087 elif isinstance(prop, String):
1088 if value is not None and type(value) != type(''):
1089 raise TypeError, 'new property "%s" not a string'%propname
1091 elif isinstance(prop, Password):
1092 if not isinstance(value, password.Password):
1093 raise TypeError, 'new property "%s" not a Password'%propname
1094 propvalues[propname] = value
1096 elif value is not None and isinstance(prop, Date):
1097 if not isinstance(value, date.Date):
1098 raise TypeError, 'new property "%s" not a Date'% propname
1099 propvalues[propname] = value
1101 elif value is not None and isinstance(prop, Interval):
1102 if not isinstance(value, date.Interval):
1103 raise TypeError, 'new property "%s" not an '\
1104 'Interval'%propname
1105 propvalues[propname] = value
1107 elif value is not None and isinstance(prop, Number):
1108 try:
1109 float(value)
1110 except ValueError:
1111 raise TypeError, 'new property "%s" not numeric'%propname
1113 elif value is not None and isinstance(prop, Boolean):
1114 try:
1115 int(value)
1116 except ValueError:
1117 raise TypeError, 'new property "%s" not boolean'%propname
1119 node[propname] = value
1121 # nothing to do?
1122 if not propvalues:
1123 return
1125 # do the set, and journal it
1126 self.db.setnode(self.classname, nodeid, node)
1128 if self.do_journal:
1129 propvalues.update(journalvalues)
1130 self.db.addjournal(self.classname, nodeid, 'set', propvalues)
1132 self.fireReactors('set', nodeid, oldvalues)
1134 def retire(self, nodeid):
1135 """Retire a node.
1137 The properties on the node remain available from the get() method,
1138 and the node's id is never reused.
1140 Retired nodes are not returned by the find(), list(), or lookup()
1141 methods, and other nodes may reuse the values of their key properties.
1143 These operations trigger detectors and can be vetoed. Attempts
1144 to modify the "creation" or "activity" properties cause a KeyError.
1145 """
1146 if self.db.journaltag is None:
1147 raise DatabaseError, 'Database open read-only'
1149 self.fireAuditors('retire', nodeid, None)
1151 node = self.db.getnode(self.classname, nodeid)
1152 node[self.db.RETIRED_FLAG] = 1
1153 self.db.setnode(self.classname, nodeid, node)
1154 if self.do_journal:
1155 self.db.addjournal(self.classname, nodeid, 'retired', None)
1157 self.fireReactors('retire', nodeid, None)
1159 def destroy(self, nodeid):
1160 """Destroy a node.
1162 WARNING: this method should never be used except in extremely rare
1163 situations where there could never be links to the node being
1164 deleted
1165 WARNING: use retire() instead
1166 WARNING: the properties of this node will not be available ever again
1167 WARNING: really, use retire() instead
1169 Well, I think that's enough warnings. This method exists mostly to
1170 support the session storage of the cgi interface.
1171 """
1172 if self.db.journaltag is None:
1173 raise DatabaseError, 'Database open read-only'
1174 self.db.destroynode(self.classname, nodeid)
1176 def history(self, nodeid):
1177 """Retrieve the journal of edits on a particular node.
1179 'nodeid' must be the id of an existing node of this class or an
1180 IndexError is raised.
1182 The returned list contains tuples of the form
1184 (date, tag, action, params)
1186 'date' is a Timestamp object specifying the time of the change and
1187 'tag' is the journaltag specified when the database was opened.
1188 """
1189 if not self.do_journal:
1190 raise ValueError, 'Journalling is disabled for this class'
1191 return self.db.getjournal(self.classname, nodeid)
1193 # Locating nodes:
1194 def hasnode(self, nodeid):
1195 '''Determine if the given nodeid actually exists
1196 '''
1197 return self.db.hasnode(self.classname, nodeid)
1199 def setkey(self, propname):
1200 """Select a String property of this class to be the key property.
1202 'propname' must be the name of a String property of this class or
1203 None, or a TypeError is raised. The values of the key property on
1204 all existing nodes must be unique or a ValueError is raised. If the
1205 property doesn't exist, KeyError is raised.
1206 """
1207 prop = self.getprops()[propname]
1208 if not isinstance(prop, String):
1209 raise TypeError, 'key properties must be String'
1210 self.key = propname
1212 def getkey(self):
1213 """Return the name of the key property for this class or None."""
1214 return self.key
1216 def labelprop(self, default_to_id=0):
1217 ''' Return the property name for a label for the given node.
1219 This method attempts to generate a consistent label for the node.
1220 It tries the following in order:
1221 1. key property
1222 2. "name" property
1223 3. "title" property
1224 4. first property from the sorted property name list
1225 '''
1226 k = self.getkey()
1227 if k:
1228 return k
1229 props = self.getprops()
1230 if props.has_key('name'):
1231 return 'name'
1232 elif props.has_key('title'):
1233 return 'title'
1234 if default_to_id:
1235 return 'id'
1236 props = props.keys()
1237 props.sort()
1238 return props[0]
1240 # TODO: set up a separate index db file for this? profile?
1241 def lookup(self, keyvalue):
1242 """Locate a particular node by its key property and return its id.
1244 If this class has no key property, a TypeError is raised. If the
1245 'keyvalue' matches one of the values for the key property among
1246 the nodes in this class, the matching node's id is returned;
1247 otherwise a KeyError is raised.
1248 """
1249 cldb = self.db.getclassdb(self.classname)
1250 try:
1251 for nodeid in self.db.getnodeids(self.classname, cldb):
1252 node = self.db.getnode(self.classname, nodeid, cldb)
1253 if node.has_key(self.db.RETIRED_FLAG):
1254 continue
1255 if node[self.key] == keyvalue:
1256 cldb.close()
1257 return nodeid
1258 finally:
1259 cldb.close()
1260 raise KeyError, keyvalue
1262 # XXX: change from spec - allows multiple props to match
1263 def find(self, **propspec):
1264 """Get the ids of nodes in this class which link to the given nodes.
1266 'propspec' consists of keyword args propname={nodeid:1,}
1267 'propname' must be the name of a property in this class, or a
1268 KeyError is raised. That property must be a Link or Multilink
1269 property, or a TypeError is raised.
1271 Any node in this class whose 'propname' property links to any of the
1272 nodeids will be returned. Used by the full text indexing, which knows
1273 that "foo" occurs in msg1, msg3 and file7, so we have hits on these issues:
1274 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1275 """
1276 propspec = propspec.items()
1277 for propname, nodeids in propspec:
1278 # check the prop is OK
1279 prop = self.properties[propname]
1280 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1281 raise TypeError, "'%s' not a Link/Multilink property"%propname
1283 # ok, now do the find
1284 cldb = self.db.getclassdb(self.classname)
1285 l = []
1286 try:
1287 for id in self.db.getnodeids(self.classname, db=cldb):
1288 node = self.db.getnode(self.classname, id, db=cldb)
1289 if node.has_key(self.db.RETIRED_FLAG):
1290 continue
1291 for propname, nodeids in propspec:
1292 # can't test if the node doesn't have this property
1293 if not node.has_key(propname):
1294 continue
1295 if type(nodeids) is type(''):
1296 nodeids = {nodeids:1}
1297 prop = self.properties[propname]
1298 value = node[propname]
1299 if isinstance(prop, Link) and nodeids.has_key(value):
1300 l.append(id)
1301 break
1302 elif isinstance(prop, Multilink):
1303 hit = 0
1304 for v in value:
1305 if nodeids.has_key(v):
1306 l.append(id)
1307 hit = 1
1308 break
1309 if hit:
1310 break
1311 finally:
1312 cldb.close()
1313 return l
1315 def stringFind(self, **requirements):
1316 """Locate a particular node by matching a set of its String
1317 properties in a caseless search.
1319 If the property is not a String property, a TypeError is raised.
1321 The return is a list of the id of all nodes that match.
1322 """
1323 for propname in requirements.keys():
1324 prop = self.properties[propname]
1325 if isinstance(not prop, String):
1326 raise TypeError, "'%s' not a String property"%propname
1327 requirements[propname] = requirements[propname].lower()
1328 l = []
1329 cldb = self.db.getclassdb(self.classname)
1330 try:
1331 for nodeid in self.db.getnodeids(self.classname, cldb):
1332 node = self.db.getnode(self.classname, nodeid, cldb)
1333 if node.has_key(self.db.RETIRED_FLAG):
1334 continue
1335 for key, value in requirements.items():
1336 if node[key] and node[key].lower() != value:
1337 break
1338 else:
1339 l.append(nodeid)
1340 finally:
1341 cldb.close()
1342 return l
1344 def list(self):
1345 """Return a list of the ids of the active nodes in this class."""
1346 l = []
1347 cn = self.classname
1348 cldb = self.db.getclassdb(cn)
1349 try:
1350 for nodeid in self.db.getnodeids(cn, cldb):
1351 node = self.db.getnode(cn, nodeid, cldb)
1352 if node.has_key(self.db.RETIRED_FLAG):
1353 continue
1354 l.append(nodeid)
1355 finally:
1356 cldb.close()
1357 l.sort()
1358 return l
1360 # XXX not in spec
1361 def filter(self, search_matches, filterspec, sort, group,
1362 num_re = re.compile('^\d+$')):
1363 ''' Return a list of the ids of the active nodes in this class that
1364 match the 'filter' spec, sorted by the group spec and then the
1365 sort spec
1366 '''
1367 cn = self.classname
1369 # optimise filterspec
1370 l = []
1371 props = self.getprops()
1372 for k, v in filterspec.items():
1373 propclass = props[k]
1374 if isinstance(propclass, Link):
1375 if type(v) is not type([]):
1376 v = [v]
1377 # replace key values with node ids
1378 u = []
1379 link_class = self.db.classes[propclass.classname]
1380 for entry in v:
1381 if entry == '-1': entry = None
1382 elif not num_re.match(entry):
1383 try:
1384 entry = link_class.lookup(entry)
1385 except (TypeError,KeyError):
1386 raise ValueError, 'property "%s": %s not a %s'%(
1387 k, entry, self.properties[k].classname)
1388 u.append(entry)
1390 l.append((0, k, u))
1391 elif isinstance(propclass, Multilink):
1392 if type(v) is not type([]):
1393 v = [v]
1394 # replace key values with node ids
1395 u = []
1396 link_class = self.db.classes[propclass.classname]
1397 for entry in v:
1398 if not num_re.match(entry):
1399 try:
1400 entry = link_class.lookup(entry)
1401 except (TypeError,KeyError):
1402 raise ValueError, 'new property "%s": %s not a %s'%(
1403 k, entry, self.properties[k].classname)
1404 u.append(entry)
1405 l.append((1, k, u))
1406 elif isinstance(propclass, String):
1407 # simple glob searching
1408 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
1409 v = v.replace('?', '.')
1410 v = v.replace('*', '.*?')
1411 l.append((2, k, re.compile(v, re.I)))
1412 elif isinstance(propclass, Boolean):
1413 if type(v) is type(''):
1414 bv = v.lower() in ('yes', 'true', 'on', '1')
1415 else:
1416 bv = v
1417 l.append((6, k, bv))
1418 elif isinstance(propclass, Number):
1419 l.append((6, k, int(v)))
1420 else:
1421 l.append((6, k, v))
1422 filterspec = l
1424 # now, find all the nodes that are active and pass filtering
1425 l = []
1426 cldb = self.db.getclassdb(cn)
1427 try:
1428 for nodeid in self.db.getnodeids(cn, cldb):
1429 node = self.db.getnode(cn, nodeid, cldb)
1430 if node.has_key(self.db.RETIRED_FLAG):
1431 continue
1432 # apply filter
1433 for t, k, v in filterspec:
1434 # this node doesn't have this property, so reject it
1435 if not node.has_key(k): break
1437 if t == 0 and node[k] not in v:
1438 # link - if this node'd property doesn't appear in the
1439 # filterspec's nodeid list, skip it
1440 break
1441 elif t == 1:
1442 # multilink - if any of the nodeids required by the
1443 # filterspec aren't in this node's property, then skip
1444 # it
1445 for value in v:
1446 if value not in node[k]:
1447 break
1448 else:
1449 continue
1450 break
1451 elif t == 2 and (node[k] is None or not v.search(node[k])):
1452 # RE search
1453 break
1454 elif t == 6 and node[k] != v:
1455 # straight value comparison for the other types
1456 break
1457 else:
1458 l.append((nodeid, node))
1459 finally:
1460 cldb.close()
1461 l.sort()
1463 # filter based on full text search
1464 if search_matches is not None:
1465 k = []
1466 l_debug = []
1467 for v in l:
1468 l_debug.append(v[0])
1469 if search_matches.has_key(v[0]):
1470 k.append(v)
1471 l = k
1473 # optimise sort
1474 m = []
1475 for entry in sort:
1476 if entry[0] != '-':
1477 m.append(('+', entry))
1478 else:
1479 m.append((entry[0], entry[1:]))
1480 sort = m
1482 # optimise group
1483 m = []
1484 for entry in group:
1485 if entry[0] != '-':
1486 m.append(('+', entry))
1487 else:
1488 m.append((entry[0], entry[1:]))
1489 group = m
1490 # now, sort the result
1491 def sortfun(a, b, sort=sort, group=group, properties=self.getprops(),
1492 db = self.db, cl=self):
1493 a_id, an = a
1494 b_id, bn = b
1495 # sort by group and then sort
1496 for list in group, sort:
1497 for dir, prop in list:
1498 # sorting is class-specific
1499 propclass = properties[prop]
1501 # handle the properties that might be "faked"
1502 # also, handle possible missing properties
1503 try:
1504 if not an.has_key(prop):
1505 an[prop] = cl.get(a_id, prop)
1506 av = an[prop]
1507 except KeyError:
1508 # the node doesn't have a value for this property
1509 if isinstance(propclass, Multilink): av = []
1510 else: av = ''
1511 try:
1512 if not bn.has_key(prop):
1513 bn[prop] = cl.get(b_id, prop)
1514 bv = bn[prop]
1515 except KeyError:
1516 # the node doesn't have a value for this property
1517 if isinstance(propclass, Multilink): bv = []
1518 else: bv = ''
1520 # String and Date values are sorted in the natural way
1521 if isinstance(propclass, String):
1522 # clean up the strings
1523 if av and av[0] in string.uppercase:
1524 av = an[prop] = av.lower()
1525 if bv and bv[0] in string.uppercase:
1526 bv = bn[prop] = bv.lower()
1527 if (isinstance(propclass, String) or
1528 isinstance(propclass, Date)):
1529 # it might be a string that's really an integer
1530 try:
1531 av = int(av)
1532 bv = int(bv)
1533 except:
1534 pass
1535 if dir == '+':
1536 r = cmp(av, bv)
1537 if r != 0: return r
1538 elif dir == '-':
1539 r = cmp(bv, av)
1540 if r != 0: return r
1542 # Link properties are sorted according to the value of
1543 # the "order" property on the linked nodes if it is
1544 # present; or otherwise on the key string of the linked
1545 # nodes; or finally on the node ids.
1546 elif isinstance(propclass, Link):
1547 link = db.classes[propclass.classname]
1548 if av is None and bv is not None: return -1
1549 if av is not None and bv is None: return 1
1550 if av is None and bv is None: continue
1551 if link.getprops().has_key('order'):
1552 if dir == '+':
1553 r = cmp(link.get(av, 'order'),
1554 link.get(bv, 'order'))
1555 if r != 0: return r
1556 elif dir == '-':
1557 r = cmp(link.get(bv, 'order'),
1558 link.get(av, 'order'))
1559 if r != 0: return r
1560 elif link.getkey():
1561 key = link.getkey()
1562 if dir == '+':
1563 r = cmp(link.get(av, key), link.get(bv, key))
1564 if r != 0: return r
1565 elif dir == '-':
1566 r = cmp(link.get(bv, key), link.get(av, key))
1567 if r != 0: return r
1568 else:
1569 if dir == '+':
1570 r = cmp(av, bv)
1571 if r != 0: return r
1572 elif dir == '-':
1573 r = cmp(bv, av)
1574 if r != 0: return r
1576 # Multilink properties are sorted according to how many
1577 # links are present.
1578 elif isinstance(propclass, Multilink):
1579 if dir == '+':
1580 r = cmp(len(av), len(bv))
1581 if r != 0: return r
1582 elif dir == '-':
1583 r = cmp(len(bv), len(av))
1584 if r != 0: return r
1585 elif isinstance(propclass, Number) or isinstance(propclass, Boolean):
1586 if dir == '+':
1587 r = cmp(av, bv)
1588 elif dir == '-':
1589 r = cmp(bv, av)
1591 # end for dir, prop in list:
1592 # end for list in sort, group:
1593 # if all else fails, compare the ids
1594 return cmp(a[0], b[0])
1596 l.sort(sortfun)
1597 return [i[0] for i in l]
1599 def count(self):
1600 """Get the number of nodes in this class.
1602 If the returned integer is 'numnodes', the ids of all the nodes
1603 in this class run from 1 to numnodes, and numnodes+1 will be the
1604 id of the next node to be created in this class.
1605 """
1606 return self.db.countnodes(self.classname)
1608 # Manipulating properties:
1610 def getprops(self, protected=1):
1611 """Return a dictionary mapping property names to property objects.
1612 If the "protected" flag is true, we include protected properties -
1613 those which may not be modified.
1615 In addition to the actual properties on the node, these
1616 methods provide the "creation" and "activity" properties. If the
1617 "protected" flag is true, we include protected properties - those
1618 which may not be modified.
1619 """
1620 d = self.properties.copy()
1621 if protected:
1622 d['id'] = String()
1623 d['creation'] = hyperdb.Date()
1624 d['activity'] = hyperdb.Date()
1625 d['creator'] = hyperdb.Link("user")
1626 return d
1628 def addprop(self, **properties):
1629 """Add properties to this class.
1631 The keyword arguments in 'properties' must map names to property
1632 objects, or a TypeError is raised. None of the keys in 'properties'
1633 may collide with the names of existing properties, or a ValueError
1634 is raised before any properties have been added.
1635 """
1636 for key in properties.keys():
1637 if self.properties.has_key(key):
1638 raise ValueError, key
1639 self.properties.update(properties)
1641 def index(self, nodeid):
1642 '''Add (or refresh) the node to search indexes
1643 '''
1644 # find all the String properties that have indexme
1645 for prop, propclass in self.getprops().items():
1646 if isinstance(propclass, String) and propclass.indexme:
1647 try:
1648 value = str(self.get(nodeid, prop))
1649 except IndexError:
1650 # node no longer exists - entry should be removed
1651 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1652 else:
1653 # and index them under (classname, nodeid, property)
1654 self.db.indexer.add_text((self.classname, nodeid, prop),
1655 value)
1657 #
1658 # Detector interface
1659 #
1660 def audit(self, event, detector):
1661 """Register a detector
1662 """
1663 l = self.auditors[event]
1664 if detector not in l:
1665 self.auditors[event].append(detector)
1667 def fireAuditors(self, action, nodeid, newvalues):
1668 """Fire all registered auditors.
1669 """
1670 for audit in self.auditors[action]:
1671 audit(self.db, self, nodeid, newvalues)
1673 def react(self, event, detector):
1674 """Register a detector
1675 """
1676 l = self.reactors[event]
1677 if detector not in l:
1678 self.reactors[event].append(detector)
1680 def fireReactors(self, action, nodeid, oldvalues):
1681 """Fire all registered reactors.
1682 """
1683 for react in self.reactors[action]:
1684 react(self.db, self, nodeid, oldvalues)
1686 class FileClass(Class):
1687 '''This class defines a large chunk of data. To support this, it has a
1688 mandatory String property "content" which is typically saved off
1689 externally to the hyperdb.
1691 The default MIME type of this data is defined by the
1692 "default_mime_type" class attribute, which may be overridden by each
1693 node if the class defines a "type" String property.
1694 '''
1695 default_mime_type = 'text/plain'
1697 def create(self, **propvalues):
1698 ''' snaffle the file propvalue and store in a file
1699 '''
1700 content = propvalues['content']
1701 del propvalues['content']
1702 newid = Class.create(self, **propvalues)
1703 self.db.storefile(self.classname, newid, None, content)
1704 return newid
1706 def get(self, nodeid, propname, default=_marker, cache=1):
1707 ''' trap the content propname and get it from the file
1708 '''
1710 poss_msg = 'Possibly a access right configuration problem.'
1711 if propname == 'content':
1712 try:
1713 return self.db.getfile(self.classname, nodeid, None)
1714 except IOError, (strerror):
1715 # BUG: by catching this we donot see an error in the log.
1716 return 'ERROR reading file: %s%s\n%s\n%s'%(
1717 self.classname, nodeid, poss_msg, strerror)
1718 if default is not _marker:
1719 return Class.get(self, nodeid, propname, default, cache=cache)
1720 else:
1721 return Class.get(self, nodeid, propname, cache=cache)
1723 def getprops(self, protected=1):
1724 ''' In addition to the actual properties on the node, these methods
1725 provide the "content" property. If the "protected" flag is true,
1726 we include protected properties - those which may not be
1727 modified.
1728 '''
1729 d = Class.getprops(self, protected=protected).copy()
1730 if protected:
1731 d['content'] = hyperdb.String()
1732 return d
1734 def index(self, nodeid):
1735 ''' Index the node in the search index.
1737 We want to index the content in addition to the normal String
1738 property indexing.
1739 '''
1740 # perform normal indexing
1741 Class.index(self, nodeid)
1743 # get the content to index
1744 content = self.get(nodeid, 'content')
1746 # figure the mime type
1747 if self.properties.has_key('type'):
1748 mime_type = self.get(nodeid, 'type')
1749 else:
1750 mime_type = self.default_mime_type
1752 # and index!
1753 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
1754 mime_type)
1756 # XXX deviation from spec - was called ItemClass
1757 class IssueClass(Class, roundupdb.IssueClass):
1758 # Overridden methods:
1759 def __init__(self, db, classname, **properties):
1760 """The newly-created class automatically includes the "messages",
1761 "files", "nosy", and "superseder" properties. If the 'properties'
1762 dictionary attempts to specify any of these properties or a
1763 "creation" or "activity" property, a ValueError is raised.
1764 """
1765 if not properties.has_key('title'):
1766 properties['title'] = hyperdb.String(indexme='yes')
1767 if not properties.has_key('messages'):
1768 properties['messages'] = hyperdb.Multilink("msg")
1769 if not properties.has_key('files'):
1770 properties['files'] = hyperdb.Multilink("file")
1771 if not properties.has_key('nosy'):
1772 properties['nosy'] = hyperdb.Multilink("user")
1773 if not properties.has_key('superseder'):
1774 properties['superseder'] = hyperdb.Multilink(classname)
1775 Class.__init__(self, db, classname, **properties)
1777 #
1778 #$Log: not supported by cvs2svn $
1779 #Revision 1.52 2002/07/19 03:36:34 richard
1780 #Implemented the destroy() method needed by the session database (and possibly
1781 #others). At the same time, I removed the leading underscores from the hyperdb
1782 #methods that Really Didn't Need Them.
1783 #The journal also raises IndexError now for all situations where there is a
1784 #request for the journal of a node that doesn't have one. It used to return
1785 #[] in _some_ situations, but not all. This _may_ break code, but the tests
1786 #pass...
1787 #
1788 #Revision 1.51 2002/07/18 23:07:08 richard
1789 #Unit tests and a few fixes.
1790 #
1791 #Revision 1.50 2002/07/18 11:50:58 richard
1792 #added tests for number type too
1793 #
1794 #Revision 1.49 2002/07/18 11:41:10 richard
1795 #added tests for boolean type, and fixes to anydbm backend
1796 #
1797 #Revision 1.48 2002/07/18 11:17:31 gmcm
1798 #Add Number and Boolean types to hyperdb.
1799 #Add conversion cases to web, mail & admin interfaces.
1800 #Add storage/serialization cases to back_anydbm & back_metakit.
1801 #
1802 #Revision 1.47 2002/07/14 23:18:20 richard
1803 #. fixed the journal bloat from multilink changes - we just log the add or
1804 # remove operations, not the whole list
1805 #
1806 #Revision 1.46 2002/07/14 06:06:34 richard
1807 #Did some old TODOs
1808 #
1809 #Revision 1.45 2002/07/14 04:03:14 richard
1810 #Implemented a switch to disable journalling for a Class. CGI session
1811 #database now uses it.
1812 #
1813 #Revision 1.44 2002/07/14 02:05:53 richard
1814 #. all storage-specific code (ie. backend) is now implemented by the backends
1815 #
1816 #Revision 1.43 2002/07/10 06:30:30 richard
1817 #...except of course it's nice to use valid Python syntax
1818 #
1819 #Revision 1.42 2002/07/10 06:21:38 richard
1820 #Be extra safe
1821 #
1822 #Revision 1.41 2002/07/10 00:21:45 richard
1823 #explicit database closing
1824 #
1825 #Revision 1.40 2002/07/09 04:19:09 richard
1826 #Added reindex command to roundup-admin.
1827 #Fixed reindex on first access.
1828 #Also fixed reindexing of entries that change.
1829 #
1830 #Revision 1.39 2002/07/09 03:02:52 richard
1831 #More indexer work:
1832 #- all String properties may now be indexed too. Currently there's a bit of
1833 # "issue" specific code in the actual searching which needs to be
1834 # addressed. In a nutshell:
1835 # + pass 'indexme="yes"' as a String() property initialisation arg, eg:
1836 # file = FileClass(db, "file", name=String(), type=String(),
1837 # comment=String(indexme="yes"))
1838 # + the comment will then be indexed and be searchable, with the results
1839 # related back to the issue that the file is linked to
1840 #- as a result of this work, the FileClass has a default MIME type that may
1841 # be overridden in a subclass, or by the use of a "type" property as is
1842 # done in the default templates.
1843 #- the regeneration of the indexes (if necessary) is done once the schema is
1844 # set up in the dbinit.
1845 #
1846 #Revision 1.38 2002/07/08 06:58:15 richard
1847 #cleaned up the indexer code:
1848 # - it splits more words out (much simpler, faster splitter)
1849 # - removed code we'll never use (roundup.roundup_indexer has the full
1850 # implementation, and replaces roundup.indexer)
1851 # - only index text/plain and rfc822/message (ideas for other text formats to
1852 # index are welcome)
1853 # - added simple unit test for indexer. Needs more tests for regression.
1854 #
1855 #Revision 1.37 2002/06/20 23:52:35 richard
1856 #More informative error message
1857 #
1858 #Revision 1.36 2002/06/19 03:07:19 richard
1859 #Moved the file storage commit into blobfiles where it belongs.
1860 #
1861 #Revision 1.35 2002/05/25 07:16:24 rochecompaan
1862 #Merged search_indexing-branch with HEAD
1863 #
1864 #Revision 1.34 2002/05/15 06:21:21 richard
1865 # . node caching now works, and gives a small boost in performance
1866 #
1867 #As a part of this, I cleaned up the DEBUG output and implemented TRACE
1868 #output (HYPERDBTRACE='file to trace to') with checkpoints at the start of
1869 #CGI requests. Run roundup with python -O to skip all the DEBUG/TRACE stuff
1870 #(using if __debug__ which is compiled out with -O)
1871 #
1872 #Revision 1.33 2002/04/24 10:38:26 rochecompaan
1873 #All database files are now created group readable and writable.
1874 #
1875 #Revision 1.32 2002/04/15 23:25:15 richard
1876 #. node ids are now generated from a lockable store - no more race conditions
1877 #
1878 #We're using the portalocker code by Jonathan Feinberg that was contributed
1879 #to the ASPN Python cookbook. This gives us locking across Unix and Windows.
1880 #
1881 #Revision 1.31 2002/04/03 05:54:31 richard
1882 #Fixed serialisation problem by moving the serialisation step out of the
1883 #hyperdb.Class (get, set) into the hyperdb.Database.
1884 #
1885 #Also fixed htmltemplate after the showid changes I made yesterday.
1886 #
1887 #Unit tests for all of the above written.
1888 #
1889 #Revision 1.30.2.1 2002/04/03 11:55:57 rochecompaan
1890 # . Added feature #526730 - search for messages capability
1891 #
1892 #Revision 1.30 2002/02/27 03:40:59 richard
1893 #Ran it through pychecker, made fixes
1894 #
1895 #Revision 1.29 2002/02/25 14:34:31 grubert
1896 # . use blobfiles in back_anydbm which is used in back_bsddb.
1897 # change test_db as dirlist does not work for subdirectories.
1898 # ATTENTION: blobfiles now creates subdirectories for files.
1899 #
1900 #Revision 1.28 2002/02/16 09:14:17 richard
1901 # . #514854 ] History: "User" is always ticket creator
1902 #
1903 #Revision 1.27 2002/01/22 07:21:13 richard
1904 #. fixed back_bsddb so it passed the journal tests
1905 #
1906 #... it didn't seem happy using the back_anydbm _open method, which is odd.
1907 #Yet another occurrance of whichdb not being able to recognise older bsddb
1908 #databases. Yadda yadda. Made the HYPERDBDEBUG stuff more sane in the
1909 #process.
1910 #
1911 #Revision 1.26 2002/01/22 05:18:38 rochecompaan
1912 #last_set_entry was referenced before assignment
1913 #
1914 #Revision 1.25 2002/01/22 05:06:08 rochecompaan
1915 #We need to keep the last 'set' entry in the journal to preserve
1916 #information on 'activity' for nodes.
1917 #
1918 #Revision 1.24 2002/01/21 16:33:20 rochecompaan
1919 #You can now use the roundup-admin tool to pack the database
1920 #
1921 #Revision 1.23 2002/01/18 04:32:04 richard
1922 #Rollback was breaking because a message hadn't actually been written to the file. Needs
1923 #more investigation.
1924 #
1925 #Revision 1.22 2002/01/14 02:20:15 richard
1926 # . changed all config accesses so they access either the instance or the
1927 # config attriubute on the db. This means that all config is obtained from
1928 # instance_config instead of the mish-mash of classes. This will make
1929 # switching to a ConfigParser setup easier too, I hope.
1930 #
1931 #At a minimum, this makes migration a _little_ easier (a lot easier in the
1932 #0.5.0 switch, I hope!)
1933 #
1934 #Revision 1.21 2002/01/02 02:31:38 richard
1935 #Sorry for the huge checkin message - I was only intending to implement #496356
1936 #but I found a number of places where things had been broken by transactions:
1937 # . modified ROUNDUPDBSENDMAILDEBUG to be SENDMAILDEBUG and hold a filename
1938 # for _all_ roundup-generated smtp messages to be sent to.
1939 # . the transaction cache had broken the roundupdb.Class set() reactors
1940 # . newly-created author users in the mailgw weren't being committed to the db
1941 #
1942 #Stuff that made it into CHANGES.txt (ie. the stuff I was actually working
1943 #on when I found that stuff :):
1944 # . #496356 ] Use threading in messages
1945 # . detectors were being registered multiple times
1946 # . added tests for mailgw
1947 # . much better attaching of erroneous messages in the mail gateway
1948 #
1949 #Revision 1.20 2001/12/18 15:30:34 rochecompaan
1950 #Fixed bugs:
1951 # . Fixed file creation and retrieval in same transaction in anydbm
1952 # backend
1953 # . Cgi interface now renders new issue after issue creation
1954 # . Could not set issue status to resolved through cgi interface
1955 # . Mail gateway was changing status back to 'chatting' if status was
1956 # omitted as an argument
1957 #
1958 #Revision 1.19 2001/12/17 03:52:48 richard
1959 #Implemented file store rollback. As a bonus, the hyperdb is now capable of
1960 #storing more than one file per node - if a property name is supplied,
1961 #the file is called designator.property.
1962 #I decided not to migrate the existing files stored over to the new naming
1963 #scheme - the FileClass just doesn't specify the property name.
1964 #
1965 #Revision 1.18 2001/12/16 10:53:38 richard
1966 #take a copy of the node dict so that the subsequent set
1967 #operation doesn't modify the oldvalues structure
1968 #
1969 #Revision 1.17 2001/12/14 23:42:57 richard
1970 #yuck, a gdbm instance tests false :(
1971 #I've left the debugging code in - it should be removed one day if we're ever
1972 #_really_ anal about performace :)
1973 #
1974 #Revision 1.16 2001/12/12 03:23:14 richard
1975 #Cor blimey this anydbm/whichdb stuff is yecchy. Turns out that whichdb
1976 #incorrectly identifies a dbm file as a dbhash file on my system. This has
1977 #been submitted to the python bug tracker as issue #491888:
1978 #https://sourceforge.net/tracker/index.php?func=detail&aid=491888&group_id=5470&atid=105470
1979 #
1980 #Revision 1.15 2001/12/12 02:30:51 richard
1981 #I fixed the problems with people whose anydbm was using the dbm module at the
1982 #backend. It turns out the dbm module modifies the file name to append ".db"
1983 #and my check to determine if we're opening an existing or new db just
1984 #tested os.path.exists() on the filename. Well, no longer! We now perform a
1985 #much better check _and_ cope with the anydbm implementation module changing
1986 #too!
1987 #I also fixed the backends __init__ so only ImportError is squashed.
1988 #
1989 #Revision 1.14 2001/12/10 22:20:01 richard
1990 #Enabled transaction support in the bsddb backend. It uses the anydbm code
1991 #where possible, only replacing methods where the db is opened (it uses the
1992 #btree opener specifically.)
1993 #Also cleaned up some change note generation.
1994 #Made the backends package work with pydoc too.
1995 #
1996 #Revision 1.13 2001/12/02 05:06:16 richard
1997 #. We now use weakrefs in the Classes to keep the database reference, so
1998 # the close() method on the database is no longer needed.
1999 # I bumped the minimum python requirement up to 2.1 accordingly.
2000 #. #487480 ] roundup-server
2001 #. #487476 ] INSTALL.txt
2002 #
2003 #I also cleaned up the change message / post-edit stuff in the cgi client.
2004 #There's now a clearly marked "TODO: append the change note" where I believe
2005 #the change note should be added there. The "changes" list will obviously
2006 #have to be modified to be a dict of the changes, or somesuch.
2007 #
2008 #More testing needed.
2009 #
2010 #Revision 1.12 2001/12/01 07:17:50 richard
2011 #. We now have basic transaction support! Information is only written to
2012 # the database when the commit() method is called. Only the anydbm
2013 # backend is modified in this way - neither of the bsddb backends have been.
2014 # The mail, admin and cgi interfaces all use commit (except the admin tool
2015 # doesn't have a commit command, so interactive users can't commit...)
2016 #. Fixed login/registration forwarding the user to the right page (or not,
2017 # on a failure)
2018 #
2019 #Revision 1.11 2001/11/21 02:34:18 richard
2020 #Added a target version field to the extended issue schema
2021 #
2022 #Revision 1.10 2001/10/09 23:58:10 richard
2023 #Moved the data stringification up into the hyperdb.Class class' get, set
2024 #and create methods. This means that the data is also stringified for the
2025 #journal call, and removes duplication of code from the backends. The
2026 #backend code now only sees strings.
2027 #
2028 #Revision 1.9 2001/10/09 07:25:59 richard
2029 #Added the Password property type. See "pydoc roundup.password" for
2030 #implementation details. Have updated some of the documentation too.
2031 #
2032 #Revision 1.8 2001/09/29 13:27:00 richard
2033 #CGI interfaces now spit up a top-level index of all the instances they can
2034 #serve.
2035 #
2036 #Revision 1.7 2001/08/12 06:32:36 richard
2037 #using isinstance(blah, Foo) now instead of isFooType
2038 #
2039 #Revision 1.6 2001/08/07 00:24:42 richard
2040 #stupid typo
2041 #
2042 #Revision 1.5 2001/08/07 00:15:51 richard
2043 #Added the copyright/license notice to (nearly) all files at request of
2044 #Bizar Software.
2045 #
2046 #Revision 1.4 2001/07/30 01:41:36 richard
2047 #Makes schema changes mucho easier.
2048 #
2049 #Revision 1.3 2001/07/25 01:23:07 richard
2050 #Added the Roundup spec to the new documentation directory.
2051 #
2052 #Revision 1.2 2001/07/23 08:20:44 richard
2053 #Moved over to using marshal in the bsddb and anydbm backends.
2054 #roundup-admin now has a "freshen" command that'll load/save all nodes (not
2055 # retired - mod hyperdb.Class.list() so it lists retired nodes)
2056 #
2057 #