1 # $Id: back_gadfly.py,v 1.4 2002-08-23 05:00:38 richard Exp $
2 __doc__ = '''
3 About Gadfly
4 ============
6 Gadfly is a collection of python modules that provides relational
7 database functionality entirely implemented in Python. It supports a
8 subset of the intergalactic standard RDBMS Structured Query Language
9 SQL.
12 Basic Structure
13 ===============
15 We map roundup classes to relational tables. Automatically detect schema
16 changes and modify the gadfly table schemas appropriately. Multilinks
17 (which represent a many-to-many relationship) are handled through
18 intermediate tables.
20 Journals are stored adjunct to the per-class tables.
22 Table names and columns have "_" prepended so the names can't
23 clash with restricted names (like "order"). Retirement is determined by the
24 __retired__ column being true.
26 All columns are defined as VARCHAR, since it really doesn't matter what
27 type they're defined as. We stuff all kinds of data in there ;) [as long as
28 it's marshallable, gadfly doesn't care]
31 Additional Instance Requirements
32 ================================
34 The instance configuration must specify where the database is. It does this
35 with GADFLY_DATABASE, which is used as the arguments to the gadfly.gadfly()
36 method:
38 Using an on-disk database directly (not a good idea):
39 GADFLY_DATABASE = (database name, directory)
41 Using a network database (much better idea):
42 GADFLY_DATABASE = (policy, password, address, port)
44 Because multiple accesses directly to a gadfly database aren't handled, but
45 multiple network accesses are, it's strongly advised that the latter setup be
46 used.
48 '''
50 # standard python modules
51 import sys, os, time, re, errno, weakref, copy
53 # roundup modules
54 from roundup import hyperdb, date, password, roundupdb, security
55 from roundup.hyperdb import String, Password, Date, Interval, Link, \
56 Multilink, DatabaseError, Boolean, Number
58 # the all-important gadfly :)
59 import gadfly
60 import gadfly.client
61 import gadfly.database
63 # support
64 from blobfiles import FileStorage
65 from roundup.indexer import Indexer
66 from sessions import Sessions
68 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
69 # flag to set on retired entries
70 RETIRED_FLAG = '__hyperdb_retired'
72 def __init__(self, config, journaltag=None):
73 ''' Open the database and load the schema from it.
74 '''
75 self.config, self.journaltag = config, journaltag
76 self.dir = config.DATABASE
77 self.classes = {}
78 self.indexer = Indexer(self.dir)
79 self.sessions = Sessions(self.config)
80 self.security = security.Security(self)
82 # additional transaction support for external files and the like
83 self.transactions = []
85 db = config.GADFLY_DATABASE
86 if len(db) == 2:
87 # ensure files are group readable and writable
88 os.umask(0002)
89 try:
90 self.conn = gadfly.gadfly(*db)
91 except IOError, error:
92 if error.errno != errno.ENOENT:
93 raise
94 self.database_schema = {}
95 self.conn = gadfly.gadfly()
96 self.conn.startup(*db)
97 cursor = self.conn.cursor()
98 cursor.execute('create table schema (schema varchar)')
99 cursor.execute('create table ids (name varchar, num integer)')
100 else:
101 cursor = self.conn.cursor()
102 cursor.execute('select schema from schema')
103 self.database_schema = cursor.fetchone()[0]
104 else:
105 self.conn = gadfly.client.gfclient(*db)
106 cursor = self.conn.cursor()
107 cursor.execute('select schema from schema')
108 self.database_schema = cursor.fetchone()[0]
110 def __repr__(self):
111 return '<radfly 0x%x>'%id(self)
113 def post_init(self):
114 ''' Called once the schema initialisation has finished.
116 We should now confirm that the schema defined by our "classes"
117 attribute actually matches the schema in the database.
118 '''
119 # now detect changes in the schema
120 for classname, spec in self.classes.items():
121 if self.database_schema.has_key(classname):
122 dbspec = self.database_schema[classname]
123 self.update_class(spec, dbspec)
124 self.database_schema[classname] = spec.schema()
125 else:
126 self.create_class(spec)
127 self.database_schema[classname] = spec.schema()
129 for classname in self.database_schema.keys():
130 if not self.classes.has_key(classname):
131 self.drop_class(classname)
133 # update the database version of the schema
134 cursor = self.conn.cursor()
135 cursor.execute('delete from schema')
136 cursor.execute('insert into schema values (?)', (self.database_schema,))
138 # reindex the db if necessary
139 if self.indexer.should_reindex():
140 self.reindex()
142 # commit
143 self.conn.commit()
145 def reindex(self):
146 for klass in self.classes.values():
147 for nodeid in klass.list():
148 klass.index(nodeid)
149 self.indexer.save_index()
151 def determine_columns(self, spec):
152 ''' Figure the column names and multilink properties from the spec
153 '''
154 cols = []
155 mls = []
156 # add the multilinks separately
157 for col, prop in spec.properties.items():
158 if isinstance(prop, Multilink):
159 mls.append(col)
160 else:
161 cols.append('_'+col)
162 cols.sort()
163 return cols, mls
165 def update_class(self, spec, dbspec):
166 ''' Determine the differences between the current spec and the
167 database version of the spec, and update where necessary
169 NOTE that this doesn't work for adding/deleting properties!
170 ... until gadfly grows an ALTER TABLE command, it's not going to!
171 '''
172 spec_schema = spec.schema()
173 if spec_schema == dbspec:
174 return
175 if __debug__:
176 print >>hyperdb.DEBUG, 'update_class FIRING'
178 # key property changed?
179 if dbspec[0] != spec_schema[0]:
180 if __debug__:
181 print >>hyperdb.DEBUG, 'update_class setting keyprop', `spec[0]`
182 # XXX turn on indexing for the key property
184 # dict 'em up
185 spec_propnames,spec_props = [],{}
186 for propname,prop in spec_schema[1]:
187 spec_propnames.append(propname)
188 spec_props[propname] = prop
189 dbspec_propnames,dbspec_props = [],{}
190 for propname,prop in dbspec[1]:
191 dbspec_propnames.append(propname)
192 dbspec_props[propname] = prop
194 # we're going to need one of these
195 cursor = self.conn.cursor()
197 # now compare
198 for propname in spec_propnames:
199 prop = spec_props[propname]
200 if __debug__:
201 print >>hyperdb.DEBUG, 'update_class ...', `prop`
202 if dbspec_props.has_key(propname) and prop==dbspec_props[propname]:
203 continue
204 if __debug__:
205 print >>hyperdb.DEBUG, 'update_class', `prop`
207 if not dbspec_props.has_key(propname):
208 # add the property
209 if isinstance(prop, Multilink):
210 sql = 'create table %s_%s (linkid varchar, nodeid '\
211 'varchar)'%(spec.classname, prop)
212 if __debug__:
213 print >>hyperdb.DEBUG, 'update_class', (self, sql)
214 cursor.execute(sql)
215 else:
216 # XXX gadfly doesn't have an ALTER TABLE command
217 raise NotImplementedError
218 sql = 'alter table _%s add column (_%s varchar)'%(
219 spec.classname, propname)
220 if __debug__:
221 print >>hyperdb.DEBUG, 'update_class', (self, sql)
222 cursor.execute(sql)
223 else:
224 # modify the property
225 if __debug__:
226 print >>hyperdb.DEBUG, 'update_class NOOP'
227 pass # NOOP in gadfly
229 # and the other way - only worry about deletions here
230 for propname in dbspec_propnames:
231 prop = dbspec_props[propname]
232 if spec_props.has_key(propname):
233 continue
234 if __debug__:
235 print >>hyperdb.DEBUG, 'update_class', `prop`
237 # delete the property
238 if isinstance(prop, Multilink):
239 sql = 'drop table %s_%s'%(spec.classname, prop)
240 if __debug__:
241 print >>hyperdb.DEBUG, 'update_class', (self, sql)
242 cursor.execute(sql)
243 else:
244 # XXX gadfly doesn't have an ALTER TABLE command
245 raise NotImplementedError
246 sql = 'alter table _%s delete column _%s'%(spec.classname,
247 propname)
248 if __debug__:
249 print >>hyperdb.DEBUG, 'update_class', (self, sql)
250 cursor.execute(sql)
252 def create_class(self, spec):
253 ''' Create a database table according to the given spec.
254 '''
255 cols, mls = self.determine_columns(spec)
257 # add on our special columns
258 cols.append('id')
259 cols.append('__retired__')
261 cursor = self.conn.cursor()
263 # create the base table
264 cols = ','.join(['%s varchar'%x for x in cols])
265 sql = 'create table _%s (%s)'%(spec.classname, cols)
266 if __debug__:
267 print >>hyperdb.DEBUG, 'create_class', (self, sql)
268 cursor.execute(sql)
270 # journal table
271 cols = ','.join(['%s varchar'%x
272 for x in 'nodeid date tag action params'.split()])
273 sql = 'create table %s__journal (%s)'%(spec.classname, cols)
274 if __debug__:
275 print >>hyperdb.DEBUG, 'create_class', (self, sql)
276 cursor.execute(sql)
278 # now create the multilink tables
279 for ml in mls:
280 sql = 'create table %s_%s (linkid varchar, nodeid varchar)'%(
281 spec.classname, ml)
282 if __debug__:
283 print >>hyperdb.DEBUG, 'create_class', (self, sql)
284 cursor.execute(sql)
286 # ID counter
287 sql = 'insert into ids (name, num) values (?,?)'
288 vals = (spec.classname, 1)
289 if __debug__:
290 print >>hyperdb.DEBUG, 'create_class', (self, sql, vals)
291 cursor.execute(sql, vals)
293 def drop_class(self, spec):
294 ''' Drop the given table from the database.
296 Drop the journal and multilink tables too.
297 '''
298 # figure the multilinks
299 mls = []
300 for col, prop in spec.properties.items():
301 if isinstance(prop, Multilink):
302 mls.append(col)
303 cursor = self.conn.cursor()
305 sql = 'drop table _%s'%spec.classname
306 if __debug__:
307 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
308 cursor.execute(sql)
310 sql = 'drop table %s__journal'%spec.classname
311 if __debug__:
312 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
313 cursor.execute(sql)
315 for ml in mls:
316 sql = 'drop table %s_%s'%(spec.classname, ml)
317 if __debug__:
318 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
319 cursor.execute(sql)
321 #
322 # Classes
323 #
324 def __getattr__(self, classname):
325 ''' A convenient way of calling self.getclass(classname).
326 '''
327 if self.classes.has_key(classname):
328 if __debug__:
329 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
330 return self.classes[classname]
331 raise AttributeError, classname
333 def addclass(self, cl):
334 ''' Add a Class to the hyperdatabase.
335 '''
336 if __debug__:
337 print >>hyperdb.DEBUG, 'addclass', (self, cl)
338 cn = cl.classname
339 if self.classes.has_key(cn):
340 raise ValueError, cn
341 self.classes[cn] = cl
343 def getclasses(self):
344 ''' Return a list of the names of all existing classes.
345 '''
346 if __debug__:
347 print >>hyperdb.DEBUG, 'getclasses', (self,)
348 l = self.classes.keys()
349 l.sort()
350 return l
352 def getclass(self, classname):
353 '''Get the Class object representing a particular class.
355 If 'classname' is not a valid class name, a KeyError is raised.
356 '''
357 if __debug__:
358 print >>hyperdb.DEBUG, 'getclass', (self, classname)
359 return self.classes[classname]
361 def clear(self):
362 ''' Delete all database contents.
364 Note: I don't commit here, which is different behaviour to the
365 "nuke from orbit" behaviour in the *dbms.
366 '''
367 if __debug__:
368 print >>hyperdb.DEBUG, 'clear', (self,)
369 cursor = self.conn.cursor()
370 for cn in self.classes.keys():
371 sql = 'delete from _%s'%cn
372 if __debug__:
373 print >>hyperdb.DEBUG, 'clear', (self, sql)
374 cursor.execute(sql)
376 #
377 # Node IDs
378 #
379 def newid(self, classname):
380 ''' Generate a new id for the given class
381 '''
382 # get the next ID
383 cursor = self.conn.cursor()
384 sql = 'select num from ids where name=?'
385 if __debug__:
386 print >>hyperdb.DEBUG, 'newid', (self, sql, classname)
387 cursor.execute(sql, (classname, ))
388 newid = cursor.fetchone()[0]
390 # update the counter
391 sql = 'update ids set num=? where name=?'
392 vals = (newid+1, classname)
393 if __debug__:
394 print >>hyperdb.DEBUG, 'newid', (self, sql, vals)
395 cursor.execute(sql, vals)
397 # return as string
398 return str(newid)
400 def setid(self, classname, setid):
401 ''' Set the id counter: used during import of database
402 '''
403 cursor = self.conn.cursor()
404 sql = 'update ids set num=? where name=?'
405 vals = (setid, spec.classname)
406 if __debug__:
407 print >>hyperdb.DEBUG, 'setid', (self, sql, vals)
408 cursor.execute(sql, vals)
410 #
411 # Nodes
412 #
414 def addnode(self, classname, nodeid, node):
415 ''' Add the specified node to its class's db.
416 '''
417 if __debug__:
418 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
419 # gadfly requires values for all non-multilink columns
420 cl = self.classes[classname]
421 cols, mls = self.determine_columns(cl)
423 # default the non-multilink columns
424 for col, prop in cl.properties.items():
425 if not isinstance(col, Multilink):
426 if not node.has_key(col):
427 node[col] = None
429 node = self.serialise(classname, node)
431 # make sure the ordering is correct for column name -> column value
432 vals = tuple([node[col[1:]] for col in cols]) + (nodeid, 0)
433 s = ','.join(['?' for x in cols]) + ',?,?'
434 cols = ','.join(cols) + ',id,__retired__'
436 # perform the inserts
437 cursor = self.conn.cursor()
438 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
439 if __debug__:
440 print >>hyperdb.DEBUG, 'addnode', (self, sql, vals)
441 cursor.execute(sql, vals)
443 # insert the multilink rows
444 for col in mls:
445 t = '%s_%s'%(classname, col)
446 for entry in node[col]:
447 sql = 'insert into %s (linkid, nodeid) values (?,?)'%t
448 vals = (entry, nodeid)
449 if __debug__:
450 print >>hyperdb.DEBUG, 'addnode', (self, sql, vals)
451 cursor.execute(sql, vals)
453 # make sure we do the commit-time extra stuff for this node
454 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
456 def setnode(self, classname, nodeid, node):
457 ''' Change the specified node.
458 '''
459 if __debug__:
460 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
461 node = self.serialise(classname, node)
463 cl = self.classes[classname]
464 cols = []
465 mls = []
466 # add the multilinks separately
467 for col in node.keys():
468 prop = cl.properties[col]
469 if isinstance(prop, Multilink):
470 mls.append(col)
471 else:
472 cols.append('_'+col)
473 cols.sort()
475 # make sure the ordering is correct for column name -> column value
476 vals = tuple([node[col[1:]] for col in cols])
477 s = ','.join(['%s=?'%x for x in cols])
478 cols = ','.join(cols)
480 # perform the update
481 cursor = self.conn.cursor()
482 sql = 'update _%s set %s'%(classname, s)
483 if __debug__:
484 print >>hyperdb.DEBUG, 'setnode', (self, sql, vals)
485 cursor.execute(sql, vals)
487 # now the fun bit, updating the multilinks ;)
488 # XXX TODO XXX
490 # make sure we do the commit-time extra stuff for this node
491 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
493 def getnode(self, classname, nodeid):
494 ''' Get a node from the database.
495 '''
496 if __debug__:
497 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid)
498 # figure the columns we're fetching
499 cl = self.classes[classname]
500 cols, mls = self.determine_columns(cl)
501 scols = ','.join(cols)
503 # perform the basic property fetch
504 cursor = self.conn.cursor()
505 sql = 'select %s from _%s where id=?'%(scols, classname)
506 if __debug__:
507 print >>hyperdb.DEBUG, 'getnode', (self, sql, nodeid)
508 cursor.execute(sql, (nodeid,))
509 try:
510 values = cursor.fetchone()
511 except gadfly.database.error, message:
512 if message == 'no more results':
513 raise IndexError, 'no such %s node %s'%(classname, nodeid)
514 raise
516 # make up the node
517 node = {}
518 for col in range(len(cols)):
519 node[cols[col][1:]] = values[col]
521 # now the multilinks
522 for col in mls:
523 # get the link ids
524 sql = 'select linkid from %s_%s where nodeid=?'%(classname, col)
525 if __debug__:
526 print >>hyperdb.DEBUG, 'getnode', (self, sql, nodeid)
527 cursor.execute(sql, (nodeid,))
528 # extract the first column from the result
529 node[col] = [x[0] for x in cursor.fetchall()]
531 return self.unserialise(classname, node)
533 def destroynode(self, classname, nodeid):
534 '''Remove a node from the database. Called exclusively by the
535 destroy() method on Class.
536 '''
537 if __debug__:
538 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
540 # make sure the node exists
541 if not self.hasnode(classname, nodeid):
542 raise IndexError, '%s has no node %s'%(classname, nodeid)
544 # see if there's any obvious commit actions that we should get rid of
545 for entry in self.transactions[:]:
546 if entry[1][:2] == (classname, nodeid):
547 self.transactions.remove(entry)
549 # now do the SQL
550 cursor = self.conn.cursor()
551 sql = 'delete from _%s where id=?'%(classname)
552 if __debug__:
553 print >>hyperdb.DEBUG, 'destroynode', (self, sql, nodeid)
554 cursor.execute(sql, (nodeid,))
556 def serialise(self, classname, node):
557 '''Copy the node contents, converting non-marshallable data into
558 marshallable data.
559 '''
560 if __debug__:
561 print >>hyperdb.DEBUG, 'serialise', classname, node
562 properties = self.getclass(classname).getprops()
563 d = {}
564 for k, v in node.items():
565 # if the property doesn't exist, or is the "retired" flag then
566 # it won't be in the properties dict
567 if not properties.has_key(k):
568 d[k] = v
569 continue
571 # get the property spec
572 prop = properties[k]
574 if isinstance(prop, Password):
575 d[k] = str(v)
576 elif isinstance(prop, Date) and v is not None:
577 d[k] = v.serialise()
578 elif isinstance(prop, Interval) and v is not None:
579 d[k] = v.serialise()
580 else:
581 d[k] = v
582 return d
584 def unserialise(self, classname, node):
585 '''Decode the marshalled node data
586 '''
587 if __debug__:
588 print >>hyperdb.DEBUG, 'unserialise', classname, node
589 properties = self.getclass(classname).getprops()
590 d = {}
591 for k, v in node.items():
592 # if the property doesn't exist, or is the "retired" flag then
593 # it won't be in the properties dict
594 if not properties.has_key(k):
595 d[k] = v
596 continue
598 # get the property spec
599 prop = properties[k]
601 if isinstance(prop, Date) and v is not None:
602 d[k] = date.Date(v)
603 elif isinstance(prop, Interval) and v is not None:
604 d[k] = date.Interval(v)
605 elif isinstance(prop, Password):
606 p = password.Password()
607 p.unpack(v)
608 d[k] = p
609 else:
610 d[k] = v
611 return d
613 def hasnode(self, classname, nodeid):
614 ''' Determine if the database has a given node.
615 '''
616 cursor = self.conn.cursor()
617 sql = 'select count(*) from _%s where id=?'%classname
618 if __debug__:
619 print >>hyperdb.DEBUG, 'hasnode', (self, sql, nodeid)
620 cursor.execute(sql, (nodeid,))
621 return cursor.fetchone()[0]
623 def countnodes(self, classname):
624 ''' Count the number of nodes that exist for a particular Class.
625 '''
626 cursor = self.conn.cursor()
627 sql = 'select count(*) from _%s'%classname
628 if __debug__:
629 print >>hyperdb.DEBUG, 'countnodes', (self, sql)
630 cursor.execute(sql)
631 return cursor.fetchone()[0]
633 def getnodeids(self, classname, retired=0):
634 ''' Retrieve all the ids of the nodes for a particular Class.
636 Set retired=None to get all nodes. Otherwise it'll get all the
637 retired or non-retired nodes, depending on the flag.
638 '''
639 cursor = self.conn.cursor()
640 # flip the sense of the flag if we don't want all of them
641 if retired is not None:
642 retired = not retired
643 sql = 'select id from _%s where __retired__ <> ?'%classname
644 if __debug__:
645 print >>hyperdb.DEBUG, 'getnodeids', (self, sql, retired)
646 cursor.execute(sql, (retired,))
647 return [x[0] for x in cursor.fetchall()]
649 def addjournal(self, classname, nodeid, action, params):
650 ''' Journal the Action
651 'action' may be:
653 'create' or 'set' -- 'params' is a dictionary of property values
654 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
655 'retire' -- 'params' is None
656 '''
657 if isinstance(params, type({})):
658 if params.has_key('creator'):
659 journaltag = self.user.get(params['creator'], 'username')
660 del params['creator']
661 else:
662 journaltag = self.journaltag
663 if params.has_key('created'):
664 journaldate = params['created'].serialise()
665 del params['created']
666 else:
667 journaldate = date.Date().serialise()
668 if params.has_key('activity'):
669 del params['activity']
671 # serialise the parameters now
672 if action in ('set', 'create'):
673 params = self.serialise(classname, params)
674 else:
675 journaltag = self.journaltag
676 journaldate = date.Date().serialise()
678 # create the journal entry
679 cols = ','.join('nodeid date tag action params'.split())
680 entry = (nodeid, journaldate, journaltag, action, params)
682 if __debug__:
683 print >>hyperdb.DEBUG, 'doSaveJournal', entry
685 # do the insert
686 cursor = self.conn.cursor()
687 sql = 'insert into %s__journal (%s) values (?,?,?,?,?)'%(classname,
688 cols)
689 if __debug__:
690 print >>hyperdb.DEBUG, 'addjournal', (self, sql, entry)
691 cursor.execute(sql, entry)
693 def getjournal(self, classname, nodeid):
694 ''' get the journal for id
695 '''
696 # make sure the node exists
697 if not self.hasnode(classname, nodeid):
698 raise IndexError, '%s has no node %s'%(classname, nodeid)
700 # now get the journal entries
701 cols = ','.join('nodeid date tag action params'.split())
702 cursor = self.conn.cursor()
703 sql = 'select %s from %s__journal where nodeid=?'%(cols, classname)
704 if __debug__:
705 print >>hyperdb.DEBUG, 'getjournal', (self, sql, nodeid)
706 cursor.execute(sql, (nodeid,))
707 res = []
708 for nodeid, date_stamp, user, action, params in cursor.fetchall():
709 res.append((nodeid, date.Date(date_stamp), user, action, params))
710 return res
712 def pack(self, pack_before):
713 ''' Delete all journal entries except "create" before 'pack_before'.
714 '''
715 # get a 'yyyymmddhhmmss' version of the date
716 date_stamp = pack_before.serialise()
718 # do the delete
719 cursor = self.conn.cursor()
720 for classname in self.classes.keys():
721 sql = "delete from %s__journal where date<? and "\
722 "action<>'create'"%classname
723 if __debug__:
724 print >>hyperdb.DEBUG, 'pack', (self, sql, date_stamp)
725 cursor.execute(sql, (date_stamp,))
727 def commit(self):
728 ''' Commit the current transactions.
730 Save all data changed since the database was opened or since the
731 last commit() or rollback().
732 '''
733 if __debug__:
734 print >>hyperdb.DEBUG, 'commit', (self,)
736 # commit gadfly
737 self.conn.commit()
739 # now, do all the other transaction stuff
740 reindex = {}
741 for method, args in self.transactions:
742 reindex[method(*args)] = 1
744 # reindex the nodes that request it
745 for classname, nodeid in filter(None, reindex.keys()):
746 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
747 self.getclass(classname).index(nodeid)
749 # save the indexer state
750 self.indexer.save_index()
752 # clear out the transactions
753 self.transactions = []
755 def rollback(self):
756 ''' Reverse all actions from the current transaction.
758 Undo all the changes made since the database was opened or the last
759 commit() or rollback() was performed.
760 '''
761 if __debug__:
762 print >>hyperdb.DEBUG, 'rollback', (self,)
764 # roll back gadfly
765 self.conn.rollback()
767 # roll back "other" transaction stuff
768 for method, args in self.transactions:
769 # delete temporary files
770 if method == self.doStoreFile:
771 self.rollbackStoreFile(*args)
772 self.transactions = []
774 def doSaveNode(self, classname, nodeid, node):
775 ''' dummy that just generates a reindex event
776 '''
777 # return the classname, nodeid so we reindex this content
778 return (classname, nodeid)
780 #
781 # The base Class class
782 #
783 class Class(hyperdb.Class):
784 ''' The handle to a particular class of nodes in a hyperdatabase.
786 All methods except __repr__ and getnode must be implemented by a
787 concrete backend Class.
788 '''
790 def __init__(self, db, classname, **properties):
791 '''Create a new class with a given name and property specification.
793 'classname' must not collide with the name of an existing class,
794 or a ValueError is raised. The keyword arguments in 'properties'
795 must map names to property objects, or a TypeError is raised.
796 '''
797 if (properties.has_key('creation') or properties.has_key('activity')
798 or properties.has_key('creator')):
799 raise ValueError, '"creation", "activity" and "creator" are '\
800 'reserved'
802 self.classname = classname
803 self.properties = properties
804 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
805 self.key = ''
807 # should we journal changes (default yes)
808 self.do_journal = 1
810 # do the db-related init stuff
811 db.addclass(self)
813 self.auditors = {'create': [], 'set': [], 'retire': []}
814 self.reactors = {'create': [], 'set': [], 'retire': []}
816 def schema(self):
817 ''' A dumpable version of the schema that we can store in the
818 database
819 '''
820 return (self.key, [(x, repr(y)) for x,y in self.properties.items()])
822 def enableJournalling(self):
823 '''Turn journalling on for this class
824 '''
825 self.do_journal = 1
827 def disableJournalling(self):
828 '''Turn journalling off for this class
829 '''
830 self.do_journal = 0
832 # Editing nodes:
833 def create(self, **propvalues):
834 ''' Create a new node of this class and return its id.
836 The keyword arguments in 'propvalues' map property names to values.
838 The values of arguments must be acceptable for the types of their
839 corresponding properties or a TypeError is raised.
841 If this class has a key property, it must be present and its value
842 must not collide with other key strings or a ValueError is raised.
844 Any other properties on this class that are missing from the
845 'propvalues' dictionary are set to None.
847 If an id in a link or multilink property does not refer to a valid
848 node, an IndexError is raised.
849 '''
850 if propvalues.has_key('id'):
851 raise KeyError, '"id" is reserved'
853 if self.db.journaltag is None:
854 raise DatabaseError, 'Database open read-only'
856 if propvalues.has_key('creation') or propvalues.has_key('activity'):
857 raise KeyError, '"creation" and "activity" are reserved'
859 self.fireAuditors('create', None, propvalues)
861 # new node's id
862 newid = self.db.newid(self.classname)
864 # validate propvalues
865 num_re = re.compile('^\d+$')
866 for key, value in propvalues.items():
867 if key == self.key:
868 try:
869 self.lookup(value)
870 except KeyError:
871 pass
872 else:
873 raise ValueError, 'node with key "%s" exists'%value
875 # try to handle this property
876 try:
877 prop = self.properties[key]
878 except KeyError:
879 raise KeyError, '"%s" has no property "%s"'%(self.classname,
880 key)
882 if value is not None and isinstance(prop, Link):
883 if type(value) != type(''):
884 raise ValueError, 'link value must be String'
885 link_class = self.properties[key].classname
886 # if it isn't a number, it's a key
887 if not num_re.match(value):
888 try:
889 value = self.db.classes[link_class].lookup(value)
890 except (TypeError, KeyError):
891 raise IndexError, 'new property "%s": %s not a %s'%(
892 key, value, link_class)
893 elif not self.db.getclass(link_class).hasnode(value):
894 raise IndexError, '%s has no node %s'%(link_class, value)
896 # save off the value
897 propvalues[key] = value
899 # register the link with the newly linked node
900 if self.do_journal and self.properties[key].do_journal:
901 self.db.addjournal(link_class, value, 'link',
902 (self.classname, newid, key))
904 elif isinstance(prop, Multilink):
905 if type(value) != type([]):
906 raise TypeError, 'new property "%s" not a list of ids'%key
908 # clean up and validate the list of links
909 link_class = self.properties[key].classname
910 l = []
911 for entry in value:
912 if type(entry) != type(''):
913 raise ValueError, '"%s" link value (%s) must be '\
914 'String'%(key, value)
915 # if it isn't a number, it's a key
916 if not num_re.match(entry):
917 try:
918 entry = self.db.classes[link_class].lookup(entry)
919 except (TypeError, KeyError):
920 raise IndexError, 'new property "%s": %s not a %s'%(
921 key, entry, self.properties[key].classname)
922 l.append(entry)
923 value = l
924 propvalues[key] = value
926 # handle additions
927 for nodeid in value:
928 if not self.db.getclass(link_class).hasnode(nodeid):
929 raise IndexError, '%s has no node %s'%(link_class,
930 nodeid)
931 # register the link with the newly linked node
932 if self.do_journal and self.properties[key].do_journal:
933 self.db.addjournal(link_class, nodeid, 'link',
934 (self.classname, newid, key))
936 elif isinstance(prop, String):
937 if type(value) != type(''):
938 raise TypeError, 'new property "%s" not a string'%key
940 elif isinstance(prop, Password):
941 if not isinstance(value, password.Password):
942 raise TypeError, 'new property "%s" not a Password'%key
944 elif isinstance(prop, Date):
945 if value is not None and not isinstance(value, date.Date):
946 raise TypeError, 'new property "%s" not a Date'%key
948 elif isinstance(prop, Interval):
949 if value is not None and not isinstance(value, date.Interval):
950 raise TypeError, 'new property "%s" not an Interval'%key
952 elif value is not None and isinstance(prop, Number):
953 try:
954 float(value)
955 except ValueError:
956 raise TypeError, 'new property "%s" not numeric'%key
958 elif value is not None and isinstance(prop, Boolean):
959 try:
960 int(value)
961 except ValueError:
962 raise TypeError, 'new property "%s" not boolean'%key
964 # make sure there's data where there needs to be
965 for key, prop in self.properties.items():
966 if propvalues.has_key(key):
967 continue
968 if key == self.key:
969 raise ValueError, 'key property "%s" is required'%key
970 if isinstance(prop, Multilink):
971 propvalues[key] = []
972 else:
973 propvalues[key] = None
975 # done
976 self.db.addnode(self.classname, newid, propvalues)
977 if self.do_journal:
978 self.db.addjournal(self.classname, newid, 'create', propvalues)
980 self.fireReactors('create', newid, None)
982 return newid
984 _marker = []
985 def get(self, nodeid, propname, default=_marker, cache=1):
986 '''Get the value of a property on an existing node of this class.
988 'nodeid' must be the id of an existing node of this class or an
989 IndexError is raised. 'propname' must be the name of a property
990 of this class or a KeyError is raised.
992 'cache' indicates whether the transaction cache should be queried
993 for the node. If the node has been modified and you need to
994 determine what its values prior to modification are, you need to
995 set cache=0.
996 '''
997 if propname == 'id':
998 return nodeid
1000 if propname == 'creation':
1001 if not self.do_journal:
1002 raise ValueError, 'Journalling is disabled for this class'
1003 journal = self.db.getjournal(self.classname, nodeid)
1004 if journal:
1005 return self.db.getjournal(self.classname, nodeid)[0][1]
1006 else:
1007 # on the strange chance that there's no journal
1008 return date.Date()
1009 if propname == 'activity':
1010 if not self.do_journal:
1011 raise ValueError, 'Journalling is disabled for this class'
1012 journal = self.db.getjournal(self.classname, nodeid)
1013 if journal:
1014 return self.db.getjournal(self.classname, nodeid)[-1][1]
1015 else:
1016 # on the strange chance that there's no journal
1017 return date.Date()
1018 if propname == 'creator':
1019 if not self.do_journal:
1020 raise ValueError, 'Journalling is disabled for this class'
1021 journal = self.db.getjournal(self.classname, nodeid)
1022 if journal:
1023 name = self.db.getjournal(self.classname, nodeid)[0][2]
1024 else:
1025 return None
1026 return self.db.user.lookup(name)
1028 # get the property (raises KeyErorr if invalid)
1029 prop = self.properties[propname]
1031 # get the node's dict
1032 d = self.db.getnode(self.classname, nodeid) #, cache=cache)
1034 if not d.has_key(propname):
1035 if default is self._marker:
1036 if isinstance(prop, Multilink):
1037 return []
1038 else:
1039 return None
1040 else:
1041 return default
1043 return d[propname]
1045 def getnode(self, nodeid, cache=1):
1046 ''' Return a convenience wrapper for the node.
1048 'nodeid' must be the id of an existing node of this class or an
1049 IndexError is raised.
1051 'cache' indicates whether the transaction cache should be queried
1052 for the node. If the node has been modified and you need to
1053 determine what its values prior to modification are, you need to
1054 set cache=0.
1055 '''
1056 return Node(self, nodeid, cache=cache)
1058 def set(self, nodeid, **propvalues):
1059 '''Modify a property on an existing node of this class.
1061 'nodeid' must be the id of an existing node of this class or an
1062 IndexError is raised.
1064 Each key in 'propvalues' must be the name of a property of this
1065 class or a KeyError is raised.
1067 All values in 'propvalues' must be acceptable types for their
1068 corresponding properties or a TypeError is raised.
1070 If the value of the key property is set, it must not collide with
1071 other key strings or a ValueError is raised.
1073 If the value of a Link or Multilink property contains an invalid
1074 node id, a ValueError is raised.
1075 '''
1076 if not propvalues:
1077 return propvalues
1079 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1080 raise KeyError, '"creation" and "activity" are reserved'
1082 if propvalues.has_key('id'):
1083 raise KeyError, '"id" is reserved'
1085 if self.db.journaltag is None:
1086 raise DatabaseError, 'Database open read-only'
1088 self.fireAuditors('set', nodeid, propvalues)
1089 # Take a copy of the node dict so that the subsequent set
1090 # operation doesn't modify the oldvalues structure.
1091 # XXX used to try the cache here first
1092 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1094 node = self.db.getnode(self.classname, nodeid)
1095 if self.is_retired(nodeid):
1096 raise IndexError
1097 num_re = re.compile('^\d+$')
1099 # if the journal value is to be different, store it in here
1100 journalvalues = {}
1102 for propname, value in propvalues.items():
1103 # check to make sure we're not duplicating an existing key
1104 if propname == self.key and node[propname] != value:
1105 try:
1106 self.lookup(value)
1107 except KeyError:
1108 pass
1109 else:
1110 raise ValueError, 'node with key "%s" exists'%value
1112 # this will raise the KeyError if the property isn't valid
1113 # ... we don't use getprops() here because we only care about
1114 # the writeable properties.
1115 prop = self.properties[propname]
1117 # if the value's the same as the existing value, no sense in
1118 # doing anything
1119 if node.has_key(propname) and value == node[propname]:
1120 del propvalues[propname]
1121 continue
1123 # do stuff based on the prop type
1124 if isinstance(prop, Link):
1125 link_class = prop.classname
1126 # if it isn't a number, it's a key
1127 if value is not None and not isinstance(value, type('')):
1128 raise ValueError, 'property "%s" link value be a string'%(
1129 propname)
1130 if isinstance(value, type('')) and not num_re.match(value):
1131 try:
1132 value = self.db.classes[link_class].lookup(value)
1133 except (TypeError, KeyError):
1134 raise IndexError, 'new property "%s": %s not a %s'%(
1135 propname, value, prop.classname)
1137 if (value is not None and
1138 not self.db.getclass(link_class).hasnode(value)):
1139 raise IndexError, '%s has no node %s'%(link_class, value)
1141 if self.do_journal and prop.do_journal:
1142 # register the unlink with the old linked node
1143 if node[propname] is not None:
1144 self.db.addjournal(link_class, node[propname], 'unlink',
1145 (self.classname, nodeid, propname))
1147 # register the link with the newly linked node
1148 if value is not None:
1149 self.db.addjournal(link_class, value, 'link',
1150 (self.classname, nodeid, propname))
1152 elif isinstance(prop, Multilink):
1153 if type(value) != type([]):
1154 raise TypeError, 'new property "%s" not a list of'\
1155 ' ids'%propname
1156 link_class = self.properties[propname].classname
1157 l = []
1158 for entry in value:
1159 # if it isn't a number, it's a key
1160 if type(entry) != type(''):
1161 raise ValueError, 'new property "%s" link value ' \
1162 'must be a string'%propname
1163 if not num_re.match(entry):
1164 try:
1165 entry = self.db.classes[link_class].lookup(entry)
1166 except (TypeError, KeyError):
1167 raise IndexError, 'new property "%s": %s not a %s'%(
1168 propname, entry,
1169 self.properties[propname].classname)
1170 l.append(entry)
1171 value = l
1172 propvalues[propname] = value
1174 # figure the journal entry for this property
1175 add = []
1176 remove = []
1178 # handle removals
1179 if node.has_key(propname):
1180 l = node[propname]
1181 else:
1182 l = []
1183 for id in l[:]:
1184 if id in value:
1185 continue
1186 # register the unlink with the old linked node
1187 if self.do_journal and self.properties[propname].do_journal:
1188 self.db.addjournal(link_class, id, 'unlink',
1189 (self.classname, nodeid, propname))
1190 l.remove(id)
1191 remove.append(id)
1193 # handle additions
1194 for id in value:
1195 if not self.db.getclass(link_class).hasnode(id):
1196 raise IndexError, '%s has no node %s'%(link_class, id)
1197 if id in l:
1198 continue
1199 # register the link with the newly linked node
1200 if self.do_journal and self.properties[propname].do_journal:
1201 self.db.addjournal(link_class, id, 'link',
1202 (self.classname, nodeid, propname))
1203 l.append(id)
1204 add.append(id)
1206 # figure the journal entry
1207 l = []
1208 if add:
1209 l.append(('+', add))
1210 if remove:
1211 l.append(('-', remove))
1212 if l:
1213 journalvalues[propname] = tuple(l)
1215 elif isinstance(prop, String):
1216 if value is not None and type(value) != type(''):
1217 raise TypeError, 'new property "%s" not a string'%propname
1219 elif isinstance(prop, Password):
1220 if not isinstance(value, password.Password):
1221 raise TypeError, 'new property "%s" not a Password'%propname
1222 propvalues[propname] = value
1224 elif value is not None and isinstance(prop, Date):
1225 if not isinstance(value, date.Date):
1226 raise TypeError, 'new property "%s" not a Date'% propname
1227 propvalues[propname] = value
1229 elif value is not None and isinstance(prop, Interval):
1230 if not isinstance(value, date.Interval):
1231 raise TypeError, 'new property "%s" not an '\
1232 'Interval'%propname
1233 propvalues[propname] = value
1235 elif value is not None and isinstance(prop, Number):
1236 try:
1237 float(value)
1238 except ValueError:
1239 raise TypeError, 'new property "%s" not numeric'%propname
1241 elif value is not None and isinstance(prop, Boolean):
1242 try:
1243 int(value)
1244 except ValueError:
1245 raise TypeError, 'new property "%s" not boolean'%propname
1247 node[propname] = value
1249 # nothing to do?
1250 if not propvalues:
1251 return propvalues
1253 # do the set, and journal it
1254 self.db.setnode(self.classname, nodeid, node)
1256 if self.do_journal:
1257 propvalues.update(journalvalues)
1258 self.db.addjournal(self.classname, nodeid, 'set', propvalues)
1260 self.fireReactors('set', nodeid, oldvalues)
1262 return propvalues
1264 def retire(self, nodeid):
1265 '''Retire a node.
1267 The properties on the node remain available from the get() method,
1268 and the node's id is never reused.
1270 Retired nodes are not returned by the find(), list(), or lookup()
1271 methods, and other nodes may reuse the values of their key properties.
1272 '''
1273 if self.db.journaltag is None:
1274 raise DatabaseError, 'Database open read-only'
1276 cursor = self.db.conn.cursor()
1277 sql = 'update _%s set __retired__=1 where id=?'%self.classname
1278 if __debug__:
1279 print >>hyperdb.DEBUG, 'retire', (self, sql, nodeid)
1280 cursor.execute(sql, (nodeid,))
1282 def is_retired(self, nodeid):
1283 '''Return true if the node is rerired
1284 '''
1285 cursor = self.db.conn.cursor()
1286 sql = 'select __retired__ from _%s where id=?'%self.classname
1287 if __debug__:
1288 print >>hyperdb.DEBUG, 'is_retired', (self, sql, nodeid)
1289 cursor.execute(sql, (nodeid,))
1290 return cursor.fetchone()[0]
1292 def destroy(self, nodeid):
1293 '''Destroy a node.
1295 WARNING: this method should never be used except in extremely rare
1296 situations where there could never be links to the node being
1297 deleted
1298 WARNING: use retire() instead
1299 WARNING: the properties of this node will not be available ever again
1300 WARNING: really, use retire() instead
1302 Well, I think that's enough warnings. This method exists mostly to
1303 support the session storage of the cgi interface.
1305 The node is completely removed from the hyperdb, including all journal
1306 entries. It will no longer be available, and will generally break code
1307 if there are any references to the node.
1308 '''
1309 if self.db.journaltag is None:
1310 raise DatabaseError, 'Database open read-only'
1311 self.db.destroynode(self.classname, nodeid)
1313 def history(self, nodeid):
1314 '''Retrieve the journal of edits on a particular node.
1316 'nodeid' must be the id of an existing node of this class or an
1317 IndexError is raised.
1319 The returned list contains tuples of the form
1321 (date, tag, action, params)
1323 'date' is a Timestamp object specifying the time of the change and
1324 'tag' is the journaltag specified when the database was opened.
1325 '''
1326 if not self.do_journal:
1327 raise ValueError, 'Journalling is disabled for this class'
1328 return self.db.getjournal(self.classname, nodeid)
1330 # Locating nodes:
1331 def hasnode(self, nodeid):
1332 '''Determine if the given nodeid actually exists
1333 '''
1334 return self.db.hasnode(self.classname, nodeid)
1336 def setkey(self, propname):
1337 '''Select a String property of this class to be the key property.
1339 'propname' must be the name of a String property of this class or
1340 None, or a TypeError is raised. The values of the key property on
1341 all existing nodes must be unique or a ValueError is raised.
1342 '''
1343 # XXX create an index on the key prop column
1344 prop = self.getprops()[propname]
1345 if not isinstance(prop, String):
1346 raise TypeError, 'key properties must be String'
1347 self.key = propname
1349 def getkey(self):
1350 '''Return the name of the key property for this class or None.'''
1351 return self.key
1353 def labelprop(self, default_to_id=0):
1354 ''' Return the property name for a label for the given node.
1356 This method attempts to generate a consistent label for the node.
1357 It tries the following in order:
1358 1. key property
1359 2. "name" property
1360 3. "title" property
1361 4. first property from the sorted property name list
1362 '''
1363 k = self.getkey()
1364 if k:
1365 return k
1366 props = self.getprops()
1367 if props.has_key('name'):
1368 return 'name'
1369 elif props.has_key('title'):
1370 return 'title'
1371 if default_to_id:
1372 return 'id'
1373 props = props.keys()
1374 props.sort()
1375 return props[0]
1377 def lookup(self, keyvalue):
1378 '''Locate a particular node by its key property and return its id.
1380 If this class has no key property, a TypeError is raised. If the
1381 'keyvalue' matches one of the values for the key property among
1382 the nodes in this class, the matching node's id is returned;
1383 otherwise a KeyError is raised.
1384 '''
1385 if not self.key:
1386 raise TypeError, 'No key property set'
1388 cursor = self.db.conn.cursor()
1389 sql = 'select id from _%s where _%s=?'%(self.classname, self.key)
1390 if __debug__:
1391 print >>hyperdb.DEBUG, 'lookup', (self, sql, keyvalue)
1392 cursor.execute(sql, (keyvalue,))
1394 # see if there was a result
1395 l = cursor.fetchall()
1396 if not l:
1397 raise KeyError, keyvalue
1399 # return the id
1400 return l[0][0]
1402 def find(self, **propspec):
1403 '''Get the ids of nodes in this class which link to the given nodes.
1405 'propspec' consists of keyword args propname={nodeid:1,}
1406 'propname' must be the name of a property in this class, or a
1407 KeyError is raised. That property must be a Link or Multilink
1408 property, or a TypeError is raised.
1410 Any node in this class whose 'propname' property links to any of the
1411 nodeids will be returned. Used by the full text indexing, which knows
1412 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1413 issues:
1415 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1416 '''
1417 if __debug__:
1418 print >>hyperdb.DEBUG, 'find', (self, propspec)
1419 if not propspec:
1420 return []
1421 queries = []
1422 tables = []
1423 allvalues = ()
1424 for prop, values in propspec.items():
1425 allvalues += tuple(values.keys())
1426 tables.append('select nodeid from %s_%s where linkid in (%s)'%(
1427 self.classname, prop, ','.join(['?' for x in values.keys()])))
1428 sql = '\nintersect\n'.join(tables)
1429 if __debug__:
1430 print >>hyperdb.DEBUG, 'find', (self, sql, allvalues)
1431 cursor = self.db.conn.cursor()
1432 cursor.execute(sql, allvalues)
1433 try:
1434 l = [x[0] for x in cursor.fetchall()]
1435 except gadfly.database.error, message:
1436 if message == 'no more results':
1437 l = []
1438 raise
1439 if __debug__:
1440 print >>hyperdb.DEBUG, 'find ... ', l
1441 return l
1443 def list(self):
1444 ''' Return a list of the ids of the active nodes in this class.
1445 '''
1446 return self.db.getnodeids(self.classname, retired=0)
1448 def filter(self, search_matches, filterspec, sort, group,
1449 num_re = re.compile('^\d+$')):
1450 ''' Return a list of the ids of the active nodes in this class that
1451 match the 'filter' spec, sorted by the group spec and then the
1452 sort spec
1453 '''
1454 raise NotImplementedError
1456 def count(self):
1457 '''Get the number of nodes in this class.
1459 If the returned integer is 'numnodes', the ids of all the nodes
1460 in this class run from 1 to numnodes, and numnodes+1 will be the
1461 id of the next node to be created in this class.
1462 '''
1463 return self.db.countnodes(self.classname)
1465 # Manipulating properties:
1466 def getprops(self, protected=1):
1467 '''Return a dictionary mapping property names to property objects.
1468 If the "protected" flag is true, we include protected properties -
1469 those which may not be modified.
1470 '''
1471 d = self.properties.copy()
1472 if protected:
1473 d['id'] = String()
1474 d['creation'] = hyperdb.Date()
1475 d['activity'] = hyperdb.Date()
1476 d['creator'] = hyperdb.Link("user")
1477 return d
1479 def addprop(self, **properties):
1480 '''Add properties to this class.
1482 The keyword arguments in 'properties' must map names to property
1483 objects, or a TypeError is raised. None of the keys in 'properties'
1484 may collide with the names of existing properties, or a ValueError
1485 is raised before any properties have been added.
1486 '''
1487 for key in properties.keys():
1488 if self.properties.has_key(key):
1489 raise ValueError, key
1490 self.properties.update(properties)
1492 def index(self, nodeid):
1493 '''Add (or refresh) the node to search indexes
1494 '''
1495 # find all the String properties that have indexme
1496 for prop, propclass in self.getprops().items():
1497 if isinstance(propclass, String) and propclass.indexme:
1498 try:
1499 value = str(self.get(nodeid, prop))
1500 except IndexError:
1501 # node no longer exists - entry should be removed
1502 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1503 else:
1504 # and index them under (classname, nodeid, property)
1505 self.db.indexer.add_text((self.classname, nodeid, prop),
1506 value)
1509 #
1510 # Detector interface
1511 #
1512 def audit(self, event, detector):
1513 '''Register a detector
1514 '''
1515 l = self.auditors[event]
1516 if detector not in l:
1517 self.auditors[event].append(detector)
1519 def fireAuditors(self, action, nodeid, newvalues):
1520 '''Fire all registered auditors.
1521 '''
1522 for audit in self.auditors[action]:
1523 audit(self.db, self, nodeid, newvalues)
1525 def react(self, event, detector):
1526 '''Register a detector
1527 '''
1528 l = self.reactors[event]
1529 if detector not in l:
1530 self.reactors[event].append(detector)
1532 def fireReactors(self, action, nodeid, oldvalues):
1533 '''Fire all registered reactors.
1534 '''
1535 for react in self.reactors[action]:
1536 react(self.db, self, nodeid, oldvalues)
1538 class FileClass(Class):
1539 '''This class defines a large chunk of data. To support this, it has a
1540 mandatory String property "content" which is typically saved off
1541 externally to the hyperdb.
1543 The default MIME type of this data is defined by the
1544 "default_mime_type" class attribute, which may be overridden by each
1545 node if the class defines a "type" String property.
1546 '''
1547 default_mime_type = 'text/plain'
1549 def create(self, **propvalues):
1550 ''' snaffle the file propvalue and store in a file
1551 '''
1552 content = propvalues['content']
1553 del propvalues['content']
1554 newid = Class.create(self, **propvalues)
1555 self.db.storefile(self.classname, newid, None, content)
1556 return newid
1558 def import_list(self, propnames, proplist):
1559 ''' Trap the "content" property...
1560 '''
1561 # dupe this list so we don't affect others
1562 propnames = propnames[:]
1564 # extract the "content" property from the proplist
1565 i = propnames.index('content')
1566 content = proplist[i]
1567 del propnames[i]
1568 del proplist[i]
1570 # do the normal import
1571 newid = Class.import_list(self, propnames, proplist)
1573 # save off the "content" file
1574 self.db.storefile(self.classname, newid, None, content)
1575 return newid
1577 _marker = []
1578 def get(self, nodeid, propname, default=_marker, cache=1):
1579 ''' trap the content propname and get it from the file
1580 '''
1582 poss_msg = 'Possibly a access right configuration problem.'
1583 if propname == 'content':
1584 try:
1585 return self.db.getfile(self.classname, nodeid, None)
1586 except IOError, (strerror):
1587 # BUG: by catching this we donot see an error in the log.
1588 return 'ERROR reading file: %s%s\n%s\n%s'%(
1589 self.classname, nodeid, poss_msg, strerror)
1590 if default is not self._marker:
1591 return Class.get(self, nodeid, propname, default, cache=cache)
1592 else:
1593 return Class.get(self, nodeid, propname, cache=cache)
1595 def getprops(self, protected=1):
1596 ''' In addition to the actual properties on the node, these methods
1597 provide the "content" property. If the "protected" flag is true,
1598 we include protected properties - those which may not be
1599 modified.
1600 '''
1601 d = Class.getprops(self, protected=protected).copy()
1602 if protected:
1603 d['content'] = hyperdb.String()
1604 return d
1606 def index(self, nodeid):
1607 ''' Index the node in the search index.
1609 We want to index the content in addition to the normal String
1610 property indexing.
1611 '''
1612 # perform normal indexing
1613 Class.index(self, nodeid)
1615 # get the content to index
1616 content = self.get(nodeid, 'content')
1618 # figure the mime type
1619 if self.properties.has_key('type'):
1620 mime_type = self.get(nodeid, 'type')
1621 else:
1622 mime_type = self.default_mime_type
1624 # and index!
1625 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
1626 mime_type)
1628 # XXX deviation from spec - was called ItemClass
1629 class IssueClass(Class, roundupdb.IssueClass):
1630 # Overridden methods:
1631 def __init__(self, db, classname, **properties):
1632 '''The newly-created class automatically includes the "messages",
1633 "files", "nosy", and "superseder" properties. If the 'properties'
1634 dictionary attempts to specify any of these properties or a
1635 "creation" or "activity" property, a ValueError is raised.
1636 '''
1637 if not properties.has_key('title'):
1638 properties['title'] = hyperdb.String(indexme='yes')
1639 if not properties.has_key('messages'):
1640 properties['messages'] = hyperdb.Multilink("msg")
1641 if not properties.has_key('files'):
1642 properties['files'] = hyperdb.Multilink("file")
1643 if not properties.has_key('nosy'):
1644 properties['nosy'] = hyperdb.Multilink("user")
1645 if not properties.has_key('superseder'):
1646 properties['superseder'] = hyperdb.Multilink(classname)
1647 Class.__init__(self, db, classname, **properties)
1649 #
1650 # $Log: not supported by cvs2svn $
1651 # Revision 1.3 2002/08/23 04:58:00 richard
1652 # ahhh, I understand now
1653 #
1654 # Revision 1.2 2002/08/23 04:48:10 richard
1655 # That's gadfly done, mostly. Things left:
1656 # - Class.filter (I'm a wuss ;)
1657 # - schema changes adding new non-multilink properties are not implemented.
1658 # gadfly doesn't have an ALTER TABLE command, making that quite difficult :)
1659 #
1660 # I had to mangle two unit tests to get this all working:
1661 # - gadfly also can't handle two handles open on the one database, so
1662 # testIDGeneration doesn't try that.
1663 # - testNewProperty is disabled as per the second comment above.
1664 #
1665 # I noticed test_pack was incorrect, and the *dbm tests fail there now.
1666 # Looking into it...
1667 #
1668 # Revision 1.1 2002/08/22 07:56:51 richard
1669 # Whee! It's not finished yet, but I can create a new instance and play with
1670 # it a little bit :)
1671 #
1672 # Revision 1.80 2002/08/16 04:28:13 richard
1673 # added is_retired query to Class
1674 #
1675 # Revision 1.79 2002/07/29 23:30:14 richard
1676 # documentation reorg post-new-security
1677 #
1678 # Revision 1.78 2002/07/21 03:26:37 richard
1679 # Gordon, does this help?
1680 #
1681 # Revision 1.77 2002/07/18 11:27:47 richard
1682 # ws
1683 #
1684 # Revision 1.76 2002/07/18 11:17:30 gmcm
1685 # Add Number and Boolean types to hyperdb.
1686 # Add conversion cases to web, mail & admin interfaces.
1687 # Add storage/serialization cases to back_anydbm & back_metakit.
1688 #
1689 # Revision 1.75 2002/07/14 02:05:53 richard
1690 # . all storage-specific code (ie. backend) is now implemented by the backends
1691 #
1692 # Revision 1.74 2002/07/10 00:24:10 richard
1693 # braino
1694 #
1695 # Revision 1.73 2002/07/10 00:19:48 richard
1696 # Added explicit closing of backend database handles.
1697 #
1698 # Revision 1.72 2002/07/09 21:53:38 gmcm
1699 # Optimize Class.find so that the propspec can contain a set of ids to match.
1700 # This is used by indexer.search so it can do just one find for all the index matches.
1701 # This was already confusing code, but for common terms (lots of index matches),
1702 # it is enormously faster.
1703 #
1704 # Revision 1.71 2002/07/09 03:02:52 richard
1705 # More indexer work:
1706 # - all String properties may now be indexed too. Currently there's a bit of
1707 # "issue" specific code in the actual searching which needs to be
1708 # addressed. In a nutshell:
1709 # + pass 'indexme="yes"' as a String() property initialisation arg, eg:
1710 # file = FileClass(db, "file", name=String(), type=String(),
1711 # comment=String(indexme="yes"))
1712 # + the comment will then be indexed and be searchable, with the results
1713 # related back to the issue that the file is linked to
1714 # - as a result of this work, the FileClass has a default MIME type that may
1715 # be overridden in a subclass, or by the use of a "type" property as is
1716 # done in the default templates.
1717 # - the regeneration of the indexes (if necessary) is done once the schema is
1718 # set up in the dbinit.
1719 #
1720 # Revision 1.70 2002/06/27 12:06:20 gmcm
1721 # Improve an error message.
1722 #
1723 # Revision 1.69 2002/06/17 23:15:29 richard
1724 # Can debug to stdout now
1725 #
1726 # Revision 1.68 2002/06/11 06:52:03 richard
1727 # . #564271 ] find() and new properties
1728 #
1729 # Revision 1.67 2002/06/11 05:02:37 richard
1730 # . #565979 ] code error in hyperdb.Class.find
1731 #
1732 # Revision 1.66 2002/05/25 07:16:24 rochecompaan
1733 # Merged search_indexing-branch with HEAD
1734 #
1735 # Revision 1.65 2002/05/22 04:12:05 richard
1736 # . applied patch #558876 ] cgi client customization
1737 # ... with significant additions and modifications ;)
1738 # - extended handling of ML assignedto to all places it's handled
1739 # - added more NotFound info
1740 #
1741 # Revision 1.64 2002/05/15 06:21:21 richard
1742 # . node caching now works, and gives a small boost in performance
1743 #
1744 # As a part of this, I cleaned up the DEBUG output and implemented TRACE
1745 # output (HYPERDBTRACE='file to trace to') with checkpoints at the start of
1746 # CGI requests. Run roundup with python -O to skip all the DEBUG/TRACE stuff
1747 # (using if __debug__ which is compiled out with -O)
1748 #
1749 # Revision 1.63 2002/04/15 23:25:15 richard
1750 # . node ids are now generated from a lockable store - no more race conditions
1751 #
1752 # We're using the portalocker code by Jonathan Feinberg that was contributed
1753 # to the ASPN Python cookbook. This gives us locking across Unix and Windows.
1754 #
1755 # Revision 1.62 2002/04/03 07:05:50 richard
1756 # d'oh! killed retirement of nodes :(
1757 # all better now...
1758 #
1759 # Revision 1.61 2002/04/03 06:11:51 richard
1760 # Fix for old databases that contain properties that don't exist any more.
1761 #
1762 # Revision 1.60 2002/04/03 05:54:31 richard
1763 # Fixed serialisation problem by moving the serialisation step out of the
1764 # hyperdb.Class (get, set) into the hyperdb.Database.
1765 #
1766 # Also fixed htmltemplate after the showid changes I made yesterday.
1767 #
1768 # Unit tests for all of the above written.
1769 #
1770 # Revision 1.59.2.2 2002/04/20 13:23:33 rochecompaan
1771 # We now have a separate search page for nodes. Search links for
1772 # different classes can be customized in instance_config similar to
1773 # index links.
1774 #
1775 # Revision 1.59.2.1 2002/04/19 19:54:42 rochecompaan
1776 # cgi_client.py
1777 # removed search link for the time being
1778 # moved rendering of matches to htmltemplate
1779 # hyperdb.py
1780 # filtering of nodes on full text search incorporated in filter method
1781 # roundupdb.py
1782 # added paramater to call of filter method
1783 # roundup_indexer.py
1784 # added search method to RoundupIndexer class
1785 #
1786 # Revision 1.59 2002/03/12 22:52:26 richard
1787 # more pychecker warnings removed
1788 #
1789 # Revision 1.58 2002/02/27 03:23:16 richard
1790 # Ran it through pychecker, made fixes
1791 #
1792 # Revision 1.57 2002/02/20 05:23:24 richard
1793 # Didn't accomodate new values for new properties
1794 #
1795 # Revision 1.56 2002/02/20 05:05:28 richard
1796 # . Added simple editing for classes that don't define a templated interface.
1797 # - access using the admin "class list" interface
1798 # - limited to admin-only
1799 # - requires the csv module from object-craft (url given if it's missing)
1800 #
1801 # Revision 1.55 2002/02/15 07:27:12 richard
1802 # Oops, precedences around the way w0rng.
1803 #
1804 # Revision 1.54 2002/02/15 07:08:44 richard
1805 # . Alternate email addresses are now available for users. See the MIGRATION
1806 # file for info on how to activate the feature.
1807 #
1808 # Revision 1.53 2002/01/22 07:21:13 richard
1809 # . fixed back_bsddb so it passed the journal tests
1810 #
1811 # ... it didn't seem happy using the back_anydbm _open method, which is odd.
1812 # Yet another occurrance of whichdb not being able to recognise older bsddb
1813 # databases. Yadda yadda. Made the HYPERDBDEBUG stuff more sane in the
1814 # process.
1815 #
1816 # Revision 1.52 2002/01/21 16:33:19 rochecompaan
1817 # You can now use the roundup-admin tool to pack the database
1818 #
1819 # Revision 1.51 2002/01/21 03:01:29 richard
1820 # brief docco on the do_journal argument
1821 #
1822 # Revision 1.50 2002/01/19 13:16:04 rochecompaan
1823 # Journal entries for link and multilink properties can now be switched on
1824 # or off.
1825 #
1826 # Revision 1.49 2002/01/16 07:02:57 richard
1827 # . lots of date/interval related changes:
1828 # - more relaxed date format for input
1829 #
1830 # Revision 1.48 2002/01/14 06:32:34 richard
1831 # . #502951 ] adding new properties to old database
1832 #
1833 # Revision 1.47 2002/01/14 02:20:15 richard
1834 # . changed all config accesses so they access either the instance or the
1835 # config attriubute on the db. This means that all config is obtained from
1836 # instance_config instead of the mish-mash of classes. This will make
1837 # switching to a ConfigParser setup easier too, I hope.
1838 #
1839 # At a minimum, this makes migration a _little_ easier (a lot easier in the
1840 # 0.5.0 switch, I hope!)
1841 #
1842 # Revision 1.46 2002/01/07 10:42:23 richard
1843 # oops
1844 #
1845 # Revision 1.45 2002/01/02 04:18:17 richard
1846 # hyperdb docstrings
1847 #
1848 # Revision 1.44 2002/01/02 02:31:38 richard
1849 # Sorry for the huge checkin message - I was only intending to implement #496356
1850 # but I found a number of places where things had been broken by transactions:
1851 # . modified ROUNDUPDBSENDMAILDEBUG to be SENDMAILDEBUG and hold a filename
1852 # for _all_ roundup-generated smtp messages to be sent to.
1853 # . the transaction cache had broken the roundupdb.Class set() reactors
1854 # . newly-created author users in the mailgw weren't being committed to the db
1855 #
1856 # Stuff that made it into CHANGES.txt (ie. the stuff I was actually working
1857 # on when I found that stuff :):
1858 # . #496356 ] Use threading in messages
1859 # . detectors were being registered multiple times
1860 # . added tests for mailgw
1861 # . much better attaching of erroneous messages in the mail gateway
1862 #
1863 # Revision 1.43 2001/12/20 06:13:24 rochecompaan
1864 # Bugs fixed:
1865 # . Exception handling in hyperdb for strings-that-look-like numbers got
1866 # lost somewhere
1867 # . Internet Explorer submits full path for filename - we now strip away
1868 # the path
1869 # Features added:
1870 # . Link and multilink properties are now displayed sorted in the cgi
1871 # interface
1872 #
1873 # Revision 1.42 2001/12/16 10:53:37 richard
1874 # take a copy of the node dict so that the subsequent set
1875 # operation doesn't modify the oldvalues structure
1876 #
1877 # Revision 1.41 2001/12/15 23:47:47 richard
1878 # Cleaned up some bare except statements
1879 #
1880 # Revision 1.40 2001/12/14 23:42:57 richard
1881 # yuck, a gdbm instance tests false :(
1882 # I've left the debugging code in - it should be removed one day if we're ever
1883 # _really_ anal about performace :)
1884 #
1885 # Revision 1.39 2001/12/02 05:06:16 richard
1886 # . We now use weakrefs in the Classes to keep the database reference, so
1887 # the close() method on the database is no longer needed.
1888 # I bumped the minimum python requirement up to 2.1 accordingly.
1889 # . #487480 ] roundup-server
1890 # . #487476 ] INSTALL.txt
1891 #
1892 # I also cleaned up the change message / post-edit stuff in the cgi client.
1893 # There's now a clearly marked "TODO: append the change note" where I believe
1894 # the change note should be added there. The "changes" list will obviously
1895 # have to be modified to be a dict of the changes, or somesuch.
1896 #
1897 # More testing needed.
1898 #
1899 # Revision 1.38 2001/12/01 07:17:50 richard
1900 # . We now have basic transaction support! Information is only written to
1901 # the database when the commit() method is called. Only the anydbm
1902 # backend is modified in this way - neither of the bsddb backends have been.
1903 # The mail, admin and cgi interfaces all use commit (except the admin tool
1904 # doesn't have a commit command, so interactive users can't commit...)
1905 # . Fixed login/registration forwarding the user to the right page (or not,
1906 # on a failure)
1907 #
1908 # Revision 1.37 2001/11/28 21:55:35 richard
1909 # . login_action and newuser_action return values were being ignored
1910 # . Woohoo! Found that bloody re-login bug that was killing the mail
1911 # gateway.
1912 # (also a minor cleanup in hyperdb)
1913 #
1914 # Revision 1.36 2001/11/27 03:16:09 richard
1915 # Another place that wasn't handling missing properties.
1916 #
1917 # Revision 1.35 2001/11/22 15:46:42 jhermann
1918 # Added module docstrings to all modules.
1919 #
1920 # Revision 1.34 2001/11/21 04:04:43 richard
1921 # *sigh* more missing value handling
1922 #
1923 # Revision 1.33 2001/11/21 03:40:54 richard
1924 # more new property handling
1925 #
1926 # Revision 1.32 2001/11/21 03:11:28 richard
1927 # Better handling of new properties.
1928 #
1929 # Revision 1.31 2001/11/12 22:01:06 richard
1930 # Fixed issues with nosy reaction and author copies.
1931 #
1932 # Revision 1.30 2001/11/09 10:11:08 richard
1933 # . roundup-admin now handles all hyperdb exceptions
1934 #
1935 # Revision 1.29 2001/10/27 00:17:41 richard
1936 # Made Class.stringFind() do caseless matching.
1937 #
1938 # Revision 1.28 2001/10/21 04:44:50 richard
1939 # bug #473124: UI inconsistency with Link fields.
1940 # This also prompted me to fix a fairly long-standing usability issue -
1941 # that of being able to turn off certain filters.
1942 #
1943 # Revision 1.27 2001/10/20 23:44:27 richard
1944 # Hyperdatabase sorts strings-that-look-like-numbers as numbers now.
1945 #
1946 # Revision 1.26 2001/10/16 03:48:01 richard
1947 # admin tool now complains if a "find" is attempted with a non-link property.
1948 #
1949 # Revision 1.25 2001/10/11 00:17:51 richard
1950 # Reverted a change in hyperdb so the default value for missing property
1951 # values in a create() is None and not '' (the empty string.) This obviously
1952 # breaks CSV import/export - the string 'None' will be created in an
1953 # export/import operation.
1954 #
1955 # Revision 1.24 2001/10/10 03:54:57 richard
1956 # Added database importing and exporting through CSV files.
1957 # Uses the csv module from object-craft for exporting if it's available.
1958 # Requires the csv module for importing.
1959 #
1960 # Revision 1.23 2001/10/09 23:58:10 richard
1961 # Moved the data stringification up into the hyperdb.Class class' get, set
1962 # and create methods. This means that the data is also stringified for the
1963 # journal call, and removes duplication of code from the backends. The
1964 # backend code now only sees strings.
1965 #
1966 # Revision 1.22 2001/10/09 07:25:59 richard
1967 # Added the Password property type. See "pydoc roundup.password" for
1968 # implementation details. Have updated some of the documentation too.
1969 #
1970 # Revision 1.21 2001/10/05 02:23:24 richard
1971 # . roundup-admin create now prompts for property info if none is supplied
1972 # on the command-line.
1973 # . hyperdb Class getprops() method may now return only the mutable
1974 # properties.
1975 # . Login now uses cookies, which makes it a whole lot more flexible. We can
1976 # now support anonymous user access (read-only, unless there's an
1977 # "anonymous" user, in which case write access is permitted). Login
1978 # handling has been moved into cgi_client.Client.main()
1979 # . The "extended" schema is now the default in roundup init.
1980 # . The schemas have had their page headings modified to cope with the new
1981 # login handling. Existing installations should copy the interfaces.py
1982 # file from the roundup lib directory to their instance home.
1983 # . Incorrectly had a Bizar Software copyright on the cgitb.py module from
1984 # Ping - has been removed.
1985 # . Fixed a whole bunch of places in the CGI interface where we should have
1986 # been returning Not Found instead of throwing an exception.
1987 # . Fixed a deviation from the spec: trying to modify the 'id' property of
1988 # an item now throws an exception.
1989 #
1990 # Revision 1.20 2001/10/04 02:12:42 richard
1991 # Added nicer command-line item adding: passing no arguments will enter an
1992 # interactive more which asks for each property in turn. While I was at it, I
1993 # fixed an implementation problem WRT the spec - I wasn't raising a
1994 # ValueError if the key property was missing from a create(). Also added a
1995 # protected=boolean argument to getprops() so we can list only the mutable
1996 # properties (defaults to yes, which lists the immutables).
1997 #
1998 # Revision 1.19 2001/08/29 04:47:18 richard
1999 # Fixed CGI client change messages so they actually include the properties
2000 # changed (again).
2001 #
2002 # Revision 1.18 2001/08/16 07:34:59 richard
2003 # better CGI text searching - but hidden filter fields are disappearing...
2004 #
2005 # Revision 1.17 2001/08/16 06:59:58 richard
2006 # all searches use re now - and they're all case insensitive
2007 #
2008 # Revision 1.16 2001/08/15 23:43:18 richard
2009 # Fixed some isFooTypes that I missed.
2010 # Refactored some code in the CGI code.
2011 #
2012 # Revision 1.15 2001/08/12 06:32:36 richard
2013 # using isinstance(blah, Foo) now instead of isFooType
2014 #
2015 # Revision 1.14 2001/08/07 00:24:42 richard
2016 # stupid typo
2017 #
2018 # Revision 1.13 2001/08/07 00:15:51 richard
2019 # Added the copyright/license notice to (nearly) all files at request of
2020 # Bizar Software.
2021 #
2022 # Revision 1.12 2001/08/02 06:38:17 richard
2023 # Roundupdb now appends "mailing list" information to its messages which
2024 # include the e-mail address and web interface address. Templates may
2025 # override this in their db classes to include specific information (support
2026 # instructions, etc).
2027 #
2028 # Revision 1.11 2001/08/01 04:24:21 richard
2029 # mailgw was assuming certain properties existed on the issues being created.
2030 #
2031 # Revision 1.10 2001/07/30 02:38:31 richard
2032 # get() now has a default arg - for migration only.
2033 #
2034 # Revision 1.9 2001/07/29 09:28:23 richard
2035 # Fixed sorting by clicking on column headings.
2036 #
2037 # Revision 1.8 2001/07/29 08:27:40 richard
2038 # Fixed handling of passed-in values in form elements (ie. during a
2039 # drill-down)
2040 #
2041 # Revision 1.7 2001/07/29 07:01:39 richard
2042 # Added vim command to all source so that we don't get no steenkin' tabs :)
2043 #
2044 # Revision 1.6 2001/07/29 05:36:14 richard
2045 # Cleanup of the link label generation.
2046 #
2047 # Revision 1.5 2001/07/29 04:05:37 richard
2048 # Added the fabricated property "id".
2049 #
2050 # Revision 1.4 2001/07/27 06:25:35 richard
2051 # Fixed some of the exceptions so they're the right type.
2052 # Removed the str()-ification of node ids so we don't mask oopsy errors any
2053 # more.
2054 #
2055 # Revision 1.3 2001/07/27 05:17:14 richard
2056 # just some comments
2057 #
2058 # Revision 1.2 2001/07/22 12:09:32 richard
2059 # Final commit of Grande Splite
2060 #
2061 # Revision 1.1 2001/07/22 11:58:35 richard
2062 # More Grande Splite
2063 #
2064 #
2065 # vim: set filetype=python ts=4 sw=4 et si