1 # $Id: back_gadfly.py,v 1.17 2002-09-12 05:51:42 richard Exp $
2 __doc__ = '''
3 About Gadfly
4 ============
6 Gadfly is a collection of python modules that provides relational
7 database functionality entirely implemented in Python. It supports a
8 subset of the intergalactic standard RDBMS Structured Query Language
9 SQL.
12 Basic Structure
13 ===============
15 We map roundup classes to relational tables. Automatically detect schema
16 changes and modify the gadfly table schemas appropriately. Multilinks
17 (which represent a many-to-many relationship) are handled through
18 intermediate tables.
20 Journals are stored adjunct to the per-class tables.
22 Table names and columns have "_" prepended so the names can't
23 clash with restricted names (like "order"). Retirement is determined by the
24 __retired__ column being true.
26 All columns are defined as VARCHAR, since it really doesn't matter what
27 type they're defined as. We stuff all kinds of data in there ;) [as long as
28 it's marshallable, gadfly doesn't care]
31 Additional Instance Requirements
32 ================================
34 The instance configuration must specify where the database is. It does this
35 with GADFLY_DATABASE, which is used as the arguments to the gadfly.gadfly()
36 method:
38 Using an on-disk database directly (not a good idea):
39 GADFLY_DATABASE = (database name, directory)
41 Using a network database (much better idea):
42 GADFLY_DATABASE = (policy, password, address, port)
44 Because multiple accesses directly to a gadfly database aren't handled, but
45 multiple network accesses are, it's strongly advised that the latter setup be
46 used.
48 '''
50 # standard python modules
51 import sys, os, time, re, errno, weakref, copy
53 # roundup modules
54 from roundup import hyperdb, date, password, roundupdb, security
55 from roundup.hyperdb import String, Password, Date, Interval, Link, \
56 Multilink, DatabaseError, Boolean, Number
58 # the all-important gadfly :)
59 import gadfly
60 import gadfly.client
61 import gadfly.database
63 # support
64 from blobfiles import FileStorage
65 from roundup.indexer import Indexer
66 from sessions import Sessions
68 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
69 # flag to set on retired entries
70 RETIRED_FLAG = '__hyperdb_retired'
72 def __init__(self, config, journaltag=None):
73 ''' Open the database and load the schema from it.
74 '''
75 self.config, self.journaltag = config, journaltag
76 self.dir = config.DATABASE
77 self.classes = {}
78 self.indexer = Indexer(self.dir)
79 self.sessions = Sessions(self.config)
80 self.security = security.Security(self)
82 # additional transaction support for external files and the like
83 self.transactions = []
85 db = getattr(config, 'GADFLY_DATABASE', ('database', self.dir))
86 if len(db) == 2:
87 # ensure files are group readable and writable
88 os.umask(0002)
89 try:
90 self.conn = gadfly.gadfly(*db)
91 except IOError, error:
92 if error.errno != errno.ENOENT:
93 raise
94 self.database_schema = {}
95 self.conn = gadfly.gadfly()
96 self.conn.startup(*db)
97 cursor = self.conn.cursor()
98 cursor.execute('create table schema (schema varchar)')
99 cursor.execute('create table ids (name varchar, num integer)')
100 else:
101 cursor = self.conn.cursor()
102 cursor.execute('select schema from schema')
103 self.database_schema = cursor.fetchone()[0]
104 else:
105 self.conn = gadfly.client.gfclient(*db)
106 cursor = self.conn.cursor()
107 cursor.execute('select schema from schema')
108 self.database_schema = cursor.fetchone()[0]
110 def __repr__(self):
111 return '<roundfly 0x%x>'%id(self)
113 def post_init(self):
114 ''' Called once the schema initialisation has finished.
116 We should now confirm that the schema defined by our "classes"
117 attribute actually matches the schema in the database.
118 '''
119 # now detect changes in the schema
120 for classname, spec in self.classes.items():
121 if self.database_schema.has_key(classname):
122 dbspec = self.database_schema[classname]
123 self.update_class(spec, dbspec)
124 self.database_schema[classname] = spec.schema()
125 else:
126 self.create_class(spec)
127 self.database_schema[classname] = spec.schema()
129 for classname in self.database_schema.keys():
130 if not self.classes.has_key(classname):
131 self.drop_class(classname)
133 # update the database version of the schema
134 cursor = self.conn.cursor()
135 cursor.execute('delete from schema')
136 cursor.execute('insert into schema values (?)', (self.database_schema,))
138 # reindex the db if necessary
139 if self.indexer.should_reindex():
140 self.reindex()
142 # commit
143 self.conn.commit()
145 def reindex(self):
146 for klass in self.classes.values():
147 for nodeid in klass.list():
148 klass.index(nodeid)
149 self.indexer.save_index()
151 def determine_columns(self, spec):
152 ''' Figure the column names and multilink properties from the spec
153 '''
154 cols = []
155 mls = []
156 # add the multilinks separately
157 for col, prop in spec.properties.items():
158 if isinstance(prop, Multilink):
159 mls.append(col)
160 else:
161 cols.append('_'+col)
162 cols.sort()
163 return cols, mls
165 def update_class(self, spec, dbspec):
166 ''' Determine the differences between the current spec and the
167 database version of the spec, and update where necessary
169 NOTE that this doesn't work for adding/deleting properties!
170 ... until gadfly grows an ALTER TABLE command, it's not going to!
171 '''
172 spec_schema = spec.schema()
173 if spec_schema == dbspec:
174 return
175 if __debug__:
176 print >>hyperdb.DEBUG, 'update_class FIRING'
178 # key property changed?
179 if dbspec[0] != spec_schema[0]:
180 if __debug__:
181 print >>hyperdb.DEBUG, 'update_class setting keyprop', `spec[0]`
182 # XXX turn on indexing for the key property
184 # dict 'em up
185 spec_propnames,spec_props = [],{}
186 for propname,prop in spec_schema[1]:
187 spec_propnames.append(propname)
188 spec_props[propname] = prop
189 dbspec_propnames,dbspec_props = [],{}
190 for propname,prop in dbspec[1]:
191 dbspec_propnames.append(propname)
192 dbspec_props[propname] = prop
194 # we're going to need one of these
195 cursor = self.conn.cursor()
197 # now compare
198 for propname in spec_propnames:
199 prop = spec_props[propname]
200 if __debug__:
201 print >>hyperdb.DEBUG, 'update_class ...', `prop`
202 if dbspec_props.has_key(propname) and prop==dbspec_props[propname]:
203 continue
204 if __debug__:
205 print >>hyperdb.DEBUG, 'update_class', `prop`
207 if not dbspec_props.has_key(propname):
208 # add the property
209 if isinstance(prop, Multilink):
210 sql = 'create table %s_%s (linkid varchar, nodeid '\
211 'varchar)'%(spec.classname, prop)
212 if __debug__:
213 print >>hyperdb.DEBUG, 'update_class', (self, sql)
214 cursor.execute(sql)
215 else:
216 # XXX gadfly doesn't have an ALTER TABLE command
217 raise NotImplementedError
218 sql = 'alter table _%s add column (_%s varchar)'%(
219 spec.classname, propname)
220 if __debug__:
221 print >>hyperdb.DEBUG, 'update_class', (self, sql)
222 cursor.execute(sql)
223 else:
224 # modify the property
225 if __debug__:
226 print >>hyperdb.DEBUG, 'update_class NOOP'
227 pass # NOOP in gadfly
229 # and the other way - only worry about deletions here
230 for propname in dbspec_propnames:
231 prop = dbspec_props[propname]
232 if spec_props.has_key(propname):
233 continue
234 if __debug__:
235 print >>hyperdb.DEBUG, 'update_class', `prop`
237 # delete the property
238 if isinstance(prop, Multilink):
239 sql = 'drop table %s_%s'%(spec.classname, prop)
240 if __debug__:
241 print >>hyperdb.DEBUG, 'update_class', (self, sql)
242 cursor.execute(sql)
243 else:
244 # XXX gadfly doesn't have an ALTER TABLE command
245 raise NotImplementedError
246 sql = 'alter table _%s delete column _%s'%(spec.classname,
247 propname)
248 if __debug__:
249 print >>hyperdb.DEBUG, 'update_class', (self, sql)
250 cursor.execute(sql)
252 def create_class(self, spec):
253 ''' Create a database table according to the given spec.
254 '''
255 cols, mls = self.determine_columns(spec)
257 # add on our special columns
258 cols.append('id')
259 cols.append('__retired__')
261 cursor = self.conn.cursor()
263 # create the base table
264 cols = ','.join(['%s varchar'%x for x in cols])
265 sql = 'create table _%s (%s)'%(spec.classname, cols)
266 if __debug__:
267 print >>hyperdb.DEBUG, 'create_class', (self, sql)
268 cursor.execute(sql)
270 # journal table
271 cols = ','.join(['%s varchar'%x
272 for x in 'nodeid date tag action params'.split()])
273 sql = 'create table %s__journal (%s)'%(spec.classname, cols)
274 if __debug__:
275 print >>hyperdb.DEBUG, 'create_class', (self, sql)
276 cursor.execute(sql)
278 # now create the multilink tables
279 for ml in mls:
280 sql = 'create table %s_%s (linkid varchar, nodeid varchar)'%(
281 spec.classname, ml)
282 if __debug__:
283 print >>hyperdb.DEBUG, 'create_class', (self, sql)
284 cursor.execute(sql)
286 # ID counter
287 sql = 'insert into ids (name, num) values (?,?)'
288 vals = (spec.classname, 1)
289 if __debug__:
290 print >>hyperdb.DEBUG, 'create_class', (self, sql, vals)
291 cursor.execute(sql, vals)
293 def drop_class(self, spec):
294 ''' Drop the given table from the database.
296 Drop the journal and multilink tables too.
297 '''
298 # figure the multilinks
299 mls = []
300 for col, prop in spec.properties.items():
301 if isinstance(prop, Multilink):
302 mls.append(col)
303 cursor = self.conn.cursor()
305 sql = 'drop table _%s'%spec.classname
306 if __debug__:
307 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
308 cursor.execute(sql)
310 sql = 'drop table %s__journal'%spec.classname
311 if __debug__:
312 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
313 cursor.execute(sql)
315 for ml in mls:
316 sql = 'drop table %s_%s'%(spec.classname, ml)
317 if __debug__:
318 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
319 cursor.execute(sql)
321 #
322 # Classes
323 #
324 def __getattr__(self, classname):
325 ''' A convenient way of calling self.getclass(classname).
326 '''
327 if self.classes.has_key(classname):
328 if __debug__:
329 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
330 return self.classes[classname]
331 raise AttributeError, classname
333 def addclass(self, cl):
334 ''' Add a Class to the hyperdatabase.
335 '''
336 if __debug__:
337 print >>hyperdb.DEBUG, 'addclass', (self, cl)
338 cn = cl.classname
339 if self.classes.has_key(cn):
340 raise ValueError, cn
341 self.classes[cn] = cl
343 def getclasses(self):
344 ''' Return a list of the names of all existing classes.
345 '''
346 if __debug__:
347 print >>hyperdb.DEBUG, 'getclasses', (self,)
348 l = self.classes.keys()
349 l.sort()
350 return l
352 def getclass(self, classname):
353 '''Get the Class object representing a particular class.
355 If 'classname' is not a valid class name, a KeyError is raised.
356 '''
357 if __debug__:
358 print >>hyperdb.DEBUG, 'getclass', (self, classname)
359 return self.classes[classname]
361 def clear(self):
362 ''' Delete all database contents.
364 Note: I don't commit here, which is different behaviour to the
365 "nuke from orbit" behaviour in the *dbms.
366 '''
367 if __debug__:
368 print >>hyperdb.DEBUG, 'clear', (self,)
369 cursor = self.conn.cursor()
370 for cn in self.classes.keys():
371 sql = 'delete from _%s'%cn
372 if __debug__:
373 print >>hyperdb.DEBUG, 'clear', (self, sql)
374 cursor.execute(sql)
376 #
377 # Node IDs
378 #
379 def newid(self, classname):
380 ''' Generate a new id for the given class
381 '''
382 # get the next ID
383 cursor = self.conn.cursor()
384 sql = 'select num from ids where name=?'
385 if __debug__:
386 print >>hyperdb.DEBUG, 'newid', (self, sql, classname)
387 cursor.execute(sql, (classname, ))
388 newid = cursor.fetchone()[0]
390 # update the counter
391 sql = 'update ids set num=? where name=?'
392 vals = (newid+1, classname)
393 if __debug__:
394 print >>hyperdb.DEBUG, 'newid', (self, sql, vals)
395 cursor.execute(sql, vals)
397 # return as string
398 return str(newid)
400 def setid(self, classname, setid):
401 ''' Set the id counter: used during import of database
402 '''
403 cursor = self.conn.cursor()
404 sql = 'update ids set num=? where name=?'
405 vals = (setid, spec.classname)
406 if __debug__:
407 print >>hyperdb.DEBUG, 'setid', (self, sql, vals)
408 cursor.execute(sql, vals)
410 #
411 # Nodes
412 #
414 def addnode(self, classname, nodeid, node):
415 ''' Add the specified node to its class's db.
416 '''
417 if __debug__:
418 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
419 # gadfly requires values for all non-multilink columns
420 cl = self.classes[classname]
421 cols, mls = self.determine_columns(cl)
423 # default the non-multilink columns
424 for col, prop in cl.properties.items():
425 if not isinstance(col, Multilink):
426 if not node.has_key(col):
427 node[col] = None
429 node = self.serialise(classname, node)
431 # make sure the ordering is correct for column name -> column value
432 vals = tuple([node[col[1:]] for col in cols]) + (nodeid, 0)
433 s = ','.join(['?' for x in cols]) + ',?,?'
434 cols = ','.join(cols) + ',id,__retired__'
436 # perform the inserts
437 cursor = self.conn.cursor()
438 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
439 if __debug__:
440 print >>hyperdb.DEBUG, 'addnode', (self, sql, vals)
441 cursor.execute(sql, vals)
443 # insert the multilink rows
444 for col in mls:
445 t = '%s_%s'%(classname, col)
446 for entry in node[col]:
447 sql = 'insert into %s (linkid, nodeid) values (?,?)'%t
448 vals = (entry, nodeid)
449 if __debug__:
450 print >>hyperdb.DEBUG, 'addnode', (self, sql, vals)
451 cursor.execute(sql, vals)
453 # make sure we do the commit-time extra stuff for this node
454 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
456 def setnode(self, classname, nodeid, node, multilink_changes):
457 ''' Change the specified node.
458 '''
459 if __debug__:
460 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
461 node = self.serialise(classname, node)
463 cl = self.classes[classname]
464 cols = []
465 mls = []
466 # add the multilinks separately
467 for col in node.keys():
468 prop = cl.properties[col]
469 if isinstance(prop, Multilink):
470 mls.append(col)
471 else:
472 cols.append('_'+col)
473 cols.sort()
475 # make sure the ordering is correct for column name -> column value
476 vals = tuple([node[col[1:]] for col in cols])
477 s = ','.join(['%s=?'%x for x in cols])
478 cols = ','.join(cols)
480 # perform the update
481 cursor = self.conn.cursor()
482 sql = 'update _%s set %s'%(classname, s)
483 if __debug__:
484 print >>hyperdb.DEBUG, 'setnode', (self, sql, vals)
485 cursor.execute(sql, vals)
487 # now the fun bit, updating the multilinks ;)
488 for col, (add, remove) in multilink_changes.items():
489 tn = '%s_%s'%(classname, col)
490 if add:
491 sql = 'insert into %s (nodeid, linkid) values (?,?)'%tn
492 vals = [(nodeid, addid) for addid in add]
493 if __debug__:
494 print >>hyperdb.DEBUG, 'setnode (add)', (self, sql, vals)
495 cursor.execute(sql, vals)
496 if remove:
497 sql = 'delete from %s where nodeid=? and linkid=?'%tn
498 vals = [(nodeid, removeid) for removeid in remove]
499 if __debug__:
500 print >>hyperdb.DEBUG, 'setnode (rem)', (self, sql, vals)
501 cursor.execute(sql, vals)
503 # make sure we do the commit-time extra stuff for this node
504 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
506 def getnode(self, classname, nodeid):
507 ''' Get a node from the database.
508 '''
509 if __debug__:
510 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid)
511 # figure the columns we're fetching
512 cl = self.classes[classname]
513 cols, mls = self.determine_columns(cl)
514 scols = ','.join(cols)
516 # perform the basic property fetch
517 cursor = self.conn.cursor()
518 sql = 'select %s from _%s where id=?'%(scols, classname)
519 if __debug__:
520 print >>hyperdb.DEBUG, 'getnode', (self, sql, nodeid)
521 cursor.execute(sql, (nodeid,))
522 try:
523 values = cursor.fetchone()
524 except gadfly.database.error, message:
525 if message == 'no more results':
526 raise IndexError, 'no such %s node %s'%(classname, nodeid)
527 raise
529 # make up the node
530 node = {}
531 for col in range(len(cols)):
532 node[cols[col][1:]] = values[col]
534 # now the multilinks
535 for col in mls:
536 # get the link ids
537 sql = 'select linkid from %s_%s where nodeid=?'%(classname, col)
538 if __debug__:
539 print >>hyperdb.DEBUG, 'getnode', (self, sql, nodeid)
540 cursor.execute(sql, (nodeid,))
541 # extract the first column from the result
542 node[col] = [x[0] for x in cursor.fetchall()]
544 return self.unserialise(classname, node)
546 def destroynode(self, classname, nodeid):
547 '''Remove a node from the database. Called exclusively by the
548 destroy() method on Class.
549 '''
550 if __debug__:
551 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
553 # make sure the node exists
554 if not self.hasnode(classname, nodeid):
555 raise IndexError, '%s has no node %s'%(classname, nodeid)
557 # see if there's any obvious commit actions that we should get rid of
558 for entry in self.transactions[:]:
559 if entry[1][:2] == (classname, nodeid):
560 self.transactions.remove(entry)
562 # now do the SQL
563 cursor = self.conn.cursor()
564 sql = 'delete from _%s where id=?'%(classname)
565 if __debug__:
566 print >>hyperdb.DEBUG, 'destroynode', (self, sql, nodeid)
567 cursor.execute(sql, (nodeid,))
569 def serialise(self, classname, node):
570 '''Copy the node contents, converting non-marshallable data into
571 marshallable data.
572 '''
573 if __debug__:
574 print >>hyperdb.DEBUG, 'serialise', classname, node
575 properties = self.getclass(classname).getprops()
576 d = {}
577 for k, v in node.items():
578 # if the property doesn't exist, or is the "retired" flag then
579 # it won't be in the properties dict
580 if not properties.has_key(k):
581 d[k] = v
582 continue
584 # get the property spec
585 prop = properties[k]
587 if isinstance(prop, Password):
588 d[k] = str(v)
589 elif isinstance(prop, Date) and v is not None:
590 d[k] = v.serialise()
591 elif isinstance(prop, Interval) and v is not None:
592 d[k] = v.serialise()
593 else:
594 d[k] = v
595 return d
597 def unserialise(self, classname, node):
598 '''Decode the marshalled node data
599 '''
600 if __debug__:
601 print >>hyperdb.DEBUG, 'unserialise', classname, node
602 properties = self.getclass(classname).getprops()
603 d = {}
604 for k, v in node.items():
605 # if the property doesn't exist, or is the "retired" flag then
606 # it won't be in the properties dict
607 if not properties.has_key(k):
608 d[k] = v
609 continue
611 # get the property spec
612 prop = properties[k]
614 if isinstance(prop, Date) and v is not None:
615 d[k] = date.Date(v)
616 elif isinstance(prop, Interval) and v is not None:
617 d[k] = date.Interval(v)
618 elif isinstance(prop, Password):
619 p = password.Password()
620 p.unpack(v)
621 d[k] = p
622 else:
623 d[k] = v
624 return d
626 def hasnode(self, classname, nodeid):
627 ''' Determine if the database has a given node.
628 '''
629 cursor = self.conn.cursor()
630 sql = 'select count(*) from _%s where id=?'%classname
631 if __debug__:
632 print >>hyperdb.DEBUG, 'hasnode', (self, sql, nodeid)
633 cursor.execute(sql, (nodeid,))
634 return cursor.fetchone()[0]
636 def countnodes(self, classname):
637 ''' Count the number of nodes that exist for a particular Class.
638 '''
639 cursor = self.conn.cursor()
640 sql = 'select count(*) from _%s'%classname
641 if __debug__:
642 print >>hyperdb.DEBUG, 'countnodes', (self, sql)
643 cursor.execute(sql)
644 return cursor.fetchone()[0]
646 def getnodeids(self, classname, retired=0):
647 ''' Retrieve all the ids of the nodes for a particular Class.
649 Set retired=None to get all nodes. Otherwise it'll get all the
650 retired or non-retired nodes, depending on the flag.
651 '''
652 cursor = self.conn.cursor()
653 # flip the sense of the flag if we don't want all of them
654 if retired is not None:
655 retired = not retired
656 sql = 'select id from _%s where __retired__ <> ?'%classname
657 if __debug__:
658 print >>hyperdb.DEBUG, 'getnodeids', (self, sql, retired)
659 cursor.execute(sql, (retired,))
660 return [x[0] for x in cursor.fetchall()]
662 def addjournal(self, classname, nodeid, action, params):
663 ''' Journal the Action
664 'action' may be:
666 'create' or 'set' -- 'params' is a dictionary of property values
667 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
668 'retire' -- 'params' is None
669 '''
670 if isinstance(params, type({})):
671 if params.has_key('creator'):
672 journaltag = self.user.get(params['creator'], 'username')
673 del params['creator']
674 else:
675 journaltag = self.journaltag
676 if params.has_key('created'):
677 journaldate = params['created'].serialise()
678 del params['created']
679 else:
680 journaldate = date.Date().serialise()
681 if params.has_key('activity'):
682 del params['activity']
684 # serialise the parameters now
685 if action in ('set', 'create'):
686 params = self.serialise(classname, params)
687 else:
688 journaltag = self.journaltag
689 journaldate = date.Date().serialise()
691 # create the journal entry
692 cols = ','.join('nodeid date tag action params'.split())
693 entry = (nodeid, journaldate, journaltag, action, params)
695 if __debug__:
696 print >>hyperdb.DEBUG, 'doSaveJournal', entry
698 # do the insert
699 cursor = self.conn.cursor()
700 sql = 'insert into %s__journal (%s) values (?,?,?,?,?)'%(classname,
701 cols)
702 if __debug__:
703 print >>hyperdb.DEBUG, 'addjournal', (self, sql, entry)
704 cursor.execute(sql, entry)
706 def getjournal(self, classname, nodeid):
707 ''' get the journal for id
708 '''
709 # make sure the node exists
710 if not self.hasnode(classname, nodeid):
711 raise IndexError, '%s has no node %s'%(classname, nodeid)
713 # now get the journal entries
714 cols = ','.join('nodeid date tag action params'.split())
715 cursor = self.conn.cursor()
716 sql = 'select %s from %s__journal where nodeid=?'%(cols, classname)
717 if __debug__:
718 print >>hyperdb.DEBUG, 'getjournal', (self, sql, nodeid)
719 cursor.execute(sql, (nodeid,))
720 res = []
721 for nodeid, date_stamp, user, action, params in cursor.fetchall():
722 res.append((nodeid, date.Date(date_stamp), user, action, params))
723 return res
725 def pack(self, pack_before):
726 ''' Delete all journal entries except "create" before 'pack_before'.
727 '''
728 # get a 'yyyymmddhhmmss' version of the date
729 date_stamp = pack_before.serialise()
731 # do the delete
732 cursor = self.conn.cursor()
733 for classname in self.classes.keys():
734 sql = "delete from %s__journal where date<? and "\
735 "action<>'create'"%classname
736 if __debug__:
737 print >>hyperdb.DEBUG, 'pack', (self, sql, date_stamp)
738 cursor.execute(sql, (date_stamp,))
740 def commit(self):
741 ''' Commit the current transactions.
743 Save all data changed since the database was opened or since the
744 last commit() or rollback().
745 '''
746 if __debug__:
747 print >>hyperdb.DEBUG, 'commit', (self,)
749 # commit gadfly
750 self.conn.commit()
752 # now, do all the other transaction stuff
753 reindex = {}
754 for method, args in self.transactions:
755 reindex[method(*args)] = 1
757 # reindex the nodes that request it
758 for classname, nodeid in filter(None, reindex.keys()):
759 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
760 self.getclass(classname).index(nodeid)
762 # save the indexer state
763 self.indexer.save_index()
765 # clear out the transactions
766 self.transactions = []
768 def rollback(self):
769 ''' Reverse all actions from the current transaction.
771 Undo all the changes made since the database was opened or the last
772 commit() or rollback() was performed.
773 '''
774 if __debug__:
775 print >>hyperdb.DEBUG, 'rollback', (self,)
777 # roll back gadfly
778 self.conn.rollback()
780 # roll back "other" transaction stuff
781 for method, args in self.transactions:
782 # delete temporary files
783 if method == self.doStoreFile:
784 self.rollbackStoreFile(*args)
785 self.transactions = []
787 def doSaveNode(self, classname, nodeid, node):
788 ''' dummy that just generates a reindex event
789 '''
790 # return the classname, nodeid so we reindex this content
791 return (classname, nodeid)
793 #
794 # The base Class class
795 #
796 class Class(hyperdb.Class):
797 ''' The handle to a particular class of nodes in a hyperdatabase.
799 All methods except __repr__ and getnode must be implemented by a
800 concrete backend Class.
801 '''
803 def __init__(self, db, classname, **properties):
804 '''Create a new class with a given name and property specification.
806 'classname' must not collide with the name of an existing class,
807 or a ValueError is raised. The keyword arguments in 'properties'
808 must map names to property objects, or a TypeError is raised.
809 '''
810 if (properties.has_key('creation') or properties.has_key('activity')
811 or properties.has_key('creator')):
812 raise ValueError, '"creation", "activity" and "creator" are '\
813 'reserved'
815 self.classname = classname
816 self.properties = properties
817 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
818 self.key = ''
820 # should we journal changes (default yes)
821 self.do_journal = 1
823 # do the db-related init stuff
824 db.addclass(self)
826 self.auditors = {'create': [], 'set': [], 'retire': []}
827 self.reactors = {'create': [], 'set': [], 'retire': []}
829 def schema(self):
830 ''' A dumpable version of the schema that we can store in the
831 database
832 '''
833 return (self.key, [(x, repr(y)) for x,y in self.properties.items()])
835 def enableJournalling(self):
836 '''Turn journalling on for this class
837 '''
838 self.do_journal = 1
840 def disableJournalling(self):
841 '''Turn journalling off for this class
842 '''
843 self.do_journal = 0
845 # Editing nodes:
846 def create(self, **propvalues):
847 ''' Create a new node of this class and return its id.
849 The keyword arguments in 'propvalues' map property names to values.
851 The values of arguments must be acceptable for the types of their
852 corresponding properties or a TypeError is raised.
854 If this class has a key property, it must be present and its value
855 must not collide with other key strings or a ValueError is raised.
857 Any other properties on this class that are missing from the
858 'propvalues' dictionary are set to None.
860 If an id in a link or multilink property does not refer to a valid
861 node, an IndexError is raised.
862 '''
863 if propvalues.has_key('id'):
864 raise KeyError, '"id" is reserved'
866 if self.db.journaltag is None:
867 raise DatabaseError, 'Database open read-only'
869 if propvalues.has_key('creation') or propvalues.has_key('activity'):
870 raise KeyError, '"creation" and "activity" are reserved'
872 self.fireAuditors('create', None, propvalues)
874 # new node's id
875 newid = self.db.newid(self.classname)
877 # validate propvalues
878 num_re = re.compile('^\d+$')
879 for key, value in propvalues.items():
880 if key == self.key:
881 try:
882 self.lookup(value)
883 except KeyError:
884 pass
885 else:
886 raise ValueError, 'node with key "%s" exists'%value
888 # try to handle this property
889 try:
890 prop = self.properties[key]
891 except KeyError:
892 raise KeyError, '"%s" has no property "%s"'%(self.classname,
893 key)
895 if value is not None and isinstance(prop, Link):
896 if type(value) != type(''):
897 raise ValueError, 'link value must be String'
898 link_class = self.properties[key].classname
899 # if it isn't a number, it's a key
900 if not num_re.match(value):
901 try:
902 value = self.db.classes[link_class].lookup(value)
903 except (TypeError, KeyError):
904 raise IndexError, 'new property "%s": %s not a %s'%(
905 key, value, link_class)
906 elif not self.db.getclass(link_class).hasnode(value):
907 raise IndexError, '%s has no node %s'%(link_class, value)
909 # save off the value
910 propvalues[key] = value
912 # register the link with the newly linked node
913 if self.do_journal and self.properties[key].do_journal:
914 self.db.addjournal(link_class, value, 'link',
915 (self.classname, newid, key))
917 elif isinstance(prop, Multilink):
918 if type(value) != type([]):
919 raise TypeError, 'new property "%s" not a list of ids'%key
921 # clean up and validate the list of links
922 link_class = self.properties[key].classname
923 l = []
924 for entry in value:
925 if type(entry) != type(''):
926 raise ValueError, '"%s" multilink value (%r) '\
927 'must contain Strings'%(key, value)
928 # if it isn't a number, it's a key
929 if not num_re.match(entry):
930 try:
931 entry = self.db.classes[link_class].lookup(entry)
932 except (TypeError, KeyError):
933 raise IndexError, 'new property "%s": %s not a %s'%(
934 key, entry, self.properties[key].classname)
935 l.append(entry)
936 value = l
937 propvalues[key] = value
939 # handle additions
940 for nodeid in value:
941 if not self.db.getclass(link_class).hasnode(nodeid):
942 raise IndexError, '%s has no node %s'%(link_class,
943 nodeid)
944 # register the link with the newly linked node
945 if self.do_journal and self.properties[key].do_journal:
946 self.db.addjournal(link_class, nodeid, 'link',
947 (self.classname, newid, key))
949 elif isinstance(prop, String):
950 if type(value) != type(''):
951 raise TypeError, 'new property "%s" not a string'%key
953 elif isinstance(prop, Password):
954 if not isinstance(value, password.Password):
955 raise TypeError, 'new property "%s" not a Password'%key
957 elif isinstance(prop, Date):
958 if value is not None and not isinstance(value, date.Date):
959 raise TypeError, 'new property "%s" not a Date'%key
961 elif isinstance(prop, Interval):
962 if value is not None and not isinstance(value, date.Interval):
963 raise TypeError, 'new property "%s" not an Interval'%key
965 elif value is not None and isinstance(prop, Number):
966 try:
967 float(value)
968 except ValueError:
969 raise TypeError, 'new property "%s" not numeric'%key
971 elif value is not None and isinstance(prop, Boolean):
972 try:
973 int(value)
974 except ValueError:
975 raise TypeError, 'new property "%s" not boolean'%key
977 # make sure there's data where there needs to be
978 for key, prop in self.properties.items():
979 if propvalues.has_key(key):
980 continue
981 if key == self.key:
982 raise ValueError, 'key property "%s" is required'%key
983 if isinstance(prop, Multilink):
984 propvalues[key] = []
985 else:
986 propvalues[key] = None
988 # done
989 self.db.addnode(self.classname, newid, propvalues)
990 if self.do_journal:
991 self.db.addjournal(self.classname, newid, 'create', propvalues)
993 self.fireReactors('create', newid, None)
995 return newid
997 _marker = []
998 def get(self, nodeid, propname, default=_marker, cache=1):
999 '''Get the value of a property on an existing node of this class.
1001 'nodeid' must be the id of an existing node of this class or an
1002 IndexError is raised. 'propname' must be the name of a property
1003 of this class or a KeyError is raised.
1005 'cache' indicates whether the transaction cache should be queried
1006 for the node. If the node has been modified and you need to
1007 determine what its values prior to modification are, you need to
1008 set cache=0.
1009 '''
1010 if propname == 'id':
1011 return nodeid
1013 if propname == 'creation':
1014 if not self.do_journal:
1015 raise ValueError, 'Journalling is disabled for this class'
1016 journal = self.db.getjournal(self.classname, nodeid)
1017 if journal:
1018 return self.db.getjournal(self.classname, nodeid)[0][1]
1019 else:
1020 # on the strange chance that there's no journal
1021 return date.Date()
1022 if propname == 'activity':
1023 if not self.do_journal:
1024 raise ValueError, 'Journalling is disabled for this class'
1025 journal = self.db.getjournal(self.classname, nodeid)
1026 if journal:
1027 return self.db.getjournal(self.classname, nodeid)[-1][1]
1028 else:
1029 # on the strange chance that there's no journal
1030 return date.Date()
1031 if propname == 'creator':
1032 if not self.do_journal:
1033 raise ValueError, 'Journalling is disabled for this class'
1034 journal = self.db.getjournal(self.classname, nodeid)
1035 if journal:
1036 name = self.db.getjournal(self.classname, nodeid)[0][2]
1037 else:
1038 return None
1039 try:
1040 return self.db.user.lookup(name)
1041 except KeyError:
1042 # the journaltag user doesn't exist any more
1043 return None
1045 # get the property (raises KeyErorr if invalid)
1046 prop = self.properties[propname]
1048 # get the node's dict
1049 d = self.db.getnode(self.classname, nodeid) #, cache=cache)
1051 if not d.has_key(propname):
1052 if default is self._marker:
1053 if isinstance(prop, Multilink):
1054 return []
1055 else:
1056 return None
1057 else:
1058 return default
1060 # don't pass our list to other code
1061 if isinstance(prop, Multilink):
1062 return d[propname][:]
1064 return d[propname]
1066 def getnode(self, nodeid, cache=1):
1067 ''' Return a convenience wrapper for the node.
1069 'nodeid' must be the id of an existing node of this class or an
1070 IndexError is raised.
1072 'cache' indicates whether the transaction cache should be queried
1073 for the node. If the node has been modified and you need to
1074 determine what its values prior to modification are, you need to
1075 set cache=0.
1076 '''
1077 return Node(self, nodeid, cache=cache)
1079 def set(self, nodeid, **propvalues):
1080 '''Modify a property on an existing node of this class.
1082 'nodeid' must be the id of an existing node of this class or an
1083 IndexError is raised.
1085 Each key in 'propvalues' must be the name of a property of this
1086 class or a KeyError is raised.
1088 All values in 'propvalues' must be acceptable types for their
1089 corresponding properties or a TypeError is raised.
1091 If the value of the key property is set, it must not collide with
1092 other key strings or a ValueError is raised.
1094 If the value of a Link or Multilink property contains an invalid
1095 node id, a ValueError is raised.
1096 '''
1097 if not propvalues:
1098 return propvalues
1100 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1101 raise KeyError, '"creation" and "activity" are reserved'
1103 if propvalues.has_key('id'):
1104 raise KeyError, '"id" is reserved'
1106 if self.db.journaltag is None:
1107 raise DatabaseError, 'Database open read-only'
1109 self.fireAuditors('set', nodeid, propvalues)
1110 # Take a copy of the node dict so that the subsequent set
1111 # operation doesn't modify the oldvalues structure.
1112 # XXX used to try the cache here first
1113 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1115 node = self.db.getnode(self.classname, nodeid)
1116 if self.is_retired(nodeid):
1117 raise IndexError
1118 num_re = re.compile('^\d+$')
1120 # if the journal value is to be different, store it in here
1121 journalvalues = {}
1123 # remember the add/remove stuff for multilinks, making it easier
1124 # for the Database layer to do its stuff
1125 multilink_changes = {}
1127 for propname, value in propvalues.items():
1128 # check to make sure we're not duplicating an existing key
1129 if propname == self.key and node[propname] != value:
1130 try:
1131 self.lookup(value)
1132 except KeyError:
1133 pass
1134 else:
1135 raise ValueError, 'node with key "%s" exists'%value
1137 # this will raise the KeyError if the property isn't valid
1138 # ... we don't use getprops() here because we only care about
1139 # the writeable properties.
1140 prop = self.properties[propname]
1142 # if the value's the same as the existing value, no sense in
1143 # doing anything
1144 if node.has_key(propname) and value == node[propname]:
1145 del propvalues[propname]
1146 continue
1148 # do stuff based on the prop type
1149 if isinstance(prop, Link):
1150 link_class = prop.classname
1151 # if it isn't a number, it's a key
1152 if value is not None and not isinstance(value, type('')):
1153 raise ValueError, 'property "%s" link value be a string'%(
1154 propname)
1155 if isinstance(value, type('')) and not num_re.match(value):
1156 try:
1157 value = self.db.classes[link_class].lookup(value)
1158 except (TypeError, KeyError):
1159 raise IndexError, 'new property "%s": %s not a %s'%(
1160 propname, value, prop.classname)
1162 if (value is not None and
1163 not self.db.getclass(link_class).hasnode(value)):
1164 raise IndexError, '%s has no node %s'%(link_class, value)
1166 if self.do_journal and prop.do_journal:
1167 # register the unlink with the old linked node
1168 if node[propname] is not None:
1169 self.db.addjournal(link_class, node[propname], 'unlink',
1170 (self.classname, nodeid, propname))
1172 # register the link with the newly linked node
1173 if value is not None:
1174 self.db.addjournal(link_class, value, 'link',
1175 (self.classname, nodeid, propname))
1177 elif isinstance(prop, Multilink):
1178 if type(value) != type([]):
1179 raise TypeError, 'new property "%s" not a list of'\
1180 ' ids'%propname
1181 link_class = self.properties[propname].classname
1182 l = []
1183 for entry in value:
1184 # if it isn't a number, it's a key
1185 if type(entry) != type(''):
1186 raise ValueError, 'new property "%s" link value ' \
1187 'must be a string'%propname
1188 if not num_re.match(entry):
1189 try:
1190 entry = self.db.classes[link_class].lookup(entry)
1191 except (TypeError, KeyError):
1192 raise IndexError, 'new property "%s": %s not a %s'%(
1193 propname, entry,
1194 self.properties[propname].classname)
1195 l.append(entry)
1196 value = l
1197 propvalues[propname] = value
1199 # figure the journal entry for this property
1200 add = []
1201 remove = []
1203 # handle removals
1204 if node.has_key(propname):
1205 l = node[propname]
1206 else:
1207 l = []
1208 for id in l[:]:
1209 if id in value:
1210 continue
1211 # register the unlink with the old linked node
1212 if self.do_journal and self.properties[propname].do_journal:
1213 self.db.addjournal(link_class, id, 'unlink',
1214 (self.classname, nodeid, propname))
1215 l.remove(id)
1216 remove.append(id)
1218 # handle additions
1219 for id in value:
1220 if not self.db.getclass(link_class).hasnode(id):
1221 raise IndexError, '%s has no node %s'%(link_class, id)
1222 if id in l:
1223 continue
1224 # register the link with the newly linked node
1225 if self.do_journal and self.properties[propname].do_journal:
1226 self.db.addjournal(link_class, id, 'link',
1227 (self.classname, nodeid, propname))
1228 l.append(id)
1229 add.append(id)
1231 # figure the journal entry
1232 l = []
1233 if add:
1234 l.append(('+', add))
1235 if remove:
1236 l.append(('-', remove))
1237 multilink_changes[propname] = (add, remove)
1238 if l:
1239 journalvalues[propname] = tuple(l)
1241 elif isinstance(prop, String):
1242 if value is not None and type(value) != type(''):
1243 raise TypeError, 'new property "%s" not a string'%propname
1245 elif isinstance(prop, Password):
1246 if not isinstance(value, password.Password):
1247 raise TypeError, 'new property "%s" not a Password'%propname
1248 propvalues[propname] = value
1250 elif value is not None and isinstance(prop, Date):
1251 if not isinstance(value, date.Date):
1252 raise TypeError, 'new property "%s" not a Date'% propname
1253 propvalues[propname] = value
1255 elif value is not None and isinstance(prop, Interval):
1256 if not isinstance(value, date.Interval):
1257 raise TypeError, 'new property "%s" not an '\
1258 'Interval'%propname
1259 propvalues[propname] = value
1261 elif value is not None and isinstance(prop, Number):
1262 try:
1263 float(value)
1264 except ValueError:
1265 raise TypeError, 'new property "%s" not numeric'%propname
1267 elif value is not None and isinstance(prop, Boolean):
1268 try:
1269 int(value)
1270 except ValueError:
1271 raise TypeError, 'new property "%s" not boolean'%propname
1273 node[propname] = value
1275 # nothing to do?
1276 if not propvalues:
1277 return propvalues
1279 # do the set, and journal it
1280 self.db.setnode(self.classname, nodeid, node, multilink_changes)
1282 if self.do_journal:
1283 propvalues.update(journalvalues)
1284 self.db.addjournal(self.classname, nodeid, 'set', propvalues)
1286 self.fireReactors('set', nodeid, oldvalues)
1288 return propvalues
1290 def retire(self, nodeid):
1291 '''Retire a node.
1293 The properties on the node remain available from the get() method,
1294 and the node's id is never reused.
1296 Retired nodes are not returned by the find(), list(), or lookup()
1297 methods, and other nodes may reuse the values of their key properties.
1298 '''
1299 if self.db.journaltag is None:
1300 raise DatabaseError, 'Database open read-only'
1302 cursor = self.db.conn.cursor()
1303 sql = 'update _%s set __retired__=1 where id=?'%self.classname
1304 if __debug__:
1305 print >>hyperdb.DEBUG, 'retire', (self, sql, nodeid)
1306 cursor.execute(sql, (nodeid,))
1308 def is_retired(self, nodeid):
1309 '''Return true if the node is rerired
1310 '''
1311 cursor = self.db.conn.cursor()
1312 sql = 'select __retired__ from _%s where id=?'%self.classname
1313 if __debug__:
1314 print >>hyperdb.DEBUG, 'is_retired', (self, sql, nodeid)
1315 cursor.execute(sql, (nodeid,))
1316 return cursor.fetchone()[0]
1318 def destroy(self, nodeid):
1319 '''Destroy a node.
1321 WARNING: this method should never be used except in extremely rare
1322 situations where there could never be links to the node being
1323 deleted
1324 WARNING: use retire() instead
1325 WARNING: the properties of this node will not be available ever again
1326 WARNING: really, use retire() instead
1328 Well, I think that's enough warnings. This method exists mostly to
1329 support the session storage of the cgi interface.
1331 The node is completely removed from the hyperdb, including all journal
1332 entries. It will no longer be available, and will generally break code
1333 if there are any references to the node.
1334 '''
1335 if self.db.journaltag is None:
1336 raise DatabaseError, 'Database open read-only'
1337 self.db.destroynode(self.classname, nodeid)
1339 def history(self, nodeid):
1340 '''Retrieve the journal of edits on a particular node.
1342 'nodeid' must be the id of an existing node of this class or an
1343 IndexError is raised.
1345 The returned list contains tuples of the form
1347 (date, tag, action, params)
1349 'date' is a Timestamp object specifying the time of the change and
1350 'tag' is the journaltag specified when the database was opened.
1351 '''
1352 if not self.do_journal:
1353 raise ValueError, 'Journalling is disabled for this class'
1354 return self.db.getjournal(self.classname, nodeid)
1356 # Locating nodes:
1357 def hasnode(self, nodeid):
1358 '''Determine if the given nodeid actually exists
1359 '''
1360 return self.db.hasnode(self.classname, nodeid)
1362 def setkey(self, propname):
1363 '''Select a String property of this class to be the key property.
1365 'propname' must be the name of a String property of this class or
1366 None, or a TypeError is raised. The values of the key property on
1367 all existing nodes must be unique or a ValueError is raised.
1368 '''
1369 # XXX create an index on the key prop column
1370 prop = self.getprops()[propname]
1371 if not isinstance(prop, String):
1372 raise TypeError, 'key properties must be String'
1373 self.key = propname
1375 def getkey(self):
1376 '''Return the name of the key property for this class or None.'''
1377 return self.key
1379 def labelprop(self, default_to_id=0):
1380 ''' Return the property name for a label for the given node.
1382 This method attempts to generate a consistent label for the node.
1383 It tries the following in order:
1384 1. key property
1385 2. "name" property
1386 3. "title" property
1387 4. first property from the sorted property name list
1388 '''
1389 k = self.getkey()
1390 if k:
1391 return k
1392 props = self.getprops()
1393 if props.has_key('name'):
1394 return 'name'
1395 elif props.has_key('title'):
1396 return 'title'
1397 if default_to_id:
1398 return 'id'
1399 props = props.keys()
1400 props.sort()
1401 return props[0]
1403 def lookup(self, keyvalue):
1404 '''Locate a particular node by its key property and return its id.
1406 If this class has no key property, a TypeError is raised. If the
1407 'keyvalue' matches one of the values for the key property among
1408 the nodes in this class, the matching node's id is returned;
1409 otherwise a KeyError is raised.
1410 '''
1411 if not self.key:
1412 raise TypeError, 'No key property set for class %s'%self.classname
1414 cursor = self.db.conn.cursor()
1415 sql = 'select id from _%s where _%s=?'%(self.classname, self.key)
1416 if __debug__:
1417 print >>hyperdb.DEBUG, 'lookup', (self, sql, keyvalue)
1418 cursor.execute(sql, (keyvalue,))
1420 # see if there was a result
1421 l = cursor.fetchall()
1422 if not l:
1423 raise KeyError, keyvalue
1425 # return the id
1426 return l[0][0]
1428 def find(self, **propspec):
1429 '''Get the ids of nodes in this class which link to the given nodes.
1431 'propspec' consists of keyword args propname={nodeid:1,}
1432 'propname' must be the name of a property in this class, or a
1433 KeyError is raised. That property must be a Link or Multilink
1434 property, or a TypeError is raised.
1436 Any node in this class whose 'propname' property links to any of the
1437 nodeids will be returned. Used by the full text indexing, which knows
1438 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1439 issues:
1441 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1442 '''
1443 if __debug__:
1444 print >>hyperdb.DEBUG, 'find', (self, propspec)
1445 if not propspec:
1446 return []
1447 queries = []
1448 tables = []
1449 allvalues = ()
1450 for prop, values in propspec.items():
1451 allvalues += tuple(values.keys())
1452 tables.append('select nodeid from %s_%s where linkid in (%s)'%(
1453 self.classname, prop, ','.join(['?' for x in values.keys()])))
1454 sql = '\nintersect\n'.join(tables)
1455 if __debug__:
1456 print >>hyperdb.DEBUG, 'find', (self, sql, allvalues)
1457 cursor = self.db.conn.cursor()
1458 cursor.execute(sql, allvalues)
1459 try:
1460 l = [x[0] for x in cursor.fetchall()]
1461 except gadfly.database.error, message:
1462 if message == 'no more results':
1463 l = []
1464 raise
1465 if __debug__:
1466 print >>hyperdb.DEBUG, 'find ... ', l
1467 return l
1469 def list(self):
1470 ''' Return a list of the ids of the active nodes in this class.
1471 '''
1472 return self.db.getnodeids(self.classname, retired=0)
1474 def filter(self, search_matches, filterspec, sort, group):
1475 ''' Return a list of the ids of the active nodes in this class that
1476 match the 'filter' spec, sorted by the group spec and then the
1477 sort spec
1479 "filterspec" is {propname: value(s)}
1480 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1481 and prop is a prop name or None
1482 "search_matches" is {nodeid: marker}
1483 '''
1484 cn = self.classname
1486 # figure the WHERE clause from the filterspec
1487 props = self.getprops()
1488 frum = ['_'+cn]
1489 where = []
1490 args = []
1491 for k, v in filterspec.items():
1492 propclass = props[k]
1493 if isinstance(propclass, Multilink):
1494 tn = '%s_%s'%(cn, k)
1495 frum.append(tn)
1496 if isinstance(v, type([])):
1497 s = ','.join(['?' for x in v])
1498 where.append('id=%s.nodeid and %s.linkid in (%s)'%(tn,tn,s))
1499 args = args + v
1500 else:
1501 where.append('id=%s.nodeid and %s.linkid = ?'%(tn, tn))
1502 args.append(v)
1503 else:
1504 if isinstance(v, type([])):
1505 s = ','.join(['?' for x in v])
1506 where.append('_%s in (%s)'%(k, s))
1507 args = args + v
1508 else:
1509 where.append('_%s=?'%k)
1510 args.append(v)
1512 # add results of full text search
1513 if search_matches is not None:
1514 v = search_matches.keys()
1515 s = ','.join(['?' for x in v])
1516 where.append('id in (%s)'%s)
1517 args = args + v
1519 # figure the order by clause
1520 orderby = []
1521 ordercols = []
1522 if sort[0] is not None and sort[1] is not None:
1523 if sort[0] != '-':
1524 orderby.append('_'+sort[1])
1525 ordercols.append(sort[1])
1526 else:
1527 orderby.append('_'+sort[1]+' desc')
1528 ordercols.append(sort[1])
1530 # figure the group by clause
1531 groupby = []
1532 groupcols = []
1533 if group[0] is not None and group[1] is not None:
1534 if group[0] != '-':
1535 groupby.append('_'+group[1])
1536 groupcols.append(group[1])
1537 else:
1538 groupby.append('_'+group[1]+' desc')
1539 groupcols.append(group[1])
1541 # construct the SQL
1542 frum = ','.join(frum)
1543 where = ' and '.join(where)
1544 cols = ['id']
1545 if orderby:
1546 cols = cols + ordercols
1547 order = ' order by %s'%(','.join(orderby))
1548 else:
1549 order = ''
1550 if groupby:
1551 cols = cols + groupcols
1552 group = ' group by %s'%(','.join(groupby))
1553 else:
1554 group = ''
1555 cols = ','.join(cols)
1556 sql = 'select %s from %s where %s%s%s'%(cols, frum, where, order,
1557 group)
1558 args = tuple(args)
1559 if __debug__:
1560 print >>hyperdb.DEBUG, 'find', (self, sql, args)
1561 cursor = self.db.conn.cursor()
1562 cursor.execute(sql, args)
1564 def count(self):
1565 '''Get the number of nodes in this class.
1567 If the returned integer is 'numnodes', the ids of all the nodes
1568 in this class run from 1 to numnodes, and numnodes+1 will be the
1569 id of the next node to be created in this class.
1570 '''
1571 return self.db.countnodes(self.classname)
1573 # Manipulating properties:
1574 def getprops(self, protected=1):
1575 '''Return a dictionary mapping property names to property objects.
1576 If the "protected" flag is true, we include protected properties -
1577 those which may not be modified.
1578 '''
1579 d = self.properties.copy()
1580 if protected:
1581 d['id'] = String()
1582 d['creation'] = hyperdb.Date()
1583 d['activity'] = hyperdb.Date()
1584 d['creator'] = hyperdb.Link("user")
1585 return d
1587 def addprop(self, **properties):
1588 '''Add properties to this class.
1590 The keyword arguments in 'properties' must map names to property
1591 objects, or a TypeError is raised. None of the keys in 'properties'
1592 may collide with the names of existing properties, or a ValueError
1593 is raised before any properties have been added.
1594 '''
1595 for key in properties.keys():
1596 if self.properties.has_key(key):
1597 raise ValueError, key
1598 self.properties.update(properties)
1600 def index(self, nodeid):
1601 '''Add (or refresh) the node to search indexes
1602 '''
1603 # find all the String properties that have indexme
1604 for prop, propclass in self.getprops().items():
1605 if isinstance(propclass, String) and propclass.indexme:
1606 try:
1607 value = str(self.get(nodeid, prop))
1608 except IndexError:
1609 # node no longer exists - entry should be removed
1610 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1611 else:
1612 # and index them under (classname, nodeid, property)
1613 self.db.indexer.add_text((self.classname, nodeid, prop),
1614 value)
1617 #
1618 # Detector interface
1619 #
1620 def audit(self, event, detector):
1621 '''Register a detector
1622 '''
1623 l = self.auditors[event]
1624 if detector not in l:
1625 self.auditors[event].append(detector)
1627 def fireAuditors(self, action, nodeid, newvalues):
1628 '''Fire all registered auditors.
1629 '''
1630 for audit in self.auditors[action]:
1631 audit(self.db, self, nodeid, newvalues)
1633 def react(self, event, detector):
1634 '''Register a detector
1635 '''
1636 l = self.reactors[event]
1637 if detector not in l:
1638 self.reactors[event].append(detector)
1640 def fireReactors(self, action, nodeid, oldvalues):
1641 '''Fire all registered reactors.
1642 '''
1643 for react in self.reactors[action]:
1644 react(self.db, self, nodeid, oldvalues)
1646 class FileClass(Class):
1647 '''This class defines a large chunk of data. To support this, it has a
1648 mandatory String property "content" which is typically saved off
1649 externally to the hyperdb.
1651 The default MIME type of this data is defined by the
1652 "default_mime_type" class attribute, which may be overridden by each
1653 node if the class defines a "type" String property.
1654 '''
1655 default_mime_type = 'text/plain'
1657 def create(self, **propvalues):
1658 ''' snaffle the file propvalue and store in a file
1659 '''
1660 content = propvalues['content']
1661 del propvalues['content']
1662 newid = Class.create(self, **propvalues)
1663 self.db.storefile(self.classname, newid, None, content)
1664 return newid
1666 def import_list(self, propnames, proplist):
1667 ''' Trap the "content" property...
1668 '''
1669 # dupe this list so we don't affect others
1670 propnames = propnames[:]
1672 # extract the "content" property from the proplist
1673 i = propnames.index('content')
1674 content = proplist[i]
1675 del propnames[i]
1676 del proplist[i]
1678 # do the normal import
1679 newid = Class.import_list(self, propnames, proplist)
1681 # save off the "content" file
1682 self.db.storefile(self.classname, newid, None, content)
1683 return newid
1685 _marker = []
1686 def get(self, nodeid, propname, default=_marker, cache=1):
1687 ''' trap the content propname and get it from the file
1688 '''
1690 poss_msg = 'Possibly a access right configuration problem.'
1691 if propname == 'content':
1692 try:
1693 return self.db.getfile(self.classname, nodeid, None)
1694 except IOError, (strerror):
1695 # BUG: by catching this we donot see an error in the log.
1696 return 'ERROR reading file: %s%s\n%s\n%s'%(
1697 self.classname, nodeid, poss_msg, strerror)
1698 if default is not self._marker:
1699 return Class.get(self, nodeid, propname, default, cache=cache)
1700 else:
1701 return Class.get(self, nodeid, propname, cache=cache)
1703 def getprops(self, protected=1):
1704 ''' In addition to the actual properties on the node, these methods
1705 provide the "content" property. If the "protected" flag is true,
1706 we include protected properties - those which may not be
1707 modified.
1708 '''
1709 d = Class.getprops(self, protected=protected).copy()
1710 if protected:
1711 d['content'] = hyperdb.String()
1712 return d
1714 def index(self, nodeid):
1715 ''' Index the node in the search index.
1717 We want to index the content in addition to the normal String
1718 property indexing.
1719 '''
1720 # perform normal indexing
1721 Class.index(self, nodeid)
1723 # get the content to index
1724 content = self.get(nodeid, 'content')
1726 # figure the mime type
1727 if self.properties.has_key('type'):
1728 mime_type = self.get(nodeid, 'type')
1729 else:
1730 mime_type = self.default_mime_type
1732 # and index!
1733 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
1734 mime_type)
1736 # XXX deviation from spec - was called ItemClass
1737 class IssueClass(Class, roundupdb.IssueClass):
1738 # Overridden methods:
1739 def __init__(self, db, classname, **properties):
1740 '''The newly-created class automatically includes the "messages",
1741 "files", "nosy", and "superseder" properties. If the 'properties'
1742 dictionary attempts to specify any of these properties or a
1743 "creation" or "activity" property, a ValueError is raised.
1744 '''
1745 if not properties.has_key('title'):
1746 properties['title'] = hyperdb.String(indexme='yes')
1747 if not properties.has_key('messages'):
1748 properties['messages'] = hyperdb.Multilink("msg")
1749 if not properties.has_key('files'):
1750 properties['files'] = hyperdb.Multilink("file")
1751 if not properties.has_key('nosy'):
1752 # note: journalling is turned off as it really just wastes
1753 # space. this behaviour may be overridden in an instance
1754 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
1755 if not properties.has_key('superseder'):
1756 properties['superseder'] = hyperdb.Multilink(classname)
1757 Class.__init__(self, db, classname, **properties)