1 # $Id: back_gadfly.py,v 1.18 2002-09-12 07:23:23 richard Exp $
2 __doc__ = '''
3 About Gadfly
4 ============
6 Gadfly is a collection of python modules that provides relational
7 database functionality entirely implemented in Python. It supports a
8 subset of the intergalactic standard RDBMS Structured Query Language
9 SQL.
12 Basic Structure
13 ===============
15 We map roundup classes to relational tables. Automatically detect schema
16 changes and modify the gadfly table schemas appropriately. Multilinks
17 (which represent a many-to-many relationship) are handled through
18 intermediate tables.
20 Journals are stored adjunct to the per-class tables.
22 Table names and columns have "_" prepended so the names can't
23 clash with restricted names (like "order"). Retirement is determined by the
24 __retired__ column being true.
26 All columns are defined as VARCHAR, since it really doesn't matter what
27 type they're defined as. We stuff all kinds of data in there ;) [as long as
28 it's marshallable, gadfly doesn't care]
31 Additional Instance Requirements
32 ================================
34 The instance configuration must specify where the database is. It does this
35 with GADFLY_DATABASE, which is used as the arguments to the gadfly.gadfly()
36 method:
38 Using an on-disk database directly (not a good idea):
39 GADFLY_DATABASE = (database name, directory)
41 Using a network database (much better idea):
42 GADFLY_DATABASE = (policy, password, address, port)
44 Because multiple accesses directly to a gadfly database aren't handled, but
45 multiple network accesses are, it's strongly advised that the latter setup be
46 used.
48 '''
50 # standard python modules
51 import sys, os, time, re, errno, weakref, copy
53 # roundup modules
54 from roundup import hyperdb, date, password, roundupdb, security
55 from roundup.hyperdb import String, Password, Date, Interval, Link, \
56 Multilink, DatabaseError, Boolean, Number
58 # the all-important gadfly :)
59 import gadfly
60 import gadfly.client
61 import gadfly.database
63 # support
64 from blobfiles import FileStorage
65 from roundup.indexer import Indexer
66 from sessions import Sessions
68 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
69 # flag to set on retired entries
70 RETIRED_FLAG = '__hyperdb_retired'
72 def __init__(self, config, journaltag=None):
73 ''' Open the database and load the schema from it.
74 '''
75 self.config, self.journaltag = config, journaltag
76 self.dir = config.DATABASE
77 self.classes = {}
78 self.indexer = Indexer(self.dir)
79 self.sessions = Sessions(self.config)
80 self.security = security.Security(self)
82 # additional transaction support for external files and the like
83 self.transactions = []
85 db = getattr(config, 'GADFLY_DATABASE', ('database', self.dir))
86 if len(db) == 2:
87 # ensure files are group readable and writable
88 os.umask(0002)
89 try:
90 self.conn = gadfly.gadfly(*db)
91 except IOError, error:
92 if error.errno != errno.ENOENT:
93 raise
94 self.database_schema = {}
95 self.conn = gadfly.gadfly()
96 self.conn.startup(*db)
97 cursor = self.conn.cursor()
98 cursor.execute('create table schema (schema varchar)')
99 cursor.execute('create table ids (name varchar, num integer)')
100 else:
101 cursor = self.conn.cursor()
102 cursor.execute('select schema from schema')
103 self.database_schema = cursor.fetchone()[0]
104 else:
105 self.conn = gadfly.client.gfclient(*db)
106 cursor = self.conn.cursor()
107 cursor.execute('select schema from schema')
108 self.database_schema = cursor.fetchone()[0]
110 def __repr__(self):
111 return '<roundfly 0x%x>'%id(self)
113 def post_init(self):
114 ''' Called once the schema initialisation has finished.
116 We should now confirm that the schema defined by our "classes"
117 attribute actually matches the schema in the database.
118 '''
119 # now detect changes in the schema
120 for classname, spec in self.classes.items():
121 if self.database_schema.has_key(classname):
122 dbspec = self.database_schema[classname]
123 self.update_class(spec, dbspec)
124 self.database_schema[classname] = spec.schema()
125 else:
126 self.create_class(spec)
127 self.database_schema[classname] = spec.schema()
129 for classname in self.database_schema.keys():
130 if not self.classes.has_key(classname):
131 self.drop_class(classname)
133 # update the database version of the schema
134 cursor = self.conn.cursor()
135 cursor.execute('delete from schema')
136 cursor.execute('insert into schema values (?)', (self.database_schema,))
138 # reindex the db if necessary
139 if self.indexer.should_reindex():
140 self.reindex()
142 # commit
143 self.conn.commit()
145 def reindex(self):
146 for klass in self.classes.values():
147 for nodeid in klass.list():
148 klass.index(nodeid)
149 self.indexer.save_index()
151 def determine_columns(self, spec):
152 ''' Figure the column names and multilink properties from the spec
153 '''
154 cols = []
155 mls = []
156 # add the multilinks separately
157 for col, prop in spec.properties.items():
158 if isinstance(prop, Multilink):
159 mls.append(col)
160 else:
161 cols.append('_'+col)
162 cols.sort()
163 return cols, mls
165 def update_class(self, spec, dbspec):
166 ''' Determine the differences between the current spec and the
167 database version of the spec, and update where necessary
169 NOTE that this doesn't work for adding/deleting properties!
170 ... until gadfly grows an ALTER TABLE command, it's not going to!
171 '''
172 spec_schema = spec.schema()
173 if spec_schema == dbspec:
174 return
175 if __debug__:
176 print >>hyperdb.DEBUG, 'update_class FIRING'
178 # key property changed?
179 if dbspec[0] != spec_schema[0]:
180 if __debug__:
181 print >>hyperdb.DEBUG, 'update_class setting keyprop', `spec[0]`
182 # XXX turn on indexing for the key property
184 # dict 'em up
185 spec_propnames,spec_props = [],{}
186 for propname,prop in spec_schema[1]:
187 spec_propnames.append(propname)
188 spec_props[propname] = prop
189 dbspec_propnames,dbspec_props = [],{}
190 for propname,prop in dbspec[1]:
191 dbspec_propnames.append(propname)
192 dbspec_props[propname] = prop
194 # we're going to need one of these
195 cursor = self.conn.cursor()
197 # now compare
198 for propname in spec_propnames:
199 prop = spec_props[propname]
200 if __debug__:
201 print >>hyperdb.DEBUG, 'update_class ...', `prop`
202 if dbspec_props.has_key(propname) and prop==dbspec_props[propname]:
203 continue
204 if __debug__:
205 print >>hyperdb.DEBUG, 'update_class', `prop`
207 if not dbspec_props.has_key(propname):
208 # add the property
209 if isinstance(prop, Multilink):
210 sql = 'create table %s_%s (linkid varchar, nodeid '\
211 'varchar)'%(spec.classname, prop)
212 if __debug__:
213 print >>hyperdb.DEBUG, 'update_class', (self, sql)
214 cursor.execute(sql)
215 else:
216 # XXX gadfly doesn't have an ALTER TABLE command
217 raise NotImplementedError
218 sql = 'alter table _%s add column (_%s varchar)'%(
219 spec.classname, propname)
220 if __debug__:
221 print >>hyperdb.DEBUG, 'update_class', (self, sql)
222 cursor.execute(sql)
223 else:
224 # modify the property
225 if __debug__:
226 print >>hyperdb.DEBUG, 'update_class NOOP'
227 pass # NOOP in gadfly
229 # and the other way - only worry about deletions here
230 for propname in dbspec_propnames:
231 prop = dbspec_props[propname]
232 if spec_props.has_key(propname):
233 continue
234 if __debug__:
235 print >>hyperdb.DEBUG, 'update_class', `prop`
237 # delete the property
238 if isinstance(prop, Multilink):
239 sql = 'drop table %s_%s'%(spec.classname, prop)
240 if __debug__:
241 print >>hyperdb.DEBUG, 'update_class', (self, sql)
242 cursor.execute(sql)
243 else:
244 # XXX gadfly doesn't have an ALTER TABLE command
245 raise NotImplementedError
246 sql = 'alter table _%s delete column _%s'%(spec.classname,
247 propname)
248 if __debug__:
249 print >>hyperdb.DEBUG, 'update_class', (self, sql)
250 cursor.execute(sql)
252 def create_class(self, spec):
253 ''' Create a database table according to the given spec.
254 '''
255 cols, mls = self.determine_columns(spec)
257 # add on our special columns
258 cols.append('id')
259 cols.append('__retired__')
261 cursor = self.conn.cursor()
263 # create the base table
264 cols = ','.join(['%s varchar'%x for x in cols])
265 sql = 'create table _%s (%s)'%(spec.classname, cols)
266 if __debug__:
267 print >>hyperdb.DEBUG, 'create_class', (self, sql)
268 cursor.execute(sql)
270 # journal table
271 cols = ','.join(['%s varchar'%x
272 for x in 'nodeid date tag action params'.split()])
273 sql = 'create table %s__journal (%s)'%(spec.classname, cols)
274 if __debug__:
275 print >>hyperdb.DEBUG, 'create_class', (self, sql)
276 cursor.execute(sql)
278 # now create the multilink tables
279 for ml in mls:
280 sql = 'create table %s_%s (linkid varchar, nodeid varchar)'%(
281 spec.classname, ml)
282 if __debug__:
283 print >>hyperdb.DEBUG, 'create_class', (self, sql)
284 cursor.execute(sql)
286 # ID counter
287 sql = 'insert into ids (name, num) values (?,?)'
288 vals = (spec.classname, 1)
289 if __debug__:
290 print >>hyperdb.DEBUG, 'create_class', (self, sql, vals)
291 cursor.execute(sql, vals)
293 def drop_class(self, spec):
294 ''' Drop the given table from the database.
296 Drop the journal and multilink tables too.
297 '''
298 # figure the multilinks
299 mls = []
300 for col, prop in spec.properties.items():
301 if isinstance(prop, Multilink):
302 mls.append(col)
303 cursor = self.conn.cursor()
305 sql = 'drop table _%s'%spec.classname
306 if __debug__:
307 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
308 cursor.execute(sql)
310 sql = 'drop table %s__journal'%spec.classname
311 if __debug__:
312 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
313 cursor.execute(sql)
315 for ml in mls:
316 sql = 'drop table %s_%s'%(spec.classname, ml)
317 if __debug__:
318 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
319 cursor.execute(sql)
321 #
322 # Classes
323 #
324 def __getattr__(self, classname):
325 ''' A convenient way of calling self.getclass(classname).
326 '''
327 if self.classes.has_key(classname):
328 if __debug__:
329 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
330 return self.classes[classname]
331 raise AttributeError, classname
333 def addclass(self, cl):
334 ''' Add a Class to the hyperdatabase.
335 '''
336 if __debug__:
337 print >>hyperdb.DEBUG, 'addclass', (self, cl)
338 cn = cl.classname
339 if self.classes.has_key(cn):
340 raise ValueError, cn
341 self.classes[cn] = cl
343 def getclasses(self):
344 ''' Return a list of the names of all existing classes.
345 '''
346 if __debug__:
347 print >>hyperdb.DEBUG, 'getclasses', (self,)
348 l = self.classes.keys()
349 l.sort()
350 return l
352 def getclass(self, classname):
353 '''Get the Class object representing a particular class.
355 If 'classname' is not a valid class name, a KeyError is raised.
356 '''
357 if __debug__:
358 print >>hyperdb.DEBUG, 'getclass', (self, classname)
359 return self.classes[classname]
361 def clear(self):
362 ''' Delete all database contents.
364 Note: I don't commit here, which is different behaviour to the
365 "nuke from orbit" behaviour in the *dbms.
366 '''
367 if __debug__:
368 print >>hyperdb.DEBUG, 'clear', (self,)
369 cursor = self.conn.cursor()
370 for cn in self.classes.keys():
371 sql = 'delete from _%s'%cn
372 if __debug__:
373 print >>hyperdb.DEBUG, 'clear', (self, sql)
374 cursor.execute(sql)
376 #
377 # Node IDs
378 #
379 def newid(self, classname):
380 ''' Generate a new id for the given class
381 '''
382 # get the next ID
383 cursor = self.conn.cursor()
384 sql = 'select num from ids where name=?'
385 if __debug__:
386 print >>hyperdb.DEBUG, 'newid', (self, sql, classname)
387 cursor.execute(sql, (classname, ))
388 newid = cursor.fetchone()[0]
390 # update the counter
391 sql = 'update ids set num=? where name=?'
392 vals = (newid+1, classname)
393 if __debug__:
394 print >>hyperdb.DEBUG, 'newid', (self, sql, vals)
395 cursor.execute(sql, vals)
397 # return as string
398 return str(newid)
400 def setid(self, classname, setid):
401 ''' Set the id counter: used during import of database
402 '''
403 cursor = self.conn.cursor()
404 sql = 'update ids set num=? where name=?'
405 vals = (setid, spec.classname)
406 if __debug__:
407 print >>hyperdb.DEBUG, 'setid', (self, sql, vals)
408 cursor.execute(sql, vals)
410 #
411 # Nodes
412 #
414 def addnode(self, classname, nodeid, node):
415 ''' Add the specified node to its class's db.
416 '''
417 if __debug__:
418 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
419 # gadfly requires values for all non-multilink columns
420 cl = self.classes[classname]
421 cols, mls = self.determine_columns(cl)
423 # default the non-multilink columns
424 for col, prop in cl.properties.items():
425 if not isinstance(col, Multilink):
426 if not node.has_key(col):
427 node[col] = None
429 node = self.serialise(classname, node)
431 # make sure the ordering is correct for column name -> column value
432 vals = tuple([node[col[1:]] for col in cols]) + (nodeid, 0)
433 s = ','.join(['?' for x in cols]) + ',?,?'
434 cols = ','.join(cols) + ',id,__retired__'
436 # perform the inserts
437 cursor = self.conn.cursor()
438 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
439 if __debug__:
440 print >>hyperdb.DEBUG, 'addnode', (self, sql, vals)
441 cursor.execute(sql, vals)
443 # insert the multilink rows
444 for col in mls:
445 t = '%s_%s'%(classname, col)
446 for entry in node[col]:
447 sql = 'insert into %s (linkid, nodeid) values (?,?)'%t
448 vals = (entry, nodeid)
449 if __debug__:
450 print >>hyperdb.DEBUG, 'addnode', (self, sql, vals)
451 cursor.execute(sql, vals)
453 # make sure we do the commit-time extra stuff for this node
454 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
456 def setnode(self, classname, nodeid, node, multilink_changes):
457 ''' Change the specified node.
458 '''
459 if __debug__:
460 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
461 node = self.serialise(classname, node)
463 cl = self.classes[classname]
464 cols = []
465 mls = []
466 # add the multilinks separately
467 for col in node.keys():
468 prop = cl.properties[col]
469 if isinstance(prop, Multilink):
470 mls.append(col)
471 else:
472 cols.append('_'+col)
473 cols.sort()
475 # make sure the ordering is correct for column name -> column value
476 vals = tuple([node[col[1:]] for col in cols])
477 s = ','.join(['%s=?'%x for x in cols])
478 cols = ','.join(cols)
480 # perform the update
481 cursor = self.conn.cursor()
482 sql = 'update _%s set %s'%(classname, s)
483 if __debug__:
484 print >>hyperdb.DEBUG, 'setnode', (self, sql, vals)
485 cursor.execute(sql, vals)
487 # now the fun bit, updating the multilinks ;)
488 for col, (add, remove) in multilink_changes.items():
489 tn = '%s_%s'%(classname, col)
490 if add:
491 sql = 'insert into %s (nodeid, linkid) values (?,?)'%tn
492 vals = [(nodeid, addid) for addid in add]
493 if __debug__:
494 print >>hyperdb.DEBUG, 'setnode (add)', (self, sql, vals)
495 cursor.execute(sql, vals)
496 if remove:
497 sql = 'delete from %s where nodeid=? and linkid=?'%tn
498 vals = [(nodeid, removeid) for removeid in remove]
499 if __debug__:
500 print >>hyperdb.DEBUG, 'setnode (rem)', (self, sql, vals)
501 cursor.execute(sql, vals)
503 # make sure we do the commit-time extra stuff for this node
504 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
506 def getnode(self, classname, nodeid):
507 ''' Get a node from the database.
508 '''
509 if __debug__:
510 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid)
511 # figure the columns we're fetching
512 cl = self.classes[classname]
513 cols, mls = self.determine_columns(cl)
514 scols = ','.join(cols)
516 # perform the basic property fetch
517 cursor = self.conn.cursor()
518 sql = 'select %s from _%s where id=?'%(scols, classname)
519 if __debug__:
520 print >>hyperdb.DEBUG, 'getnode', (self, sql, nodeid)
521 cursor.execute(sql, (nodeid,))
522 try:
523 values = cursor.fetchone()
524 except gadfly.database.error, message:
525 if message == 'no more results':
526 raise IndexError, 'no such %s node %s'%(classname, nodeid)
527 raise
529 # make up the node
530 node = {}
531 for col in range(len(cols)):
532 node[cols[col][1:]] = values[col]
534 # now the multilinks
535 for col in mls:
536 # get the link ids
537 sql = 'select linkid from %s_%s where nodeid=?'%(classname, col)
538 if __debug__:
539 print >>hyperdb.DEBUG, 'getnode', (self, sql, nodeid)
540 cursor.execute(sql, (nodeid,))
541 # extract the first column from the result
542 node[col] = [x[0] for x in cursor.fetchall()]
544 return self.unserialise(classname, node)
546 def destroynode(self, classname, nodeid):
547 '''Remove a node from the database. Called exclusively by the
548 destroy() method on Class.
549 '''
550 if __debug__:
551 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
553 # make sure the node exists
554 if not self.hasnode(classname, nodeid):
555 raise IndexError, '%s has no node %s'%(classname, nodeid)
557 # see if there's any obvious commit actions that we should get rid of
558 for entry in self.transactions[:]:
559 if entry[1][:2] == (classname, nodeid):
560 self.transactions.remove(entry)
562 # now do the SQL
563 cursor = self.conn.cursor()
564 sql = 'delete from _%s where id=?'%(classname)
565 if __debug__:
566 print >>hyperdb.DEBUG, 'destroynode', (self, sql, nodeid)
567 cursor.execute(sql, (nodeid,))
569 def serialise(self, classname, node):
570 '''Copy the node contents, converting non-marshallable data into
571 marshallable data.
572 '''
573 if __debug__:
574 print >>hyperdb.DEBUG, 'serialise', classname, node
575 properties = self.getclass(classname).getprops()
576 d = {}
577 for k, v in node.items():
578 # if the property doesn't exist, or is the "retired" flag then
579 # it won't be in the properties dict
580 if not properties.has_key(k):
581 d[k] = v
582 continue
584 # get the property spec
585 prop = properties[k]
587 if isinstance(prop, Password):
588 d[k] = str(v)
589 elif isinstance(prop, Date) and v is not None:
590 d[k] = v.serialise()
591 elif isinstance(prop, Interval) and v is not None:
592 d[k] = v.serialise()
593 else:
594 d[k] = v
595 return d
597 def unserialise(self, classname, node):
598 '''Decode the marshalled node data
599 '''
600 if __debug__:
601 print >>hyperdb.DEBUG, 'unserialise', classname, node
602 properties = self.getclass(classname).getprops()
603 d = {}
604 for k, v in node.items():
605 # if the property doesn't exist, or is the "retired" flag then
606 # it won't be in the properties dict
607 if not properties.has_key(k):
608 d[k] = v
609 continue
611 # get the property spec
612 prop = properties[k]
614 if isinstance(prop, Date) and v is not None:
615 d[k] = date.Date(v)
616 elif isinstance(prop, Interval) and v is not None:
617 d[k] = date.Interval(v)
618 elif isinstance(prop, Password):
619 p = password.Password()
620 p.unpack(v)
621 d[k] = p
622 else:
623 d[k] = v
624 return d
626 def hasnode(self, classname, nodeid):
627 ''' Determine if the database has a given node.
628 '''
629 cursor = self.conn.cursor()
630 sql = 'select count(*) from _%s where id=?'%classname
631 if __debug__:
632 print >>hyperdb.DEBUG, 'hasnode', (self, sql, nodeid)
633 cursor.execute(sql, (nodeid,))
634 return cursor.fetchone()[0]
636 def countnodes(self, classname):
637 ''' Count the number of nodes that exist for a particular Class.
638 '''
639 cursor = self.conn.cursor()
640 sql = 'select count(*) from _%s'%classname
641 if __debug__:
642 print >>hyperdb.DEBUG, 'countnodes', (self, sql)
643 cursor.execute(sql)
644 return cursor.fetchone()[0]
646 def getnodeids(self, classname, retired=0):
647 ''' Retrieve all the ids of the nodes for a particular Class.
649 Set retired=None to get all nodes. Otherwise it'll get all the
650 retired or non-retired nodes, depending on the flag.
651 '''
652 cursor = self.conn.cursor()
653 # flip the sense of the flag if we don't want all of them
654 if retired is not None:
655 retired = not retired
656 sql = 'select id from _%s where __retired__ <> ?'%classname
657 if __debug__:
658 print >>hyperdb.DEBUG, 'getnodeids', (self, sql, retired)
659 cursor.execute(sql, (retired,))
660 return [x[0] for x in cursor.fetchall()]
662 def addjournal(self, classname, nodeid, action, params):
663 ''' Journal the Action
664 'action' may be:
666 'create' or 'set' -- 'params' is a dictionary of property values
667 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
668 'retire' -- 'params' is None
669 '''
670 if isinstance(params, type({})):
671 if params.has_key('creator'):
672 journaltag = self.user.get(params['creator'], 'username')
673 del params['creator']
674 else:
675 journaltag = self.journaltag
676 if params.has_key('created'):
677 journaldate = params['created'].serialise()
678 del params['created']
679 else:
680 journaldate = date.Date().serialise()
681 if params.has_key('activity'):
682 del params['activity']
684 # serialise the parameters now
685 if action in ('set', 'create'):
686 params = self.serialise(classname, params)
687 else:
688 journaltag = self.journaltag
689 journaldate = date.Date().serialise()
691 # create the journal entry
692 cols = ','.join('nodeid date tag action params'.split())
693 entry = (nodeid, journaldate, journaltag, action, params)
695 if __debug__:
696 print >>hyperdb.DEBUG, 'doSaveJournal', entry
698 # do the insert
699 cursor = self.conn.cursor()
700 sql = 'insert into %s__journal (%s) values (?,?,?,?,?)'%(classname,
701 cols)
702 if __debug__:
703 print >>hyperdb.DEBUG, 'addjournal', (self, sql, entry)
704 cursor.execute(sql, entry)
706 def getjournal(self, classname, nodeid):
707 ''' get the journal for id
708 '''
709 # make sure the node exists
710 if not self.hasnode(classname, nodeid):
711 raise IndexError, '%s has no node %s'%(classname, nodeid)
713 # now get the journal entries
714 cols = ','.join('nodeid date tag action params'.split())
715 cursor = self.conn.cursor()
716 sql = 'select %s from %s__journal where nodeid=?'%(cols, classname)
717 if __debug__:
718 print >>hyperdb.DEBUG, 'getjournal', (self, sql, nodeid)
719 cursor.execute(sql, (nodeid,))
720 res = []
721 for nodeid, date_stamp, user, action, params in cursor.fetchall():
722 res.append((nodeid, date.Date(date_stamp), user, action, params))
723 return res
725 def pack(self, pack_before):
726 ''' Delete all journal entries except "create" before 'pack_before'.
727 '''
728 # get a 'yyyymmddhhmmss' version of the date
729 date_stamp = pack_before.serialise()
731 # do the delete
732 cursor = self.conn.cursor()
733 for classname in self.classes.keys():
734 sql = "delete from %s__journal where date<? and "\
735 "action<>'create'"%classname
736 if __debug__:
737 print >>hyperdb.DEBUG, 'pack', (self, sql, date_stamp)
738 cursor.execute(sql, (date_stamp,))
740 def commit(self):
741 ''' Commit the current transactions.
743 Save all data changed since the database was opened or since the
744 last commit() or rollback().
745 '''
746 if __debug__:
747 print >>hyperdb.DEBUG, 'commit', (self,)
749 # commit gadfly
750 self.conn.commit()
752 # now, do all the other transaction stuff
753 reindex = {}
754 for method, args in self.transactions:
755 reindex[method(*args)] = 1
757 # reindex the nodes that request it
758 for classname, nodeid in filter(None, reindex.keys()):
759 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
760 self.getclass(classname).index(nodeid)
762 # save the indexer state
763 self.indexer.save_index()
765 # clear out the transactions
766 self.transactions = []
768 def rollback(self):
769 ''' Reverse all actions from the current transaction.
771 Undo all the changes made since the database was opened or the last
772 commit() or rollback() was performed.
773 '''
774 if __debug__:
775 print >>hyperdb.DEBUG, 'rollback', (self,)
777 # roll back gadfly
778 self.conn.rollback()
780 # roll back "other" transaction stuff
781 for method, args in self.transactions:
782 # delete temporary files
783 if method == self.doStoreFile:
784 self.rollbackStoreFile(*args)
785 self.transactions = []
787 def doSaveNode(self, classname, nodeid, node):
788 ''' dummy that just generates a reindex event
789 '''
790 # return the classname, nodeid so we reindex this content
791 return (classname, nodeid)
793 def close(self):
794 ''' Close off the connection.
795 '''
796 self.conn.close()
798 #
799 # The base Class class
800 #
801 class Class(hyperdb.Class):
802 ''' The handle to a particular class of nodes in a hyperdatabase.
804 All methods except __repr__ and getnode must be implemented by a
805 concrete backend Class.
806 '''
808 def __init__(self, db, classname, **properties):
809 '''Create a new class with a given name and property specification.
811 'classname' must not collide with the name of an existing class,
812 or a ValueError is raised. The keyword arguments in 'properties'
813 must map names to property objects, or a TypeError is raised.
814 '''
815 if (properties.has_key('creation') or properties.has_key('activity')
816 or properties.has_key('creator')):
817 raise ValueError, '"creation", "activity" and "creator" are '\
818 'reserved'
820 self.classname = classname
821 self.properties = properties
822 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
823 self.key = ''
825 # should we journal changes (default yes)
826 self.do_journal = 1
828 # do the db-related init stuff
829 db.addclass(self)
831 self.auditors = {'create': [], 'set': [], 'retire': []}
832 self.reactors = {'create': [], 'set': [], 'retire': []}
834 def schema(self):
835 ''' A dumpable version of the schema that we can store in the
836 database
837 '''
838 return (self.key, [(x, repr(y)) for x,y in self.properties.items()])
840 def enableJournalling(self):
841 '''Turn journalling on for this class
842 '''
843 self.do_journal = 1
845 def disableJournalling(self):
846 '''Turn journalling off for this class
847 '''
848 self.do_journal = 0
850 # Editing nodes:
851 def create(self, **propvalues):
852 ''' Create a new node of this class and return its id.
854 The keyword arguments in 'propvalues' map property names to values.
856 The values of arguments must be acceptable for the types of their
857 corresponding properties or a TypeError is raised.
859 If this class has a key property, it must be present and its value
860 must not collide with other key strings or a ValueError is raised.
862 Any other properties on this class that are missing from the
863 'propvalues' dictionary are set to None.
865 If an id in a link or multilink property does not refer to a valid
866 node, an IndexError is raised.
867 '''
868 if propvalues.has_key('id'):
869 raise KeyError, '"id" is reserved'
871 if self.db.journaltag is None:
872 raise DatabaseError, 'Database open read-only'
874 if propvalues.has_key('creation') or propvalues.has_key('activity'):
875 raise KeyError, '"creation" and "activity" are reserved'
877 self.fireAuditors('create', None, propvalues)
879 # new node's id
880 newid = self.db.newid(self.classname)
882 # validate propvalues
883 num_re = re.compile('^\d+$')
884 for key, value in propvalues.items():
885 if key == self.key:
886 try:
887 self.lookup(value)
888 except KeyError:
889 pass
890 else:
891 raise ValueError, 'node with key "%s" exists'%value
893 # try to handle this property
894 try:
895 prop = self.properties[key]
896 except KeyError:
897 raise KeyError, '"%s" has no property "%s"'%(self.classname,
898 key)
900 if value is not None and isinstance(prop, Link):
901 if type(value) != type(''):
902 raise ValueError, 'link value must be String'
903 link_class = self.properties[key].classname
904 # if it isn't a number, it's a key
905 if not num_re.match(value):
906 try:
907 value = self.db.classes[link_class].lookup(value)
908 except (TypeError, KeyError):
909 raise IndexError, 'new property "%s": %s not a %s'%(
910 key, value, link_class)
911 elif not self.db.getclass(link_class).hasnode(value):
912 raise IndexError, '%s has no node %s'%(link_class, value)
914 # save off the value
915 propvalues[key] = value
917 # register the link with the newly linked node
918 if self.do_journal and self.properties[key].do_journal:
919 self.db.addjournal(link_class, value, 'link',
920 (self.classname, newid, key))
922 elif isinstance(prop, Multilink):
923 if type(value) != type([]):
924 raise TypeError, 'new property "%s" not a list of ids'%key
926 # clean up and validate the list of links
927 link_class = self.properties[key].classname
928 l = []
929 for entry in value:
930 if type(entry) != type(''):
931 raise ValueError, '"%s" multilink value (%r) '\
932 'must contain Strings'%(key, value)
933 # if it isn't a number, it's a key
934 if not num_re.match(entry):
935 try:
936 entry = self.db.classes[link_class].lookup(entry)
937 except (TypeError, KeyError):
938 raise IndexError, 'new property "%s": %s not a %s'%(
939 key, entry, self.properties[key].classname)
940 l.append(entry)
941 value = l
942 propvalues[key] = value
944 # handle additions
945 for nodeid in value:
946 if not self.db.getclass(link_class).hasnode(nodeid):
947 raise IndexError, '%s has no node %s'%(link_class,
948 nodeid)
949 # register the link with the newly linked node
950 if self.do_journal and self.properties[key].do_journal:
951 self.db.addjournal(link_class, nodeid, 'link',
952 (self.classname, newid, key))
954 elif isinstance(prop, String):
955 if type(value) != type(''):
956 raise TypeError, 'new property "%s" not a string'%key
958 elif isinstance(prop, Password):
959 if not isinstance(value, password.Password):
960 raise TypeError, 'new property "%s" not a Password'%key
962 elif isinstance(prop, Date):
963 if value is not None and not isinstance(value, date.Date):
964 raise TypeError, 'new property "%s" not a Date'%key
966 elif isinstance(prop, Interval):
967 if value is not None and not isinstance(value, date.Interval):
968 raise TypeError, 'new property "%s" not an Interval'%key
970 elif value is not None and isinstance(prop, Number):
971 try:
972 float(value)
973 except ValueError:
974 raise TypeError, 'new property "%s" not numeric'%key
976 elif value is not None and isinstance(prop, Boolean):
977 try:
978 int(value)
979 except ValueError:
980 raise TypeError, 'new property "%s" not boolean'%key
982 # make sure there's data where there needs to be
983 for key, prop in self.properties.items():
984 if propvalues.has_key(key):
985 continue
986 if key == self.key:
987 raise ValueError, 'key property "%s" is required'%key
988 if isinstance(prop, Multilink):
989 propvalues[key] = []
990 else:
991 propvalues[key] = None
993 # done
994 self.db.addnode(self.classname, newid, propvalues)
995 if self.do_journal:
996 self.db.addjournal(self.classname, newid, 'create', propvalues)
998 self.fireReactors('create', newid, None)
1000 return newid
1002 _marker = []
1003 def get(self, nodeid, propname, default=_marker, cache=1):
1004 '''Get the value of a property on an existing node of this class.
1006 'nodeid' must be the id of an existing node of this class or an
1007 IndexError is raised. 'propname' must be the name of a property
1008 of this class or a KeyError is raised.
1010 'cache' indicates whether the transaction cache should be queried
1011 for the node. If the node has been modified and you need to
1012 determine what its values prior to modification are, you need to
1013 set cache=0.
1014 '''
1015 if propname == 'id':
1016 return nodeid
1018 if propname == 'creation':
1019 if not self.do_journal:
1020 raise ValueError, 'Journalling is disabled for this class'
1021 journal = self.db.getjournal(self.classname, nodeid)
1022 if journal:
1023 return self.db.getjournal(self.classname, nodeid)[0][1]
1024 else:
1025 # on the strange chance that there's no journal
1026 return date.Date()
1027 if propname == 'activity':
1028 if not self.do_journal:
1029 raise ValueError, 'Journalling is disabled for this class'
1030 journal = self.db.getjournal(self.classname, nodeid)
1031 if journal:
1032 return self.db.getjournal(self.classname, nodeid)[-1][1]
1033 else:
1034 # on the strange chance that there's no journal
1035 return date.Date()
1036 if propname == 'creator':
1037 if not self.do_journal:
1038 raise ValueError, 'Journalling is disabled for this class'
1039 journal = self.db.getjournal(self.classname, nodeid)
1040 if journal:
1041 name = self.db.getjournal(self.classname, nodeid)[0][2]
1042 else:
1043 return None
1044 try:
1045 return self.db.user.lookup(name)
1046 except KeyError:
1047 # the journaltag user doesn't exist any more
1048 return None
1050 # get the property (raises KeyErorr if invalid)
1051 prop = self.properties[propname]
1053 # get the node's dict
1054 d = self.db.getnode(self.classname, nodeid) #, cache=cache)
1056 if not d.has_key(propname):
1057 if default is self._marker:
1058 if isinstance(prop, Multilink):
1059 return []
1060 else:
1061 return None
1062 else:
1063 return default
1065 # don't pass our list to other code
1066 if isinstance(prop, Multilink):
1067 return d[propname][:]
1069 return d[propname]
1071 def getnode(self, nodeid, cache=1):
1072 ''' Return a convenience wrapper for the node.
1074 'nodeid' must be the id of an existing node of this class or an
1075 IndexError is raised.
1077 'cache' indicates whether the transaction cache should be queried
1078 for the node. If the node has been modified and you need to
1079 determine what its values prior to modification are, you need to
1080 set cache=0.
1081 '''
1082 return Node(self, nodeid, cache=cache)
1084 def set(self, nodeid, **propvalues):
1085 '''Modify a property on an existing node of this class.
1087 'nodeid' must be the id of an existing node of this class or an
1088 IndexError is raised.
1090 Each key in 'propvalues' must be the name of a property of this
1091 class or a KeyError is raised.
1093 All values in 'propvalues' must be acceptable types for their
1094 corresponding properties or a TypeError is raised.
1096 If the value of the key property is set, it must not collide with
1097 other key strings or a ValueError is raised.
1099 If the value of a Link or Multilink property contains an invalid
1100 node id, a ValueError is raised.
1101 '''
1102 if not propvalues:
1103 return propvalues
1105 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1106 raise KeyError, '"creation" and "activity" are reserved'
1108 if propvalues.has_key('id'):
1109 raise KeyError, '"id" is reserved'
1111 if self.db.journaltag is None:
1112 raise DatabaseError, 'Database open read-only'
1114 self.fireAuditors('set', nodeid, propvalues)
1115 # Take a copy of the node dict so that the subsequent set
1116 # operation doesn't modify the oldvalues structure.
1117 # XXX used to try the cache here first
1118 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1120 node = self.db.getnode(self.classname, nodeid)
1121 if self.is_retired(nodeid):
1122 raise IndexError
1123 num_re = re.compile('^\d+$')
1125 # if the journal value is to be different, store it in here
1126 journalvalues = {}
1128 # remember the add/remove stuff for multilinks, making it easier
1129 # for the Database layer to do its stuff
1130 multilink_changes = {}
1132 for propname, value in propvalues.items():
1133 # check to make sure we're not duplicating an existing key
1134 if propname == self.key and node[propname] != value:
1135 try:
1136 self.lookup(value)
1137 except KeyError:
1138 pass
1139 else:
1140 raise ValueError, 'node with key "%s" exists'%value
1142 # this will raise the KeyError if the property isn't valid
1143 # ... we don't use getprops() here because we only care about
1144 # the writeable properties.
1145 prop = self.properties[propname]
1147 # if the value's the same as the existing value, no sense in
1148 # doing anything
1149 if node.has_key(propname) and value == node[propname]:
1150 del propvalues[propname]
1151 continue
1153 # do stuff based on the prop type
1154 if isinstance(prop, Link):
1155 link_class = prop.classname
1156 # if it isn't a number, it's a key
1157 if value is not None and not isinstance(value, type('')):
1158 raise ValueError, 'property "%s" link value be a string'%(
1159 propname)
1160 if isinstance(value, type('')) and not num_re.match(value):
1161 try:
1162 value = self.db.classes[link_class].lookup(value)
1163 except (TypeError, KeyError):
1164 raise IndexError, 'new property "%s": %s not a %s'%(
1165 propname, value, prop.classname)
1167 if (value is not None and
1168 not self.db.getclass(link_class).hasnode(value)):
1169 raise IndexError, '%s has no node %s'%(link_class, value)
1171 if self.do_journal and prop.do_journal:
1172 # register the unlink with the old linked node
1173 if node[propname] is not None:
1174 self.db.addjournal(link_class, node[propname], 'unlink',
1175 (self.classname, nodeid, propname))
1177 # register the link with the newly linked node
1178 if value is not None:
1179 self.db.addjournal(link_class, value, 'link',
1180 (self.classname, nodeid, propname))
1182 elif isinstance(prop, Multilink):
1183 if type(value) != type([]):
1184 raise TypeError, 'new property "%s" not a list of'\
1185 ' ids'%propname
1186 link_class = self.properties[propname].classname
1187 l = []
1188 for entry in value:
1189 # if it isn't a number, it's a key
1190 if type(entry) != type(''):
1191 raise ValueError, 'new property "%s" link value ' \
1192 'must be a string'%propname
1193 if not num_re.match(entry):
1194 try:
1195 entry = self.db.classes[link_class].lookup(entry)
1196 except (TypeError, KeyError):
1197 raise IndexError, 'new property "%s": %s not a %s'%(
1198 propname, entry,
1199 self.properties[propname].classname)
1200 l.append(entry)
1201 value = l
1202 propvalues[propname] = value
1204 # figure the journal entry for this property
1205 add = []
1206 remove = []
1208 # handle removals
1209 if node.has_key(propname):
1210 l = node[propname]
1211 else:
1212 l = []
1213 for id in l[:]:
1214 if id in value:
1215 continue
1216 # register the unlink with the old linked node
1217 if self.do_journal and self.properties[propname].do_journal:
1218 self.db.addjournal(link_class, id, 'unlink',
1219 (self.classname, nodeid, propname))
1220 l.remove(id)
1221 remove.append(id)
1223 # handle additions
1224 for id in value:
1225 if not self.db.getclass(link_class).hasnode(id):
1226 raise IndexError, '%s has no node %s'%(link_class, id)
1227 if id in l:
1228 continue
1229 # register the link with the newly linked node
1230 if self.do_journal and self.properties[propname].do_journal:
1231 self.db.addjournal(link_class, id, 'link',
1232 (self.classname, nodeid, propname))
1233 l.append(id)
1234 add.append(id)
1236 # figure the journal entry
1237 l = []
1238 if add:
1239 l.append(('+', add))
1240 if remove:
1241 l.append(('-', remove))
1242 multilink_changes[propname] = (add, remove)
1243 if l:
1244 journalvalues[propname] = tuple(l)
1246 elif isinstance(prop, String):
1247 if value is not None and type(value) != type(''):
1248 raise TypeError, 'new property "%s" not a string'%propname
1250 elif isinstance(prop, Password):
1251 if not isinstance(value, password.Password):
1252 raise TypeError, 'new property "%s" not a Password'%propname
1253 propvalues[propname] = value
1255 elif value is not None and isinstance(prop, Date):
1256 if not isinstance(value, date.Date):
1257 raise TypeError, 'new property "%s" not a Date'% propname
1258 propvalues[propname] = value
1260 elif value is not None and isinstance(prop, Interval):
1261 if not isinstance(value, date.Interval):
1262 raise TypeError, 'new property "%s" not an '\
1263 'Interval'%propname
1264 propvalues[propname] = value
1266 elif value is not None and isinstance(prop, Number):
1267 try:
1268 float(value)
1269 except ValueError:
1270 raise TypeError, 'new property "%s" not numeric'%propname
1272 elif value is not None and isinstance(prop, Boolean):
1273 try:
1274 int(value)
1275 except ValueError:
1276 raise TypeError, 'new property "%s" not boolean'%propname
1278 node[propname] = value
1280 # nothing to do?
1281 if not propvalues:
1282 return propvalues
1284 # do the set, and journal it
1285 self.db.setnode(self.classname, nodeid, node, multilink_changes)
1287 if self.do_journal:
1288 propvalues.update(journalvalues)
1289 self.db.addjournal(self.classname, nodeid, 'set', propvalues)
1291 self.fireReactors('set', nodeid, oldvalues)
1293 return propvalues
1295 def retire(self, nodeid):
1296 '''Retire a node.
1298 The properties on the node remain available from the get() method,
1299 and the node's id is never reused.
1301 Retired nodes are not returned by the find(), list(), or lookup()
1302 methods, and other nodes may reuse the values of their key properties.
1303 '''
1304 if self.db.journaltag is None:
1305 raise DatabaseError, 'Database open read-only'
1307 cursor = self.db.conn.cursor()
1308 sql = 'update _%s set __retired__=1 where id=?'%self.classname
1309 if __debug__:
1310 print >>hyperdb.DEBUG, 'retire', (self, sql, nodeid)
1311 cursor.execute(sql, (nodeid,))
1313 def is_retired(self, nodeid):
1314 '''Return true if the node is rerired
1315 '''
1316 cursor = self.db.conn.cursor()
1317 sql = 'select __retired__ from _%s where id=?'%self.classname
1318 if __debug__:
1319 print >>hyperdb.DEBUG, 'is_retired', (self, sql, nodeid)
1320 cursor.execute(sql, (nodeid,))
1321 return cursor.fetchone()[0]
1323 def destroy(self, nodeid):
1324 '''Destroy a node.
1326 WARNING: this method should never be used except in extremely rare
1327 situations where there could never be links to the node being
1328 deleted
1329 WARNING: use retire() instead
1330 WARNING: the properties of this node will not be available ever again
1331 WARNING: really, use retire() instead
1333 Well, I think that's enough warnings. This method exists mostly to
1334 support the session storage of the cgi interface.
1336 The node is completely removed from the hyperdb, including all journal
1337 entries. It will no longer be available, and will generally break code
1338 if there are any references to the node.
1339 '''
1340 if self.db.journaltag is None:
1341 raise DatabaseError, 'Database open read-only'
1342 self.db.destroynode(self.classname, nodeid)
1344 def history(self, nodeid):
1345 '''Retrieve the journal of edits on a particular node.
1347 'nodeid' must be the id of an existing node of this class or an
1348 IndexError is raised.
1350 The returned list contains tuples of the form
1352 (date, tag, action, params)
1354 'date' is a Timestamp object specifying the time of the change and
1355 'tag' is the journaltag specified when the database was opened.
1356 '''
1357 if not self.do_journal:
1358 raise ValueError, 'Journalling is disabled for this class'
1359 return self.db.getjournal(self.classname, nodeid)
1361 # Locating nodes:
1362 def hasnode(self, nodeid):
1363 '''Determine if the given nodeid actually exists
1364 '''
1365 return self.db.hasnode(self.classname, nodeid)
1367 def setkey(self, propname):
1368 '''Select a String property of this class to be the key property.
1370 'propname' must be the name of a String property of this class or
1371 None, or a TypeError is raised. The values of the key property on
1372 all existing nodes must be unique or a ValueError is raised.
1373 '''
1374 # XXX create an index on the key prop column
1375 prop = self.getprops()[propname]
1376 if not isinstance(prop, String):
1377 raise TypeError, 'key properties must be String'
1378 self.key = propname
1380 def getkey(self):
1381 '''Return the name of the key property for this class or None.'''
1382 return self.key
1384 def labelprop(self, default_to_id=0):
1385 ''' Return the property name for a label for the given node.
1387 This method attempts to generate a consistent label for the node.
1388 It tries the following in order:
1389 1. key property
1390 2. "name" property
1391 3. "title" property
1392 4. first property from the sorted property name list
1393 '''
1394 k = self.getkey()
1395 if k:
1396 return k
1397 props = self.getprops()
1398 if props.has_key('name'):
1399 return 'name'
1400 elif props.has_key('title'):
1401 return 'title'
1402 if default_to_id:
1403 return 'id'
1404 props = props.keys()
1405 props.sort()
1406 return props[0]
1408 def lookup(self, keyvalue):
1409 '''Locate a particular node by its key property and return its id.
1411 If this class has no key property, a TypeError is raised. If the
1412 'keyvalue' matches one of the values for the key property among
1413 the nodes in this class, the matching node's id is returned;
1414 otherwise a KeyError is raised.
1415 '''
1416 if not self.key:
1417 raise TypeError, 'No key property set for class %s'%self.classname
1419 cursor = self.db.conn.cursor()
1420 sql = 'select id from _%s where _%s=?'%(self.classname, self.key)
1421 if __debug__:
1422 print >>hyperdb.DEBUG, 'lookup', (self, sql, keyvalue)
1423 cursor.execute(sql, (keyvalue,))
1425 # see if there was a result
1426 l = cursor.fetchall()
1427 if not l:
1428 raise KeyError, keyvalue
1430 # return the id
1431 return l[0][0]
1433 def find(self, **propspec):
1434 '''Get the ids of nodes in this class which link to the given nodes.
1436 'propspec' consists of keyword args propname={nodeid:1,}
1437 'propname' must be the name of a property in this class, or a
1438 KeyError is raised. That property must be a Link or Multilink
1439 property, or a TypeError is raised.
1441 Any node in this class whose 'propname' property links to any of the
1442 nodeids will be returned. Used by the full text indexing, which knows
1443 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1444 issues:
1446 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1447 '''
1448 if __debug__:
1449 print >>hyperdb.DEBUG, 'find', (self, propspec)
1450 if not propspec:
1451 return []
1452 queries = []
1453 tables = []
1454 allvalues = ()
1455 for prop, values in propspec.items():
1456 allvalues += tuple(values.keys())
1457 tables.append('select nodeid from %s_%s where linkid in (%s)'%(
1458 self.classname, prop, ','.join(['?' for x in values.keys()])))
1459 sql = '\nintersect\n'.join(tables)
1460 if __debug__:
1461 print >>hyperdb.DEBUG, 'find', (self, sql, allvalues)
1462 cursor = self.db.conn.cursor()
1463 cursor.execute(sql, allvalues)
1464 try:
1465 l = [x[0] for x in cursor.fetchall()]
1466 except gadfly.database.error, message:
1467 if message == 'no more results':
1468 l = []
1469 raise
1470 if __debug__:
1471 print >>hyperdb.DEBUG, 'find ... ', l
1472 return l
1474 def list(self):
1475 ''' Return a list of the ids of the active nodes in this class.
1476 '''
1477 return self.db.getnodeids(self.classname, retired=0)
1479 def filter(self, search_matches, filterspec, sort, group):
1480 ''' Return a list of the ids of the active nodes in this class that
1481 match the 'filter' spec, sorted by the group spec and then the
1482 sort spec
1484 "filterspec" is {propname: value(s)}
1485 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1486 and prop is a prop name or None
1487 "search_matches" is {nodeid: marker}
1488 '''
1489 cn = self.classname
1491 # figure the WHERE clause from the filterspec
1492 props = self.getprops()
1493 frum = ['_'+cn]
1494 where = []
1495 args = []
1496 for k, v in filterspec.items():
1497 propclass = props[k]
1498 if isinstance(propclass, Multilink):
1499 tn = '%s_%s'%(cn, k)
1500 frum.append(tn)
1501 if isinstance(v, type([])):
1502 s = ','.join(['?' for x in v])
1503 where.append('id=%s.nodeid and %s.linkid in (%s)'%(tn,tn,s))
1504 args = args + v
1505 else:
1506 where.append('id=%s.nodeid and %s.linkid = ?'%(tn, tn))
1507 args.append(v)
1508 else:
1509 if isinstance(v, type([])):
1510 s = ','.join(['?' for x in v])
1511 where.append('_%s in (%s)'%(k, s))
1512 args = args + v
1513 else:
1514 where.append('_%s=?'%k)
1515 args.append(v)
1517 # add results of full text search
1518 if search_matches is not None:
1519 v = search_matches.keys()
1520 s = ','.join(['?' for x in v])
1521 where.append('id in (%s)'%s)
1522 args = args + v
1524 # figure the order by clause
1525 orderby = []
1526 ordercols = []
1527 if sort[0] is not None and sort[1] is not None:
1528 if sort[0] != '-':
1529 orderby.append('_'+sort[1])
1530 ordercols.append(sort[1])
1531 else:
1532 orderby.append('_'+sort[1]+' desc')
1533 ordercols.append(sort[1])
1535 # figure the group by clause
1536 groupby = []
1537 groupcols = []
1538 if group[0] is not None and group[1] is not None:
1539 if group[0] != '-':
1540 groupby.append('_'+group[1])
1541 groupcols.append(group[1])
1542 else:
1543 groupby.append('_'+group[1]+' desc')
1544 groupcols.append(group[1])
1546 # construct the SQL
1547 frum = ','.join(frum)
1548 where = ' and '.join(where)
1549 cols = ['id']
1550 if orderby:
1551 cols = cols + ordercols
1552 order = ' order by %s'%(','.join(orderby))
1553 else:
1554 order = ''
1555 if groupby:
1556 cols = cols + groupcols
1557 group = ' group by %s'%(','.join(groupby))
1558 else:
1559 group = ''
1560 cols = ','.join(cols)
1561 sql = 'select %s from %s where %s%s%s'%(cols, frum, where, order,
1562 group)
1563 args = tuple(args)
1564 if __debug__:
1565 print >>hyperdb.DEBUG, 'find', (self, sql, args)
1566 cursor = self.db.conn.cursor()
1567 cursor.execute(sql, args)
1569 def count(self):
1570 '''Get the number of nodes in this class.
1572 If the returned integer is 'numnodes', the ids of all the nodes
1573 in this class run from 1 to numnodes, and numnodes+1 will be the
1574 id of the next node to be created in this class.
1575 '''
1576 return self.db.countnodes(self.classname)
1578 # Manipulating properties:
1579 def getprops(self, protected=1):
1580 '''Return a dictionary mapping property names to property objects.
1581 If the "protected" flag is true, we include protected properties -
1582 those which may not be modified.
1583 '''
1584 d = self.properties.copy()
1585 if protected:
1586 d['id'] = String()
1587 d['creation'] = hyperdb.Date()
1588 d['activity'] = hyperdb.Date()
1589 d['creator'] = hyperdb.Link("user")
1590 return d
1592 def addprop(self, **properties):
1593 '''Add properties to this class.
1595 The keyword arguments in 'properties' must map names to property
1596 objects, or a TypeError is raised. None of the keys in 'properties'
1597 may collide with the names of existing properties, or a ValueError
1598 is raised before any properties have been added.
1599 '''
1600 for key in properties.keys():
1601 if self.properties.has_key(key):
1602 raise ValueError, key
1603 self.properties.update(properties)
1605 def index(self, nodeid):
1606 '''Add (or refresh) the node to search indexes
1607 '''
1608 # find all the String properties that have indexme
1609 for prop, propclass in self.getprops().items():
1610 if isinstance(propclass, String) and propclass.indexme:
1611 try:
1612 value = str(self.get(nodeid, prop))
1613 except IndexError:
1614 # node no longer exists - entry should be removed
1615 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1616 else:
1617 # and index them under (classname, nodeid, property)
1618 self.db.indexer.add_text((self.classname, nodeid, prop),
1619 value)
1622 #
1623 # Detector interface
1624 #
1625 def audit(self, event, detector):
1626 '''Register a detector
1627 '''
1628 l = self.auditors[event]
1629 if detector not in l:
1630 self.auditors[event].append(detector)
1632 def fireAuditors(self, action, nodeid, newvalues):
1633 '''Fire all registered auditors.
1634 '''
1635 for audit in self.auditors[action]:
1636 audit(self.db, self, nodeid, newvalues)
1638 def react(self, event, detector):
1639 '''Register a detector
1640 '''
1641 l = self.reactors[event]
1642 if detector not in l:
1643 self.reactors[event].append(detector)
1645 def fireReactors(self, action, nodeid, oldvalues):
1646 '''Fire all registered reactors.
1647 '''
1648 for react in self.reactors[action]:
1649 react(self.db, self, nodeid, oldvalues)
1651 class FileClass(Class):
1652 '''This class defines a large chunk of data. To support this, it has a
1653 mandatory String property "content" which is typically saved off
1654 externally to the hyperdb.
1656 The default MIME type of this data is defined by the
1657 "default_mime_type" class attribute, which may be overridden by each
1658 node if the class defines a "type" String property.
1659 '''
1660 default_mime_type = 'text/plain'
1662 def create(self, **propvalues):
1663 ''' snaffle the file propvalue and store in a file
1664 '''
1665 content = propvalues['content']
1666 del propvalues['content']
1667 newid = Class.create(self, **propvalues)
1668 self.db.storefile(self.classname, newid, None, content)
1669 return newid
1671 def import_list(self, propnames, proplist):
1672 ''' Trap the "content" property...
1673 '''
1674 # dupe this list so we don't affect others
1675 propnames = propnames[:]
1677 # extract the "content" property from the proplist
1678 i = propnames.index('content')
1679 content = proplist[i]
1680 del propnames[i]
1681 del proplist[i]
1683 # do the normal import
1684 newid = Class.import_list(self, propnames, proplist)
1686 # save off the "content" file
1687 self.db.storefile(self.classname, newid, None, content)
1688 return newid
1690 _marker = []
1691 def get(self, nodeid, propname, default=_marker, cache=1):
1692 ''' trap the content propname and get it from the file
1693 '''
1695 poss_msg = 'Possibly a access right configuration problem.'
1696 if propname == 'content':
1697 try:
1698 return self.db.getfile(self.classname, nodeid, None)
1699 except IOError, (strerror):
1700 # BUG: by catching this we donot see an error in the log.
1701 return 'ERROR reading file: %s%s\n%s\n%s'%(
1702 self.classname, nodeid, poss_msg, strerror)
1703 if default is not self._marker:
1704 return Class.get(self, nodeid, propname, default, cache=cache)
1705 else:
1706 return Class.get(self, nodeid, propname, cache=cache)
1708 def getprops(self, protected=1):
1709 ''' In addition to the actual properties on the node, these methods
1710 provide the "content" property. If the "protected" flag is true,
1711 we include protected properties - those which may not be
1712 modified.
1713 '''
1714 d = Class.getprops(self, protected=protected).copy()
1715 if protected:
1716 d['content'] = hyperdb.String()
1717 return d
1719 def index(self, nodeid):
1720 ''' Index the node in the search index.
1722 We want to index the content in addition to the normal String
1723 property indexing.
1724 '''
1725 # perform normal indexing
1726 Class.index(self, nodeid)
1728 # get the content to index
1729 content = self.get(nodeid, 'content')
1731 # figure the mime type
1732 if self.properties.has_key('type'):
1733 mime_type = self.get(nodeid, 'type')
1734 else:
1735 mime_type = self.default_mime_type
1737 # and index!
1738 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
1739 mime_type)
1741 # XXX deviation from spec - was called ItemClass
1742 class IssueClass(Class, roundupdb.IssueClass):
1743 # Overridden methods:
1744 def __init__(self, db, classname, **properties):
1745 '''The newly-created class automatically includes the "messages",
1746 "files", "nosy", and "superseder" properties. If the 'properties'
1747 dictionary attempts to specify any of these properties or a
1748 "creation" or "activity" property, a ValueError is raised.
1749 '''
1750 if not properties.has_key('title'):
1751 properties['title'] = hyperdb.String(indexme='yes')
1752 if not properties.has_key('messages'):
1753 properties['messages'] = hyperdb.Multilink("msg")
1754 if not properties.has_key('files'):
1755 properties['files'] = hyperdb.Multilink("file")
1756 if not properties.has_key('nosy'):
1757 # note: journalling is turned off as it really just wastes
1758 # space. this behaviour may be overridden in an instance
1759 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
1760 if not properties.has_key('superseder'):
1761 properties['superseder'] = hyperdb.Multilink(classname)
1762 Class.__init__(self, db, classname, **properties)