d9a108181169bd0978d403933bf6e00bd2527a68
1 # $Id: back_gadfly.py,v 1.21 2002-09-16 08:04:46 richard Exp $
2 __doc__ = '''
3 About Gadfly
4 ============
6 Gadfly is a collection of python modules that provides relational
7 database functionality entirely implemented in Python. It supports a
8 subset of the intergalactic standard RDBMS Structured Query Language
9 SQL.
12 Basic Structure
13 ===============
15 We map roundup classes to relational tables. Automatically detect schema
16 changes and modify the gadfly table schemas appropriately. Multilinks
17 (which represent a many-to-many relationship) are handled through
18 intermediate tables.
20 Journals are stored adjunct to the per-class tables.
22 Table names and columns have "_" prepended so the names can't
23 clash with restricted names (like "order"). Retirement is determined by the
24 __retired__ column being true.
26 All columns are defined as VARCHAR, since it really doesn't matter what
27 type they're defined as. We stuff all kinds of data in there ;) [as long as
28 it's marshallable, gadfly doesn't care]
31 Additional Instance Requirements
32 ================================
34 The instance configuration must specify where the database is. It does this
35 with GADFLY_DATABASE, which is used as the arguments to the gadfly.gadfly()
36 method:
38 Using an on-disk database directly (not a good idea):
39 GADFLY_DATABASE = (database name, directory)
41 Using a network database (much better idea):
42 GADFLY_DATABASE = (policy, password, address, port)
44 Because multiple accesses directly to a gadfly database aren't handled, but
45 multiple network accesses are, it's strongly advised that the latter setup be
46 used.
48 '''
50 # standard python modules
51 import sys, os, time, re, errno, weakref, copy
53 # roundup modules
54 from roundup import hyperdb, date, password, roundupdb, security
55 from roundup.hyperdb import String, Password, Date, Interval, Link, \
56 Multilink, DatabaseError, Boolean, Number
58 # the all-important gadfly :)
59 import gadfly
60 import gadfly.client
61 import gadfly.database
63 # support
64 from blobfiles import FileStorage
65 from roundup.indexer import Indexer
66 from sessions import Sessions
68 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
69 # flag to set on retired entries
70 RETIRED_FLAG = '__hyperdb_retired'
72 def __init__(self, config, journaltag=None):
73 ''' Open the database and load the schema from it.
74 '''
75 self.config, self.journaltag = config, journaltag
76 self.dir = config.DATABASE
77 self.classes = {}
78 self.indexer = Indexer(self.dir)
79 self.sessions = Sessions(self.config)
80 self.security = security.Security(self)
82 # additional transaction support for external files and the like
83 self.transactions = []
85 db = getattr(config, 'GADFLY_DATABASE', ('database', self.dir))
86 if len(db) == 2:
87 # ensure files are group readable and writable
88 os.umask(0002)
89 try:
90 self.conn = gadfly.gadfly(*db)
91 except IOError, error:
92 if error.errno != errno.ENOENT:
93 raise
94 self.database_schema = {}
95 self.conn = gadfly.gadfly()
96 self.conn.startup(*db)
97 cursor = self.conn.cursor()
98 cursor.execute('create table schema (schema varchar)')
99 cursor.execute('create table ids (name varchar, num integer)')
100 else:
101 cursor = self.conn.cursor()
102 cursor.execute('select schema from schema')
103 self.database_schema = cursor.fetchone()[0]
104 else:
105 self.conn = gadfly.client.gfclient(*db)
106 cursor = self.conn.cursor()
107 cursor.execute('select schema from schema')
108 self.database_schema = cursor.fetchone()[0]
110 def __repr__(self):
111 return '<roundfly 0x%x>'%id(self)
113 def post_init(self):
114 ''' Called once the schema initialisation has finished.
116 We should now confirm that the schema defined by our "classes"
117 attribute actually matches the schema in the database.
118 '''
119 # now detect changes in the schema
120 for classname, spec in self.classes.items():
121 if self.database_schema.has_key(classname):
122 dbspec = self.database_schema[classname]
123 self.update_class(spec, dbspec)
124 self.database_schema[classname] = spec.schema()
125 else:
126 self.create_class(spec)
127 self.database_schema[classname] = spec.schema()
129 for classname in self.database_schema.keys():
130 if not self.classes.has_key(classname):
131 self.drop_class(classname)
133 # update the database version of the schema
134 cursor = self.conn.cursor()
135 cursor.execute('delete from schema')
136 cursor.execute('insert into schema values (?)', (self.database_schema,))
138 # reindex the db if necessary
139 if self.indexer.should_reindex():
140 self.reindex()
142 # commit
143 self.conn.commit()
145 def reindex(self):
146 for klass in self.classes.values():
147 for nodeid in klass.list():
148 klass.index(nodeid)
149 self.indexer.save_index()
151 def determine_columns(self, properties):
152 ''' Figure the column names and multilink properties from the spec
154 "properties" is a list of (name, prop) where prop may be an
155 instance of a hyperdb "type" _or_ a string repr of that type.
156 '''
157 cols = []
158 mls = []
159 # add the multilinks separately
160 for col, prop in properties:
161 if isinstance(prop, Multilink):
162 mls.append(col)
163 elif isinstance(prop, type('')) and prop.find('Multilink') != -1:
164 mls.append(col)
165 else:
166 cols.append('_'+col)
167 cols.sort()
168 return cols, mls
170 def update_class(self, spec, dbspec):
171 ''' Determine the differences between the current spec and the
172 database version of the spec, and update where necessary
173 '''
174 spec_schema = spec.schema()
175 if spec_schema == dbspec:
176 return
177 if __debug__:
178 print >>hyperdb.DEBUG, 'update_class FIRING'
180 # key property changed?
181 if dbspec[0] != spec_schema[0]:
182 if __debug__:
183 print >>hyperdb.DEBUG, 'update_class setting keyprop', `spec[0]`
184 # XXX turn on indexing for the key property
186 # dict 'em up
187 spec_propnames,spec_props = [],{}
188 for propname,prop in spec_schema[1]:
189 spec_propnames.append(propname)
190 spec_props[propname] = prop
191 dbspec_propnames,dbspec_props = [],{}
192 for propname,prop in dbspec[1]:
193 dbspec_propnames.append(propname)
194 dbspec_props[propname] = prop
196 # we're going to need one of these
197 cursor = self.conn.cursor()
199 # now compare
200 for propname in spec_propnames:
201 prop = spec_props[propname]
202 if dbspec_props.has_key(propname) and prop==dbspec_props[propname]:
203 continue
204 if __debug__:
205 print >>hyperdb.DEBUG, 'update_class ADD', (propname, prop)
207 if not dbspec_props.has_key(propname):
208 # add the property
209 if isinstance(prop, Multilink):
210 # all we have to do here is create a new table, easy!
211 self.create_multilink_table(cursor, spec, propname)
212 continue
214 # no ALTER TABLE, so we:
215 # 1. pull out the data, including an extra None column
216 oldcols, x = self.determine_columns(dbspec[1])
217 oldcols.append('id')
218 oldcols.append('__retired__')
219 cn = spec.classname
220 sql = 'select %s,? from _%s'%(','.join(oldcols), cn)
221 if __debug__:
222 print >>hyperdb.DEBUG, 'update_class', (self, sql, None)
223 cursor.execute(sql, (None,))
224 olddata = cursor.fetchall()
226 # 2. drop the old table
227 cursor.execute('drop table _%s'%cn)
229 # 3. create the new table
230 cols, mls = self.create_class_table(cursor, spec)
231 # ensure the new column is last
232 cols.remove('_'+propname)
233 assert oldcols == cols, "Column lists don't match!"
234 cols.append('_'+propname)
236 # 4. populate with the data from step one
237 s = ','.join(['?' for x in cols])
238 scols = ','.join(cols)
239 sql = 'insert into _%s (%s) values (%s)'%(cn, scols, s)
241 # GAH, nothing had better go wrong from here on in... but
242 # we have to commit the drop...
243 self.conn.commit()
245 # we're safe to insert now
246 if __debug__:
247 print >>hyperdb.DEBUG, 'update_class', (self, sql, olddata)
248 cursor.execute(sql, olddata)
250 else:
251 # modify the property
252 if __debug__:
253 print >>hyperdb.DEBUG, 'update_class NOOP'
254 pass # NOOP in gadfly
256 # and the other way - only worry about deletions here
257 for propname in dbspec_propnames:
258 prop = dbspec_props[propname]
259 if spec_props.has_key(propname):
260 continue
261 if __debug__:
262 print >>hyperdb.DEBUG, 'update_class REMOVE', `prop`
264 # delete the property
265 if isinstance(prop, Multilink):
266 sql = 'drop table %s_%s'%(spec.classname, prop)
267 if __debug__:
268 print >>hyperdb.DEBUG, 'update_class', (self, sql)
269 cursor.execute(sql)
270 else:
271 # no ALTER TABLE, so we:
272 # 1. pull out the data, excluding the removed column
273 oldcols, x = self.determine_columns(spec.properties.items())
274 oldcols.append('id')
275 oldcols.append('__retired__')
276 # remove the missing column
277 oldcols.remove('_'+propname)
278 cn = spec.classname
279 sql = 'select %s from _%s'%(','.join(oldcols), cn)
280 cursor.execute(sql, (None,))
281 olddata = sql.fetchall()
283 # 2. drop the old table
284 cursor.execute('drop table _%s'%cn)
286 # 3. create the new table
287 cols, mls = self.create_class_table(self, cursor, spec)
288 assert oldcols != cols, "Column lists don't match!"
290 # 4. populate with the data from step one
291 qs = ','.join(['?' for x in cols])
292 sql = 'insert into _%s values (%s)'%(cn, s)
293 cursor.execute(sql, olddata)
295 def create_class_table(self, cursor, spec):
296 ''' create the class table for the given spec
297 '''
298 cols, mls = self.determine_columns(spec.properties.items())
300 # add on our special columns
301 cols.append('id')
302 cols.append('__retired__')
304 # create the base table
305 scols = ','.join(['%s varchar'%x for x in cols])
306 sql = 'create table _%s (%s)'%(spec.classname, scols)
307 if __debug__:
308 print >>hyperdb.DEBUG, 'create_class', (self, sql)
309 cursor.execute(sql)
311 return cols, mls
313 def create_journal_table(self, cursor, spec):
314 ''' create the journal table for a class given the spec and
315 already-determined cols
316 '''
317 # journal table
318 cols = ','.join(['%s varchar'%x
319 for x in 'nodeid date tag action params'.split()])
320 sql = 'create table %s__journal (%s)'%(spec.classname, cols)
321 if __debug__:
322 print >>hyperdb.DEBUG, 'create_class', (self, sql)
323 cursor.execute(sql)
325 def create_multilink_table(self, cursor, spec, ml):
326 ''' Create a multilink table for the "ml" property of the class
327 given by the spec
328 '''
329 sql = 'create table %s_%s (linkid varchar, nodeid varchar)'%(
330 spec.classname, ml)
331 if __debug__:
332 print >>hyperdb.DEBUG, 'create_class', (self, sql)
333 cursor.execute(sql)
335 def create_class(self, spec):
336 ''' Create a database table according to the given spec.
337 '''
338 cursor = self.conn.cursor()
339 cols, mls = self.create_class_table(cursor, spec)
340 self.create_journal_table(cursor, spec)
342 # now create the multilink tables
343 for ml in mls:
344 self.create_multilink_table(cursor, spec, ml)
346 # ID counter
347 sql = 'insert into ids (name, num) values (?,?)'
348 vals = (spec.classname, 1)
349 if __debug__:
350 print >>hyperdb.DEBUG, 'create_class', (self, sql, vals)
351 cursor.execute(sql, vals)
353 def drop_class(self, spec):
354 ''' Drop the given table from the database.
356 Drop the journal and multilink tables too.
357 '''
358 # figure the multilinks
359 mls = []
360 for col, prop in spec.properties.items():
361 if isinstance(prop, Multilink):
362 mls.append(col)
363 cursor = self.conn.cursor()
365 sql = 'drop table _%s'%spec.classname
366 if __debug__:
367 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
368 cursor.execute(sql)
370 sql = 'drop table %s__journal'%spec.classname
371 if __debug__:
372 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
373 cursor.execute(sql)
375 for ml in mls:
376 sql = 'drop table %s_%s'%(spec.classname, ml)
377 if __debug__:
378 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
379 cursor.execute(sql)
381 #
382 # Classes
383 #
384 def __getattr__(self, classname):
385 ''' A convenient way of calling self.getclass(classname).
386 '''
387 if self.classes.has_key(classname):
388 if __debug__:
389 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
390 return self.classes[classname]
391 raise AttributeError, classname
393 def addclass(self, cl):
394 ''' Add a Class to the hyperdatabase.
395 '''
396 if __debug__:
397 print >>hyperdb.DEBUG, 'addclass', (self, cl)
398 cn = cl.classname
399 if self.classes.has_key(cn):
400 raise ValueError, cn
401 self.classes[cn] = cl
403 def getclasses(self):
404 ''' Return a list of the names of all existing classes.
405 '''
406 if __debug__:
407 print >>hyperdb.DEBUG, 'getclasses', (self,)
408 l = self.classes.keys()
409 l.sort()
410 return l
412 def getclass(self, classname):
413 '''Get the Class object representing a particular class.
415 If 'classname' is not a valid class name, a KeyError is raised.
416 '''
417 if __debug__:
418 print >>hyperdb.DEBUG, 'getclass', (self, classname)
419 try:
420 return self.classes[classname]
421 except KeyError:
422 raise KeyError, 'There is no class called "%s"'%classname
424 def clear(self):
425 ''' Delete all database contents.
427 Note: I don't commit here, which is different behaviour to the
428 "nuke from orbit" behaviour in the *dbms.
429 '''
430 if __debug__:
431 print >>hyperdb.DEBUG, 'clear', (self,)
432 cursor = self.conn.cursor()
433 for cn in self.classes.keys():
434 sql = 'delete from _%s'%cn
435 if __debug__:
436 print >>hyperdb.DEBUG, 'clear', (self, sql)
437 cursor.execute(sql)
439 #
440 # Node IDs
441 #
442 def newid(self, classname):
443 ''' Generate a new id for the given class
444 '''
445 # get the next ID
446 cursor = self.conn.cursor()
447 sql = 'select num from ids where name=?'
448 if __debug__:
449 print >>hyperdb.DEBUG, 'newid', (self, sql, classname)
450 cursor.execute(sql, (classname, ))
451 newid = cursor.fetchone()[0]
453 # update the counter
454 sql = 'update ids set num=? where name=?'
455 vals = (newid+1, classname)
456 if __debug__:
457 print >>hyperdb.DEBUG, 'newid', (self, sql, vals)
458 cursor.execute(sql, vals)
460 # return as string
461 return str(newid)
463 def setid(self, classname, setid):
464 ''' Set the id counter: used during import of database
465 '''
466 cursor = self.conn.cursor()
467 sql = 'update ids set num=? where name=?'
468 vals = (setid, spec.classname)
469 if __debug__:
470 print >>hyperdb.DEBUG, 'setid', (self, sql, vals)
471 cursor.execute(sql, vals)
473 #
474 # Nodes
475 #
477 def addnode(self, classname, nodeid, node):
478 ''' Add the specified node to its class's db.
479 '''
480 if __debug__:
481 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
482 # gadfly requires values for all non-multilink columns
483 cl = self.classes[classname]
484 cols, mls = self.determine_columns(cl.properties.items())
486 # default the non-multilink columns
487 for col, prop in cl.properties.items():
488 if not isinstance(col, Multilink):
489 if not node.has_key(col):
490 node[col] = None
492 node = self.serialise(classname, node)
494 # make sure the ordering is correct for column name -> column value
495 vals = tuple([node[col[1:]] for col in cols]) + (nodeid, 0)
496 s = ','.join(['?' for x in cols]) + ',?,?'
497 cols = ','.join(cols) + ',id,__retired__'
499 # perform the inserts
500 cursor = self.conn.cursor()
501 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
502 if __debug__:
503 print >>hyperdb.DEBUG, 'addnode', (self, sql, vals)
504 cursor.execute(sql, vals)
506 # insert the multilink rows
507 for col in mls:
508 t = '%s_%s'%(classname, col)
509 for entry in node[col]:
510 sql = 'insert into %s (linkid, nodeid) values (?,?)'%t
511 vals = (entry, nodeid)
512 if __debug__:
513 print >>hyperdb.DEBUG, 'addnode', (self, sql, vals)
514 cursor.execute(sql, vals)
516 # make sure we do the commit-time extra stuff for this node
517 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
519 def setnode(self, classname, nodeid, node, multilink_changes):
520 ''' Change the specified node.
521 '''
522 if __debug__:
523 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
524 node = self.serialise(classname, node)
526 cl = self.classes[classname]
527 cols = []
528 mls = []
529 # add the multilinks separately
530 for col in node.keys():
531 prop = cl.properties[col]
532 if isinstance(prop, Multilink):
533 mls.append(col)
534 else:
535 cols.append('_'+col)
536 cols.sort()
538 # make sure the ordering is correct for column name -> column value
539 vals = tuple([node[col[1:]] for col in cols])
540 s = ','.join(['%s=?'%x for x in cols])
541 cols = ','.join(cols)
543 # perform the update
544 cursor = self.conn.cursor()
545 sql = 'update _%s set %s'%(classname, s)
546 if __debug__:
547 print >>hyperdb.DEBUG, 'setnode', (self, sql, vals)
548 cursor.execute(sql, vals)
550 # now the fun bit, updating the multilinks ;)
551 for col, (add, remove) in multilink_changes.items():
552 tn = '%s_%s'%(classname, col)
553 if add:
554 sql = 'insert into %s (nodeid, linkid) values (?,?)'%tn
555 vals = [(nodeid, addid) for addid in add]
556 if __debug__:
557 print >>hyperdb.DEBUG, 'setnode (add)', (self, sql, vals)
558 cursor.execute(sql, vals)
559 if remove:
560 sql = 'delete from %s where nodeid=? and linkid=?'%tn
561 vals = [(nodeid, removeid) for removeid in remove]
562 if __debug__:
563 print >>hyperdb.DEBUG, 'setnode (rem)', (self, sql, vals)
564 cursor.execute(sql, vals)
566 # make sure we do the commit-time extra stuff for this node
567 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
569 def getnode(self, classname, nodeid):
570 ''' Get a node from the database.
571 '''
572 if __debug__:
573 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid)
574 # figure the columns we're fetching
575 cl = self.classes[classname]
576 cols, mls = self.determine_columns(cl.properties.items())
577 scols = ','.join(cols)
579 # perform the basic property fetch
580 cursor = self.conn.cursor()
581 sql = 'select %s from _%s where id=?'%(scols, classname)
582 if __debug__:
583 print >>hyperdb.DEBUG, 'getnode', (self, sql, nodeid)
584 cursor.execute(sql, (nodeid,))
585 try:
586 values = cursor.fetchone()
587 except gadfly.database.error, message:
588 if message == 'no more results':
589 raise IndexError, 'no such %s node %s'%(classname, nodeid)
590 raise
592 # make up the node
593 node = {}
594 for col in range(len(cols)):
595 node[cols[col][1:]] = values[col]
597 # now the multilinks
598 for col in mls:
599 # get the link ids
600 sql = 'select linkid from %s_%s where nodeid=?'%(classname, col)
601 if __debug__:
602 print >>hyperdb.DEBUG, 'getnode', (self, sql, nodeid)
603 cursor.execute(sql, (nodeid,))
604 # extract the first column from the result
605 node[col] = [x[0] for x in cursor.fetchall()]
607 return self.unserialise(classname, node)
609 def destroynode(self, classname, nodeid):
610 '''Remove a node from the database. Called exclusively by the
611 destroy() method on Class.
612 '''
613 if __debug__:
614 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
616 # make sure the node exists
617 if not self.hasnode(classname, nodeid):
618 raise IndexError, '%s has no node %s'%(classname, nodeid)
620 # see if there's any obvious commit actions that we should get rid of
621 for entry in self.transactions[:]:
622 if entry[1][:2] == (classname, nodeid):
623 self.transactions.remove(entry)
625 # now do the SQL
626 cursor = self.conn.cursor()
627 sql = 'delete from _%s where id=?'%(classname)
628 if __debug__:
629 print >>hyperdb.DEBUG, 'destroynode', (self, sql, nodeid)
630 cursor.execute(sql, (nodeid,))
632 def serialise(self, classname, node):
633 '''Copy the node contents, converting non-marshallable data into
634 marshallable data.
635 '''
636 if __debug__:
637 print >>hyperdb.DEBUG, 'serialise', classname, node
638 properties = self.getclass(classname).getprops()
639 d = {}
640 for k, v in node.items():
641 # if the property doesn't exist, or is the "retired" flag then
642 # it won't be in the properties dict
643 if not properties.has_key(k):
644 d[k] = v
645 continue
647 # get the property spec
648 prop = properties[k]
650 if isinstance(prop, Password):
651 d[k] = str(v)
652 elif isinstance(prop, Date) and v is not None:
653 d[k] = v.serialise()
654 elif isinstance(prop, Interval) and v is not None:
655 d[k] = v.serialise()
656 else:
657 d[k] = v
658 return d
660 def unserialise(self, classname, node):
661 '''Decode the marshalled node data
662 '''
663 if __debug__:
664 print >>hyperdb.DEBUG, 'unserialise', classname, node
665 properties = self.getclass(classname).getprops()
666 d = {}
667 for k, v in node.items():
668 # if the property doesn't exist, or is the "retired" flag then
669 # it won't be in the properties dict
670 if not properties.has_key(k):
671 d[k] = v
672 continue
674 # get the property spec
675 prop = properties[k]
677 if isinstance(prop, Date) and v is not None:
678 d[k] = date.Date(v)
679 elif isinstance(prop, Interval) and v is not None:
680 d[k] = date.Interval(v)
681 elif isinstance(prop, Password):
682 p = password.Password()
683 p.unpack(v)
684 d[k] = p
685 else:
686 d[k] = v
687 return d
689 def hasnode(self, classname, nodeid):
690 ''' Determine if the database has a given node.
691 '''
692 cursor = self.conn.cursor()
693 sql = 'select count(*) from _%s where id=?'%classname
694 if __debug__:
695 print >>hyperdb.DEBUG, 'hasnode', (self, sql, nodeid)
696 cursor.execute(sql, (nodeid,))
697 return cursor.fetchone()[0]
699 def countnodes(self, classname):
700 ''' Count the number of nodes that exist for a particular Class.
701 '''
702 cursor = self.conn.cursor()
703 sql = 'select count(*) from _%s'%classname
704 if __debug__:
705 print >>hyperdb.DEBUG, 'countnodes', (self, sql)
706 cursor.execute(sql)
707 return cursor.fetchone()[0]
709 def getnodeids(self, classname, retired=0):
710 ''' Retrieve all the ids of the nodes for a particular Class.
712 Set retired=None to get all nodes. Otherwise it'll get all the
713 retired or non-retired nodes, depending on the flag.
714 '''
715 cursor = self.conn.cursor()
716 # flip the sense of the flag if we don't want all of them
717 if retired is not None:
718 retired = not retired
719 sql = 'select id from _%s where __retired__ <> ?'%classname
720 if __debug__:
721 print >>hyperdb.DEBUG, 'getnodeids', (self, sql, retired)
722 cursor.execute(sql, (retired,))
723 return [x[0] for x in cursor.fetchall()]
725 def addjournal(self, classname, nodeid, action, params, creator=None,
726 creation=None):
727 ''' Journal the Action
728 'action' may be:
730 'create' or 'set' -- 'params' is a dictionary of property values
731 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
732 'retire' -- 'params' is None
733 '''
734 # serialise the parameters now if necessary
735 if isinstance(params, type({})):
736 if action in ('set', 'create'):
737 params = self.serialise(classname, params)
739 # handle supply of the special journalling parameters (usually
740 # supplied on importing an existing database)
741 if creator:
742 journaltag = creator
743 else:
744 journaltag = self.journaltag
745 if creation:
746 journaldate = creation.serialise()
747 else:
748 journaldate = date.Date().serialise()
750 # create the journal entry
751 cols = ','.join('nodeid date tag action params'.split())
752 entry = (nodeid, journaldate, journaltag, action, params)
754 if __debug__:
755 print >>hyperdb.DEBUG, 'addjournal', entry
757 # do the insert
758 cursor = self.conn.cursor()
759 sql = 'insert into %s__journal (%s) values (?,?,?,?,?)'%(classname,
760 cols)
761 if __debug__:
762 print >>hyperdb.DEBUG, 'addjournal', (self, sql, entry)
763 cursor.execute(sql, entry)
765 def getjournal(self, classname, nodeid):
766 ''' get the journal for id
767 '''
768 # make sure the node exists
769 if not self.hasnode(classname, nodeid):
770 raise IndexError, '%s has no node %s'%(classname, nodeid)
772 # now get the journal entries
773 cols = ','.join('nodeid date tag action params'.split())
774 cursor = self.conn.cursor()
775 sql = 'select %s from %s__journal where nodeid=?'%(cols, classname)
776 if __debug__:
777 print >>hyperdb.DEBUG, 'getjournal', (self, sql, nodeid)
778 cursor.execute(sql, (nodeid,))
779 res = []
780 for nodeid, date_stamp, user, action, params in cursor.fetchall():
781 res.append((nodeid, date.Date(date_stamp), user, action, params))
782 return res
784 def pack(self, pack_before):
785 ''' Delete all journal entries except "create" before 'pack_before'.
786 '''
787 # get a 'yyyymmddhhmmss' version of the date
788 date_stamp = pack_before.serialise()
790 # do the delete
791 cursor = self.conn.cursor()
792 for classname in self.classes.keys():
793 sql = "delete from %s__journal where date<? and "\
794 "action<>'create'"%classname
795 if __debug__:
796 print >>hyperdb.DEBUG, 'pack', (self, sql, date_stamp)
797 cursor.execute(sql, (date_stamp,))
799 def commit(self):
800 ''' Commit the current transactions.
802 Save all data changed since the database was opened or since the
803 last commit() or rollback().
804 '''
805 if __debug__:
806 print >>hyperdb.DEBUG, 'commit', (self,)
808 # commit gadfly
809 self.conn.commit()
811 # now, do all the other transaction stuff
812 reindex = {}
813 for method, args in self.transactions:
814 reindex[method(*args)] = 1
816 # reindex the nodes that request it
817 for classname, nodeid in filter(None, reindex.keys()):
818 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
819 self.getclass(classname).index(nodeid)
821 # save the indexer state
822 self.indexer.save_index()
824 # clear out the transactions
825 self.transactions = []
827 def rollback(self):
828 ''' Reverse all actions from the current transaction.
830 Undo all the changes made since the database was opened or the last
831 commit() or rollback() was performed.
832 '''
833 if __debug__:
834 print >>hyperdb.DEBUG, 'rollback', (self,)
836 # roll back gadfly
837 self.conn.rollback()
839 # roll back "other" transaction stuff
840 for method, args in self.transactions:
841 # delete temporary files
842 if method == self.doStoreFile:
843 self.rollbackStoreFile(*args)
844 self.transactions = []
846 def doSaveNode(self, classname, nodeid, node):
847 ''' dummy that just generates a reindex event
848 '''
849 # return the classname, nodeid so we reindex this content
850 return (classname, nodeid)
852 def close(self):
853 ''' Close off the connection.
854 '''
855 self.conn.close()
857 #
858 # The base Class class
859 #
860 class Class(hyperdb.Class):
861 ''' The handle to a particular class of nodes in a hyperdatabase.
863 All methods except __repr__ and getnode must be implemented by a
864 concrete backend Class.
865 '''
867 def __init__(self, db, classname, **properties):
868 '''Create a new class with a given name and property specification.
870 'classname' must not collide with the name of an existing class,
871 or a ValueError is raised. The keyword arguments in 'properties'
872 must map names to property objects, or a TypeError is raised.
873 '''
874 if (properties.has_key('creation') or properties.has_key('activity')
875 or properties.has_key('creator')):
876 raise ValueError, '"creation", "activity" and "creator" are '\
877 'reserved'
879 self.classname = classname
880 self.properties = properties
881 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
882 self.key = ''
884 # should we journal changes (default yes)
885 self.do_journal = 1
887 # do the db-related init stuff
888 db.addclass(self)
890 self.auditors = {'create': [], 'set': [], 'retire': []}
891 self.reactors = {'create': [], 'set': [], 'retire': []}
893 def schema(self):
894 ''' A dumpable version of the schema that we can store in the
895 database
896 '''
897 return (self.key, [(x, repr(y)) for x,y in self.properties.items()])
899 def enableJournalling(self):
900 '''Turn journalling on for this class
901 '''
902 self.do_journal = 1
904 def disableJournalling(self):
905 '''Turn journalling off for this class
906 '''
907 self.do_journal = 0
909 # Editing nodes:
910 def create(self, **propvalues):
911 ''' Create a new node of this class and return its id.
913 The keyword arguments in 'propvalues' map property names to values.
915 The values of arguments must be acceptable for the types of their
916 corresponding properties or a TypeError is raised.
918 If this class has a key property, it must be present and its value
919 must not collide with other key strings or a ValueError is raised.
921 Any other properties on this class that are missing from the
922 'propvalues' dictionary are set to None.
924 If an id in a link or multilink property does not refer to a valid
925 node, an IndexError is raised.
926 '''
927 if propvalues.has_key('id'):
928 raise KeyError, '"id" is reserved'
930 if self.db.journaltag is None:
931 raise DatabaseError, 'Database open read-only'
933 if propvalues.has_key('creation') or propvalues.has_key('activity'):
934 raise KeyError, '"creation" and "activity" are reserved'
936 self.fireAuditors('create', None, propvalues)
938 # new node's id
939 newid = self.db.newid(self.classname)
941 # validate propvalues
942 num_re = re.compile('^\d+$')
943 for key, value in propvalues.items():
944 if key == self.key:
945 try:
946 self.lookup(value)
947 except KeyError:
948 pass
949 else:
950 raise ValueError, 'node with key "%s" exists'%value
952 # try to handle this property
953 try:
954 prop = self.properties[key]
955 except KeyError:
956 raise KeyError, '"%s" has no property "%s"'%(self.classname,
957 key)
959 if value is not None and isinstance(prop, Link):
960 if type(value) != type(''):
961 raise ValueError, 'link value must be String'
962 link_class = self.properties[key].classname
963 # if it isn't a number, it's a key
964 if not num_re.match(value):
965 try:
966 value = self.db.classes[link_class].lookup(value)
967 except (TypeError, KeyError):
968 raise IndexError, 'new property "%s": %s not a %s'%(
969 key, value, link_class)
970 elif not self.db.getclass(link_class).hasnode(value):
971 raise IndexError, '%s has no node %s'%(link_class, value)
973 # save off the value
974 propvalues[key] = value
976 # register the link with the newly linked node
977 if self.do_journal and self.properties[key].do_journal:
978 self.db.addjournal(link_class, value, 'link',
979 (self.classname, newid, key))
981 elif isinstance(prop, Multilink):
982 if type(value) != type([]):
983 raise TypeError, 'new property "%s" not a list of ids'%key
985 # clean up and validate the list of links
986 link_class = self.properties[key].classname
987 l = []
988 for entry in value:
989 if type(entry) != type(''):
990 raise ValueError, '"%s" multilink value (%r) '\
991 'must contain Strings'%(key, value)
992 # if it isn't a number, it's a key
993 if not num_re.match(entry):
994 try:
995 entry = self.db.classes[link_class].lookup(entry)
996 except (TypeError, KeyError):
997 raise IndexError, 'new property "%s": %s not a %s'%(
998 key, entry, self.properties[key].classname)
999 l.append(entry)
1000 value = l
1001 propvalues[key] = value
1003 # handle additions
1004 for nodeid in value:
1005 if not self.db.getclass(link_class).hasnode(nodeid):
1006 raise IndexError, '%s has no node %s'%(link_class,
1007 nodeid)
1008 # register the link with the newly linked node
1009 if self.do_journal and self.properties[key].do_journal:
1010 self.db.addjournal(link_class, nodeid, 'link',
1011 (self.classname, newid, key))
1013 elif isinstance(prop, String):
1014 if type(value) != type(''):
1015 raise TypeError, 'new property "%s" not a string'%key
1017 elif isinstance(prop, Password):
1018 if not isinstance(value, password.Password):
1019 raise TypeError, 'new property "%s" not a Password'%key
1021 elif isinstance(prop, Date):
1022 if value is not None and not isinstance(value, date.Date):
1023 raise TypeError, 'new property "%s" not a Date'%key
1025 elif isinstance(prop, Interval):
1026 if value is not None and not isinstance(value, date.Interval):
1027 raise TypeError, 'new property "%s" not an Interval'%key
1029 elif value is not None and isinstance(prop, Number):
1030 try:
1031 float(value)
1032 except ValueError:
1033 raise TypeError, 'new property "%s" not numeric'%key
1035 elif value is not None and isinstance(prop, Boolean):
1036 try:
1037 int(value)
1038 except ValueError:
1039 raise TypeError, 'new property "%s" not boolean'%key
1041 # make sure there's data where there needs to be
1042 for key, prop in self.properties.items():
1043 if propvalues.has_key(key):
1044 continue
1045 if key == self.key:
1046 raise ValueError, 'key property "%s" is required'%key
1047 if isinstance(prop, Multilink):
1048 propvalues[key] = []
1049 else:
1050 propvalues[key] = None
1052 # done
1053 self.db.addnode(self.classname, newid, propvalues)
1054 if self.do_journal:
1055 self.db.addjournal(self.classname, newid, 'create', propvalues)
1057 self.fireReactors('create', newid, None)
1059 return newid
1061 def export_list(self, propnames, nodeid):
1062 ''' Export a node - generate a list of CSV-able data in the order
1063 specified by propnames for the given node.
1064 '''
1065 properties = self.getprops()
1066 l = []
1067 for prop in propnames:
1068 proptype = properties[prop]
1069 value = self.get(nodeid, prop)
1070 # "marshal" data where needed
1071 if value is None:
1072 pass
1073 elif isinstance(proptype, hyperdb.Date):
1074 value = value.get_tuple()
1075 elif isinstance(proptype, hyperdb.Interval):
1076 value = value.get_tuple()
1077 elif isinstance(proptype, hyperdb.Password):
1078 value = str(value)
1079 l.append(repr(value))
1080 return l
1082 def import_list(self, propnames, proplist):
1083 ''' Import a node - all information including "id" is present and
1084 should not be sanity checked. Triggers are not triggered. The
1085 journal should be initialised using the "creator" and "created"
1086 information.
1088 Return the nodeid of the node imported.
1089 '''
1090 if self.db.journaltag is None:
1091 raise DatabaseError, 'Database open read-only'
1092 properties = self.getprops()
1094 # make the new node's property map
1095 d = {}
1096 for i in range(len(propnames)):
1097 # Use eval to reverse the repr() used to output the CSV
1098 value = eval(proplist[i])
1100 # Figure the property for this column
1101 propname = propnames[i]
1102 prop = properties[propname]
1104 # "unmarshal" where necessary
1105 if propname == 'id':
1106 newid = value
1107 continue
1108 elif value is None:
1109 # don't set Nones
1110 continue
1111 elif isinstance(prop, hyperdb.Date):
1112 value = date.Date(value)
1113 elif isinstance(prop, hyperdb.Interval):
1114 value = date.Interval(value)
1115 elif isinstance(prop, hyperdb.Password):
1116 pwd = password.Password()
1117 pwd.unpack(value)
1118 value = pwd
1119 d[propname] = value
1121 # extract the extraneous journalling gumpf and nuke it
1122 if d.has_key('creator'):
1123 creator = d['creator']
1124 del d['creator']
1125 if d.has_key('creation'):
1126 creation = d['creation']
1127 del d['creation']
1128 if d.has_key('activity'):
1129 del d['activity']
1131 # add the node and journal
1132 self.db.addnode(self.classname, newid, d)
1133 self.db.addjournal(self.classname, newid, 'create', d, creator,
1134 creation)
1135 return newid
1137 _marker = []
1138 def get(self, nodeid, propname, default=_marker, cache=1):
1139 '''Get the value of a property on an existing node of this class.
1141 'nodeid' must be the id of an existing node of this class or an
1142 IndexError is raised. 'propname' must be the name of a property
1143 of this class or a KeyError is raised.
1145 'cache' indicates whether the transaction cache should be queried
1146 for the node. If the node has been modified and you need to
1147 determine what its values prior to modification are, you need to
1148 set cache=0.
1149 '''
1150 if propname == 'id':
1151 return nodeid
1153 if propname == 'creation':
1154 if not self.do_journal:
1155 raise ValueError, 'Journalling is disabled for this class'
1156 journal = self.db.getjournal(self.classname, nodeid)
1157 if journal:
1158 return self.db.getjournal(self.classname, nodeid)[0][1]
1159 else:
1160 # on the strange chance that there's no journal
1161 return date.Date()
1162 if propname == 'activity':
1163 if not self.do_journal:
1164 raise ValueError, 'Journalling is disabled for this class'
1165 journal = self.db.getjournal(self.classname, nodeid)
1166 if journal:
1167 return self.db.getjournal(self.classname, nodeid)[-1][1]
1168 else:
1169 # on the strange chance that there's no journal
1170 return date.Date()
1171 if propname == 'creator':
1172 if not self.do_journal:
1173 raise ValueError, 'Journalling is disabled for this class'
1174 journal = self.db.getjournal(self.classname, nodeid)
1175 if journal:
1176 name = self.db.getjournal(self.classname, nodeid)[0][2]
1177 else:
1178 return None
1179 try:
1180 return self.db.user.lookup(name)
1181 except KeyError:
1182 # the journaltag user doesn't exist any more
1183 return None
1185 # get the property (raises KeyErorr if invalid)
1186 prop = self.properties[propname]
1188 # get the node's dict
1189 d = self.db.getnode(self.classname, nodeid) #, cache=cache)
1191 if not d.has_key(propname):
1192 if default is self._marker:
1193 if isinstance(prop, Multilink):
1194 return []
1195 else:
1196 return None
1197 else:
1198 return default
1200 # don't pass our list to other code
1201 if isinstance(prop, Multilink):
1202 return d[propname][:]
1204 return d[propname]
1206 def getnode(self, nodeid, cache=1):
1207 ''' Return a convenience wrapper for the node.
1209 'nodeid' must be the id of an existing node of this class or an
1210 IndexError is raised.
1212 'cache' indicates whether the transaction cache should be queried
1213 for the node. If the node has been modified and you need to
1214 determine what its values prior to modification are, you need to
1215 set cache=0.
1216 '''
1217 return Node(self, nodeid, cache=cache)
1219 def set(self, nodeid, **propvalues):
1220 '''Modify a property on an existing node of this class.
1222 'nodeid' must be the id of an existing node of this class or an
1223 IndexError is raised.
1225 Each key in 'propvalues' must be the name of a property of this
1226 class or a KeyError is raised.
1228 All values in 'propvalues' must be acceptable types for their
1229 corresponding properties or a TypeError is raised.
1231 If the value of the key property is set, it must not collide with
1232 other key strings or a ValueError is raised.
1234 If the value of a Link or Multilink property contains an invalid
1235 node id, a ValueError is raised.
1236 '''
1237 if not propvalues:
1238 return propvalues
1240 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1241 raise KeyError, '"creation" and "activity" are reserved'
1243 if propvalues.has_key('id'):
1244 raise KeyError, '"id" is reserved'
1246 if self.db.journaltag is None:
1247 raise DatabaseError, 'Database open read-only'
1249 self.fireAuditors('set', nodeid, propvalues)
1250 # Take a copy of the node dict so that the subsequent set
1251 # operation doesn't modify the oldvalues structure.
1252 # XXX used to try the cache here first
1253 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1255 node = self.db.getnode(self.classname, nodeid)
1256 if self.is_retired(nodeid):
1257 raise IndexError
1258 num_re = re.compile('^\d+$')
1260 # if the journal value is to be different, store it in here
1261 journalvalues = {}
1263 # remember the add/remove stuff for multilinks, making it easier
1264 # for the Database layer to do its stuff
1265 multilink_changes = {}
1267 for propname, value in propvalues.items():
1268 # check to make sure we're not duplicating an existing key
1269 if propname == self.key and node[propname] != value:
1270 try:
1271 self.lookup(value)
1272 except KeyError:
1273 pass
1274 else:
1275 raise ValueError, 'node with key "%s" exists'%value
1277 # this will raise the KeyError if the property isn't valid
1278 # ... we don't use getprops() here because we only care about
1279 # the writeable properties.
1280 prop = self.properties[propname]
1282 # if the value's the same as the existing value, no sense in
1283 # doing anything
1284 if node.has_key(propname) and value == node[propname]:
1285 del propvalues[propname]
1286 continue
1288 # do stuff based on the prop type
1289 if isinstance(prop, Link):
1290 link_class = prop.classname
1291 # if it isn't a number, it's a key
1292 if value is not None and not isinstance(value, type('')):
1293 raise ValueError, 'property "%s" link value be a string'%(
1294 propname)
1295 if isinstance(value, type('')) and not num_re.match(value):
1296 try:
1297 value = self.db.classes[link_class].lookup(value)
1298 except (TypeError, KeyError):
1299 raise IndexError, 'new property "%s": %s not a %s'%(
1300 propname, value, prop.classname)
1302 if (value is not None and
1303 not self.db.getclass(link_class).hasnode(value)):
1304 raise IndexError, '%s has no node %s'%(link_class, value)
1306 if self.do_journal and prop.do_journal:
1307 # register the unlink with the old linked node
1308 if node[propname] is not None:
1309 self.db.addjournal(link_class, node[propname], 'unlink',
1310 (self.classname, nodeid, propname))
1312 # register the link with the newly linked node
1313 if value is not None:
1314 self.db.addjournal(link_class, value, 'link',
1315 (self.classname, nodeid, propname))
1317 elif isinstance(prop, Multilink):
1318 if type(value) != type([]):
1319 raise TypeError, 'new property "%s" not a list of'\
1320 ' ids'%propname
1321 link_class = self.properties[propname].classname
1322 l = []
1323 for entry in value:
1324 # if it isn't a number, it's a key
1325 if type(entry) != type(''):
1326 raise ValueError, 'new property "%s" link value ' \
1327 'must be a string'%propname
1328 if not num_re.match(entry):
1329 try:
1330 entry = self.db.classes[link_class].lookup(entry)
1331 except (TypeError, KeyError):
1332 raise IndexError, 'new property "%s": %s not a %s'%(
1333 propname, entry,
1334 self.properties[propname].classname)
1335 l.append(entry)
1336 value = l
1337 propvalues[propname] = value
1339 # figure the journal entry for this property
1340 add = []
1341 remove = []
1343 # handle removals
1344 if node.has_key(propname):
1345 l = node[propname]
1346 else:
1347 l = []
1348 for id in l[:]:
1349 if id in value:
1350 continue
1351 # register the unlink with the old linked node
1352 if self.do_journal and self.properties[propname].do_journal:
1353 self.db.addjournal(link_class, id, 'unlink',
1354 (self.classname, nodeid, propname))
1355 l.remove(id)
1356 remove.append(id)
1358 # handle additions
1359 for id in value:
1360 if not self.db.getclass(link_class).hasnode(id):
1361 raise IndexError, '%s has no node %s'%(link_class, id)
1362 if id in l:
1363 continue
1364 # register the link with the newly linked node
1365 if self.do_journal and self.properties[propname].do_journal:
1366 self.db.addjournal(link_class, id, 'link',
1367 (self.classname, nodeid, propname))
1368 l.append(id)
1369 add.append(id)
1371 # figure the journal entry
1372 l = []
1373 if add:
1374 l.append(('+', add))
1375 if remove:
1376 l.append(('-', remove))
1377 multilink_changes[propname] = (add, remove)
1378 if l:
1379 journalvalues[propname] = tuple(l)
1381 elif isinstance(prop, String):
1382 if value is not None and type(value) != type(''):
1383 raise TypeError, 'new property "%s" not a string'%propname
1385 elif isinstance(prop, Password):
1386 if not isinstance(value, password.Password):
1387 raise TypeError, 'new property "%s" not a Password'%propname
1388 propvalues[propname] = value
1390 elif value is not None and isinstance(prop, Date):
1391 if not isinstance(value, date.Date):
1392 raise TypeError, 'new property "%s" not a Date'% propname
1393 propvalues[propname] = value
1395 elif value is not None and isinstance(prop, Interval):
1396 if not isinstance(value, date.Interval):
1397 raise TypeError, 'new property "%s" not an '\
1398 'Interval'%propname
1399 propvalues[propname] = value
1401 elif value is not None and isinstance(prop, Number):
1402 try:
1403 float(value)
1404 except ValueError:
1405 raise TypeError, 'new property "%s" not numeric'%propname
1407 elif value is not None and isinstance(prop, Boolean):
1408 try:
1409 int(value)
1410 except ValueError:
1411 raise TypeError, 'new property "%s" not boolean'%propname
1413 node[propname] = value
1415 # nothing to do?
1416 if not propvalues:
1417 return propvalues
1419 # do the set, and journal it
1420 self.db.setnode(self.classname, nodeid, node, multilink_changes)
1422 if self.do_journal:
1423 propvalues.update(journalvalues)
1424 self.db.addjournal(self.classname, nodeid, 'set', propvalues)
1426 self.fireReactors('set', nodeid, oldvalues)
1428 return propvalues
1430 def retire(self, nodeid):
1431 '''Retire a node.
1433 The properties on the node remain available from the get() method,
1434 and the node's id is never reused.
1436 Retired nodes are not returned by the find(), list(), or lookup()
1437 methods, and other nodes may reuse the values of their key properties.
1438 '''
1439 if self.db.journaltag is None:
1440 raise DatabaseError, 'Database open read-only'
1442 cursor = self.db.conn.cursor()
1443 sql = 'update _%s set __retired__=1 where id=?'%self.classname
1444 if __debug__:
1445 print >>hyperdb.DEBUG, 'retire', (self, sql, nodeid)
1446 cursor.execute(sql, (nodeid,))
1448 def is_retired(self, nodeid):
1449 '''Return true if the node is rerired
1450 '''
1451 cursor = self.db.conn.cursor()
1452 sql = 'select __retired__ from _%s where id=?'%self.classname
1453 if __debug__:
1454 print >>hyperdb.DEBUG, 'is_retired', (self, sql, nodeid)
1455 cursor.execute(sql, (nodeid,))
1456 return cursor.fetchone()[0]
1458 def destroy(self, nodeid):
1459 '''Destroy a node.
1461 WARNING: this method should never be used except in extremely rare
1462 situations where there could never be links to the node being
1463 deleted
1464 WARNING: use retire() instead
1465 WARNING: the properties of this node will not be available ever again
1466 WARNING: really, use retire() instead
1468 Well, I think that's enough warnings. This method exists mostly to
1469 support the session storage of the cgi interface.
1471 The node is completely removed from the hyperdb, including all journal
1472 entries. It will no longer be available, and will generally break code
1473 if there are any references to the node.
1474 '''
1475 if self.db.journaltag is None:
1476 raise DatabaseError, 'Database open read-only'
1477 self.db.destroynode(self.classname, nodeid)
1479 def history(self, nodeid):
1480 '''Retrieve the journal of edits on a particular node.
1482 'nodeid' must be the id of an existing node of this class or an
1483 IndexError is raised.
1485 The returned list contains tuples of the form
1487 (date, tag, action, params)
1489 'date' is a Timestamp object specifying the time of the change and
1490 'tag' is the journaltag specified when the database was opened.
1491 '''
1492 if not self.do_journal:
1493 raise ValueError, 'Journalling is disabled for this class'
1494 return self.db.getjournal(self.classname, nodeid)
1496 # Locating nodes:
1497 def hasnode(self, nodeid):
1498 '''Determine if the given nodeid actually exists
1499 '''
1500 return self.db.hasnode(self.classname, nodeid)
1502 def setkey(self, propname):
1503 '''Select a String property of this class to be the key property.
1505 'propname' must be the name of a String property of this class or
1506 None, or a TypeError is raised. The values of the key property on
1507 all existing nodes must be unique or a ValueError is raised.
1508 '''
1509 # XXX create an index on the key prop column
1510 prop = self.getprops()[propname]
1511 if not isinstance(prop, String):
1512 raise TypeError, 'key properties must be String'
1513 self.key = propname
1515 def getkey(self):
1516 '''Return the name of the key property for this class or None.'''
1517 return self.key
1519 def labelprop(self, default_to_id=0):
1520 ''' Return the property name for a label for the given node.
1522 This method attempts to generate a consistent label for the node.
1523 It tries the following in order:
1524 1. key property
1525 2. "name" property
1526 3. "title" property
1527 4. first property from the sorted property name list
1528 '''
1529 k = self.getkey()
1530 if k:
1531 return k
1532 props = self.getprops()
1533 if props.has_key('name'):
1534 return 'name'
1535 elif props.has_key('title'):
1536 return 'title'
1537 if default_to_id:
1538 return 'id'
1539 props = props.keys()
1540 props.sort()
1541 return props[0]
1543 def lookup(self, keyvalue):
1544 '''Locate a particular node by its key property and return its id.
1546 If this class has no key property, a TypeError is raised. If the
1547 'keyvalue' matches one of the values for the key property among
1548 the nodes in this class, the matching node's id is returned;
1549 otherwise a KeyError is raised.
1550 '''
1551 if not self.key:
1552 raise TypeError, 'No key property set for class %s'%self.classname
1554 cursor = self.db.conn.cursor()
1555 sql = 'select id from _%s where _%s=?'%(self.classname, self.key)
1556 if __debug__:
1557 print >>hyperdb.DEBUG, 'lookup', (self, sql, keyvalue)
1558 cursor.execute(sql, (keyvalue,))
1560 # see if there was a result
1561 l = cursor.fetchall()
1562 if not l:
1563 raise KeyError, keyvalue
1565 # return the id
1566 return l[0][0]
1568 def find(self, **propspec):
1569 '''Get the ids of nodes in this class which link to the given nodes.
1571 'propspec' consists of keyword args propname={nodeid:1,}
1572 'propname' must be the name of a property in this class, or a
1573 KeyError is raised. That property must be a Link or Multilink
1574 property, or a TypeError is raised.
1576 Any node in this class whose 'propname' property links to any of the
1577 nodeids will be returned. Used by the full text indexing, which knows
1578 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1579 issues:
1581 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1582 '''
1583 if __debug__:
1584 print >>hyperdb.DEBUG, 'find', (self, propspec)
1585 if not propspec:
1586 return []
1587 queries = []
1588 tables = []
1589 allvalues = ()
1590 for prop, values in propspec.items():
1591 allvalues += tuple(values.keys())
1592 tables.append('select nodeid from %s_%s where linkid in (%s)'%(
1593 self.classname, prop, ','.join(['?' for x in values.keys()])))
1594 sql = '\nintersect\n'.join(tables)
1595 if __debug__:
1596 print >>hyperdb.DEBUG, 'find', (self, sql, allvalues)
1597 cursor = self.db.conn.cursor()
1598 cursor.execute(sql, allvalues)
1599 try:
1600 l = [x[0] for x in cursor.fetchall()]
1601 except gadfly.database.error, message:
1602 if message == 'no more results':
1603 l = []
1604 raise
1605 if __debug__:
1606 print >>hyperdb.DEBUG, 'find ... ', l
1607 return l
1609 def list(self):
1610 ''' Return a list of the ids of the active nodes in this class.
1611 '''
1612 return self.db.getnodeids(self.classname, retired=0)
1614 def filter(self, search_matches, filterspec, sort, group):
1615 ''' Return a list of the ids of the active nodes in this class that
1616 match the 'filter' spec, sorted by the group spec and then the
1617 sort spec
1619 "filterspec" is {propname: value(s)}
1620 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1621 and prop is a prop name or None
1622 "search_matches" is {nodeid: marker}
1623 '''
1624 cn = self.classname
1626 # figure the WHERE clause from the filterspec
1627 props = self.getprops()
1628 frum = ['_'+cn]
1629 where = []
1630 args = []
1631 for k, v in filterspec.items():
1632 propclass = props[k]
1633 if isinstance(propclass, Multilink):
1634 tn = '%s_%s'%(cn, k)
1635 frum.append(tn)
1636 if isinstance(v, type([])):
1637 s = ','.join(['?' for x in v])
1638 where.append('id=%s.nodeid and %s.linkid in (%s)'%(tn,tn,s))
1639 args = args + v
1640 else:
1641 where.append('id=%s.nodeid and %s.linkid = ?'%(tn, tn))
1642 args.append(v)
1643 else:
1644 if isinstance(v, type([])):
1645 s = ','.join(['?' for x in v])
1646 where.append('_%s in (%s)'%(k, s))
1647 args = args + v
1648 else:
1649 where.append('_%s=?'%k)
1650 args.append(v)
1652 # add results of full text search
1653 if search_matches is not None:
1654 v = search_matches.keys()
1655 s = ','.join(['?' for x in v])
1656 where.append('id in (%s)'%s)
1657 args = args + v
1659 # figure the order by clause
1660 orderby = []
1661 ordercols = []
1662 if sort[0] is not None and sort[1] is not None:
1663 if sort[0] != '-':
1664 orderby.append('_'+sort[1])
1665 ordercols.append(sort[1])
1666 else:
1667 orderby.append('_'+sort[1]+' desc')
1668 ordercols.append(sort[1])
1670 # figure the group by clause
1671 groupby = []
1672 groupcols = []
1673 if group[0] is not None and group[1] is not None:
1674 if group[0] != '-':
1675 groupby.append('_'+group[1])
1676 groupcols.append(group[1])
1677 else:
1678 groupby.append('_'+group[1]+' desc')
1679 groupcols.append(group[1])
1681 # construct the SQL
1682 frum = ','.join(frum)
1683 where = ' and '.join(where)
1684 cols = ['id']
1685 if orderby:
1686 cols = cols + ordercols
1687 order = ' order by %s'%(','.join(orderby))
1688 else:
1689 order = ''
1690 if groupby:
1691 cols = cols + groupcols
1692 group = ' group by %s'%(','.join(groupby))
1693 else:
1694 group = ''
1695 cols = ','.join(cols)
1696 sql = 'select %s from %s where %s%s%s'%(cols, frum, where, order,
1697 group)
1698 args = tuple(args)
1699 if __debug__:
1700 print >>hyperdb.DEBUG, 'find', (self, sql, args)
1701 cursor = self.db.conn.cursor()
1702 cursor.execute(sql, args)
1704 def count(self):
1705 '''Get the number of nodes in this class.
1707 If the returned integer is 'numnodes', the ids of all the nodes
1708 in this class run from 1 to numnodes, and numnodes+1 will be the
1709 id of the next node to be created in this class.
1710 '''
1711 return self.db.countnodes(self.classname)
1713 # Manipulating properties:
1714 def getprops(self, protected=1):
1715 '''Return a dictionary mapping property names to property objects.
1716 If the "protected" flag is true, we include protected properties -
1717 those which may not be modified.
1718 '''
1719 d = self.properties.copy()
1720 if protected:
1721 d['id'] = String()
1722 d['creation'] = hyperdb.Date()
1723 d['activity'] = hyperdb.Date()
1724 d['creator'] = hyperdb.Link("user")
1725 return d
1727 def addprop(self, **properties):
1728 '''Add properties to this class.
1730 The keyword arguments in 'properties' must map names to property
1731 objects, or a TypeError is raised. None of the keys in 'properties'
1732 may collide with the names of existing properties, or a ValueError
1733 is raised before any properties have been added.
1734 '''
1735 for key in properties.keys():
1736 if self.properties.has_key(key):
1737 raise ValueError, key
1738 self.properties.update(properties)
1740 def index(self, nodeid):
1741 '''Add (or refresh) the node to search indexes
1742 '''
1743 # find all the String properties that have indexme
1744 for prop, propclass in self.getprops().items():
1745 if isinstance(propclass, String) and propclass.indexme:
1746 try:
1747 value = str(self.get(nodeid, prop))
1748 except IndexError:
1749 # node no longer exists - entry should be removed
1750 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1751 else:
1752 # and index them under (classname, nodeid, property)
1753 self.db.indexer.add_text((self.classname, nodeid, prop),
1754 value)
1757 #
1758 # Detector interface
1759 #
1760 def audit(self, event, detector):
1761 '''Register a detector
1762 '''
1763 l = self.auditors[event]
1764 if detector not in l:
1765 self.auditors[event].append(detector)
1767 def fireAuditors(self, action, nodeid, newvalues):
1768 '''Fire all registered auditors.
1769 '''
1770 for audit in self.auditors[action]:
1771 audit(self.db, self, nodeid, newvalues)
1773 def react(self, event, detector):
1774 '''Register a detector
1775 '''
1776 l = self.reactors[event]
1777 if detector not in l:
1778 self.reactors[event].append(detector)
1780 def fireReactors(self, action, nodeid, oldvalues):
1781 '''Fire all registered reactors.
1782 '''
1783 for react in self.reactors[action]:
1784 react(self.db, self, nodeid, oldvalues)
1786 class FileClass(Class):
1787 '''This class defines a large chunk of data. To support this, it has a
1788 mandatory String property "content" which is typically saved off
1789 externally to the hyperdb.
1791 The default MIME type of this data is defined by the
1792 "default_mime_type" class attribute, which may be overridden by each
1793 node if the class defines a "type" String property.
1794 '''
1795 default_mime_type = 'text/plain'
1797 def create(self, **propvalues):
1798 ''' snaffle the file propvalue and store in a file
1799 '''
1800 content = propvalues['content']
1801 del propvalues['content']
1802 newid = Class.create(self, **propvalues)
1803 self.db.storefile(self.classname, newid, None, content)
1804 return newid
1806 def import_list(self, propnames, proplist):
1807 ''' Trap the "content" property...
1808 '''
1809 # dupe this list so we don't affect others
1810 propnames = propnames[:]
1812 # extract the "content" property from the proplist
1813 i = propnames.index('content')
1814 content = eval(proplist[i])
1815 del propnames[i]
1816 del proplist[i]
1818 # do the normal import
1819 newid = Class.import_list(self, propnames, proplist)
1821 # save off the "content" file
1822 self.db.storefile(self.classname, newid, None, content)
1823 return newid
1825 _marker = []
1826 def get(self, nodeid, propname, default=_marker, cache=1):
1827 ''' trap the content propname and get it from the file
1828 '''
1830 poss_msg = 'Possibly a access right configuration problem.'
1831 if propname == 'content':
1832 try:
1833 return self.db.getfile(self.classname, nodeid, None)
1834 except IOError, (strerror):
1835 # BUG: by catching this we donot see an error in the log.
1836 return 'ERROR reading file: %s%s\n%s\n%s'%(
1837 self.classname, nodeid, poss_msg, strerror)
1838 if default is not self._marker:
1839 return Class.get(self, nodeid, propname, default, cache=cache)
1840 else:
1841 return Class.get(self, nodeid, propname, cache=cache)
1843 def getprops(self, protected=1):
1844 ''' In addition to the actual properties on the node, these methods
1845 provide the "content" property. If the "protected" flag is true,
1846 we include protected properties - those which may not be
1847 modified.
1848 '''
1849 d = Class.getprops(self, protected=protected).copy()
1850 if protected:
1851 d['content'] = hyperdb.String()
1852 return d
1854 def index(self, nodeid):
1855 ''' Index the node in the search index.
1857 We want to index the content in addition to the normal String
1858 property indexing.
1859 '''
1860 # perform normal indexing
1861 Class.index(self, nodeid)
1863 # get the content to index
1864 content = self.get(nodeid, 'content')
1866 # figure the mime type
1867 if self.properties.has_key('type'):
1868 mime_type = self.get(nodeid, 'type')
1869 else:
1870 mime_type = self.default_mime_type
1872 # and index!
1873 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
1874 mime_type)
1876 # XXX deviation from spec - was called ItemClass
1877 class IssueClass(Class, roundupdb.IssueClass):
1878 # Overridden methods:
1879 def __init__(self, db, classname, **properties):
1880 '''The newly-created class automatically includes the "messages",
1881 "files", "nosy", and "superseder" properties. If the 'properties'
1882 dictionary attempts to specify any of these properties or a
1883 "creation" or "activity" property, a ValueError is raised.
1884 '''
1885 if not properties.has_key('title'):
1886 properties['title'] = hyperdb.String(indexme='yes')
1887 if not properties.has_key('messages'):
1888 properties['messages'] = hyperdb.Multilink("msg")
1889 if not properties.has_key('files'):
1890 properties['files'] = hyperdb.Multilink("file")
1891 if not properties.has_key('nosy'):
1892 # note: journalling is turned off as it really just wastes
1893 # space. this behaviour may be overridden in an instance
1894 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
1895 if not properties.has_key('superseder'):
1896 properties['superseder'] = hyperdb.Multilink(classname)
1897 Class.__init__(self, db, classname, **properties)