42d832d5f42d54f44de90d5771984c25053f01ba
1 # $Id: rdbms_common.py,v 1.3 2002-09-19 02:37:41 richard Exp $
3 # standard python modules
4 import sys, os, time, re, errno, weakref, copy
6 # roundup modules
7 from roundup import hyperdb, date, password, roundupdb, security
8 from roundup.hyperdb import String, Password, Date, Interval, Link, \
9 Multilink, DatabaseError, Boolean, Number
11 # support
12 from blobfiles import FileStorage
13 from roundup.indexer import Indexer
14 from sessions import Sessions
16 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
17 # flag to set on retired entries
18 RETIRED_FLAG = '__hyperdb_retired'
20 def __init__(self, config, journaltag=None):
21 ''' Open the database and load the schema from it.
22 '''
23 self.config, self.journaltag = config, journaltag
24 self.dir = config.DATABASE
25 self.classes = {}
26 self.indexer = Indexer(self.dir)
27 self.sessions = Sessions(self.config)
28 self.security = security.Security(self)
30 # additional transaction support for external files and the like
31 self.transactions = []
33 # open a connection to the database, creating the "conn" attribute
34 self.open_connection()
36 def open_connection(self):
37 ''' Open a connection to the database, creating it if necessary
38 '''
39 raise NotImplemented
41 def sql(self, cursor, sql, args=None):
42 ''' Execute the sql with the optional args.
43 '''
44 if __debug__:
45 print >>hyperdb.DEBUG, (self, sql, args)
46 if args:
47 cursor.execute(sql, args)
48 else:
49 cursor.execute(sql)
51 def sql_fetchone(self, cursor):
52 ''' Fetch a single row. If there's nothing to fetch, return None.
53 '''
54 raise NotImplemented
56 def sql_stringquote(self, value):
57 ''' Quote the string so it's safe to put in the 'sql quotes'
58 '''
59 return re.sub("'", "''", str(value))
61 def save_dbschema(self, cursor, schema):
62 ''' Save the schema definition that the database currently implements
63 '''
64 raise NotImplemented
66 def load_dbschema(self, cursor):
67 ''' Load the schema definition that the database currently implements
68 '''
69 raise NotImplemented
71 def post_init(self):
72 ''' Called once the schema initialisation has finished.
74 We should now confirm that the schema defined by our "classes"
75 attribute actually matches the schema in the database.
76 '''
77 # now detect changes in the schema
78 save = 0
79 for classname, spec in self.classes.items():
80 if self.database_schema.has_key(classname):
81 dbspec = self.database_schema[classname]
82 if self.update_class(spec, dbspec):
83 self.database_schema[classname] = spec.schema()
84 save = 1
85 else:
86 self.create_class(spec)
87 self.database_schema[classname] = spec.schema()
88 save = 1
90 for classname in self.database_schema.keys():
91 if not self.classes.has_key(classname):
92 self.drop_class(classname)
94 # update the database version of the schema
95 if save:
96 cursor = self.conn.cursor()
97 self.sql(cursor, 'delete from schema')
98 self.save_dbschema(cursor, self.database_schema)
100 # reindex the db if necessary
101 if self.indexer.should_reindex():
102 self.reindex()
104 # commit
105 self.conn.commit()
107 def reindex(self):
108 for klass in self.classes.values():
109 for nodeid in klass.list():
110 klass.index(nodeid)
111 self.indexer.save_index()
113 def determine_columns(self, properties):
114 ''' Figure the column names and multilink properties from the spec
116 "properties" is a list of (name, prop) where prop may be an
117 instance of a hyperdb "type" _or_ a string repr of that type.
118 '''
119 cols = []
120 mls = []
121 # add the multilinks separately
122 for col, prop in properties:
123 if isinstance(prop, Multilink):
124 mls.append(col)
125 elif isinstance(prop, type('')) and prop.find('Multilink') != -1:
126 mls.append(col)
127 else:
128 cols.append('_'+col)
129 cols.sort()
130 return cols, mls
132 def update_class(self, spec, dbspec):
133 ''' Determine the differences between the current spec and the
134 database version of the spec, and update where necessary
135 '''
136 spec_schema = spec.schema()
137 if spec_schema == dbspec:
138 # no save needed for this one
139 return 0
140 if __debug__:
141 print >>hyperdb.DEBUG, 'update_class FIRING'
143 # key property changed?
144 if dbspec[0] != spec_schema[0]:
145 if __debug__:
146 print >>hyperdb.DEBUG, 'update_class setting keyprop', `spec[0]`
147 # XXX turn on indexing for the key property
149 # dict 'em up
150 spec_propnames,spec_props = [],{}
151 for propname,prop in spec_schema[1]:
152 spec_propnames.append(propname)
153 spec_props[propname] = prop
154 dbspec_propnames,dbspec_props = [],{}
155 for propname,prop in dbspec[1]:
156 dbspec_propnames.append(propname)
157 dbspec_props[propname] = prop
159 # we're going to need one of these
160 cursor = self.conn.cursor()
162 # now compare
163 for propname in spec_propnames:
164 prop = spec_props[propname]
165 if dbspec_props.has_key(propname) and prop==dbspec_props[propname]:
166 continue
167 if __debug__:
168 print >>hyperdb.DEBUG, 'update_class ADD', (propname, prop)
170 if not dbspec_props.has_key(propname):
171 # add the property
172 if isinstance(prop, Multilink):
173 # all we have to do here is create a new table, easy!
174 self.create_multilink_table(cursor, spec, propname)
175 continue
177 # no ALTER TABLE, so we:
178 # 1. pull out the data, including an extra None column
179 oldcols, x = self.determine_columns(dbspec[1])
180 oldcols.append('id')
181 oldcols.append('__retired__')
182 cn = spec.classname
183 sql = 'select %s,%s from _%s'%(','.join(oldcols), self.arg, cn)
184 if __debug__:
185 print >>hyperdb.DEBUG, 'update_class', (self, sql, None)
186 cursor.execute(sql, (None,))
187 olddata = cursor.fetchall()
189 # 2. drop the old table
190 cursor.execute('drop table _%s'%cn)
192 # 3. create the new table
193 cols, mls = self.create_class_table(cursor, spec)
194 # ensure the new column is last
195 cols.remove('_'+propname)
196 assert oldcols == cols, "Column lists don't match!"
197 cols.append('_'+propname)
199 # 4. populate with the data from step one
200 s = ','.join([self.arg for x in cols])
201 scols = ','.join(cols)
202 sql = 'insert into _%s (%s) values (%s)'%(cn, scols, s)
204 # GAH, nothing had better go wrong from here on in... but
205 # we have to commit the drop...
206 # XXX this isn't necessary in sqlite :(
207 self.conn.commit()
209 # do the insert
210 for row in olddata:
211 self.sql(cursor, sql, tuple(row))
213 else:
214 # modify the property
215 if __debug__:
216 print >>hyperdb.DEBUG, 'update_class NOOP'
217 pass # NOOP in gadfly
219 # and the other way - only worry about deletions here
220 for propname in dbspec_propnames:
221 prop = dbspec_props[propname]
222 if spec_props.has_key(propname):
223 continue
224 if __debug__:
225 print >>hyperdb.DEBUG, 'update_class REMOVE', `prop`
227 # delete the property
228 if isinstance(prop, Multilink):
229 sql = 'drop table %s_%s'%(spec.classname, prop)
230 if __debug__:
231 print >>hyperdb.DEBUG, 'update_class', (self, sql)
232 cursor.execute(sql)
233 else:
234 # no ALTER TABLE, so we:
235 # 1. pull out the data, excluding the removed column
236 oldcols, x = self.determine_columns(spec.properties.items())
237 oldcols.append('id')
238 oldcols.append('__retired__')
239 # remove the missing column
240 oldcols.remove('_'+propname)
241 cn = spec.classname
242 sql = 'select %s from _%s'%(','.join(oldcols), cn)
243 cursor.execute(sql, (None,))
244 olddata = sql.fetchall()
246 # 2. drop the old table
247 cursor.execute('drop table _%s'%cn)
249 # 3. create the new table
250 cols, mls = self.create_class_table(self, cursor, spec)
251 assert oldcols != cols, "Column lists don't match!"
253 # 4. populate with the data from step one
254 qs = ','.join([self.arg for x in cols])
255 sql = 'insert into _%s values (%s)'%(cn, s)
256 cursor.execute(sql, olddata)
257 return 1
259 def create_class_table(self, cursor, spec):
260 ''' create the class table for the given spec
261 '''
262 cols, mls = self.determine_columns(spec.properties.items())
264 # add on our special columns
265 cols.append('id')
266 cols.append('__retired__')
268 # create the base table
269 scols = ','.join(['%s varchar'%x for x in cols])
270 sql = 'create table _%s (%s)'%(spec.classname, scols)
271 if __debug__:
272 print >>hyperdb.DEBUG, 'create_class', (self, sql)
273 cursor.execute(sql)
275 return cols, mls
277 def create_journal_table(self, cursor, spec):
278 ''' create the journal table for a class given the spec and
279 already-determined cols
280 '''
281 # journal table
282 cols = ','.join(['%s varchar'%x
283 for x in 'nodeid date tag action params'.split()])
284 sql = 'create table %s__journal (%s)'%(spec.classname, cols)
285 if __debug__:
286 print >>hyperdb.DEBUG, 'create_class', (self, sql)
287 cursor.execute(sql)
289 def create_multilink_table(self, cursor, spec, ml):
290 ''' Create a multilink table for the "ml" property of the class
291 given by the spec
292 '''
293 sql = 'create table %s_%s (linkid varchar, nodeid varchar)'%(
294 spec.classname, ml)
295 if __debug__:
296 print >>hyperdb.DEBUG, 'create_class', (self, sql)
297 cursor.execute(sql)
299 def create_class(self, spec):
300 ''' Create a database table according to the given spec.
301 '''
302 cursor = self.conn.cursor()
303 cols, mls = self.create_class_table(cursor, spec)
304 self.create_journal_table(cursor, spec)
306 # now create the multilink tables
307 for ml in mls:
308 self.create_multilink_table(cursor, spec, ml)
310 # ID counter
311 sql = 'insert into ids (name, num) values (%s,%s)'%(self.arg, self.arg)
312 vals = (spec.classname, 1)
313 if __debug__:
314 print >>hyperdb.DEBUG, 'create_class', (self, sql, vals)
315 cursor.execute(sql, vals)
317 def drop_class(self, spec):
318 ''' Drop the given table from the database.
320 Drop the journal and multilink tables too.
321 '''
322 # figure the multilinks
323 mls = []
324 for col, prop in spec.properties.items():
325 if isinstance(prop, Multilink):
326 mls.append(col)
327 cursor = self.conn.cursor()
329 sql = 'drop table _%s'%spec.classname
330 if __debug__:
331 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
332 cursor.execute(sql)
334 sql = 'drop table %s__journal'%spec.classname
335 if __debug__:
336 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
337 cursor.execute(sql)
339 for ml in mls:
340 sql = 'drop table %s_%s'%(spec.classname, ml)
341 if __debug__:
342 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
343 cursor.execute(sql)
345 #
346 # Classes
347 #
348 def __getattr__(self, classname):
349 ''' A convenient way of calling self.getclass(classname).
350 '''
351 if self.classes.has_key(classname):
352 if __debug__:
353 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
354 return self.classes[classname]
355 raise AttributeError, classname
357 def addclass(self, cl):
358 ''' Add a Class to the hyperdatabase.
359 '''
360 if __debug__:
361 print >>hyperdb.DEBUG, 'addclass', (self, cl)
362 cn = cl.classname
363 if self.classes.has_key(cn):
364 raise ValueError, cn
365 self.classes[cn] = cl
367 def getclasses(self):
368 ''' Return a list of the names of all existing classes.
369 '''
370 if __debug__:
371 print >>hyperdb.DEBUG, 'getclasses', (self,)
372 l = self.classes.keys()
373 l.sort()
374 return l
376 def getclass(self, classname):
377 '''Get the Class object representing a particular class.
379 If 'classname' is not a valid class name, a KeyError is raised.
380 '''
381 if __debug__:
382 print >>hyperdb.DEBUG, 'getclass', (self, classname)
383 try:
384 return self.classes[classname]
385 except KeyError:
386 raise KeyError, 'There is no class called "%s"'%classname
388 def clear(self):
389 ''' Delete all database contents.
391 Note: I don't commit here, which is different behaviour to the
392 "nuke from orbit" behaviour in the *dbms.
393 '''
394 if __debug__:
395 print >>hyperdb.DEBUG, 'clear', (self,)
396 cursor = self.conn.cursor()
397 for cn in self.classes.keys():
398 sql = 'delete from _%s'%cn
399 if __debug__:
400 print >>hyperdb.DEBUG, 'clear', (self, sql)
401 cursor.execute(sql)
403 #
404 # Node IDs
405 #
406 def newid(self, classname):
407 ''' Generate a new id for the given class
408 '''
409 # get the next ID
410 cursor = self.conn.cursor()
411 sql = 'select num from ids where name=%s'%self.arg
412 if __debug__:
413 print >>hyperdb.DEBUG, 'newid', (self, sql, classname)
414 cursor.execute(sql, (classname, ))
415 newid = cursor.fetchone()[0]
417 # update the counter
418 sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
419 vals = (int(newid)+1, classname)
420 if __debug__:
421 print >>hyperdb.DEBUG, 'newid', (self, sql, vals)
422 cursor.execute(sql, vals)
424 # return as string
425 return str(newid)
427 def setid(self, classname, setid):
428 ''' Set the id counter: used during import of database
429 '''
430 cursor = self.conn.cursor()
431 sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
432 vals = (setid, spec.classname)
433 if __debug__:
434 print >>hyperdb.DEBUG, 'setid', (self, sql, vals)
435 cursor.execute(sql, vals)
437 #
438 # Nodes
439 #
441 def addnode(self, classname, nodeid, node):
442 ''' Add the specified node to its class's db.
443 '''
444 if __debug__:
445 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
446 # gadfly requires values for all non-multilink columns
447 cl = self.classes[classname]
448 cols, mls = self.determine_columns(cl.properties.items())
450 # default the non-multilink columns
451 for col, prop in cl.properties.items():
452 if not isinstance(col, Multilink):
453 if not node.has_key(col):
454 node[col] = None
456 node = self.serialise(classname, node)
458 # make sure the ordering is correct for column name -> column value
459 vals = tuple([node[col[1:]] for col in cols]) + (nodeid, 0)
460 s = ','.join([self.arg for x in cols]) + ',%s,%s'%(self.arg, self.arg)
461 cols = ','.join(cols) + ',id,__retired__'
463 # perform the inserts
464 cursor = self.conn.cursor()
465 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
466 if __debug__:
467 print >>hyperdb.DEBUG, 'addnode', (self, sql, vals)
468 cursor.execute(sql, vals)
470 # insert the multilink rows
471 for col in mls:
472 t = '%s_%s'%(classname, col)
473 for entry in node[col]:
474 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
475 self.arg, self.arg)
476 self.sql(cursor, sql, (entry, nodeid))
478 # make sure we do the commit-time extra stuff for this node
479 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
481 def setnode(self, classname, nodeid, node, multilink_changes):
482 ''' Change the specified node.
483 '''
484 if __debug__:
485 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
486 node = self.serialise(classname, node)
488 cl = self.classes[classname]
489 cols = []
490 mls = []
491 # add the multilinks separately
492 for col in node.keys():
493 prop = cl.properties[col]
494 if isinstance(prop, Multilink):
495 mls.append(col)
496 else:
497 cols.append('_'+col)
498 cols.sort()
500 # make sure the ordering is correct for column name -> column value
501 vals = tuple([node[col[1:]] for col in cols])
502 s = ','.join(['%s=%s'%(x, self.arg) for x in cols])
503 cols = ','.join(cols)
505 # perform the update
506 cursor = self.conn.cursor()
507 sql = 'update _%s set %s'%(classname, s)
508 if __debug__:
509 print >>hyperdb.DEBUG, 'setnode', (self, sql, vals)
510 cursor.execute(sql, vals)
512 # now the fun bit, updating the multilinks ;)
513 for col, (add, remove) in multilink_changes.items():
514 tn = '%s_%s'%(classname, col)
515 if add:
516 sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
517 self.arg, self.arg)
518 for addid in add:
519 self.sql(cursor, sql, (nodeid, addid))
520 if remove:
521 sql = 'delete from %s where nodeid=%s and linkid=%s'%(tn,
522 self.arg, self.arg)
523 for removeid in remove:
524 self.sql(cursor, sql, (nodeid, removeid))
526 # make sure we do the commit-time extra stuff for this node
527 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
529 def getnode(self, classname, nodeid):
530 ''' Get a node from the database.
531 '''
532 if __debug__:
533 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid)
534 # figure the columns we're fetching
535 cl = self.classes[classname]
536 cols, mls = self.determine_columns(cl.properties.items())
537 scols = ','.join(cols)
539 # perform the basic property fetch
540 cursor = self.conn.cursor()
541 sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
542 self.sql(cursor, sql, (nodeid,))
544 values = self.sql_fetchone(cursor)
545 if values is None:
546 raise IndexError, 'no such %s node %s'%(classname, nodeid)
548 # make up the node
549 node = {}
550 for col in range(len(cols)):
551 node[cols[col][1:]] = values[col]
553 # now the multilinks
554 for col in mls:
555 # get the link ids
556 sql = 'select linkid from %s_%s where nodeid=%s'%(classname, col,
557 self.arg)
558 cursor.execute(sql, (nodeid,))
559 # extract the first column from the result
560 node[col] = [x[0] for x in cursor.fetchall()]
562 return self.unserialise(classname, node)
564 def destroynode(self, classname, nodeid):
565 '''Remove a node from the database. Called exclusively by the
566 destroy() method on Class.
567 '''
568 if __debug__:
569 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
571 # make sure the node exists
572 if not self.hasnode(classname, nodeid):
573 raise IndexError, '%s has no node %s'%(classname, nodeid)
575 # see if there's any obvious commit actions that we should get rid of
576 for entry in self.transactions[:]:
577 if entry[1][:2] == (classname, nodeid):
578 self.transactions.remove(entry)
580 # now do the SQL
581 cursor = self.conn.cursor()
582 sql = 'delete from _%s where id=%s'%(classname, self.arg)
583 self.sql(cursor, sql, (nodeid,))
585 # remove from multilnks
586 cl = self.getclass(classname)
587 x, mls = self.determine_columns(cl.properties.items())
588 for col in mls:
589 # get the link ids
590 sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
591 cursor.execute(sql, (nodeid,))
593 # remove journal entries
594 sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
595 self.sql(cursor, sql, (nodeid,))
597 def serialise(self, classname, node):
598 '''Copy the node contents, converting non-marshallable data into
599 marshallable data.
600 '''
601 if __debug__:
602 print >>hyperdb.DEBUG, 'serialise', classname, node
603 properties = self.getclass(classname).getprops()
604 d = {}
605 for k, v in node.items():
606 # if the property doesn't exist, or is the "retired" flag then
607 # it won't be in the properties dict
608 if not properties.has_key(k):
609 d[k] = v
610 continue
612 # get the property spec
613 prop = properties[k]
615 if isinstance(prop, Password):
616 d[k] = str(v)
617 elif isinstance(prop, Date) and v is not None:
618 d[k] = v.serialise()
619 elif isinstance(prop, Interval) and v is not None:
620 d[k] = v.serialise()
621 else:
622 d[k] = v
623 return d
625 def unserialise(self, classname, node):
626 '''Decode the marshalled node data
627 '''
628 if __debug__:
629 print >>hyperdb.DEBUG, 'unserialise', classname, node
630 properties = self.getclass(classname).getprops()
631 d = {}
632 for k, v in node.items():
633 # if the property doesn't exist, or is the "retired" flag then
634 # it won't be in the properties dict
635 if not properties.has_key(k):
636 d[k] = v
637 continue
639 # get the property spec
640 prop = properties[k]
642 if isinstance(prop, Date) and v is not None:
643 d[k] = date.Date(v)
644 elif isinstance(prop, Interval) and v is not None:
645 d[k] = date.Interval(v)
646 elif isinstance(prop, Password):
647 p = password.Password()
648 p.unpack(v)
649 d[k] = p
650 else:
651 d[k] = v
652 return d
654 def hasnode(self, classname, nodeid):
655 ''' Determine if the database has a given node.
656 '''
657 cursor = self.conn.cursor()
658 sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
659 if __debug__:
660 print >>hyperdb.DEBUG, 'hasnode', (self, sql, nodeid)
661 cursor.execute(sql, (nodeid,))
662 return int(cursor.fetchone()[0])
664 def countnodes(self, classname):
665 ''' Count the number of nodes that exist for a particular Class.
666 '''
667 cursor = self.conn.cursor()
668 sql = 'select count(*) from _%s'%classname
669 if __debug__:
670 print >>hyperdb.DEBUG, 'countnodes', (self, sql)
671 cursor.execute(sql)
672 return cursor.fetchone()[0]
674 def getnodeids(self, classname, retired=0):
675 ''' Retrieve all the ids of the nodes for a particular Class.
677 Set retired=None to get all nodes. Otherwise it'll get all the
678 retired or non-retired nodes, depending on the flag.
679 '''
680 cursor = self.conn.cursor()
681 # flip the sense of the flag if we don't want all of them
682 if retired is not None:
683 retired = not retired
684 sql = 'select id from _%s where __retired__ <> %s'%(classname, self.arg)
685 if __debug__:
686 print >>hyperdb.DEBUG, 'getnodeids', (self, sql, retired)
687 cursor.execute(sql, (retired,))
688 return [x[0] for x in cursor.fetchall()]
690 def addjournal(self, classname, nodeid, action, params, creator=None,
691 creation=None):
692 ''' Journal the Action
693 'action' may be:
695 'create' or 'set' -- 'params' is a dictionary of property values
696 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
697 'retire' -- 'params' is None
698 '''
699 # serialise the parameters now if necessary
700 if isinstance(params, type({})):
701 if action in ('set', 'create'):
702 params = self.serialise(classname, params)
704 # handle supply of the special journalling parameters (usually
705 # supplied on importing an existing database)
706 if creator:
707 journaltag = creator
708 else:
709 journaltag = self.journaltag
710 if creation:
711 journaldate = creation.serialise()
712 else:
713 journaldate = date.Date().serialise()
715 # create the journal entry
716 cols = ','.join('nodeid date tag action params'.split())
718 if __debug__:
719 print >>hyperdb.DEBUG, 'addjournal', (nodeid, journaldate,
720 journaltag, action, params)
722 cursor = self.conn.cursor()
723 self.save_journal(cursor, classname, cols, nodeid, journaldate,
724 journaltag, action, params)
726 def save_journal(self, cursor, classname, cols, nodeid, journaldate,
727 journaltag, action, params):
728 ''' Save the journal entry to the database
729 '''
730 raise NotImplemented
732 def getjournal(self, classname, nodeid):
733 ''' get the journal for id
734 '''
735 # make sure the node exists
736 if not self.hasnode(classname, nodeid):
737 raise IndexError, '%s has no node %s'%(classname, nodeid)
739 cursor = self.conn.cursor()
740 cols = ','.join('nodeid date tag action params'.split())
741 return self.load_journal(cursor, classname, cols, nodeid)
743 def load_journal(self, cursor, classname, cols, nodeid):
744 ''' Load the journal from the database
745 '''
746 raise NotImplemented
748 def pack(self, pack_before):
749 ''' Delete all journal entries except "create" before 'pack_before'.
750 '''
751 # get a 'yyyymmddhhmmss' version of the date
752 date_stamp = pack_before.serialise()
754 # do the delete
755 cursor = self.conn.cursor()
756 for classname in self.classes.keys():
757 sql = "delete from %s__journal where date<%s and "\
758 "action<>'create'"%(classname, self.arg)
759 if __debug__:
760 print >>hyperdb.DEBUG, 'pack', (self, sql, date_stamp)
761 cursor.execute(sql, (date_stamp,))
763 def sql_commit(self):
764 ''' Actually commit to the database.
765 '''
766 self.conn.commit()
768 def commit(self):
769 ''' Commit the current transactions.
771 Save all data changed since the database was opened or since the
772 last commit() or rollback().
773 '''
774 if __debug__:
775 print >>hyperdb.DEBUG, 'commit', (self,)
777 # commit the database
778 self.sql_commit()
780 # now, do all the other transaction stuff
781 reindex = {}
782 for method, args in self.transactions:
783 reindex[method(*args)] = 1
785 # reindex the nodes that request it
786 for classname, nodeid in filter(None, reindex.keys()):
787 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
788 self.getclass(classname).index(nodeid)
790 # save the indexer state
791 self.indexer.save_index()
793 # clear out the transactions
794 self.transactions = []
796 def rollback(self):
797 ''' Reverse all actions from the current transaction.
799 Undo all the changes made since the database was opened or the last
800 commit() or rollback() was performed.
801 '''
802 if __debug__:
803 print >>hyperdb.DEBUG, 'rollback', (self,)
805 # roll back
806 self.conn.rollback()
808 # roll back "other" transaction stuff
809 for method, args in self.transactions:
810 # delete temporary files
811 if method == self.doStoreFile:
812 self.rollbackStoreFile(*args)
813 self.transactions = []
815 def doSaveNode(self, classname, nodeid, node):
816 ''' dummy that just generates a reindex event
817 '''
818 # return the classname, nodeid so we reindex this content
819 return (classname, nodeid)
821 def close(self):
822 ''' Close off the connection.
823 '''
824 self.conn.close()
826 #
827 # The base Class class
828 #
829 class Class(hyperdb.Class):
830 ''' The handle to a particular class of nodes in a hyperdatabase.
832 All methods except __repr__ and getnode must be implemented by a
833 concrete backend Class.
834 '''
836 def __init__(self, db, classname, **properties):
837 '''Create a new class with a given name and property specification.
839 'classname' must not collide with the name of an existing class,
840 or a ValueError is raised. The keyword arguments in 'properties'
841 must map names to property objects, or a TypeError is raised.
842 '''
843 if (properties.has_key('creation') or properties.has_key('activity')
844 or properties.has_key('creator')):
845 raise ValueError, '"creation", "activity" and "creator" are '\
846 'reserved'
848 self.classname = classname
849 self.properties = properties
850 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
851 self.key = ''
853 # should we journal changes (default yes)
854 self.do_journal = 1
856 # do the db-related init stuff
857 db.addclass(self)
859 self.auditors = {'create': [], 'set': [], 'retire': []}
860 self.reactors = {'create': [], 'set': [], 'retire': []}
862 def schema(self):
863 ''' A dumpable version of the schema that we can store in the
864 database
865 '''
866 return (self.key, [(x, repr(y)) for x,y in self.properties.items()])
868 def enableJournalling(self):
869 '''Turn journalling on for this class
870 '''
871 self.do_journal = 1
873 def disableJournalling(self):
874 '''Turn journalling off for this class
875 '''
876 self.do_journal = 0
878 # Editing nodes:
879 def create(self, **propvalues):
880 ''' Create a new node of this class and return its id.
882 The keyword arguments in 'propvalues' map property names to values.
884 The values of arguments must be acceptable for the types of their
885 corresponding properties or a TypeError is raised.
887 If this class has a key property, it must be present and its value
888 must not collide with other key strings or a ValueError is raised.
890 Any other properties on this class that are missing from the
891 'propvalues' dictionary are set to None.
893 If an id in a link or multilink property does not refer to a valid
894 node, an IndexError is raised.
895 '''
896 if propvalues.has_key('id'):
897 raise KeyError, '"id" is reserved'
899 if self.db.journaltag is None:
900 raise DatabaseError, 'Database open read-only'
902 if propvalues.has_key('creation') or propvalues.has_key('activity'):
903 raise KeyError, '"creation" and "activity" are reserved'
905 self.fireAuditors('create', None, propvalues)
907 # new node's id
908 newid = self.db.newid(self.classname)
910 # validate propvalues
911 num_re = re.compile('^\d+$')
912 for key, value in propvalues.items():
913 if key == self.key:
914 try:
915 self.lookup(value)
916 except KeyError:
917 pass
918 else:
919 raise ValueError, 'node with key "%s" exists'%value
921 # try to handle this property
922 try:
923 prop = self.properties[key]
924 except KeyError:
925 raise KeyError, '"%s" has no property "%s"'%(self.classname,
926 key)
928 if value is not None and isinstance(prop, Link):
929 if type(value) != type(''):
930 raise ValueError, 'link value must be String'
931 link_class = self.properties[key].classname
932 # if it isn't a number, it's a key
933 if not num_re.match(value):
934 try:
935 value = self.db.classes[link_class].lookup(value)
936 except (TypeError, KeyError):
937 raise IndexError, 'new property "%s": %s not a %s'%(
938 key, value, link_class)
939 elif not self.db.getclass(link_class).hasnode(value):
940 raise IndexError, '%s has no node %s'%(link_class, value)
942 # save off the value
943 propvalues[key] = value
945 # register the link with the newly linked node
946 if self.do_journal and self.properties[key].do_journal:
947 self.db.addjournal(link_class, value, 'link',
948 (self.classname, newid, key))
950 elif isinstance(prop, Multilink):
951 if type(value) != type([]):
952 raise TypeError, 'new property "%s" not a list of ids'%key
954 # clean up and validate the list of links
955 link_class = self.properties[key].classname
956 l = []
957 for entry in value:
958 if type(entry) != type(''):
959 raise ValueError, '"%s" multilink value (%r) '\
960 'must contain Strings'%(key, value)
961 # if it isn't a number, it's a key
962 if not num_re.match(entry):
963 try:
964 entry = self.db.classes[link_class].lookup(entry)
965 except (TypeError, KeyError):
966 raise IndexError, 'new property "%s": %s not a %s'%(
967 key, entry, self.properties[key].classname)
968 l.append(entry)
969 value = l
970 propvalues[key] = value
972 # handle additions
973 for nodeid in value:
974 if not self.db.getclass(link_class).hasnode(nodeid):
975 raise IndexError, '%s has no node %s'%(link_class,
976 nodeid)
977 # register the link with the newly linked node
978 if self.do_journal and self.properties[key].do_journal:
979 self.db.addjournal(link_class, nodeid, 'link',
980 (self.classname, newid, key))
982 elif isinstance(prop, String):
983 if type(value) != type(''):
984 raise TypeError, 'new property "%s" not a string'%key
986 elif isinstance(prop, Password):
987 if not isinstance(value, password.Password):
988 raise TypeError, 'new property "%s" not a Password'%key
990 elif isinstance(prop, Date):
991 if value is not None and not isinstance(value, date.Date):
992 raise TypeError, 'new property "%s" not a Date'%key
994 elif isinstance(prop, Interval):
995 if value is not None and not isinstance(value, date.Interval):
996 raise TypeError, 'new property "%s" not an Interval'%key
998 elif value is not None and isinstance(prop, Number):
999 try:
1000 float(value)
1001 except ValueError:
1002 raise TypeError, 'new property "%s" not numeric'%key
1004 elif value is not None and isinstance(prop, Boolean):
1005 try:
1006 int(value)
1007 except ValueError:
1008 raise TypeError, 'new property "%s" not boolean'%key
1010 # make sure there's data where there needs to be
1011 for key, prop in self.properties.items():
1012 if propvalues.has_key(key):
1013 continue
1014 if key == self.key:
1015 raise ValueError, 'key property "%s" is required'%key
1016 if isinstance(prop, Multilink):
1017 propvalues[key] = []
1018 else:
1019 propvalues[key] = None
1021 # done
1022 self.db.addnode(self.classname, newid, propvalues)
1023 if self.do_journal:
1024 self.db.addjournal(self.classname, newid, 'create', propvalues)
1026 self.fireReactors('create', newid, None)
1028 return newid
1030 def export_list(self, propnames, nodeid):
1031 ''' Export a node - generate a list of CSV-able data in the order
1032 specified by propnames for the given node.
1033 '''
1034 properties = self.getprops()
1035 l = []
1036 for prop in propnames:
1037 proptype = properties[prop]
1038 value = self.get(nodeid, prop)
1039 # "marshal" data where needed
1040 if value is None:
1041 pass
1042 elif isinstance(proptype, hyperdb.Date):
1043 value = value.get_tuple()
1044 elif isinstance(proptype, hyperdb.Interval):
1045 value = value.get_tuple()
1046 elif isinstance(proptype, hyperdb.Password):
1047 value = str(value)
1048 l.append(repr(value))
1049 return l
1051 def import_list(self, propnames, proplist):
1052 ''' Import a node - all information including "id" is present and
1053 should not be sanity checked. Triggers are not triggered. The
1054 journal should be initialised using the "creator" and "created"
1055 information.
1057 Return the nodeid of the node imported.
1058 '''
1059 if self.db.journaltag is None:
1060 raise DatabaseError, 'Database open read-only'
1061 properties = self.getprops()
1063 # make the new node's property map
1064 d = {}
1065 for i in range(len(propnames)):
1066 # Use eval to reverse the repr() used to output the CSV
1067 value = eval(proplist[i])
1069 # Figure the property for this column
1070 propname = propnames[i]
1071 prop = properties[propname]
1073 # "unmarshal" where necessary
1074 if propname == 'id':
1075 newid = value
1076 continue
1077 elif value is None:
1078 # don't set Nones
1079 continue
1080 elif isinstance(prop, hyperdb.Date):
1081 value = date.Date(value)
1082 elif isinstance(prop, hyperdb.Interval):
1083 value = date.Interval(value)
1084 elif isinstance(prop, hyperdb.Password):
1085 pwd = password.Password()
1086 pwd.unpack(value)
1087 value = pwd
1088 d[propname] = value
1090 # extract the extraneous journalling gumpf and nuke it
1091 if d.has_key('creator'):
1092 creator = d['creator']
1093 del d['creator']
1094 if d.has_key('creation'):
1095 creation = d['creation']
1096 del d['creation']
1097 if d.has_key('activity'):
1098 del d['activity']
1100 # add the node and journal
1101 self.db.addnode(self.classname, newid, d)
1102 self.db.addjournal(self.classname, newid, 'create', d, creator,
1103 creation)
1104 return newid
1106 _marker = []
1107 def get(self, nodeid, propname, default=_marker, cache=1):
1108 '''Get the value of a property on an existing node of this class.
1110 'nodeid' must be the id of an existing node of this class or an
1111 IndexError is raised. 'propname' must be the name of a property
1112 of this class or a KeyError is raised.
1114 'cache' indicates whether the transaction cache should be queried
1115 for the node. If the node has been modified and you need to
1116 determine what its values prior to modification are, you need to
1117 set cache=0.
1118 '''
1119 if propname == 'id':
1120 return nodeid
1122 if propname == 'creation':
1123 if not self.do_journal:
1124 raise ValueError, 'Journalling is disabled for this class'
1125 journal = self.db.getjournal(self.classname, nodeid)
1126 if journal:
1127 return self.db.getjournal(self.classname, nodeid)[0][1]
1128 else:
1129 # on the strange chance that there's no journal
1130 return date.Date()
1131 if propname == 'activity':
1132 if not self.do_journal:
1133 raise ValueError, 'Journalling is disabled for this class'
1134 journal = self.db.getjournal(self.classname, nodeid)
1135 if journal:
1136 return self.db.getjournal(self.classname, nodeid)[-1][1]
1137 else:
1138 # on the strange chance that there's no journal
1139 return date.Date()
1140 if propname == 'creator':
1141 if not self.do_journal:
1142 raise ValueError, 'Journalling is disabled for this class'
1143 journal = self.db.getjournal(self.classname, nodeid)
1144 if journal:
1145 name = self.db.getjournal(self.classname, nodeid)[0][2]
1146 else:
1147 return None
1148 try:
1149 return self.db.user.lookup(name)
1150 except KeyError:
1151 # the journaltag user doesn't exist any more
1152 return None
1154 # get the property (raises KeyErorr if invalid)
1155 prop = self.properties[propname]
1157 # get the node's dict
1158 d = self.db.getnode(self.classname, nodeid) #, cache=cache)
1160 if not d.has_key(propname):
1161 if default is self._marker:
1162 if isinstance(prop, Multilink):
1163 return []
1164 else:
1165 return None
1166 else:
1167 return default
1169 # don't pass our list to other code
1170 if isinstance(prop, Multilink):
1171 return d[propname][:]
1173 return d[propname]
1175 def getnode(self, nodeid, cache=1):
1176 ''' Return a convenience wrapper for the node.
1178 'nodeid' must be the id of an existing node of this class or an
1179 IndexError is raised.
1181 'cache' indicates whether the transaction cache should be queried
1182 for the node. If the node has been modified and you need to
1183 determine what its values prior to modification are, you need to
1184 set cache=0.
1185 '''
1186 return Node(self, nodeid, cache=cache)
1188 def set(self, nodeid, **propvalues):
1189 '''Modify a property on an existing node of this class.
1191 'nodeid' must be the id of an existing node of this class or an
1192 IndexError is raised.
1194 Each key in 'propvalues' must be the name of a property of this
1195 class or a KeyError is raised.
1197 All values in 'propvalues' must be acceptable types for their
1198 corresponding properties or a TypeError is raised.
1200 If the value of the key property is set, it must not collide with
1201 other key strings or a ValueError is raised.
1203 If the value of a Link or Multilink property contains an invalid
1204 node id, a ValueError is raised.
1205 '''
1206 if not propvalues:
1207 return propvalues
1209 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1210 raise KeyError, '"creation" and "activity" are reserved'
1212 if propvalues.has_key('id'):
1213 raise KeyError, '"id" is reserved'
1215 if self.db.journaltag is None:
1216 raise DatabaseError, 'Database open read-only'
1218 self.fireAuditors('set', nodeid, propvalues)
1219 # Take a copy of the node dict so that the subsequent set
1220 # operation doesn't modify the oldvalues structure.
1221 # XXX used to try the cache here first
1222 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1224 node = self.db.getnode(self.classname, nodeid)
1225 if self.is_retired(nodeid):
1226 raise IndexError, 'Requested item is retired'
1227 num_re = re.compile('^\d+$')
1229 # if the journal value is to be different, store it in here
1230 journalvalues = {}
1232 # remember the add/remove stuff for multilinks, making it easier
1233 # for the Database layer to do its stuff
1234 multilink_changes = {}
1236 for propname, value in propvalues.items():
1237 # check to make sure we're not duplicating an existing key
1238 if propname == self.key and node[propname] != value:
1239 try:
1240 self.lookup(value)
1241 except KeyError:
1242 pass
1243 else:
1244 raise ValueError, 'node with key "%s" exists'%value
1246 # this will raise the KeyError if the property isn't valid
1247 # ... we don't use getprops() here because we only care about
1248 # the writeable properties.
1249 prop = self.properties[propname]
1251 # if the value's the same as the existing value, no sense in
1252 # doing anything
1253 if node.has_key(propname) and value == node[propname]:
1254 del propvalues[propname]
1255 continue
1257 # do stuff based on the prop type
1258 if isinstance(prop, Link):
1259 link_class = prop.classname
1260 # if it isn't a number, it's a key
1261 if value is not None and not isinstance(value, type('')):
1262 raise ValueError, 'property "%s" link value be a string'%(
1263 propname)
1264 if isinstance(value, type('')) and not num_re.match(value):
1265 try:
1266 value = self.db.classes[link_class].lookup(value)
1267 except (TypeError, KeyError):
1268 raise IndexError, 'new property "%s": %s not a %s'%(
1269 propname, value, prop.classname)
1271 if (value is not None and
1272 not self.db.getclass(link_class).hasnode(value)):
1273 raise IndexError, '%s has no node %s'%(link_class, value)
1275 if self.do_journal and prop.do_journal:
1276 # register the unlink with the old linked node
1277 if node[propname] is not None:
1278 self.db.addjournal(link_class, node[propname], 'unlink',
1279 (self.classname, nodeid, propname))
1281 # register the link with the newly linked node
1282 if value is not None:
1283 self.db.addjournal(link_class, value, 'link',
1284 (self.classname, nodeid, propname))
1286 elif isinstance(prop, Multilink):
1287 if type(value) != type([]):
1288 raise TypeError, 'new property "%s" not a list of'\
1289 ' ids'%propname
1290 link_class = self.properties[propname].classname
1291 l = []
1292 for entry in value:
1293 # if it isn't a number, it's a key
1294 if type(entry) != type(''):
1295 raise ValueError, 'new property "%s" link value ' \
1296 'must be a string'%propname
1297 if not num_re.match(entry):
1298 try:
1299 entry = self.db.classes[link_class].lookup(entry)
1300 except (TypeError, KeyError):
1301 raise IndexError, 'new property "%s": %s not a %s'%(
1302 propname, entry,
1303 self.properties[propname].classname)
1304 l.append(entry)
1305 value = l
1306 propvalues[propname] = value
1308 # figure the journal entry for this property
1309 add = []
1310 remove = []
1312 # handle removals
1313 if node.has_key(propname):
1314 l = node[propname]
1315 else:
1316 l = []
1317 for id in l[:]:
1318 if id in value:
1319 continue
1320 # register the unlink with the old linked node
1321 if self.do_journal and self.properties[propname].do_journal:
1322 self.db.addjournal(link_class, id, 'unlink',
1323 (self.classname, nodeid, propname))
1324 l.remove(id)
1325 remove.append(id)
1327 # handle additions
1328 for id in value:
1329 if not self.db.getclass(link_class).hasnode(id):
1330 raise IndexError, '%s has no node %s'%(link_class, id)
1331 if id in l:
1332 continue
1333 # register the link with the newly linked node
1334 if self.do_journal and self.properties[propname].do_journal:
1335 self.db.addjournal(link_class, id, 'link',
1336 (self.classname, nodeid, propname))
1337 l.append(id)
1338 add.append(id)
1340 # figure the journal entry
1341 l = []
1342 if add:
1343 l.append(('+', add))
1344 if remove:
1345 l.append(('-', remove))
1346 multilink_changes[propname] = (add, remove)
1347 if l:
1348 journalvalues[propname] = tuple(l)
1350 elif isinstance(prop, String):
1351 if value is not None and type(value) != type(''):
1352 raise TypeError, 'new property "%s" not a string'%propname
1354 elif isinstance(prop, Password):
1355 if not isinstance(value, password.Password):
1356 raise TypeError, 'new property "%s" not a Password'%propname
1357 propvalues[propname] = value
1359 elif value is not None and isinstance(prop, Date):
1360 if not isinstance(value, date.Date):
1361 raise TypeError, 'new property "%s" not a Date'% propname
1362 propvalues[propname] = value
1364 elif value is not None and isinstance(prop, Interval):
1365 if not isinstance(value, date.Interval):
1366 raise TypeError, 'new property "%s" not an '\
1367 'Interval'%propname
1368 propvalues[propname] = value
1370 elif value is not None and isinstance(prop, Number):
1371 try:
1372 float(value)
1373 except ValueError:
1374 raise TypeError, 'new property "%s" not numeric'%propname
1376 elif value is not None and isinstance(prop, Boolean):
1377 try:
1378 int(value)
1379 except ValueError:
1380 raise TypeError, 'new property "%s" not boolean'%propname
1382 node[propname] = value
1384 # nothing to do?
1385 if not propvalues:
1386 return propvalues
1388 # do the set, and journal it
1389 self.db.setnode(self.classname, nodeid, node, multilink_changes)
1391 if self.do_journal:
1392 propvalues.update(journalvalues)
1393 self.db.addjournal(self.classname, nodeid, 'set', propvalues)
1395 self.fireReactors('set', nodeid, oldvalues)
1397 return propvalues
1399 def retire(self, nodeid):
1400 '''Retire a node.
1402 The properties on the node remain available from the get() method,
1403 and the node's id is never reused.
1405 Retired nodes are not returned by the find(), list(), or lookup()
1406 methods, and other nodes may reuse the values of their key properties.
1407 '''
1408 if self.db.journaltag is None:
1409 raise DatabaseError, 'Database open read-only'
1411 cursor = self.db.conn.cursor()
1412 sql = 'update _%s set __retired__=1 where id=%s'%(self.classname,
1413 self.db.arg)
1414 if __debug__:
1415 print >>hyperdb.DEBUG, 'retire', (self, sql, nodeid)
1416 cursor.execute(sql, (nodeid,))
1418 def is_retired(self, nodeid):
1419 '''Return true if the node is rerired
1420 '''
1421 cursor = self.db.conn.cursor()
1422 sql = 'select __retired__ from _%s where id=%s'%(self.classname,
1423 self.db.arg)
1424 if __debug__:
1425 print >>hyperdb.DEBUG, 'is_retired', (self, sql, nodeid)
1426 cursor.execute(sql, (nodeid,))
1427 return int(cursor.fetchone()[0])
1429 def destroy(self, nodeid):
1430 '''Destroy a node.
1432 WARNING: this method should never be used except in extremely rare
1433 situations where there could never be links to the node being
1434 deleted
1435 WARNING: use retire() instead
1436 WARNING: the properties of this node will not be available ever again
1437 WARNING: really, use retire() instead
1439 Well, I think that's enough warnings. This method exists mostly to
1440 support the session storage of the cgi interface.
1442 The node is completely removed from the hyperdb, including all journal
1443 entries. It will no longer be available, and will generally break code
1444 if there are any references to the node.
1445 '''
1446 if self.db.journaltag is None:
1447 raise DatabaseError, 'Database open read-only'
1448 self.db.destroynode(self.classname, nodeid)
1450 def history(self, nodeid):
1451 '''Retrieve the journal of edits on a particular node.
1453 'nodeid' must be the id of an existing node of this class or an
1454 IndexError is raised.
1456 The returned list contains tuples of the form
1458 (date, tag, action, params)
1460 'date' is a Timestamp object specifying the time of the change and
1461 'tag' is the journaltag specified when the database was opened.
1462 '''
1463 if not self.do_journal:
1464 raise ValueError, 'Journalling is disabled for this class'
1465 return self.db.getjournal(self.classname, nodeid)
1467 # Locating nodes:
1468 def hasnode(self, nodeid):
1469 '''Determine if the given nodeid actually exists
1470 '''
1471 return self.db.hasnode(self.classname, nodeid)
1473 def setkey(self, propname):
1474 '''Select a String property of this class to be the key property.
1476 'propname' must be the name of a String property of this class or
1477 None, or a TypeError is raised. The values of the key property on
1478 all existing nodes must be unique or a ValueError is raised.
1479 '''
1480 # XXX create an index on the key prop column
1481 prop = self.getprops()[propname]
1482 if not isinstance(prop, String):
1483 raise TypeError, 'key properties must be String'
1484 self.key = propname
1486 def getkey(self):
1487 '''Return the name of the key property for this class or None.'''
1488 return self.key
1490 def labelprop(self, default_to_id=0):
1491 ''' Return the property name for a label for the given node.
1493 This method attempts to generate a consistent label for the node.
1494 It tries the following in order:
1495 1. key property
1496 2. "name" property
1497 3. "title" property
1498 4. first property from the sorted property name list
1499 '''
1500 k = self.getkey()
1501 if k:
1502 return k
1503 props = self.getprops()
1504 if props.has_key('name'):
1505 return 'name'
1506 elif props.has_key('title'):
1507 return 'title'
1508 if default_to_id:
1509 return 'id'
1510 props = props.keys()
1511 props.sort()
1512 return props[0]
1514 def lookup(self, keyvalue):
1515 '''Locate a particular node by its key property and return its id.
1517 If this class has no key property, a TypeError is raised. If the
1518 'keyvalue' matches one of the values for the key property among
1519 the nodes in this class, the matching node's id is returned;
1520 otherwise a KeyError is raised.
1521 '''
1522 if not self.key:
1523 raise TypeError, 'No key property set for class %s'%self.classname
1525 cursor = self.db.conn.cursor()
1526 sql = 'select id from _%s where _%s=%s'%(self.classname, self.key,
1527 self.db.arg)
1528 if __debug__:
1529 print >>hyperdb.DEBUG, 'lookup', (self, sql, keyvalue)
1530 cursor.execute(sql, (keyvalue,))
1532 # see if there was a result
1533 l = cursor.fetchall()
1534 if not l:
1535 raise KeyError, keyvalue
1537 # return the id
1538 return l[0][0]
1540 def find(self, **propspec):
1541 '''Get the ids of nodes in this class which link to the given nodes.
1543 'propspec' consists of keyword args propname={nodeid:1,}
1544 'propname' must be the name of a property in this class, or a
1545 KeyError is raised. That property must be a Link or Multilink
1546 property, or a TypeError is raised.
1548 Any node in this class whose 'propname' property links to any of the
1549 nodeids will be returned. Used by the full text indexing, which knows
1550 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1551 issues:
1553 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1554 '''
1555 if __debug__:
1556 print >>hyperdb.DEBUG, 'find', (self, propspec)
1557 if not propspec:
1558 return []
1559 queries = []
1560 tables = []
1561 allvalues = ()
1562 for prop, values in propspec.items():
1563 allvalues += tuple(values.keys())
1564 a = self.db.arg
1565 tables.append('select nodeid from %s_%s where linkid in (%s)'%(
1566 self.classname, prop, ','.join([a for x in values.keys()])))
1567 sql = '\nintersect\n'.join(tables)
1568 if __debug__:
1569 print >>hyperdb.DEBUG, 'find', (self, sql, allvalues)
1570 cursor = self.db.conn.cursor()
1571 cursor.execute(sql, allvalues)
1572 try:
1573 l = [x[0] for x in cursor.fetchall()]
1574 except gadfly.database.error, message:
1575 if message == 'no more results':
1576 l = []
1577 raise
1578 if __debug__:
1579 print >>hyperdb.DEBUG, 'find ... ', l
1580 return l
1582 def list(self):
1583 ''' Return a list of the ids of the active nodes in this class.
1584 '''
1585 return self.db.getnodeids(self.classname, retired=0)
1587 def filter(self, search_matches, filterspec, sort, group):
1588 ''' Return a list of the ids of the active nodes in this class that
1589 match the 'filter' spec, sorted by the group spec and then the
1590 sort spec
1592 "filterspec" is {propname: value(s)}
1593 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1594 and prop is a prop name or None
1595 "search_matches" is {nodeid: marker}
1597 The filter must match all properties specificed - but if the
1598 property value to match is a list, any one of the values in the
1599 list may match for that property to match.
1600 '''
1601 cn = self.classname
1603 # figure the WHERE clause from the filterspec
1604 props = self.getprops()
1605 frum = ['_'+cn]
1606 where = []
1607 args = []
1608 a = self.db.arg
1609 for k, v in filterspec.items():
1610 propclass = props[k]
1611 # now do other where clause stuff
1612 if isinstance(propclass, Multilink):
1613 tn = '%s_%s'%(cn, k)
1614 frum.append(tn)
1615 if isinstance(v, type([])):
1616 s = ','.join([self.arg for x in v])
1617 where.append('id=%s.nodeid and %s.linkid in (%s)'%(tn,tn,s))
1618 args = args + v
1619 else:
1620 where.append('id=%s.nodeid and %s.linkid = %s'%(tn, tn, a))
1621 args.append(v)
1622 elif isinstance(propclass, String):
1623 if not isinstance(v, type([])):
1624 v = [v]
1626 # Quote the bits in the string that need it and then embed
1627 # in a "substring" search. Note - need to quote the '%' so
1628 # they make it through the python layer happily
1629 v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
1631 # now add to the where clause
1632 where.append(' or '.join(["_%s LIKE '%s'"%(k, s) for s in v]))
1633 # note: args are embedded in the query string now
1634 elif isinstance(propclass, Link):
1635 if isinstance(v, type([])):
1636 if '-1' in v:
1637 v.remove('-1')
1638 xtra = ' or _%s is NULL'%k
1639 s = ','.join([a for x in v])
1640 where.append('(_%s in (%s)%s)'%(k, s, xtra))
1641 args = args + v
1642 else:
1643 if v == '-1':
1644 v = None
1645 where.append('_%s is NULL'%k)
1646 else:
1647 where.append('_%s=%s'%(k, a))
1648 args.append(v)
1649 else:
1650 if isinstance(v, type([])):
1651 s = ','.join([a for x in v])
1652 where.append('_%s in (%s)'%(k, s))
1653 args = args + v
1654 else:
1655 where.append('_%s=%s'%(k, a))
1656 args.append(v)
1658 # add results of full text search
1659 if search_matches is not None:
1660 v = search_matches.keys()
1661 s = ','.join([a for x in v])
1662 where.append('id in (%s)'%s)
1663 args = args + v
1665 # "grouping" is just the first-order sorting in the SQL fetch
1666 # can modify it...)
1667 orderby = []
1668 ordercols = []
1669 if group[0] is not None and group[1] is not None:
1670 if group[0] != '-':
1671 orderby.append('_'+group[1])
1672 ordercols.append('_'+group[1])
1673 else:
1674 orderby.append('_'+group[1]+' desc')
1675 ordercols.append('_'+group[1])
1677 # now add in the sorting
1678 group = ''
1679 if sort[0] is not None and sort[1] is not None:
1680 direction, colname = sort
1681 if direction != '-':
1682 if colname == 'activity':
1683 orderby.append('activity')
1684 ordercols.append('max(%s__journal.date) as activity'%cn)
1685 frum.append('%s__journal'%cn)
1686 where.append('%s__journal.nodeid = _%s.id'%(cn, cn))
1687 # we need to group by id
1688 group = ' group by id'
1689 elif colname == 'id':
1690 orderby.append(colname)
1691 else:
1692 orderby.append('_'+colname)
1693 ordercols.append('_'+colname)
1694 else:
1695 if colname == 'activity':
1696 orderby.append('activity desc')
1697 ordercols.append('max(%s__journal.date) as activity'%cn)
1698 frum.append('%s__journal'%cn)
1699 where.append('%s__journal.nodeid = _%s.id'%(cn, cn))
1700 # we need to group by id
1701 group = ' group by id'
1702 elif colname == 'id':
1703 orderby.append(colname+' desc')
1704 ordercols.append(colname)
1705 else:
1706 orderby.append('_'+colname+' desc')
1707 ordercols.append('_'+colname)
1709 # construct the SQL
1710 frum = ','.join(frum)
1711 if where:
1712 where = ' where ' + (' and '.join(where))
1713 else:
1714 where = ''
1715 cols = ['id']
1716 if orderby:
1717 cols = cols + ordercols
1718 order = ' order by %s'%(','.join(orderby))
1719 else:
1720 order = ''
1721 cols = ','.join(cols)
1722 sql = 'select %s from %s %s%s%s'%(cols, frum, where, group, order)
1723 args = tuple(args)
1724 if __debug__:
1725 print >>hyperdb.DEBUG, 'filter', (self, sql, args)
1726 cursor = self.db.conn.cursor()
1727 print (sql, args)
1728 cursor.execute(sql, args)
1729 l = cursor.fetchall()
1730 print l
1732 # return the IDs (the first column)
1733 # XXX The filter(None, l) bit is sqlite-specific... if there's _NO_
1734 # XXX matches to a fetch, it returns NULL instead of nothing!?!
1735 return filter(None, [row[0] for row in l])
1737 def count(self):
1738 '''Get the number of nodes in this class.
1740 If the returned integer is 'numnodes', the ids of all the nodes
1741 in this class run from 1 to numnodes, and numnodes+1 will be the
1742 id of the next node to be created in this class.
1743 '''
1744 return self.db.countnodes(self.classname)
1746 # Manipulating properties:
1747 def getprops(self, protected=1):
1748 '''Return a dictionary mapping property names to property objects.
1749 If the "protected" flag is true, we include protected properties -
1750 those which may not be modified.
1751 '''
1752 d = self.properties.copy()
1753 if protected:
1754 d['id'] = String()
1755 d['creation'] = hyperdb.Date()
1756 d['activity'] = hyperdb.Date()
1757 d['creator'] = hyperdb.Link("user")
1758 return d
1760 def addprop(self, **properties):
1761 '''Add properties to this class.
1763 The keyword arguments in 'properties' must map names to property
1764 objects, or a TypeError is raised. None of the keys in 'properties'
1765 may collide with the names of existing properties, or a ValueError
1766 is raised before any properties have been added.
1767 '''
1768 for key in properties.keys():
1769 if self.properties.has_key(key):
1770 raise ValueError, key
1771 self.properties.update(properties)
1773 def index(self, nodeid):
1774 '''Add (or refresh) the node to search indexes
1775 '''
1776 # find all the String properties that have indexme
1777 for prop, propclass in self.getprops().items():
1778 if isinstance(propclass, String) and propclass.indexme:
1779 try:
1780 value = str(self.get(nodeid, prop))
1781 except IndexError:
1782 # node no longer exists - entry should be removed
1783 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1784 else:
1785 # and index them under (classname, nodeid, property)
1786 self.db.indexer.add_text((self.classname, nodeid, prop),
1787 value)
1790 #
1791 # Detector interface
1792 #
1793 def audit(self, event, detector):
1794 '''Register a detector
1795 '''
1796 l = self.auditors[event]
1797 if detector not in l:
1798 self.auditors[event].append(detector)
1800 def fireAuditors(self, action, nodeid, newvalues):
1801 '''Fire all registered auditors.
1802 '''
1803 for audit in self.auditors[action]:
1804 audit(self.db, self, nodeid, newvalues)
1806 def react(self, event, detector):
1807 '''Register a detector
1808 '''
1809 l = self.reactors[event]
1810 if detector not in l:
1811 self.reactors[event].append(detector)
1813 def fireReactors(self, action, nodeid, oldvalues):
1814 '''Fire all registered reactors.
1815 '''
1816 for react in self.reactors[action]:
1817 react(self.db, self, nodeid, oldvalues)
1819 class FileClass(Class):
1820 '''This class defines a large chunk of data. To support this, it has a
1821 mandatory String property "content" which is typically saved off
1822 externally to the hyperdb.
1824 The default MIME type of this data is defined by the
1825 "default_mime_type" class attribute, which may be overridden by each
1826 node if the class defines a "type" String property.
1827 '''
1828 default_mime_type = 'text/plain'
1830 def create(self, **propvalues):
1831 ''' snaffle the file propvalue and store in a file
1832 '''
1833 content = propvalues['content']
1834 del propvalues['content']
1835 newid = Class.create(self, **propvalues)
1836 self.db.storefile(self.classname, newid, None, content)
1837 return newid
1839 def import_list(self, propnames, proplist):
1840 ''' Trap the "content" property...
1841 '''
1842 # dupe this list so we don't affect others
1843 propnames = propnames[:]
1845 # extract the "content" property from the proplist
1846 i = propnames.index('content')
1847 content = eval(proplist[i])
1848 del propnames[i]
1849 del proplist[i]
1851 # do the normal import
1852 newid = Class.import_list(self, propnames, proplist)
1854 # save off the "content" file
1855 self.db.storefile(self.classname, newid, None, content)
1856 return newid
1858 _marker = []
1859 def get(self, nodeid, propname, default=_marker, cache=1):
1860 ''' trap the content propname and get it from the file
1861 '''
1863 poss_msg = 'Possibly a access right configuration problem.'
1864 if propname == 'content':
1865 try:
1866 return self.db.getfile(self.classname, nodeid, None)
1867 except IOError, (strerror):
1868 # BUG: by catching this we donot see an error in the log.
1869 return 'ERROR reading file: %s%s\n%s\n%s'%(
1870 self.classname, nodeid, poss_msg, strerror)
1871 if default is not self._marker:
1872 return Class.get(self, nodeid, propname, default, cache=cache)
1873 else:
1874 return Class.get(self, nodeid, propname, cache=cache)
1876 def getprops(self, protected=1):
1877 ''' In addition to the actual properties on the node, these methods
1878 provide the "content" property. If the "protected" flag is true,
1879 we include protected properties - those which may not be
1880 modified.
1881 '''
1882 d = Class.getprops(self, protected=protected).copy()
1883 d['content'] = hyperdb.String()
1884 return d
1886 def index(self, nodeid):
1887 ''' Index the node in the search index.
1889 We want to index the content in addition to the normal String
1890 property indexing.
1891 '''
1892 # perform normal indexing
1893 Class.index(self, nodeid)
1895 # get the content to index
1896 content = self.get(nodeid, 'content')
1898 # figure the mime type
1899 if self.properties.has_key('type'):
1900 mime_type = self.get(nodeid, 'type')
1901 else:
1902 mime_type = self.default_mime_type
1904 # and index!
1905 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
1906 mime_type)
1908 # XXX deviation from spec - was called ItemClass
1909 class IssueClass(Class, roundupdb.IssueClass):
1910 # Overridden methods:
1911 def __init__(self, db, classname, **properties):
1912 '''The newly-created class automatically includes the "messages",
1913 "files", "nosy", and "superseder" properties. If the 'properties'
1914 dictionary attempts to specify any of these properties or a
1915 "creation" or "activity" property, a ValueError is raised.
1916 '''
1917 if not properties.has_key('title'):
1918 properties['title'] = hyperdb.String(indexme='yes')
1919 if not properties.has_key('messages'):
1920 properties['messages'] = hyperdb.Multilink("msg")
1921 if not properties.has_key('files'):
1922 properties['files'] = hyperdb.Multilink("file")
1923 if not properties.has_key('nosy'):
1924 # note: journalling is turned off as it really just wastes
1925 # space. this behaviour may be overridden in an instance
1926 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
1927 if not properties.has_key('superseder'):
1928 properties['superseder'] = hyperdb.Multilink(classname)
1929 Class.__init__(self, db, classname, **properties)