93d3e8bc463a26455f5406d26aafcfd482071c1d
1 # $Id: rdbms_common.py,v 1.7 2002-09-20 01:20:32 richard Exp $
3 # standard python modules
4 import sys, os, time, re, errno, weakref, copy
6 # roundup modules
7 from roundup import hyperdb, date, password, roundupdb, security
8 from roundup.hyperdb import String, Password, Date, Interval, Link, \
9 Multilink, DatabaseError, Boolean, Number
11 # support
12 from blobfiles import FileStorage
13 from roundup.indexer import Indexer
14 from sessions import Sessions
16 # number of rows to keep in memory
17 ROW_CACHE_SIZE = 100
19 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
20 ''' Wrapper around an SQL database that presents a hyperdb interface.
22 - some functionality is specific to the actual SQL database, hence
23 the sql_* methods that are NotImplemented
24 - we keep a cache of the latest ROW_CACHE_SIZE row fetches.
25 '''
26 def __init__(self, config, journaltag=None):
27 ''' Open the database and load the schema from it.
28 '''
29 self.config, self.journaltag = config, journaltag
30 self.dir = config.DATABASE
31 self.classes = {}
32 self.indexer = Indexer(self.dir)
33 self.sessions = Sessions(self.config)
34 self.security = security.Security(self)
36 # additional transaction support for external files and the like
37 self.transactions = []
39 # keep a cache of the N most recently retrieved rows of any kind
40 # (classname, nodeid) = row
41 self.cache = {}
42 self.cache_lru = []
44 # open a connection to the database, creating the "conn" attribute
45 self.open_connection()
47 def open_connection(self):
48 ''' Open a connection to the database, creating it if necessary
49 '''
50 raise NotImplemented
52 def sql(self, cursor, sql, args=None):
53 ''' Execute the sql with the optional args.
54 '''
55 if __debug__:
56 print >>hyperdb.DEBUG, (self, sql, args)
57 if args:
58 cursor.execute(sql, args)
59 else:
60 cursor.execute(sql)
62 def sql_fetchone(self, cursor):
63 ''' Fetch a single row. If there's nothing to fetch, return None.
64 '''
65 raise NotImplemented
67 def sql_stringquote(self, value):
68 ''' Quote the string so it's safe to put in the 'sql quotes'
69 '''
70 return re.sub("'", "''", str(value))
72 def save_dbschema(self, cursor, schema):
73 ''' Save the schema definition that the database currently implements
74 '''
75 raise NotImplemented
77 def load_dbschema(self, cursor):
78 ''' Load the schema definition that the database currently implements
79 '''
80 raise NotImplemented
82 def post_init(self):
83 ''' Called once the schema initialisation has finished.
85 We should now confirm that the schema defined by our "classes"
86 attribute actually matches the schema in the database.
87 '''
88 # now detect changes in the schema
89 save = 0
90 for classname, spec in self.classes.items():
91 if self.database_schema.has_key(classname):
92 dbspec = self.database_schema[classname]
93 if self.update_class(spec, dbspec):
94 self.database_schema[classname] = spec.schema()
95 save = 1
96 else:
97 self.create_class(spec)
98 self.database_schema[classname] = spec.schema()
99 save = 1
101 for classname in self.database_schema.keys():
102 if not self.classes.has_key(classname):
103 self.drop_class(classname)
105 # update the database version of the schema
106 if save:
107 cursor = self.conn.cursor()
108 self.sql(cursor, 'delete from schema')
109 self.save_dbschema(cursor, self.database_schema)
111 # reindex the db if necessary
112 if self.indexer.should_reindex():
113 self.reindex()
115 # commit
116 self.conn.commit()
118 def reindex(self):
119 for klass in self.classes.values():
120 for nodeid in klass.list():
121 klass.index(nodeid)
122 self.indexer.save_index()
124 def determine_columns(self, properties):
125 ''' Figure the column names and multilink properties from the spec
127 "properties" is a list of (name, prop) where prop may be an
128 instance of a hyperdb "type" _or_ a string repr of that type.
129 '''
130 cols = ['_activity', '_creator', '_creation']
131 mls = []
132 # add the multilinks separately
133 for col, prop in properties:
134 if isinstance(prop, Multilink):
135 mls.append(col)
136 elif isinstance(prop, type('')) and prop.find('Multilink') != -1:
137 mls.append(col)
138 else:
139 cols.append('_'+col)
140 cols.sort()
141 return cols, mls
143 def update_class(self, spec, dbspec):
144 ''' Determine the differences between the current spec and the
145 database version of the spec, and update where necessary
146 '''
147 spec_schema = spec.schema()
148 if spec_schema == dbspec:
149 # no save needed for this one
150 return 0
151 if __debug__:
152 print >>hyperdb.DEBUG, 'update_class FIRING'
154 # key property changed?
155 if dbspec[0] != spec_schema[0]:
156 if __debug__:
157 print >>hyperdb.DEBUG, 'update_class setting keyprop', `spec[0]`
158 # XXX turn on indexing for the key property
160 # dict 'em up
161 spec_propnames,spec_props = [],{}
162 for propname,prop in spec_schema[1]:
163 spec_propnames.append(propname)
164 spec_props[propname] = prop
165 dbspec_propnames,dbspec_props = [],{}
166 for propname,prop in dbspec[1]:
167 dbspec_propnames.append(propname)
168 dbspec_props[propname] = prop
170 # we're going to need one of these
171 cursor = self.conn.cursor()
173 # now compare
174 for propname in spec_propnames:
175 prop = spec_props[propname]
176 if dbspec_props.has_key(propname) and prop==dbspec_props[propname]:
177 continue
178 if __debug__:
179 print >>hyperdb.DEBUG, 'update_class ADD', (propname, prop)
181 if not dbspec_props.has_key(propname):
182 # add the property
183 if isinstance(prop, Multilink):
184 # all we have to do here is create a new table, easy!
185 self.create_multilink_table(cursor, spec, propname)
186 continue
188 # no ALTER TABLE, so we:
189 # 1. pull out the data, including an extra None column
190 oldcols, x = self.determine_columns(dbspec[1])
191 oldcols.append('id')
192 oldcols.append('__retired__')
193 cn = spec.classname
194 sql = 'select %s,%s from _%s'%(','.join(oldcols), self.arg, cn)
195 if __debug__:
196 print >>hyperdb.DEBUG, 'update_class', (self, sql, None)
197 cursor.execute(sql, (None,))
198 olddata = cursor.fetchall()
200 # 2. drop the old table
201 cursor.execute('drop table _%s'%cn)
203 # 3. create the new table
204 cols, mls = self.create_class_table(cursor, spec)
205 # ensure the new column is last
206 cols.remove('_'+propname)
207 assert oldcols == cols, "Column lists don't match!"
208 cols.append('_'+propname)
210 # 4. populate with the data from step one
211 s = ','.join([self.arg for x in cols])
212 scols = ','.join(cols)
213 sql = 'insert into _%s (%s) values (%s)'%(cn, scols, s)
215 # GAH, nothing had better go wrong from here on in... but
216 # we have to commit the drop...
217 # XXX this isn't necessary in sqlite :(
218 self.conn.commit()
220 # do the insert
221 for row in olddata:
222 self.sql(cursor, sql, tuple(row))
224 else:
225 # modify the property
226 if __debug__:
227 print >>hyperdb.DEBUG, 'update_class NOOP'
228 pass # NOOP in gadfly
230 # and the other way - only worry about deletions here
231 for propname in dbspec_propnames:
232 prop = dbspec_props[propname]
233 if spec_props.has_key(propname):
234 continue
235 if __debug__:
236 print >>hyperdb.DEBUG, 'update_class REMOVE', `prop`
238 # delete the property
239 if isinstance(prop, Multilink):
240 sql = 'drop table %s_%s'%(spec.classname, prop)
241 if __debug__:
242 print >>hyperdb.DEBUG, 'update_class', (self, sql)
243 cursor.execute(sql)
244 else:
245 # no ALTER TABLE, so we:
246 # 1. pull out the data, excluding the removed column
247 oldcols, x = self.determine_columns(spec.properties.items())
248 oldcols.append('id')
249 oldcols.append('__retired__')
250 # remove the missing column
251 oldcols.remove('_'+propname)
252 cn = spec.classname
253 sql = 'select %s from _%s'%(','.join(oldcols), cn)
254 cursor.execute(sql, (None,))
255 olddata = sql.fetchall()
257 # 2. drop the old table
258 cursor.execute('drop table _%s'%cn)
260 # 3. create the new table
261 cols, mls = self.create_class_table(self, cursor, spec)
262 assert oldcols != cols, "Column lists don't match!"
264 # 4. populate with the data from step one
265 qs = ','.join([self.arg for x in cols])
266 sql = 'insert into _%s values (%s)'%(cn, s)
267 cursor.execute(sql, olddata)
268 return 1
270 def create_class_table(self, cursor, spec):
271 ''' create the class table for the given spec
272 '''
273 cols, mls = self.determine_columns(spec.properties.items())
275 # add on our special columns
276 cols.append('id')
277 cols.append('__retired__')
279 # create the base table
280 scols = ','.join(['%s varchar'%x for x in cols])
281 sql = 'create table _%s (%s)'%(spec.classname, scols)
282 if __debug__:
283 print >>hyperdb.DEBUG, 'create_class', (self, sql)
284 cursor.execute(sql)
286 return cols, mls
288 def create_journal_table(self, cursor, spec):
289 ''' create the journal table for a class given the spec and
290 already-determined cols
291 '''
292 # journal table
293 cols = ','.join(['%s varchar'%x
294 for x in 'nodeid date tag action params'.split()])
295 sql = 'create table %s__journal (%s)'%(spec.classname, cols)
296 if __debug__:
297 print >>hyperdb.DEBUG, 'create_class', (self, sql)
298 cursor.execute(sql)
300 def create_multilink_table(self, cursor, spec, ml):
301 ''' Create a multilink table for the "ml" property of the class
302 given by the spec
303 '''
304 sql = 'create table %s_%s (linkid varchar, nodeid varchar)'%(
305 spec.classname, ml)
306 if __debug__:
307 print >>hyperdb.DEBUG, 'create_class', (self, sql)
308 cursor.execute(sql)
310 def create_class(self, spec):
311 ''' Create a database table according to the given spec.
312 '''
313 cursor = self.conn.cursor()
314 cols, mls = self.create_class_table(cursor, spec)
315 self.create_journal_table(cursor, spec)
317 # now create the multilink tables
318 for ml in mls:
319 self.create_multilink_table(cursor, spec, ml)
321 # ID counter
322 sql = 'insert into ids (name, num) values (%s,%s)'%(self.arg, self.arg)
323 vals = (spec.classname, 1)
324 if __debug__:
325 print >>hyperdb.DEBUG, 'create_class', (self, sql, vals)
326 cursor.execute(sql, vals)
328 def drop_class(self, spec):
329 ''' Drop the given table from the database.
331 Drop the journal and multilink tables too.
332 '''
333 # figure the multilinks
334 mls = []
335 for col, prop in spec.properties.items():
336 if isinstance(prop, Multilink):
337 mls.append(col)
338 cursor = self.conn.cursor()
340 sql = 'drop table _%s'%spec.classname
341 if __debug__:
342 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
343 cursor.execute(sql)
345 sql = 'drop table %s__journal'%spec.classname
346 if __debug__:
347 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
348 cursor.execute(sql)
350 for ml in mls:
351 sql = 'drop table %s_%s'%(spec.classname, ml)
352 if __debug__:
353 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
354 cursor.execute(sql)
356 #
357 # Classes
358 #
359 def __getattr__(self, classname):
360 ''' A convenient way of calling self.getclass(classname).
361 '''
362 if self.classes.has_key(classname):
363 if __debug__:
364 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
365 return self.classes[classname]
366 raise AttributeError, classname
368 def addclass(self, cl):
369 ''' Add a Class to the hyperdatabase.
370 '''
371 if __debug__:
372 print >>hyperdb.DEBUG, 'addclass', (self, cl)
373 cn = cl.classname
374 if self.classes.has_key(cn):
375 raise ValueError, cn
376 self.classes[cn] = cl
378 def getclasses(self):
379 ''' Return a list of the names of all existing classes.
380 '''
381 if __debug__:
382 print >>hyperdb.DEBUG, 'getclasses', (self,)
383 l = self.classes.keys()
384 l.sort()
385 return l
387 def getclass(self, classname):
388 '''Get the Class object representing a particular class.
390 If 'classname' is not a valid class name, a KeyError is raised.
391 '''
392 if __debug__:
393 print >>hyperdb.DEBUG, 'getclass', (self, classname)
394 try:
395 return self.classes[classname]
396 except KeyError:
397 raise KeyError, 'There is no class called "%s"'%classname
399 def clear(self):
400 ''' Delete all database contents.
402 Note: I don't commit here, which is different behaviour to the
403 "nuke from orbit" behaviour in the *dbms.
404 '''
405 if __debug__:
406 print >>hyperdb.DEBUG, 'clear', (self,)
407 cursor = self.conn.cursor()
408 for cn in self.classes.keys():
409 sql = 'delete from _%s'%cn
410 if __debug__:
411 print >>hyperdb.DEBUG, 'clear', (self, sql)
412 cursor.execute(sql)
414 #
415 # Node IDs
416 #
417 def newid(self, classname):
418 ''' Generate a new id for the given class
419 '''
420 # get the next ID
421 cursor = self.conn.cursor()
422 sql = 'select num from ids where name=%s'%self.arg
423 if __debug__:
424 print >>hyperdb.DEBUG, 'newid', (self, sql, classname)
425 cursor.execute(sql, (classname, ))
426 newid = cursor.fetchone()[0]
428 # update the counter
429 sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
430 vals = (int(newid)+1, classname)
431 if __debug__:
432 print >>hyperdb.DEBUG, 'newid', (self, sql, vals)
433 cursor.execute(sql, vals)
435 # return as string
436 return str(newid)
438 def setid(self, classname, setid):
439 ''' Set the id counter: used during import of database
440 '''
441 cursor = self.conn.cursor()
442 sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
443 vals = (setid, classname)
444 if __debug__:
445 print >>hyperdb.DEBUG, 'setid', (self, sql, vals)
446 cursor.execute(sql, vals)
448 #
449 # Nodes
450 #
452 def addnode(self, classname, nodeid, node):
453 ''' Add the specified node to its class's db.
454 '''
455 if __debug__:
456 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
457 # gadfly requires values for all non-multilink columns
458 cl = self.classes[classname]
459 cols, mls = self.determine_columns(cl.properties.items())
461 # add the special props
462 node = node.copy()
463 node['creation'] = node['activity'] = date.Date()
464 node['creator'] = self.journaltag
466 # default the non-multilink columns
467 for col, prop in cl.properties.items():
468 if not isinstance(col, Multilink):
469 if not node.has_key(col):
470 node[col] = None
472 # clear this node out of the cache if it's in there
473 key = (classname, nodeid)
474 if self.cache.has_key(key):
475 del self.cache[key]
476 self.cache_lru.remove(key)
478 # make the node data safe for the DB
479 node = self.serialise(classname, node)
481 # make sure the ordering is correct for column name -> column value
482 vals = tuple([node[col[1:]] for col in cols]) + (nodeid, 0)
483 s = ','.join([self.arg for x in cols]) + ',%s,%s'%(self.arg, self.arg)
484 cols = ','.join(cols) + ',id,__retired__'
486 # perform the inserts
487 cursor = self.conn.cursor()
488 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
489 if __debug__:
490 print >>hyperdb.DEBUG, 'addnode', (self, sql, vals)
491 cursor.execute(sql, vals)
493 # insert the multilink rows
494 for col in mls:
495 t = '%s_%s'%(classname, col)
496 for entry in node[col]:
497 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
498 self.arg, self.arg)
499 self.sql(cursor, sql, (entry, nodeid))
501 # make sure we do the commit-time extra stuff for this node
502 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
504 def setnode(self, classname, nodeid, values, multilink_changes):
505 ''' Change the specified node.
506 '''
507 if __debug__:
508 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, values)
510 # clear this node out of the cache if it's in there
511 key = (classname, nodeid)
512 if self.cache.has_key(key):
513 del self.cache[key]
514 self.cache_lru.remove(key)
516 # add the special props
517 values = values.copy()
518 values['activity'] = date.Date()
520 # make db-friendly
521 values = self.serialise(classname, values)
523 cl = self.classes[classname]
524 cols = []
525 mls = []
526 # add the multilinks separately
527 props = cl.getprops()
528 for col in values.keys():
529 prop = props[col]
530 if isinstance(prop, Multilink):
531 mls.append(col)
532 else:
533 cols.append('_'+col)
534 cols.sort()
536 cursor = self.conn.cursor()
538 # if there's any updates to regular columns, do them
539 if cols:
540 # make sure the ordering is correct for column name -> column value
541 sqlvals = tuple([values[col[1:]] for col in cols]) + (nodeid,)
542 s = ','.join(['%s=%s'%(x, self.arg) for x in cols])
543 cols = ','.join(cols)
545 # perform the update
546 sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
547 if __debug__:
548 print >>hyperdb.DEBUG, 'setnode', (self, sql, sqlvals)
549 cursor.execute(sql, sqlvals)
551 # now the fun bit, updating the multilinks ;)
552 for col, (add, remove) in multilink_changes.items():
553 tn = '%s_%s'%(classname, col)
554 if add:
555 sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
556 self.arg, self.arg)
557 for addid in add:
558 self.sql(cursor, sql, (nodeid, addid))
559 if remove:
560 sql = 'delete from %s where nodeid=%s and linkid=%s'%(tn,
561 self.arg, self.arg)
562 for removeid in remove:
563 self.sql(cursor, sql, (nodeid, removeid))
565 # make sure we do the commit-time extra stuff for this node
566 self.transactions.append((self.doSaveNode, (classname, nodeid, values)))
568 def getnode(self, classname, nodeid):
569 ''' Get a node from the database.
570 '''
571 if __debug__:
572 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid)
574 # see if we have this node cached
575 key = (classname, nodeid)
576 if self.cache.has_key(key):
577 # push us back to the top of the LRU
578 self.cache_lru.remove(key)
579 self.cache_lry.insert(0, key)
580 # return the cached information
581 return self.cache[key]
583 # figure the columns we're fetching
584 cl = self.classes[classname]
585 cols, mls = self.determine_columns(cl.properties.items())
586 scols = ','.join(cols)
588 # perform the basic property fetch
589 cursor = self.conn.cursor()
590 sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
591 self.sql(cursor, sql, (nodeid,))
593 values = self.sql_fetchone(cursor)
594 if values is None:
595 raise IndexError, 'no such %s node %s'%(classname, nodeid)
597 # make up the node
598 node = {}
599 for col in range(len(cols)):
600 node[cols[col][1:]] = values[col]
602 # now the multilinks
603 for col in mls:
604 # get the link ids
605 sql = 'select linkid from %s_%s where nodeid=%s'%(classname, col,
606 self.arg)
607 cursor.execute(sql, (nodeid,))
608 # extract the first column from the result
609 node[col] = [x[0] for x in cursor.fetchall()]
611 # un-dbificate the node data
612 node = self.unserialise(classname, node)
614 # save off in the cache
615 key = (classname, nodeid)
616 self.cache[key] = node
617 # update the LRU
618 self.cache_lru.insert(0, key)
619 del self.cache[self.cache_lru.pop()]
621 return node
623 def destroynode(self, classname, nodeid):
624 '''Remove a node from the database. Called exclusively by the
625 destroy() method on Class.
626 '''
627 if __debug__:
628 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
630 # make sure the node exists
631 if not self.hasnode(classname, nodeid):
632 raise IndexError, '%s has no node %s'%(classname, nodeid)
634 # see if we have this node cached
635 if self.cache.has_key((classname, nodeid)):
636 del self.cache[(classname, nodeid)]
638 # see if there's any obvious commit actions that we should get rid of
639 for entry in self.transactions[:]:
640 if entry[1][:2] == (classname, nodeid):
641 self.transactions.remove(entry)
643 # now do the SQL
644 cursor = self.conn.cursor()
645 sql = 'delete from _%s where id=%s'%(classname, self.arg)
646 self.sql(cursor, sql, (nodeid,))
648 # remove from multilnks
649 cl = self.getclass(classname)
650 x, mls = self.determine_columns(cl.properties.items())
651 for col in mls:
652 # get the link ids
653 sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
654 cursor.execute(sql, (nodeid,))
656 # remove journal entries
657 sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
658 self.sql(cursor, sql, (nodeid,))
660 def serialise(self, classname, node):
661 '''Copy the node contents, converting non-marshallable data into
662 marshallable data.
663 '''
664 if __debug__:
665 print >>hyperdb.DEBUG, 'serialise', classname, node
666 properties = self.getclass(classname).getprops()
667 d = {}
668 for k, v in node.items():
669 # if the property doesn't exist, or is the "retired" flag then
670 # it won't be in the properties dict
671 if not properties.has_key(k):
672 d[k] = v
673 continue
675 # get the property spec
676 prop = properties[k]
678 if isinstance(prop, Password):
679 d[k] = str(v)
680 elif isinstance(prop, Date) and v is not None:
681 d[k] = v.serialise()
682 elif isinstance(prop, Interval) and v is not None:
683 d[k] = v.serialise()
684 else:
685 d[k] = v
686 return d
688 def unserialise(self, classname, node):
689 '''Decode the marshalled node data
690 '''
691 if __debug__:
692 print >>hyperdb.DEBUG, 'unserialise', classname, node
693 properties = self.getclass(classname).getprops()
694 d = {}
695 for k, v in node.items():
696 # if the property doesn't exist, or is the "retired" flag then
697 # it won't be in the properties dict
698 if not properties.has_key(k):
699 d[k] = v
700 continue
702 # get the property spec
703 prop = properties[k]
705 if isinstance(prop, Date) and v is not None:
706 d[k] = date.Date(v)
707 elif isinstance(prop, Interval) and v is not None:
708 d[k] = date.Interval(v)
709 elif isinstance(prop, Password):
710 p = password.Password()
711 p.unpack(v)
712 d[k] = p
713 else:
714 d[k] = v
715 return d
717 def hasnode(self, classname, nodeid):
718 ''' Determine if the database has a given node.
719 '''
720 cursor = self.conn.cursor()
721 sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
722 if __debug__:
723 print >>hyperdb.DEBUG, 'hasnode', (self, sql, nodeid)
724 cursor.execute(sql, (nodeid,))
725 return int(cursor.fetchone()[0])
727 def countnodes(self, classname):
728 ''' Count the number of nodes that exist for a particular Class.
729 '''
730 cursor = self.conn.cursor()
731 sql = 'select count(*) from _%s'%classname
732 if __debug__:
733 print >>hyperdb.DEBUG, 'countnodes', (self, sql)
734 cursor.execute(sql)
735 return cursor.fetchone()[0]
737 def getnodeids(self, classname, retired=0):
738 ''' Retrieve all the ids of the nodes for a particular Class.
740 Set retired=None to get all nodes. Otherwise it'll get all the
741 retired or non-retired nodes, depending on the flag.
742 '''
743 cursor = self.conn.cursor()
744 # flip the sense of the flag if we don't want all of them
745 if retired is not None:
746 retired = not retired
747 sql = 'select id from _%s where __retired__ <> %s'%(classname, self.arg)
748 if __debug__:
749 print >>hyperdb.DEBUG, 'getnodeids', (self, sql, retired)
750 cursor.execute(sql, (retired,))
751 return [x[0] for x in cursor.fetchall()]
753 def addjournal(self, classname, nodeid, action, params, creator=None,
754 creation=None):
755 ''' Journal the Action
756 'action' may be:
758 'create' or 'set' -- 'params' is a dictionary of property values
759 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
760 'retire' -- 'params' is None
761 '''
762 # serialise the parameters now if necessary
763 if isinstance(params, type({})):
764 if action in ('set', 'create'):
765 params = self.serialise(classname, params)
767 # handle supply of the special journalling parameters (usually
768 # supplied on importing an existing database)
769 if creator:
770 journaltag = creator
771 else:
772 journaltag = self.journaltag
773 if creation:
774 journaldate = creation.serialise()
775 else:
776 journaldate = date.Date().serialise()
778 # create the journal entry
779 cols = ','.join('nodeid date tag action params'.split())
781 if __debug__:
782 print >>hyperdb.DEBUG, 'addjournal', (nodeid, journaldate,
783 journaltag, action, params)
785 cursor = self.conn.cursor()
786 self.save_journal(cursor, classname, cols, nodeid, journaldate,
787 journaltag, action, params)
789 def save_journal(self, cursor, classname, cols, nodeid, journaldate,
790 journaltag, action, params):
791 ''' Save the journal entry to the database
792 '''
793 raise NotImplemented
795 def getjournal(self, classname, nodeid):
796 ''' get the journal for id
797 '''
798 # make sure the node exists
799 if not self.hasnode(classname, nodeid):
800 raise IndexError, '%s has no node %s'%(classname, nodeid)
802 cursor = self.conn.cursor()
803 cols = ','.join('nodeid date tag action params'.split())
804 return self.load_journal(cursor, classname, cols, nodeid)
806 def load_journal(self, cursor, classname, cols, nodeid):
807 ''' Load the journal from the database
808 '''
809 raise NotImplemented
811 def pack(self, pack_before):
812 ''' Delete all journal entries except "create" before 'pack_before'.
813 '''
814 # get a 'yyyymmddhhmmss' version of the date
815 date_stamp = pack_before.serialise()
817 # do the delete
818 cursor = self.conn.cursor()
819 for classname in self.classes.keys():
820 sql = "delete from %s__journal where date<%s and "\
821 "action<>'create'"%(classname, self.arg)
822 if __debug__:
823 print >>hyperdb.DEBUG, 'pack', (self, sql, date_stamp)
824 cursor.execute(sql, (date_stamp,))
826 def sql_commit(self):
827 ''' Actually commit to the database.
828 '''
829 self.conn.commit()
831 def commit(self):
832 ''' Commit the current transactions.
834 Save all data changed since the database was opened or since the
835 last commit() or rollback().
836 '''
837 if __debug__:
838 print >>hyperdb.DEBUG, 'commit', (self,)
840 # commit the database
841 self.sql_commit()
843 # now, do all the other transaction stuff
844 reindex = {}
845 for method, args in self.transactions:
846 reindex[method(*args)] = 1
848 # reindex the nodes that request it
849 for classname, nodeid in filter(None, reindex.keys()):
850 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
851 self.getclass(classname).index(nodeid)
853 # save the indexer state
854 self.indexer.save_index()
856 # clear out the transactions
857 self.transactions = []
859 def rollback(self):
860 ''' Reverse all actions from the current transaction.
862 Undo all the changes made since the database was opened or the last
863 commit() or rollback() was performed.
864 '''
865 if __debug__:
866 print >>hyperdb.DEBUG, 'rollback', (self,)
868 # roll back
869 self.conn.rollback()
871 # roll back "other" transaction stuff
872 for method, args in self.transactions:
873 # delete temporary files
874 if method == self.doStoreFile:
875 self.rollbackStoreFile(*args)
876 self.transactions = []
878 def doSaveNode(self, classname, nodeid, node):
879 ''' dummy that just generates a reindex event
880 '''
881 # return the classname, nodeid so we reindex this content
882 return (classname, nodeid)
884 def close(self):
885 ''' Close off the connection.
886 '''
887 self.conn.close()
889 #
890 # The base Class class
891 #
892 class Class(hyperdb.Class):
893 ''' The handle to a particular class of nodes in a hyperdatabase.
895 All methods except __repr__ and getnode must be implemented by a
896 concrete backend Class.
897 '''
899 def __init__(self, db, classname, **properties):
900 '''Create a new class with a given name and property specification.
902 'classname' must not collide with the name of an existing class,
903 or a ValueError is raised. The keyword arguments in 'properties'
904 must map names to property objects, or a TypeError is raised.
905 '''
906 if (properties.has_key('creation') or properties.has_key('activity')
907 or properties.has_key('creator')):
908 raise ValueError, '"creation", "activity" and "creator" are '\
909 'reserved'
911 self.classname = classname
912 self.properties = properties
913 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
914 self.key = ''
916 # should we journal changes (default yes)
917 self.do_journal = 1
919 # do the db-related init stuff
920 db.addclass(self)
922 self.auditors = {'create': [], 'set': [], 'retire': []}
923 self.reactors = {'create': [], 'set': [], 'retire': []}
925 def schema(self):
926 ''' A dumpable version of the schema that we can store in the
927 database
928 '''
929 return (self.key, [(x, repr(y)) for x,y in self.properties.items()])
931 def enableJournalling(self):
932 '''Turn journalling on for this class
933 '''
934 self.do_journal = 1
936 def disableJournalling(self):
937 '''Turn journalling off for this class
938 '''
939 self.do_journal = 0
941 # Editing nodes:
942 def create(self, **propvalues):
943 ''' Create a new node of this class and return its id.
945 The keyword arguments in 'propvalues' map property names to values.
947 The values of arguments must be acceptable for the types of their
948 corresponding properties or a TypeError is raised.
950 If this class has a key property, it must be present and its value
951 must not collide with other key strings or a ValueError is raised.
953 Any other properties on this class that are missing from the
954 'propvalues' dictionary are set to None.
956 If an id in a link or multilink property does not refer to a valid
957 node, an IndexError is raised.
958 '''
959 if propvalues.has_key('id'):
960 raise KeyError, '"id" is reserved'
962 if self.db.journaltag is None:
963 raise DatabaseError, 'Database open read-only'
965 if propvalues.has_key('creation') or propvalues.has_key('activity'):
966 raise KeyError, '"creation" and "activity" are reserved'
968 self.fireAuditors('create', None, propvalues)
970 # new node's id
971 newid = self.db.newid(self.classname)
973 # validate propvalues
974 num_re = re.compile('^\d+$')
975 for key, value in propvalues.items():
976 if key == self.key:
977 try:
978 self.lookup(value)
979 except KeyError:
980 pass
981 else:
982 raise ValueError, 'node with key "%s" exists'%value
984 # try to handle this property
985 try:
986 prop = self.properties[key]
987 except KeyError:
988 raise KeyError, '"%s" has no property "%s"'%(self.classname,
989 key)
991 if value is not None and isinstance(prop, Link):
992 if type(value) != type(''):
993 raise ValueError, 'link value must be String'
994 link_class = self.properties[key].classname
995 # if it isn't a number, it's a key
996 if not num_re.match(value):
997 try:
998 value = self.db.classes[link_class].lookup(value)
999 except (TypeError, KeyError):
1000 raise IndexError, 'new property "%s": %s not a %s'%(
1001 key, value, link_class)
1002 elif not self.db.getclass(link_class).hasnode(value):
1003 raise IndexError, '%s has no node %s'%(link_class, value)
1005 # save off the value
1006 propvalues[key] = value
1008 # register the link with the newly linked node
1009 if self.do_journal and self.properties[key].do_journal:
1010 self.db.addjournal(link_class, value, 'link',
1011 (self.classname, newid, key))
1013 elif isinstance(prop, Multilink):
1014 if type(value) != type([]):
1015 raise TypeError, 'new property "%s" not a list of ids'%key
1017 # clean up and validate the list of links
1018 link_class = self.properties[key].classname
1019 l = []
1020 for entry in value:
1021 if type(entry) != type(''):
1022 raise ValueError, '"%s" multilink value (%r) '\
1023 'must contain Strings'%(key, value)
1024 # if it isn't a number, it's a key
1025 if not num_re.match(entry):
1026 try:
1027 entry = self.db.classes[link_class].lookup(entry)
1028 except (TypeError, KeyError):
1029 raise IndexError, 'new property "%s": %s not a %s'%(
1030 key, entry, self.properties[key].classname)
1031 l.append(entry)
1032 value = l
1033 propvalues[key] = value
1035 # handle additions
1036 for nodeid in value:
1037 if not self.db.getclass(link_class).hasnode(nodeid):
1038 raise IndexError, '%s has no node %s'%(link_class,
1039 nodeid)
1040 # register the link with the newly linked node
1041 if self.do_journal and self.properties[key].do_journal:
1042 self.db.addjournal(link_class, nodeid, 'link',
1043 (self.classname, newid, key))
1045 elif isinstance(prop, String):
1046 if type(value) != type(''):
1047 raise TypeError, 'new property "%s" not a string'%key
1049 elif isinstance(prop, Password):
1050 if not isinstance(value, password.Password):
1051 raise TypeError, 'new property "%s" not a Password'%key
1053 elif isinstance(prop, Date):
1054 if value is not None and not isinstance(value, date.Date):
1055 raise TypeError, 'new property "%s" not a Date'%key
1057 elif isinstance(prop, Interval):
1058 if value is not None and not isinstance(value, date.Interval):
1059 raise TypeError, 'new property "%s" not an Interval'%key
1061 elif value is not None and isinstance(prop, Number):
1062 try:
1063 float(value)
1064 except ValueError:
1065 raise TypeError, 'new property "%s" not numeric'%key
1067 elif value is not None and isinstance(prop, Boolean):
1068 try:
1069 int(value)
1070 except ValueError:
1071 raise TypeError, 'new property "%s" not boolean'%key
1073 # make sure there's data where there needs to be
1074 for key, prop in self.properties.items():
1075 if propvalues.has_key(key):
1076 continue
1077 if key == self.key:
1078 raise ValueError, 'key property "%s" is required'%key
1079 if isinstance(prop, Multilink):
1080 propvalues[key] = []
1081 else:
1082 propvalues[key] = None
1084 # done
1085 self.db.addnode(self.classname, newid, propvalues)
1086 if self.do_journal:
1087 self.db.addjournal(self.classname, newid, 'create', propvalues)
1089 self.fireReactors('create', newid, None)
1091 return newid
1093 def export_list(self, propnames, nodeid):
1094 ''' Export a node - generate a list of CSV-able data in the order
1095 specified by propnames for the given node.
1096 '''
1097 properties = self.getprops()
1098 l = []
1099 for prop in propnames:
1100 proptype = properties[prop]
1101 value = self.get(nodeid, prop)
1102 # "marshal" data where needed
1103 if value is None:
1104 pass
1105 elif isinstance(proptype, hyperdb.Date):
1106 value = value.get_tuple()
1107 elif isinstance(proptype, hyperdb.Interval):
1108 value = value.get_tuple()
1109 elif isinstance(proptype, hyperdb.Password):
1110 value = str(value)
1111 l.append(repr(value))
1112 return l
1114 def import_list(self, propnames, proplist):
1115 ''' Import a node - all information including "id" is present and
1116 should not be sanity checked. Triggers are not triggered. The
1117 journal should be initialised using the "creator" and "created"
1118 information.
1120 Return the nodeid of the node imported.
1121 '''
1122 if self.db.journaltag is None:
1123 raise DatabaseError, 'Database open read-only'
1124 properties = self.getprops()
1126 # make the new node's property map
1127 d = {}
1128 for i in range(len(propnames)):
1129 # Use eval to reverse the repr() used to output the CSV
1130 value = eval(proplist[i])
1132 # Figure the property for this column
1133 propname = propnames[i]
1134 prop = properties[propname]
1136 # "unmarshal" where necessary
1137 if propname == 'id':
1138 newid = value
1139 continue
1140 elif value is None:
1141 # don't set Nones
1142 continue
1143 elif isinstance(prop, hyperdb.Date):
1144 value = date.Date(value)
1145 elif isinstance(prop, hyperdb.Interval):
1146 value = date.Interval(value)
1147 elif isinstance(prop, hyperdb.Password):
1148 pwd = password.Password()
1149 pwd.unpack(value)
1150 value = pwd
1151 d[propname] = value
1153 # extract the extraneous journalling gumpf and nuke it
1154 if d.has_key('creator'):
1155 creator = d['creator']
1156 del d['creator']
1157 if d.has_key('creation'):
1158 creation = d['creation']
1159 del d['creation']
1160 if d.has_key('activity'):
1161 del d['activity']
1163 # add the node and journal
1164 self.db.addnode(self.classname, newid, d)
1165 self.db.addjournal(self.classname, newid, 'create', d, creator,
1166 creation)
1167 return newid
1169 _marker = []
1170 def get(self, nodeid, propname, default=_marker, cache=1):
1171 '''Get the value of a property on an existing node of this class.
1173 'nodeid' must be the id of an existing node of this class or an
1174 IndexError is raised. 'propname' must be the name of a property
1175 of this class or a KeyError is raised.
1177 'cache' indicates whether the transaction cache should be queried
1178 for the node. If the node has been modified and you need to
1179 determine what its values prior to modification are, you need to
1180 set cache=0.
1181 '''
1182 if propname == 'id':
1183 return nodeid
1185 # get the node's dict
1186 d = self.db.getnode(self.classname, nodeid)
1188 if propname == 'creation':
1189 if d.has_key('creation'):
1190 return d['creation']
1191 else:
1192 return date.Date()
1193 if propname == 'activity':
1194 if d.has_key('activity'):
1195 return d['activity']
1196 else:
1197 return date.Date()
1198 if propname == 'creator':
1199 if d.has_key('creator'):
1200 return d['creator']
1201 else:
1202 return self.db.journaltag
1204 # get the property (raises KeyErorr if invalid)
1205 prop = self.properties[propname]
1207 if not d.has_key(propname):
1208 if default is self._marker:
1209 if isinstance(prop, Multilink):
1210 return []
1211 else:
1212 return None
1213 else:
1214 return default
1216 # don't pass our list to other code
1217 if isinstance(prop, Multilink):
1218 return d[propname][:]
1220 return d[propname]
1222 def getnode(self, nodeid, cache=1):
1223 ''' Return a convenience wrapper for the node.
1225 'nodeid' must be the id of an existing node of this class or an
1226 IndexError is raised.
1228 'cache' indicates whether the transaction cache should be queried
1229 for the node. If the node has been modified and you need to
1230 determine what its values prior to modification are, you need to
1231 set cache=0.
1232 '''
1233 return Node(self, nodeid, cache=cache)
1235 def set(self, nodeid, **propvalues):
1236 '''Modify a property on an existing node of this class.
1238 'nodeid' must be the id of an existing node of this class or an
1239 IndexError is raised.
1241 Each key in 'propvalues' must be the name of a property of this
1242 class or a KeyError is raised.
1244 All values in 'propvalues' must be acceptable types for their
1245 corresponding properties or a TypeError is raised.
1247 If the value of the key property is set, it must not collide with
1248 other key strings or a ValueError is raised.
1250 If the value of a Link or Multilink property contains an invalid
1251 node id, a ValueError is raised.
1252 '''
1253 if not propvalues:
1254 return propvalues
1256 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1257 raise KeyError, '"creation" and "activity" are reserved'
1259 if propvalues.has_key('id'):
1260 raise KeyError, '"id" is reserved'
1262 if self.db.journaltag is None:
1263 raise DatabaseError, 'Database open read-only'
1265 self.fireAuditors('set', nodeid, propvalues)
1266 # Take a copy of the node dict so that the subsequent set
1267 # operation doesn't modify the oldvalues structure.
1268 # XXX used to try the cache here first
1269 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1271 node = self.db.getnode(self.classname, nodeid)
1272 if self.is_retired(nodeid):
1273 raise IndexError, 'Requested item is retired'
1274 num_re = re.compile('^\d+$')
1276 # if the journal value is to be different, store it in here
1277 journalvalues = {}
1279 # remember the add/remove stuff for multilinks, making it easier
1280 # for the Database layer to do its stuff
1281 multilink_changes = {}
1283 for propname, value in propvalues.items():
1284 # check to make sure we're not duplicating an existing key
1285 if propname == self.key and node[propname] != value:
1286 try:
1287 self.lookup(value)
1288 except KeyError:
1289 pass
1290 else:
1291 raise ValueError, 'node with key "%s" exists'%value
1293 # this will raise the KeyError if the property isn't valid
1294 # ... we don't use getprops() here because we only care about
1295 # the writeable properties.
1296 try:
1297 prop = self.properties[propname]
1298 except KeyError:
1299 raise KeyError, '"%s" has no property named "%s"'%(
1300 self.classname, propname)
1302 # if the value's the same as the existing value, no sense in
1303 # doing anything
1304 if node.has_key(propname) and value == node[propname]:
1305 del propvalues[propname]
1306 continue
1308 # do stuff based on the prop type
1309 if isinstance(prop, Link):
1310 link_class = prop.classname
1311 # if it isn't a number, it's a key
1312 if value is not None and not isinstance(value, type('')):
1313 raise ValueError, 'property "%s" link value be a string'%(
1314 propname)
1315 if isinstance(value, type('')) and not num_re.match(value):
1316 try:
1317 value = self.db.classes[link_class].lookup(value)
1318 except (TypeError, KeyError):
1319 raise IndexError, 'new property "%s": %s not a %s'%(
1320 propname, value, prop.classname)
1322 if (value is not None and
1323 not self.db.getclass(link_class).hasnode(value)):
1324 raise IndexError, '%s has no node %s'%(link_class, value)
1326 if self.do_journal and prop.do_journal:
1327 # register the unlink with the old linked node
1328 if node[propname] is not None:
1329 self.db.addjournal(link_class, node[propname], 'unlink',
1330 (self.classname, nodeid, propname))
1332 # register the link with the newly linked node
1333 if value is not None:
1334 self.db.addjournal(link_class, value, 'link',
1335 (self.classname, nodeid, propname))
1337 elif isinstance(prop, Multilink):
1338 if type(value) != type([]):
1339 raise TypeError, 'new property "%s" not a list of'\
1340 ' ids'%propname
1341 link_class = self.properties[propname].classname
1342 l = []
1343 for entry in value:
1344 # if it isn't a number, it's a key
1345 if type(entry) != type(''):
1346 raise ValueError, 'new property "%s" link value ' \
1347 'must be a string'%propname
1348 if not num_re.match(entry):
1349 try:
1350 entry = self.db.classes[link_class].lookup(entry)
1351 except (TypeError, KeyError):
1352 raise IndexError, 'new property "%s": %s not a %s'%(
1353 propname, entry,
1354 self.properties[propname].classname)
1355 l.append(entry)
1356 value = l
1357 propvalues[propname] = value
1359 # figure the journal entry for this property
1360 add = []
1361 remove = []
1363 # handle removals
1364 if node.has_key(propname):
1365 l = node[propname]
1366 else:
1367 l = []
1368 for id in l[:]:
1369 if id in value:
1370 continue
1371 # register the unlink with the old linked node
1372 if self.do_journal and self.properties[propname].do_journal:
1373 self.db.addjournal(link_class, id, 'unlink',
1374 (self.classname, nodeid, propname))
1375 l.remove(id)
1376 remove.append(id)
1378 # handle additions
1379 for id in value:
1380 if not self.db.getclass(link_class).hasnode(id):
1381 raise IndexError, '%s has no node %s'%(link_class, id)
1382 if id in l:
1383 continue
1384 # register the link with the newly linked node
1385 if self.do_journal and self.properties[propname].do_journal:
1386 self.db.addjournal(link_class, id, 'link',
1387 (self.classname, nodeid, propname))
1388 l.append(id)
1389 add.append(id)
1391 # figure the journal entry
1392 l = []
1393 if add:
1394 l.append(('+', add))
1395 if remove:
1396 l.append(('-', remove))
1397 multilink_changes[propname] = (add, remove)
1398 if l:
1399 journalvalues[propname] = tuple(l)
1401 elif isinstance(prop, String):
1402 if value is not None and type(value) != type(''):
1403 raise TypeError, 'new property "%s" not a string'%propname
1405 elif isinstance(prop, Password):
1406 if not isinstance(value, password.Password):
1407 raise TypeError, 'new property "%s" not a Password'%propname
1408 propvalues[propname] = value
1410 elif value is not None and isinstance(prop, Date):
1411 if not isinstance(value, date.Date):
1412 raise TypeError, 'new property "%s" not a Date'% propname
1413 propvalues[propname] = value
1415 elif value is not None and isinstance(prop, Interval):
1416 if not isinstance(value, date.Interval):
1417 raise TypeError, 'new property "%s" not an '\
1418 'Interval'%propname
1419 propvalues[propname] = value
1421 elif value is not None and isinstance(prop, Number):
1422 try:
1423 float(value)
1424 except ValueError:
1425 raise TypeError, 'new property "%s" not numeric'%propname
1427 elif value is not None and isinstance(prop, Boolean):
1428 try:
1429 int(value)
1430 except ValueError:
1431 raise TypeError, 'new property "%s" not boolean'%propname
1433 # nothing to do?
1434 if not propvalues:
1435 return propvalues
1437 # do the set, and journal it
1438 self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
1440 if self.do_journal:
1441 propvalues.update(journalvalues)
1442 self.db.addjournal(self.classname, nodeid, 'set', propvalues)
1444 self.fireReactors('set', nodeid, oldvalues)
1446 return propvalues
1448 def retire(self, nodeid):
1449 '''Retire a node.
1451 The properties on the node remain available from the get() method,
1452 and the node's id is never reused.
1454 Retired nodes are not returned by the find(), list(), or lookup()
1455 methods, and other nodes may reuse the values of their key properties.
1456 '''
1457 if self.db.journaltag is None:
1458 raise DatabaseError, 'Database open read-only'
1460 cursor = self.db.conn.cursor()
1461 sql = 'update _%s set __retired__=1 where id=%s'%(self.classname,
1462 self.db.arg)
1463 if __debug__:
1464 print >>hyperdb.DEBUG, 'retire', (self, sql, nodeid)
1465 cursor.execute(sql, (nodeid,))
1467 def is_retired(self, nodeid):
1468 '''Return true if the node is rerired
1469 '''
1470 cursor = self.db.conn.cursor()
1471 sql = 'select __retired__ from _%s where id=%s'%(self.classname,
1472 self.db.arg)
1473 if __debug__:
1474 print >>hyperdb.DEBUG, 'is_retired', (self, sql, nodeid)
1475 cursor.execute(sql, (nodeid,))
1476 return int(cursor.fetchone()[0])
1478 def destroy(self, nodeid):
1479 '''Destroy a node.
1481 WARNING: this method should never be used except in extremely rare
1482 situations where there could never be links to the node being
1483 deleted
1484 WARNING: use retire() instead
1485 WARNING: the properties of this node will not be available ever again
1486 WARNING: really, use retire() instead
1488 Well, I think that's enough warnings. This method exists mostly to
1489 support the session storage of the cgi interface.
1491 The node is completely removed from the hyperdb, including all journal
1492 entries. It will no longer be available, and will generally break code
1493 if there are any references to the node.
1494 '''
1495 if self.db.journaltag is None:
1496 raise DatabaseError, 'Database open read-only'
1497 self.db.destroynode(self.classname, nodeid)
1499 def history(self, nodeid):
1500 '''Retrieve the journal of edits on a particular node.
1502 'nodeid' must be the id of an existing node of this class or an
1503 IndexError is raised.
1505 The returned list contains tuples of the form
1507 (date, tag, action, params)
1509 'date' is a Timestamp object specifying the time of the change and
1510 'tag' is the journaltag specified when the database was opened.
1511 '''
1512 if not self.do_journal:
1513 raise ValueError, 'Journalling is disabled for this class'
1514 return self.db.getjournal(self.classname, nodeid)
1516 # Locating nodes:
1517 def hasnode(self, nodeid):
1518 '''Determine if the given nodeid actually exists
1519 '''
1520 return self.db.hasnode(self.classname, nodeid)
1522 def setkey(self, propname):
1523 '''Select a String property of this class to be the key property.
1525 'propname' must be the name of a String property of this class or
1526 None, or a TypeError is raised. The values of the key property on
1527 all existing nodes must be unique or a ValueError is raised.
1528 '''
1529 # XXX create an index on the key prop column
1530 prop = self.getprops()[propname]
1531 if not isinstance(prop, String):
1532 raise TypeError, 'key properties must be String'
1533 self.key = propname
1535 def getkey(self):
1536 '''Return the name of the key property for this class or None.'''
1537 return self.key
1539 def labelprop(self, default_to_id=0):
1540 ''' Return the property name for a label for the given node.
1542 This method attempts to generate a consistent label for the node.
1543 It tries the following in order:
1544 1. key property
1545 2. "name" property
1546 3. "title" property
1547 4. first property from the sorted property name list
1548 '''
1549 k = self.getkey()
1550 if k:
1551 return k
1552 props = self.getprops()
1553 if props.has_key('name'):
1554 return 'name'
1555 elif props.has_key('title'):
1556 return 'title'
1557 if default_to_id:
1558 return 'id'
1559 props = props.keys()
1560 props.sort()
1561 return props[0]
1563 def lookup(self, keyvalue):
1564 '''Locate a particular node by its key property and return its id.
1566 If this class has no key property, a TypeError is raised. If the
1567 'keyvalue' matches one of the values for the key property among
1568 the nodes in this class, the matching node's id is returned;
1569 otherwise a KeyError is raised.
1570 '''
1571 if not self.key:
1572 raise TypeError, 'No key property set for class %s'%self.classname
1574 cursor = self.db.conn.cursor()
1575 sql = 'select id,__retired__ from _%s where _%s=%s'%(self.classname,
1576 self.key, self.db.arg)
1577 self.db.sql(cursor, sql, (keyvalue,))
1579 # see if there was a result that's not retired
1580 l = cursor.fetchall()
1581 if not l or int(l[0][1]):
1582 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1583 keyvalue, self.classname)
1585 # return the id
1586 return l[0][0]
1588 def find(self, **propspec):
1589 '''Get the ids of nodes in this class which link to the given nodes.
1591 'propspec' consists of keyword args propname={nodeid:1,}
1592 'propname' must be the name of a property in this class, or a
1593 KeyError is raised. That property must be a Link or Multilink
1594 property, or a TypeError is raised.
1596 Any node in this class whose 'propname' property links to any of the
1597 nodeids will be returned. Used by the full text indexing, which knows
1598 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1599 issues:
1601 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1602 '''
1603 if __debug__:
1604 print >>hyperdb.DEBUG, 'find', (self, propspec)
1605 if not propspec:
1606 return []
1607 queries = []
1608 tables = []
1609 allvalues = ()
1610 for prop, values in propspec.items():
1611 allvalues += tuple(values.keys())
1612 a = self.db.arg
1613 tables.append('select nodeid from %s_%s where linkid in (%s)'%(
1614 self.classname, prop, ','.join([a for x in values.keys()])))
1615 sql = '\nintersect\n'.join(tables)
1616 if __debug__:
1617 print >>hyperdb.DEBUG, 'find', (self, sql, allvalues)
1618 cursor = self.db.conn.cursor()
1619 cursor.execute(sql, allvalues)
1620 try:
1621 l = [x[0] for x in cursor.fetchall()]
1622 except gadfly.database.error, message:
1623 if message == 'no more results':
1624 l = []
1625 raise
1626 if __debug__:
1627 print >>hyperdb.DEBUG, 'find ... ', l
1628 return l
1630 def list(self):
1631 ''' Return a list of the ids of the active nodes in this class.
1632 '''
1633 return self.db.getnodeids(self.classname, retired=0)
1635 def filter(self, search_matches, filterspec, sort, group):
1636 ''' Return a list of the ids of the active nodes in this class that
1637 match the 'filter' spec, sorted by the group spec and then the
1638 sort spec
1640 "filterspec" is {propname: value(s)}
1641 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1642 and prop is a prop name or None
1643 "search_matches" is {nodeid: marker}
1645 The filter must match all properties specificed - but if the
1646 property value to match is a list, any one of the values in the
1647 list may match for that property to match.
1648 '''
1649 cn = self.classname
1651 # figure the WHERE clause from the filterspec
1652 props = self.getprops()
1653 frum = ['_'+cn]
1654 where = []
1655 args = []
1656 a = self.db.arg
1657 for k, v in filterspec.items():
1658 propclass = props[k]
1659 # now do other where clause stuff
1660 if isinstance(propclass, Multilink):
1661 tn = '%s_%s'%(cn, k)
1662 frum.append(tn)
1663 if isinstance(v, type([])):
1664 s = ','.join([a for x in v])
1665 where.append('id=%s.nodeid and %s.linkid in (%s)'%(tn,tn,s))
1666 args = args + v
1667 else:
1668 where.append('id=%s.nodeid and %s.linkid = %s'%(tn, tn, a))
1669 args.append(v)
1670 elif isinstance(propclass, String):
1671 if not isinstance(v, type([])):
1672 v = [v]
1674 # Quote the bits in the string that need it and then embed
1675 # in a "substring" search. Note - need to quote the '%' so
1676 # they make it through the python layer happily
1677 v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
1679 # now add to the where clause
1680 where.append(' or '.join(["_%s LIKE '%s'"%(k, s) for s in v]))
1681 # note: args are embedded in the query string now
1682 elif isinstance(propclass, Link):
1683 if isinstance(v, type([])):
1684 if '-1' in v:
1685 v.remove('-1')
1686 xtra = ' or _%s is NULL'%k
1687 else:
1688 xtra = ''
1689 s = ','.join([a for x in v])
1690 where.append('(_%s in (%s)%s)'%(k, s, xtra))
1691 args = args + v
1692 else:
1693 if v == '-1':
1694 v = None
1695 where.append('_%s is NULL'%k)
1696 else:
1697 where.append('_%s=%s'%(k, a))
1698 args.append(v)
1699 else:
1700 if isinstance(v, type([])):
1701 s = ','.join([a for x in v])
1702 where.append('_%s in (%s)'%(k, s))
1703 args = args + v
1704 else:
1705 where.append('_%s=%s'%(k, a))
1706 args.append(v)
1708 # add results of full text search
1709 if search_matches is not None:
1710 v = search_matches.keys()
1711 s = ','.join([a for x in v])
1712 where.append('id in (%s)'%s)
1713 args = args + v
1715 # "grouping" is just the first-order sorting in the SQL fetch
1716 # can modify it...)
1717 orderby = []
1718 ordercols = []
1719 if group[0] is not None and group[1] is not None:
1720 if group[0] != '-':
1721 orderby.append('_'+group[1])
1722 ordercols.append('_'+group[1])
1723 else:
1724 orderby.append('_'+group[1]+' desc')
1725 ordercols.append('_'+group[1])
1727 # now add in the sorting
1728 group = ''
1729 if sort[0] is not None and sort[1] is not None:
1730 direction, colname = sort
1731 if direction != '-':
1732 if colname == 'id':
1733 orderby.append(colname)
1734 else:
1735 orderby.append('_'+colname)
1736 ordercols.append('_'+colname)
1737 else:
1738 if colname == 'id':
1739 orderby.append(colname+' desc')
1740 ordercols.append(colname)
1741 else:
1742 orderby.append('_'+colname+' desc')
1743 ordercols.append('_'+colname)
1745 # construct the SQL
1746 frum = ','.join(frum)
1747 if where:
1748 where = ' where ' + (' and '.join(where))
1749 else:
1750 where = ''
1751 cols = ['id']
1752 if orderby:
1753 cols = cols + ordercols
1754 order = ' order by %s'%(','.join(orderby))
1755 else:
1756 order = ''
1757 cols = ','.join(cols)
1758 sql = 'select %s from %s %s%s%s'%(cols, frum, where, group, order)
1759 args = tuple(args)
1760 if __debug__:
1761 print >>hyperdb.DEBUG, 'filter', (self, sql, args)
1762 cursor = self.db.conn.cursor()
1763 cursor.execute(sql, args)
1764 l = cursor.fetchall()
1766 # return the IDs (the first column)
1767 # XXX The filter(None, l) bit is sqlite-specific... if there's _NO_
1768 # XXX matches to a fetch, it returns NULL instead of nothing!?!
1769 return filter(None, [row[0] for row in l])
1771 def count(self):
1772 '''Get the number of nodes in this class.
1774 If the returned integer is 'numnodes', the ids of all the nodes
1775 in this class run from 1 to numnodes, and numnodes+1 will be the
1776 id of the next node to be created in this class.
1777 '''
1778 return self.db.countnodes(self.classname)
1780 # Manipulating properties:
1781 def getprops(self, protected=1):
1782 '''Return a dictionary mapping property names to property objects.
1783 If the "protected" flag is true, we include protected properties -
1784 those which may not be modified.
1785 '''
1786 d = self.properties.copy()
1787 if protected:
1788 d['id'] = String()
1789 d['creation'] = hyperdb.Date()
1790 d['activity'] = hyperdb.Date()
1791 d['creator'] = hyperdb.Link("user")
1792 return d
1794 def addprop(self, **properties):
1795 '''Add properties to this class.
1797 The keyword arguments in 'properties' must map names to property
1798 objects, or a TypeError is raised. None of the keys in 'properties'
1799 may collide with the names of existing properties, or a ValueError
1800 is raised before any properties have been added.
1801 '''
1802 for key in properties.keys():
1803 if self.properties.has_key(key):
1804 raise ValueError, key
1805 self.properties.update(properties)
1807 def index(self, nodeid):
1808 '''Add (or refresh) the node to search indexes
1809 '''
1810 # find all the String properties that have indexme
1811 for prop, propclass in self.getprops().items():
1812 if isinstance(propclass, String) and propclass.indexme:
1813 try:
1814 value = str(self.get(nodeid, prop))
1815 except IndexError:
1816 # node no longer exists - entry should be removed
1817 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1818 else:
1819 # and index them under (classname, nodeid, property)
1820 self.db.indexer.add_text((self.classname, nodeid, prop),
1821 value)
1824 #
1825 # Detector interface
1826 #
1827 def audit(self, event, detector):
1828 '''Register a detector
1829 '''
1830 l = self.auditors[event]
1831 if detector not in l:
1832 self.auditors[event].append(detector)
1834 def fireAuditors(self, action, nodeid, newvalues):
1835 '''Fire all registered auditors.
1836 '''
1837 for audit in self.auditors[action]:
1838 audit(self.db, self, nodeid, newvalues)
1840 def react(self, event, detector):
1841 '''Register a detector
1842 '''
1843 l = self.reactors[event]
1844 if detector not in l:
1845 self.reactors[event].append(detector)
1847 def fireReactors(self, action, nodeid, oldvalues):
1848 '''Fire all registered reactors.
1849 '''
1850 for react in self.reactors[action]:
1851 react(self.db, self, nodeid, oldvalues)
1853 class FileClass(Class):
1854 '''This class defines a large chunk of data. To support this, it has a
1855 mandatory String property "content" which is typically saved off
1856 externally to the hyperdb.
1858 The default MIME type of this data is defined by the
1859 "default_mime_type" class attribute, which may be overridden by each
1860 node if the class defines a "type" String property.
1861 '''
1862 default_mime_type = 'text/plain'
1864 def create(self, **propvalues):
1865 ''' snaffle the file propvalue and store in a file
1866 '''
1867 content = propvalues['content']
1868 del propvalues['content']
1869 newid = Class.create(self, **propvalues)
1870 self.db.storefile(self.classname, newid, None, content)
1871 return newid
1873 def import_list(self, propnames, proplist):
1874 ''' Trap the "content" property...
1875 '''
1876 # dupe this list so we don't affect others
1877 propnames = propnames[:]
1879 # extract the "content" property from the proplist
1880 i = propnames.index('content')
1881 content = eval(proplist[i])
1882 del propnames[i]
1883 del proplist[i]
1885 # do the normal import
1886 newid = Class.import_list(self, propnames, proplist)
1888 # save off the "content" file
1889 self.db.storefile(self.classname, newid, None, content)
1890 return newid
1892 _marker = []
1893 def get(self, nodeid, propname, default=_marker, cache=1):
1894 ''' trap the content propname and get it from the file
1895 '''
1897 poss_msg = 'Possibly a access right configuration problem.'
1898 if propname == 'content':
1899 try:
1900 return self.db.getfile(self.classname, nodeid, None)
1901 except IOError, (strerror):
1902 # BUG: by catching this we donot see an error in the log.
1903 return 'ERROR reading file: %s%s\n%s\n%s'%(
1904 self.classname, nodeid, poss_msg, strerror)
1905 if default is not self._marker:
1906 return Class.get(self, nodeid, propname, default, cache=cache)
1907 else:
1908 return Class.get(self, nodeid, propname, cache=cache)
1910 def getprops(self, protected=1):
1911 ''' In addition to the actual properties on the node, these methods
1912 provide the "content" property. If the "protected" flag is true,
1913 we include protected properties - those which may not be
1914 modified.
1915 '''
1916 d = Class.getprops(self, protected=protected).copy()
1917 d['content'] = hyperdb.String()
1918 return d
1920 def index(self, nodeid):
1921 ''' Index the node in the search index.
1923 We want to index the content in addition to the normal String
1924 property indexing.
1925 '''
1926 # perform normal indexing
1927 Class.index(self, nodeid)
1929 # get the content to index
1930 content = self.get(nodeid, 'content')
1932 # figure the mime type
1933 if self.properties.has_key('type'):
1934 mime_type = self.get(nodeid, 'type')
1935 else:
1936 mime_type = self.default_mime_type
1938 # and index!
1939 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
1940 mime_type)
1942 # XXX deviation from spec - was called ItemClass
1943 class IssueClass(Class, roundupdb.IssueClass):
1944 # Overridden methods:
1945 def __init__(self, db, classname, **properties):
1946 '''The newly-created class automatically includes the "messages",
1947 "files", "nosy", and "superseder" properties. If the 'properties'
1948 dictionary attempts to specify any of these properties or a
1949 "creation" or "activity" property, a ValueError is raised.
1950 '''
1951 if not properties.has_key('title'):
1952 properties['title'] = hyperdb.String(indexme='yes')
1953 if not properties.has_key('messages'):
1954 properties['messages'] = hyperdb.Multilink("msg")
1955 if not properties.has_key('files'):
1956 properties['files'] = hyperdb.Multilink("file")
1957 if not properties.has_key('nosy'):
1958 # note: journalling is turned off as it really just wastes
1959 # space. this behaviour may be overridden in an instance
1960 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
1961 if not properties.has_key('superseder'):
1962 properties['superseder'] = hyperdb.Multilink(classname)
1963 Class.__init__(self, db, classname, **properties)