1 # $Id: rdbms_common.py,v 1.36 2003-02-26 23:42:54 richard Exp $
2 ''' Relational database (SQL) backend common code.
4 Basics:
6 - map roundup classes to relational tables
7 - automatically detect schema changes and modify the table schemas
8 appropriately (we store the "database version" of the schema in the
9 database itself as the only row of the "schema" table)
10 - multilinks (which represent a many-to-many relationship) are handled through
11 intermediate tables
12 - journals are stored adjunct to the per-class tables
13 - table names and columns have "_" prepended so the names can't clash with
14 restricted names (like "order")
15 - retirement is determined by the __retired__ column being true
17 Database-specific changes may generally be pushed out to the overridable
18 sql_* methods, since everything else should be fairly generic. There's
19 probably a bit of work to be done if a database is used that actually
20 honors column typing, since the initial databases don't (sqlite stores
21 everything as a string, and gadfly stores anything that's marsallable).
22 '''
24 # standard python modules
25 import sys, os, time, re, errno, weakref, copy
27 # roundup modules
28 from roundup import hyperdb, date, password, roundupdb, security
29 from roundup.hyperdb import String, Password, Date, Interval, Link, \
30 Multilink, DatabaseError, Boolean, Number, Node
31 from roundup.backends import locking
33 # support
34 from blobfiles import FileStorage
35 from roundup.indexer import Indexer
36 from sessions import Sessions, OneTimeKeys
38 # number of rows to keep in memory
39 ROW_CACHE_SIZE = 100
41 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
42 ''' Wrapper around an SQL database that presents a hyperdb interface.
44 - some functionality is specific to the actual SQL database, hence
45 the sql_* methods that are NotImplemented
46 - we keep a cache of the latest ROW_CACHE_SIZE row fetches.
47 '''
48 def __init__(self, config, journaltag=None):
49 ''' Open the database and load the schema from it.
50 '''
51 self.config, self.journaltag = config, journaltag
52 self.dir = config.DATABASE
53 self.classes = {}
54 self.indexer = Indexer(self.dir)
55 self.sessions = Sessions(self.config)
56 self.otks = OneTimeKeys(self.config)
57 self.security = security.Security(self)
59 # additional transaction support for external files and the like
60 self.transactions = []
62 # keep a cache of the N most recently retrieved rows of any kind
63 # (classname, nodeid) = row
64 self.cache = {}
65 self.cache_lru = []
67 # database lock
68 self.lockfile = None
70 # open a connection to the database, creating the "conn" attribute
71 self.open_connection()
73 def clearCache(self):
74 self.cache = {}
75 self.cache_lru = []
77 def open_connection(self):
78 ''' Open a connection to the database, creating it if necessary
79 '''
80 raise NotImplemented
82 def sql(self, sql, args=None):
83 ''' Execute the sql with the optional args.
84 '''
85 if __debug__:
86 print >>hyperdb.DEBUG, (self, sql, args)
87 if args:
88 self.cursor.execute(sql, args)
89 else:
90 self.cursor.execute(sql)
92 def sql_fetchone(self):
93 ''' Fetch a single row. If there's nothing to fetch, return None.
94 '''
95 raise NotImplemented
97 def sql_stringquote(self, value):
98 ''' Quote the string so it's safe to put in the 'sql quotes'
99 '''
100 return re.sub("'", "''", str(value))
102 def save_dbschema(self, schema):
103 ''' Save the schema definition that the database currently implements
104 '''
105 raise NotImplemented
107 def load_dbschema(self):
108 ''' Load the schema definition that the database currently implements
109 '''
110 raise NotImplemented
112 def post_init(self):
113 ''' Called once the schema initialisation has finished.
115 We should now confirm that the schema defined by our "classes"
116 attribute actually matches the schema in the database.
117 '''
118 # now detect changes in the schema
119 save = 0
120 for classname, spec in self.classes.items():
121 if self.database_schema.has_key(classname):
122 dbspec = self.database_schema[classname]
123 if self.update_class(spec, dbspec):
124 self.database_schema[classname] = spec.schema()
125 save = 1
126 else:
127 self.create_class(spec)
128 self.database_schema[classname] = spec.schema()
129 save = 1
131 for classname in self.database_schema.keys():
132 if not self.classes.has_key(classname):
133 self.drop_class(classname)
135 # update the database version of the schema
136 if save:
137 self.sql('delete from schema')
138 self.save_dbschema(self.database_schema)
140 # reindex the db if necessary
141 if self.indexer.should_reindex():
142 self.reindex()
144 # commit
145 self.conn.commit()
147 # figure the "curuserid"
148 if self.journaltag is None:
149 self.curuserid = None
150 elif self.journaltag == 'admin':
151 # admin user may not exist, but always has ID 1
152 self.curuserid = '1'
153 else:
154 self.curuserid = self.user.lookup(self.journaltag)
156 def reindex(self):
157 for klass in self.classes.values():
158 for nodeid in klass.list():
159 klass.index(nodeid)
160 self.indexer.save_index()
162 def determine_columns(self, properties):
163 ''' Figure the column names and multilink properties from the spec
165 "properties" is a list of (name, prop) where prop may be an
166 instance of a hyperdb "type" _or_ a string repr of that type.
167 '''
168 cols = ['_activity', '_creator', '_creation']
169 mls = []
170 # add the multilinks separately
171 for col, prop in properties:
172 if isinstance(prop, Multilink):
173 mls.append(col)
174 elif isinstance(prop, type('')) and prop.find('Multilink') != -1:
175 mls.append(col)
176 else:
177 cols.append('_'+col)
178 cols.sort()
179 return cols, mls
181 def update_class(self, spec, dbspec):
182 ''' Determine the differences between the current spec and the
183 database version of the spec, and update where necessary
184 '''
185 spec_schema = spec.schema()
186 if spec_schema == dbspec:
187 # no save needed for this one
188 return 0
189 if __debug__:
190 print >>hyperdb.DEBUG, 'update_class FIRING'
192 # key property changed?
193 if dbspec[0] != spec_schema[0]:
194 if __debug__:
195 print >>hyperdb.DEBUG, 'update_class setting keyprop', `spec[0]`
196 # XXX turn on indexing for the key property
198 # dict 'em up
199 spec_propnames,spec_props = [],{}
200 for propname,prop in spec_schema[1]:
201 spec_propnames.append(propname)
202 spec_props[propname] = prop
203 dbspec_propnames,dbspec_props = [],{}
204 for propname,prop in dbspec[1]:
205 dbspec_propnames.append(propname)
206 dbspec_props[propname] = prop
208 # now compare
209 for propname in spec_propnames:
210 prop = spec_props[propname]
211 if dbspec_props.has_key(propname) and prop==dbspec_props[propname]:
212 continue
213 if __debug__:
214 print >>hyperdb.DEBUG, 'update_class ADD', (propname, prop)
216 if not dbspec_props.has_key(propname):
217 # add the property
218 if isinstance(prop, Multilink):
219 # all we have to do here is create a new table, easy!
220 self.create_multilink_table(spec, propname)
221 continue
223 # no ALTER TABLE, so we:
224 # 1. pull out the data, including an extra None column
225 oldcols, x = self.determine_columns(dbspec[1])
226 oldcols.append('id')
227 oldcols.append('__retired__')
228 cn = spec.classname
229 sql = 'select %s,%s from _%s'%(','.join(oldcols), self.arg, cn)
230 if __debug__:
231 print >>hyperdb.DEBUG, 'update_class', (self, sql, None)
232 self.cursor.execute(sql, (None,))
233 olddata = self.cursor.fetchall()
235 # 2. drop the old table
236 self.cursor.execute('drop table _%s'%cn)
238 # 3. create the new table
239 cols, mls = self.create_class_table(spec)
240 # ensure the new column is last
241 cols.remove('_'+propname)
242 assert oldcols == cols, "Column lists don't match!"
243 cols.append('_'+propname)
245 # 4. populate with the data from step one
246 s = ','.join([self.arg for x in cols])
247 scols = ','.join(cols)
248 sql = 'insert into _%s (%s) values (%s)'%(cn, scols, s)
250 # GAH, nothing had better go wrong from here on in... but
251 # we have to commit the drop...
252 # XXX this isn't necessary in sqlite :(
253 self.conn.commit()
255 # do the insert
256 for row in olddata:
257 self.sql(sql, tuple(row))
259 else:
260 # modify the property
261 if __debug__:
262 print >>hyperdb.DEBUG, 'update_class NOOP'
263 pass # NOOP in gadfly
265 # and the other way - only worry about deletions here
266 for propname in dbspec_propnames:
267 prop = dbspec_props[propname]
268 if spec_props.has_key(propname):
269 continue
270 if __debug__:
271 print >>hyperdb.DEBUG, 'update_class REMOVE', `prop`
273 # delete the property
274 if isinstance(prop, Multilink):
275 sql = 'drop table %s_%s'%(spec.classname, prop)
276 if __debug__:
277 print >>hyperdb.DEBUG, 'update_class', (self, sql)
278 self.cursor.execute(sql)
279 else:
280 # no ALTER TABLE, so we:
281 # 1. pull out the data, excluding the removed column
282 oldcols, x = self.determine_columns(spec.properties.items())
283 oldcols.append('id')
284 oldcols.append('__retired__')
285 # remove the missing column
286 oldcols.remove('_'+propname)
287 cn = spec.classname
288 sql = 'select %s from _%s'%(','.join(oldcols), cn)
289 self.cursor.execute(sql, (None,))
290 olddata = sql.fetchall()
292 # 2. drop the old table
293 self.cursor.execute('drop table _%s'%cn)
295 # 3. create the new table
296 cols, mls = self.create_class_table(self, spec)
297 assert oldcols != cols, "Column lists don't match!"
299 # 4. populate with the data from step one
300 qs = ','.join([self.arg for x in cols])
301 sql = 'insert into _%s values (%s)'%(cn, s)
302 self.cursor.execute(sql, olddata)
303 return 1
305 def create_class_table(self, spec):
306 ''' create the class table for the given spec
307 '''
308 cols, mls = self.determine_columns(spec.properties.items())
310 # add on our special columns
311 cols.append('id')
312 cols.append('__retired__')
314 # create the base table
315 scols = ','.join(['%s varchar'%x for x in cols])
316 sql = 'create table _%s (%s)'%(spec.classname, scols)
317 if __debug__:
318 print >>hyperdb.DEBUG, 'create_class', (self, sql)
319 self.cursor.execute(sql)
321 return cols, mls
323 def create_journal_table(self, spec):
324 ''' create the journal table for a class given the spec and
325 already-determined cols
326 '''
327 # journal table
328 cols = ','.join(['%s varchar'%x
329 for x in 'nodeid date tag action params'.split()])
330 sql = 'create table %s__journal (%s)'%(spec.classname, cols)
331 if __debug__:
332 print >>hyperdb.DEBUG, 'create_class', (self, sql)
333 self.cursor.execute(sql)
335 def create_multilink_table(self, spec, ml):
336 ''' Create a multilink table for the "ml" property of the class
337 given by the spec
338 '''
339 sql = 'create table %s_%s (linkid varchar, nodeid varchar)'%(
340 spec.classname, ml)
341 if __debug__:
342 print >>hyperdb.DEBUG, 'create_class', (self, sql)
343 self.cursor.execute(sql)
345 def create_class(self, spec):
346 ''' Create a database table according to the given spec.
347 '''
348 cols, mls = self.create_class_table(spec)
349 self.create_journal_table(spec)
351 # now create the multilink tables
352 for ml in mls:
353 self.create_multilink_table(spec, ml)
355 # ID counter
356 sql = 'insert into ids (name, num) values (%s,%s)'%(self.arg, self.arg)
357 vals = (spec.classname, 1)
358 if __debug__:
359 print >>hyperdb.DEBUG, 'create_class', (self, sql, vals)
360 self.cursor.execute(sql, vals)
362 def drop_class(self, spec):
363 ''' Drop the given table from the database.
365 Drop the journal and multilink tables too.
366 '''
367 # figure the multilinks
368 mls = []
369 for col, prop in spec.properties.items():
370 if isinstance(prop, Multilink):
371 mls.append(col)
373 sql = 'drop table _%s'%spec.classname
374 if __debug__:
375 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
376 self.cursor.execute(sql)
378 sql = 'drop table %s__journal'%spec.classname
379 if __debug__:
380 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
381 self.cursor.execute(sql)
383 for ml in mls:
384 sql = 'drop table %s_%s'%(spec.classname, ml)
385 if __debug__:
386 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
387 self.cursor.execute(sql)
389 #
390 # Classes
391 #
392 def __getattr__(self, classname):
393 ''' A convenient way of calling self.getclass(classname).
394 '''
395 if self.classes.has_key(classname):
396 if __debug__:
397 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
398 return self.classes[classname]
399 raise AttributeError, classname
401 def addclass(self, cl):
402 ''' Add a Class to the hyperdatabase.
403 '''
404 if __debug__:
405 print >>hyperdb.DEBUG, 'addclass', (self, cl)
406 cn = cl.classname
407 if self.classes.has_key(cn):
408 raise ValueError, cn
409 self.classes[cn] = cl
411 def getclasses(self):
412 ''' Return a list of the names of all existing classes.
413 '''
414 if __debug__:
415 print >>hyperdb.DEBUG, 'getclasses', (self,)
416 l = self.classes.keys()
417 l.sort()
418 return l
420 def getclass(self, classname):
421 '''Get the Class object representing a particular class.
423 If 'classname' is not a valid class name, a KeyError is raised.
424 '''
425 if __debug__:
426 print >>hyperdb.DEBUG, 'getclass', (self, classname)
427 try:
428 return self.classes[classname]
429 except KeyError:
430 raise KeyError, 'There is no class called "%s"'%classname
432 def clear(self):
433 ''' Delete all database contents.
435 Note: I don't commit here, which is different behaviour to the
436 "nuke from orbit" behaviour in the *dbms.
437 '''
438 if __debug__:
439 print >>hyperdb.DEBUG, 'clear', (self,)
440 for cn in self.classes.keys():
441 sql = 'delete from _%s'%cn
442 if __debug__:
443 print >>hyperdb.DEBUG, 'clear', (self, sql)
444 self.cursor.execute(sql)
446 #
447 # Node IDs
448 #
449 def newid(self, classname):
450 ''' Generate a new id for the given class
451 '''
452 # get the next ID
453 sql = 'select num from ids where name=%s'%self.arg
454 if __debug__:
455 print >>hyperdb.DEBUG, 'newid', (self, sql, classname)
456 self.cursor.execute(sql, (classname, ))
457 newid = self.cursor.fetchone()[0]
459 # update the counter
460 sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
461 vals = (int(newid)+1, classname)
462 if __debug__:
463 print >>hyperdb.DEBUG, 'newid', (self, sql, vals)
464 self.cursor.execute(sql, vals)
466 # return as string
467 return str(newid)
469 def setid(self, classname, setid):
470 ''' Set the id counter: used during import of database
471 '''
472 sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
473 vals = (setid, classname)
474 if __debug__:
475 print >>hyperdb.DEBUG, 'setid', (self, sql, vals)
476 self.cursor.execute(sql, vals)
478 #
479 # Nodes
480 #
482 def addnode(self, classname, nodeid, node):
483 ''' Add the specified node to its class's db.
484 '''
485 if __debug__:
486 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
487 # gadfly requires values for all non-multilink columns
488 cl = self.classes[classname]
489 cols, mls = self.determine_columns(cl.properties.items())
491 # we'll be supplied these props if we're doing an import
492 if not node.has_key('creator'):
493 # add in the "calculated" properties (dupe so we don't affect
494 # calling code's node assumptions)
495 node = node.copy()
496 node['creation'] = node['activity'] = date.Date()
497 node['creator'] = self.curuserid
499 # default the non-multilink columns
500 for col, prop in cl.properties.items():
501 if not isinstance(col, Multilink):
502 if not node.has_key(col):
503 node[col] = None
505 # clear this node out of the cache if it's in there
506 key = (classname, nodeid)
507 if self.cache.has_key(key):
508 del self.cache[key]
509 self.cache_lru.remove(key)
511 # make the node data safe for the DB
512 node = self.serialise(classname, node)
514 # make sure the ordering is correct for column name -> column value
515 vals = tuple([node[col[1:]] for col in cols]) + (nodeid, 0)
516 s = ','.join([self.arg for x in cols]) + ',%s,%s'%(self.arg, self.arg)
517 cols = ','.join(cols) + ',id,__retired__'
519 # perform the inserts
520 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
521 if __debug__:
522 print >>hyperdb.DEBUG, 'addnode', (self, sql, vals)
523 self.cursor.execute(sql, vals)
525 # insert the multilink rows
526 for col in mls:
527 t = '%s_%s'%(classname, col)
528 for entry in node[col]:
529 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
530 self.arg, self.arg)
531 self.sql(sql, (entry, nodeid))
533 # make sure we do the commit-time extra stuff for this node
534 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
536 def setnode(self, classname, nodeid, values, multilink_changes):
537 ''' Change the specified node.
538 '''
539 if __debug__:
540 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, values)
542 # clear this node out of the cache if it's in there
543 key = (classname, nodeid)
544 if self.cache.has_key(key):
545 del self.cache[key]
546 self.cache_lru.remove(key)
548 # add the special props
549 values = values.copy()
550 values['activity'] = date.Date()
552 # make db-friendly
553 values = self.serialise(classname, values)
555 cl = self.classes[classname]
556 cols = []
557 mls = []
558 # add the multilinks separately
559 props = cl.getprops()
560 for col in values.keys():
561 prop = props[col]
562 if isinstance(prop, Multilink):
563 mls.append(col)
564 else:
565 cols.append('_'+col)
566 cols.sort()
568 # if there's any updates to regular columns, do them
569 if cols:
570 # make sure the ordering is correct for column name -> column value
571 sqlvals = tuple([values[col[1:]] for col in cols]) + (nodeid,)
572 s = ','.join(['%s=%s'%(x, self.arg) for x in cols])
573 cols = ','.join(cols)
575 # perform the update
576 sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
577 if __debug__:
578 print >>hyperdb.DEBUG, 'setnode', (self, sql, sqlvals)
579 self.cursor.execute(sql, sqlvals)
581 # now the fun bit, updating the multilinks ;)
582 for col, (add, remove) in multilink_changes.items():
583 tn = '%s_%s'%(classname, col)
584 if add:
585 sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
586 self.arg, self.arg)
587 for addid in add:
588 self.sql(sql, (nodeid, addid))
589 if remove:
590 sql = 'delete from %s where nodeid=%s and linkid=%s'%(tn,
591 self.arg, self.arg)
592 for removeid in remove:
593 self.sql(sql, (nodeid, removeid))
595 # make sure we do the commit-time extra stuff for this node
596 self.transactions.append((self.doSaveNode, (classname, nodeid, values)))
598 def getnode(self, classname, nodeid):
599 ''' Get a node from the database.
600 '''
601 if __debug__:
602 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid)
604 # see if we have this node cached
605 key = (classname, nodeid)
606 if self.cache.has_key(key):
607 # push us back to the top of the LRU
608 self.cache_lru.remove(key)
609 self.cache_lru.insert(0, key)
610 # return the cached information
611 return self.cache[key]
613 # figure the columns we're fetching
614 cl = self.classes[classname]
615 cols, mls = self.determine_columns(cl.properties.items())
616 scols = ','.join(cols)
618 # perform the basic property fetch
619 sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
620 self.sql(sql, (nodeid,))
622 values = self.sql_fetchone()
623 if values is None:
624 raise IndexError, 'no such %s node %s'%(classname, nodeid)
626 # make up the node
627 node = {}
628 for col in range(len(cols)):
629 node[cols[col][1:]] = values[col]
631 # now the multilinks
632 for col in mls:
633 # get the link ids
634 sql = 'select linkid from %s_%s where nodeid=%s'%(classname, col,
635 self.arg)
636 self.cursor.execute(sql, (nodeid,))
637 # extract the first column from the result
638 node[col] = [x[0] for x in self.cursor.fetchall()]
640 # un-dbificate the node data
641 node = self.unserialise(classname, node)
643 # save off in the cache
644 key = (classname, nodeid)
645 self.cache[key] = node
646 # update the LRU
647 self.cache_lru.insert(0, key)
648 if len(self.cache_lru) > ROW_CACHE_SIZE:
649 del self.cache[self.cache_lru.pop()]
651 return node
653 def destroynode(self, classname, nodeid):
654 '''Remove a node from the database. Called exclusively by the
655 destroy() method on Class.
656 '''
657 if __debug__:
658 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
660 # make sure the node exists
661 if not self.hasnode(classname, nodeid):
662 raise IndexError, '%s has no node %s'%(classname, nodeid)
664 # see if we have this node cached
665 if self.cache.has_key((classname, nodeid)):
666 del self.cache[(classname, nodeid)]
668 # see if there's any obvious commit actions that we should get rid of
669 for entry in self.transactions[:]:
670 if entry[1][:2] == (classname, nodeid):
671 self.transactions.remove(entry)
673 # now do the SQL
674 sql = 'delete from _%s where id=%s'%(classname, self.arg)
675 self.sql(sql, (nodeid,))
677 # remove from multilnks
678 cl = self.getclass(classname)
679 x, mls = self.determine_columns(cl.properties.items())
680 for col in mls:
681 # get the link ids
682 sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
683 self.cursor.execute(sql, (nodeid,))
685 # remove journal entries
686 sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
687 self.sql(sql, (nodeid,))
689 def serialise(self, classname, node):
690 '''Copy the node contents, converting non-marshallable data into
691 marshallable data.
692 '''
693 if __debug__:
694 print >>hyperdb.DEBUG, 'serialise', classname, node
695 properties = self.getclass(classname).getprops()
696 d = {}
697 for k, v in node.items():
698 # if the property doesn't exist, or is the "retired" flag then
699 # it won't be in the properties dict
700 if not properties.has_key(k):
701 d[k] = v
702 continue
704 # get the property spec
705 prop = properties[k]
707 if isinstance(prop, Password) and v is not None:
708 d[k] = str(v)
709 elif isinstance(prop, Date) and v is not None:
710 d[k] = v.serialise()
711 elif isinstance(prop, Interval) and v is not None:
712 d[k] = v.serialise()
713 else:
714 d[k] = v
715 return d
717 def unserialise(self, classname, node):
718 '''Decode the marshalled node data
719 '''
720 if __debug__:
721 print >>hyperdb.DEBUG, 'unserialise', classname, node
722 properties = self.getclass(classname).getprops()
723 d = {}
724 for k, v in node.items():
725 # if the property doesn't exist, or is the "retired" flag then
726 # it won't be in the properties dict
727 if not properties.has_key(k):
728 d[k] = v
729 continue
731 # get the property spec
732 prop = properties[k]
734 if isinstance(prop, Date) and v is not None:
735 d[k] = date.Date(v)
736 elif isinstance(prop, Interval) and v is not None:
737 d[k] = date.Interval(v)
738 elif isinstance(prop, Password) and v is not None:
739 p = password.Password()
740 p.unpack(v)
741 d[k] = p
742 elif (isinstance(prop, Boolean) or isinstance(prop, Number)) and v is not None:
743 d[k]=float(v)
744 else:
745 d[k] = v
746 return d
748 def hasnode(self, classname, nodeid):
749 ''' Determine if the database has a given node.
750 '''
751 sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
752 if __debug__:
753 print >>hyperdb.DEBUG, 'hasnode', (self, sql, nodeid)
754 self.cursor.execute(sql, (nodeid,))
755 return int(self.cursor.fetchone()[0])
757 def countnodes(self, classname):
758 ''' Count the number of nodes that exist for a particular Class.
759 '''
760 sql = 'select count(*) from _%s'%classname
761 if __debug__:
762 print >>hyperdb.DEBUG, 'countnodes', (self, sql)
763 self.cursor.execute(sql)
764 return self.cursor.fetchone()[0]
766 def getnodeids(self, classname, retired=0):
767 ''' Retrieve all the ids of the nodes for a particular Class.
769 Set retired=None to get all nodes. Otherwise it'll get all the
770 retired or non-retired nodes, depending on the flag.
771 '''
772 # flip the sense of the flag if we don't want all of them
773 if retired is not None:
774 retired = not retired
775 sql = 'select id from _%s where __retired__ <> %s'%(classname, self.arg)
776 if __debug__:
777 print >>hyperdb.DEBUG, 'getnodeids', (self, sql, retired)
778 self.cursor.execute(sql, (retired,))
779 return [x[0] for x in self.cursor.fetchall()]
781 def addjournal(self, classname, nodeid, action, params, creator=None,
782 creation=None):
783 ''' Journal the Action
784 'action' may be:
786 'create' or 'set' -- 'params' is a dictionary of property values
787 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
788 'retire' -- 'params' is None
789 '''
790 # serialise the parameters now if necessary
791 if isinstance(params, type({})):
792 if action in ('set', 'create'):
793 params = self.serialise(classname, params)
795 # handle supply of the special journalling parameters (usually
796 # supplied on importing an existing database)
797 if creator:
798 journaltag = creator
799 else:
800 journaltag = self.curuserid
801 if creation:
802 journaldate = creation.serialise()
803 else:
804 journaldate = date.Date().serialise()
806 # create the journal entry
807 cols = ','.join('nodeid date tag action params'.split())
809 if __debug__:
810 print >>hyperdb.DEBUG, 'addjournal', (nodeid, journaldate,
811 journaltag, action, params)
813 self.save_journal(classname, cols, nodeid, journaldate,
814 journaltag, action, params)
816 def save_journal(self, classname, cols, nodeid, journaldate,
817 journaltag, action, params):
818 ''' Save the journal entry to the database
819 '''
820 raise NotImplemented
822 def getjournal(self, classname, nodeid):
823 ''' get the journal for id
824 '''
825 # make sure the node exists
826 if not self.hasnode(classname, nodeid):
827 raise IndexError, '%s has no node %s'%(classname, nodeid)
829 cols = ','.join('nodeid date tag action params'.split())
830 return self.load_journal(classname, cols, nodeid)
832 def load_journal(self, classname, cols, nodeid):
833 ''' Load the journal from the database
834 '''
835 raise NotImplemented
837 def pack(self, pack_before):
838 ''' Delete all journal entries except "create" before 'pack_before'.
839 '''
840 # get a 'yyyymmddhhmmss' version of the date
841 date_stamp = pack_before.serialise()
843 # do the delete
844 for classname in self.classes.keys():
845 sql = "delete from %s__journal where date<%s and "\
846 "action<>'create'"%(classname, self.arg)
847 if __debug__:
848 print >>hyperdb.DEBUG, 'pack', (self, sql, date_stamp)
849 self.cursor.execute(sql, (date_stamp,))
851 def sql_commit(self):
852 ''' Actually commit to the database.
853 '''
854 self.conn.commit()
856 def commit(self):
857 ''' Commit the current transactions.
859 Save all data changed since the database was opened or since the
860 last commit() or rollback().
861 '''
862 if __debug__:
863 print >>hyperdb.DEBUG, 'commit', (self,)
865 # commit the database
866 self.sql_commit()
868 # now, do all the other transaction stuff
869 reindex = {}
870 for method, args in self.transactions:
871 reindex[method(*args)] = 1
873 # reindex the nodes that request it
874 for classname, nodeid in filter(None, reindex.keys()):
875 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
876 self.getclass(classname).index(nodeid)
878 # save the indexer state
879 self.indexer.save_index()
881 # clear out the transactions
882 self.transactions = []
884 def rollback(self):
885 ''' Reverse all actions from the current transaction.
887 Undo all the changes made since the database was opened or the last
888 commit() or rollback() was performed.
889 '''
890 if __debug__:
891 print >>hyperdb.DEBUG, 'rollback', (self,)
893 # roll back
894 self.conn.rollback()
896 # roll back "other" transaction stuff
897 for method, args in self.transactions:
898 # delete temporary files
899 if method == self.doStoreFile:
900 self.rollbackStoreFile(*args)
901 self.transactions = []
903 def doSaveNode(self, classname, nodeid, node):
904 ''' dummy that just generates a reindex event
905 '''
906 # return the classname, nodeid so we reindex this content
907 return (classname, nodeid)
909 def close(self):
910 ''' Close off the connection.
911 '''
912 self.conn.close()
913 if self.lockfile is not None:
914 locking.release_lock(self.lockfile)
915 if self.lockfile is not None:
916 self.lockfile.close()
917 self.lockfile = None
919 #
920 # The base Class class
921 #
922 class Class(hyperdb.Class):
923 ''' The handle to a particular class of nodes in a hyperdatabase.
925 All methods except __repr__ and getnode must be implemented by a
926 concrete backend Class.
927 '''
929 def __init__(self, db, classname, **properties):
930 '''Create a new class with a given name and property specification.
932 'classname' must not collide with the name of an existing class,
933 or a ValueError is raised. The keyword arguments in 'properties'
934 must map names to property objects, or a TypeError is raised.
935 '''
936 if (properties.has_key('creation') or properties.has_key('activity')
937 or properties.has_key('creator')):
938 raise ValueError, '"creation", "activity" and "creator" are '\
939 'reserved'
941 self.classname = classname
942 self.properties = properties
943 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
944 self.key = ''
946 # should we journal changes (default yes)
947 self.do_journal = 1
949 # do the db-related init stuff
950 db.addclass(self)
952 self.auditors = {'create': [], 'set': [], 'retire': []}
953 self.reactors = {'create': [], 'set': [], 'retire': []}
955 def schema(self):
956 ''' A dumpable version of the schema that we can store in the
957 database
958 '''
959 return (self.key, [(x, repr(y)) for x,y in self.properties.items()])
961 def enableJournalling(self):
962 '''Turn journalling on for this class
963 '''
964 self.do_journal = 1
966 def disableJournalling(self):
967 '''Turn journalling off for this class
968 '''
969 self.do_journal = 0
971 # Editing nodes:
972 def create(self, **propvalues):
973 ''' Create a new node of this class and return its id.
975 The keyword arguments in 'propvalues' map property names to values.
977 The values of arguments must be acceptable for the types of their
978 corresponding properties or a TypeError is raised.
980 If this class has a key property, it must be present and its value
981 must not collide with other key strings or a ValueError is raised.
983 Any other properties on this class that are missing from the
984 'propvalues' dictionary are set to None.
986 If an id in a link or multilink property does not refer to a valid
987 node, an IndexError is raised.
988 '''
989 self.fireAuditors('create', None, propvalues)
990 newid = self.create_inner(**propvalues)
991 self.fireReactors('create', newid, None)
992 return newid
994 def create_inner(self, **propvalues):
995 ''' Called by create, in-between the audit and react calls.
996 '''
997 if propvalues.has_key('id'):
998 raise KeyError, '"id" is reserved'
1000 if self.db.journaltag is None:
1001 raise DatabaseError, 'Database open read-only'
1003 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1004 raise KeyError, '"creation" and "activity" are reserved'
1006 # new node's id
1007 newid = self.db.newid(self.classname)
1009 # validate propvalues
1010 num_re = re.compile('^\d+$')
1011 for key, value in propvalues.items():
1012 if key == self.key:
1013 try:
1014 self.lookup(value)
1015 except KeyError:
1016 pass
1017 else:
1018 raise ValueError, 'node with key "%s" exists'%value
1020 # try to handle this property
1021 try:
1022 prop = self.properties[key]
1023 except KeyError:
1024 raise KeyError, '"%s" has no property "%s"'%(self.classname,
1025 key)
1027 if value is not None and isinstance(prop, Link):
1028 if type(value) != type(''):
1029 raise ValueError, 'link value must be String'
1030 link_class = self.properties[key].classname
1031 # if it isn't a number, it's a key
1032 if not num_re.match(value):
1033 try:
1034 value = self.db.classes[link_class].lookup(value)
1035 except (TypeError, KeyError):
1036 raise IndexError, 'new property "%s": %s not a %s'%(
1037 key, value, link_class)
1038 elif not self.db.getclass(link_class).hasnode(value):
1039 raise IndexError, '%s has no node %s'%(link_class, value)
1041 # save off the value
1042 propvalues[key] = value
1044 # register the link with the newly linked node
1045 if self.do_journal and self.properties[key].do_journal:
1046 self.db.addjournal(link_class, value, 'link',
1047 (self.classname, newid, key))
1049 elif isinstance(prop, Multilink):
1050 if type(value) != type([]):
1051 raise TypeError, 'new property "%s" not a list of ids'%key
1053 # clean up and validate the list of links
1054 link_class = self.properties[key].classname
1055 l = []
1056 for entry in value:
1057 if type(entry) != type(''):
1058 raise ValueError, '"%s" multilink value (%r) '\
1059 'must contain Strings'%(key, value)
1060 # if it isn't a number, it's a key
1061 if not num_re.match(entry):
1062 try:
1063 entry = self.db.classes[link_class].lookup(entry)
1064 except (TypeError, KeyError):
1065 raise IndexError, 'new property "%s": %s not a %s'%(
1066 key, entry, self.properties[key].classname)
1067 l.append(entry)
1068 value = l
1069 propvalues[key] = value
1071 # handle additions
1072 for nodeid in value:
1073 if not self.db.getclass(link_class).hasnode(nodeid):
1074 raise IndexError, '%s has no node %s'%(link_class,
1075 nodeid)
1076 # register the link with the newly linked node
1077 if self.do_journal and self.properties[key].do_journal:
1078 self.db.addjournal(link_class, nodeid, 'link',
1079 (self.classname, newid, key))
1081 elif isinstance(prop, String):
1082 if type(value) != type('') and type(value) != type(u''):
1083 raise TypeError, 'new property "%s" not a string'%key
1085 elif isinstance(prop, Password):
1086 if not isinstance(value, password.Password):
1087 raise TypeError, 'new property "%s" not a Password'%key
1089 elif isinstance(prop, Date):
1090 if value is not None and not isinstance(value, date.Date):
1091 raise TypeError, 'new property "%s" not a Date'%key
1093 elif isinstance(prop, Interval):
1094 if value is not None and not isinstance(value, date.Interval):
1095 raise TypeError, 'new property "%s" not an Interval'%key
1097 elif value is not None and isinstance(prop, Number):
1098 try:
1099 float(value)
1100 except ValueError:
1101 raise TypeError, 'new property "%s" not numeric'%key
1103 elif value is not None and isinstance(prop, Boolean):
1104 try:
1105 int(value)
1106 except ValueError:
1107 raise TypeError, 'new property "%s" not boolean'%key
1109 # make sure there's data where there needs to be
1110 for key, prop in self.properties.items():
1111 if propvalues.has_key(key):
1112 continue
1113 if key == self.key:
1114 raise ValueError, 'key property "%s" is required'%key
1115 if isinstance(prop, Multilink):
1116 propvalues[key] = []
1117 else:
1118 propvalues[key] = None
1120 # done
1121 self.db.addnode(self.classname, newid, propvalues)
1122 if self.do_journal:
1123 self.db.addjournal(self.classname, newid, 'create', {})
1125 return newid
1127 def export_list(self, propnames, nodeid):
1128 ''' Export a node - generate a list of CSV-able data in the order
1129 specified by propnames for the given node.
1130 '''
1131 properties = self.getprops()
1132 l = []
1133 for prop in propnames:
1134 proptype = properties[prop]
1135 value = self.get(nodeid, prop)
1136 # "marshal" data where needed
1137 if value is None:
1138 pass
1139 elif isinstance(proptype, hyperdb.Date):
1140 value = value.get_tuple()
1141 elif isinstance(proptype, hyperdb.Interval):
1142 value = value.get_tuple()
1143 elif isinstance(proptype, hyperdb.Password):
1144 value = str(value)
1145 l.append(repr(value))
1146 l.append(self.is_retired(nodeid))
1147 return l
1149 def import_list(self, propnames, proplist):
1150 ''' Import a node - all information including "id" is present and
1151 should not be sanity checked. Triggers are not triggered. The
1152 journal should be initialised using the "creator" and "created"
1153 information.
1155 Return the nodeid of the node imported.
1156 '''
1157 if self.db.journaltag is None:
1158 raise DatabaseError, 'Database open read-only'
1159 properties = self.getprops()
1161 # make the new node's property map
1162 d = {}
1163 for i in range(len(propnames)):
1164 # Use eval to reverse the repr() used to output the CSV
1165 value = eval(proplist[i])
1167 # Figure the property for this column
1168 propname = propnames[i]
1169 prop = properties[propname]
1171 # "unmarshal" where necessary
1172 if propname == 'id':
1173 newid = value
1174 continue
1175 elif value is None:
1176 # don't set Nones
1177 continue
1178 elif isinstance(prop, hyperdb.Date):
1179 value = date.Date(value)
1180 elif isinstance(prop, hyperdb.Interval):
1181 value = date.Interval(value)
1182 elif isinstance(prop, hyperdb.Password):
1183 pwd = password.Password()
1184 pwd.unpack(value)
1185 value = pwd
1186 d[propname] = value
1188 # retire?
1189 if int(proplist[-1]):
1190 # use the arg for __retired__ to cope with any odd database type
1191 # conversion (hello, sqlite)
1192 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1193 self.db.arg, self.db.arg)
1194 if __debug__:
1195 print >>hyperdb.DEBUG, 'retire', (self, sql, newid)
1196 self.db.cursor.execute(sql, (1, newid))
1198 # add the node and journal
1199 self.db.addnode(self.classname, newid, d)
1201 # extract the extraneous journalling gumpf and nuke it
1202 if d.has_key('creator'):
1203 creator = d['creator']
1204 del d['creator']
1205 else:
1206 creator = None
1207 if d.has_key('creation'):
1208 creation = d['creation']
1209 del d['creation']
1210 else:
1211 creation = None
1212 if d.has_key('activity'):
1213 del d['activity']
1214 self.db.addjournal(self.classname, newid, 'create', {}, creator,
1215 creation)
1216 return newid
1218 _marker = []
1219 def get(self, nodeid, propname, default=_marker, cache=1):
1220 '''Get the value of a property on an existing node of this class.
1222 'nodeid' must be the id of an existing node of this class or an
1223 IndexError is raised. 'propname' must be the name of a property
1224 of this class or a KeyError is raised.
1226 'cache' indicates whether the transaction cache should be queried
1227 for the node. If the node has been modified and you need to
1228 determine what its values prior to modification are, you need to
1229 set cache=0.
1230 '''
1231 if propname == 'id':
1232 return nodeid
1234 # get the node's dict
1235 d = self.db.getnode(self.classname, nodeid)
1237 if propname == 'creation':
1238 if d.has_key('creation'):
1239 return d['creation']
1240 else:
1241 return date.Date()
1242 if propname == 'activity':
1243 if d.has_key('activity'):
1244 return d['activity']
1245 else:
1246 return date.Date()
1247 if propname == 'creator':
1248 if d.has_key('creator'):
1249 return d['creator']
1250 else:
1251 return self.db.curuserid
1253 # get the property (raises KeyErorr if invalid)
1254 prop = self.properties[propname]
1256 if not d.has_key(propname):
1257 if default is self._marker:
1258 if isinstance(prop, Multilink):
1259 return []
1260 else:
1261 return None
1262 else:
1263 return default
1265 # don't pass our list to other code
1266 if isinstance(prop, Multilink):
1267 return d[propname][:]
1269 return d[propname]
1271 def getnode(self, nodeid, cache=1):
1272 ''' Return a convenience wrapper for the node.
1274 'nodeid' must be the id of an existing node of this class or an
1275 IndexError is raised.
1277 'cache' indicates whether the transaction cache should be queried
1278 for the node. If the node has been modified and you need to
1279 determine what its values prior to modification are, you need to
1280 set cache=0.
1281 '''
1282 return Node(self, nodeid, cache=cache)
1284 def set(self, nodeid, **propvalues):
1285 '''Modify a property on an existing node of this class.
1287 'nodeid' must be the id of an existing node of this class or an
1288 IndexError is raised.
1290 Each key in 'propvalues' must be the name of a property of this
1291 class or a KeyError is raised.
1293 All values in 'propvalues' must be acceptable types for their
1294 corresponding properties or a TypeError is raised.
1296 If the value of the key property is set, it must not collide with
1297 other key strings or a ValueError is raised.
1299 If the value of a Link or Multilink property contains an invalid
1300 node id, a ValueError is raised.
1301 '''
1302 if not propvalues:
1303 return propvalues
1305 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1306 raise KeyError, '"creation" and "activity" are reserved'
1308 if propvalues.has_key('id'):
1309 raise KeyError, '"id" is reserved'
1311 if self.db.journaltag is None:
1312 raise DatabaseError, 'Database open read-only'
1314 self.fireAuditors('set', nodeid, propvalues)
1315 # Take a copy of the node dict so that the subsequent set
1316 # operation doesn't modify the oldvalues structure.
1317 # XXX used to try the cache here first
1318 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1320 node = self.db.getnode(self.classname, nodeid)
1321 if self.is_retired(nodeid):
1322 raise IndexError, 'Requested item is retired'
1323 num_re = re.compile('^\d+$')
1325 # if the journal value is to be different, store it in here
1326 journalvalues = {}
1328 # remember the add/remove stuff for multilinks, making it easier
1329 # for the Database layer to do its stuff
1330 multilink_changes = {}
1332 for propname, value in propvalues.items():
1333 # check to make sure we're not duplicating an existing key
1334 if propname == self.key and node[propname] != value:
1335 try:
1336 self.lookup(value)
1337 except KeyError:
1338 pass
1339 else:
1340 raise ValueError, 'node with key "%s" exists'%value
1342 # this will raise the KeyError if the property isn't valid
1343 # ... we don't use getprops() here because we only care about
1344 # the writeable properties.
1345 try:
1346 prop = self.properties[propname]
1347 except KeyError:
1348 raise KeyError, '"%s" has no property named "%s"'%(
1349 self.classname, propname)
1351 # if the value's the same as the existing value, no sense in
1352 # doing anything
1353 current = node.get(propname, None)
1354 if value == current:
1355 del propvalues[propname]
1356 continue
1357 journalvalues[propname] = current
1359 # do stuff based on the prop type
1360 if isinstance(prop, Link):
1361 link_class = prop.classname
1362 # if it isn't a number, it's a key
1363 if value is not None and not isinstance(value, type('')):
1364 raise ValueError, 'property "%s" link value be a string'%(
1365 propname)
1366 if isinstance(value, type('')) and not num_re.match(value):
1367 try:
1368 value = self.db.classes[link_class].lookup(value)
1369 except (TypeError, KeyError):
1370 raise IndexError, 'new property "%s": %s not a %s'%(
1371 propname, value, prop.classname)
1373 if (value is not None and
1374 not self.db.getclass(link_class).hasnode(value)):
1375 raise IndexError, '%s has no node %s'%(link_class, value)
1377 if self.do_journal and prop.do_journal:
1378 # register the unlink with the old linked node
1379 if node[propname] is not None:
1380 self.db.addjournal(link_class, node[propname], 'unlink',
1381 (self.classname, nodeid, propname))
1383 # register the link with the newly linked node
1384 if value is not None:
1385 self.db.addjournal(link_class, value, 'link',
1386 (self.classname, nodeid, propname))
1388 elif isinstance(prop, Multilink):
1389 if type(value) != type([]):
1390 raise TypeError, 'new property "%s" not a list of'\
1391 ' ids'%propname
1392 link_class = self.properties[propname].classname
1393 l = []
1394 for entry in value:
1395 # if it isn't a number, it's a key
1396 if type(entry) != type(''):
1397 raise ValueError, 'new property "%s" link value ' \
1398 'must be a string'%propname
1399 if not num_re.match(entry):
1400 try:
1401 entry = self.db.classes[link_class].lookup(entry)
1402 except (TypeError, KeyError):
1403 raise IndexError, 'new property "%s": %s not a %s'%(
1404 propname, entry,
1405 self.properties[propname].classname)
1406 l.append(entry)
1407 value = l
1408 propvalues[propname] = value
1410 # figure the journal entry for this property
1411 add = []
1412 remove = []
1414 # handle removals
1415 if node.has_key(propname):
1416 l = node[propname]
1417 else:
1418 l = []
1419 for id in l[:]:
1420 if id in value:
1421 continue
1422 # register the unlink with the old linked node
1423 if self.do_journal and self.properties[propname].do_journal:
1424 self.db.addjournal(link_class, id, 'unlink',
1425 (self.classname, nodeid, propname))
1426 l.remove(id)
1427 remove.append(id)
1429 # handle additions
1430 for id in value:
1431 if not self.db.getclass(link_class).hasnode(id):
1432 raise IndexError, '%s has no node %s'%(link_class, id)
1433 if id in l:
1434 continue
1435 # register the link with the newly linked node
1436 if self.do_journal and self.properties[propname].do_journal:
1437 self.db.addjournal(link_class, id, 'link',
1438 (self.classname, nodeid, propname))
1439 l.append(id)
1440 add.append(id)
1442 # figure the journal entry
1443 l = []
1444 if add:
1445 l.append(('+', add))
1446 if remove:
1447 l.append(('-', remove))
1448 multilink_changes[propname] = (add, remove)
1449 if l:
1450 journalvalues[propname] = tuple(l)
1452 elif isinstance(prop, String):
1453 if value is not None and type(value) != type('') and type(value) != type(u''):
1454 raise TypeError, 'new property "%s" not a string'%propname
1456 elif isinstance(prop, Password):
1457 if not isinstance(value, password.Password):
1458 raise TypeError, 'new property "%s" not a Password'%propname
1459 propvalues[propname] = value
1461 elif value is not None and isinstance(prop, Date):
1462 if not isinstance(value, date.Date):
1463 raise TypeError, 'new property "%s" not a Date'% propname
1464 propvalues[propname] = value
1466 elif value is not None and isinstance(prop, Interval):
1467 if not isinstance(value, date.Interval):
1468 raise TypeError, 'new property "%s" not an '\
1469 'Interval'%propname
1470 propvalues[propname] = value
1472 elif value is not None and isinstance(prop, Number):
1473 try:
1474 float(value)
1475 except ValueError:
1476 raise TypeError, 'new property "%s" not numeric'%propname
1478 elif value is not None and isinstance(prop, Boolean):
1479 try:
1480 int(value)
1481 except ValueError:
1482 raise TypeError, 'new property "%s" not boolean'%propname
1484 # nothing to do?
1485 if not propvalues:
1486 return propvalues
1488 # do the set, and journal it
1489 self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
1491 if self.do_journal:
1492 self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
1494 self.fireReactors('set', nodeid, oldvalues)
1496 return propvalues
1498 def retire(self, nodeid):
1499 '''Retire a node.
1501 The properties on the node remain available from the get() method,
1502 and the node's id is never reused.
1504 Retired nodes are not returned by the find(), list(), or lookup()
1505 methods, and other nodes may reuse the values of their key properties.
1506 '''
1507 if self.db.journaltag is None:
1508 raise DatabaseError, 'Database open read-only'
1510 self.fireAuditors('retire', nodeid, None)
1512 # use the arg for __retired__ to cope with any odd database type
1513 # conversion (hello, sqlite)
1514 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1515 self.db.arg, self.db.arg)
1516 if __debug__:
1517 print >>hyperdb.DEBUG, 'retire', (self, sql, nodeid)
1518 self.db.cursor.execute(sql, (1, nodeid))
1520 self.fireReactors('retire', nodeid, None)
1522 def is_retired(self, nodeid):
1523 '''Return true if the node is rerired
1524 '''
1525 sql = 'select __retired__ from _%s where id=%s'%(self.classname,
1526 self.db.arg)
1527 if __debug__:
1528 print >>hyperdb.DEBUG, 'is_retired', (self, sql, nodeid)
1529 self.db.cursor.execute(sql, (nodeid,))
1530 return int(self.db.sql_fetchone()[0])
1532 def destroy(self, nodeid):
1533 '''Destroy a node.
1535 WARNING: this method should never be used except in extremely rare
1536 situations where there could never be links to the node being
1537 deleted
1538 WARNING: use retire() instead
1539 WARNING: the properties of this node will not be available ever again
1540 WARNING: really, use retire() instead
1542 Well, I think that's enough warnings. This method exists mostly to
1543 support the session storage of the cgi interface.
1545 The node is completely removed from the hyperdb, including all journal
1546 entries. It will no longer be available, and will generally break code
1547 if there are any references to the node.
1548 '''
1549 if self.db.journaltag is None:
1550 raise DatabaseError, 'Database open read-only'
1551 self.db.destroynode(self.classname, nodeid)
1553 def history(self, nodeid):
1554 '''Retrieve the journal of edits on a particular node.
1556 'nodeid' must be the id of an existing node of this class or an
1557 IndexError is raised.
1559 The returned list contains tuples of the form
1561 (nodeid, date, tag, action, params)
1563 'date' is a Timestamp object specifying the time of the change and
1564 'tag' is the journaltag specified when the database was opened.
1565 '''
1566 if not self.do_journal:
1567 raise ValueError, 'Journalling is disabled for this class'
1568 return self.db.getjournal(self.classname, nodeid)
1570 # Locating nodes:
1571 def hasnode(self, nodeid):
1572 '''Determine if the given nodeid actually exists
1573 '''
1574 return self.db.hasnode(self.classname, nodeid)
1576 def setkey(self, propname):
1577 '''Select a String property of this class to be the key property.
1579 'propname' must be the name of a String property of this class or
1580 None, or a TypeError is raised. The values of the key property on
1581 all existing nodes must be unique or a ValueError is raised.
1582 '''
1583 # XXX create an index on the key prop column
1584 prop = self.getprops()[propname]
1585 if not isinstance(prop, String):
1586 raise TypeError, 'key properties must be String'
1587 self.key = propname
1589 def getkey(self):
1590 '''Return the name of the key property for this class or None.'''
1591 return self.key
1593 def labelprop(self, default_to_id=0):
1594 ''' Return the property name for a label for the given node.
1596 This method attempts to generate a consistent label for the node.
1597 It tries the following in order:
1598 1. key property
1599 2. "name" property
1600 3. "title" property
1601 4. first property from the sorted property name list
1602 '''
1603 k = self.getkey()
1604 if k:
1605 return k
1606 props = self.getprops()
1607 if props.has_key('name'):
1608 return 'name'
1609 elif props.has_key('title'):
1610 return 'title'
1611 if default_to_id:
1612 return 'id'
1613 props = props.keys()
1614 props.sort()
1615 return props[0]
1617 def lookup(self, keyvalue):
1618 '''Locate a particular node by its key property and return its id.
1620 If this class has no key property, a TypeError is raised. If the
1621 'keyvalue' matches one of the values for the key property among
1622 the nodes in this class, the matching node's id is returned;
1623 otherwise a KeyError is raised.
1624 '''
1625 if not self.key:
1626 raise TypeError, 'No key property set for class %s'%self.classname
1628 # use the arg to handle any odd database type conversion (hello,
1629 # sqlite)
1630 sql = "select id from _%s where _%s=%s and __retired__ <> %s"%(
1631 self.classname, self.key, self.db.arg, self.db.arg)
1632 self.db.sql(sql, (keyvalue, 1))
1634 # see if there was a result that's not retired
1635 row = self.db.sql_fetchone()
1636 if not row:
1637 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1638 keyvalue, self.classname)
1640 # return the id
1641 return row[0]
1643 def find(self, **propspec):
1644 '''Get the ids of nodes in this class which link to the given nodes.
1646 'propspec' consists of keyword args propname=nodeid or
1647 propname={nodeid:1, }
1648 'propname' must be the name of a property in this class, or a
1649 KeyError is raised. That property must be a Link or Multilink
1650 property, or a TypeError is raised.
1652 Any node in this class whose 'propname' property links to any of the
1653 nodeids will be returned. Used by the full text indexing, which knows
1654 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1655 issues:
1657 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1658 '''
1659 if __debug__:
1660 print >>hyperdb.DEBUG, 'find', (self, propspec)
1662 # shortcut
1663 if not propspec:
1664 return []
1666 # validate the args
1667 props = self.getprops()
1668 propspec = propspec.items()
1669 for propname, nodeids in propspec:
1670 # check the prop is OK
1671 prop = props[propname]
1672 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1673 raise TypeError, "'%s' not a Link/Multilink property"%propname
1675 # first, links
1676 where = []
1677 allvalues = ()
1678 a = self.db.arg
1679 for prop, values in propspec:
1680 if not isinstance(props[prop], hyperdb.Link):
1681 continue
1682 if type(values) is type(''):
1683 allvalues += (values,)
1684 where.append('_%s = %s'%(prop, a))
1685 else:
1686 allvalues += tuple(values.keys())
1687 where.append('_%s in (%s)'%(prop, ','.join([a]*len(values))))
1688 tables = []
1689 if where:
1690 tables.append('select id as nodeid from _%s where %s'%(
1691 self.classname, ' and '.join(where)))
1693 # now multilinks
1694 for prop, values in propspec:
1695 if not isinstance(props[prop], hyperdb.Multilink):
1696 continue
1697 if type(values) is type(''):
1698 allvalues += (values,)
1699 s = a
1700 else:
1701 allvalues += tuple(values.keys())
1702 s = ','.join([a]*len(values))
1703 tables.append('select nodeid from %s_%s where linkid in (%s)'%(
1704 self.classname, prop, s))
1705 sql = '\nunion\n'.join(tables)
1706 self.db.sql(sql, allvalues)
1707 l = [x[0] for x in self.db.sql_fetchall()]
1708 if __debug__:
1709 print >>hyperdb.DEBUG, 'find ... ', l
1710 return l
1712 def stringFind(self, **requirements):
1713 '''Locate a particular node by matching a set of its String
1714 properties in a caseless search.
1716 If the property is not a String property, a TypeError is raised.
1718 The return is a list of the id of all nodes that match.
1719 '''
1720 where = []
1721 args = []
1722 for propname in requirements.keys():
1723 prop = self.properties[propname]
1724 if isinstance(not prop, String):
1725 raise TypeError, "'%s' not a String property"%propname
1726 where.append(propname)
1727 args.append(requirements[propname].lower())
1729 # generate the where clause
1730 s = ' and '.join(['_%s=%s'%(col, self.db.arg) for col in where])
1731 sql = 'select id from _%s where %s'%(self.classname, s)
1732 self.db.sql(sql, tuple(args))
1733 l = [x[0] for x in self.db.sql_fetchall()]
1734 if __debug__:
1735 print >>hyperdb.DEBUG, 'find ... ', l
1736 return l
1738 def list(self):
1739 ''' Return a list of the ids of the active nodes in this class.
1740 '''
1741 return self.db.getnodeids(self.classname, retired=0)
1743 def filter(self, search_matches, filterspec, sort=(None,None),
1744 group=(None,None)):
1745 ''' Return a list of the ids of the active nodes in this class that
1746 match the 'filter' spec, sorted by the group spec and then the
1747 sort spec
1749 "filterspec" is {propname: value(s)}
1750 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1751 and prop is a prop name or None
1752 "search_matches" is {nodeid: marker}
1754 The filter must match all properties specificed - but if the
1755 property value to match is a list, any one of the values in the
1756 list may match for that property to match.
1757 '''
1758 # just don't bother if the full-text search matched diddly
1759 if search_matches == {}:
1760 return []
1762 cn = self.classname
1764 # figure the WHERE clause from the filterspec
1765 props = self.getprops()
1766 frum = ['_'+cn]
1767 where = []
1768 args = []
1769 a = self.db.arg
1770 for k, v in filterspec.items():
1771 propclass = props[k]
1772 # now do other where clause stuff
1773 if isinstance(propclass, Multilink):
1774 tn = '%s_%s'%(cn, k)
1775 frum.append(tn)
1776 if isinstance(v, type([])):
1777 s = ','.join([a for x in v])
1778 where.append('id=%s.nodeid and %s.linkid in (%s)'%(tn,tn,s))
1779 args = args + v
1780 else:
1781 where.append('id=%s.nodeid and %s.linkid = %s'%(tn, tn, a))
1782 args.append(v)
1783 elif k == 'id':
1784 if isinstance(v, type([])):
1785 s = ','.join([a for x in v])
1786 where.append('%s in (%s)'%(k, s))
1787 args = args + v
1788 else:
1789 where.append('%s=%s'%(k, a))
1790 args.append(v)
1791 elif isinstance(propclass, String):
1792 if not isinstance(v, type([])):
1793 v = [v]
1795 # Quote the bits in the string that need it and then embed
1796 # in a "substring" search. Note - need to quote the '%' so
1797 # they make it through the python layer happily
1798 v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
1800 # now add to the where clause
1801 where.append(' or '.join(["_%s LIKE '%s'"%(k, s) for s in v]))
1802 # note: args are embedded in the query string now
1803 elif isinstance(propclass, Link):
1804 if isinstance(v, type([])):
1805 if '-1' in v:
1806 v.remove('-1')
1807 xtra = ' or _%s is NULL'%k
1808 else:
1809 xtra = ''
1810 if v:
1811 s = ','.join([a for x in v])
1812 where.append('(_%s in (%s)%s)'%(k, s, xtra))
1813 args = args + v
1814 else:
1815 where.append('_%s is NULL'%k)
1816 else:
1817 if v == '-1':
1818 v = None
1819 where.append('_%s is NULL'%k)
1820 else:
1821 where.append('_%s=%s'%(k, a))
1822 args.append(v)
1823 elif isinstance(propclass, Date):
1824 if isinstance(v, type([])):
1825 s = ','.join([a for x in v])
1826 where.append('_%s in (%s)'%(k, s))
1827 args = args + [date.Date(x).serialise() for x in v]
1828 else:
1829 where.append('_%s=%s'%(k, a))
1830 args.append(date.Date(v).serialise())
1831 elif isinstance(propclass, Interval):
1832 if isinstance(v, type([])):
1833 s = ','.join([a for x in v])
1834 where.append('_%s in (%s)'%(k, s))
1835 args = args + [date.Interval(x).serialise() for x in v]
1836 else:
1837 where.append('_%s=%s'%(k, a))
1838 args.append(date.Interval(v).serialise())
1839 else:
1840 if isinstance(v, type([])):
1841 s = ','.join([a for x in v])
1842 where.append('_%s in (%s)'%(k, s))
1843 args = args + v
1844 else:
1845 where.append('_%s=%s'%(k, a))
1846 args.append(v)
1848 # add results of full text search
1849 if search_matches is not None:
1850 v = search_matches.keys()
1851 s = ','.join([a for x in v])
1852 where.append('id in (%s)'%s)
1853 args = args + v
1855 # "grouping" is just the first-order sorting in the SQL fetch
1856 # can modify it...)
1857 orderby = []
1858 ordercols = []
1859 if group[0] is not None and group[1] is not None:
1860 if group[0] != '-':
1861 orderby.append('_'+group[1])
1862 ordercols.append('_'+group[1])
1863 else:
1864 orderby.append('_'+group[1]+' desc')
1865 ordercols.append('_'+group[1])
1867 # now add in the sorting
1868 group = ''
1869 if sort[0] is not None and sort[1] is not None:
1870 direction, colname = sort
1871 if direction != '-':
1872 if colname == 'id':
1873 orderby.append(colname)
1874 else:
1875 orderby.append('_'+colname)
1876 ordercols.append('_'+colname)
1877 else:
1878 if colname == 'id':
1879 orderby.append(colname+' desc')
1880 ordercols.append(colname)
1881 else:
1882 orderby.append('_'+colname+' desc')
1883 ordercols.append('_'+colname)
1885 # construct the SQL
1886 frum = ','.join(frum)
1887 if where:
1888 where = ' where ' + (' and '.join(where))
1889 else:
1890 where = ''
1891 cols = ['id']
1892 if orderby:
1893 cols = cols + ordercols
1894 order = ' order by %s'%(','.join(orderby))
1895 else:
1896 order = ''
1897 cols = ','.join(cols)
1898 sql = 'select %s from %s %s%s%s'%(cols, frum, where, group, order)
1899 args = tuple(args)
1900 if __debug__:
1901 print >>hyperdb.DEBUG, 'filter', (self, sql, args)
1902 self.db.cursor.execute(sql, args)
1903 l = self.db.cursor.fetchall()
1905 # return the IDs (the first column)
1906 # XXX The filter(None, l) bit is sqlite-specific... if there's _NO_
1907 # XXX matches to a fetch, it returns NULL instead of nothing!?!
1908 return filter(None, [row[0] for row in l])
1910 def count(self):
1911 '''Get the number of nodes in this class.
1913 If the returned integer is 'numnodes', the ids of all the nodes
1914 in this class run from 1 to numnodes, and numnodes+1 will be the
1915 id of the next node to be created in this class.
1916 '''
1917 return self.db.countnodes(self.classname)
1919 # Manipulating properties:
1920 def getprops(self, protected=1):
1921 '''Return a dictionary mapping property names to property objects.
1922 If the "protected" flag is true, we include protected properties -
1923 those which may not be modified.
1924 '''
1925 d = self.properties.copy()
1926 if protected:
1927 d['id'] = String()
1928 d['creation'] = hyperdb.Date()
1929 d['activity'] = hyperdb.Date()
1930 d['creator'] = hyperdb.Link('user')
1931 return d
1933 def addprop(self, **properties):
1934 '''Add properties to this class.
1936 The keyword arguments in 'properties' must map names to property
1937 objects, or a TypeError is raised. None of the keys in 'properties'
1938 may collide with the names of existing properties, or a ValueError
1939 is raised before any properties have been added.
1940 '''
1941 for key in properties.keys():
1942 if self.properties.has_key(key):
1943 raise ValueError, key
1944 self.properties.update(properties)
1946 def index(self, nodeid):
1947 '''Add (or refresh) the node to search indexes
1948 '''
1949 # find all the String properties that have indexme
1950 for prop, propclass in self.getprops().items():
1951 if isinstance(propclass, String) and propclass.indexme:
1952 try:
1953 value = str(self.get(nodeid, prop))
1954 except IndexError:
1955 # node no longer exists - entry should be removed
1956 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1957 else:
1958 # and index them under (classname, nodeid, property)
1959 self.db.indexer.add_text((self.classname, nodeid, prop),
1960 value)
1963 #
1964 # Detector interface
1965 #
1966 def audit(self, event, detector):
1967 '''Register a detector
1968 '''
1969 l = self.auditors[event]
1970 if detector not in l:
1971 self.auditors[event].append(detector)
1973 def fireAuditors(self, action, nodeid, newvalues):
1974 '''Fire all registered auditors.
1975 '''
1976 for audit in self.auditors[action]:
1977 audit(self.db, self, nodeid, newvalues)
1979 def react(self, event, detector):
1980 '''Register a detector
1981 '''
1982 l = self.reactors[event]
1983 if detector not in l:
1984 self.reactors[event].append(detector)
1986 def fireReactors(self, action, nodeid, oldvalues):
1987 '''Fire all registered reactors.
1988 '''
1989 for react in self.reactors[action]:
1990 react(self.db, self, nodeid, oldvalues)
1992 class FileClass(Class, hyperdb.FileClass):
1993 '''This class defines a large chunk of data. To support this, it has a
1994 mandatory String property "content" which is typically saved off
1995 externally to the hyperdb.
1997 The default MIME type of this data is defined by the
1998 "default_mime_type" class attribute, which may be overridden by each
1999 node if the class defines a "type" String property.
2000 '''
2001 default_mime_type = 'text/plain'
2003 def create(self, **propvalues):
2004 ''' snaffle the file propvalue and store in a file
2005 '''
2006 # we need to fire the auditors now, or the content property won't
2007 # be in propvalues for the auditors to play with
2008 self.fireAuditors('create', None, propvalues)
2010 # now remove the content property so it's not stored in the db
2011 content = propvalues['content']
2012 del propvalues['content']
2014 # do the database create
2015 newid = Class.create_inner(self, **propvalues)
2017 # fire reactors
2018 self.fireReactors('create', newid, None)
2020 # store off the content as a file
2021 self.db.storefile(self.classname, newid, None, content)
2022 return newid
2024 def import_list(self, propnames, proplist):
2025 ''' Trap the "content" property...
2026 '''
2027 # dupe this list so we don't affect others
2028 propnames = propnames[:]
2030 # extract the "content" property from the proplist
2031 i = propnames.index('content')
2032 content = eval(proplist[i])
2033 del propnames[i]
2034 del proplist[i]
2036 # do the normal import
2037 newid = Class.import_list(self, propnames, proplist)
2039 # save off the "content" file
2040 self.db.storefile(self.classname, newid, None, content)
2041 return newid
2043 _marker = []
2044 def get(self, nodeid, propname, default=_marker, cache=1):
2045 ''' trap the content propname and get it from the file
2046 '''
2047 poss_msg = 'Possibly a access right configuration problem.'
2048 if propname == 'content':
2049 try:
2050 return self.db.getfile(self.classname, nodeid, None)
2051 except IOError, (strerror):
2052 # BUG: by catching this we donot see an error in the log.
2053 return 'ERROR reading file: %s%s\n%s\n%s'%(
2054 self.classname, nodeid, poss_msg, strerror)
2055 if default is not self._marker:
2056 return Class.get(self, nodeid, propname, default, cache=cache)
2057 else:
2058 return Class.get(self, nodeid, propname, cache=cache)
2060 def getprops(self, protected=1):
2061 ''' In addition to the actual properties on the node, these methods
2062 provide the "content" property. If the "protected" flag is true,
2063 we include protected properties - those which may not be
2064 modified.
2065 '''
2066 d = Class.getprops(self, protected=protected).copy()
2067 d['content'] = hyperdb.String()
2068 return d
2070 def index(self, nodeid):
2071 ''' Index the node in the search index.
2073 We want to index the content in addition to the normal String
2074 property indexing.
2075 '''
2076 # perform normal indexing
2077 Class.index(self, nodeid)
2079 # get the content to index
2080 content = self.get(nodeid, 'content')
2082 # figure the mime type
2083 if self.properties.has_key('type'):
2084 mime_type = self.get(nodeid, 'type')
2085 else:
2086 mime_type = self.default_mime_type
2088 # and index!
2089 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
2090 mime_type)
2092 # XXX deviation from spec - was called ItemClass
2093 class IssueClass(Class, roundupdb.IssueClass):
2094 # Overridden methods:
2095 def __init__(self, db, classname, **properties):
2096 '''The newly-created class automatically includes the "messages",
2097 "files", "nosy", and "superseder" properties. If the 'properties'
2098 dictionary attempts to specify any of these properties or a
2099 "creation" or "activity" property, a ValueError is raised.
2100 '''
2101 if not properties.has_key('title'):
2102 properties['title'] = hyperdb.String(indexme='yes')
2103 if not properties.has_key('messages'):
2104 properties['messages'] = hyperdb.Multilink("msg")
2105 if not properties.has_key('files'):
2106 properties['files'] = hyperdb.Multilink("file")
2107 if not properties.has_key('nosy'):
2108 # note: journalling is turned off as it really just wastes
2109 # space. this behaviour may be overridden in an instance
2110 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2111 if not properties.has_key('superseder'):
2112 properties['superseder'] = hyperdb.Multilink(classname)
2113 Class.__init__(self, db, classname, **properties)