ced7ccaf31b56211202d91c24395e6806799052e
1 # $Id: rdbms_common.py,v 1.20 2002-10-03 06:56:29 richard Exp $
2 ''' Relational database (SQL) backend common code.
4 Basics:
6 - map roundup classes to relational tables
7 - automatically detect schema changes and modify the table schemas
8 appropriately (we store the "database version" of the schema in the
9 database itself as the only row of the "schema" table)
10 - multilinks (which represent a many-to-many relationship) are handled through
11 intermediate tables
12 - journals are stored adjunct to the per-class tables
13 - table names and columns have "_" prepended so the names can't clash with
14 restricted names (like "order")
15 - retirement is determined by the __retired__ column being true
17 Database-specific changes may generally be pushed out to the overridable
18 sql_* methods, since everything else should be fairly generic. There's
19 probably a bit of work to be done if a database is used that actually
20 honors column typing, since the initial databases don't (sqlite stores
21 everything as a string, and gadfly stores anything that's marsallable).
22 '''
24 # standard python modules
25 import sys, os, time, re, errno, weakref, copy
27 # roundup modules
28 from roundup import hyperdb, date, password, roundupdb, security
29 from roundup.hyperdb import String, Password, Date, Interval, Link, \
30 Multilink, DatabaseError, Boolean, Number
32 # support
33 from blobfiles import FileStorage
34 from roundup.indexer import Indexer
35 from sessions import Sessions
37 # number of rows to keep in memory
38 ROW_CACHE_SIZE = 100
40 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
41 ''' Wrapper around an SQL database that presents a hyperdb interface.
43 - some functionality is specific to the actual SQL database, hence
44 the sql_* methods that are NotImplemented
45 - we keep a cache of the latest ROW_CACHE_SIZE row fetches.
46 '''
47 def __init__(self, config, journaltag=None):
48 ''' Open the database and load the schema from it.
49 '''
50 self.config, self.journaltag = config, journaltag
51 self.dir = config.DATABASE
52 self.classes = {}
53 self.indexer = Indexer(self.dir)
54 self.sessions = Sessions(self.config)
55 self.security = security.Security(self)
57 # additional transaction support for external files and the like
58 self.transactions = []
60 # keep a cache of the N most recently retrieved rows of any kind
61 # (classname, nodeid) = row
62 self.cache = {}
63 self.cache_lru = []
65 # open a connection to the database, creating the "conn" attribute
66 self.open_connection()
68 def clearCache(self):
69 self.cache = {}
70 self.cache_lru = []
72 def open_connection(self):
73 ''' Open a connection to the database, creating it if necessary
74 '''
75 raise NotImplemented
77 def sql(self, sql, args=None):
78 ''' Execute the sql with the optional args.
79 '''
80 if __debug__:
81 print >>hyperdb.DEBUG, (self, sql, args)
82 if args:
83 self.cursor.execute(sql, args)
84 else:
85 self.cursor.execute(sql)
87 def sql_fetchone(self):
88 ''' Fetch a single row. If there's nothing to fetch, return None.
89 '''
90 raise NotImplemented
92 def sql_stringquote(self, value):
93 ''' Quote the string so it's safe to put in the 'sql quotes'
94 '''
95 return re.sub("'", "''", str(value))
97 def save_dbschema(self, schema):
98 ''' Save the schema definition that the database currently implements
99 '''
100 raise NotImplemented
102 def load_dbschema(self):
103 ''' Load the schema definition that the database currently implements
104 '''
105 raise NotImplemented
107 def post_init(self):
108 ''' Called once the schema initialisation has finished.
110 We should now confirm that the schema defined by our "classes"
111 attribute actually matches the schema in the database.
112 '''
113 # now detect changes in the schema
114 save = 0
115 for classname, spec in self.classes.items():
116 if self.database_schema.has_key(classname):
117 dbspec = self.database_schema[classname]
118 if self.update_class(spec, dbspec):
119 self.database_schema[classname] = spec.schema()
120 save = 1
121 else:
122 self.create_class(spec)
123 self.database_schema[classname] = spec.schema()
124 save = 1
126 for classname in self.database_schema.keys():
127 if not self.classes.has_key(classname):
128 self.drop_class(classname)
130 # update the database version of the schema
131 if save:
132 self.sql('delete from schema')
133 self.save_dbschema(self.database_schema)
135 # reindex the db if necessary
136 if self.indexer.should_reindex():
137 self.reindex()
139 # commit
140 self.conn.commit()
142 # figure the "curuserid"
143 if self.journaltag is None:
144 self.curuserid = None
145 elif self.journaltag == 'admin':
146 # admin user may not exist, but always has ID 1
147 self.curuserid = '1'
148 else:
149 self.curuserid = self.user.lookup(self.journaltag)
151 def reindex(self):
152 for klass in self.classes.values():
153 for nodeid in klass.list():
154 klass.index(nodeid)
155 self.indexer.save_index()
157 def determine_columns(self, properties):
158 ''' Figure the column names and multilink properties from the spec
160 "properties" is a list of (name, prop) where prop may be an
161 instance of a hyperdb "type" _or_ a string repr of that type.
162 '''
163 cols = ['_activity', '_creator', '_creation']
164 mls = []
165 # add the multilinks separately
166 for col, prop in properties:
167 if isinstance(prop, Multilink):
168 mls.append(col)
169 elif isinstance(prop, type('')) and prop.find('Multilink') != -1:
170 mls.append(col)
171 else:
172 cols.append('_'+col)
173 cols.sort()
174 return cols, mls
176 def update_class(self, spec, dbspec):
177 ''' Determine the differences between the current spec and the
178 database version of the spec, and update where necessary
179 '''
180 spec_schema = spec.schema()
181 if spec_schema == dbspec:
182 # no save needed for this one
183 return 0
184 if __debug__:
185 print >>hyperdb.DEBUG, 'update_class FIRING'
187 # key property changed?
188 if dbspec[0] != spec_schema[0]:
189 if __debug__:
190 print >>hyperdb.DEBUG, 'update_class setting keyprop', `spec[0]`
191 # XXX turn on indexing for the key property
193 # dict 'em up
194 spec_propnames,spec_props = [],{}
195 for propname,prop in spec_schema[1]:
196 spec_propnames.append(propname)
197 spec_props[propname] = prop
198 dbspec_propnames,dbspec_props = [],{}
199 for propname,prop in dbspec[1]:
200 dbspec_propnames.append(propname)
201 dbspec_props[propname] = prop
203 # now compare
204 for propname in spec_propnames:
205 prop = spec_props[propname]
206 if dbspec_props.has_key(propname) and prop==dbspec_props[propname]:
207 continue
208 if __debug__:
209 print >>hyperdb.DEBUG, 'update_class ADD', (propname, prop)
211 if not dbspec_props.has_key(propname):
212 # add the property
213 if isinstance(prop, Multilink):
214 # all we have to do here is create a new table, easy!
215 self.create_multilink_table(spec, propname)
216 continue
218 # no ALTER TABLE, so we:
219 # 1. pull out the data, including an extra None column
220 oldcols, x = self.determine_columns(dbspec[1])
221 oldcols.append('id')
222 oldcols.append('__retired__')
223 cn = spec.classname
224 sql = 'select %s,%s from _%s'%(','.join(oldcols), self.arg, cn)
225 if __debug__:
226 print >>hyperdb.DEBUG, 'update_class', (self, sql, None)
227 self.cursor.execute(sql, (None,))
228 olddata = self.cursor.fetchall()
230 # 2. drop the old table
231 self.cursor.execute('drop table _%s'%cn)
233 # 3. create the new table
234 cols, mls = self.create_class_table(spec)
235 # ensure the new column is last
236 cols.remove('_'+propname)
237 assert oldcols == cols, "Column lists don't match!"
238 cols.append('_'+propname)
240 # 4. populate with the data from step one
241 s = ','.join([self.arg for x in cols])
242 scols = ','.join(cols)
243 sql = 'insert into _%s (%s) values (%s)'%(cn, scols, s)
245 # GAH, nothing had better go wrong from here on in... but
246 # we have to commit the drop...
247 # XXX this isn't necessary in sqlite :(
248 self.conn.commit()
250 # do the insert
251 for row in olddata:
252 self.sql(sql, tuple(row))
254 else:
255 # modify the property
256 if __debug__:
257 print >>hyperdb.DEBUG, 'update_class NOOP'
258 pass # NOOP in gadfly
260 # and the other way - only worry about deletions here
261 for propname in dbspec_propnames:
262 prop = dbspec_props[propname]
263 if spec_props.has_key(propname):
264 continue
265 if __debug__:
266 print >>hyperdb.DEBUG, 'update_class REMOVE', `prop`
268 # delete the property
269 if isinstance(prop, Multilink):
270 sql = 'drop table %s_%s'%(spec.classname, prop)
271 if __debug__:
272 print >>hyperdb.DEBUG, 'update_class', (self, sql)
273 self.cursor.execute(sql)
274 else:
275 # no ALTER TABLE, so we:
276 # 1. pull out the data, excluding the removed column
277 oldcols, x = self.determine_columns(spec.properties.items())
278 oldcols.append('id')
279 oldcols.append('__retired__')
280 # remove the missing column
281 oldcols.remove('_'+propname)
282 cn = spec.classname
283 sql = 'select %s from _%s'%(','.join(oldcols), cn)
284 self.cursor.execute(sql, (None,))
285 olddata = sql.fetchall()
287 # 2. drop the old table
288 self.cursor.execute('drop table _%s'%cn)
290 # 3. create the new table
291 cols, mls = self.create_class_table(self, spec)
292 assert oldcols != cols, "Column lists don't match!"
294 # 4. populate with the data from step one
295 qs = ','.join([self.arg for x in cols])
296 sql = 'insert into _%s values (%s)'%(cn, s)
297 self.cursor.execute(sql, olddata)
298 return 1
300 def create_class_table(self, spec):
301 ''' create the class table for the given spec
302 '''
303 cols, mls = self.determine_columns(spec.properties.items())
305 # add on our special columns
306 cols.append('id')
307 cols.append('__retired__')
309 # create the base table
310 scols = ','.join(['%s varchar'%x for x in cols])
311 sql = 'create table _%s (%s)'%(spec.classname, scols)
312 if __debug__:
313 print >>hyperdb.DEBUG, 'create_class', (self, sql)
314 self.cursor.execute(sql)
316 return cols, mls
318 def create_journal_table(self, spec):
319 ''' create the journal table for a class given the spec and
320 already-determined cols
321 '''
322 # journal table
323 cols = ','.join(['%s varchar'%x
324 for x in 'nodeid date tag action params'.split()])
325 sql = 'create table %s__journal (%s)'%(spec.classname, cols)
326 if __debug__:
327 print >>hyperdb.DEBUG, 'create_class', (self, sql)
328 self.cursor.execute(sql)
330 def create_multilink_table(self, spec, ml):
331 ''' Create a multilink table for the "ml" property of the class
332 given by the spec
333 '''
334 sql = 'create table %s_%s (linkid varchar, nodeid varchar)'%(
335 spec.classname, ml)
336 if __debug__:
337 print >>hyperdb.DEBUG, 'create_class', (self, sql)
338 self.cursor.execute(sql)
340 def create_class(self, spec):
341 ''' Create a database table according to the given spec.
342 '''
343 cols, mls = self.create_class_table(spec)
344 self.create_journal_table(spec)
346 # now create the multilink tables
347 for ml in mls:
348 self.create_multilink_table(spec, ml)
350 # ID counter
351 sql = 'insert into ids (name, num) values (%s,%s)'%(self.arg, self.arg)
352 vals = (spec.classname, 1)
353 if __debug__:
354 print >>hyperdb.DEBUG, 'create_class', (self, sql, vals)
355 self.cursor.execute(sql, vals)
357 def drop_class(self, spec):
358 ''' Drop the given table from the database.
360 Drop the journal and multilink tables too.
361 '''
362 # figure the multilinks
363 mls = []
364 for col, prop in spec.properties.items():
365 if isinstance(prop, Multilink):
366 mls.append(col)
368 sql = 'drop table _%s'%spec.classname
369 if __debug__:
370 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
371 self.cursor.execute(sql)
373 sql = 'drop table %s__journal'%spec.classname
374 if __debug__:
375 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
376 self.cursor.execute(sql)
378 for ml in mls:
379 sql = 'drop table %s_%s'%(spec.classname, ml)
380 if __debug__:
381 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
382 self.cursor.execute(sql)
384 #
385 # Classes
386 #
387 def __getattr__(self, classname):
388 ''' A convenient way of calling self.getclass(classname).
389 '''
390 if self.classes.has_key(classname):
391 if __debug__:
392 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
393 return self.classes[classname]
394 raise AttributeError, classname
396 def addclass(self, cl):
397 ''' Add a Class to the hyperdatabase.
398 '''
399 if __debug__:
400 print >>hyperdb.DEBUG, 'addclass', (self, cl)
401 cn = cl.classname
402 if self.classes.has_key(cn):
403 raise ValueError, cn
404 self.classes[cn] = cl
406 def getclasses(self):
407 ''' Return a list of the names of all existing classes.
408 '''
409 if __debug__:
410 print >>hyperdb.DEBUG, 'getclasses', (self,)
411 l = self.classes.keys()
412 l.sort()
413 return l
415 def getclass(self, classname):
416 '''Get the Class object representing a particular class.
418 If 'classname' is not a valid class name, a KeyError is raised.
419 '''
420 if __debug__:
421 print >>hyperdb.DEBUG, 'getclass', (self, classname)
422 try:
423 return self.classes[classname]
424 except KeyError:
425 raise KeyError, 'There is no class called "%s"'%classname
427 def clear(self):
428 ''' Delete all database contents.
430 Note: I don't commit here, which is different behaviour to the
431 "nuke from orbit" behaviour in the *dbms.
432 '''
433 if __debug__:
434 print >>hyperdb.DEBUG, 'clear', (self,)
435 for cn in self.classes.keys():
436 sql = 'delete from _%s'%cn
437 if __debug__:
438 print >>hyperdb.DEBUG, 'clear', (self, sql)
439 self.cursor.execute(sql)
441 #
442 # Node IDs
443 #
444 def newid(self, classname):
445 ''' Generate a new id for the given class
446 '''
447 # get the next ID
448 sql = 'select num from ids where name=%s'%self.arg
449 if __debug__:
450 print >>hyperdb.DEBUG, 'newid', (self, sql, classname)
451 self.cursor.execute(sql, (classname, ))
452 newid = self.cursor.fetchone()[0]
454 # update the counter
455 sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
456 vals = (int(newid)+1, classname)
457 if __debug__:
458 print >>hyperdb.DEBUG, 'newid', (self, sql, vals)
459 self.cursor.execute(sql, vals)
461 # return as string
462 return str(newid)
464 def setid(self, classname, setid):
465 ''' Set the id counter: used during import of database
466 '''
467 sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
468 vals = (setid, classname)
469 if __debug__:
470 print >>hyperdb.DEBUG, 'setid', (self, sql, vals)
471 self.cursor.execute(sql, vals)
473 #
474 # Nodes
475 #
477 def addnode(self, classname, nodeid, node):
478 ''' Add the specified node to its class's db.
479 '''
480 if __debug__:
481 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
482 # gadfly requires values for all non-multilink columns
483 cl = self.classes[classname]
484 cols, mls = self.determine_columns(cl.properties.items())
486 # we'll be supplied these props if we're doing an import
487 if not node.has_key('creator'):
488 # add in the "calculated" properties (dupe so we don't affect
489 # calling code's node assumptions)
490 node = node.copy()
491 node['creation'] = node['activity'] = date.Date()
492 node['creator'] = self.curuserid
494 # default the non-multilink columns
495 for col, prop in cl.properties.items():
496 if not isinstance(col, Multilink):
497 if not node.has_key(col):
498 node[col] = None
500 # clear this node out of the cache if it's in there
501 key = (classname, nodeid)
502 if self.cache.has_key(key):
503 del self.cache[key]
504 self.cache_lru.remove(key)
506 # make the node data safe for the DB
507 node = self.serialise(classname, node)
509 # make sure the ordering is correct for column name -> column value
510 vals = tuple([node[col[1:]] for col in cols]) + (nodeid, 0)
511 s = ','.join([self.arg for x in cols]) + ',%s,%s'%(self.arg, self.arg)
512 cols = ','.join(cols) + ',id,__retired__'
514 # perform the inserts
515 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
516 if __debug__:
517 print >>hyperdb.DEBUG, 'addnode', (self, sql, vals)
518 self.cursor.execute(sql, vals)
520 # insert the multilink rows
521 for col in mls:
522 t = '%s_%s'%(classname, col)
523 for entry in node[col]:
524 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
525 self.arg, self.arg)
526 self.sql(sql, (entry, nodeid))
528 # make sure we do the commit-time extra stuff for this node
529 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
531 def setnode(self, classname, nodeid, values, multilink_changes):
532 ''' Change the specified node.
533 '''
534 if __debug__:
535 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, values)
537 # clear this node out of the cache if it's in there
538 key = (classname, nodeid)
539 if self.cache.has_key(key):
540 del self.cache[key]
541 self.cache_lru.remove(key)
543 # add the special props
544 values = values.copy()
545 values['activity'] = date.Date()
547 # make db-friendly
548 values = self.serialise(classname, values)
550 cl = self.classes[classname]
551 cols = []
552 mls = []
553 # add the multilinks separately
554 props = cl.getprops()
555 for col in values.keys():
556 prop = props[col]
557 if isinstance(prop, Multilink):
558 mls.append(col)
559 else:
560 cols.append('_'+col)
561 cols.sort()
563 # if there's any updates to regular columns, do them
564 if cols:
565 # make sure the ordering is correct for column name -> column value
566 sqlvals = tuple([values[col[1:]] for col in cols]) + (nodeid,)
567 s = ','.join(['%s=%s'%(x, self.arg) for x in cols])
568 cols = ','.join(cols)
570 # perform the update
571 sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
572 if __debug__:
573 print >>hyperdb.DEBUG, 'setnode', (self, sql, sqlvals)
574 self.cursor.execute(sql, sqlvals)
576 # now the fun bit, updating the multilinks ;)
577 for col, (add, remove) in multilink_changes.items():
578 tn = '%s_%s'%(classname, col)
579 if add:
580 sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
581 self.arg, self.arg)
582 for addid in add:
583 self.sql(sql, (nodeid, addid))
584 if remove:
585 sql = 'delete from %s where nodeid=%s and linkid=%s'%(tn,
586 self.arg, self.arg)
587 for removeid in remove:
588 self.sql(sql, (nodeid, removeid))
590 # make sure we do the commit-time extra stuff for this node
591 self.transactions.append((self.doSaveNode, (classname, nodeid, values)))
593 def getnode(self, classname, nodeid):
594 ''' Get a node from the database.
595 '''
596 if __debug__:
597 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid)
599 # see if we have this node cached
600 key = (classname, nodeid)
601 if self.cache.has_key(key):
602 # push us back to the top of the LRU
603 self.cache_lru.remove(key)
604 self.cache_lru.insert(0, key)
605 # return the cached information
606 return self.cache[key]
608 # figure the columns we're fetching
609 cl = self.classes[classname]
610 cols, mls = self.determine_columns(cl.properties.items())
611 scols = ','.join(cols)
613 # perform the basic property fetch
614 sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
615 self.sql(sql, (nodeid,))
617 values = self.sql_fetchone()
618 if values is None:
619 raise IndexError, 'no such %s node %s'%(classname, nodeid)
621 # make up the node
622 node = {}
623 for col in range(len(cols)):
624 node[cols[col][1:]] = values[col]
626 # now the multilinks
627 for col in mls:
628 # get the link ids
629 sql = 'select linkid from %s_%s where nodeid=%s'%(classname, col,
630 self.arg)
631 self.cursor.execute(sql, (nodeid,))
632 # extract the first column from the result
633 node[col] = [x[0] for x in self.cursor.fetchall()]
635 # un-dbificate the node data
636 node = self.unserialise(classname, node)
638 # save off in the cache
639 key = (classname, nodeid)
640 self.cache[key] = node
641 # update the LRU
642 self.cache_lru.insert(0, key)
643 if len(self.cache_lru) > ROW_CACHE_SIZE:
644 del self.cache[self.cache_lru.pop()]
646 return node
648 def destroynode(self, classname, nodeid):
649 '''Remove a node from the database. Called exclusively by the
650 destroy() method on Class.
651 '''
652 if __debug__:
653 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
655 # make sure the node exists
656 if not self.hasnode(classname, nodeid):
657 raise IndexError, '%s has no node %s'%(classname, nodeid)
659 # see if we have this node cached
660 if self.cache.has_key((classname, nodeid)):
661 del self.cache[(classname, nodeid)]
663 # see if there's any obvious commit actions that we should get rid of
664 for entry in self.transactions[:]:
665 if entry[1][:2] == (classname, nodeid):
666 self.transactions.remove(entry)
668 # now do the SQL
669 sql = 'delete from _%s where id=%s'%(classname, self.arg)
670 self.sql(sql, (nodeid,))
672 # remove from multilnks
673 cl = self.getclass(classname)
674 x, mls = self.determine_columns(cl.properties.items())
675 for col in mls:
676 # get the link ids
677 sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
678 self.cursor.execute(sql, (nodeid,))
680 # remove journal entries
681 sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
682 self.sql(sql, (nodeid,))
684 def serialise(self, classname, node):
685 '''Copy the node contents, converting non-marshallable data into
686 marshallable data.
687 '''
688 if __debug__:
689 print >>hyperdb.DEBUG, 'serialise', classname, node
690 properties = self.getclass(classname).getprops()
691 d = {}
692 for k, v in node.items():
693 # if the property doesn't exist, or is the "retired" flag then
694 # it won't be in the properties dict
695 if not properties.has_key(k):
696 d[k] = v
697 continue
699 # get the property spec
700 prop = properties[k]
702 if isinstance(prop, Password):
703 d[k] = str(v)
704 elif isinstance(prop, Date) and v is not None:
705 d[k] = v.serialise()
706 elif isinstance(prop, Interval) and v is not None:
707 d[k] = v.serialise()
708 else:
709 d[k] = v
710 return d
712 def unserialise(self, classname, node):
713 '''Decode the marshalled node data
714 '''
715 if __debug__:
716 print >>hyperdb.DEBUG, 'unserialise', classname, node
717 properties = self.getclass(classname).getprops()
718 d = {}
719 for k, v in node.items():
720 # if the property doesn't exist, or is the "retired" flag then
721 # it won't be in the properties dict
722 if not properties.has_key(k):
723 d[k] = v
724 continue
726 # get the property spec
727 prop = properties[k]
729 if isinstance(prop, Date) and v is not None:
730 d[k] = date.Date(v)
731 elif isinstance(prop, Interval) and v is not None:
732 d[k] = date.Interval(v)
733 elif isinstance(prop, Password):
734 p = password.Password()
735 p.unpack(v)
736 d[k] = p
737 else:
738 d[k] = v
739 return d
741 def hasnode(self, classname, nodeid):
742 ''' Determine if the database has a given node.
743 '''
744 sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
745 if __debug__:
746 print >>hyperdb.DEBUG, 'hasnode', (self, sql, nodeid)
747 self.cursor.execute(sql, (nodeid,))
748 return int(self.cursor.fetchone()[0])
750 def countnodes(self, classname):
751 ''' Count the number of nodes that exist for a particular Class.
752 '''
753 sql = 'select count(*) from _%s'%classname
754 if __debug__:
755 print >>hyperdb.DEBUG, 'countnodes', (self, sql)
756 self.cursor.execute(sql)
757 return self.cursor.fetchone()[0]
759 def getnodeids(self, classname, retired=0):
760 ''' Retrieve all the ids of the nodes for a particular Class.
762 Set retired=None to get all nodes. Otherwise it'll get all the
763 retired or non-retired nodes, depending on the flag.
764 '''
765 # flip the sense of the flag if we don't want all of them
766 if retired is not None:
767 retired = not retired
768 sql = 'select id from _%s where __retired__ <> %s'%(classname, self.arg)
769 if __debug__:
770 print >>hyperdb.DEBUG, 'getnodeids', (self, sql, retired)
771 self.cursor.execute(sql, (retired,))
772 return [x[0] for x in self.cursor.fetchall()]
774 def addjournal(self, classname, nodeid, action, params, creator=None,
775 creation=None):
776 ''' Journal the Action
777 'action' may be:
779 'create' or 'set' -- 'params' is a dictionary of property values
780 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
781 'retire' -- 'params' is None
782 '''
783 # serialise the parameters now if necessary
784 if isinstance(params, type({})):
785 if action in ('set', 'create'):
786 params = self.serialise(classname, params)
788 # handle supply of the special journalling parameters (usually
789 # supplied on importing an existing database)
790 if creator:
791 journaltag = creator
792 else:
793 journaltag = self.curuserid
794 if creation:
795 journaldate = creation.serialise()
796 else:
797 journaldate = date.Date().serialise()
799 # create the journal entry
800 cols = ','.join('nodeid date tag action params'.split())
802 if __debug__:
803 print >>hyperdb.DEBUG, 'addjournal', (nodeid, journaldate,
804 journaltag, action, params)
806 self.save_journal(classname, cols, nodeid, journaldate,
807 journaltag, action, params)
809 def save_journal(self, classname, cols, nodeid, journaldate,
810 journaltag, action, params):
811 ''' Save the journal entry to the database
812 '''
813 raise NotImplemented
815 def getjournal(self, classname, nodeid):
816 ''' get the journal for id
817 '''
818 # make sure the node exists
819 if not self.hasnode(classname, nodeid):
820 raise IndexError, '%s has no node %s'%(classname, nodeid)
822 cols = ','.join('nodeid date tag action params'.split())
823 return self.load_journal(classname, cols, nodeid)
825 def load_journal(self, classname, cols, nodeid):
826 ''' Load the journal from the database
827 '''
828 raise NotImplemented
830 def pack(self, pack_before):
831 ''' Delete all journal entries except "create" before 'pack_before'.
832 '''
833 # get a 'yyyymmddhhmmss' version of the date
834 date_stamp = pack_before.serialise()
836 # do the delete
837 for classname in self.classes.keys():
838 sql = "delete from %s__journal where date<%s and "\
839 "action<>'create'"%(classname, self.arg)
840 if __debug__:
841 print >>hyperdb.DEBUG, 'pack', (self, sql, date_stamp)
842 self.cursor.execute(sql, (date_stamp,))
844 def sql_commit(self):
845 ''' Actually commit to the database.
846 '''
847 self.conn.commit()
849 def commit(self):
850 ''' Commit the current transactions.
852 Save all data changed since the database was opened or since the
853 last commit() or rollback().
854 '''
855 if __debug__:
856 print >>hyperdb.DEBUG, 'commit', (self,)
858 # commit the database
859 self.sql_commit()
861 # now, do all the other transaction stuff
862 reindex = {}
863 for method, args in self.transactions:
864 reindex[method(*args)] = 1
866 # reindex the nodes that request it
867 for classname, nodeid in filter(None, reindex.keys()):
868 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
869 self.getclass(classname).index(nodeid)
871 # save the indexer state
872 self.indexer.save_index()
874 # clear out the transactions
875 self.transactions = []
877 def rollback(self):
878 ''' Reverse all actions from the current transaction.
880 Undo all the changes made since the database was opened or the last
881 commit() or rollback() was performed.
882 '''
883 if __debug__:
884 print >>hyperdb.DEBUG, 'rollback', (self,)
886 # roll back
887 self.conn.rollback()
889 # roll back "other" transaction stuff
890 for method, args in self.transactions:
891 # delete temporary files
892 if method == self.doStoreFile:
893 self.rollbackStoreFile(*args)
894 self.transactions = []
896 def doSaveNode(self, classname, nodeid, node):
897 ''' dummy that just generates a reindex event
898 '''
899 # return the classname, nodeid so we reindex this content
900 return (classname, nodeid)
902 def close(self):
903 ''' Close off the connection.
904 '''
905 self.conn.close()
907 #
908 # The base Class class
909 #
910 class Class(hyperdb.Class):
911 ''' The handle to a particular class of nodes in a hyperdatabase.
913 All methods except __repr__ and getnode must be implemented by a
914 concrete backend Class.
915 '''
917 def __init__(self, db, classname, **properties):
918 '''Create a new class with a given name and property specification.
920 'classname' must not collide with the name of an existing class,
921 or a ValueError is raised. The keyword arguments in 'properties'
922 must map names to property objects, or a TypeError is raised.
923 '''
924 if (properties.has_key('creation') or properties.has_key('activity')
925 or properties.has_key('creator')):
926 raise ValueError, '"creation", "activity" and "creator" are '\
927 'reserved'
929 self.classname = classname
930 self.properties = properties
931 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
932 self.key = ''
934 # should we journal changes (default yes)
935 self.do_journal = 1
937 # do the db-related init stuff
938 db.addclass(self)
940 self.auditors = {'create': [], 'set': [], 'retire': []}
941 self.reactors = {'create': [], 'set': [], 'retire': []}
943 def schema(self):
944 ''' A dumpable version of the schema that we can store in the
945 database
946 '''
947 return (self.key, [(x, repr(y)) for x,y in self.properties.items()])
949 def enableJournalling(self):
950 '''Turn journalling on for this class
951 '''
952 self.do_journal = 1
954 def disableJournalling(self):
955 '''Turn journalling off for this class
956 '''
957 self.do_journal = 0
959 # Editing nodes:
960 def create(self, **propvalues):
961 ''' Create a new node of this class and return its id.
963 The keyword arguments in 'propvalues' map property names to values.
965 The values of arguments must be acceptable for the types of their
966 corresponding properties or a TypeError is raised.
968 If this class has a key property, it must be present and its value
969 must not collide with other key strings or a ValueError is raised.
971 Any other properties on this class that are missing from the
972 'propvalues' dictionary are set to None.
974 If an id in a link or multilink property does not refer to a valid
975 node, an IndexError is raised.
976 '''
977 if propvalues.has_key('id'):
978 raise KeyError, '"id" is reserved'
980 if self.db.journaltag is None:
981 raise DatabaseError, 'Database open read-only'
983 if propvalues.has_key('creation') or propvalues.has_key('activity'):
984 raise KeyError, '"creation" and "activity" are reserved'
986 self.fireAuditors('create', None, propvalues)
988 # new node's id
989 newid = self.db.newid(self.classname)
991 # validate propvalues
992 num_re = re.compile('^\d+$')
993 for key, value in propvalues.items():
994 if key == self.key:
995 try:
996 self.lookup(value)
997 except KeyError:
998 pass
999 else:
1000 raise ValueError, 'node with key "%s" exists'%value
1002 # try to handle this property
1003 try:
1004 prop = self.properties[key]
1005 except KeyError:
1006 raise KeyError, '"%s" has no property "%s"'%(self.classname,
1007 key)
1009 if value is not None and isinstance(prop, Link):
1010 if type(value) != type(''):
1011 raise ValueError, 'link value must be String'
1012 link_class = self.properties[key].classname
1013 # if it isn't a number, it's a key
1014 if not num_re.match(value):
1015 try:
1016 value = self.db.classes[link_class].lookup(value)
1017 except (TypeError, KeyError):
1018 raise IndexError, 'new property "%s": %s not a %s'%(
1019 key, value, link_class)
1020 elif not self.db.getclass(link_class).hasnode(value):
1021 raise IndexError, '%s has no node %s'%(link_class, value)
1023 # save off the value
1024 propvalues[key] = value
1026 # register the link with the newly linked node
1027 if self.do_journal and self.properties[key].do_journal:
1028 self.db.addjournal(link_class, value, 'link',
1029 (self.classname, newid, key))
1031 elif isinstance(prop, Multilink):
1032 if type(value) != type([]):
1033 raise TypeError, 'new property "%s" not a list of ids'%key
1035 # clean up and validate the list of links
1036 link_class = self.properties[key].classname
1037 l = []
1038 for entry in value:
1039 if type(entry) != type(''):
1040 raise ValueError, '"%s" multilink value (%r) '\
1041 'must contain Strings'%(key, value)
1042 # if it isn't a number, it's a key
1043 if not num_re.match(entry):
1044 try:
1045 entry = self.db.classes[link_class].lookup(entry)
1046 except (TypeError, KeyError):
1047 raise IndexError, 'new property "%s": %s not a %s'%(
1048 key, entry, self.properties[key].classname)
1049 l.append(entry)
1050 value = l
1051 propvalues[key] = value
1053 # handle additions
1054 for nodeid in value:
1055 if not self.db.getclass(link_class).hasnode(nodeid):
1056 raise IndexError, '%s has no node %s'%(link_class,
1057 nodeid)
1058 # register the link with the newly linked node
1059 if self.do_journal and self.properties[key].do_journal:
1060 self.db.addjournal(link_class, nodeid, 'link',
1061 (self.classname, newid, key))
1063 elif isinstance(prop, String):
1064 if type(value) != type(''):
1065 raise TypeError, 'new property "%s" not a string'%key
1067 elif isinstance(prop, Password):
1068 if not isinstance(value, password.Password):
1069 raise TypeError, 'new property "%s" not a Password'%key
1071 elif isinstance(prop, Date):
1072 if value is not None and not isinstance(value, date.Date):
1073 raise TypeError, 'new property "%s" not a Date'%key
1075 elif isinstance(prop, Interval):
1076 if value is not None and not isinstance(value, date.Interval):
1077 raise TypeError, 'new property "%s" not an Interval'%key
1079 elif value is not None and isinstance(prop, Number):
1080 try:
1081 float(value)
1082 except ValueError:
1083 raise TypeError, 'new property "%s" not numeric'%key
1085 elif value is not None and isinstance(prop, Boolean):
1086 try:
1087 int(value)
1088 except ValueError:
1089 raise TypeError, 'new property "%s" not boolean'%key
1091 # make sure there's data where there needs to be
1092 for key, prop in self.properties.items():
1093 if propvalues.has_key(key):
1094 continue
1095 if key == self.key:
1096 raise ValueError, 'key property "%s" is required'%key
1097 if isinstance(prop, Multilink):
1098 propvalues[key] = []
1099 else:
1100 propvalues[key] = None
1102 # done
1103 self.db.addnode(self.classname, newid, propvalues)
1104 if self.do_journal:
1105 self.db.addjournal(self.classname, newid, 'create', propvalues)
1107 self.fireReactors('create', newid, None)
1109 return newid
1111 def export_list(self, propnames, nodeid):
1112 ''' Export a node - generate a list of CSV-able data in the order
1113 specified by propnames for the given node.
1114 '''
1115 properties = self.getprops()
1116 l = []
1117 for prop in propnames:
1118 proptype = properties[prop]
1119 value = self.get(nodeid, prop)
1120 # "marshal" data where needed
1121 if value is None:
1122 pass
1123 elif isinstance(proptype, hyperdb.Date):
1124 value = value.get_tuple()
1125 elif isinstance(proptype, hyperdb.Interval):
1126 value = value.get_tuple()
1127 elif isinstance(proptype, hyperdb.Password):
1128 value = str(value)
1129 l.append(repr(value))
1130 return l
1132 def import_list(self, propnames, proplist):
1133 ''' Import a node - all information including "id" is present and
1134 should not be sanity checked. Triggers are not triggered. The
1135 journal should be initialised using the "creator" and "created"
1136 information.
1138 Return the nodeid of the node imported.
1139 '''
1140 if self.db.journaltag is None:
1141 raise DatabaseError, 'Database open read-only'
1142 properties = self.getprops()
1144 # make the new node's property map
1145 d = {}
1146 for i in range(len(propnames)):
1147 # Use eval to reverse the repr() used to output the CSV
1148 value = eval(proplist[i])
1150 # Figure the property for this column
1151 propname = propnames[i]
1152 prop = properties[propname]
1154 # "unmarshal" where necessary
1155 if propname == 'id':
1156 newid = value
1157 continue
1158 elif value is None:
1159 # don't set Nones
1160 continue
1161 elif isinstance(prop, hyperdb.Date):
1162 value = date.Date(value)
1163 elif isinstance(prop, hyperdb.Interval):
1164 value = date.Interval(value)
1165 elif isinstance(prop, hyperdb.Password):
1166 pwd = password.Password()
1167 pwd.unpack(value)
1168 value = pwd
1169 d[propname] = value
1171 # add the node and journal
1172 self.db.addnode(self.classname, newid, d)
1174 # extract the extraneous journalling gumpf and nuke it
1175 if d.has_key('creator'):
1176 creator = d['creator']
1177 del d['creator']
1178 else:
1179 creator = None
1180 if d.has_key('creation'):
1181 creation = d['creation']
1182 del d['creation']
1183 else:
1184 creation = None
1185 if d.has_key('activity'):
1186 del d['activity']
1187 self.db.addjournal(self.classname, newid, 'create', d, creator,
1188 creation)
1189 return newid
1191 _marker = []
1192 def get(self, nodeid, propname, default=_marker, cache=1):
1193 '''Get the value of a property on an existing node of this class.
1195 'nodeid' must be the id of an existing node of this class or an
1196 IndexError is raised. 'propname' must be the name of a property
1197 of this class or a KeyError is raised.
1199 'cache' indicates whether the transaction cache should be queried
1200 for the node. If the node has been modified and you need to
1201 determine what its values prior to modification are, you need to
1202 set cache=0.
1203 '''
1204 if propname == 'id':
1205 return nodeid
1207 # get the node's dict
1208 d = self.db.getnode(self.classname, nodeid)
1210 if propname == 'creation':
1211 if d.has_key('creation'):
1212 return d['creation']
1213 else:
1214 return date.Date()
1215 if propname == 'activity':
1216 if d.has_key('activity'):
1217 return d['activity']
1218 else:
1219 return date.Date()
1220 if propname == 'creator':
1221 if d.has_key('creator'):
1222 return d['creator']
1223 else:
1224 return self.db.curuserid
1226 # get the property (raises KeyErorr if invalid)
1227 prop = self.properties[propname]
1229 if not d.has_key(propname):
1230 if default is self._marker:
1231 if isinstance(prop, Multilink):
1232 return []
1233 else:
1234 return None
1235 else:
1236 return default
1238 # don't pass our list to other code
1239 if isinstance(prop, Multilink):
1240 return d[propname][:]
1242 return d[propname]
1244 def getnode(self, nodeid, cache=1):
1245 ''' Return a convenience wrapper for the node.
1247 'nodeid' must be the id of an existing node of this class or an
1248 IndexError is raised.
1250 'cache' indicates whether the transaction cache should be queried
1251 for the node. If the node has been modified and you need to
1252 determine what its values prior to modification are, you need to
1253 set cache=0.
1254 '''
1255 return Node(self, nodeid, cache=cache)
1257 def set(self, nodeid, **propvalues):
1258 '''Modify a property on an existing node of this class.
1260 'nodeid' must be the id of an existing node of this class or an
1261 IndexError is raised.
1263 Each key in 'propvalues' must be the name of a property of this
1264 class or a KeyError is raised.
1266 All values in 'propvalues' must be acceptable types for their
1267 corresponding properties or a TypeError is raised.
1269 If the value of the key property is set, it must not collide with
1270 other key strings or a ValueError is raised.
1272 If the value of a Link or Multilink property contains an invalid
1273 node id, a ValueError is raised.
1274 '''
1275 if not propvalues:
1276 return propvalues
1278 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1279 raise KeyError, '"creation" and "activity" are reserved'
1281 if propvalues.has_key('id'):
1282 raise KeyError, '"id" is reserved'
1284 if self.db.journaltag is None:
1285 raise DatabaseError, 'Database open read-only'
1287 self.fireAuditors('set', nodeid, propvalues)
1288 # Take a copy of the node dict so that the subsequent set
1289 # operation doesn't modify the oldvalues structure.
1290 # XXX used to try the cache here first
1291 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1293 node = self.db.getnode(self.classname, nodeid)
1294 if self.is_retired(nodeid):
1295 raise IndexError, 'Requested item is retired'
1296 num_re = re.compile('^\d+$')
1298 # if the journal value is to be different, store it in here
1299 journalvalues = {}
1301 # remember the add/remove stuff for multilinks, making it easier
1302 # for the Database layer to do its stuff
1303 multilink_changes = {}
1305 for propname, value in propvalues.items():
1306 # check to make sure we're not duplicating an existing key
1307 if propname == self.key and node[propname] != value:
1308 try:
1309 self.lookup(value)
1310 except KeyError:
1311 pass
1312 else:
1313 raise ValueError, 'node with key "%s" exists'%value
1315 # this will raise the KeyError if the property isn't valid
1316 # ... we don't use getprops() here because we only care about
1317 # the writeable properties.
1318 try:
1319 prop = self.properties[propname]
1320 except KeyError:
1321 raise KeyError, '"%s" has no property named "%s"'%(
1322 self.classname, propname)
1324 # if the value's the same as the existing value, no sense in
1325 # doing anything
1326 if node.has_key(propname) and value == node[propname]:
1327 del propvalues[propname]
1328 continue
1330 # do stuff based on the prop type
1331 if isinstance(prop, Link):
1332 link_class = prop.classname
1333 # if it isn't a number, it's a key
1334 if value is not None and not isinstance(value, type('')):
1335 raise ValueError, 'property "%s" link value be a string'%(
1336 propname)
1337 if isinstance(value, type('')) and not num_re.match(value):
1338 try:
1339 value = self.db.classes[link_class].lookup(value)
1340 except (TypeError, KeyError):
1341 raise IndexError, 'new property "%s": %s not a %s'%(
1342 propname, value, prop.classname)
1344 if (value is not None and
1345 not self.db.getclass(link_class).hasnode(value)):
1346 raise IndexError, '%s has no node %s'%(link_class, value)
1348 if self.do_journal and prop.do_journal:
1349 # register the unlink with the old linked node
1350 if node[propname] is not None:
1351 self.db.addjournal(link_class, node[propname], 'unlink',
1352 (self.classname, nodeid, propname))
1354 # register the link with the newly linked node
1355 if value is not None:
1356 self.db.addjournal(link_class, value, 'link',
1357 (self.classname, nodeid, propname))
1359 elif isinstance(prop, Multilink):
1360 if type(value) != type([]):
1361 raise TypeError, 'new property "%s" not a list of'\
1362 ' ids'%propname
1363 link_class = self.properties[propname].classname
1364 l = []
1365 for entry in value:
1366 # if it isn't a number, it's a key
1367 if type(entry) != type(''):
1368 raise ValueError, 'new property "%s" link value ' \
1369 'must be a string'%propname
1370 if not num_re.match(entry):
1371 try:
1372 entry = self.db.classes[link_class].lookup(entry)
1373 except (TypeError, KeyError):
1374 raise IndexError, 'new property "%s": %s not a %s'%(
1375 propname, entry,
1376 self.properties[propname].classname)
1377 l.append(entry)
1378 value = l
1379 propvalues[propname] = value
1381 # figure the journal entry for this property
1382 add = []
1383 remove = []
1385 # handle removals
1386 if node.has_key(propname):
1387 l = node[propname]
1388 else:
1389 l = []
1390 for id in l[:]:
1391 if id in value:
1392 continue
1393 # register the unlink with the old linked node
1394 if self.do_journal and self.properties[propname].do_journal:
1395 self.db.addjournal(link_class, id, 'unlink',
1396 (self.classname, nodeid, propname))
1397 l.remove(id)
1398 remove.append(id)
1400 # handle additions
1401 for id in value:
1402 if not self.db.getclass(link_class).hasnode(id):
1403 raise IndexError, '%s has no node %s'%(link_class, id)
1404 if id in l:
1405 continue
1406 # register the link with the newly linked node
1407 if self.do_journal and self.properties[propname].do_journal:
1408 self.db.addjournal(link_class, id, 'link',
1409 (self.classname, nodeid, propname))
1410 l.append(id)
1411 add.append(id)
1413 # figure the journal entry
1414 l = []
1415 if add:
1416 l.append(('+', add))
1417 if remove:
1418 l.append(('-', remove))
1419 multilink_changes[propname] = (add, remove)
1420 if l:
1421 journalvalues[propname] = tuple(l)
1423 elif isinstance(prop, String):
1424 if value is not None and type(value) != type(''):
1425 raise TypeError, 'new property "%s" not a string'%propname
1427 elif isinstance(prop, Password):
1428 if not isinstance(value, password.Password):
1429 raise TypeError, 'new property "%s" not a Password'%propname
1430 propvalues[propname] = value
1432 elif value is not None and isinstance(prop, Date):
1433 if not isinstance(value, date.Date):
1434 raise TypeError, 'new property "%s" not a Date'% propname
1435 propvalues[propname] = value
1437 elif value is not None and isinstance(prop, Interval):
1438 if not isinstance(value, date.Interval):
1439 raise TypeError, 'new property "%s" not an '\
1440 'Interval'%propname
1441 propvalues[propname] = value
1443 elif value is not None and isinstance(prop, Number):
1444 try:
1445 float(value)
1446 except ValueError:
1447 raise TypeError, 'new property "%s" not numeric'%propname
1449 elif value is not None and isinstance(prop, Boolean):
1450 try:
1451 int(value)
1452 except ValueError:
1453 raise TypeError, 'new property "%s" not boolean'%propname
1455 # nothing to do?
1456 if not propvalues:
1457 return propvalues
1459 # do the set, and journal it
1460 self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
1462 if self.do_journal:
1463 propvalues.update(journalvalues)
1464 self.db.addjournal(self.classname, nodeid, 'set', propvalues)
1466 self.fireReactors('set', nodeid, oldvalues)
1468 return propvalues
1470 def retire(self, nodeid):
1471 '''Retire a node.
1473 The properties on the node remain available from the get() method,
1474 and the node's id is never reused.
1476 Retired nodes are not returned by the find(), list(), or lookup()
1477 methods, and other nodes may reuse the values of their key properties.
1478 '''
1479 if self.db.journaltag is None:
1480 raise DatabaseError, 'Database open read-only'
1482 # use the arg for __retired__ to cope with any odd database type
1483 # conversion (hello, sqlite)
1484 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1485 self.db.arg, self.db.arg)
1486 if __debug__:
1487 print >>hyperdb.DEBUG, 'retire', (self, sql, nodeid)
1488 self.db.cursor.execute(sql, (1, nodeid))
1490 def is_retired(self, nodeid):
1491 '''Return true if the node is rerired
1492 '''
1493 sql = 'select __retired__ from _%s where id=%s'%(self.classname,
1494 self.db.arg)
1495 if __debug__:
1496 print >>hyperdb.DEBUG, 'is_retired', (self, sql, nodeid)
1497 self.db.cursor.execute(sql, (nodeid,))
1498 return int(self.db.sql_fetchone()[0])
1500 def destroy(self, nodeid):
1501 '''Destroy a node.
1503 WARNING: this method should never be used except in extremely rare
1504 situations where there could never be links to the node being
1505 deleted
1506 WARNING: use retire() instead
1507 WARNING: the properties of this node will not be available ever again
1508 WARNING: really, use retire() instead
1510 Well, I think that's enough warnings. This method exists mostly to
1511 support the session storage of the cgi interface.
1513 The node is completely removed from the hyperdb, including all journal
1514 entries. It will no longer be available, and will generally break code
1515 if there are any references to the node.
1516 '''
1517 if self.db.journaltag is None:
1518 raise DatabaseError, 'Database open read-only'
1519 self.db.destroynode(self.classname, nodeid)
1521 def history(self, nodeid):
1522 '''Retrieve the journal of edits on a particular node.
1524 'nodeid' must be the id of an existing node of this class or an
1525 IndexError is raised.
1527 The returned list contains tuples of the form
1529 (date, tag, action, params)
1531 'date' is a Timestamp object specifying the time of the change and
1532 'tag' is the journaltag specified when the database was opened.
1533 '''
1534 if not self.do_journal:
1535 raise ValueError, 'Journalling is disabled for this class'
1536 return self.db.getjournal(self.classname, nodeid)
1538 # Locating nodes:
1539 def hasnode(self, nodeid):
1540 '''Determine if the given nodeid actually exists
1541 '''
1542 return self.db.hasnode(self.classname, nodeid)
1544 def setkey(self, propname):
1545 '''Select a String property of this class to be the key property.
1547 'propname' must be the name of a String property of this class or
1548 None, or a TypeError is raised. The values of the key property on
1549 all existing nodes must be unique or a ValueError is raised.
1550 '''
1551 # XXX create an index on the key prop column
1552 prop = self.getprops()[propname]
1553 if not isinstance(prop, String):
1554 raise TypeError, 'key properties must be String'
1555 self.key = propname
1557 def getkey(self):
1558 '''Return the name of the key property for this class or None.'''
1559 return self.key
1561 def labelprop(self, default_to_id=0):
1562 ''' Return the property name for a label for the given node.
1564 This method attempts to generate a consistent label for the node.
1565 It tries the following in order:
1566 1. key property
1567 2. "name" property
1568 3. "title" property
1569 4. first property from the sorted property name list
1570 '''
1571 k = self.getkey()
1572 if k:
1573 return k
1574 props = self.getprops()
1575 if props.has_key('name'):
1576 return 'name'
1577 elif props.has_key('title'):
1578 return 'title'
1579 if default_to_id:
1580 return 'id'
1581 props = props.keys()
1582 props.sort()
1583 return props[0]
1585 def lookup(self, keyvalue):
1586 '''Locate a particular node by its key property and return its id.
1588 If this class has no key property, a TypeError is raised. If the
1589 'keyvalue' matches one of the values for the key property among
1590 the nodes in this class, the matching node's id is returned;
1591 otherwise a KeyError is raised.
1592 '''
1593 if not self.key:
1594 raise TypeError, 'No key property set for class %s'%self.classname
1596 # use the arg to handle any odd database type conversion (hello,
1597 # sqlite)
1598 sql = "select id from _%s where _%s=%s and __retired__ <> %s"%(
1599 self.classname, self.key, self.db.arg, self.db.arg)
1600 self.db.sql(sql, (keyvalue, 1))
1602 # see if there was a result that's not retired
1603 row = self.db.sql_fetchone()
1604 if not row:
1605 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1606 keyvalue, self.classname)
1608 # return the id
1609 return row[0]
1611 def find(self, **propspec):
1612 '''Get the ids of nodes in this class which link to the given nodes.
1614 'propspec' consists of keyword args propname=nodeid or
1615 propname={nodeid:1, }
1616 'propname' must be the name of a property in this class, or a
1617 KeyError is raised. That property must be a Link or Multilink
1618 property, or a TypeError is raised.
1620 Any node in this class whose 'propname' property links to any of the
1621 nodeids will be returned. Used by the full text indexing, which knows
1622 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1623 issues:
1625 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1626 '''
1627 if __debug__:
1628 print >>hyperdb.DEBUG, 'find', (self, propspec)
1630 # shortcut
1631 if not propspec:
1632 return []
1634 # validate the args
1635 props = self.getprops()
1636 propspec = propspec.items()
1637 for propname, nodeids in propspec:
1638 # check the prop is OK
1639 prop = props[propname]
1640 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1641 raise TypeError, "'%s' not a Link/Multilink property"%propname
1643 # first, links
1644 where = []
1645 allvalues = ()
1646 a = self.db.arg
1647 for prop, values in propspec:
1648 if not isinstance(props[prop], hyperdb.Link):
1649 continue
1650 if type(values) is type(''):
1651 allvalues += (values,)
1652 where.append('_%s = %s'%(prop, a))
1653 else:
1654 allvalues += tuple(values.keys())
1655 where.append('_%s in (%s)'%(prop, ','.join([a]*len(values))))
1656 tables = []
1657 if where:
1658 tables.append('select id as nodeid from _%s where %s'%(
1659 self.classname, ' and '.join(where)))
1661 # now multilinks
1662 for prop, values in propspec:
1663 if not isinstance(props[prop], hyperdb.Multilink):
1664 continue
1665 if type(values) is type(''):
1666 allvalues += (values,)
1667 s = a
1668 else:
1669 allvalues += tuple(values.keys())
1670 s = ','.join([a]*len(values))
1671 tables.append('select nodeid from %s_%s where linkid in (%s)'%(
1672 self.classname, prop, s))
1673 sql = '\nunion\n'.join(tables)
1674 self.db.sql(sql, allvalues)
1675 l = [x[0] for x in self.db.sql_fetchall()]
1676 if __debug__:
1677 print >>hyperdb.DEBUG, 'find ... ', l
1678 return l
1680 def stringFind(self, **requirements):
1681 '''Locate a particular node by matching a set of its String
1682 properties in a caseless search.
1684 If the property is not a String property, a TypeError is raised.
1686 The return is a list of the id of all nodes that match.
1687 '''
1688 where = []
1689 args = []
1690 for propname in requirements.keys():
1691 prop = self.properties[propname]
1692 if isinstance(not prop, String):
1693 raise TypeError, "'%s' not a String property"%propname
1694 where.append(propname)
1695 args.append(requirements[propname].lower())
1697 # generate the where clause
1698 s = ' and '.join(['_%s=%s'%(col, self.db.arg) for col in where])
1699 sql = 'select id from _%s where %s'%(self.classname, s)
1700 self.db.sql(sql, tuple(args))
1701 l = [x[0] for x in self.db.sql_fetchall()]
1702 if __debug__:
1703 print >>hyperdb.DEBUG, 'find ... ', l
1704 return l
1706 def list(self):
1707 ''' Return a list of the ids of the active nodes in this class.
1708 '''
1709 return self.db.getnodeids(self.classname, retired=0)
1711 def filter(self, search_matches, filterspec, sort, group):
1712 ''' Return a list of the ids of the active nodes in this class that
1713 match the 'filter' spec, sorted by the group spec and then the
1714 sort spec
1716 "filterspec" is {propname: value(s)}
1717 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1718 and prop is a prop name or None
1719 "search_matches" is {nodeid: marker}
1721 The filter must match all properties specificed - but if the
1722 property value to match is a list, any one of the values in the
1723 list may match for that property to match.
1724 '''
1725 # just don't bother if the full-text search matched diddly
1726 if search_matches == {}:
1727 return []
1729 cn = self.classname
1731 # figure the WHERE clause from the filterspec
1732 props = self.getprops()
1733 frum = ['_'+cn]
1734 where = []
1735 args = []
1736 a = self.db.arg
1737 for k, v in filterspec.items():
1738 propclass = props[k]
1739 # now do other where clause stuff
1740 if isinstance(propclass, Multilink):
1741 tn = '%s_%s'%(cn, k)
1742 frum.append(tn)
1743 if isinstance(v, type([])):
1744 s = ','.join([a for x in v])
1745 where.append('id=%s.nodeid and %s.linkid in (%s)'%(tn,tn,s))
1746 args = args + v
1747 else:
1748 where.append('id=%s.nodeid and %s.linkid = %s'%(tn, tn, a))
1749 args.append(v)
1750 elif isinstance(propclass, String):
1751 if not isinstance(v, type([])):
1752 v = [v]
1754 # Quote the bits in the string that need it and then embed
1755 # in a "substring" search. Note - need to quote the '%' so
1756 # they make it through the python layer happily
1757 v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
1759 # now add to the where clause
1760 where.append(' or '.join(["_%s LIKE '%s'"%(k, s) for s in v]))
1761 # note: args are embedded in the query string now
1762 elif isinstance(propclass, Link):
1763 if isinstance(v, type([])):
1764 if '-1' in v:
1765 v.remove('-1')
1766 xtra = ' or _%s is NULL'%k
1767 else:
1768 xtra = ''
1769 s = ','.join([a for x in v])
1770 where.append('(_%s in (%s)%s)'%(k, s, xtra))
1771 args = args + v
1772 else:
1773 if v == '-1':
1774 v = None
1775 where.append('_%s is NULL'%k)
1776 else:
1777 where.append('_%s=%s'%(k, a))
1778 args.append(v)
1779 else:
1780 if isinstance(v, type([])):
1781 s = ','.join([a for x in v])
1782 where.append('_%s in (%s)'%(k, s))
1783 args = args + v
1784 else:
1785 where.append('_%s=%s'%(k, a))
1786 args.append(v)
1788 # add results of full text search
1789 if search_matches is not None:
1790 v = search_matches.keys()
1791 s = ','.join([a for x in v])
1792 where.append('id in (%s)'%s)
1793 args = args + v
1795 # "grouping" is just the first-order sorting in the SQL fetch
1796 # can modify it...)
1797 orderby = []
1798 ordercols = []
1799 if group[0] is not None and group[1] is not None:
1800 if group[0] != '-':
1801 orderby.append('_'+group[1])
1802 ordercols.append('_'+group[1])
1803 else:
1804 orderby.append('_'+group[1]+' desc')
1805 ordercols.append('_'+group[1])
1807 # now add in the sorting
1808 group = ''
1809 if sort[0] is not None and sort[1] is not None:
1810 direction, colname = sort
1811 if direction != '-':
1812 if colname == 'id':
1813 orderby.append(colname)
1814 else:
1815 orderby.append('_'+colname)
1816 ordercols.append('_'+colname)
1817 else:
1818 if colname == 'id':
1819 orderby.append(colname+' desc')
1820 ordercols.append(colname)
1821 else:
1822 orderby.append('_'+colname+' desc')
1823 ordercols.append('_'+colname)
1825 # construct the SQL
1826 frum = ','.join(frum)
1827 if where:
1828 where = ' where ' + (' and '.join(where))
1829 else:
1830 where = ''
1831 cols = ['id']
1832 if orderby:
1833 cols = cols + ordercols
1834 order = ' order by %s'%(','.join(orderby))
1835 else:
1836 order = ''
1837 cols = ','.join(cols)
1838 sql = 'select %s from %s %s%s%s'%(cols, frum, where, group, order)
1839 args = tuple(args)
1840 if __debug__:
1841 print >>hyperdb.DEBUG, 'filter', (self, sql, args)
1842 self.db.cursor.execute(sql, args)
1843 l = self.db.cursor.fetchall()
1845 # return the IDs (the first column)
1846 # XXX The filter(None, l) bit is sqlite-specific... if there's _NO_
1847 # XXX matches to a fetch, it returns NULL instead of nothing!?!
1848 return filter(None, [row[0] for row in l])
1850 def count(self):
1851 '''Get the number of nodes in this class.
1853 If the returned integer is 'numnodes', the ids of all the nodes
1854 in this class run from 1 to numnodes, and numnodes+1 will be the
1855 id of the next node to be created in this class.
1856 '''
1857 return self.db.countnodes(self.classname)
1859 # Manipulating properties:
1860 def getprops(self, protected=1):
1861 '''Return a dictionary mapping property names to property objects.
1862 If the "protected" flag is true, we include protected properties -
1863 those which may not be modified.
1864 '''
1865 d = self.properties.copy()
1866 if protected:
1867 d['id'] = String()
1868 d['creation'] = hyperdb.Date()
1869 d['activity'] = hyperdb.Date()
1870 d['creator'] = hyperdb.Link('user')
1871 return d
1873 def addprop(self, **properties):
1874 '''Add properties to this class.
1876 The keyword arguments in 'properties' must map names to property
1877 objects, or a TypeError is raised. None of the keys in 'properties'
1878 may collide with the names of existing properties, or a ValueError
1879 is raised before any properties have been added.
1880 '''
1881 for key in properties.keys():
1882 if self.properties.has_key(key):
1883 raise ValueError, key
1884 self.properties.update(properties)
1886 def index(self, nodeid):
1887 '''Add (or refresh) the node to search indexes
1888 '''
1889 # find all the String properties that have indexme
1890 for prop, propclass in self.getprops().items():
1891 if isinstance(propclass, String) and propclass.indexme:
1892 try:
1893 value = str(self.get(nodeid, prop))
1894 except IndexError:
1895 # node no longer exists - entry should be removed
1896 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1897 else:
1898 # and index them under (classname, nodeid, property)
1899 self.db.indexer.add_text((self.classname, nodeid, prop),
1900 value)
1903 #
1904 # Detector interface
1905 #
1906 def audit(self, event, detector):
1907 '''Register a detector
1908 '''
1909 l = self.auditors[event]
1910 if detector not in l:
1911 self.auditors[event].append(detector)
1913 def fireAuditors(self, action, nodeid, newvalues):
1914 '''Fire all registered auditors.
1915 '''
1916 for audit in self.auditors[action]:
1917 audit(self.db, self, nodeid, newvalues)
1919 def react(self, event, detector):
1920 '''Register a detector
1921 '''
1922 l = self.reactors[event]
1923 if detector not in l:
1924 self.reactors[event].append(detector)
1926 def fireReactors(self, action, nodeid, oldvalues):
1927 '''Fire all registered reactors.
1928 '''
1929 for react in self.reactors[action]:
1930 react(self.db, self, nodeid, oldvalues)
1932 class FileClass(Class):
1933 '''This class defines a large chunk of data. To support this, it has a
1934 mandatory String property "content" which is typically saved off
1935 externally to the hyperdb.
1937 The default MIME type of this data is defined by the
1938 "default_mime_type" class attribute, which may be overridden by each
1939 node if the class defines a "type" String property.
1940 '''
1941 default_mime_type = 'text/plain'
1943 def create(self, **propvalues):
1944 ''' snaffle the file propvalue and store in a file
1945 '''
1946 content = propvalues['content']
1947 del propvalues['content']
1948 newid = Class.create(self, **propvalues)
1949 self.db.storefile(self.classname, newid, None, content)
1950 return newid
1952 def import_list(self, propnames, proplist):
1953 ''' Trap the "content" property...
1954 '''
1955 # dupe this list so we don't affect others
1956 propnames = propnames[:]
1958 # extract the "content" property from the proplist
1959 i = propnames.index('content')
1960 content = eval(proplist[i])
1961 del propnames[i]
1962 del proplist[i]
1964 # do the normal import
1965 newid = Class.import_list(self, propnames, proplist)
1967 # save off the "content" file
1968 self.db.storefile(self.classname, newid, None, content)
1969 return newid
1971 _marker = []
1972 def get(self, nodeid, propname, default=_marker, cache=1):
1973 ''' trap the content propname and get it from the file
1974 '''
1976 poss_msg = 'Possibly a access right configuration problem.'
1977 if propname == 'content':
1978 try:
1979 return self.db.getfile(self.classname, nodeid, None)
1980 except IOError, (strerror):
1981 # BUG: by catching this we donot see an error in the log.
1982 return 'ERROR reading file: %s%s\n%s\n%s'%(
1983 self.classname, nodeid, poss_msg, strerror)
1984 if default is not self._marker:
1985 return Class.get(self, nodeid, propname, default, cache=cache)
1986 else:
1987 return Class.get(self, nodeid, propname, cache=cache)
1989 def getprops(self, protected=1):
1990 ''' In addition to the actual properties on the node, these methods
1991 provide the "content" property. If the "protected" flag is true,
1992 we include protected properties - those which may not be
1993 modified.
1994 '''
1995 d = Class.getprops(self, protected=protected).copy()
1996 d['content'] = hyperdb.String()
1997 return d
1999 def index(self, nodeid):
2000 ''' Index the node in the search index.
2002 We want to index the content in addition to the normal String
2003 property indexing.
2004 '''
2005 # perform normal indexing
2006 Class.index(self, nodeid)
2008 # get the content to index
2009 content = self.get(nodeid, 'content')
2011 # figure the mime type
2012 if self.properties.has_key('type'):
2013 mime_type = self.get(nodeid, 'type')
2014 else:
2015 mime_type = self.default_mime_type
2017 # and index!
2018 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
2019 mime_type)
2021 # XXX deviation from spec - was called ItemClass
2022 class IssueClass(Class, roundupdb.IssueClass):
2023 # Overridden methods:
2024 def __init__(self, db, classname, **properties):
2025 '''The newly-created class automatically includes the "messages",
2026 "files", "nosy", and "superseder" properties. If the 'properties'
2027 dictionary attempts to specify any of these properties or a
2028 "creation" or "activity" property, a ValueError is raised.
2029 '''
2030 if not properties.has_key('title'):
2031 properties['title'] = hyperdb.String(indexme='yes')
2032 if not properties.has_key('messages'):
2033 properties['messages'] = hyperdb.Multilink("msg")
2034 if not properties.has_key('files'):
2035 properties['files'] = hyperdb.Multilink("file")
2036 if not properties.has_key('nosy'):
2037 # note: journalling is turned off as it really just wastes
2038 # space. this behaviour may be overridden in an instance
2039 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2040 if not properties.has_key('superseder'):
2041 properties['superseder'] = hyperdb.Multilink(classname)
2042 Class.__init__(self, db, classname, **properties)