1 # $Id: rdbms_common.py,v 1.64 2003-10-07 08:34:58 anthonybaxter Exp $
2 ''' Relational database (SQL) backend common code.
4 Basics:
6 - map roundup classes to relational tables
7 - automatically detect schema changes and modify the table schemas
8 appropriately (we store the "database version" of the schema in the
9 database itself as the only row of the "schema" table)
10 - multilinks (which represent a many-to-many relationship) are handled through
11 intermediate tables
12 - journals are stored adjunct to the per-class tables
13 - table names and columns have "_" prepended so the names can't clash with
14 restricted names (like "order")
15 - retirement is determined by the __retired__ column being true
17 Database-specific changes may generally be pushed out to the overridable
18 sql_* methods, since everything else should be fairly generic. There's
19 probably a bit of work to be done if a database is used that actually
20 honors column typing, since the initial databases don't (sqlite stores
21 everything as a string.)
22 '''
24 # standard python modules
25 import sys, os, time, re, errno, weakref, copy
27 # roundup modules
28 from roundup import hyperdb, date, password, roundupdb, security
29 from roundup.hyperdb import String, Password, Date, Interval, Link, \
30 Multilink, DatabaseError, Boolean, Number, Node
31 from roundup.backends import locking
33 # support
34 from blobfiles import FileStorage
35 from roundup.indexer import Indexer
36 from sessions import Sessions, OneTimeKeys
37 from roundup.date import Range
39 # number of rows to keep in memory
40 ROW_CACHE_SIZE = 100
42 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
43 ''' Wrapper around an SQL database that presents a hyperdb interface.
45 - some functionality is specific to the actual SQL database, hence
46 the sql_* methods that are NotImplemented
47 - we keep a cache of the latest ROW_CACHE_SIZE row fetches.
48 '''
49 def __init__(self, config, journaltag=None):
50 ''' Open the database and load the schema from it.
51 '''
52 self.config, self.journaltag = config, journaltag
53 self.dir = config.DATABASE
54 self.classes = {}
55 self.indexer = Indexer(self.dir)
56 self.sessions = Sessions(self.config)
57 self.otks = OneTimeKeys(self.config)
58 self.security = security.Security(self)
60 # additional transaction support for external files and the like
61 self.transactions = []
63 # keep a cache of the N most recently retrieved rows of any kind
64 # (classname, nodeid) = row
65 self.cache = {}
66 self.cache_lru = []
68 # database lock
69 self.lockfile = None
71 # open a connection to the database, creating the "conn" attribute
72 self.open_connection()
74 def clearCache(self):
75 self.cache = {}
76 self.cache_lru = []
78 def open_connection(self):
79 ''' Open a connection to the database, creating it if necessary
80 '''
81 raise NotImplemented
83 def sql(self, sql, args=None):
84 ''' Execute the sql with the optional args.
85 '''
86 if __debug__:
87 print >>hyperdb.DEBUG, (self, sql, args)
88 if args:
89 self.cursor.execute(sql, args)
90 else:
91 self.cursor.execute(sql)
93 def sql_fetchone(self):
94 ''' Fetch a single row. If there's nothing to fetch, return None.
95 '''
96 raise NotImplemented
98 def sql_stringquote(self, value):
99 ''' Quote the string so it's safe to put in the 'sql quotes'
100 '''
101 return re.sub("'", "''", str(value))
103 def save_dbschema(self, schema):
104 ''' Save the schema definition that the database currently implements
105 '''
106 raise NotImplemented
108 def load_dbschema(self):
109 ''' Load the schema definition that the database currently implements
110 '''
111 raise NotImplemented
113 def post_init(self):
114 ''' Called once the schema initialisation has finished.
116 We should now confirm that the schema defined by our "classes"
117 attribute actually matches the schema in the database.
118 '''
119 # now detect changes in the schema
120 save = 0
121 for classname, spec in self.classes.items():
122 if self.database_schema.has_key(classname):
123 dbspec = self.database_schema[classname]
124 if self.update_class(spec, dbspec):
125 self.database_schema[classname] = spec.schema()
126 save = 1
127 else:
128 self.create_class(spec)
129 self.database_schema[classname] = spec.schema()
130 save = 1
132 for classname in self.database_schema.keys():
133 if not self.classes.has_key(classname):
134 self.drop_class(classname)
136 # update the database version of the schema
137 if save:
138 self.sql('delete from schema')
139 self.save_dbschema(self.database_schema)
141 # reindex the db if necessary
142 if self.indexer.should_reindex():
143 self.reindex()
145 # commit
146 self.conn.commit()
148 def reindex(self):
149 for klass in self.classes.values():
150 for nodeid in klass.list():
151 klass.index(nodeid)
152 self.indexer.save_index()
154 def determine_columns(self, properties):
155 ''' Figure the column names and multilink properties from the spec
157 "properties" is a list of (name, prop) where prop may be an
158 instance of a hyperdb "type" _or_ a string repr of that type.
159 '''
160 cols = ['_activity', '_creator', '_creation']
161 mls = []
162 # add the multilinks separately
163 for col, prop in properties:
164 if isinstance(prop, Multilink):
165 mls.append(col)
166 elif isinstance(prop, type('')) and prop.find('Multilink') != -1:
167 mls.append(col)
168 else:
169 cols.append('_'+col)
170 cols.sort()
171 return cols, mls
173 def update_class(self, spec, old_spec):
174 ''' Determine the differences between the current spec and the
175 database version of the spec, and update where necessary
176 '''
177 new_spec = spec
178 new_has = new_spec.properties.has_key
180 new_spec = new_spec.schema()
181 new_spec[1].sort()
182 old_spec[1].sort()
183 if new_spec == old_spec:
184 # no changes
185 return 0
187 if __debug__:
188 print >>hyperdb.DEBUG, 'update_class FIRING'
190 # key property changed?
191 if old_spec[0] != new_spec[0]:
192 if __debug__:
193 print >>hyperdb.DEBUG, 'update_class setting keyprop', `spec[0]`
194 # XXX turn on indexing for the key property.
196 # detect multilinks that have been removed, and drop their table
197 old_has = {}
198 for name,prop in old_spec[1]:
199 old_has[name] = 1
200 if not new_has(name) and isinstance(prop, Multilink):
201 # it's a multilink, and it's been removed - drop the old
202 # table. First drop indexes.
203 index_sqls = [ 'drop index %s_%s_l_idx'%(spec.classname, ml),
204 'drop index %s_%s_n_idx'%(spec.classname, ml) ]
205 for index_sql in index_sqls:
206 if __debug__:
207 print >>hyperdb.DEBUG, 'drop_index', (self, index_sql)
208 try:
209 self.cursor.execute(index_sql)
210 except:
211 # The database may not actually have any indexes.
212 # assume the worst.
213 pass
214 sql = 'drop table %s_%s'%(spec.classname, prop)
215 if __debug__:
216 print >>hyperdb.DEBUG, 'update_class', (self, sql)
217 self.cursor.execute(sql)
218 continue
219 old_has = old_has.has_key
221 # now figure how we populate the new table
222 fetch = ['_activity', '_creation', '_creator']
223 properties = spec.getprops()
224 for propname,x in new_spec[1]:
225 prop = properties[propname]
226 if isinstance(prop, Multilink):
227 if not old_has(propname):
228 # we need to create the new table
229 self.create_multilink_table(spec, propname)
230 elif old_has(propname):
231 # we copy this col over from the old table
232 fetch.append('_'+propname)
234 # select the data out of the old table
235 fetch.append('id')
236 fetch.append('__retired__')
237 fetchcols = ','.join(fetch)
238 cn = spec.classname
239 sql = 'select %s from _%s'%(fetchcols, cn)
240 if __debug__:
241 print >>hyperdb.DEBUG, 'update_class', (self, sql)
242 self.cursor.execute(sql)
243 olddata = self.cursor.fetchall()
245 # drop the old table - indexes first
246 index_sqls = [ 'drop index _%s_id_idx'%cn,
247 'drop index _%s_retired_idx'%cn ]
248 for index_sql in index_sqls:
249 if __debug__:
250 print >>hyperdb.DEBUG, 'drop_index', (self, index_sql)
251 try:
252 self.cursor.execute(index_sql)
253 except:
254 # The database may not actually have any indexes.
255 # assume the worst.
256 pass
257 self.cursor.execute('drop table _%s'%cn)
258 # create the new table
259 self.create_class_table(spec)
261 if olddata:
262 # do the insert
263 args = ','.join([self.arg for x in fetch])
264 sql = 'insert into _%s (%s) values (%s)'%(cn, fetchcols, args)
265 if __debug__:
266 print >>hyperdb.DEBUG, 'update_class', (self, sql, olddata[0])
267 for entry in olddata:
268 self.cursor.execute(sql, tuple(entry))
270 return 1
272 def create_class_table(self, spec):
273 ''' create the class table for the given spec
274 '''
275 cols, mls = self.determine_columns(spec.properties.items())
277 # add on our special columns
278 cols.append('id')
279 cols.append('__retired__')
281 # create the base table
282 scols = ','.join(['%s varchar'%x for x in cols])
283 sql = 'create table _%s (%s)'%(spec.classname, scols)
284 if __debug__:
285 print >>hyperdb.DEBUG, 'create_class', (self, sql)
286 self.cursor.execute(sql)
287 index_sql1 = 'create index _%s_id_idx on _%s(id)'%(
288 spec.classname, spec.classname)
289 if __debug__:
290 print >>hyperdb.DEBUG, 'create_index', (self, index_sql1)
291 self.cursor.execute(index_sql1)
292 index_sql2 = 'create index _%s_retired_idx on _%s(__retired__)'%(
293 spec.classname, spec.classname)
294 if __debug__:
295 print >>hyperdb.DEBUG, 'create_index', (self, index_sql2)
296 self.cursor.execute(index_sql2)
298 return cols, mls
300 def create_journal_table(self, spec):
301 ''' create the journal table for a class given the spec and
302 already-determined cols
303 '''
304 # journal table
305 cols = ','.join(['%s varchar'%x
306 for x in 'nodeid date tag action params'.split()])
307 sql = 'create table %s__journal (%s)'%(spec.classname, cols)
308 if __debug__:
309 print >>hyperdb.DEBUG, 'create_class', (self, sql)
310 self.cursor.execute(sql)
311 index_sql = 'create index %s_journ_idx on %s__journal(nodeid)'%(
312 spec.classname, spec.classname)
313 if __debug__:
314 print >>hyperdb.DEBUG, 'create_index', (self, index_sql)
315 self.cursor.execute(index_sql)
317 def create_multilink_table(self, spec, ml):
318 ''' Create a multilink table for the "ml" property of the class
319 given by the spec
320 '''
321 sql = 'create table %s_%s (linkid varchar, nodeid varchar)'%(
322 spec.classname, ml)
323 if __debug__:
324 print >>hyperdb.DEBUG, 'create_class', (self, sql)
325 self.cursor.execute(sql)
326 index_sql = 'create index %s_%s_l_idx on %s_%s(linkid)'%(
327 spec.classname, ml, spec.classname, ml)
328 if __debug__:
329 print >>hyperdb.DEBUG, 'create_index', (self, index_sql)
330 self.cursor.execute(index_sql)
331 index_sql = 'create index %s_%s_n_idx on %s_%s(nodeid)'%(
332 spec.classname, ml, spec.classname, ml)
333 if __debug__:
334 print >>hyperdb.DEBUG, 'create_index', (self, index_sql)
335 self.cursor.execute(index_sql)
337 def create_class(self, spec):
338 ''' Create a database table according to the given spec.
339 '''
340 cols, mls = self.create_class_table(spec)
341 self.create_journal_table(spec)
343 # now create the multilink tables
344 for ml in mls:
345 self.create_multilink_table(spec, ml)
347 # ID counter
348 sql = 'insert into ids (name, num) values (%s,%s)'%(self.arg, self.arg)
349 vals = (spec.classname, 1)
350 if __debug__:
351 print >>hyperdb.DEBUG, 'create_class', (self, sql, vals)
352 self.cursor.execute(sql, vals)
354 def drop_class(self, spec):
355 ''' Drop the given table from the database.
357 Drop the journal and multilink tables too.
358 '''
359 # figure the multilinks
360 mls = []
361 for col, prop in spec.properties.items():
362 if isinstance(prop, Multilink):
363 mls.append(col)
365 index_sqls = [ 'drop index _%s_id_idx'%cn,
366 'drop index _%s_retired_idx'%cn,
367 'drop index %s_journ_idx'%cn ]
368 for index_sql in index_sqls:
369 if __debug__:
370 print >>hyperdb.DEBUG, 'drop_index', (self, index_sql)
371 try:
372 self.cursor.execute(index_sql)
373 except:
374 # The database may not actually have any indexes.
375 # assume the worst.
376 pass
378 sql = 'drop table _%s'%spec.classname
379 if __debug__:
380 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
381 self.cursor.execute(sql)
382 sql = 'drop table %s__journal'%spec.classname
383 if __debug__:
384 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
385 self.cursor.execute(sql)
387 for ml in mls:
388 index_sqls = [
389 'drop index %s_%s_n_idx'%(spec.classname, ml),
390 'drop index %s_%s_l_idx'%(spec.classname, ml),
391 ]
392 for index_sql in index_sqls:
393 if __debug__:
394 print >>hyperdb.DEBUG, 'drop_index', (self, index_sql)
395 try:
396 self.cursor.execute(index_sql)
397 except:
398 # The database may not actually have any indexes.
399 # assume the worst.
400 pass
401 sql = 'drop table %s_%s'%(spec.classname, ml)
402 if __debug__:
403 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
404 self.cursor.execute(sql)
406 #
407 # Classes
408 #
409 def __getattr__(self, classname):
410 ''' A convenient way of calling self.getclass(classname).
411 '''
412 if self.classes.has_key(classname):
413 if __debug__:
414 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
415 return self.classes[classname]
416 raise AttributeError, classname
418 def addclass(self, cl):
419 ''' Add a Class to the hyperdatabase.
420 '''
421 if __debug__:
422 print >>hyperdb.DEBUG, 'addclass', (self, cl)
423 cn = cl.classname
424 if self.classes.has_key(cn):
425 raise ValueError, cn
426 self.classes[cn] = cl
428 def getclasses(self):
429 ''' Return a list of the names of all existing classes.
430 '''
431 if __debug__:
432 print >>hyperdb.DEBUG, 'getclasses', (self,)
433 l = self.classes.keys()
434 l.sort()
435 return l
437 def getclass(self, classname):
438 '''Get the Class object representing a particular class.
440 If 'classname' is not a valid class name, a KeyError is raised.
441 '''
442 if __debug__:
443 print >>hyperdb.DEBUG, 'getclass', (self, classname)
444 try:
445 return self.classes[classname]
446 except KeyError:
447 raise KeyError, 'There is no class called "%s"'%classname
449 def clear(self):
450 ''' Delete all database contents.
452 Note: I don't commit here, which is different behaviour to the
453 "nuke from orbit" behaviour in the *dbms.
454 '''
455 if __debug__:
456 print >>hyperdb.DEBUG, 'clear', (self,)
457 for cn in self.classes.keys():
458 sql = 'delete from _%s'%cn
459 if __debug__:
460 print >>hyperdb.DEBUG, 'clear', (self, sql)
461 self.cursor.execute(sql)
463 #
464 # Node IDs
465 #
466 def newid(self, classname):
467 ''' Generate a new id for the given class
468 '''
469 # get the next ID
470 sql = 'select num from ids where name=%s'%self.arg
471 if __debug__:
472 print >>hyperdb.DEBUG, 'newid', (self, sql, classname)
473 self.cursor.execute(sql, (classname, ))
474 newid = self.cursor.fetchone()[0]
476 # update the counter
477 sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
478 vals = (int(newid)+1, classname)
479 if __debug__:
480 print >>hyperdb.DEBUG, 'newid', (self, sql, vals)
481 self.cursor.execute(sql, vals)
483 # return as string
484 return str(newid)
486 def setid(self, classname, setid):
487 ''' Set the id counter: used during import of database
488 '''
489 sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
490 vals = (setid, classname)
491 if __debug__:
492 print >>hyperdb.DEBUG, 'setid', (self, sql, vals)
493 self.cursor.execute(sql, vals)
495 #
496 # Nodes
497 #
498 def addnode(self, classname, nodeid, node):
499 ''' Add the specified node to its class's db.
500 '''
501 if __debug__:
502 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
504 # determine the column definitions and multilink tables
505 cl = self.classes[classname]
506 cols, mls = self.determine_columns(cl.properties.items())
508 # we'll be supplied these props if we're doing an import
509 if not node.has_key('creator'):
510 # add in the "calculated" properties (dupe so we don't affect
511 # calling code's node assumptions)
512 node = node.copy()
513 node['creation'] = node['activity'] = date.Date()
514 node['creator'] = self.getuid()
516 # default the non-multilink columns
517 for col, prop in cl.properties.items():
518 if not node.has_key(col):
519 if isinstance(prop, Multilink):
520 node[col] = []
521 else:
522 node[col] = None
524 # clear this node out of the cache if it's in there
525 key = (classname, nodeid)
526 if self.cache.has_key(key):
527 del self.cache[key]
528 self.cache_lru.remove(key)
530 # make the node data safe for the DB
531 node = self.serialise(classname, node)
533 # make sure the ordering is correct for column name -> column value
534 vals = tuple([node[col[1:]] for col in cols]) + (nodeid, 0)
535 s = ','.join([self.arg for x in cols]) + ',%s,%s'%(self.arg, self.arg)
536 cols = ','.join(cols) + ',id,__retired__'
538 # perform the inserts
539 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
540 if __debug__:
541 print >>hyperdb.DEBUG, 'addnode', (self, sql, vals)
542 self.cursor.execute(sql, vals)
544 # insert the multilink rows
545 for col in mls:
546 t = '%s_%s'%(classname, col)
547 for entry in node[col]:
548 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
549 self.arg, self.arg)
550 self.sql(sql, (entry, nodeid))
552 # make sure we do the commit-time extra stuff for this node
553 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
555 def setnode(self, classname, nodeid, values, multilink_changes):
556 ''' Change the specified node.
557 '''
558 if __debug__:
559 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, values)
561 # clear this node out of the cache if it's in there
562 key = (classname, nodeid)
563 if self.cache.has_key(key):
564 del self.cache[key]
565 self.cache_lru.remove(key)
567 # add the special props
568 values = values.copy()
569 values['activity'] = date.Date()
571 # make db-friendly
572 values = self.serialise(classname, values)
574 cl = self.classes[classname]
575 cols = []
576 mls = []
577 # add the multilinks separately
578 props = cl.getprops()
579 for col in values.keys():
580 prop = props[col]
581 if isinstance(prop, Multilink):
582 mls.append(col)
583 else:
584 cols.append('_'+col)
585 cols.sort()
587 # if there's any updates to regular columns, do them
588 if cols:
589 # make sure the ordering is correct for column name -> column value
590 sqlvals = tuple([values[col[1:]] for col in cols]) + (nodeid,)
591 s = ','.join(['%s=%s'%(x, self.arg) for x in cols])
592 cols = ','.join(cols)
594 # perform the update
595 sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
596 if __debug__:
597 print >>hyperdb.DEBUG, 'setnode', (self, sql, sqlvals)
598 self.cursor.execute(sql, sqlvals)
600 # now the fun bit, updating the multilinks ;)
601 for col, (add, remove) in multilink_changes.items():
602 tn = '%s_%s'%(classname, col)
603 if add:
604 sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
605 self.arg, self.arg)
606 for addid in add:
607 self.sql(sql, (nodeid, addid))
608 if remove:
609 sql = 'delete from %s where nodeid=%s and linkid=%s'%(tn,
610 self.arg, self.arg)
611 for removeid in remove:
612 self.sql(sql, (nodeid, removeid))
614 # make sure we do the commit-time extra stuff for this node
615 self.transactions.append((self.doSaveNode, (classname, nodeid, values)))
617 def getnode(self, classname, nodeid):
618 ''' Get a node from the database.
619 '''
620 if __debug__:
621 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid)
623 # see if we have this node cached
624 key = (classname, nodeid)
625 if self.cache.has_key(key):
626 # push us back to the top of the LRU
627 self.cache_lru.remove(key)
628 self.cache_lru.insert(0, key)
629 # return the cached information
630 return self.cache[key]
632 # figure the columns we're fetching
633 cl = self.classes[classname]
634 cols, mls = self.determine_columns(cl.properties.items())
635 scols = ','.join(cols)
637 # perform the basic property fetch
638 sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
639 self.sql(sql, (nodeid,))
641 values = self.sql_fetchone()
642 if values is None:
643 raise IndexError, 'no such %s node %s'%(classname, nodeid)
645 # make up the node
646 node = {}
647 for col in range(len(cols)):
648 node[cols[col][1:]] = values[col]
650 # now the multilinks
651 for col in mls:
652 # get the link ids
653 sql = 'select linkid from %s_%s where nodeid=%s'%(classname, col,
654 self.arg)
655 self.cursor.execute(sql, (nodeid,))
656 # extract the first column from the result
657 node[col] = [x[0] for x in self.cursor.fetchall()]
659 # un-dbificate the node data
660 node = self.unserialise(classname, node)
662 # save off in the cache
663 key = (classname, nodeid)
664 self.cache[key] = node
665 # update the LRU
666 self.cache_lru.insert(0, key)
667 if len(self.cache_lru) > ROW_CACHE_SIZE:
668 del self.cache[self.cache_lru.pop()]
670 return node
672 def destroynode(self, classname, nodeid):
673 '''Remove a node from the database. Called exclusively by the
674 destroy() method on Class.
675 '''
676 if __debug__:
677 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
679 # make sure the node exists
680 if not self.hasnode(classname, nodeid):
681 raise IndexError, '%s has no node %s'%(classname, nodeid)
683 # see if we have this node cached
684 if self.cache.has_key((classname, nodeid)):
685 del self.cache[(classname, nodeid)]
687 # see if there's any obvious commit actions that we should get rid of
688 for entry in self.transactions[:]:
689 if entry[1][:2] == (classname, nodeid):
690 self.transactions.remove(entry)
692 # now do the SQL
693 sql = 'delete from _%s where id=%s'%(classname, self.arg)
694 self.sql(sql, (nodeid,))
696 # remove from multilnks
697 cl = self.getclass(classname)
698 x, mls = self.determine_columns(cl.properties.items())
699 for col in mls:
700 # get the link ids
701 sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
702 self.cursor.execute(sql, (nodeid,))
704 # remove journal entries
705 sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
706 self.sql(sql, (nodeid,))
708 def serialise(self, classname, node):
709 '''Copy the node contents, converting non-marshallable data into
710 marshallable data.
711 '''
712 if __debug__:
713 print >>hyperdb.DEBUG, 'serialise', classname, node
714 properties = self.getclass(classname).getprops()
715 d = {}
716 for k, v in node.items():
717 # if the property doesn't exist, or is the "retired" flag then
718 # it won't be in the properties dict
719 if not properties.has_key(k):
720 d[k] = v
721 continue
723 # get the property spec
724 prop = properties[k]
726 if isinstance(prop, Password) and v is not None:
727 d[k] = str(v)
728 elif isinstance(prop, Date) and v is not None:
729 d[k] = v.serialise()
730 elif isinstance(prop, Interval) and v is not None:
731 d[k] = v.serialise()
732 else:
733 d[k] = v
734 return d
736 def unserialise(self, classname, node):
737 '''Decode the marshalled node data
738 '''
739 if __debug__:
740 print >>hyperdb.DEBUG, 'unserialise', classname, node
741 properties = self.getclass(classname).getprops()
742 d = {}
743 for k, v in node.items():
744 # if the property doesn't exist, or is the "retired" flag then
745 # it won't be in the properties dict
746 if not properties.has_key(k):
747 d[k] = v
748 continue
750 # get the property spec
751 prop = properties[k]
753 if isinstance(prop, Date) and v is not None:
754 d[k] = date.Date(v)
755 elif isinstance(prop, Interval) and v is not None:
756 d[k] = date.Interval(v)
757 elif isinstance(prop, Password) and v is not None:
758 p = password.Password()
759 p.unpack(v)
760 d[k] = p
761 elif (isinstance(prop, Boolean) or isinstance(prop, Number)) and v is not None:
762 d[k]=float(v)
763 else:
764 d[k] = v
765 return d
767 def hasnode(self, classname, nodeid):
768 ''' Determine if the database has a given node.
769 '''
770 sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
771 if __debug__:
772 print >>hyperdb.DEBUG, 'hasnode', (self, sql, nodeid)
773 self.cursor.execute(sql, (nodeid,))
774 return int(self.cursor.fetchone()[0])
776 def countnodes(self, classname):
777 ''' Count the number of nodes that exist for a particular Class.
778 '''
779 sql = 'select count(*) from _%s'%classname
780 if __debug__:
781 print >>hyperdb.DEBUG, 'countnodes', (self, sql)
782 self.cursor.execute(sql)
783 return self.cursor.fetchone()[0]
785 def addjournal(self, classname, nodeid, action, params, creator=None,
786 creation=None):
787 ''' Journal the Action
788 'action' may be:
790 'create' or 'set' -- 'params' is a dictionary of property values
791 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
792 'retire' -- 'params' is None
793 '''
794 # serialise the parameters now if necessary
795 if isinstance(params, type({})):
796 if action in ('set', 'create'):
797 params = self.serialise(classname, params)
799 # handle supply of the special journalling parameters (usually
800 # supplied on importing an existing database)
801 if creator:
802 journaltag = creator
803 else:
804 journaltag = self.getuid()
805 if creation:
806 journaldate = creation.serialise()
807 else:
808 journaldate = date.Date().serialise()
810 # create the journal entry
811 cols = ','.join('nodeid date tag action params'.split())
813 if __debug__:
814 print >>hyperdb.DEBUG, 'addjournal', (nodeid, journaldate,
815 journaltag, action, params)
817 self.save_journal(classname, cols, nodeid, journaldate,
818 journaltag, action, params)
820 def save_journal(self, classname, cols, nodeid, journaldate,
821 journaltag, action, params):
822 ''' Save the journal entry to the database
823 '''
824 raise NotImplemented
826 def getjournal(self, classname, nodeid):
827 ''' get the journal for id
828 '''
829 # make sure the node exists
830 if not self.hasnode(classname, nodeid):
831 raise IndexError, '%s has no node %s'%(classname, nodeid)
833 cols = ','.join('nodeid date tag action params'.split())
834 return self.load_journal(classname, cols, nodeid)
836 def load_journal(self, classname, cols, nodeid):
837 ''' Load the journal from the database
838 '''
839 raise NotImplemented
841 def pack(self, pack_before):
842 ''' Delete all journal entries except "create" before 'pack_before'.
843 '''
844 # get a 'yyyymmddhhmmss' version of the date
845 date_stamp = pack_before.serialise()
847 # do the delete
848 for classname in self.classes.keys():
849 sql = "delete from %s__journal where date<%s and "\
850 "action<>'create'"%(classname, self.arg)
851 if __debug__:
852 print >>hyperdb.DEBUG, 'pack', (self, sql, date_stamp)
853 self.cursor.execute(sql, (date_stamp,))
855 def sql_commit(self):
856 ''' Actually commit to the database.
857 '''
858 self.conn.commit()
860 def commit(self):
861 ''' Commit the current transactions.
863 Save all data changed since the database was opened or since the
864 last commit() or rollback().
865 '''
866 if __debug__:
867 print >>hyperdb.DEBUG, 'commit', (self,)
869 # commit the database
870 self.sql_commit()
872 # now, do all the other transaction stuff
873 reindex = {}
874 for method, args in self.transactions:
875 reindex[method(*args)] = 1
877 # reindex the nodes that request it
878 for classname, nodeid in filter(None, reindex.keys()):
879 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
880 self.getclass(classname).index(nodeid)
882 # save the indexer state
883 self.indexer.save_index()
885 # clear out the transactions
886 self.transactions = []
888 def rollback(self):
889 ''' Reverse all actions from the current transaction.
891 Undo all the changes made since the database was opened or the last
892 commit() or rollback() was performed.
893 '''
894 if __debug__:
895 print >>hyperdb.DEBUG, 'rollback', (self,)
897 # roll back
898 self.conn.rollback()
900 # roll back "other" transaction stuff
901 for method, args in self.transactions:
902 # delete temporary files
903 if method == self.doStoreFile:
904 self.rollbackStoreFile(*args)
905 self.transactions = []
907 # clear the cache
908 self.clearCache()
910 def doSaveNode(self, classname, nodeid, node):
911 ''' dummy that just generates a reindex event
912 '''
913 # return the classname, nodeid so we reindex this content
914 return (classname, nodeid)
916 def close(self):
917 ''' Close off the connection.
918 '''
919 self.conn.close()
920 if self.lockfile is not None:
921 locking.release_lock(self.lockfile)
922 if self.lockfile is not None:
923 self.lockfile.close()
924 self.lockfile = None
926 #
927 # The base Class class
928 #
929 class Class(hyperdb.Class):
930 ''' The handle to a particular class of nodes in a hyperdatabase.
932 All methods except __repr__ and getnode must be implemented by a
933 concrete backend Class.
934 '''
936 def __init__(self, db, classname, **properties):
937 '''Create a new class with a given name and property specification.
939 'classname' must not collide with the name of an existing class,
940 or a ValueError is raised. The keyword arguments in 'properties'
941 must map names to property objects, or a TypeError is raised.
942 '''
943 if (properties.has_key('creation') or properties.has_key('activity')
944 or properties.has_key('creator')):
945 raise ValueError, '"creation", "activity" and "creator" are '\
946 'reserved'
948 self.classname = classname
949 self.properties = properties
950 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
951 self.key = ''
953 # should we journal changes (default yes)
954 self.do_journal = 1
956 # do the db-related init stuff
957 db.addclass(self)
959 self.auditors = {'create': [], 'set': [], 'retire': [], 'restore': []}
960 self.reactors = {'create': [], 'set': [], 'retire': [], 'restore': []}
962 def schema(self):
963 ''' A dumpable version of the schema that we can store in the
964 database
965 '''
966 return (self.key, [(x, repr(y)) for x,y in self.properties.items()])
968 def enableJournalling(self):
969 '''Turn journalling on for this class
970 '''
971 self.do_journal = 1
973 def disableJournalling(self):
974 '''Turn journalling off for this class
975 '''
976 self.do_journal = 0
978 # Editing nodes:
979 def create(self, **propvalues):
980 ''' Create a new node of this class and return its id.
982 The keyword arguments in 'propvalues' map property names to values.
984 The values of arguments must be acceptable for the types of their
985 corresponding properties or a TypeError is raised.
987 If this class has a key property, it must be present and its value
988 must not collide with other key strings or a ValueError is raised.
990 Any other properties on this class that are missing from the
991 'propvalues' dictionary are set to None.
993 If an id in a link or multilink property does not refer to a valid
994 node, an IndexError is raised.
995 '''
996 self.fireAuditors('create', None, propvalues)
997 newid = self.create_inner(**propvalues)
998 self.fireReactors('create', newid, None)
999 return newid
1001 def create_inner(self, **propvalues):
1002 ''' Called by create, in-between the audit and react calls.
1003 '''
1004 if propvalues.has_key('id'):
1005 raise KeyError, '"id" is reserved'
1007 if self.db.journaltag is None:
1008 raise DatabaseError, 'Database open read-only'
1010 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1011 raise KeyError, '"creation" and "activity" are reserved'
1013 # new node's id
1014 newid = self.db.newid(self.classname)
1016 # validate propvalues
1017 num_re = re.compile('^\d+$')
1018 for key, value in propvalues.items():
1019 if key == self.key:
1020 try:
1021 self.lookup(value)
1022 except KeyError:
1023 pass
1024 else:
1025 raise ValueError, 'node with key "%s" exists'%value
1027 # try to handle this property
1028 try:
1029 prop = self.properties[key]
1030 except KeyError:
1031 raise KeyError, '"%s" has no property "%s"'%(self.classname,
1032 key)
1034 if value is not None and isinstance(prop, Link):
1035 if type(value) != type(''):
1036 raise ValueError, 'link value must be String'
1037 link_class = self.properties[key].classname
1038 # if it isn't a number, it's a key
1039 if not num_re.match(value):
1040 try:
1041 value = self.db.classes[link_class].lookup(value)
1042 except (TypeError, KeyError):
1043 raise IndexError, 'new property "%s": %s not a %s'%(
1044 key, value, link_class)
1045 elif not self.db.getclass(link_class).hasnode(value):
1046 raise IndexError, '%s has no node %s'%(link_class, value)
1048 # save off the value
1049 propvalues[key] = value
1051 # register the link with the newly linked node
1052 if self.do_journal and self.properties[key].do_journal:
1053 self.db.addjournal(link_class, value, 'link',
1054 (self.classname, newid, key))
1056 elif isinstance(prop, Multilink):
1057 if type(value) != type([]):
1058 raise TypeError, 'new property "%s" not a list of ids'%key
1060 # clean up and validate the list of links
1061 link_class = self.properties[key].classname
1062 l = []
1063 for entry in value:
1064 if type(entry) != type(''):
1065 raise ValueError, '"%s" multilink value (%r) '\
1066 'must contain Strings'%(key, value)
1067 # if it isn't a number, it's a key
1068 if not num_re.match(entry):
1069 try:
1070 entry = self.db.classes[link_class].lookup(entry)
1071 except (TypeError, KeyError):
1072 raise IndexError, 'new property "%s": %s not a %s'%(
1073 key, entry, self.properties[key].classname)
1074 l.append(entry)
1075 value = l
1076 propvalues[key] = value
1078 # handle additions
1079 for nodeid in value:
1080 if not self.db.getclass(link_class).hasnode(nodeid):
1081 raise IndexError, '%s has no node %s'%(link_class,
1082 nodeid)
1083 # register the link with the newly linked node
1084 if self.do_journal and self.properties[key].do_journal:
1085 self.db.addjournal(link_class, nodeid, 'link',
1086 (self.classname, newid, key))
1088 elif isinstance(prop, String):
1089 if type(value) != type('') and type(value) != type(u''):
1090 raise TypeError, 'new property "%s" not a string'%key
1092 elif isinstance(prop, Password):
1093 if not isinstance(value, password.Password):
1094 raise TypeError, 'new property "%s" not a Password'%key
1096 elif isinstance(prop, Date):
1097 if value is not None and not isinstance(value, date.Date):
1098 raise TypeError, 'new property "%s" not a Date'%key
1100 elif isinstance(prop, Interval):
1101 if value is not None and not isinstance(value, date.Interval):
1102 raise TypeError, 'new property "%s" not an Interval'%key
1104 elif value is not None and isinstance(prop, Number):
1105 try:
1106 float(value)
1107 except ValueError:
1108 raise TypeError, 'new property "%s" not numeric'%key
1110 elif value is not None and isinstance(prop, Boolean):
1111 try:
1112 int(value)
1113 except ValueError:
1114 raise TypeError, 'new property "%s" not boolean'%key
1116 # make sure there's data where there needs to be
1117 for key, prop in self.properties.items():
1118 if propvalues.has_key(key):
1119 continue
1120 if key == self.key:
1121 raise ValueError, 'key property "%s" is required'%key
1122 if isinstance(prop, Multilink):
1123 propvalues[key] = []
1124 else:
1125 propvalues[key] = None
1127 # done
1128 self.db.addnode(self.classname, newid, propvalues)
1129 if self.do_journal:
1130 self.db.addjournal(self.classname, newid, 'create', {})
1132 return newid
1134 def export_list(self, propnames, nodeid):
1135 ''' Export a node - generate a list of CSV-able data in the order
1136 specified by propnames for the given node.
1137 '''
1138 properties = self.getprops()
1139 l = []
1140 for prop in propnames:
1141 proptype = properties[prop]
1142 value = self.get(nodeid, prop)
1143 # "marshal" data where needed
1144 if value is None:
1145 pass
1146 elif isinstance(proptype, hyperdb.Date):
1147 value = value.get_tuple()
1148 elif isinstance(proptype, hyperdb.Interval):
1149 value = value.get_tuple()
1150 elif isinstance(proptype, hyperdb.Password):
1151 value = str(value)
1152 l.append(repr(value))
1153 l.append(self.is_retired(nodeid))
1154 return l
1156 def import_list(self, propnames, proplist):
1157 ''' Import a node - all information including "id" is present and
1158 should not be sanity checked. Triggers are not triggered. The
1159 journal should be initialised using the "creator" and "created"
1160 information.
1162 Return the nodeid of the node imported.
1163 '''
1164 if self.db.journaltag is None:
1165 raise DatabaseError, 'Database open read-only'
1166 properties = self.getprops()
1168 # make the new node's property map
1169 d = {}
1170 retire = 0
1171 newid = None
1172 for i in range(len(propnames)):
1173 # Use eval to reverse the repr() used to output the CSV
1174 value = eval(proplist[i])
1176 # Figure the property for this column
1177 propname = propnames[i]
1179 # "unmarshal" where necessary
1180 if propname == 'id':
1181 newid = value
1182 continue
1183 elif propname == 'is retired':
1184 # is the item retired?
1185 if int(value):
1186 retire = 1
1187 continue
1188 elif value is None:
1189 d[propname] = None
1190 continue
1192 prop = properties[propname]
1193 if value is None:
1194 # don't set Nones
1195 continue
1196 elif isinstance(prop, hyperdb.Date):
1197 value = date.Date(value)
1198 elif isinstance(prop, hyperdb.Interval):
1199 value = date.Interval(value)
1200 elif isinstance(prop, hyperdb.Password):
1201 pwd = password.Password()
1202 pwd.unpack(value)
1203 value = pwd
1204 d[propname] = value
1206 # get a new id if necessary
1207 if newid is None:
1208 newid = self.db.newid(self.classname)
1210 # retire?
1211 if retire:
1212 # use the arg for __retired__ to cope with any odd database type
1213 # conversion (hello, sqlite)
1214 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1215 self.db.arg, self.db.arg)
1216 if __debug__:
1217 print >>hyperdb.DEBUG, 'retire', (self, sql, newid)
1218 self.db.cursor.execute(sql, (1, newid))
1220 # add the node and journal
1221 self.db.addnode(self.classname, newid, d)
1223 # extract the extraneous journalling gumpf and nuke it
1224 if d.has_key('creator'):
1225 creator = d['creator']
1226 del d['creator']
1227 else:
1228 creator = None
1229 if d.has_key('creation'):
1230 creation = d['creation']
1231 del d['creation']
1232 else:
1233 creation = None
1234 if d.has_key('activity'):
1235 del d['activity']
1236 self.db.addjournal(self.classname, newid, 'create', {}, creator,
1237 creation)
1238 return newid
1240 _marker = []
1241 def get(self, nodeid, propname, default=_marker, cache=1):
1242 '''Get the value of a property on an existing node of this class.
1244 'nodeid' must be the id of an existing node of this class or an
1245 IndexError is raised. 'propname' must be the name of a property
1246 of this class or a KeyError is raised.
1248 'cache' exists for backwards compatibility, and is not used.
1249 '''
1250 if propname == 'id':
1251 return nodeid
1253 # get the node's dict
1254 d = self.db.getnode(self.classname, nodeid)
1256 if propname == 'creation':
1257 if d.has_key('creation'):
1258 return d['creation']
1259 else:
1260 return date.Date()
1261 if propname == 'activity':
1262 if d.has_key('activity'):
1263 return d['activity']
1264 else:
1265 return date.Date()
1266 if propname == 'creator':
1267 if d.has_key('creator'):
1268 return d['creator']
1269 else:
1270 return self.db.getuid()
1272 # get the property (raises KeyErorr if invalid)
1273 prop = self.properties[propname]
1275 if not d.has_key(propname):
1276 if default is self._marker:
1277 if isinstance(prop, Multilink):
1278 return []
1279 else:
1280 return None
1281 else:
1282 return default
1284 # don't pass our list to other code
1285 if isinstance(prop, Multilink):
1286 return d[propname][:]
1288 return d[propname]
1290 def getnode(self, nodeid, cache=1):
1291 ''' Return a convenience wrapper for the node.
1293 'nodeid' must be the id of an existing node of this class or an
1294 IndexError is raised.
1296 'cache' exists for backwards compatibility, and is not used.
1297 '''
1298 return Node(self, nodeid)
1300 def set(self, nodeid, **propvalues):
1301 '''Modify a property on an existing node of this class.
1303 'nodeid' must be the id of an existing node of this class or an
1304 IndexError is raised.
1306 Each key in 'propvalues' must be the name of a property of this
1307 class or a KeyError is raised.
1309 All values in 'propvalues' must be acceptable types for their
1310 corresponding properties or a TypeError is raised.
1312 If the value of the key property is set, it must not collide with
1313 other key strings or a ValueError is raised.
1315 If the value of a Link or Multilink property contains an invalid
1316 node id, a ValueError is raised.
1317 '''
1318 if not propvalues:
1319 return propvalues
1321 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1322 raise KeyError, '"creation" and "activity" are reserved'
1324 if propvalues.has_key('id'):
1325 raise KeyError, '"id" is reserved'
1327 if self.db.journaltag is None:
1328 raise DatabaseError, 'Database open read-only'
1330 self.fireAuditors('set', nodeid, propvalues)
1331 # Take a copy of the node dict so that the subsequent set
1332 # operation doesn't modify the oldvalues structure.
1333 # XXX used to try the cache here first
1334 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1336 node = self.db.getnode(self.classname, nodeid)
1337 if self.is_retired(nodeid):
1338 raise IndexError, 'Requested item is retired'
1339 num_re = re.compile('^\d+$')
1341 # if the journal value is to be different, store it in here
1342 journalvalues = {}
1344 # remember the add/remove stuff for multilinks, making it easier
1345 # for the Database layer to do its stuff
1346 multilink_changes = {}
1348 for propname, value in propvalues.items():
1349 # check to make sure we're not duplicating an existing key
1350 if propname == self.key and node[propname] != value:
1351 try:
1352 self.lookup(value)
1353 except KeyError:
1354 pass
1355 else:
1356 raise ValueError, 'node with key "%s" exists'%value
1358 # this will raise the KeyError if the property isn't valid
1359 # ... we don't use getprops() here because we only care about
1360 # the writeable properties.
1361 try:
1362 prop = self.properties[propname]
1363 except KeyError:
1364 raise KeyError, '"%s" has no property named "%s"'%(
1365 self.classname, propname)
1367 # if the value's the same as the existing value, no sense in
1368 # doing anything
1369 current = node.get(propname, None)
1370 if value == current:
1371 del propvalues[propname]
1372 continue
1373 journalvalues[propname] = current
1375 # do stuff based on the prop type
1376 if isinstance(prop, Link):
1377 link_class = prop.classname
1378 # if it isn't a number, it's a key
1379 if value is not None and not isinstance(value, type('')):
1380 raise ValueError, 'property "%s" link value be a string'%(
1381 propname)
1382 if isinstance(value, type('')) and not num_re.match(value):
1383 try:
1384 value = self.db.classes[link_class].lookup(value)
1385 except (TypeError, KeyError):
1386 raise IndexError, 'new property "%s": %s not a %s'%(
1387 propname, value, prop.classname)
1389 if (value is not None and
1390 not self.db.getclass(link_class).hasnode(value)):
1391 raise IndexError, '%s has no node %s'%(link_class, value)
1393 if self.do_journal and prop.do_journal:
1394 # register the unlink with the old linked node
1395 if node[propname] is not None:
1396 self.db.addjournal(link_class, node[propname], 'unlink',
1397 (self.classname, nodeid, propname))
1399 # register the link with the newly linked node
1400 if value is not None:
1401 self.db.addjournal(link_class, value, 'link',
1402 (self.classname, nodeid, propname))
1404 elif isinstance(prop, Multilink):
1405 if type(value) != type([]):
1406 raise TypeError, 'new property "%s" not a list of'\
1407 ' ids'%propname
1408 link_class = self.properties[propname].classname
1409 l = []
1410 for entry in value:
1411 # if it isn't a number, it's a key
1412 if type(entry) != type(''):
1413 raise ValueError, 'new property "%s" link value ' \
1414 'must be a string'%propname
1415 if not num_re.match(entry):
1416 try:
1417 entry = self.db.classes[link_class].lookup(entry)
1418 except (TypeError, KeyError):
1419 raise IndexError, 'new property "%s": %s not a %s'%(
1420 propname, entry,
1421 self.properties[propname].classname)
1422 l.append(entry)
1423 value = l
1424 propvalues[propname] = value
1426 # figure the journal entry for this property
1427 add = []
1428 remove = []
1430 # handle removals
1431 if node.has_key(propname):
1432 l = node[propname]
1433 else:
1434 l = []
1435 for id in l[:]:
1436 if id in value:
1437 continue
1438 # register the unlink with the old linked node
1439 if self.do_journal and self.properties[propname].do_journal:
1440 self.db.addjournal(link_class, id, 'unlink',
1441 (self.classname, nodeid, propname))
1442 l.remove(id)
1443 remove.append(id)
1445 # handle additions
1446 for id in value:
1447 if not self.db.getclass(link_class).hasnode(id):
1448 raise IndexError, '%s has no node %s'%(link_class, id)
1449 if id in l:
1450 continue
1451 # register the link with the newly linked node
1452 if self.do_journal and self.properties[propname].do_journal:
1453 self.db.addjournal(link_class, id, 'link',
1454 (self.classname, nodeid, propname))
1455 l.append(id)
1456 add.append(id)
1458 # figure the journal entry
1459 l = []
1460 if add:
1461 l.append(('+', add))
1462 if remove:
1463 l.append(('-', remove))
1464 multilink_changes[propname] = (add, remove)
1465 if l:
1466 journalvalues[propname] = tuple(l)
1468 elif isinstance(prop, String):
1469 if value is not None and type(value) != type('') and type(value) != type(u''):
1470 raise TypeError, 'new property "%s" not a string'%propname
1472 elif isinstance(prop, Password):
1473 if not isinstance(value, password.Password):
1474 raise TypeError, 'new property "%s" not a Password'%propname
1475 propvalues[propname] = value
1477 elif value is not None and isinstance(prop, Date):
1478 if not isinstance(value, date.Date):
1479 raise TypeError, 'new property "%s" not a Date'% propname
1480 propvalues[propname] = value
1482 elif value is not None and isinstance(prop, Interval):
1483 if not isinstance(value, date.Interval):
1484 raise TypeError, 'new property "%s" not an '\
1485 'Interval'%propname
1486 propvalues[propname] = value
1488 elif value is not None and isinstance(prop, Number):
1489 try:
1490 float(value)
1491 except ValueError:
1492 raise TypeError, 'new property "%s" not numeric'%propname
1494 elif value is not None and isinstance(prop, Boolean):
1495 try:
1496 int(value)
1497 except ValueError:
1498 raise TypeError, 'new property "%s" not boolean'%propname
1500 # nothing to do?
1501 if not propvalues:
1502 return propvalues
1504 # do the set, and journal it
1505 self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
1507 if self.do_journal:
1508 self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
1510 self.fireReactors('set', nodeid, oldvalues)
1512 return propvalues
1514 def retire(self, nodeid):
1515 '''Retire a node.
1517 The properties on the node remain available from the get() method,
1518 and the node's id is never reused.
1520 Retired nodes are not returned by the find(), list(), or lookup()
1521 methods, and other nodes may reuse the values of their key properties.
1522 '''
1523 if self.db.journaltag is None:
1524 raise DatabaseError, 'Database open read-only'
1526 self.fireAuditors('retire', nodeid, None)
1528 # use the arg for __retired__ to cope with any odd database type
1529 # conversion (hello, sqlite)
1530 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1531 self.db.arg, self.db.arg)
1532 if __debug__:
1533 print >>hyperdb.DEBUG, 'retire', (self, sql, nodeid)
1534 self.db.cursor.execute(sql, (1, nodeid))
1535 if self.do_journal:
1536 self.db.addjournal(self.classname, nodeid, 'retired', None)
1538 self.fireReactors('retire', nodeid, None)
1540 def restore(self, nodeid):
1541 '''Restore a retired node.
1543 Make node available for all operations like it was before retirement.
1544 '''
1545 if self.db.journaltag is None:
1546 raise DatabaseError, 'Database open read-only'
1548 node = self.db.getnode(self.classname, nodeid)
1549 # check if key property was overrided
1550 key = self.getkey()
1551 try:
1552 id = self.lookup(node[key])
1553 except KeyError:
1554 pass
1555 else:
1556 raise KeyError, "Key property (%s) of retired node clashes with \
1557 existing one (%s)" % (key, node[key])
1559 self.fireAuditors('restore', nodeid, None)
1560 # use the arg for __retired__ to cope with any odd database type
1561 # conversion (hello, sqlite)
1562 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1563 self.db.arg, self.db.arg)
1564 if __debug__:
1565 print >>hyperdb.DEBUG, 'restore', (self, sql, nodeid)
1566 self.db.cursor.execute(sql, (0, nodeid))
1567 if self.do_journal:
1568 self.db.addjournal(self.classname, nodeid, 'restored', None)
1570 self.fireReactors('restore', nodeid, None)
1572 def is_retired(self, nodeid):
1573 '''Return true if the node is rerired
1574 '''
1575 sql = 'select __retired__ from _%s where id=%s'%(self.classname,
1576 self.db.arg)
1577 if __debug__:
1578 print >>hyperdb.DEBUG, 'is_retired', (self, sql, nodeid)
1579 self.db.cursor.execute(sql, (nodeid,))
1580 return int(self.db.sql_fetchone()[0])
1582 def destroy(self, nodeid):
1583 '''Destroy a node.
1585 WARNING: this method should never be used except in extremely rare
1586 situations where there could never be links to the node being
1587 deleted
1588 WARNING: use retire() instead
1589 WARNING: the properties of this node will not be available ever again
1590 WARNING: really, use retire() instead
1592 Well, I think that's enough warnings. This method exists mostly to
1593 support the session storage of the cgi interface.
1595 The node is completely removed from the hyperdb, including all journal
1596 entries. It will no longer be available, and will generally break code
1597 if there are any references to the node.
1598 '''
1599 if self.db.journaltag is None:
1600 raise DatabaseError, 'Database open read-only'
1601 self.db.destroynode(self.classname, nodeid)
1603 def history(self, nodeid):
1604 '''Retrieve the journal of edits on a particular node.
1606 'nodeid' must be the id of an existing node of this class or an
1607 IndexError is raised.
1609 The returned list contains tuples of the form
1611 (nodeid, date, tag, action, params)
1613 'date' is a Timestamp object specifying the time of the change and
1614 'tag' is the journaltag specified when the database was opened.
1615 '''
1616 if not self.do_journal:
1617 raise ValueError, 'Journalling is disabled for this class'
1618 return self.db.getjournal(self.classname, nodeid)
1620 # Locating nodes:
1621 def hasnode(self, nodeid):
1622 '''Determine if the given nodeid actually exists
1623 '''
1624 return self.db.hasnode(self.classname, nodeid)
1626 def setkey(self, propname):
1627 '''Select a String property of this class to be the key property.
1629 'propname' must be the name of a String property of this class or
1630 None, or a TypeError is raised. The values of the key property on
1631 all existing nodes must be unique or a ValueError is raised.
1632 '''
1633 # XXX create an index on the key prop column. We should also
1634 # record that we've created this index in the schema somewhere.
1635 prop = self.getprops()[propname]
1636 if not isinstance(prop, String):
1637 raise TypeError, 'key properties must be String'
1638 self.key = propname
1640 def getkey(self):
1641 '''Return the name of the key property for this class or None.'''
1642 return self.key
1644 def labelprop(self, default_to_id=0):
1645 ''' Return the property name for a label for the given node.
1647 This method attempts to generate a consistent label for the node.
1648 It tries the following in order:
1649 1. key property
1650 2. "name" property
1651 3. "title" property
1652 4. first property from the sorted property name list
1653 '''
1654 k = self.getkey()
1655 if k:
1656 return k
1657 props = self.getprops()
1658 if props.has_key('name'):
1659 return 'name'
1660 elif props.has_key('title'):
1661 return 'title'
1662 if default_to_id:
1663 return 'id'
1664 props = props.keys()
1665 props.sort()
1666 return props[0]
1668 def lookup(self, keyvalue):
1669 '''Locate a particular node by its key property and return its id.
1671 If this class has no key property, a TypeError is raised. If the
1672 'keyvalue' matches one of the values for the key property among
1673 the nodes in this class, the matching node's id is returned;
1674 otherwise a KeyError is raised.
1675 '''
1676 if not self.key:
1677 raise TypeError, 'No key property set for class %s'%self.classname
1679 # use the arg to handle any odd database type conversion (hello,
1680 # sqlite)
1681 sql = "select id from _%s where _%s=%s and __retired__ <> %s"%(
1682 self.classname, self.key, self.db.arg, self.db.arg)
1683 self.db.sql(sql, (keyvalue, 1))
1685 # see if there was a result that's not retired
1686 row = self.db.sql_fetchone()
1687 if not row:
1688 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1689 keyvalue, self.classname)
1691 # return the id
1692 return row[0]
1694 def find(self, **propspec):
1695 '''Get the ids of nodes in this class which link to the given nodes.
1697 'propspec' consists of keyword args propname=nodeid or
1698 propname={nodeid:1, }
1699 'propname' must be the name of a property in this class, or a
1700 KeyError is raised. That property must be a Link or Multilink
1701 property, or a TypeError is raised.
1703 Any node in this class whose 'propname' property links to any of the
1704 nodeids will be returned. Used by the full text indexing, which knows
1705 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1706 issues:
1708 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1709 '''
1710 if __debug__:
1711 print >>hyperdb.DEBUG, 'find', (self, propspec)
1713 # shortcut
1714 if not propspec:
1715 return []
1717 # validate the args
1718 props = self.getprops()
1719 propspec = propspec.items()
1720 for propname, nodeids in propspec:
1721 # check the prop is OK
1722 prop = props[propname]
1723 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1724 raise TypeError, "'%s' not a Link/Multilink property"%propname
1726 # first, links
1727 where = []
1728 allvalues = ()
1729 a = self.db.arg
1730 for prop, values in propspec:
1731 if not isinstance(props[prop], hyperdb.Link):
1732 continue
1733 if type(values) is type(''):
1734 allvalues += (values,)
1735 where.append('_%s = %s'%(prop, a))
1736 elif values is None:
1737 where.append('_%s is NULL'%prop)
1738 else:
1739 allvalues += tuple(values.keys())
1740 where.append('_%s in (%s)'%(prop, ','.join([a]*len(values))))
1741 tables = []
1742 if where:
1743 tables.append('select id as nodeid from _%s where %s'%(
1744 self.classname, ' and '.join(where)))
1746 # now multilinks
1747 for prop, values in propspec:
1748 if not isinstance(props[prop], hyperdb.Multilink):
1749 continue
1750 if type(values) is type(''):
1751 allvalues += (values,)
1752 s = a
1753 else:
1754 allvalues += tuple(values.keys())
1755 s = ','.join([a]*len(values))
1756 tables.append('select nodeid from %s_%s where linkid in (%s)'%(
1757 self.classname, prop, s))
1758 sql = '\nunion\n'.join(tables)
1759 self.db.sql(sql, allvalues)
1760 l = [x[0] for x in self.db.sql_fetchall()]
1761 if __debug__:
1762 print >>hyperdb.DEBUG, 'find ... ', l
1763 return l
1765 def stringFind(self, **requirements):
1766 '''Locate a particular node by matching a set of its String
1767 properties in a caseless search.
1769 If the property is not a String property, a TypeError is raised.
1771 The return is a list of the id of all nodes that match.
1772 '''
1773 where = []
1774 args = []
1775 for propname in requirements.keys():
1776 prop = self.properties[propname]
1777 if isinstance(not prop, String):
1778 raise TypeError, "'%s' not a String property"%propname
1779 where.append(propname)
1780 args.append(requirements[propname].lower())
1782 # generate the where clause
1783 s = ' and '.join(['lower(_%s)=%s'%(col, self.db.arg) for col in where])
1784 sql = 'select id from _%s where %s'%(self.classname, s)
1785 self.db.sql(sql, tuple(args))
1786 l = [x[0] for x in self.db.sql_fetchall()]
1787 if __debug__:
1788 print >>hyperdb.DEBUG, 'find ... ', l
1789 return l
1791 def list(self):
1792 ''' Return a list of the ids of the active nodes in this class.
1793 '''
1794 return self.getnodeids(retired=0)
1796 def getnodeids(self, retired=None):
1797 ''' Retrieve all the ids of the nodes for a particular Class.
1799 Set retired=None to get all nodes. Otherwise it'll get all the
1800 retired or non-retired nodes, depending on the flag.
1801 '''
1802 # flip the sense of the 'retired' flag if we don't want all of them
1803 if retired is not None:
1804 if retired:
1805 args = (0, )
1806 else:
1807 args = (1, )
1808 sql = 'select id from _%s where __retired__ <> %s'%(self.classname,
1809 self.db.arg)
1810 else:
1811 args = ()
1812 sql = 'select id from _%s'%self.classname
1813 if __debug__:
1814 print >>hyperdb.DEBUG, 'getnodeids', (self, sql, retired)
1815 self.db.cursor.execute(sql, args)
1816 ids = [x[0] for x in self.db.cursor.fetchall()]
1817 return ids
1819 def filter(self, search_matches, filterspec, sort=(None,None),
1820 group=(None,None)):
1821 ''' Return a list of the ids of the active nodes in this class that
1822 match the 'filter' spec, sorted by the group spec and then the
1823 sort spec
1825 "filterspec" is {propname: value(s)}
1826 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1827 and prop is a prop name or None
1828 "search_matches" is {nodeid: marker}
1830 The filter must match all properties specificed - but if the
1831 property value to match is a list, any one of the values in the
1832 list may match for that property to match.
1833 '''
1834 # just don't bother if the full-text search matched diddly
1835 if search_matches == {}:
1836 return []
1838 cn = self.classname
1840 timezone = self.db.getUserTimezone()
1842 # figure the WHERE clause from the filterspec
1843 props = self.getprops()
1844 frum = ['_'+cn]
1845 where = []
1846 args = []
1847 a = self.db.arg
1848 for k, v in filterspec.items():
1849 propclass = props[k]
1850 # now do other where clause stuff
1851 if isinstance(propclass, Multilink):
1852 tn = '%s_%s'%(cn, k)
1853 if v in ('-1', ['-1']):
1854 # only match rows that have count(linkid)=0 in the
1855 # corresponding multilink table)
1856 where.append('id not in (select nodeid from %s)'%tn)
1857 elif isinstance(v, type([])):
1858 frum.append(tn)
1859 s = ','.join([a for x in v])
1860 where.append('id=%s.nodeid and %s.linkid in (%s)'%(tn,tn,s))
1861 args = args + v
1862 else:
1863 frum.append(tn)
1864 where.append('id=%s.nodeid and %s.linkid=%s'%(tn, tn, a))
1865 args.append(v)
1866 elif k == 'id':
1867 if isinstance(v, type([])):
1868 s = ','.join([a for x in v])
1869 where.append('%s in (%s)'%(k, s))
1870 args = args + v
1871 else:
1872 where.append('%s=%s'%(k, a))
1873 args.append(v)
1874 elif isinstance(propclass, String):
1875 if not isinstance(v, type([])):
1876 v = [v]
1878 # Quote the bits in the string that need it and then embed
1879 # in a "substring" search. Note - need to quote the '%' so
1880 # they make it through the python layer happily
1881 v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
1883 # now add to the where clause
1884 where.append(' or '.join(["_%s LIKE '%s'"%(k, s) for s in v]))
1885 # note: args are embedded in the query string now
1886 elif isinstance(propclass, Link):
1887 if isinstance(v, type([])):
1888 if '-1' in v:
1889 v = v[:]
1890 v.remove('-1')
1891 xtra = ' or _%s is NULL'%k
1892 else:
1893 xtra = ''
1894 if v:
1895 s = ','.join([a for x in v])
1896 where.append('(_%s in (%s)%s)'%(k, s, xtra))
1897 args = args + v
1898 else:
1899 where.append('_%s is NULL'%k)
1900 else:
1901 if v == '-1':
1902 v = None
1903 where.append('_%s is NULL'%k)
1904 else:
1905 where.append('_%s=%s'%(k, a))
1906 args.append(v)
1907 elif isinstance(propclass, Date):
1908 if isinstance(v, type([])):
1909 s = ','.join([a for x in v])
1910 where.append('_%s in (%s)'%(k, s))
1911 args = args + [date.Date(x).serialise() for x in v]
1912 else:
1913 try:
1914 # Try to filter on range of dates
1915 date_rng = Range(v, date.Date, offset=timezone)
1916 if (date_rng.from_value):
1917 where.append('_%s >= %s'%(k, a))
1918 args.append(date_rng.from_value.serialise())
1919 if (date_rng.to_value):
1920 where.append('_%s <= %s'%(k, a))
1921 args.append(date_rng.to_value.serialise())
1922 except ValueError:
1923 # If range creation fails - ignore that search parameter
1924 pass
1925 elif isinstance(propclass, Interval):
1926 if isinstance(v, type([])):
1927 s = ','.join([a for x in v])
1928 where.append('_%s in (%s)'%(k, s))
1929 args = args + [date.Interval(x).serialise() for x in v]
1930 else:
1931 try:
1932 # Try to filter on range of intervals
1933 date_rng = Range(v, date.Interval)
1934 if (date_rng.from_value):
1935 where.append('_%s >= %s'%(k, a))
1936 args.append(date_rng.from_value.serialise())
1937 if (date_rng.to_value):
1938 where.append('_%s <= %s'%(k, a))
1939 args.append(date_rng.to_value.serialise())
1940 except ValueError:
1941 # If range creation fails - ignore that search parameter
1942 pass
1943 #where.append('_%s=%s'%(k, a))
1944 #args.append(date.Interval(v).serialise())
1945 else:
1946 if isinstance(v, type([])):
1947 s = ','.join([a for x in v])
1948 where.append('_%s in (%s)'%(k, s))
1949 args = args + v
1950 else:
1951 where.append('_%s=%s'%(k, a))
1952 args.append(v)
1954 # don't match retired nodes
1955 where.append('__retired__ <> 1')
1957 # add results of full text search
1958 if search_matches is not None:
1959 v = search_matches.keys()
1960 s = ','.join([a for x in v])
1961 where.append('id in (%s)'%s)
1962 args = args + v
1964 # "grouping" is just the first-order sorting in the SQL fetch
1965 # can modify it...)
1966 orderby = []
1967 ordercols = []
1968 if group[0] is not None and group[1] is not None:
1969 if group[0] != '-':
1970 orderby.append('_'+group[1])
1971 ordercols.append('_'+group[1])
1972 else:
1973 orderby.append('_'+group[1]+' desc')
1974 ordercols.append('_'+group[1])
1976 # now add in the sorting
1977 group = ''
1978 if sort[0] is not None and sort[1] is not None:
1979 direction, colname = sort
1980 if direction != '-':
1981 if colname == 'id':
1982 orderby.append(colname)
1983 else:
1984 orderby.append('_'+colname)
1985 ordercols.append('_'+colname)
1986 else:
1987 if colname == 'id':
1988 orderby.append(colname+' desc')
1989 ordercols.append(colname)
1990 else:
1991 orderby.append('_'+colname+' desc')
1992 ordercols.append('_'+colname)
1994 # construct the SQL
1995 frum = ','.join(frum)
1996 if where:
1997 where = ' where ' + (' and '.join(where))
1998 else:
1999 where = ''
2000 cols = ['id']
2001 if orderby:
2002 cols = cols + ordercols
2003 order = ' order by %s'%(','.join(orderby))
2004 else:
2005 order = ''
2006 cols = ','.join(cols)
2007 sql = 'select %s from %s %s%s%s'%(cols, frum, where, group, order)
2008 args = tuple(args)
2009 if __debug__:
2010 print >>hyperdb.DEBUG, 'filter', (self, sql, args)
2011 self.db.cursor.execute(sql, args)
2012 l = self.db.cursor.fetchall()
2014 # return the IDs (the first column)
2015 # XXX The filter(None, l) bit is sqlite-specific... if there's _NO_
2016 # XXX matches to a fetch, it returns NULL instead of nothing!?!
2017 return filter(None, [row[0] for row in l])
2019 def count(self):
2020 '''Get the number of nodes in this class.
2022 If the returned integer is 'numnodes', the ids of all the nodes
2023 in this class run from 1 to numnodes, and numnodes+1 will be the
2024 id of the next node to be created in this class.
2025 '''
2026 return self.db.countnodes(self.classname)
2028 # Manipulating properties:
2029 def getprops(self, protected=1):
2030 '''Return a dictionary mapping property names to property objects.
2031 If the "protected" flag is true, we include protected properties -
2032 those which may not be modified.
2033 '''
2034 d = self.properties.copy()
2035 if protected:
2036 d['id'] = String()
2037 d['creation'] = hyperdb.Date()
2038 d['activity'] = hyperdb.Date()
2039 d['creator'] = hyperdb.Link('user')
2040 return d
2042 def addprop(self, **properties):
2043 '''Add properties to this class.
2045 The keyword arguments in 'properties' must map names to property
2046 objects, or a TypeError is raised. None of the keys in 'properties'
2047 may collide with the names of existing properties, or a ValueError
2048 is raised before any properties have been added.
2049 '''
2050 for key in properties.keys():
2051 if self.properties.has_key(key):
2052 raise ValueError, key
2053 self.properties.update(properties)
2055 def index(self, nodeid):
2056 '''Add (or refresh) the node to search indexes
2057 '''
2058 # find all the String properties that have indexme
2059 for prop, propclass in self.getprops().items():
2060 if isinstance(propclass, String) and propclass.indexme:
2061 try:
2062 value = str(self.get(nodeid, prop))
2063 except IndexError:
2064 # node no longer exists - entry should be removed
2065 self.db.indexer.purge_entry((self.classname, nodeid, prop))
2066 else:
2067 # and index them under (classname, nodeid, property)
2068 self.db.indexer.add_text((self.classname, nodeid, prop),
2069 value)
2072 #
2073 # Detector interface
2074 #
2075 def audit(self, event, detector):
2076 '''Register a detector
2077 '''
2078 l = self.auditors[event]
2079 if detector not in l:
2080 self.auditors[event].append(detector)
2082 def fireAuditors(self, action, nodeid, newvalues):
2083 '''Fire all registered auditors.
2084 '''
2085 for audit in self.auditors[action]:
2086 audit(self.db, self, nodeid, newvalues)
2088 def react(self, event, detector):
2089 '''Register a detector
2090 '''
2091 l = self.reactors[event]
2092 if detector not in l:
2093 self.reactors[event].append(detector)
2095 def fireReactors(self, action, nodeid, oldvalues):
2096 '''Fire all registered reactors.
2097 '''
2098 for react in self.reactors[action]:
2099 react(self.db, self, nodeid, oldvalues)
2101 class FileClass(Class, hyperdb.FileClass):
2102 '''This class defines a large chunk of data. To support this, it has a
2103 mandatory String property "content" which is typically saved off
2104 externally to the hyperdb.
2106 The default MIME type of this data is defined by the
2107 "default_mime_type" class attribute, which may be overridden by each
2108 node if the class defines a "type" String property.
2109 '''
2110 default_mime_type = 'text/plain'
2112 def create(self, **propvalues):
2113 ''' snaffle the file propvalue and store in a file
2114 '''
2115 # we need to fire the auditors now, or the content property won't
2116 # be in propvalues for the auditors to play with
2117 self.fireAuditors('create', None, propvalues)
2119 # now remove the content property so it's not stored in the db
2120 content = propvalues['content']
2121 del propvalues['content']
2123 # do the database create
2124 newid = Class.create_inner(self, **propvalues)
2126 # fire reactors
2127 self.fireReactors('create', newid, None)
2129 # store off the content as a file
2130 self.db.storefile(self.classname, newid, None, content)
2131 return newid
2133 def import_list(self, propnames, proplist):
2134 ''' Trap the "content" property...
2135 '''
2136 # dupe this list so we don't affect others
2137 propnames = propnames[:]
2139 # extract the "content" property from the proplist
2140 i = propnames.index('content')
2141 content = eval(proplist[i])
2142 del propnames[i]
2143 del proplist[i]
2145 # do the normal import
2146 newid = Class.import_list(self, propnames, proplist)
2148 # save off the "content" file
2149 self.db.storefile(self.classname, newid, None, content)
2150 return newid
2152 _marker = []
2153 def get(self, nodeid, propname, default=_marker, cache=1):
2154 ''' Trap the content propname and get it from the file
2156 'cache' exists for backwards compatibility, and is not used.
2157 '''
2158 poss_msg = 'Possibly a access right configuration problem.'
2159 if propname == 'content':
2160 try:
2161 return self.db.getfile(self.classname, nodeid, None)
2162 except IOError, (strerror):
2163 # BUG: by catching this we donot see an error in the log.
2164 return 'ERROR reading file: %s%s\n%s\n%s'%(
2165 self.classname, nodeid, poss_msg, strerror)
2166 if default is not self._marker:
2167 return Class.get(self, nodeid, propname, default)
2168 else:
2169 return Class.get(self, nodeid, propname)
2171 def getprops(self, protected=1):
2172 ''' In addition to the actual properties on the node, these methods
2173 provide the "content" property. If the "protected" flag is true,
2174 we include protected properties - those which may not be
2175 modified.
2176 '''
2177 d = Class.getprops(self, protected=protected).copy()
2178 d['content'] = hyperdb.String()
2179 return d
2181 def index(self, nodeid):
2182 ''' Index the node in the search index.
2184 We want to index the content in addition to the normal String
2185 property indexing.
2186 '''
2187 # perform normal indexing
2188 Class.index(self, nodeid)
2190 # get the content to index
2191 content = self.get(nodeid, 'content')
2193 # figure the mime type
2194 if self.properties.has_key('type'):
2195 mime_type = self.get(nodeid, 'type')
2196 else:
2197 mime_type = self.default_mime_type
2199 # and index!
2200 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
2201 mime_type)
2203 # XXX deviation from spec - was called ItemClass
2204 class IssueClass(Class, roundupdb.IssueClass):
2205 # Overridden methods:
2206 def __init__(self, db, classname, **properties):
2207 '''The newly-created class automatically includes the "messages",
2208 "files", "nosy", and "superseder" properties. If the 'properties'
2209 dictionary attempts to specify any of these properties or a
2210 "creation" or "activity" property, a ValueError is raised.
2211 '''
2212 if not properties.has_key('title'):
2213 properties['title'] = hyperdb.String(indexme='yes')
2214 if not properties.has_key('messages'):
2215 properties['messages'] = hyperdb.Multilink("msg")
2216 if not properties.has_key('files'):
2217 properties['files'] = hyperdb.Multilink("file")
2218 if not properties.has_key('nosy'):
2219 # note: journalling is turned off as it really just wastes
2220 # space. this behaviour may be overridden in an instance
2221 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2222 if not properties.has_key('superseder'):
2223 properties['superseder'] = hyperdb.Multilink(classname)
2224 Class.__init__(self, db, classname, **properties)