5d67abc4ca6d56089099a2c52cfeb1065a7f980d
1 # $Id: rdbms_common.py,v 1.62 2003-09-08 20:39:18 jlgijsbers Exp $
2 ''' Relational database (SQL) backend common code.
4 Basics:
6 - map roundup classes to relational tables
7 - automatically detect schema changes and modify the table schemas
8 appropriately (we store the "database version" of the schema in the
9 database itself as the only row of the "schema" table)
10 - multilinks (which represent a many-to-many relationship) are handled through
11 intermediate tables
12 - journals are stored adjunct to the per-class tables
13 - table names and columns have "_" prepended so the names can't clash with
14 restricted names (like "order")
15 - retirement is determined by the __retired__ column being true
17 Database-specific changes may generally be pushed out to the overridable
18 sql_* methods, since everything else should be fairly generic. There's
19 probably a bit of work to be done if a database is used that actually
20 honors column typing, since the initial databases don't (sqlite stores
21 everything as a string.)
22 '''
24 # standard python modules
25 import sys, os, time, re, errno, weakref, copy
27 # roundup modules
28 from roundup import hyperdb, date, password, roundupdb, security
29 from roundup.hyperdb import String, Password, Date, Interval, Link, \
30 Multilink, DatabaseError, Boolean, Number, Node
31 from roundup.backends import locking
33 # support
34 from blobfiles import FileStorage
35 from roundup.indexer import Indexer
36 from sessions import Sessions, OneTimeKeys
37 from roundup.date import Range
39 # number of rows to keep in memory
40 ROW_CACHE_SIZE = 100
42 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
43 ''' Wrapper around an SQL database that presents a hyperdb interface.
45 - some functionality is specific to the actual SQL database, hence
46 the sql_* methods that are NotImplemented
47 - we keep a cache of the latest ROW_CACHE_SIZE row fetches.
48 '''
49 def __init__(self, config, journaltag=None):
50 ''' Open the database and load the schema from it.
51 '''
52 self.config, self.journaltag = config, journaltag
53 self.dir = config.DATABASE
54 self.classes = {}
55 self.indexer = Indexer(self.dir)
56 self.sessions = Sessions(self.config)
57 self.otks = OneTimeKeys(self.config)
58 self.security = security.Security(self)
60 # additional transaction support for external files and the like
61 self.transactions = []
63 # keep a cache of the N most recently retrieved rows of any kind
64 # (classname, nodeid) = row
65 self.cache = {}
66 self.cache_lru = []
68 # database lock
69 self.lockfile = None
71 # open a connection to the database, creating the "conn" attribute
72 self.open_connection()
74 def clearCache(self):
75 self.cache = {}
76 self.cache_lru = []
78 def open_connection(self):
79 ''' Open a connection to the database, creating it if necessary
80 '''
81 raise NotImplemented
83 def sql(self, sql, args=None):
84 ''' Execute the sql with the optional args.
85 '''
86 if __debug__:
87 print >>hyperdb.DEBUG, (self, sql, args)
88 if args:
89 self.cursor.execute(sql, args)
90 else:
91 self.cursor.execute(sql)
93 def sql_fetchone(self):
94 ''' Fetch a single row. If there's nothing to fetch, return None.
95 '''
96 raise NotImplemented
98 def sql_stringquote(self, value):
99 ''' Quote the string so it's safe to put in the 'sql quotes'
100 '''
101 return re.sub("'", "''", str(value))
103 def save_dbschema(self, schema):
104 ''' Save the schema definition that the database currently implements
105 '''
106 raise NotImplemented
108 def load_dbschema(self):
109 ''' Load the schema definition that the database currently implements
110 '''
111 raise NotImplemented
113 def post_init(self):
114 ''' Called once the schema initialisation has finished.
116 We should now confirm that the schema defined by our "classes"
117 attribute actually matches the schema in the database.
118 '''
119 # now detect changes in the schema
120 save = 0
121 for classname, spec in self.classes.items():
122 if self.database_schema.has_key(classname):
123 dbspec = self.database_schema[classname]
124 if self.update_class(spec, dbspec):
125 self.database_schema[classname] = spec.schema()
126 save = 1
127 else:
128 self.create_class(spec)
129 self.database_schema[classname] = spec.schema()
130 save = 1
132 for classname in self.database_schema.keys():
133 if not self.classes.has_key(classname):
134 self.drop_class(classname)
136 # update the database version of the schema
137 if save:
138 self.sql('delete from schema')
139 self.save_dbschema(self.database_schema)
141 # reindex the db if necessary
142 if self.indexer.should_reindex():
143 self.reindex()
145 # commit
146 self.conn.commit()
148 def reindex(self):
149 for klass in self.classes.values():
150 for nodeid in klass.list():
151 klass.index(nodeid)
152 self.indexer.save_index()
154 def determine_columns(self, properties):
155 ''' Figure the column names and multilink properties from the spec
157 "properties" is a list of (name, prop) where prop may be an
158 instance of a hyperdb "type" _or_ a string repr of that type.
159 '''
160 cols = ['_activity', '_creator', '_creation']
161 mls = []
162 # add the multilinks separately
163 for col, prop in properties:
164 if isinstance(prop, Multilink):
165 mls.append(col)
166 elif isinstance(prop, type('')) and prop.find('Multilink') != -1:
167 mls.append(col)
168 else:
169 cols.append('_'+col)
170 cols.sort()
171 return cols, mls
173 def update_class(self, spec, old_spec):
174 ''' Determine the differences between the current spec and the
175 database version of the spec, and update where necessary
176 '''
177 new_spec = spec
178 new_has = new_spec.properties.has_key
180 new_spec = new_spec.schema()
181 new_spec[1].sort()
182 old_spec[1].sort()
183 if new_spec == old_spec:
184 # no changes
185 return 0
187 if __debug__:
188 print >>hyperdb.DEBUG, 'update_class FIRING'
190 # key property changed?
191 if old_spec[0] != new_spec[0]:
192 if __debug__:
193 print >>hyperdb.DEBUG, 'update_class setting keyprop', `spec[0]`
194 # XXX turn on indexing for the key property
196 # detect multilinks that have been removed, and drop their table
197 old_has = {}
198 for name,prop in old_spec[1]:
199 old_has[name] = 1
200 if not new_has(name) and isinstance(prop, Multilink):
201 # it's a multilink, and it's been removed - drop the old
202 # table
203 sql = 'drop table %s_%s'%(spec.classname, prop)
204 if __debug__:
205 print >>hyperdb.DEBUG, 'update_class', (self, sql)
206 self.cursor.execute(sql)
207 continue
208 old_has = old_has.has_key
210 # now figure how we populate the new table
211 fetch = ['_activity', '_creation', '_creator']
212 properties = spec.getprops()
213 for propname,x in new_spec[1]:
214 prop = properties[propname]
215 if isinstance(prop, Multilink):
216 if not old_has(propname):
217 # we need to create the new table
218 self.create_multilink_table(spec, propname)
219 elif old_has(propname):
220 # we copy this col over from the old table
221 fetch.append('_'+propname)
223 # select the data out of the old table
224 fetch.append('id')
225 fetch.append('__retired__')
226 fetchcols = ','.join(fetch)
227 cn = spec.classname
228 sql = 'select %s from _%s'%(fetchcols, cn)
229 if __debug__:
230 print >>hyperdb.DEBUG, 'update_class', (self, sql)
231 self.cursor.execute(sql)
232 olddata = self.cursor.fetchall()
234 # drop the old table
235 self.cursor.execute('drop table _%s'%cn)
237 # create the new table
238 self.create_class_table(spec)
240 if olddata:
241 # do the insert
242 args = ','.join([self.arg for x in fetch])
243 sql = 'insert into _%s (%s) values (%s)'%(cn, fetchcols, args)
244 if __debug__:
245 print >>hyperdb.DEBUG, 'update_class', (self, sql, olddata[0])
246 for entry in olddata:
247 self.cursor.execute(sql, tuple(entry))
249 return 1
251 def create_class_table(self, spec):
252 ''' create the class table for the given spec
253 '''
254 cols, mls = self.determine_columns(spec.properties.items())
256 # add on our special columns
257 cols.append('id')
258 cols.append('__retired__')
260 # create the base table
261 scols = ','.join(['%s varchar'%x for x in cols])
262 sql = 'create table _%s (%s)'%(spec.classname, scols)
263 if __debug__:
264 print >>hyperdb.DEBUG, 'create_class', (self, sql)
265 self.cursor.execute(sql)
267 return cols, mls
269 def create_journal_table(self, spec):
270 ''' create the journal table for a class given the spec and
271 already-determined cols
272 '''
273 # journal table
274 cols = ','.join(['%s varchar'%x
275 for x in 'nodeid date tag action params'.split()])
276 sql = 'create table %s__journal (%s)'%(spec.classname, cols)
277 if __debug__:
278 print >>hyperdb.DEBUG, 'create_class', (self, sql)
279 self.cursor.execute(sql)
281 def create_multilink_table(self, spec, ml):
282 ''' Create a multilink table for the "ml" property of the class
283 given by the spec
284 '''
285 sql = 'create table %s_%s (linkid varchar, nodeid varchar)'%(
286 spec.classname, ml)
287 if __debug__:
288 print >>hyperdb.DEBUG, 'create_class', (self, sql)
289 self.cursor.execute(sql)
291 def create_class(self, spec):
292 ''' Create a database table according to the given spec.
293 '''
294 cols, mls = self.create_class_table(spec)
295 self.create_journal_table(spec)
297 # now create the multilink tables
298 for ml in mls:
299 self.create_multilink_table(spec, ml)
301 # ID counter
302 sql = 'insert into ids (name, num) values (%s,%s)'%(self.arg, self.arg)
303 vals = (spec.classname, 1)
304 if __debug__:
305 print >>hyperdb.DEBUG, 'create_class', (self, sql, vals)
306 self.cursor.execute(sql, vals)
308 def drop_class(self, spec):
309 ''' Drop the given table from the database.
311 Drop the journal and multilink tables too.
312 '''
313 # figure the multilinks
314 mls = []
315 for col, prop in spec.properties.items():
316 if isinstance(prop, Multilink):
317 mls.append(col)
319 sql = 'drop table _%s'%spec.classname
320 if __debug__:
321 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
322 self.cursor.execute(sql)
324 sql = 'drop table %s__journal'%spec.classname
325 if __debug__:
326 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
327 self.cursor.execute(sql)
329 for ml in mls:
330 sql = 'drop table %s_%s'%(spec.classname, ml)
331 if __debug__:
332 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
333 self.cursor.execute(sql)
335 #
336 # Classes
337 #
338 def __getattr__(self, classname):
339 ''' A convenient way of calling self.getclass(classname).
340 '''
341 if self.classes.has_key(classname):
342 if __debug__:
343 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
344 return self.classes[classname]
345 raise AttributeError, classname
347 def addclass(self, cl):
348 ''' Add a Class to the hyperdatabase.
349 '''
350 if __debug__:
351 print >>hyperdb.DEBUG, 'addclass', (self, cl)
352 cn = cl.classname
353 if self.classes.has_key(cn):
354 raise ValueError, cn
355 self.classes[cn] = cl
357 def getclasses(self):
358 ''' Return a list of the names of all existing classes.
359 '''
360 if __debug__:
361 print >>hyperdb.DEBUG, 'getclasses', (self,)
362 l = self.classes.keys()
363 l.sort()
364 return l
366 def getclass(self, classname):
367 '''Get the Class object representing a particular class.
369 If 'classname' is not a valid class name, a KeyError is raised.
370 '''
371 if __debug__:
372 print >>hyperdb.DEBUG, 'getclass', (self, classname)
373 try:
374 return self.classes[classname]
375 except KeyError:
376 raise KeyError, 'There is no class called "%s"'%classname
378 def clear(self):
379 ''' Delete all database contents.
381 Note: I don't commit here, which is different behaviour to the
382 "nuke from orbit" behaviour in the *dbms.
383 '''
384 if __debug__:
385 print >>hyperdb.DEBUG, 'clear', (self,)
386 for cn in self.classes.keys():
387 sql = 'delete from _%s'%cn
388 if __debug__:
389 print >>hyperdb.DEBUG, 'clear', (self, sql)
390 self.cursor.execute(sql)
392 #
393 # Node IDs
394 #
395 def newid(self, classname):
396 ''' Generate a new id for the given class
397 '''
398 # get the next ID
399 sql = 'select num from ids where name=%s'%self.arg
400 if __debug__:
401 print >>hyperdb.DEBUG, 'newid', (self, sql, classname)
402 self.cursor.execute(sql, (classname, ))
403 newid = self.cursor.fetchone()[0]
405 # update the counter
406 sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
407 vals = (int(newid)+1, classname)
408 if __debug__:
409 print >>hyperdb.DEBUG, 'newid', (self, sql, vals)
410 self.cursor.execute(sql, vals)
412 # return as string
413 return str(newid)
415 def setid(self, classname, setid):
416 ''' Set the id counter: used during import of database
417 '''
418 sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
419 vals = (setid, classname)
420 if __debug__:
421 print >>hyperdb.DEBUG, 'setid', (self, sql, vals)
422 self.cursor.execute(sql, vals)
424 #
425 # Nodes
426 #
427 def addnode(self, classname, nodeid, node):
428 ''' Add the specified node to its class's db.
429 '''
430 if __debug__:
431 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
433 # determine the column definitions and multilink tables
434 cl = self.classes[classname]
435 cols, mls = self.determine_columns(cl.properties.items())
437 # we'll be supplied these props if we're doing an import
438 if not node.has_key('creator'):
439 # add in the "calculated" properties (dupe so we don't affect
440 # calling code's node assumptions)
441 node = node.copy()
442 node['creation'] = node['activity'] = date.Date()
443 node['creator'] = self.getuid()
445 # default the non-multilink columns
446 for col, prop in cl.properties.items():
447 if not node.has_key(col):
448 if isinstance(prop, Multilink):
449 node[col] = []
450 else:
451 node[col] = None
453 # clear this node out of the cache if it's in there
454 key = (classname, nodeid)
455 if self.cache.has_key(key):
456 del self.cache[key]
457 self.cache_lru.remove(key)
459 # make the node data safe for the DB
460 node = self.serialise(classname, node)
462 # make sure the ordering is correct for column name -> column value
463 vals = tuple([node[col[1:]] for col in cols]) + (nodeid, 0)
464 s = ','.join([self.arg for x in cols]) + ',%s,%s'%(self.arg, self.arg)
465 cols = ','.join(cols) + ',id,__retired__'
467 # perform the inserts
468 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
469 if __debug__:
470 print >>hyperdb.DEBUG, 'addnode', (self, sql, vals)
471 self.cursor.execute(sql, vals)
473 # insert the multilink rows
474 for col in mls:
475 t = '%s_%s'%(classname, col)
476 for entry in node[col]:
477 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
478 self.arg, self.arg)
479 self.sql(sql, (entry, nodeid))
481 # make sure we do the commit-time extra stuff for this node
482 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
484 def setnode(self, classname, nodeid, values, multilink_changes):
485 ''' Change the specified node.
486 '''
487 if __debug__:
488 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, values)
490 # clear this node out of the cache if it's in there
491 key = (classname, nodeid)
492 if self.cache.has_key(key):
493 del self.cache[key]
494 self.cache_lru.remove(key)
496 # add the special props
497 values = values.copy()
498 values['activity'] = date.Date()
500 # make db-friendly
501 values = self.serialise(classname, values)
503 cl = self.classes[classname]
504 cols = []
505 mls = []
506 # add the multilinks separately
507 props = cl.getprops()
508 for col in values.keys():
509 prop = props[col]
510 if isinstance(prop, Multilink):
511 mls.append(col)
512 else:
513 cols.append('_'+col)
514 cols.sort()
516 # if there's any updates to regular columns, do them
517 if cols:
518 # make sure the ordering is correct for column name -> column value
519 sqlvals = tuple([values[col[1:]] for col in cols]) + (nodeid,)
520 s = ','.join(['%s=%s'%(x, self.arg) for x in cols])
521 cols = ','.join(cols)
523 # perform the update
524 sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
525 if __debug__:
526 print >>hyperdb.DEBUG, 'setnode', (self, sql, sqlvals)
527 self.cursor.execute(sql, sqlvals)
529 # now the fun bit, updating the multilinks ;)
530 for col, (add, remove) in multilink_changes.items():
531 tn = '%s_%s'%(classname, col)
532 if add:
533 sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
534 self.arg, self.arg)
535 for addid in add:
536 self.sql(sql, (nodeid, addid))
537 if remove:
538 sql = 'delete from %s where nodeid=%s and linkid=%s'%(tn,
539 self.arg, self.arg)
540 for removeid in remove:
541 self.sql(sql, (nodeid, removeid))
543 # make sure we do the commit-time extra stuff for this node
544 self.transactions.append((self.doSaveNode, (classname, nodeid, values)))
546 def getnode(self, classname, nodeid):
547 ''' Get a node from the database.
548 '''
549 if __debug__:
550 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid)
552 # see if we have this node cached
553 key = (classname, nodeid)
554 if self.cache.has_key(key):
555 # push us back to the top of the LRU
556 self.cache_lru.remove(key)
557 self.cache_lru.insert(0, key)
558 # return the cached information
559 return self.cache[key]
561 # figure the columns we're fetching
562 cl = self.classes[classname]
563 cols, mls = self.determine_columns(cl.properties.items())
564 scols = ','.join(cols)
566 # perform the basic property fetch
567 sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
568 self.sql(sql, (nodeid,))
570 values = self.sql_fetchone()
571 if values is None:
572 raise IndexError, 'no such %s node %s'%(classname, nodeid)
574 # make up the node
575 node = {}
576 for col in range(len(cols)):
577 node[cols[col][1:]] = values[col]
579 # now the multilinks
580 for col in mls:
581 # get the link ids
582 sql = 'select linkid from %s_%s where nodeid=%s'%(classname, col,
583 self.arg)
584 self.cursor.execute(sql, (nodeid,))
585 # extract the first column from the result
586 node[col] = [x[0] for x in self.cursor.fetchall()]
588 # un-dbificate the node data
589 node = self.unserialise(classname, node)
591 # save off in the cache
592 key = (classname, nodeid)
593 self.cache[key] = node
594 # update the LRU
595 self.cache_lru.insert(0, key)
596 if len(self.cache_lru) > ROW_CACHE_SIZE:
597 del self.cache[self.cache_lru.pop()]
599 return node
601 def destroynode(self, classname, nodeid):
602 '''Remove a node from the database. Called exclusively by the
603 destroy() method on Class.
604 '''
605 if __debug__:
606 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
608 # make sure the node exists
609 if not self.hasnode(classname, nodeid):
610 raise IndexError, '%s has no node %s'%(classname, nodeid)
612 # see if we have this node cached
613 if self.cache.has_key((classname, nodeid)):
614 del self.cache[(classname, nodeid)]
616 # see if there's any obvious commit actions that we should get rid of
617 for entry in self.transactions[:]:
618 if entry[1][:2] == (classname, nodeid):
619 self.transactions.remove(entry)
621 # now do the SQL
622 sql = 'delete from _%s where id=%s'%(classname, self.arg)
623 self.sql(sql, (nodeid,))
625 # remove from multilnks
626 cl = self.getclass(classname)
627 x, mls = self.determine_columns(cl.properties.items())
628 for col in mls:
629 # get the link ids
630 sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
631 self.cursor.execute(sql, (nodeid,))
633 # remove journal entries
634 sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
635 self.sql(sql, (nodeid,))
637 def serialise(self, classname, node):
638 '''Copy the node contents, converting non-marshallable data into
639 marshallable data.
640 '''
641 if __debug__:
642 print >>hyperdb.DEBUG, 'serialise', classname, node
643 properties = self.getclass(classname).getprops()
644 d = {}
645 for k, v in node.items():
646 # if the property doesn't exist, or is the "retired" flag then
647 # it won't be in the properties dict
648 if not properties.has_key(k):
649 d[k] = v
650 continue
652 # get the property spec
653 prop = properties[k]
655 if isinstance(prop, Password) and v is not None:
656 d[k] = str(v)
657 elif isinstance(prop, Date) and v is not None:
658 d[k] = v.serialise()
659 elif isinstance(prop, Interval) and v is not None:
660 d[k] = v.serialise()
661 else:
662 d[k] = v
663 return d
665 def unserialise(self, classname, node):
666 '''Decode the marshalled node data
667 '''
668 if __debug__:
669 print >>hyperdb.DEBUG, 'unserialise', classname, node
670 properties = self.getclass(classname).getprops()
671 d = {}
672 for k, v in node.items():
673 # if the property doesn't exist, or is the "retired" flag then
674 # it won't be in the properties dict
675 if not properties.has_key(k):
676 d[k] = v
677 continue
679 # get the property spec
680 prop = properties[k]
682 if isinstance(prop, Date) and v is not None:
683 d[k] = date.Date(v)
684 elif isinstance(prop, Interval) and v is not None:
685 d[k] = date.Interval(v)
686 elif isinstance(prop, Password) and v is not None:
687 p = password.Password()
688 p.unpack(v)
689 d[k] = p
690 elif (isinstance(prop, Boolean) or isinstance(prop, Number)) and v is not None:
691 d[k]=float(v)
692 else:
693 d[k] = v
694 return d
696 def hasnode(self, classname, nodeid):
697 ''' Determine if the database has a given node.
698 '''
699 sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
700 if __debug__:
701 print >>hyperdb.DEBUG, 'hasnode', (self, sql, nodeid)
702 self.cursor.execute(sql, (nodeid,))
703 return int(self.cursor.fetchone()[0])
705 def countnodes(self, classname):
706 ''' Count the number of nodes that exist for a particular Class.
707 '''
708 sql = 'select count(*) from _%s'%classname
709 if __debug__:
710 print >>hyperdb.DEBUG, 'countnodes', (self, sql)
711 self.cursor.execute(sql)
712 return self.cursor.fetchone()[0]
714 def addjournal(self, classname, nodeid, action, params, creator=None,
715 creation=None):
716 ''' Journal the Action
717 'action' may be:
719 'create' or 'set' -- 'params' is a dictionary of property values
720 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
721 'retire' -- 'params' is None
722 '''
723 # serialise the parameters now if necessary
724 if isinstance(params, type({})):
725 if action in ('set', 'create'):
726 params = self.serialise(classname, params)
728 # handle supply of the special journalling parameters (usually
729 # supplied on importing an existing database)
730 if creator:
731 journaltag = creator
732 else:
733 journaltag = self.getuid()
734 if creation:
735 journaldate = creation.serialise()
736 else:
737 journaldate = date.Date().serialise()
739 # create the journal entry
740 cols = ','.join('nodeid date tag action params'.split())
742 if __debug__:
743 print >>hyperdb.DEBUG, 'addjournal', (nodeid, journaldate,
744 journaltag, action, params)
746 self.save_journal(classname, cols, nodeid, journaldate,
747 journaltag, action, params)
749 def save_journal(self, classname, cols, nodeid, journaldate,
750 journaltag, action, params):
751 ''' Save the journal entry to the database
752 '''
753 raise NotImplemented
755 def getjournal(self, classname, nodeid):
756 ''' get the journal for id
757 '''
758 # make sure the node exists
759 if not self.hasnode(classname, nodeid):
760 raise IndexError, '%s has no node %s'%(classname, nodeid)
762 cols = ','.join('nodeid date tag action params'.split())
763 return self.load_journal(classname, cols, nodeid)
765 def load_journal(self, classname, cols, nodeid):
766 ''' Load the journal from the database
767 '''
768 raise NotImplemented
770 def pack(self, pack_before):
771 ''' Delete all journal entries except "create" before 'pack_before'.
772 '''
773 # get a 'yyyymmddhhmmss' version of the date
774 date_stamp = pack_before.serialise()
776 # do the delete
777 for classname in self.classes.keys():
778 sql = "delete from %s__journal where date<%s and "\
779 "action<>'create'"%(classname, self.arg)
780 if __debug__:
781 print >>hyperdb.DEBUG, 'pack', (self, sql, date_stamp)
782 self.cursor.execute(sql, (date_stamp,))
784 def sql_commit(self):
785 ''' Actually commit to the database.
786 '''
787 self.conn.commit()
789 def commit(self):
790 ''' Commit the current transactions.
792 Save all data changed since the database was opened or since the
793 last commit() or rollback().
794 '''
795 if __debug__:
796 print >>hyperdb.DEBUG, 'commit', (self,)
798 # commit the database
799 self.sql_commit()
801 # now, do all the other transaction stuff
802 reindex = {}
803 for method, args in self.transactions:
804 reindex[method(*args)] = 1
806 # reindex the nodes that request it
807 for classname, nodeid in filter(None, reindex.keys()):
808 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
809 self.getclass(classname).index(nodeid)
811 # save the indexer state
812 self.indexer.save_index()
814 # clear out the transactions
815 self.transactions = []
817 def rollback(self):
818 ''' Reverse all actions from the current transaction.
820 Undo all the changes made since the database was opened or the last
821 commit() or rollback() was performed.
822 '''
823 if __debug__:
824 print >>hyperdb.DEBUG, 'rollback', (self,)
826 # roll back
827 self.conn.rollback()
829 # roll back "other" transaction stuff
830 for method, args in self.transactions:
831 # delete temporary files
832 if method == self.doStoreFile:
833 self.rollbackStoreFile(*args)
834 self.transactions = []
836 # clear the cache
837 self.clearCache()
839 def doSaveNode(self, classname, nodeid, node):
840 ''' dummy that just generates a reindex event
841 '''
842 # return the classname, nodeid so we reindex this content
843 return (classname, nodeid)
845 def close(self):
846 ''' Close off the connection.
847 '''
848 self.conn.close()
849 if self.lockfile is not None:
850 locking.release_lock(self.lockfile)
851 if self.lockfile is not None:
852 self.lockfile.close()
853 self.lockfile = None
855 #
856 # The base Class class
857 #
858 class Class(hyperdb.Class):
859 ''' The handle to a particular class of nodes in a hyperdatabase.
861 All methods except __repr__ and getnode must be implemented by a
862 concrete backend Class.
863 '''
865 def __init__(self, db, classname, **properties):
866 '''Create a new class with a given name and property specification.
868 'classname' must not collide with the name of an existing class,
869 or a ValueError is raised. The keyword arguments in 'properties'
870 must map names to property objects, or a TypeError is raised.
871 '''
872 if (properties.has_key('creation') or properties.has_key('activity')
873 or properties.has_key('creator')):
874 raise ValueError, '"creation", "activity" and "creator" are '\
875 'reserved'
877 self.classname = classname
878 self.properties = properties
879 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
880 self.key = ''
882 # should we journal changes (default yes)
883 self.do_journal = 1
885 # do the db-related init stuff
886 db.addclass(self)
888 self.auditors = {'create': [], 'set': [], 'retire': [], 'restore': []}
889 self.reactors = {'create': [], 'set': [], 'retire': [], 'restore': []}
891 def schema(self):
892 ''' A dumpable version of the schema that we can store in the
893 database
894 '''
895 return (self.key, [(x, repr(y)) for x,y in self.properties.items()])
897 def enableJournalling(self):
898 '''Turn journalling on for this class
899 '''
900 self.do_journal = 1
902 def disableJournalling(self):
903 '''Turn journalling off for this class
904 '''
905 self.do_journal = 0
907 # Editing nodes:
908 def create(self, **propvalues):
909 ''' Create a new node of this class and return its id.
911 The keyword arguments in 'propvalues' map property names to values.
913 The values of arguments must be acceptable for the types of their
914 corresponding properties or a TypeError is raised.
916 If this class has a key property, it must be present and its value
917 must not collide with other key strings or a ValueError is raised.
919 Any other properties on this class that are missing from the
920 'propvalues' dictionary are set to None.
922 If an id in a link or multilink property does not refer to a valid
923 node, an IndexError is raised.
924 '''
925 self.fireAuditors('create', None, propvalues)
926 newid = self.create_inner(**propvalues)
927 self.fireReactors('create', newid, None)
928 return newid
930 def create_inner(self, **propvalues):
931 ''' Called by create, in-between the audit and react calls.
932 '''
933 if propvalues.has_key('id'):
934 raise KeyError, '"id" is reserved'
936 if self.db.journaltag is None:
937 raise DatabaseError, 'Database open read-only'
939 if propvalues.has_key('creation') or propvalues.has_key('activity'):
940 raise KeyError, '"creation" and "activity" are reserved'
942 # new node's id
943 newid = self.db.newid(self.classname)
945 # validate propvalues
946 num_re = re.compile('^\d+$')
947 for key, value in propvalues.items():
948 if key == self.key:
949 try:
950 self.lookup(value)
951 except KeyError:
952 pass
953 else:
954 raise ValueError, 'node with key "%s" exists'%value
956 # try to handle this property
957 try:
958 prop = self.properties[key]
959 except KeyError:
960 raise KeyError, '"%s" has no property "%s"'%(self.classname,
961 key)
963 if value is not None and isinstance(prop, Link):
964 if type(value) != type(''):
965 raise ValueError, 'link value must be String'
966 link_class = self.properties[key].classname
967 # if it isn't a number, it's a key
968 if not num_re.match(value):
969 try:
970 value = self.db.classes[link_class].lookup(value)
971 except (TypeError, KeyError):
972 raise IndexError, 'new property "%s": %s not a %s'%(
973 key, value, link_class)
974 elif not self.db.getclass(link_class).hasnode(value):
975 raise IndexError, '%s has no node %s'%(link_class, value)
977 # save off the value
978 propvalues[key] = value
980 # register the link with the newly linked node
981 if self.do_journal and self.properties[key].do_journal:
982 self.db.addjournal(link_class, value, 'link',
983 (self.classname, newid, key))
985 elif isinstance(prop, Multilink):
986 if type(value) != type([]):
987 raise TypeError, 'new property "%s" not a list of ids'%key
989 # clean up and validate the list of links
990 link_class = self.properties[key].classname
991 l = []
992 for entry in value:
993 if type(entry) != type(''):
994 raise ValueError, '"%s" multilink value (%r) '\
995 'must contain Strings'%(key, value)
996 # if it isn't a number, it's a key
997 if not num_re.match(entry):
998 try:
999 entry = self.db.classes[link_class].lookup(entry)
1000 except (TypeError, KeyError):
1001 raise IndexError, 'new property "%s": %s not a %s'%(
1002 key, entry, self.properties[key].classname)
1003 l.append(entry)
1004 value = l
1005 propvalues[key] = value
1007 # handle additions
1008 for nodeid in value:
1009 if not self.db.getclass(link_class).hasnode(nodeid):
1010 raise IndexError, '%s has no node %s'%(link_class,
1011 nodeid)
1012 # register the link with the newly linked node
1013 if self.do_journal and self.properties[key].do_journal:
1014 self.db.addjournal(link_class, nodeid, 'link',
1015 (self.classname, newid, key))
1017 elif isinstance(prop, String):
1018 if type(value) != type('') and type(value) != type(u''):
1019 raise TypeError, 'new property "%s" not a string'%key
1021 elif isinstance(prop, Password):
1022 if not isinstance(value, password.Password):
1023 raise TypeError, 'new property "%s" not a Password'%key
1025 elif isinstance(prop, Date):
1026 if value is not None and not isinstance(value, date.Date):
1027 raise TypeError, 'new property "%s" not a Date'%key
1029 elif isinstance(prop, Interval):
1030 if value is not None and not isinstance(value, date.Interval):
1031 raise TypeError, 'new property "%s" not an Interval'%key
1033 elif value is not None and isinstance(prop, Number):
1034 try:
1035 float(value)
1036 except ValueError:
1037 raise TypeError, 'new property "%s" not numeric'%key
1039 elif value is not None and isinstance(prop, Boolean):
1040 try:
1041 int(value)
1042 except ValueError:
1043 raise TypeError, 'new property "%s" not boolean'%key
1045 # make sure there's data where there needs to be
1046 for key, prop in self.properties.items():
1047 if propvalues.has_key(key):
1048 continue
1049 if key == self.key:
1050 raise ValueError, 'key property "%s" is required'%key
1051 if isinstance(prop, Multilink):
1052 propvalues[key] = []
1053 else:
1054 propvalues[key] = None
1056 # done
1057 self.db.addnode(self.classname, newid, propvalues)
1058 if self.do_journal:
1059 self.db.addjournal(self.classname, newid, 'create', {})
1061 return newid
1063 def export_list(self, propnames, nodeid):
1064 ''' Export a node - generate a list of CSV-able data in the order
1065 specified by propnames for the given node.
1066 '''
1067 properties = self.getprops()
1068 l = []
1069 for prop in propnames:
1070 proptype = properties[prop]
1071 value = self.get(nodeid, prop)
1072 # "marshal" data where needed
1073 if value is None:
1074 pass
1075 elif isinstance(proptype, hyperdb.Date):
1076 value = value.get_tuple()
1077 elif isinstance(proptype, hyperdb.Interval):
1078 value = value.get_tuple()
1079 elif isinstance(proptype, hyperdb.Password):
1080 value = str(value)
1081 l.append(repr(value))
1082 l.append(self.is_retired(nodeid))
1083 return l
1085 def import_list(self, propnames, proplist):
1086 ''' Import a node - all information including "id" is present and
1087 should not be sanity checked. Triggers are not triggered. The
1088 journal should be initialised using the "creator" and "created"
1089 information.
1091 Return the nodeid of the node imported.
1092 '''
1093 if self.db.journaltag is None:
1094 raise DatabaseError, 'Database open read-only'
1095 properties = self.getprops()
1097 # make the new node's property map
1098 d = {}
1099 retire = 0
1100 newid = None
1101 for i in range(len(propnames)):
1102 # Use eval to reverse the repr() used to output the CSV
1103 value = eval(proplist[i])
1105 # Figure the property for this column
1106 propname = propnames[i]
1108 # "unmarshal" where necessary
1109 if propname == 'id':
1110 newid = value
1111 continue
1112 elif propname == 'is retired':
1113 # is the item retired?
1114 if int(value):
1115 retire = 1
1116 continue
1117 elif value is None:
1118 d[propname] = None
1119 continue
1121 prop = properties[propname]
1122 if value is None:
1123 # don't set Nones
1124 continue
1125 elif isinstance(prop, hyperdb.Date):
1126 value = date.Date(value)
1127 elif isinstance(prop, hyperdb.Interval):
1128 value = date.Interval(value)
1129 elif isinstance(prop, hyperdb.Password):
1130 pwd = password.Password()
1131 pwd.unpack(value)
1132 value = pwd
1133 d[propname] = value
1135 # get a new id if necessary
1136 if newid is None:
1137 newid = self.db.newid(self.classname)
1139 # retire?
1140 if retire:
1141 # use the arg for __retired__ to cope with any odd database type
1142 # conversion (hello, sqlite)
1143 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1144 self.db.arg, self.db.arg)
1145 if __debug__:
1146 print >>hyperdb.DEBUG, 'retire', (self, sql, newid)
1147 self.db.cursor.execute(sql, (1, newid))
1149 # add the node and journal
1150 self.db.addnode(self.classname, newid, d)
1152 # extract the extraneous journalling gumpf and nuke it
1153 if d.has_key('creator'):
1154 creator = d['creator']
1155 del d['creator']
1156 else:
1157 creator = None
1158 if d.has_key('creation'):
1159 creation = d['creation']
1160 del d['creation']
1161 else:
1162 creation = None
1163 if d.has_key('activity'):
1164 del d['activity']
1165 self.db.addjournal(self.classname, newid, 'create', {}, creator,
1166 creation)
1167 return newid
1169 _marker = []
1170 def get(self, nodeid, propname, default=_marker, cache=1):
1171 '''Get the value of a property on an existing node of this class.
1173 'nodeid' must be the id of an existing node of this class or an
1174 IndexError is raised. 'propname' must be the name of a property
1175 of this class or a KeyError is raised.
1177 'cache' exists for backwards compatibility, and is not used.
1178 '''
1179 if propname == 'id':
1180 return nodeid
1182 # get the node's dict
1183 d = self.db.getnode(self.classname, nodeid)
1185 if propname == 'creation':
1186 if d.has_key('creation'):
1187 return d['creation']
1188 else:
1189 return date.Date()
1190 if propname == 'activity':
1191 if d.has_key('activity'):
1192 return d['activity']
1193 else:
1194 return date.Date()
1195 if propname == 'creator':
1196 if d.has_key('creator'):
1197 return d['creator']
1198 else:
1199 return self.db.getuid()
1201 # get the property (raises KeyErorr if invalid)
1202 prop = self.properties[propname]
1204 if not d.has_key(propname):
1205 if default is self._marker:
1206 if isinstance(prop, Multilink):
1207 return []
1208 else:
1209 return None
1210 else:
1211 return default
1213 # don't pass our list to other code
1214 if isinstance(prop, Multilink):
1215 return d[propname][:]
1217 return d[propname]
1219 def getnode(self, nodeid, cache=1):
1220 ''' Return a convenience wrapper for the node.
1222 'nodeid' must be the id of an existing node of this class or an
1223 IndexError is raised.
1225 'cache' exists for backwards compatibility, and is not used.
1226 '''
1227 return Node(self, nodeid)
1229 def set(self, nodeid, **propvalues):
1230 '''Modify a property on an existing node of this class.
1232 'nodeid' must be the id of an existing node of this class or an
1233 IndexError is raised.
1235 Each key in 'propvalues' must be the name of a property of this
1236 class or a KeyError is raised.
1238 All values in 'propvalues' must be acceptable types for their
1239 corresponding properties or a TypeError is raised.
1241 If the value of the key property is set, it must not collide with
1242 other key strings or a ValueError is raised.
1244 If the value of a Link or Multilink property contains an invalid
1245 node id, a ValueError is raised.
1246 '''
1247 if not propvalues:
1248 return propvalues
1250 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1251 raise KeyError, '"creation" and "activity" are reserved'
1253 if propvalues.has_key('id'):
1254 raise KeyError, '"id" is reserved'
1256 if self.db.journaltag is None:
1257 raise DatabaseError, 'Database open read-only'
1259 self.fireAuditors('set', nodeid, propvalues)
1260 # Take a copy of the node dict so that the subsequent set
1261 # operation doesn't modify the oldvalues structure.
1262 # XXX used to try the cache here first
1263 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1265 node = self.db.getnode(self.classname, nodeid)
1266 if self.is_retired(nodeid):
1267 raise IndexError, 'Requested item is retired'
1268 num_re = re.compile('^\d+$')
1270 # if the journal value is to be different, store it in here
1271 journalvalues = {}
1273 # remember the add/remove stuff for multilinks, making it easier
1274 # for the Database layer to do its stuff
1275 multilink_changes = {}
1277 for propname, value in propvalues.items():
1278 # check to make sure we're not duplicating an existing key
1279 if propname == self.key and node[propname] != value:
1280 try:
1281 self.lookup(value)
1282 except KeyError:
1283 pass
1284 else:
1285 raise ValueError, 'node with key "%s" exists'%value
1287 # this will raise the KeyError if the property isn't valid
1288 # ... we don't use getprops() here because we only care about
1289 # the writeable properties.
1290 try:
1291 prop = self.properties[propname]
1292 except KeyError:
1293 raise KeyError, '"%s" has no property named "%s"'%(
1294 self.classname, propname)
1296 # if the value's the same as the existing value, no sense in
1297 # doing anything
1298 current = node.get(propname, None)
1299 if value == current:
1300 del propvalues[propname]
1301 continue
1302 journalvalues[propname] = current
1304 # do stuff based on the prop type
1305 if isinstance(prop, Link):
1306 link_class = prop.classname
1307 # if it isn't a number, it's a key
1308 if value is not None and not isinstance(value, type('')):
1309 raise ValueError, 'property "%s" link value be a string'%(
1310 propname)
1311 if isinstance(value, type('')) and not num_re.match(value):
1312 try:
1313 value = self.db.classes[link_class].lookup(value)
1314 except (TypeError, KeyError):
1315 raise IndexError, 'new property "%s": %s not a %s'%(
1316 propname, value, prop.classname)
1318 if (value is not None and
1319 not self.db.getclass(link_class).hasnode(value)):
1320 raise IndexError, '%s has no node %s'%(link_class, value)
1322 if self.do_journal and prop.do_journal:
1323 # register the unlink with the old linked node
1324 if node[propname] is not None:
1325 self.db.addjournal(link_class, node[propname], 'unlink',
1326 (self.classname, nodeid, propname))
1328 # register the link with the newly linked node
1329 if value is not None:
1330 self.db.addjournal(link_class, value, 'link',
1331 (self.classname, nodeid, propname))
1333 elif isinstance(prop, Multilink):
1334 if type(value) != type([]):
1335 raise TypeError, 'new property "%s" not a list of'\
1336 ' ids'%propname
1337 link_class = self.properties[propname].classname
1338 l = []
1339 for entry in value:
1340 # if it isn't a number, it's a key
1341 if type(entry) != type(''):
1342 raise ValueError, 'new property "%s" link value ' \
1343 'must be a string'%propname
1344 if not num_re.match(entry):
1345 try:
1346 entry = self.db.classes[link_class].lookup(entry)
1347 except (TypeError, KeyError):
1348 raise IndexError, 'new property "%s": %s not a %s'%(
1349 propname, entry,
1350 self.properties[propname].classname)
1351 l.append(entry)
1352 value = l
1353 propvalues[propname] = value
1355 # figure the journal entry for this property
1356 add = []
1357 remove = []
1359 # handle removals
1360 if node.has_key(propname):
1361 l = node[propname]
1362 else:
1363 l = []
1364 for id in l[:]:
1365 if id in value:
1366 continue
1367 # register the unlink with the old linked node
1368 if self.do_journal and self.properties[propname].do_journal:
1369 self.db.addjournal(link_class, id, 'unlink',
1370 (self.classname, nodeid, propname))
1371 l.remove(id)
1372 remove.append(id)
1374 # handle additions
1375 for id in value:
1376 if not self.db.getclass(link_class).hasnode(id):
1377 raise IndexError, '%s has no node %s'%(link_class, id)
1378 if id in l:
1379 continue
1380 # register the link with the newly linked node
1381 if self.do_journal and self.properties[propname].do_journal:
1382 self.db.addjournal(link_class, id, 'link',
1383 (self.classname, nodeid, propname))
1384 l.append(id)
1385 add.append(id)
1387 # figure the journal entry
1388 l = []
1389 if add:
1390 l.append(('+', add))
1391 if remove:
1392 l.append(('-', remove))
1393 multilink_changes[propname] = (add, remove)
1394 if l:
1395 journalvalues[propname] = tuple(l)
1397 elif isinstance(prop, String):
1398 if value is not None and type(value) != type('') and type(value) != type(u''):
1399 raise TypeError, 'new property "%s" not a string'%propname
1401 elif isinstance(prop, Password):
1402 if not isinstance(value, password.Password):
1403 raise TypeError, 'new property "%s" not a Password'%propname
1404 propvalues[propname] = value
1406 elif value is not None and isinstance(prop, Date):
1407 if not isinstance(value, date.Date):
1408 raise TypeError, 'new property "%s" not a Date'% propname
1409 propvalues[propname] = value
1411 elif value is not None and isinstance(prop, Interval):
1412 if not isinstance(value, date.Interval):
1413 raise TypeError, 'new property "%s" not an '\
1414 'Interval'%propname
1415 propvalues[propname] = value
1417 elif value is not None and isinstance(prop, Number):
1418 try:
1419 float(value)
1420 except ValueError:
1421 raise TypeError, 'new property "%s" not numeric'%propname
1423 elif value is not None and isinstance(prop, Boolean):
1424 try:
1425 int(value)
1426 except ValueError:
1427 raise TypeError, 'new property "%s" not boolean'%propname
1429 # nothing to do?
1430 if not propvalues:
1431 return propvalues
1433 # do the set, and journal it
1434 self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
1436 if self.do_journal:
1437 self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
1439 self.fireReactors('set', nodeid, oldvalues)
1441 return propvalues
1443 def retire(self, nodeid):
1444 '''Retire a node.
1446 The properties on the node remain available from the get() method,
1447 and the node's id is never reused.
1449 Retired nodes are not returned by the find(), list(), or lookup()
1450 methods, and other nodes may reuse the values of their key properties.
1451 '''
1452 if self.db.journaltag is None:
1453 raise DatabaseError, 'Database open read-only'
1455 self.fireAuditors('retire', nodeid, None)
1457 # use the arg for __retired__ to cope with any odd database type
1458 # conversion (hello, sqlite)
1459 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1460 self.db.arg, self.db.arg)
1461 if __debug__:
1462 print >>hyperdb.DEBUG, 'retire', (self, sql, nodeid)
1463 self.db.cursor.execute(sql, (1, nodeid))
1464 if self.do_journal:
1465 self.db.addjournal(self.classname, nodeid, 'retired', None)
1467 self.fireReactors('retire', nodeid, None)
1469 def restore(self, nodeid):
1470 '''Restore a retired node.
1472 Make node available for all operations like it was before retirement.
1473 '''
1474 if self.db.journaltag is None:
1475 raise DatabaseError, 'Database open read-only'
1477 node = self.db.getnode(self.classname, nodeid)
1478 # check if key property was overrided
1479 key = self.getkey()
1480 try:
1481 id = self.lookup(node[key])
1482 except KeyError:
1483 pass
1484 else:
1485 raise KeyError, "Key property (%s) of retired node clashes with \
1486 existing one (%s)" % (key, node[key])
1488 self.fireAuditors('restore', nodeid, None)
1489 # use the arg for __retired__ to cope with any odd database type
1490 # conversion (hello, sqlite)
1491 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1492 self.db.arg, self.db.arg)
1493 if __debug__:
1494 print >>hyperdb.DEBUG, 'restore', (self, sql, nodeid)
1495 self.db.cursor.execute(sql, (0, nodeid))
1496 if self.do_journal:
1497 self.db.addjournal(self.classname, nodeid, 'restored', None)
1499 self.fireReactors('restore', nodeid, None)
1501 def is_retired(self, nodeid):
1502 '''Return true if the node is rerired
1503 '''
1504 sql = 'select __retired__ from _%s where id=%s'%(self.classname,
1505 self.db.arg)
1506 if __debug__:
1507 print >>hyperdb.DEBUG, 'is_retired', (self, sql, nodeid)
1508 self.db.cursor.execute(sql, (nodeid,))
1509 return int(self.db.sql_fetchone()[0])
1511 def destroy(self, nodeid):
1512 '''Destroy a node.
1514 WARNING: this method should never be used except in extremely rare
1515 situations where there could never be links to the node being
1516 deleted
1517 WARNING: use retire() instead
1518 WARNING: the properties of this node will not be available ever again
1519 WARNING: really, use retire() instead
1521 Well, I think that's enough warnings. This method exists mostly to
1522 support the session storage of the cgi interface.
1524 The node is completely removed from the hyperdb, including all journal
1525 entries. It will no longer be available, and will generally break code
1526 if there are any references to the node.
1527 '''
1528 if self.db.journaltag is None:
1529 raise DatabaseError, 'Database open read-only'
1530 self.db.destroynode(self.classname, nodeid)
1532 def history(self, nodeid):
1533 '''Retrieve the journal of edits on a particular node.
1535 'nodeid' must be the id of an existing node of this class or an
1536 IndexError is raised.
1538 The returned list contains tuples of the form
1540 (nodeid, date, tag, action, params)
1542 'date' is a Timestamp object specifying the time of the change and
1543 'tag' is the journaltag specified when the database was opened.
1544 '''
1545 if not self.do_journal:
1546 raise ValueError, 'Journalling is disabled for this class'
1547 return self.db.getjournal(self.classname, nodeid)
1549 # Locating nodes:
1550 def hasnode(self, nodeid):
1551 '''Determine if the given nodeid actually exists
1552 '''
1553 return self.db.hasnode(self.classname, nodeid)
1555 def setkey(self, propname):
1556 '''Select a String property of this class to be the key property.
1558 'propname' must be the name of a String property of this class or
1559 None, or a TypeError is raised. The values of the key property on
1560 all existing nodes must be unique or a ValueError is raised.
1561 '''
1562 # XXX create an index on the key prop column
1563 prop = self.getprops()[propname]
1564 if not isinstance(prop, String):
1565 raise TypeError, 'key properties must be String'
1566 self.key = propname
1568 def getkey(self):
1569 '''Return the name of the key property for this class or None.'''
1570 return self.key
1572 def labelprop(self, default_to_id=0):
1573 ''' Return the property name for a label for the given node.
1575 This method attempts to generate a consistent label for the node.
1576 It tries the following in order:
1577 1. key property
1578 2. "name" property
1579 3. "title" property
1580 4. first property from the sorted property name list
1581 '''
1582 k = self.getkey()
1583 if k:
1584 return k
1585 props = self.getprops()
1586 if props.has_key('name'):
1587 return 'name'
1588 elif props.has_key('title'):
1589 return 'title'
1590 if default_to_id:
1591 return 'id'
1592 props = props.keys()
1593 props.sort()
1594 return props[0]
1596 def lookup(self, keyvalue):
1597 '''Locate a particular node by its key property and return its id.
1599 If this class has no key property, a TypeError is raised. If the
1600 'keyvalue' matches one of the values for the key property among
1601 the nodes in this class, the matching node's id is returned;
1602 otherwise a KeyError is raised.
1603 '''
1604 if not self.key:
1605 raise TypeError, 'No key property set for class %s'%self.classname
1607 # use the arg to handle any odd database type conversion (hello,
1608 # sqlite)
1609 sql = "select id from _%s where _%s=%s and __retired__ <> %s"%(
1610 self.classname, self.key, self.db.arg, self.db.arg)
1611 self.db.sql(sql, (keyvalue, 1))
1613 # see if there was a result that's not retired
1614 row = self.db.sql_fetchone()
1615 if not row:
1616 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1617 keyvalue, self.classname)
1619 # return the id
1620 return row[0]
1622 def find(self, **propspec):
1623 '''Get the ids of nodes in this class which link to the given nodes.
1625 'propspec' consists of keyword args propname=nodeid or
1626 propname={nodeid:1, }
1627 'propname' must be the name of a property in this class, or a
1628 KeyError is raised. That property must be a Link or Multilink
1629 property, or a TypeError is raised.
1631 Any node in this class whose 'propname' property links to any of the
1632 nodeids will be returned. Used by the full text indexing, which knows
1633 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1634 issues:
1636 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1637 '''
1638 if __debug__:
1639 print >>hyperdb.DEBUG, 'find', (self, propspec)
1641 # shortcut
1642 if not propspec:
1643 return []
1645 # validate the args
1646 props = self.getprops()
1647 propspec = propspec.items()
1648 for propname, nodeids in propspec:
1649 # check the prop is OK
1650 prop = props[propname]
1651 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1652 raise TypeError, "'%s' not a Link/Multilink property"%propname
1654 # first, links
1655 where = []
1656 allvalues = ()
1657 a = self.db.arg
1658 for prop, values in propspec:
1659 if not isinstance(props[prop], hyperdb.Link):
1660 continue
1661 if type(values) is type(''):
1662 allvalues += (values,)
1663 where.append('_%s = %s'%(prop, a))
1664 elif values is None:
1665 where.append('_%s is NULL'%prop)
1666 else:
1667 allvalues += tuple(values.keys())
1668 where.append('_%s in (%s)'%(prop, ','.join([a]*len(values))))
1669 tables = []
1670 if where:
1671 tables.append('select id as nodeid from _%s where %s'%(
1672 self.classname, ' and '.join(where)))
1674 # now multilinks
1675 for prop, values in propspec:
1676 if not isinstance(props[prop], hyperdb.Multilink):
1677 continue
1678 if type(values) is type(''):
1679 allvalues += (values,)
1680 s = a
1681 else:
1682 allvalues += tuple(values.keys())
1683 s = ','.join([a]*len(values))
1684 tables.append('select nodeid from %s_%s where linkid in (%s)'%(
1685 self.classname, prop, s))
1686 sql = '\nunion\n'.join(tables)
1687 self.db.sql(sql, allvalues)
1688 l = [x[0] for x in self.db.sql_fetchall()]
1689 if __debug__:
1690 print >>hyperdb.DEBUG, 'find ... ', l
1691 return l
1693 def stringFind(self, **requirements):
1694 '''Locate a particular node by matching a set of its String
1695 properties in a caseless search.
1697 If the property is not a String property, a TypeError is raised.
1699 The return is a list of the id of all nodes that match.
1700 '''
1701 where = []
1702 args = []
1703 for propname in requirements.keys():
1704 prop = self.properties[propname]
1705 if isinstance(not prop, String):
1706 raise TypeError, "'%s' not a String property"%propname
1707 where.append(propname)
1708 args.append(requirements[propname].lower())
1710 # generate the where clause
1711 s = ' and '.join(['lower(_%s)=%s'%(col, self.db.arg) for col in where])
1712 sql = 'select id from _%s where %s'%(self.classname, s)
1713 self.db.sql(sql, tuple(args))
1714 l = [x[0] for x in self.db.sql_fetchall()]
1715 if __debug__:
1716 print >>hyperdb.DEBUG, 'find ... ', l
1717 return l
1719 def list(self):
1720 ''' Return a list of the ids of the active nodes in this class.
1721 '''
1722 return self.getnodeids(retired=0)
1724 def getnodeids(self, retired=None):
1725 ''' Retrieve all the ids of the nodes for a particular Class.
1727 Set retired=None to get all nodes. Otherwise it'll get all the
1728 retired or non-retired nodes, depending on the flag.
1729 '''
1730 # flip the sense of the 'retired' flag if we don't want all of them
1731 if retired is not None:
1732 if retired:
1733 args = (0, )
1734 else:
1735 args = (1, )
1736 sql = 'select id from _%s where __retired__ <> %s'%(self.classname,
1737 self.db.arg)
1738 else:
1739 args = ()
1740 sql = 'select id from _%s'%self.classname
1741 if __debug__:
1742 print >>hyperdb.DEBUG, 'getnodeids', (self, sql, retired)
1743 self.db.cursor.execute(sql, args)
1744 ids = [x[0] for x in self.db.cursor.fetchall()]
1745 return ids
1747 def filter(self, search_matches, filterspec, sort=(None,None),
1748 group=(None,None)):
1749 ''' Return a list of the ids of the active nodes in this class that
1750 match the 'filter' spec, sorted by the group spec and then the
1751 sort spec
1753 "filterspec" is {propname: value(s)}
1754 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1755 and prop is a prop name or None
1756 "search_matches" is {nodeid: marker}
1758 The filter must match all properties specificed - but if the
1759 property value to match is a list, any one of the values in the
1760 list may match for that property to match.
1761 '''
1762 # just don't bother if the full-text search matched diddly
1763 if search_matches == {}:
1764 return []
1766 cn = self.classname
1768 timezone = self.db.getUserTimezone()
1770 # figure the WHERE clause from the filterspec
1771 props = self.getprops()
1772 frum = ['_'+cn]
1773 where = []
1774 args = []
1775 a = self.db.arg
1776 for k, v in filterspec.items():
1777 propclass = props[k]
1778 # now do other where clause stuff
1779 if isinstance(propclass, Multilink):
1780 tn = '%s_%s'%(cn, k)
1781 if v in ('-1', ['-1']):
1782 # only match rows that have count(linkid)=0 in the
1783 # corresponding multilink table)
1784 where.append('id not in (select nodeid from %s)'%tn)
1785 elif isinstance(v, type([])):
1786 frum.append(tn)
1787 s = ','.join([a for x in v])
1788 where.append('id=%s.nodeid and %s.linkid in (%s)'%(tn,tn,s))
1789 args = args + v
1790 else:
1791 frum.append(tn)
1792 where.append('id=%s.nodeid and %s.linkid=%s'%(tn, tn, a))
1793 args.append(v)
1794 elif k == 'id':
1795 if isinstance(v, type([])):
1796 s = ','.join([a for x in v])
1797 where.append('%s in (%s)'%(k, s))
1798 args = args + v
1799 else:
1800 where.append('%s=%s'%(k, a))
1801 args.append(v)
1802 elif isinstance(propclass, String):
1803 if not isinstance(v, type([])):
1804 v = [v]
1806 # Quote the bits in the string that need it and then embed
1807 # in a "substring" search. Note - need to quote the '%' so
1808 # they make it through the python layer happily
1809 v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
1811 # now add to the where clause
1812 where.append(' or '.join(["_%s LIKE '%s'"%(k, s) for s in v]))
1813 # note: args are embedded in the query string now
1814 elif isinstance(propclass, Link):
1815 if isinstance(v, type([])):
1816 if '-1' in v:
1817 v = v[:]
1818 v.remove('-1')
1819 xtra = ' or _%s is NULL'%k
1820 else:
1821 xtra = ''
1822 if v:
1823 s = ','.join([a for x in v])
1824 where.append('(_%s in (%s)%s)'%(k, s, xtra))
1825 args = args + v
1826 else:
1827 where.append('_%s is NULL'%k)
1828 else:
1829 if v == '-1':
1830 v = None
1831 where.append('_%s is NULL'%k)
1832 else:
1833 where.append('_%s=%s'%(k, a))
1834 args.append(v)
1835 elif isinstance(propclass, Date):
1836 if isinstance(v, type([])):
1837 s = ','.join([a for x in v])
1838 where.append('_%s in (%s)'%(k, s))
1839 args = args + [date.Date(x).serialise() for x in v]
1840 else:
1841 try:
1842 # Try to filter on range of dates
1843 date_rng = Range(v, date.Date, offset=timezone)
1844 if (date_rng.from_value):
1845 where.append('_%s >= %s'%(k, a))
1846 args.append(date_rng.from_value.serialise())
1847 if (date_rng.to_value):
1848 where.append('_%s <= %s'%(k, a))
1849 args.append(date_rng.to_value.serialise())
1850 except ValueError:
1851 # If range creation fails - ignore that search parameter
1852 pass
1853 elif isinstance(propclass, Interval):
1854 if isinstance(v, type([])):
1855 s = ','.join([a for x in v])
1856 where.append('_%s in (%s)'%(k, s))
1857 args = args + [date.Interval(x).serialise() for x in v]
1858 else:
1859 try:
1860 # Try to filter on range of intervals
1861 date_rng = Range(v, date.Interval)
1862 if (date_rng.from_value):
1863 where.append('_%s >= %s'%(k, a))
1864 args.append(date_rng.from_value.serialise())
1865 if (date_rng.to_value):
1866 where.append('_%s <= %s'%(k, a))
1867 args.append(date_rng.to_value.serialise())
1868 except ValueError:
1869 # If range creation fails - ignore that search parameter
1870 pass
1871 #where.append('_%s=%s'%(k, a))
1872 #args.append(date.Interval(v).serialise())
1873 else:
1874 if isinstance(v, type([])):
1875 s = ','.join([a for x in v])
1876 where.append('_%s in (%s)'%(k, s))
1877 args = args + v
1878 else:
1879 where.append('_%s=%s'%(k, a))
1880 args.append(v)
1882 # don't match retired nodes
1883 where.append('__retired__ <> 1')
1885 # add results of full text search
1886 if search_matches is not None:
1887 v = search_matches.keys()
1888 s = ','.join([a for x in v])
1889 where.append('id in (%s)'%s)
1890 args = args + v
1892 # "grouping" is just the first-order sorting in the SQL fetch
1893 # can modify it...)
1894 orderby = []
1895 ordercols = []
1896 if group[0] is not None and group[1] is not None:
1897 if group[0] != '-':
1898 orderby.append('_'+group[1])
1899 ordercols.append('_'+group[1])
1900 else:
1901 orderby.append('_'+group[1]+' desc')
1902 ordercols.append('_'+group[1])
1904 # now add in the sorting
1905 group = ''
1906 if sort[0] is not None and sort[1] is not None:
1907 direction, colname = sort
1908 if direction != '-':
1909 if colname == 'id':
1910 orderby.append(colname)
1911 else:
1912 orderby.append('_'+colname)
1913 ordercols.append('_'+colname)
1914 else:
1915 if colname == 'id':
1916 orderby.append(colname+' desc')
1917 ordercols.append(colname)
1918 else:
1919 orderby.append('_'+colname+' desc')
1920 ordercols.append('_'+colname)
1922 # construct the SQL
1923 frum = ','.join(frum)
1924 if where:
1925 where = ' where ' + (' and '.join(where))
1926 else:
1927 where = ''
1928 cols = ['id']
1929 if orderby:
1930 cols = cols + ordercols
1931 order = ' order by %s'%(','.join(orderby))
1932 else:
1933 order = ''
1934 cols = ','.join(cols)
1935 sql = 'select %s from %s %s%s%s'%(cols, frum, where, group, order)
1936 args = tuple(args)
1937 if __debug__:
1938 print >>hyperdb.DEBUG, 'filter', (self, sql, args)
1939 self.db.cursor.execute(sql, args)
1940 l = self.db.cursor.fetchall()
1942 # return the IDs (the first column)
1943 # XXX The filter(None, l) bit is sqlite-specific... if there's _NO_
1944 # XXX matches to a fetch, it returns NULL instead of nothing!?!
1945 return filter(None, [row[0] for row in l])
1947 def count(self):
1948 '''Get the number of nodes in this class.
1950 If the returned integer is 'numnodes', the ids of all the nodes
1951 in this class run from 1 to numnodes, and numnodes+1 will be the
1952 id of the next node to be created in this class.
1953 '''
1954 return self.db.countnodes(self.classname)
1956 # Manipulating properties:
1957 def getprops(self, protected=1):
1958 '''Return a dictionary mapping property names to property objects.
1959 If the "protected" flag is true, we include protected properties -
1960 those which may not be modified.
1961 '''
1962 d = self.properties.copy()
1963 if protected:
1964 d['id'] = String()
1965 d['creation'] = hyperdb.Date()
1966 d['activity'] = hyperdb.Date()
1967 d['creator'] = hyperdb.Link('user')
1968 return d
1970 def addprop(self, **properties):
1971 '''Add properties to this class.
1973 The keyword arguments in 'properties' must map names to property
1974 objects, or a TypeError is raised. None of the keys in 'properties'
1975 may collide with the names of existing properties, or a ValueError
1976 is raised before any properties have been added.
1977 '''
1978 for key in properties.keys():
1979 if self.properties.has_key(key):
1980 raise ValueError, key
1981 self.properties.update(properties)
1983 def index(self, nodeid):
1984 '''Add (or refresh) the node to search indexes
1985 '''
1986 # find all the String properties that have indexme
1987 for prop, propclass in self.getprops().items():
1988 if isinstance(propclass, String) and propclass.indexme:
1989 try:
1990 value = str(self.get(nodeid, prop))
1991 except IndexError:
1992 # node no longer exists - entry should be removed
1993 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1994 else:
1995 # and index them under (classname, nodeid, property)
1996 self.db.indexer.add_text((self.classname, nodeid, prop),
1997 value)
2000 #
2001 # Detector interface
2002 #
2003 def audit(self, event, detector):
2004 '''Register a detector
2005 '''
2006 l = self.auditors[event]
2007 if detector not in l:
2008 self.auditors[event].append(detector)
2010 def fireAuditors(self, action, nodeid, newvalues):
2011 '''Fire all registered auditors.
2012 '''
2013 for audit in self.auditors[action]:
2014 audit(self.db, self, nodeid, newvalues)
2016 def react(self, event, detector):
2017 '''Register a detector
2018 '''
2019 l = self.reactors[event]
2020 if detector not in l:
2021 self.reactors[event].append(detector)
2023 def fireReactors(self, action, nodeid, oldvalues):
2024 '''Fire all registered reactors.
2025 '''
2026 for react in self.reactors[action]:
2027 react(self.db, self, nodeid, oldvalues)
2029 class FileClass(Class, hyperdb.FileClass):
2030 '''This class defines a large chunk of data. To support this, it has a
2031 mandatory String property "content" which is typically saved off
2032 externally to the hyperdb.
2034 The default MIME type of this data is defined by the
2035 "default_mime_type" class attribute, which may be overridden by each
2036 node if the class defines a "type" String property.
2037 '''
2038 default_mime_type = 'text/plain'
2040 def create(self, **propvalues):
2041 ''' snaffle the file propvalue and store in a file
2042 '''
2043 # we need to fire the auditors now, or the content property won't
2044 # be in propvalues for the auditors to play with
2045 self.fireAuditors('create', None, propvalues)
2047 # now remove the content property so it's not stored in the db
2048 content = propvalues['content']
2049 del propvalues['content']
2051 # do the database create
2052 newid = Class.create_inner(self, **propvalues)
2054 # fire reactors
2055 self.fireReactors('create', newid, None)
2057 # store off the content as a file
2058 self.db.storefile(self.classname, newid, None, content)
2059 return newid
2061 def import_list(self, propnames, proplist):
2062 ''' Trap the "content" property...
2063 '''
2064 # dupe this list so we don't affect others
2065 propnames = propnames[:]
2067 # extract the "content" property from the proplist
2068 i = propnames.index('content')
2069 content = eval(proplist[i])
2070 del propnames[i]
2071 del proplist[i]
2073 # do the normal import
2074 newid = Class.import_list(self, propnames, proplist)
2076 # save off the "content" file
2077 self.db.storefile(self.classname, newid, None, content)
2078 return newid
2080 _marker = []
2081 def get(self, nodeid, propname, default=_marker, cache=1):
2082 ''' Trap the content propname and get it from the file
2084 'cache' exists for backwards compatibility, and is not used.
2085 '''
2086 poss_msg = 'Possibly a access right configuration problem.'
2087 if propname == 'content':
2088 try:
2089 return self.db.getfile(self.classname, nodeid, None)
2090 except IOError, (strerror):
2091 # BUG: by catching this we donot see an error in the log.
2092 return 'ERROR reading file: %s%s\n%s\n%s'%(
2093 self.classname, nodeid, poss_msg, strerror)
2094 if default is not self._marker:
2095 return Class.get(self, nodeid, propname, default)
2096 else:
2097 return Class.get(self, nodeid, propname)
2099 def getprops(self, protected=1):
2100 ''' In addition to the actual properties on the node, these methods
2101 provide the "content" property. If the "protected" flag is true,
2102 we include protected properties - those which may not be
2103 modified.
2104 '''
2105 d = Class.getprops(self, protected=protected).copy()
2106 d['content'] = hyperdb.String()
2107 return d
2109 def index(self, nodeid):
2110 ''' Index the node in the search index.
2112 We want to index the content in addition to the normal String
2113 property indexing.
2114 '''
2115 # perform normal indexing
2116 Class.index(self, nodeid)
2118 # get the content to index
2119 content = self.get(nodeid, 'content')
2121 # figure the mime type
2122 if self.properties.has_key('type'):
2123 mime_type = self.get(nodeid, 'type')
2124 else:
2125 mime_type = self.default_mime_type
2127 # and index!
2128 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
2129 mime_type)
2131 # XXX deviation from spec - was called ItemClass
2132 class IssueClass(Class, roundupdb.IssueClass):
2133 # Overridden methods:
2134 def __init__(self, db, classname, **properties):
2135 '''The newly-created class automatically includes the "messages",
2136 "files", "nosy", and "superseder" properties. If the 'properties'
2137 dictionary attempts to specify any of these properties or a
2138 "creation" or "activity" property, a ValueError is raised.
2139 '''
2140 if not properties.has_key('title'):
2141 properties['title'] = hyperdb.String(indexme='yes')
2142 if not properties.has_key('messages'):
2143 properties['messages'] = hyperdb.Multilink("msg")
2144 if not properties.has_key('files'):
2145 properties['files'] = hyperdb.Multilink("file")
2146 if not properties.has_key('nosy'):
2147 # note: journalling is turned off as it really just wastes
2148 # space. this behaviour may be overridden in an instance
2149 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2150 if not properties.has_key('superseder'):
2151 properties['superseder'] = hyperdb.Multilink(classname)
2152 Class.__init__(self, db, classname, **properties)