1 # $Id: rdbms_common.py,v 1.29 2003-01-15 22:17:19 kedder Exp $
2 ''' Relational database (SQL) backend common code.
4 Basics:
6 - map roundup classes to relational tables
7 - automatically detect schema changes and modify the table schemas
8 appropriately (we store the "database version" of the schema in the
9 database itself as the only row of the "schema" table)
10 - multilinks (which represent a many-to-many relationship) are handled through
11 intermediate tables
12 - journals are stored adjunct to the per-class tables
13 - table names and columns have "_" prepended so the names can't clash with
14 restricted names (like "order")
15 - retirement is determined by the __retired__ column being true
17 Database-specific changes may generally be pushed out to the overridable
18 sql_* methods, since everything else should be fairly generic. There's
19 probably a bit of work to be done if a database is used that actually
20 honors column typing, since the initial databases don't (sqlite stores
21 everything as a string, and gadfly stores anything that's marsallable).
22 '''
24 # standard python modules
25 import sys, os, time, re, errno, weakref, copy
27 # roundup modules
28 from roundup import hyperdb, date, password, roundupdb, security
29 from roundup.hyperdb import String, Password, Date, Interval, Link, \
30 Multilink, DatabaseError, Boolean, Number
31 from roundup.backends import locking
33 # support
34 from blobfiles import FileStorage
35 from roundup.indexer import Indexer
36 from sessions import Sessions
38 # number of rows to keep in memory
39 ROW_CACHE_SIZE = 100
41 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
42 ''' Wrapper around an SQL database that presents a hyperdb interface.
44 - some functionality is specific to the actual SQL database, hence
45 the sql_* methods that are NotImplemented
46 - we keep a cache of the latest ROW_CACHE_SIZE row fetches.
47 '''
48 def __init__(self, config, journaltag=None):
49 ''' Open the database and load the schema from it.
50 '''
51 self.config, self.journaltag = config, journaltag
52 self.dir = config.DATABASE
53 self.classes = {}
54 self.indexer = Indexer(self.dir)
55 self.sessions = Sessions(self.config)
56 self.security = security.Security(self)
58 # additional transaction support for external files and the like
59 self.transactions = []
61 # keep a cache of the N most recently retrieved rows of any kind
62 # (classname, nodeid) = row
63 self.cache = {}
64 self.cache_lru = []
66 # database lock
67 self.lockfile = None
69 # open a connection to the database, creating the "conn" attribute
70 self.open_connection()
72 def clearCache(self):
73 self.cache = {}
74 self.cache_lru = []
76 def open_connection(self):
77 ''' Open a connection to the database, creating it if necessary
78 '''
79 raise NotImplemented
81 def sql(self, sql, args=None):
82 ''' Execute the sql with the optional args.
83 '''
84 if __debug__:
85 print >>hyperdb.DEBUG, (self, sql, args)
86 if args:
87 self.cursor.execute(sql, args)
88 else:
89 self.cursor.execute(sql)
91 def sql_fetchone(self):
92 ''' Fetch a single row. If there's nothing to fetch, return None.
93 '''
94 raise NotImplemented
96 def sql_stringquote(self, value):
97 ''' Quote the string so it's safe to put in the 'sql quotes'
98 '''
99 return re.sub("'", "''", str(value))
101 def save_dbschema(self, schema):
102 ''' Save the schema definition that the database currently implements
103 '''
104 raise NotImplemented
106 def load_dbschema(self):
107 ''' Load the schema definition that the database currently implements
108 '''
109 raise NotImplemented
111 def post_init(self):
112 ''' Called once the schema initialisation has finished.
114 We should now confirm that the schema defined by our "classes"
115 attribute actually matches the schema in the database.
116 '''
117 # now detect changes in the schema
118 save = 0
119 for classname, spec in self.classes.items():
120 if self.database_schema.has_key(classname):
121 dbspec = self.database_schema[classname]
122 if self.update_class(spec, dbspec):
123 self.database_schema[classname] = spec.schema()
124 save = 1
125 else:
126 self.create_class(spec)
127 self.database_schema[classname] = spec.schema()
128 save = 1
130 for classname in self.database_schema.keys():
131 if not self.classes.has_key(classname):
132 self.drop_class(classname)
134 # update the database version of the schema
135 if save:
136 self.sql('delete from schema')
137 self.save_dbschema(self.database_schema)
139 # reindex the db if necessary
140 if self.indexer.should_reindex():
141 self.reindex()
143 # commit
144 self.conn.commit()
146 # figure the "curuserid"
147 if self.journaltag is None:
148 self.curuserid = None
149 elif self.journaltag == 'admin':
150 # admin user may not exist, but always has ID 1
151 self.curuserid = '1'
152 else:
153 self.curuserid = self.user.lookup(self.journaltag)
155 def reindex(self):
156 for klass in self.classes.values():
157 for nodeid in klass.list():
158 klass.index(nodeid)
159 self.indexer.save_index()
161 def determine_columns(self, properties):
162 ''' Figure the column names and multilink properties from the spec
164 "properties" is a list of (name, prop) where prop may be an
165 instance of a hyperdb "type" _or_ a string repr of that type.
166 '''
167 cols = ['_activity', '_creator', '_creation']
168 mls = []
169 # add the multilinks separately
170 for col, prop in properties:
171 if isinstance(prop, Multilink):
172 mls.append(col)
173 elif isinstance(prop, type('')) and prop.find('Multilink') != -1:
174 mls.append(col)
175 else:
176 cols.append('_'+col)
177 cols.sort()
178 return cols, mls
180 def update_class(self, spec, dbspec):
181 ''' Determine the differences between the current spec and the
182 database version of the spec, and update where necessary
183 '''
184 spec_schema = spec.schema()
185 if spec_schema == dbspec:
186 # no save needed for this one
187 return 0
188 if __debug__:
189 print >>hyperdb.DEBUG, 'update_class FIRING'
191 # key property changed?
192 if dbspec[0] != spec_schema[0]:
193 if __debug__:
194 print >>hyperdb.DEBUG, 'update_class setting keyprop', `spec[0]`
195 # XXX turn on indexing for the key property
197 # dict 'em up
198 spec_propnames,spec_props = [],{}
199 for propname,prop in spec_schema[1]:
200 spec_propnames.append(propname)
201 spec_props[propname] = prop
202 dbspec_propnames,dbspec_props = [],{}
203 for propname,prop in dbspec[1]:
204 dbspec_propnames.append(propname)
205 dbspec_props[propname] = prop
207 # now compare
208 for propname in spec_propnames:
209 prop = spec_props[propname]
210 if dbspec_props.has_key(propname) and prop==dbspec_props[propname]:
211 continue
212 if __debug__:
213 print >>hyperdb.DEBUG, 'update_class ADD', (propname, prop)
215 if not dbspec_props.has_key(propname):
216 # add the property
217 if isinstance(prop, Multilink):
218 # all we have to do here is create a new table, easy!
219 self.create_multilink_table(spec, propname)
220 continue
222 # no ALTER TABLE, so we:
223 # 1. pull out the data, including an extra None column
224 oldcols, x = self.determine_columns(dbspec[1])
225 oldcols.append('id')
226 oldcols.append('__retired__')
227 cn = spec.classname
228 sql = 'select %s,%s from _%s'%(','.join(oldcols), self.arg, cn)
229 if __debug__:
230 print >>hyperdb.DEBUG, 'update_class', (self, sql, None)
231 self.cursor.execute(sql, (None,))
232 olddata = self.cursor.fetchall()
234 # 2. drop the old table
235 self.cursor.execute('drop table _%s'%cn)
237 # 3. create the new table
238 cols, mls = self.create_class_table(spec)
239 # ensure the new column is last
240 cols.remove('_'+propname)
241 assert oldcols == cols, "Column lists don't match!"
242 cols.append('_'+propname)
244 # 4. populate with the data from step one
245 s = ','.join([self.arg for x in cols])
246 scols = ','.join(cols)
247 sql = 'insert into _%s (%s) values (%s)'%(cn, scols, s)
249 # GAH, nothing had better go wrong from here on in... but
250 # we have to commit the drop...
251 # XXX this isn't necessary in sqlite :(
252 self.conn.commit()
254 # do the insert
255 for row in olddata:
256 self.sql(sql, tuple(row))
258 else:
259 # modify the property
260 if __debug__:
261 print >>hyperdb.DEBUG, 'update_class NOOP'
262 pass # NOOP in gadfly
264 # and the other way - only worry about deletions here
265 for propname in dbspec_propnames:
266 prop = dbspec_props[propname]
267 if spec_props.has_key(propname):
268 continue
269 if __debug__:
270 print >>hyperdb.DEBUG, 'update_class REMOVE', `prop`
272 # delete the property
273 if isinstance(prop, Multilink):
274 sql = 'drop table %s_%s'%(spec.classname, prop)
275 if __debug__:
276 print >>hyperdb.DEBUG, 'update_class', (self, sql)
277 self.cursor.execute(sql)
278 else:
279 # no ALTER TABLE, so we:
280 # 1. pull out the data, excluding the removed column
281 oldcols, x = self.determine_columns(spec.properties.items())
282 oldcols.append('id')
283 oldcols.append('__retired__')
284 # remove the missing column
285 oldcols.remove('_'+propname)
286 cn = spec.classname
287 sql = 'select %s from _%s'%(','.join(oldcols), cn)
288 self.cursor.execute(sql, (None,))
289 olddata = sql.fetchall()
291 # 2. drop the old table
292 self.cursor.execute('drop table _%s'%cn)
294 # 3. create the new table
295 cols, mls = self.create_class_table(self, spec)
296 assert oldcols != cols, "Column lists don't match!"
298 # 4. populate with the data from step one
299 qs = ','.join([self.arg for x in cols])
300 sql = 'insert into _%s values (%s)'%(cn, s)
301 self.cursor.execute(sql, olddata)
302 return 1
304 def create_class_table(self, spec):
305 ''' create the class table for the given spec
306 '''
307 cols, mls = self.determine_columns(spec.properties.items())
309 # add on our special columns
310 cols.append('id')
311 cols.append('__retired__')
313 # create the base table
314 scols = ','.join(['%s varchar'%x for x in cols])
315 sql = 'create table _%s (%s)'%(spec.classname, scols)
316 if __debug__:
317 print >>hyperdb.DEBUG, 'create_class', (self, sql)
318 self.cursor.execute(sql)
320 return cols, mls
322 def create_journal_table(self, spec):
323 ''' create the journal table for a class given the spec and
324 already-determined cols
325 '''
326 # journal table
327 cols = ','.join(['%s varchar'%x
328 for x in 'nodeid date tag action params'.split()])
329 sql = 'create table %s__journal (%s)'%(spec.classname, cols)
330 if __debug__:
331 print >>hyperdb.DEBUG, 'create_class', (self, sql)
332 self.cursor.execute(sql)
334 def create_multilink_table(self, spec, ml):
335 ''' Create a multilink table for the "ml" property of the class
336 given by the spec
337 '''
338 sql = 'create table %s_%s (linkid varchar, nodeid varchar)'%(
339 spec.classname, ml)
340 if __debug__:
341 print >>hyperdb.DEBUG, 'create_class', (self, sql)
342 self.cursor.execute(sql)
344 def create_class(self, spec):
345 ''' Create a database table according to the given spec.
346 '''
347 cols, mls = self.create_class_table(spec)
348 self.create_journal_table(spec)
350 # now create the multilink tables
351 for ml in mls:
352 self.create_multilink_table(spec, ml)
354 # ID counter
355 sql = 'insert into ids (name, num) values (%s,%s)'%(self.arg, self.arg)
356 vals = (spec.classname, 1)
357 if __debug__:
358 print >>hyperdb.DEBUG, 'create_class', (self, sql, vals)
359 self.cursor.execute(sql, vals)
361 def drop_class(self, spec):
362 ''' Drop the given table from the database.
364 Drop the journal and multilink tables too.
365 '''
366 # figure the multilinks
367 mls = []
368 for col, prop in spec.properties.items():
369 if isinstance(prop, Multilink):
370 mls.append(col)
372 sql = 'drop table _%s'%spec.classname
373 if __debug__:
374 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
375 self.cursor.execute(sql)
377 sql = 'drop table %s__journal'%spec.classname
378 if __debug__:
379 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
380 self.cursor.execute(sql)
382 for ml in mls:
383 sql = 'drop table %s_%s'%(spec.classname, ml)
384 if __debug__:
385 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
386 self.cursor.execute(sql)
388 #
389 # Classes
390 #
391 def __getattr__(self, classname):
392 ''' A convenient way of calling self.getclass(classname).
393 '''
394 if self.classes.has_key(classname):
395 if __debug__:
396 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
397 return self.classes[classname]
398 raise AttributeError, classname
400 def addclass(self, cl):
401 ''' Add a Class to the hyperdatabase.
402 '''
403 if __debug__:
404 print >>hyperdb.DEBUG, 'addclass', (self, cl)
405 cn = cl.classname
406 if self.classes.has_key(cn):
407 raise ValueError, cn
408 self.classes[cn] = cl
410 def getclasses(self):
411 ''' Return a list of the names of all existing classes.
412 '''
413 if __debug__:
414 print >>hyperdb.DEBUG, 'getclasses', (self,)
415 l = self.classes.keys()
416 l.sort()
417 return l
419 def getclass(self, classname):
420 '''Get the Class object representing a particular class.
422 If 'classname' is not a valid class name, a KeyError is raised.
423 '''
424 if __debug__:
425 print >>hyperdb.DEBUG, 'getclass', (self, classname)
426 try:
427 return self.classes[classname]
428 except KeyError:
429 raise KeyError, 'There is no class called "%s"'%classname
431 def clear(self):
432 ''' Delete all database contents.
434 Note: I don't commit here, which is different behaviour to the
435 "nuke from orbit" behaviour in the *dbms.
436 '''
437 if __debug__:
438 print >>hyperdb.DEBUG, 'clear', (self,)
439 for cn in self.classes.keys():
440 sql = 'delete from _%s'%cn
441 if __debug__:
442 print >>hyperdb.DEBUG, 'clear', (self, sql)
443 self.cursor.execute(sql)
445 #
446 # Node IDs
447 #
448 def newid(self, classname):
449 ''' Generate a new id for the given class
450 '''
451 # get the next ID
452 sql = 'select num from ids where name=%s'%self.arg
453 if __debug__:
454 print >>hyperdb.DEBUG, 'newid', (self, sql, classname)
455 self.cursor.execute(sql, (classname, ))
456 newid = self.cursor.fetchone()[0]
458 # update the counter
459 sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
460 vals = (int(newid)+1, classname)
461 if __debug__:
462 print >>hyperdb.DEBUG, 'newid', (self, sql, vals)
463 self.cursor.execute(sql, vals)
465 # return as string
466 return str(newid)
468 def setid(self, classname, setid):
469 ''' Set the id counter: used during import of database
470 '''
471 sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
472 vals = (setid, classname)
473 if __debug__:
474 print >>hyperdb.DEBUG, 'setid', (self, sql, vals)
475 self.cursor.execute(sql, vals)
477 #
478 # Nodes
479 #
481 def addnode(self, classname, nodeid, node):
482 ''' Add the specified node to its class's db.
483 '''
484 if __debug__:
485 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
486 # gadfly requires values for all non-multilink columns
487 cl = self.classes[classname]
488 cols, mls = self.determine_columns(cl.properties.items())
490 # we'll be supplied these props if we're doing an import
491 if not node.has_key('creator'):
492 # add in the "calculated" properties (dupe so we don't affect
493 # calling code's node assumptions)
494 node = node.copy()
495 node['creation'] = node['activity'] = date.Date()
496 node['creator'] = self.curuserid
498 # default the non-multilink columns
499 for col, prop in cl.properties.items():
500 if not isinstance(col, Multilink):
501 if not node.has_key(col):
502 node[col] = None
504 # clear this node out of the cache if it's in there
505 key = (classname, nodeid)
506 if self.cache.has_key(key):
507 del self.cache[key]
508 self.cache_lru.remove(key)
510 # make the node data safe for the DB
511 node = self.serialise(classname, node)
513 # make sure the ordering is correct for column name -> column value
514 vals = tuple([node[col[1:]] for col in cols]) + (nodeid, 0)
515 s = ','.join([self.arg for x in cols]) + ',%s,%s'%(self.arg, self.arg)
516 cols = ','.join(cols) + ',id,__retired__'
518 # perform the inserts
519 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
520 if __debug__:
521 print >>hyperdb.DEBUG, 'addnode', (self, sql, vals)
522 self.cursor.execute(sql, vals)
524 # insert the multilink rows
525 for col in mls:
526 t = '%s_%s'%(classname, col)
527 for entry in node[col]:
528 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
529 self.arg, self.arg)
530 self.sql(sql, (entry, nodeid))
532 # make sure we do the commit-time extra stuff for this node
533 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
535 def setnode(self, classname, nodeid, values, multilink_changes):
536 ''' Change the specified node.
537 '''
538 if __debug__:
539 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, values)
541 # clear this node out of the cache if it's in there
542 key = (classname, nodeid)
543 if self.cache.has_key(key):
544 del self.cache[key]
545 self.cache_lru.remove(key)
547 # add the special props
548 values = values.copy()
549 values['activity'] = date.Date()
551 # make db-friendly
552 values = self.serialise(classname, values)
554 cl = self.classes[classname]
555 cols = []
556 mls = []
557 # add the multilinks separately
558 props = cl.getprops()
559 for col in values.keys():
560 prop = props[col]
561 if isinstance(prop, Multilink):
562 mls.append(col)
563 else:
564 cols.append('_'+col)
565 cols.sort()
567 # if there's any updates to regular columns, do them
568 if cols:
569 # make sure the ordering is correct for column name -> column value
570 sqlvals = tuple([values[col[1:]] for col in cols]) + (nodeid,)
571 s = ','.join(['%s=%s'%(x, self.arg) for x in cols])
572 cols = ','.join(cols)
574 # perform the update
575 sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
576 if __debug__:
577 print >>hyperdb.DEBUG, 'setnode', (self, sql, sqlvals)
578 self.cursor.execute(sql, sqlvals)
580 # now the fun bit, updating the multilinks ;)
581 for col, (add, remove) in multilink_changes.items():
582 tn = '%s_%s'%(classname, col)
583 if add:
584 sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
585 self.arg, self.arg)
586 for addid in add:
587 self.sql(sql, (nodeid, addid))
588 if remove:
589 sql = 'delete from %s where nodeid=%s and linkid=%s'%(tn,
590 self.arg, self.arg)
591 for removeid in remove:
592 self.sql(sql, (nodeid, removeid))
594 # make sure we do the commit-time extra stuff for this node
595 self.transactions.append((self.doSaveNode, (classname, nodeid, values)))
597 def getnode(self, classname, nodeid):
598 ''' Get a node from the database.
599 '''
600 if __debug__:
601 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid)
603 # see if we have this node cached
604 key = (classname, nodeid)
605 if self.cache.has_key(key):
606 # push us back to the top of the LRU
607 self.cache_lru.remove(key)
608 self.cache_lru.insert(0, key)
609 # return the cached information
610 return self.cache[key]
612 # figure the columns we're fetching
613 cl = self.classes[classname]
614 cols, mls = self.determine_columns(cl.properties.items())
615 scols = ','.join(cols)
617 # perform the basic property fetch
618 sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
619 self.sql(sql, (nodeid,))
621 values = self.sql_fetchone()
622 if values is None:
623 raise IndexError, 'no such %s node %s'%(classname, nodeid)
625 # make up the node
626 node = {}
627 for col in range(len(cols)):
628 node[cols[col][1:]] = values[col]
630 # now the multilinks
631 for col in mls:
632 # get the link ids
633 sql = 'select linkid from %s_%s where nodeid=%s'%(classname, col,
634 self.arg)
635 self.cursor.execute(sql, (nodeid,))
636 # extract the first column from the result
637 node[col] = [x[0] for x in self.cursor.fetchall()]
639 # un-dbificate the node data
640 node = self.unserialise(classname, node)
642 # save off in the cache
643 key = (classname, nodeid)
644 self.cache[key] = node
645 # update the LRU
646 self.cache_lru.insert(0, key)
647 if len(self.cache_lru) > ROW_CACHE_SIZE:
648 del self.cache[self.cache_lru.pop()]
650 return node
652 def destroynode(self, classname, nodeid):
653 '''Remove a node from the database. Called exclusively by the
654 destroy() method on Class.
655 '''
656 if __debug__:
657 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
659 # make sure the node exists
660 if not self.hasnode(classname, nodeid):
661 raise IndexError, '%s has no node %s'%(classname, nodeid)
663 # see if we have this node cached
664 if self.cache.has_key((classname, nodeid)):
665 del self.cache[(classname, nodeid)]
667 # see if there's any obvious commit actions that we should get rid of
668 for entry in self.transactions[:]:
669 if entry[1][:2] == (classname, nodeid):
670 self.transactions.remove(entry)
672 # now do the SQL
673 sql = 'delete from _%s where id=%s'%(classname, self.arg)
674 self.sql(sql, (nodeid,))
676 # remove from multilnks
677 cl = self.getclass(classname)
678 x, mls = self.determine_columns(cl.properties.items())
679 for col in mls:
680 # get the link ids
681 sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
682 self.cursor.execute(sql, (nodeid,))
684 # remove journal entries
685 sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
686 self.sql(sql, (nodeid,))
688 def serialise(self, classname, node):
689 '''Copy the node contents, converting non-marshallable data into
690 marshallable data.
691 '''
692 if __debug__:
693 print >>hyperdb.DEBUG, 'serialise', classname, node
694 properties = self.getclass(classname).getprops()
695 d = {}
696 for k, v in node.items():
697 # if the property doesn't exist, or is the "retired" flag then
698 # it won't be in the properties dict
699 if not properties.has_key(k):
700 d[k] = v
701 continue
703 # get the property spec
704 prop = properties[k]
706 if isinstance(prop, Password) and v is not None:
707 d[k] = str(v)
708 elif isinstance(prop, Date) and v is not None:
709 d[k] = v.serialise()
710 elif isinstance(prop, Interval) and v is not None:
711 d[k] = v.serialise()
712 else:
713 d[k] = v
714 return d
716 def unserialise(self, classname, node):
717 '''Decode the marshalled node data
718 '''
719 if __debug__:
720 print >>hyperdb.DEBUG, 'unserialise', classname, node
721 properties = self.getclass(classname).getprops()
722 d = {}
723 for k, v in node.items():
724 # if the property doesn't exist, or is the "retired" flag then
725 # it won't be in the properties dict
726 if not properties.has_key(k):
727 d[k] = v
728 continue
730 # get the property spec
731 prop = properties[k]
733 if isinstance(prop, Date) and v is not None:
734 d[k] = date.Date(v)
735 elif isinstance(prop, Interval) and v is not None:
736 d[k] = date.Interval(v)
737 elif isinstance(prop, Password) and v is not None:
738 p = password.Password()
739 p.unpack(v)
740 d[k] = p
741 else:
742 d[k] = v
743 return d
745 def hasnode(self, classname, nodeid):
746 ''' Determine if the database has a given node.
747 '''
748 sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
749 if __debug__:
750 print >>hyperdb.DEBUG, 'hasnode', (self, sql, nodeid)
751 self.cursor.execute(sql, (nodeid,))
752 return int(self.cursor.fetchone()[0])
754 def countnodes(self, classname):
755 ''' Count the number of nodes that exist for a particular Class.
756 '''
757 sql = 'select count(*) from _%s'%classname
758 if __debug__:
759 print >>hyperdb.DEBUG, 'countnodes', (self, sql)
760 self.cursor.execute(sql)
761 return self.cursor.fetchone()[0]
763 def getnodeids(self, classname, retired=0):
764 ''' Retrieve all the ids of the nodes for a particular Class.
766 Set retired=None to get all nodes. Otherwise it'll get all the
767 retired or non-retired nodes, depending on the flag.
768 '''
769 # flip the sense of the flag if we don't want all of them
770 if retired is not None:
771 retired = not retired
772 sql = 'select id from _%s where __retired__ <> %s'%(classname, self.arg)
773 if __debug__:
774 print >>hyperdb.DEBUG, 'getnodeids', (self, sql, retired)
775 self.cursor.execute(sql, (retired,))
776 return [x[0] for x in self.cursor.fetchall()]
778 def addjournal(self, classname, nodeid, action, params, creator=None,
779 creation=None):
780 ''' Journal the Action
781 'action' may be:
783 'create' or 'set' -- 'params' is a dictionary of property values
784 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
785 'retire' -- 'params' is None
786 '''
787 # serialise the parameters now if necessary
788 if isinstance(params, type({})):
789 if action in ('set', 'create'):
790 params = self.serialise(classname, params)
792 # handle supply of the special journalling parameters (usually
793 # supplied on importing an existing database)
794 if creator:
795 journaltag = creator
796 else:
797 journaltag = self.curuserid
798 if creation:
799 journaldate = creation.serialise()
800 else:
801 journaldate = date.Date().serialise()
803 # create the journal entry
804 cols = ','.join('nodeid date tag action params'.split())
806 if __debug__:
807 print >>hyperdb.DEBUG, 'addjournal', (nodeid, journaldate,
808 journaltag, action, params)
810 self.save_journal(classname, cols, nodeid, journaldate,
811 journaltag, action, params)
813 def save_journal(self, classname, cols, nodeid, journaldate,
814 journaltag, action, params):
815 ''' Save the journal entry to the database
816 '''
817 raise NotImplemented
819 def getjournal(self, classname, nodeid):
820 ''' get the journal for id
821 '''
822 # make sure the node exists
823 if not self.hasnode(classname, nodeid):
824 raise IndexError, '%s has no node %s'%(classname, nodeid)
826 cols = ','.join('nodeid date tag action params'.split())
827 return self.load_journal(classname, cols, nodeid)
829 def load_journal(self, classname, cols, nodeid):
830 ''' Load the journal from the database
831 '''
832 raise NotImplemented
834 def pack(self, pack_before):
835 ''' Delete all journal entries except "create" before 'pack_before'.
836 '''
837 # get a 'yyyymmddhhmmss' version of the date
838 date_stamp = pack_before.serialise()
840 # do the delete
841 for classname in self.classes.keys():
842 sql = "delete from %s__journal where date<%s and "\
843 "action<>'create'"%(classname, self.arg)
844 if __debug__:
845 print >>hyperdb.DEBUG, 'pack', (self, sql, date_stamp)
846 self.cursor.execute(sql, (date_stamp,))
848 def sql_commit(self):
849 ''' Actually commit to the database.
850 '''
851 self.conn.commit()
853 def commit(self):
854 ''' Commit the current transactions.
856 Save all data changed since the database was opened or since the
857 last commit() or rollback().
858 '''
859 if __debug__:
860 print >>hyperdb.DEBUG, 'commit', (self,)
862 # commit the database
863 self.sql_commit()
865 # now, do all the other transaction stuff
866 reindex = {}
867 for method, args in self.transactions:
868 reindex[method(*args)] = 1
870 # reindex the nodes that request it
871 for classname, nodeid in filter(None, reindex.keys()):
872 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
873 self.getclass(classname).index(nodeid)
875 # save the indexer state
876 self.indexer.save_index()
878 # clear out the transactions
879 self.transactions = []
881 def rollback(self):
882 ''' Reverse all actions from the current transaction.
884 Undo all the changes made since the database was opened or the last
885 commit() or rollback() was performed.
886 '''
887 if __debug__:
888 print >>hyperdb.DEBUG, 'rollback', (self,)
890 # roll back
891 self.conn.rollback()
893 # roll back "other" transaction stuff
894 for method, args in self.transactions:
895 # delete temporary files
896 if method == self.doStoreFile:
897 self.rollbackStoreFile(*args)
898 self.transactions = []
900 def doSaveNode(self, classname, nodeid, node):
901 ''' dummy that just generates a reindex event
902 '''
903 # return the classname, nodeid so we reindex this content
904 return (classname, nodeid)
906 def close(self):
907 ''' Close off the connection.
908 '''
909 self.conn.close()
910 if self.lockfile is not None:
911 locking.release_lock(self.lockfile)
912 if self.lockfile is not None:
913 self.lockfile.close()
914 self.lockfile = None
916 #
917 # The base Class class
918 #
919 class Class(hyperdb.Class):
920 ''' The handle to a particular class of nodes in a hyperdatabase.
922 All methods except __repr__ and getnode must be implemented by a
923 concrete backend Class.
924 '''
926 def __init__(self, db, classname, **properties):
927 '''Create a new class with a given name and property specification.
929 'classname' must not collide with the name of an existing class,
930 or a ValueError is raised. The keyword arguments in 'properties'
931 must map names to property objects, or a TypeError is raised.
932 '''
933 if (properties.has_key('creation') or properties.has_key('activity')
934 or properties.has_key('creator')):
935 raise ValueError, '"creation", "activity" and "creator" are '\
936 'reserved'
938 self.classname = classname
939 self.properties = properties
940 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
941 self.key = ''
943 # should we journal changes (default yes)
944 self.do_journal = 1
946 # do the db-related init stuff
947 db.addclass(self)
949 self.auditors = {'create': [], 'set': [], 'retire': []}
950 self.reactors = {'create': [], 'set': [], 'retire': []}
952 def schema(self):
953 ''' A dumpable version of the schema that we can store in the
954 database
955 '''
956 return (self.key, [(x, repr(y)) for x,y in self.properties.items()])
958 def enableJournalling(self):
959 '''Turn journalling on for this class
960 '''
961 self.do_journal = 1
963 def disableJournalling(self):
964 '''Turn journalling off for this class
965 '''
966 self.do_journal = 0
968 # Editing nodes:
969 def create(self, **propvalues):
970 ''' Create a new node of this class and return its id.
972 The keyword arguments in 'propvalues' map property names to values.
974 The values of arguments must be acceptable for the types of their
975 corresponding properties or a TypeError is raised.
977 If this class has a key property, it must be present and its value
978 must not collide with other key strings or a ValueError is raised.
980 Any other properties on this class that are missing from the
981 'propvalues' dictionary are set to None.
983 If an id in a link or multilink property does not refer to a valid
984 node, an IndexError is raised.
985 '''
986 if propvalues.has_key('id'):
987 raise KeyError, '"id" is reserved'
989 if self.db.journaltag is None:
990 raise DatabaseError, 'Database open read-only'
992 if propvalues.has_key('creation') or propvalues.has_key('activity'):
993 raise KeyError, '"creation" and "activity" are reserved'
995 self.fireAuditors('create', None, propvalues)
997 # new node's id
998 newid = self.db.newid(self.classname)
1000 # validate propvalues
1001 num_re = re.compile('^\d+$')
1002 for key, value in propvalues.items():
1003 if key == self.key:
1004 try:
1005 self.lookup(value)
1006 except KeyError:
1007 pass
1008 else:
1009 raise ValueError, 'node with key "%s" exists'%value
1011 # try to handle this property
1012 try:
1013 prop = self.properties[key]
1014 except KeyError:
1015 raise KeyError, '"%s" has no property "%s"'%(self.classname,
1016 key)
1018 if value is not None and isinstance(prop, Link):
1019 if type(value) != type(''):
1020 raise ValueError, 'link value must be String'
1021 link_class = self.properties[key].classname
1022 # if it isn't a number, it's a key
1023 if not num_re.match(value):
1024 try:
1025 value = self.db.classes[link_class].lookup(value)
1026 except (TypeError, KeyError):
1027 raise IndexError, 'new property "%s": %s not a %s'%(
1028 key, value, link_class)
1029 elif not self.db.getclass(link_class).hasnode(value):
1030 raise IndexError, '%s has no node %s'%(link_class, value)
1032 # save off the value
1033 propvalues[key] = value
1035 # register the link with the newly linked node
1036 if self.do_journal and self.properties[key].do_journal:
1037 self.db.addjournal(link_class, value, 'link',
1038 (self.classname, newid, key))
1040 elif isinstance(prop, Multilink):
1041 if type(value) != type([]):
1042 raise TypeError, 'new property "%s" not a list of ids'%key
1044 # clean up and validate the list of links
1045 link_class = self.properties[key].classname
1046 l = []
1047 for entry in value:
1048 if type(entry) != type(''):
1049 raise ValueError, '"%s" multilink value (%r) '\
1050 'must contain Strings'%(key, value)
1051 # if it isn't a number, it's a key
1052 if not num_re.match(entry):
1053 try:
1054 entry = self.db.classes[link_class].lookup(entry)
1055 except (TypeError, KeyError):
1056 raise IndexError, 'new property "%s": %s not a %s'%(
1057 key, entry, self.properties[key].classname)
1058 l.append(entry)
1059 value = l
1060 propvalues[key] = value
1062 # handle additions
1063 for nodeid in value:
1064 if not self.db.getclass(link_class).hasnode(nodeid):
1065 raise IndexError, '%s has no node %s'%(link_class,
1066 nodeid)
1067 # register the link with the newly linked node
1068 if self.do_journal and self.properties[key].do_journal:
1069 self.db.addjournal(link_class, nodeid, 'link',
1070 (self.classname, newid, key))
1072 elif isinstance(prop, String):
1073 if type(value) != type('') and type(value) != type(u''):
1074 raise TypeError, 'new property "%s" not a string'%key
1076 elif isinstance(prop, Password):
1077 if not isinstance(value, password.Password):
1078 raise TypeError, 'new property "%s" not a Password'%key
1080 elif isinstance(prop, Date):
1081 if value is not None and not isinstance(value, date.Date):
1082 raise TypeError, 'new property "%s" not a Date'%key
1084 elif isinstance(prop, Interval):
1085 if value is not None and not isinstance(value, date.Interval):
1086 raise TypeError, 'new property "%s" not an Interval'%key
1088 elif value is not None and isinstance(prop, Number):
1089 try:
1090 float(value)
1091 except ValueError:
1092 raise TypeError, 'new property "%s" not numeric'%key
1094 elif value is not None and isinstance(prop, Boolean):
1095 try:
1096 int(value)
1097 except ValueError:
1098 raise TypeError, 'new property "%s" not boolean'%key
1100 # make sure there's data where there needs to be
1101 for key, prop in self.properties.items():
1102 if propvalues.has_key(key):
1103 continue
1104 if key == self.key:
1105 raise ValueError, 'key property "%s" is required'%key
1106 if isinstance(prop, Multilink):
1107 propvalues[key] = []
1108 else:
1109 propvalues[key] = None
1111 # done
1112 self.db.addnode(self.classname, newid, propvalues)
1113 if self.do_journal:
1114 self.db.addjournal(self.classname, newid, 'create', {})
1116 self.fireReactors('create', newid, None)
1118 return newid
1120 def export_list(self, propnames, nodeid):
1121 ''' Export a node - generate a list of CSV-able data in the order
1122 specified by propnames for the given node.
1123 '''
1124 properties = self.getprops()
1125 l = []
1126 for prop in propnames:
1127 proptype = properties[prop]
1128 value = self.get(nodeid, prop)
1129 # "marshal" data where needed
1130 if value is None:
1131 pass
1132 elif isinstance(proptype, hyperdb.Date):
1133 value = value.get_tuple()
1134 elif isinstance(proptype, hyperdb.Interval):
1135 value = value.get_tuple()
1136 elif isinstance(proptype, hyperdb.Password):
1137 value = str(value)
1138 l.append(repr(value))
1139 return l
1141 def import_list(self, propnames, proplist):
1142 ''' Import a node - all information including "id" is present and
1143 should not be sanity checked. Triggers are not triggered. The
1144 journal should be initialised using the "creator" and "created"
1145 information.
1147 Return the nodeid of the node imported.
1148 '''
1149 if self.db.journaltag is None:
1150 raise DatabaseError, 'Database open read-only'
1151 properties = self.getprops()
1153 # make the new node's property map
1154 d = {}
1155 for i in range(len(propnames)):
1156 # Use eval to reverse the repr() used to output the CSV
1157 value = eval(proplist[i])
1159 # Figure the property for this column
1160 propname = propnames[i]
1161 prop = properties[propname]
1163 # "unmarshal" where necessary
1164 if propname == 'id':
1165 newid = value
1166 continue
1167 elif value is None:
1168 # don't set Nones
1169 continue
1170 elif isinstance(prop, hyperdb.Date):
1171 value = date.Date(value)
1172 elif isinstance(prop, hyperdb.Interval):
1173 value = date.Interval(value)
1174 elif isinstance(prop, hyperdb.Password):
1175 pwd = password.Password()
1176 pwd.unpack(value)
1177 value = pwd
1178 d[propname] = value
1180 # add the node and journal
1181 self.db.addnode(self.classname, newid, d)
1183 # extract the extraneous journalling gumpf and nuke it
1184 if d.has_key('creator'):
1185 creator = d['creator']
1186 del d['creator']
1187 else:
1188 creator = None
1189 if d.has_key('creation'):
1190 creation = d['creation']
1191 del d['creation']
1192 else:
1193 creation = None
1194 if d.has_key('activity'):
1195 del d['activity']
1196 self.db.addjournal(self.classname, newid, 'create', {}, creator,
1197 creation)
1198 return newid
1200 _marker = []
1201 def get(self, nodeid, propname, default=_marker, cache=1):
1202 '''Get the value of a property on an existing node of this class.
1204 'nodeid' must be the id of an existing node of this class or an
1205 IndexError is raised. 'propname' must be the name of a property
1206 of this class or a KeyError is raised.
1208 'cache' indicates whether the transaction cache should be queried
1209 for the node. If the node has been modified and you need to
1210 determine what its values prior to modification are, you need to
1211 set cache=0.
1212 '''
1213 if propname == 'id':
1214 return nodeid
1216 # get the node's dict
1217 d = self.db.getnode(self.classname, nodeid)
1219 if propname == 'creation':
1220 if d.has_key('creation'):
1221 return d['creation']
1222 else:
1223 return date.Date()
1224 if propname == 'activity':
1225 if d.has_key('activity'):
1226 return d['activity']
1227 else:
1228 return date.Date()
1229 if propname == 'creator':
1230 if d.has_key('creator'):
1231 return d['creator']
1232 else:
1233 return self.db.curuserid
1235 # get the property (raises KeyErorr if invalid)
1236 prop = self.properties[propname]
1238 if not d.has_key(propname):
1239 if default is self._marker:
1240 if isinstance(prop, Multilink):
1241 return []
1242 else:
1243 return None
1244 else:
1245 return default
1247 # don't pass our list to other code
1248 if isinstance(prop, Multilink):
1249 return d[propname][:]
1251 return d[propname]
1253 def getnode(self, nodeid, cache=1):
1254 ''' Return a convenience wrapper for the node.
1256 'nodeid' must be the id of an existing node of this class or an
1257 IndexError is raised.
1259 'cache' indicates whether the transaction cache should be queried
1260 for the node. If the node has been modified and you need to
1261 determine what its values prior to modification are, you need to
1262 set cache=0.
1263 '''
1264 return Node(self, nodeid, cache=cache)
1266 def set(self, nodeid, **propvalues):
1267 '''Modify a property on an existing node of this class.
1269 'nodeid' must be the id of an existing node of this class or an
1270 IndexError is raised.
1272 Each key in 'propvalues' must be the name of a property of this
1273 class or a KeyError is raised.
1275 All values in 'propvalues' must be acceptable types for their
1276 corresponding properties or a TypeError is raised.
1278 If the value of the key property is set, it must not collide with
1279 other key strings or a ValueError is raised.
1281 If the value of a Link or Multilink property contains an invalid
1282 node id, a ValueError is raised.
1283 '''
1284 if not propvalues:
1285 return propvalues
1287 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1288 raise KeyError, '"creation" and "activity" are reserved'
1290 if propvalues.has_key('id'):
1291 raise KeyError, '"id" is reserved'
1293 if self.db.journaltag is None:
1294 raise DatabaseError, 'Database open read-only'
1296 self.fireAuditors('set', nodeid, propvalues)
1297 # Take a copy of the node dict so that the subsequent set
1298 # operation doesn't modify the oldvalues structure.
1299 # XXX used to try the cache here first
1300 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1302 node = self.db.getnode(self.classname, nodeid)
1303 if self.is_retired(nodeid):
1304 raise IndexError, 'Requested item is retired'
1305 num_re = re.compile('^\d+$')
1307 # if the journal value is to be different, store it in here
1308 journalvalues = {}
1310 # remember the add/remove stuff for multilinks, making it easier
1311 # for the Database layer to do its stuff
1312 multilink_changes = {}
1314 for propname, value in propvalues.items():
1315 # check to make sure we're not duplicating an existing key
1316 if propname == self.key and node[propname] != value:
1317 try:
1318 self.lookup(value)
1319 except KeyError:
1320 pass
1321 else:
1322 raise ValueError, 'node with key "%s" exists'%value
1324 # this will raise the KeyError if the property isn't valid
1325 # ... we don't use getprops() here because we only care about
1326 # the writeable properties.
1327 try:
1328 prop = self.properties[propname]
1329 except KeyError:
1330 raise KeyError, '"%s" has no property named "%s"'%(
1331 self.classname, propname)
1333 # if the value's the same as the existing value, no sense in
1334 # doing anything
1335 current = node.get(propname, None)
1336 if value == current:
1337 del propvalues[propname]
1338 continue
1339 journalvalues[propname] = current
1341 # do stuff based on the prop type
1342 if isinstance(prop, Link):
1343 link_class = prop.classname
1344 # if it isn't a number, it's a key
1345 if value is not None and not isinstance(value, type('')):
1346 raise ValueError, 'property "%s" link value be a string'%(
1347 propname)
1348 if isinstance(value, type('')) and not num_re.match(value):
1349 try:
1350 value = self.db.classes[link_class].lookup(value)
1351 except (TypeError, KeyError):
1352 raise IndexError, 'new property "%s": %s not a %s'%(
1353 propname, value, prop.classname)
1355 if (value is not None and
1356 not self.db.getclass(link_class).hasnode(value)):
1357 raise IndexError, '%s has no node %s'%(link_class, value)
1359 if self.do_journal and prop.do_journal:
1360 # register the unlink with the old linked node
1361 if node[propname] is not None:
1362 self.db.addjournal(link_class, node[propname], 'unlink',
1363 (self.classname, nodeid, propname))
1365 # register the link with the newly linked node
1366 if value is not None:
1367 self.db.addjournal(link_class, value, 'link',
1368 (self.classname, nodeid, propname))
1370 elif isinstance(prop, Multilink):
1371 if type(value) != type([]):
1372 raise TypeError, 'new property "%s" not a list of'\
1373 ' ids'%propname
1374 link_class = self.properties[propname].classname
1375 l = []
1376 for entry in value:
1377 # if it isn't a number, it's a key
1378 if type(entry) != type(''):
1379 raise ValueError, 'new property "%s" link value ' \
1380 'must be a string'%propname
1381 if not num_re.match(entry):
1382 try:
1383 entry = self.db.classes[link_class].lookup(entry)
1384 except (TypeError, KeyError):
1385 raise IndexError, 'new property "%s": %s not a %s'%(
1386 propname, entry,
1387 self.properties[propname].classname)
1388 l.append(entry)
1389 value = l
1390 propvalues[propname] = value
1392 # figure the journal entry for this property
1393 add = []
1394 remove = []
1396 # handle removals
1397 if node.has_key(propname):
1398 l = node[propname]
1399 else:
1400 l = []
1401 for id in l[:]:
1402 if id in value:
1403 continue
1404 # register the unlink with the old linked node
1405 if self.do_journal and self.properties[propname].do_journal:
1406 self.db.addjournal(link_class, id, 'unlink',
1407 (self.classname, nodeid, propname))
1408 l.remove(id)
1409 remove.append(id)
1411 # handle additions
1412 for id in value:
1413 if not self.db.getclass(link_class).hasnode(id):
1414 raise IndexError, '%s has no node %s'%(link_class, id)
1415 if id in l:
1416 continue
1417 # register the link with the newly linked node
1418 if self.do_journal and self.properties[propname].do_journal:
1419 self.db.addjournal(link_class, id, 'link',
1420 (self.classname, nodeid, propname))
1421 l.append(id)
1422 add.append(id)
1424 # figure the journal entry
1425 l = []
1426 if add:
1427 l.append(('+', add))
1428 if remove:
1429 l.append(('-', remove))
1430 multilink_changes[propname] = (add, remove)
1431 if l:
1432 journalvalues[propname] = tuple(l)
1434 elif isinstance(prop, String):
1435 if value is not None and type(value) != type('') and type(value) != type(u''):
1436 raise TypeError, 'new property "%s" not a string'%propname
1438 elif isinstance(prop, Password):
1439 if not isinstance(value, password.Password):
1440 raise TypeError, 'new property "%s" not a Password'%propname
1441 propvalues[propname] = value
1443 elif value is not None and isinstance(prop, Date):
1444 if not isinstance(value, date.Date):
1445 raise TypeError, 'new property "%s" not a Date'% propname
1446 propvalues[propname] = value
1448 elif value is not None and isinstance(prop, Interval):
1449 if not isinstance(value, date.Interval):
1450 raise TypeError, 'new property "%s" not an '\
1451 'Interval'%propname
1452 propvalues[propname] = value
1454 elif value is not None and isinstance(prop, Number):
1455 try:
1456 float(value)
1457 except ValueError:
1458 raise TypeError, 'new property "%s" not numeric'%propname
1460 elif value is not None and isinstance(prop, Boolean):
1461 try:
1462 int(value)
1463 except ValueError:
1464 raise TypeError, 'new property "%s" not boolean'%propname
1466 # nothing to do?
1467 if not propvalues:
1468 return propvalues
1470 # do the set, and journal it
1471 self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
1473 if self.do_journal:
1474 self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
1476 self.fireReactors('set', nodeid, oldvalues)
1478 return propvalues
1480 def retire(self, nodeid):
1481 '''Retire a node.
1483 The properties on the node remain available from the get() method,
1484 and the node's id is never reused.
1486 Retired nodes are not returned by the find(), list(), or lookup()
1487 methods, and other nodes may reuse the values of their key properties.
1488 '''
1489 if self.db.journaltag is None:
1490 raise DatabaseError, 'Database open read-only'
1492 self.fireAuditors('retire', nodeid, None)
1494 # use the arg for __retired__ to cope with any odd database type
1495 # conversion (hello, sqlite)
1496 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1497 self.db.arg, self.db.arg)
1498 if __debug__:
1499 print >>hyperdb.DEBUG, 'retire', (self, sql, nodeid)
1500 self.db.cursor.execute(sql, (1, nodeid))
1502 self.fireReactors('retire', nodeid, None)
1504 def is_retired(self, nodeid):
1505 '''Return true if the node is rerired
1506 '''
1507 sql = 'select __retired__ from _%s where id=%s'%(self.classname,
1508 self.db.arg)
1509 if __debug__:
1510 print >>hyperdb.DEBUG, 'is_retired', (self, sql, nodeid)
1511 self.db.cursor.execute(sql, (nodeid,))
1512 return int(self.db.sql_fetchone()[0])
1514 def destroy(self, nodeid):
1515 '''Destroy a node.
1517 WARNING: this method should never be used except in extremely rare
1518 situations where there could never be links to the node being
1519 deleted
1520 WARNING: use retire() instead
1521 WARNING: the properties of this node will not be available ever again
1522 WARNING: really, use retire() instead
1524 Well, I think that's enough warnings. This method exists mostly to
1525 support the session storage of the cgi interface.
1527 The node is completely removed from the hyperdb, including all journal
1528 entries. It will no longer be available, and will generally break code
1529 if there are any references to the node.
1530 '''
1531 if self.db.journaltag is None:
1532 raise DatabaseError, 'Database open read-only'
1533 self.db.destroynode(self.classname, nodeid)
1535 def history(self, nodeid):
1536 '''Retrieve the journal of edits on a particular node.
1538 'nodeid' must be the id of an existing node of this class or an
1539 IndexError is raised.
1541 The returned list contains tuples of the form
1543 (date, tag, action, params)
1545 'date' is a Timestamp object specifying the time of the change and
1546 'tag' is the journaltag specified when the database was opened.
1547 '''
1548 if not self.do_journal:
1549 raise ValueError, 'Journalling is disabled for this class'
1550 return self.db.getjournal(self.classname, nodeid)
1552 # Locating nodes:
1553 def hasnode(self, nodeid):
1554 '''Determine if the given nodeid actually exists
1555 '''
1556 return self.db.hasnode(self.classname, nodeid)
1558 def setkey(self, propname):
1559 '''Select a String property of this class to be the key property.
1561 'propname' must be the name of a String property of this class or
1562 None, or a TypeError is raised. The values of the key property on
1563 all existing nodes must be unique or a ValueError is raised.
1564 '''
1565 # XXX create an index on the key prop column
1566 prop = self.getprops()[propname]
1567 if not isinstance(prop, String):
1568 raise TypeError, 'key properties must be String'
1569 self.key = propname
1571 def getkey(self):
1572 '''Return the name of the key property for this class or None.'''
1573 return self.key
1575 def labelprop(self, default_to_id=0):
1576 ''' Return the property name for a label for the given node.
1578 This method attempts to generate a consistent label for the node.
1579 It tries the following in order:
1580 1. key property
1581 2. "name" property
1582 3. "title" property
1583 4. first property from the sorted property name list
1584 '''
1585 k = self.getkey()
1586 if k:
1587 return k
1588 props = self.getprops()
1589 if props.has_key('name'):
1590 return 'name'
1591 elif props.has_key('title'):
1592 return 'title'
1593 if default_to_id:
1594 return 'id'
1595 props = props.keys()
1596 props.sort()
1597 return props[0]
1599 def lookup(self, keyvalue):
1600 '''Locate a particular node by its key property and return its id.
1602 If this class has no key property, a TypeError is raised. If the
1603 'keyvalue' matches one of the values for the key property among
1604 the nodes in this class, the matching node's id is returned;
1605 otherwise a KeyError is raised.
1606 '''
1607 if not self.key:
1608 raise TypeError, 'No key property set for class %s'%self.classname
1610 # use the arg to handle any odd database type conversion (hello,
1611 # sqlite)
1612 sql = "select id from _%s where _%s=%s and __retired__ <> %s"%(
1613 self.classname, self.key, self.db.arg, self.db.arg)
1614 self.db.sql(sql, (keyvalue, 1))
1616 # see if there was a result that's not retired
1617 row = self.db.sql_fetchone()
1618 if not row:
1619 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1620 keyvalue, self.classname)
1622 # return the id
1623 return row[0]
1625 def find(self, **propspec):
1626 '''Get the ids of nodes in this class which link to the given nodes.
1628 'propspec' consists of keyword args propname=nodeid or
1629 propname={nodeid:1, }
1630 'propname' must be the name of a property in this class, or a
1631 KeyError is raised. That property must be a Link or Multilink
1632 property, or a TypeError is raised.
1634 Any node in this class whose 'propname' property links to any of the
1635 nodeids will be returned. Used by the full text indexing, which knows
1636 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1637 issues:
1639 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1640 '''
1641 if __debug__:
1642 print >>hyperdb.DEBUG, 'find', (self, propspec)
1644 # shortcut
1645 if not propspec:
1646 return []
1648 # validate the args
1649 props = self.getprops()
1650 propspec = propspec.items()
1651 for propname, nodeids in propspec:
1652 # check the prop is OK
1653 prop = props[propname]
1654 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1655 raise TypeError, "'%s' not a Link/Multilink property"%propname
1657 # first, links
1658 where = []
1659 allvalues = ()
1660 a = self.db.arg
1661 for prop, values in propspec:
1662 if not isinstance(props[prop], hyperdb.Link):
1663 continue
1664 if type(values) is type(''):
1665 allvalues += (values,)
1666 where.append('_%s = %s'%(prop, a))
1667 else:
1668 allvalues += tuple(values.keys())
1669 where.append('_%s in (%s)'%(prop, ','.join([a]*len(values))))
1670 tables = []
1671 if where:
1672 tables.append('select id as nodeid from _%s where %s'%(
1673 self.classname, ' and '.join(where)))
1675 # now multilinks
1676 for prop, values in propspec:
1677 if not isinstance(props[prop], hyperdb.Multilink):
1678 continue
1679 if type(values) is type(''):
1680 allvalues += (values,)
1681 s = a
1682 else:
1683 allvalues += tuple(values.keys())
1684 s = ','.join([a]*len(values))
1685 tables.append('select nodeid from %s_%s where linkid in (%s)'%(
1686 self.classname, prop, s))
1687 sql = '\nunion\n'.join(tables)
1688 self.db.sql(sql, allvalues)
1689 l = [x[0] for x in self.db.sql_fetchall()]
1690 if __debug__:
1691 print >>hyperdb.DEBUG, 'find ... ', l
1692 return l
1694 def stringFind(self, **requirements):
1695 '''Locate a particular node by matching a set of its String
1696 properties in a caseless search.
1698 If the property is not a String property, a TypeError is raised.
1700 The return is a list of the id of all nodes that match.
1701 '''
1702 where = []
1703 args = []
1704 for propname in requirements.keys():
1705 prop = self.properties[propname]
1706 if isinstance(not prop, String):
1707 raise TypeError, "'%s' not a String property"%propname
1708 where.append(propname)
1709 args.append(requirements[propname].lower())
1711 # generate the where clause
1712 s = ' and '.join(['_%s=%s'%(col, self.db.arg) for col in where])
1713 sql = 'select id from _%s where %s'%(self.classname, s)
1714 self.db.sql(sql, tuple(args))
1715 l = [x[0] for x in self.db.sql_fetchall()]
1716 if __debug__:
1717 print >>hyperdb.DEBUG, 'find ... ', l
1718 return l
1720 def list(self):
1721 ''' Return a list of the ids of the active nodes in this class.
1722 '''
1723 return self.db.getnodeids(self.classname, retired=0)
1725 def filter(self, search_matches, filterspec, sort=(None,None),
1726 group=(None,None)):
1727 ''' Return a list of the ids of the active nodes in this class that
1728 match the 'filter' spec, sorted by the group spec and then the
1729 sort spec
1731 "filterspec" is {propname: value(s)}
1732 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1733 and prop is a prop name or None
1734 "search_matches" is {nodeid: marker}
1736 The filter must match all properties specificed - but if the
1737 property value to match is a list, any one of the values in the
1738 list may match for that property to match.
1739 '''
1740 # just don't bother if the full-text search matched diddly
1741 if search_matches == {}:
1742 return []
1744 cn = self.classname
1746 # figure the WHERE clause from the filterspec
1747 props = self.getprops()
1748 frum = ['_'+cn]
1749 where = []
1750 args = []
1751 a = self.db.arg
1752 for k, v in filterspec.items():
1753 propclass = props[k]
1754 # now do other where clause stuff
1755 if isinstance(propclass, Multilink):
1756 tn = '%s_%s'%(cn, k)
1757 frum.append(tn)
1758 if isinstance(v, type([])):
1759 s = ','.join([a for x in v])
1760 where.append('id=%s.nodeid and %s.linkid in (%s)'%(tn,tn,s))
1761 args = args + v
1762 else:
1763 where.append('id=%s.nodeid and %s.linkid = %s'%(tn, tn, a))
1764 args.append(v)
1765 elif k == 'id':
1766 if isinstance(v, type([])):
1767 s = ','.join([a for x in v])
1768 where.append('%s in (%s)'%(k, s))
1769 args = args + v
1770 else:
1771 where.append('%s=%s'%(k, a))
1772 args.append(v)
1773 elif isinstance(propclass, String):
1774 if not isinstance(v, type([])):
1775 v = [v]
1777 # Quote the bits in the string that need it and then embed
1778 # in a "substring" search. Note - need to quote the '%' so
1779 # they make it through the python layer happily
1780 v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
1782 # now add to the where clause
1783 where.append(' or '.join(["_%s LIKE '%s'"%(k, s) for s in v]))
1784 # note: args are embedded in the query string now
1785 elif isinstance(propclass, Link):
1786 if isinstance(v, type([])):
1787 if '-1' in v:
1788 v.remove('-1')
1789 xtra = ' or _%s is NULL'%k
1790 else:
1791 xtra = ''
1792 if v:
1793 s = ','.join([a for x in v])
1794 where.append('(_%s in (%s)%s)'%(k, s, xtra))
1795 args = args + v
1796 else:
1797 where.append('_%s is NULL'%k)
1798 else:
1799 if v == '-1':
1800 v = None
1801 where.append('_%s is NULL'%k)
1802 else:
1803 where.append('_%s=%s'%(k, a))
1804 args.append(v)
1805 elif isinstance(propclass, Date):
1806 if isinstance(v, type([])):
1807 s = ','.join([a for x in v])
1808 where.append('_%s in (%s)'%(k, s))
1809 args = args + [date.Date(x).serialise() for x in v]
1810 else:
1811 where.append('_%s=%s'%(k, a))
1812 args.append(date.Date(v).serialise())
1813 elif isinstance(propclass, Interval):
1814 if isinstance(v, type([])):
1815 s = ','.join([a for x in v])
1816 where.append('_%s in (%s)'%(k, s))
1817 args = args + [date.Interval(x).serialise() for x in v]
1818 else:
1819 where.append('_%s=%s'%(k, a))
1820 args.append(date.Interval(v).serialise())
1821 else:
1822 if isinstance(v, type([])):
1823 s = ','.join([a for x in v])
1824 where.append('_%s in (%s)'%(k, s))
1825 args = args + v
1826 else:
1827 where.append('_%s=%s'%(k, a))
1828 args.append(v)
1830 # add results of full text search
1831 if search_matches is not None:
1832 v = search_matches.keys()
1833 s = ','.join([a for x in v])
1834 where.append('id in (%s)'%s)
1835 args = args + v
1837 # "grouping" is just the first-order sorting in the SQL fetch
1838 # can modify it...)
1839 orderby = []
1840 ordercols = []
1841 if group[0] is not None and group[1] is not None:
1842 if group[0] != '-':
1843 orderby.append('_'+group[1])
1844 ordercols.append('_'+group[1])
1845 else:
1846 orderby.append('_'+group[1]+' desc')
1847 ordercols.append('_'+group[1])
1849 # now add in the sorting
1850 group = ''
1851 if sort[0] is not None and sort[1] is not None:
1852 direction, colname = sort
1853 if direction != '-':
1854 if colname == 'id':
1855 orderby.append(colname)
1856 else:
1857 orderby.append('_'+colname)
1858 ordercols.append('_'+colname)
1859 else:
1860 if colname == 'id':
1861 orderby.append(colname+' desc')
1862 ordercols.append(colname)
1863 else:
1864 orderby.append('_'+colname+' desc')
1865 ordercols.append('_'+colname)
1867 # construct the SQL
1868 frum = ','.join(frum)
1869 if where:
1870 where = ' where ' + (' and '.join(where))
1871 else:
1872 where = ''
1873 cols = ['id']
1874 if orderby:
1875 cols = cols + ordercols
1876 order = ' order by %s'%(','.join(orderby))
1877 else:
1878 order = ''
1879 cols = ','.join(cols)
1880 sql = 'select %s from %s %s%s%s'%(cols, frum, where, group, order)
1881 args = tuple(args)
1882 if __debug__:
1883 print >>hyperdb.DEBUG, 'filter', (self, sql, args)
1884 self.db.cursor.execute(sql, args)
1885 l = self.db.cursor.fetchall()
1887 # return the IDs (the first column)
1888 # XXX The filter(None, l) bit is sqlite-specific... if there's _NO_
1889 # XXX matches to a fetch, it returns NULL instead of nothing!?!
1890 return filter(None, [row[0] for row in l])
1892 def count(self):
1893 '''Get the number of nodes in this class.
1895 If the returned integer is 'numnodes', the ids of all the nodes
1896 in this class run from 1 to numnodes, and numnodes+1 will be the
1897 id of the next node to be created in this class.
1898 '''
1899 return self.db.countnodes(self.classname)
1901 # Manipulating properties:
1902 def getprops(self, protected=1):
1903 '''Return a dictionary mapping property names to property objects.
1904 If the "protected" flag is true, we include protected properties -
1905 those which may not be modified.
1906 '''
1907 d = self.properties.copy()
1908 if protected:
1909 d['id'] = String()
1910 d['creation'] = hyperdb.Date()
1911 d['activity'] = hyperdb.Date()
1912 d['creator'] = hyperdb.Link('user')
1913 return d
1915 def addprop(self, **properties):
1916 '''Add properties to this class.
1918 The keyword arguments in 'properties' must map names to property
1919 objects, or a TypeError is raised. None of the keys in 'properties'
1920 may collide with the names of existing properties, or a ValueError
1921 is raised before any properties have been added.
1922 '''
1923 for key in properties.keys():
1924 if self.properties.has_key(key):
1925 raise ValueError, key
1926 self.properties.update(properties)
1928 def index(self, nodeid):
1929 '''Add (or refresh) the node to search indexes
1930 '''
1931 # find all the String properties that have indexme
1932 for prop, propclass in self.getprops().items():
1933 if isinstance(propclass, String) and propclass.indexme:
1934 try:
1935 value = str(self.get(nodeid, prop))
1936 except IndexError:
1937 # node no longer exists - entry should be removed
1938 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1939 else:
1940 # and index them under (classname, nodeid, property)
1941 self.db.indexer.add_text((self.classname, nodeid, prop),
1942 value)
1945 #
1946 # Detector interface
1947 #
1948 def audit(self, event, detector):
1949 '''Register a detector
1950 '''
1951 l = self.auditors[event]
1952 if detector not in l:
1953 self.auditors[event].append(detector)
1955 def fireAuditors(self, action, nodeid, newvalues):
1956 '''Fire all registered auditors.
1957 '''
1958 for audit in self.auditors[action]:
1959 audit(self.db, self, nodeid, newvalues)
1961 def react(self, event, detector):
1962 '''Register a detector
1963 '''
1964 l = self.reactors[event]
1965 if detector not in l:
1966 self.reactors[event].append(detector)
1968 def fireReactors(self, action, nodeid, oldvalues):
1969 '''Fire all registered reactors.
1970 '''
1971 for react in self.reactors[action]:
1972 react(self.db, self, nodeid, oldvalues)
1974 class FileClass(Class):
1975 '''This class defines a large chunk of data. To support this, it has a
1976 mandatory String property "content" which is typically saved off
1977 externally to the hyperdb.
1979 The default MIME type of this data is defined by the
1980 "default_mime_type" class attribute, which may be overridden by each
1981 node if the class defines a "type" String property.
1982 '''
1983 default_mime_type = 'text/plain'
1985 def create(self, **propvalues):
1986 ''' snaffle the file propvalue and store in a file
1987 '''
1988 content = propvalues['content']
1989 del propvalues['content']
1990 newid = Class.create(self, **propvalues)
1991 self.db.storefile(self.classname, newid, None, content)
1992 return newid
1994 def import_list(self, propnames, proplist):
1995 ''' Trap the "content" property...
1996 '''
1997 # dupe this list so we don't affect others
1998 propnames = propnames[:]
2000 # extract the "content" property from the proplist
2001 i = propnames.index('content')
2002 content = eval(proplist[i])
2003 del propnames[i]
2004 del proplist[i]
2006 # do the normal import
2007 newid = Class.import_list(self, propnames, proplist)
2009 # save off the "content" file
2010 self.db.storefile(self.classname, newid, None, content)
2011 return newid
2013 _marker = []
2014 def get(self, nodeid, propname, default=_marker, cache=1):
2015 ''' trap the content propname and get it from the file
2016 '''
2018 poss_msg = 'Possibly a access right configuration problem.'
2019 if propname == 'content':
2020 try:
2021 return self.db.getfile(self.classname, nodeid, None)
2022 except IOError, (strerror):
2023 # BUG: by catching this we donot see an error in the log.
2024 return 'ERROR reading file: %s%s\n%s\n%s'%(
2025 self.classname, nodeid, poss_msg, strerror)
2026 if default is not self._marker:
2027 return Class.get(self, nodeid, propname, default, cache=cache)
2028 else:
2029 return Class.get(self, nodeid, propname, cache=cache)
2031 def getprops(self, protected=1):
2032 ''' In addition to the actual properties on the node, these methods
2033 provide the "content" property. If the "protected" flag is true,
2034 we include protected properties - those which may not be
2035 modified.
2036 '''
2037 d = Class.getprops(self, protected=protected).copy()
2038 d['content'] = hyperdb.String()
2039 return d
2041 def index(self, nodeid):
2042 ''' Index the node in the search index.
2044 We want to index the content in addition to the normal String
2045 property indexing.
2046 '''
2047 # perform normal indexing
2048 Class.index(self, nodeid)
2050 # get the content to index
2051 content = self.get(nodeid, 'content')
2053 # figure the mime type
2054 if self.properties.has_key('type'):
2055 mime_type = self.get(nodeid, 'type')
2056 else:
2057 mime_type = self.default_mime_type
2059 # and index!
2060 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
2061 mime_type)
2063 # XXX deviation from spec - was called ItemClass
2064 class IssueClass(Class, roundupdb.IssueClass):
2065 # Overridden methods:
2066 def __init__(self, db, classname, **properties):
2067 '''The newly-created class automatically includes the "messages",
2068 "files", "nosy", and "superseder" properties. If the 'properties'
2069 dictionary attempts to specify any of these properties or a
2070 "creation" or "activity" property, a ValueError is raised.
2071 '''
2072 if not properties.has_key('title'):
2073 properties['title'] = hyperdb.String(indexme='yes')
2074 if not properties.has_key('messages'):
2075 properties['messages'] = hyperdb.Multilink("msg")
2076 if not properties.has_key('files'):
2077 properties['files'] = hyperdb.Multilink("file")
2078 if not properties.has_key('nosy'):
2079 # note: journalling is turned off as it really just wastes
2080 # space. this behaviour may be overridden in an instance
2081 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2082 if not properties.has_key('superseder'):
2083 properties['superseder'] = hyperdb.Multilink(classname)
2084 Class.__init__(self, db, classname, **properties)