1 # $Id: rdbms_common.py,v 1.34 2003-02-18 01:57:39 richard Exp $
2 ''' Relational database (SQL) backend common code.
4 Basics:
6 - map roundup classes to relational tables
7 - automatically detect schema changes and modify the table schemas
8 appropriately (we store the "database version" of the schema in the
9 database itself as the only row of the "schema" table)
10 - multilinks (which represent a many-to-many relationship) are handled through
11 intermediate tables
12 - journals are stored adjunct to the per-class tables
13 - table names and columns have "_" prepended so the names can't clash with
14 restricted names (like "order")
15 - retirement is determined by the __retired__ column being true
17 Database-specific changes may generally be pushed out to the overridable
18 sql_* methods, since everything else should be fairly generic. There's
19 probably a bit of work to be done if a database is used that actually
20 honors column typing, since the initial databases don't (sqlite stores
21 everything as a string, and gadfly stores anything that's marsallable).
22 '''
24 # standard python modules
25 import sys, os, time, re, errno, weakref, copy
27 # roundup modules
28 from roundup import hyperdb, date, password, roundupdb, security
29 from roundup.hyperdb import String, Password, Date, Interval, Link, \
30 Multilink, DatabaseError, Boolean, Number, Node
31 from roundup.backends import locking
33 # support
34 from blobfiles import FileStorage
35 from roundup.indexer import Indexer
36 from sessions import Sessions
38 # number of rows to keep in memory
39 ROW_CACHE_SIZE = 100
41 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
42 ''' Wrapper around an SQL database that presents a hyperdb interface.
44 - some functionality is specific to the actual SQL database, hence
45 the sql_* methods that are NotImplemented
46 - we keep a cache of the latest ROW_CACHE_SIZE row fetches.
47 '''
48 def __init__(self, config, journaltag=None):
49 ''' Open the database and load the schema from it.
50 '''
51 self.config, self.journaltag = config, journaltag
52 self.dir = config.DATABASE
53 self.classes = {}
54 self.indexer = Indexer(self.dir)
55 self.sessions = Sessions(self.config)
56 self.security = security.Security(self)
58 # additional transaction support for external files and the like
59 self.transactions = []
61 # keep a cache of the N most recently retrieved rows of any kind
62 # (classname, nodeid) = row
63 self.cache = {}
64 self.cache_lru = []
66 # database lock
67 self.lockfile = None
69 # open a connection to the database, creating the "conn" attribute
70 self.open_connection()
72 def clearCache(self):
73 self.cache = {}
74 self.cache_lru = []
76 def open_connection(self):
77 ''' Open a connection to the database, creating it if necessary
78 '''
79 raise NotImplemented
81 def sql(self, sql, args=None):
82 ''' Execute the sql with the optional args.
83 '''
84 if __debug__:
85 print >>hyperdb.DEBUG, (self, sql, args)
86 if args:
87 self.cursor.execute(sql, args)
88 else:
89 self.cursor.execute(sql)
91 def sql_fetchone(self):
92 ''' Fetch a single row. If there's nothing to fetch, return None.
93 '''
94 raise NotImplemented
96 def sql_stringquote(self, value):
97 ''' Quote the string so it's safe to put in the 'sql quotes'
98 '''
99 return re.sub("'", "''", str(value))
101 def save_dbschema(self, schema):
102 ''' Save the schema definition that the database currently implements
103 '''
104 raise NotImplemented
106 def load_dbschema(self):
107 ''' Load the schema definition that the database currently implements
108 '''
109 raise NotImplemented
111 def post_init(self):
112 ''' Called once the schema initialisation has finished.
114 We should now confirm that the schema defined by our "classes"
115 attribute actually matches the schema in the database.
116 '''
117 # now detect changes in the schema
118 save = 0
119 for classname, spec in self.classes.items():
120 if self.database_schema.has_key(classname):
121 dbspec = self.database_schema[classname]
122 if self.update_class(spec, dbspec):
123 self.database_schema[classname] = spec.schema()
124 save = 1
125 else:
126 self.create_class(spec)
127 self.database_schema[classname] = spec.schema()
128 save = 1
130 for classname in self.database_schema.keys():
131 if not self.classes.has_key(classname):
132 self.drop_class(classname)
134 # update the database version of the schema
135 if save:
136 self.sql('delete from schema')
137 self.save_dbschema(self.database_schema)
139 # reindex the db if necessary
140 if self.indexer.should_reindex():
141 self.reindex()
143 # commit
144 self.conn.commit()
146 # figure the "curuserid"
147 if self.journaltag is None:
148 self.curuserid = None
149 elif self.journaltag == 'admin':
150 # admin user may not exist, but always has ID 1
151 self.curuserid = '1'
152 else:
153 self.curuserid = self.user.lookup(self.journaltag)
155 def reindex(self):
156 for klass in self.classes.values():
157 for nodeid in klass.list():
158 klass.index(nodeid)
159 self.indexer.save_index()
161 def determine_columns(self, properties):
162 ''' Figure the column names and multilink properties from the spec
164 "properties" is a list of (name, prop) where prop may be an
165 instance of a hyperdb "type" _or_ a string repr of that type.
166 '''
167 cols = ['_activity', '_creator', '_creation']
168 mls = []
169 # add the multilinks separately
170 for col, prop in properties:
171 if isinstance(prop, Multilink):
172 mls.append(col)
173 elif isinstance(prop, type('')) and prop.find('Multilink') != -1:
174 mls.append(col)
175 else:
176 cols.append('_'+col)
177 cols.sort()
178 return cols, mls
180 def update_class(self, spec, dbspec):
181 ''' Determine the differences between the current spec and the
182 database version of the spec, and update where necessary
183 '''
184 spec_schema = spec.schema()
185 if spec_schema == dbspec:
186 # no save needed for this one
187 return 0
188 if __debug__:
189 print >>hyperdb.DEBUG, 'update_class FIRING'
191 # key property changed?
192 if dbspec[0] != spec_schema[0]:
193 if __debug__:
194 print >>hyperdb.DEBUG, 'update_class setting keyprop', `spec[0]`
195 # XXX turn on indexing for the key property
197 # dict 'em up
198 spec_propnames,spec_props = [],{}
199 for propname,prop in spec_schema[1]:
200 spec_propnames.append(propname)
201 spec_props[propname] = prop
202 dbspec_propnames,dbspec_props = [],{}
203 for propname,prop in dbspec[1]:
204 dbspec_propnames.append(propname)
205 dbspec_props[propname] = prop
207 # now compare
208 for propname in spec_propnames:
209 prop = spec_props[propname]
210 if dbspec_props.has_key(propname) and prop==dbspec_props[propname]:
211 continue
212 if __debug__:
213 print >>hyperdb.DEBUG, 'update_class ADD', (propname, prop)
215 if not dbspec_props.has_key(propname):
216 # add the property
217 if isinstance(prop, Multilink):
218 # all we have to do here is create a new table, easy!
219 self.create_multilink_table(spec, propname)
220 continue
222 # no ALTER TABLE, so we:
223 # 1. pull out the data, including an extra None column
224 oldcols, x = self.determine_columns(dbspec[1])
225 oldcols.append('id')
226 oldcols.append('__retired__')
227 cn = spec.classname
228 sql = 'select %s,%s from _%s'%(','.join(oldcols), self.arg, cn)
229 if __debug__:
230 print >>hyperdb.DEBUG, 'update_class', (self, sql, None)
231 self.cursor.execute(sql, (None,))
232 olddata = self.cursor.fetchall()
234 # 2. drop the old table
235 self.cursor.execute('drop table _%s'%cn)
237 # 3. create the new table
238 cols, mls = self.create_class_table(spec)
239 # ensure the new column is last
240 cols.remove('_'+propname)
241 assert oldcols == cols, "Column lists don't match!"
242 cols.append('_'+propname)
244 # 4. populate with the data from step one
245 s = ','.join([self.arg for x in cols])
246 scols = ','.join(cols)
247 sql = 'insert into _%s (%s) values (%s)'%(cn, scols, s)
249 # GAH, nothing had better go wrong from here on in... but
250 # we have to commit the drop...
251 # XXX this isn't necessary in sqlite :(
252 self.conn.commit()
254 # do the insert
255 for row in olddata:
256 self.sql(sql, tuple(row))
258 else:
259 # modify the property
260 if __debug__:
261 print >>hyperdb.DEBUG, 'update_class NOOP'
262 pass # NOOP in gadfly
264 # and the other way - only worry about deletions here
265 for propname in dbspec_propnames:
266 prop = dbspec_props[propname]
267 if spec_props.has_key(propname):
268 continue
269 if __debug__:
270 print >>hyperdb.DEBUG, 'update_class REMOVE', `prop`
272 # delete the property
273 if isinstance(prop, Multilink):
274 sql = 'drop table %s_%s'%(spec.classname, prop)
275 if __debug__:
276 print >>hyperdb.DEBUG, 'update_class', (self, sql)
277 self.cursor.execute(sql)
278 else:
279 # no ALTER TABLE, so we:
280 # 1. pull out the data, excluding the removed column
281 oldcols, x = self.determine_columns(spec.properties.items())
282 oldcols.append('id')
283 oldcols.append('__retired__')
284 # remove the missing column
285 oldcols.remove('_'+propname)
286 cn = spec.classname
287 sql = 'select %s from _%s'%(','.join(oldcols), cn)
288 self.cursor.execute(sql, (None,))
289 olddata = sql.fetchall()
291 # 2. drop the old table
292 self.cursor.execute('drop table _%s'%cn)
294 # 3. create the new table
295 cols, mls = self.create_class_table(self, spec)
296 assert oldcols != cols, "Column lists don't match!"
298 # 4. populate with the data from step one
299 qs = ','.join([self.arg for x in cols])
300 sql = 'insert into _%s values (%s)'%(cn, s)
301 self.cursor.execute(sql, olddata)
302 return 1
304 def create_class_table(self, spec):
305 ''' create the class table for the given spec
306 '''
307 cols, mls = self.determine_columns(spec.properties.items())
309 # add on our special columns
310 cols.append('id')
311 cols.append('__retired__')
313 # create the base table
314 scols = ','.join(['%s varchar'%x for x in cols])
315 sql = 'create table _%s (%s)'%(spec.classname, scols)
316 if __debug__:
317 print >>hyperdb.DEBUG, 'create_class', (self, sql)
318 self.cursor.execute(sql)
320 return cols, mls
322 def create_journal_table(self, spec):
323 ''' create the journal table for a class given the spec and
324 already-determined cols
325 '''
326 # journal table
327 cols = ','.join(['%s varchar'%x
328 for x in 'nodeid date tag action params'.split()])
329 sql = 'create table %s__journal (%s)'%(spec.classname, cols)
330 if __debug__:
331 print >>hyperdb.DEBUG, 'create_class', (self, sql)
332 self.cursor.execute(sql)
334 def create_multilink_table(self, spec, ml):
335 ''' Create a multilink table for the "ml" property of the class
336 given by the spec
337 '''
338 sql = 'create table %s_%s (linkid varchar, nodeid varchar)'%(
339 spec.classname, ml)
340 if __debug__:
341 print >>hyperdb.DEBUG, 'create_class', (self, sql)
342 self.cursor.execute(sql)
344 def create_class(self, spec):
345 ''' Create a database table according to the given spec.
346 '''
347 cols, mls = self.create_class_table(spec)
348 self.create_journal_table(spec)
350 # now create the multilink tables
351 for ml in mls:
352 self.create_multilink_table(spec, ml)
354 # ID counter
355 sql = 'insert into ids (name, num) values (%s,%s)'%(self.arg, self.arg)
356 vals = (spec.classname, 1)
357 if __debug__:
358 print >>hyperdb.DEBUG, 'create_class', (self, sql, vals)
359 self.cursor.execute(sql, vals)
361 def drop_class(self, spec):
362 ''' Drop the given table from the database.
364 Drop the journal and multilink tables too.
365 '''
366 # figure the multilinks
367 mls = []
368 for col, prop in spec.properties.items():
369 if isinstance(prop, Multilink):
370 mls.append(col)
372 sql = 'drop table _%s'%spec.classname
373 if __debug__:
374 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
375 self.cursor.execute(sql)
377 sql = 'drop table %s__journal'%spec.classname
378 if __debug__:
379 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
380 self.cursor.execute(sql)
382 for ml in mls:
383 sql = 'drop table %s_%s'%(spec.classname, ml)
384 if __debug__:
385 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
386 self.cursor.execute(sql)
388 #
389 # Classes
390 #
391 def __getattr__(self, classname):
392 ''' A convenient way of calling self.getclass(classname).
393 '''
394 if self.classes.has_key(classname):
395 if __debug__:
396 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
397 return self.classes[classname]
398 raise AttributeError, classname
400 def addclass(self, cl):
401 ''' Add a Class to the hyperdatabase.
402 '''
403 if __debug__:
404 print >>hyperdb.DEBUG, 'addclass', (self, cl)
405 cn = cl.classname
406 if self.classes.has_key(cn):
407 raise ValueError, cn
408 self.classes[cn] = cl
410 def getclasses(self):
411 ''' Return a list of the names of all existing classes.
412 '''
413 if __debug__:
414 print >>hyperdb.DEBUG, 'getclasses', (self,)
415 l = self.classes.keys()
416 l.sort()
417 return l
419 def getclass(self, classname):
420 '''Get the Class object representing a particular class.
422 If 'classname' is not a valid class name, a KeyError is raised.
423 '''
424 if __debug__:
425 print >>hyperdb.DEBUG, 'getclass', (self, classname)
426 try:
427 return self.classes[classname]
428 except KeyError:
429 raise KeyError, 'There is no class called "%s"'%classname
431 def clear(self):
432 ''' Delete all database contents.
434 Note: I don't commit here, which is different behaviour to the
435 "nuke from orbit" behaviour in the *dbms.
436 '''
437 if __debug__:
438 print >>hyperdb.DEBUG, 'clear', (self,)
439 for cn in self.classes.keys():
440 sql = 'delete from _%s'%cn
441 if __debug__:
442 print >>hyperdb.DEBUG, 'clear', (self, sql)
443 self.cursor.execute(sql)
445 #
446 # Node IDs
447 #
448 def newid(self, classname):
449 ''' Generate a new id for the given class
450 '''
451 # get the next ID
452 sql = 'select num from ids where name=%s'%self.arg
453 if __debug__:
454 print >>hyperdb.DEBUG, 'newid', (self, sql, classname)
455 self.cursor.execute(sql, (classname, ))
456 newid = self.cursor.fetchone()[0]
458 # update the counter
459 sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
460 vals = (int(newid)+1, classname)
461 if __debug__:
462 print >>hyperdb.DEBUG, 'newid', (self, sql, vals)
463 self.cursor.execute(sql, vals)
465 # return as string
466 return str(newid)
468 def setid(self, classname, setid):
469 ''' Set the id counter: used during import of database
470 '''
471 sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
472 vals = (setid, classname)
473 if __debug__:
474 print >>hyperdb.DEBUG, 'setid', (self, sql, vals)
475 self.cursor.execute(sql, vals)
477 #
478 # Nodes
479 #
481 def addnode(self, classname, nodeid, node):
482 ''' Add the specified node to its class's db.
483 '''
484 if __debug__:
485 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
486 # gadfly requires values for all non-multilink columns
487 cl = self.classes[classname]
488 cols, mls = self.determine_columns(cl.properties.items())
490 # we'll be supplied these props if we're doing an import
491 if not node.has_key('creator'):
492 # add in the "calculated" properties (dupe so we don't affect
493 # calling code's node assumptions)
494 node = node.copy()
495 node['creation'] = node['activity'] = date.Date()
496 node['creator'] = self.curuserid
498 # default the non-multilink columns
499 for col, prop in cl.properties.items():
500 if not isinstance(col, Multilink):
501 if not node.has_key(col):
502 node[col] = None
504 # clear this node out of the cache if it's in there
505 key = (classname, nodeid)
506 if self.cache.has_key(key):
507 del self.cache[key]
508 self.cache_lru.remove(key)
510 # make the node data safe for the DB
511 node = self.serialise(classname, node)
513 # make sure the ordering is correct for column name -> column value
514 vals = tuple([node[col[1:]] for col in cols]) + (nodeid, 0)
515 s = ','.join([self.arg for x in cols]) + ',%s,%s'%(self.arg, self.arg)
516 cols = ','.join(cols) + ',id,__retired__'
518 # perform the inserts
519 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
520 if __debug__:
521 print >>hyperdb.DEBUG, 'addnode', (self, sql, vals)
522 self.cursor.execute(sql, vals)
524 # insert the multilink rows
525 for col in mls:
526 t = '%s_%s'%(classname, col)
527 for entry in node[col]:
528 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
529 self.arg, self.arg)
530 self.sql(sql, (entry, nodeid))
532 # make sure we do the commit-time extra stuff for this node
533 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
535 def setnode(self, classname, nodeid, values, multilink_changes):
536 ''' Change the specified node.
537 '''
538 if __debug__:
539 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, values)
541 # clear this node out of the cache if it's in there
542 key = (classname, nodeid)
543 if self.cache.has_key(key):
544 del self.cache[key]
545 self.cache_lru.remove(key)
547 # add the special props
548 values = values.copy()
549 values['activity'] = date.Date()
551 # make db-friendly
552 values = self.serialise(classname, values)
554 cl = self.classes[classname]
555 cols = []
556 mls = []
557 # add the multilinks separately
558 props = cl.getprops()
559 for col in values.keys():
560 prop = props[col]
561 if isinstance(prop, Multilink):
562 mls.append(col)
563 else:
564 cols.append('_'+col)
565 cols.sort()
567 # if there's any updates to regular columns, do them
568 if cols:
569 # make sure the ordering is correct for column name -> column value
570 sqlvals = tuple([values[col[1:]] for col in cols]) + (nodeid,)
571 s = ','.join(['%s=%s'%(x, self.arg) for x in cols])
572 cols = ','.join(cols)
574 # perform the update
575 sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
576 if __debug__:
577 print >>hyperdb.DEBUG, 'setnode', (self, sql, sqlvals)
578 self.cursor.execute(sql, sqlvals)
580 # now the fun bit, updating the multilinks ;)
581 for col, (add, remove) in multilink_changes.items():
582 tn = '%s_%s'%(classname, col)
583 if add:
584 sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
585 self.arg, self.arg)
586 for addid in add:
587 self.sql(sql, (nodeid, addid))
588 if remove:
589 sql = 'delete from %s where nodeid=%s and linkid=%s'%(tn,
590 self.arg, self.arg)
591 for removeid in remove:
592 self.sql(sql, (nodeid, removeid))
594 # make sure we do the commit-time extra stuff for this node
595 self.transactions.append((self.doSaveNode, (classname, nodeid, values)))
597 def getnode(self, classname, nodeid):
598 ''' Get a node from the database.
599 '''
600 if __debug__:
601 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid)
603 # see if we have this node cached
604 key = (classname, nodeid)
605 if self.cache.has_key(key):
606 # push us back to the top of the LRU
607 self.cache_lru.remove(key)
608 self.cache_lru.insert(0, key)
609 # return the cached information
610 return self.cache[key]
612 # figure the columns we're fetching
613 cl = self.classes[classname]
614 cols, mls = self.determine_columns(cl.properties.items())
615 scols = ','.join(cols)
617 # perform the basic property fetch
618 sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
619 self.sql(sql, (nodeid,))
621 values = self.sql_fetchone()
622 if values is None:
623 raise IndexError, 'no such %s node %s'%(classname, nodeid)
625 # make up the node
626 node = {}
627 for col in range(len(cols)):
628 node[cols[col][1:]] = values[col]
630 # now the multilinks
631 for col in mls:
632 # get the link ids
633 sql = 'select linkid from %s_%s where nodeid=%s'%(classname, col,
634 self.arg)
635 self.cursor.execute(sql, (nodeid,))
636 # extract the first column from the result
637 node[col] = [x[0] for x in self.cursor.fetchall()]
639 # un-dbificate the node data
640 node = self.unserialise(classname, node)
642 # save off in the cache
643 key = (classname, nodeid)
644 self.cache[key] = node
645 # update the LRU
646 self.cache_lru.insert(0, key)
647 if len(self.cache_lru) > ROW_CACHE_SIZE:
648 del self.cache[self.cache_lru.pop()]
650 return node
652 def destroynode(self, classname, nodeid):
653 '''Remove a node from the database. Called exclusively by the
654 destroy() method on Class.
655 '''
656 if __debug__:
657 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
659 # make sure the node exists
660 if not self.hasnode(classname, nodeid):
661 raise IndexError, '%s has no node %s'%(classname, nodeid)
663 # see if we have this node cached
664 if self.cache.has_key((classname, nodeid)):
665 del self.cache[(classname, nodeid)]
667 # see if there's any obvious commit actions that we should get rid of
668 for entry in self.transactions[:]:
669 if entry[1][:2] == (classname, nodeid):
670 self.transactions.remove(entry)
672 # now do the SQL
673 sql = 'delete from _%s where id=%s'%(classname, self.arg)
674 self.sql(sql, (nodeid,))
676 # remove from multilnks
677 cl = self.getclass(classname)
678 x, mls = self.determine_columns(cl.properties.items())
679 for col in mls:
680 # get the link ids
681 sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
682 self.cursor.execute(sql, (nodeid,))
684 # remove journal entries
685 sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
686 self.sql(sql, (nodeid,))
688 def serialise(self, classname, node):
689 '''Copy the node contents, converting non-marshallable data into
690 marshallable data.
691 '''
692 if __debug__:
693 print >>hyperdb.DEBUG, 'serialise', classname, node
694 properties = self.getclass(classname).getprops()
695 d = {}
696 for k, v in node.items():
697 # if the property doesn't exist, or is the "retired" flag then
698 # it won't be in the properties dict
699 if not properties.has_key(k):
700 d[k] = v
701 continue
703 # get the property spec
704 prop = properties[k]
706 if isinstance(prop, Password) and v is not None:
707 d[k] = str(v)
708 elif isinstance(prop, Date) and v is not None:
709 d[k] = v.serialise()
710 elif isinstance(prop, Interval) and v is not None:
711 d[k] = v.serialise()
712 else:
713 d[k] = v
714 return d
716 def unserialise(self, classname, node):
717 '''Decode the marshalled node data
718 '''
719 if __debug__:
720 print >>hyperdb.DEBUG, 'unserialise', classname, node
721 properties = self.getclass(classname).getprops()
722 d = {}
723 for k, v in node.items():
724 # if the property doesn't exist, or is the "retired" flag then
725 # it won't be in the properties dict
726 if not properties.has_key(k):
727 d[k] = v
728 continue
730 # get the property spec
731 prop = properties[k]
733 if isinstance(prop, Date) and v is not None:
734 d[k] = date.Date(v)
735 elif isinstance(prop, Interval) and v is not None:
736 d[k] = date.Interval(v)
737 elif isinstance(prop, Password) and v is not None:
738 p = password.Password()
739 p.unpack(v)
740 d[k] = p
741 elif (isinstance(prop, Boolean) or isinstance(prop, Number)) and v is not None:
742 d[k]=float(v)
743 else:
744 d[k] = v
745 return d
747 def hasnode(self, classname, nodeid):
748 ''' Determine if the database has a given node.
749 '''
750 sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
751 if __debug__:
752 print >>hyperdb.DEBUG, 'hasnode', (self, sql, nodeid)
753 self.cursor.execute(sql, (nodeid,))
754 return int(self.cursor.fetchone()[0])
756 def countnodes(self, classname):
757 ''' Count the number of nodes that exist for a particular Class.
758 '''
759 sql = 'select count(*) from _%s'%classname
760 if __debug__:
761 print >>hyperdb.DEBUG, 'countnodes', (self, sql)
762 self.cursor.execute(sql)
763 return self.cursor.fetchone()[0]
765 def getnodeids(self, classname, retired=0):
766 ''' Retrieve all the ids of the nodes for a particular Class.
768 Set retired=None to get all nodes. Otherwise it'll get all the
769 retired or non-retired nodes, depending on the flag.
770 '''
771 # flip the sense of the flag if we don't want all of them
772 if retired is not None:
773 retired = not retired
774 sql = 'select id from _%s where __retired__ <> %s'%(classname, self.arg)
775 if __debug__:
776 print >>hyperdb.DEBUG, 'getnodeids', (self, sql, retired)
777 self.cursor.execute(sql, (retired,))
778 return [x[0] for x in self.cursor.fetchall()]
780 def addjournal(self, classname, nodeid, action, params, creator=None,
781 creation=None):
782 ''' Journal the Action
783 'action' may be:
785 'create' or 'set' -- 'params' is a dictionary of property values
786 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
787 'retire' -- 'params' is None
788 '''
789 # serialise the parameters now if necessary
790 if isinstance(params, type({})):
791 if action in ('set', 'create'):
792 params = self.serialise(classname, params)
794 # handle supply of the special journalling parameters (usually
795 # supplied on importing an existing database)
796 if creator:
797 journaltag = creator
798 else:
799 journaltag = self.curuserid
800 if creation:
801 journaldate = creation.serialise()
802 else:
803 journaldate = date.Date().serialise()
805 # create the journal entry
806 cols = ','.join('nodeid date tag action params'.split())
808 if __debug__:
809 print >>hyperdb.DEBUG, 'addjournal', (nodeid, journaldate,
810 journaltag, action, params)
812 self.save_journal(classname, cols, nodeid, journaldate,
813 journaltag, action, params)
815 def save_journal(self, classname, cols, nodeid, journaldate,
816 journaltag, action, params):
817 ''' Save the journal entry to the database
818 '''
819 raise NotImplemented
821 def getjournal(self, classname, nodeid):
822 ''' get the journal for id
823 '''
824 # make sure the node exists
825 if not self.hasnode(classname, nodeid):
826 raise IndexError, '%s has no node %s'%(classname, nodeid)
828 cols = ','.join('nodeid date tag action params'.split())
829 return self.load_journal(classname, cols, nodeid)
831 def load_journal(self, classname, cols, nodeid):
832 ''' Load the journal from the database
833 '''
834 raise NotImplemented
836 def pack(self, pack_before):
837 ''' Delete all journal entries except "create" before 'pack_before'.
838 '''
839 # get a 'yyyymmddhhmmss' version of the date
840 date_stamp = pack_before.serialise()
842 # do the delete
843 for classname in self.classes.keys():
844 sql = "delete from %s__journal where date<%s and "\
845 "action<>'create'"%(classname, self.arg)
846 if __debug__:
847 print >>hyperdb.DEBUG, 'pack', (self, sql, date_stamp)
848 self.cursor.execute(sql, (date_stamp,))
850 def sql_commit(self):
851 ''' Actually commit to the database.
852 '''
853 self.conn.commit()
855 def commit(self):
856 ''' Commit the current transactions.
858 Save all data changed since the database was opened or since the
859 last commit() or rollback().
860 '''
861 if __debug__:
862 print >>hyperdb.DEBUG, 'commit', (self,)
864 # commit the database
865 self.sql_commit()
867 # now, do all the other transaction stuff
868 reindex = {}
869 for method, args in self.transactions:
870 reindex[method(*args)] = 1
872 # reindex the nodes that request it
873 for classname, nodeid in filter(None, reindex.keys()):
874 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
875 self.getclass(classname).index(nodeid)
877 # save the indexer state
878 self.indexer.save_index()
880 # clear out the transactions
881 self.transactions = []
883 def rollback(self):
884 ''' Reverse all actions from the current transaction.
886 Undo all the changes made since the database was opened or the last
887 commit() or rollback() was performed.
888 '''
889 if __debug__:
890 print >>hyperdb.DEBUG, 'rollback', (self,)
892 # roll back
893 self.conn.rollback()
895 # roll back "other" transaction stuff
896 for method, args in self.transactions:
897 # delete temporary files
898 if method == self.doStoreFile:
899 self.rollbackStoreFile(*args)
900 self.transactions = []
902 def doSaveNode(self, classname, nodeid, node):
903 ''' dummy that just generates a reindex event
904 '''
905 # return the classname, nodeid so we reindex this content
906 return (classname, nodeid)
908 def close(self):
909 ''' Close off the connection.
910 '''
911 self.conn.close()
912 if self.lockfile is not None:
913 locking.release_lock(self.lockfile)
914 if self.lockfile is not None:
915 self.lockfile.close()
916 self.lockfile = None
918 #
919 # The base Class class
920 #
921 class Class(hyperdb.Class):
922 ''' The handle to a particular class of nodes in a hyperdatabase.
924 All methods except __repr__ and getnode must be implemented by a
925 concrete backend Class.
926 '''
928 def __init__(self, db, classname, **properties):
929 '''Create a new class with a given name and property specification.
931 'classname' must not collide with the name of an existing class,
932 or a ValueError is raised. The keyword arguments in 'properties'
933 must map names to property objects, or a TypeError is raised.
934 '''
935 if (properties.has_key('creation') or properties.has_key('activity')
936 or properties.has_key('creator')):
937 raise ValueError, '"creation", "activity" and "creator" are '\
938 'reserved'
940 self.classname = classname
941 self.properties = properties
942 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
943 self.key = ''
945 # should we journal changes (default yes)
946 self.do_journal = 1
948 # do the db-related init stuff
949 db.addclass(self)
951 self.auditors = {'create': [], 'set': [], 'retire': []}
952 self.reactors = {'create': [], 'set': [], 'retire': []}
954 def schema(self):
955 ''' A dumpable version of the schema that we can store in the
956 database
957 '''
958 return (self.key, [(x, repr(y)) for x,y in self.properties.items()])
960 def enableJournalling(self):
961 '''Turn journalling on for this class
962 '''
963 self.do_journal = 1
965 def disableJournalling(self):
966 '''Turn journalling off for this class
967 '''
968 self.do_journal = 0
970 # Editing nodes:
971 def create(self, **propvalues):
972 ''' Create a new node of this class and return its id.
974 The keyword arguments in 'propvalues' map property names to values.
976 The values of arguments must be acceptable for the types of their
977 corresponding properties or a TypeError is raised.
979 If this class has a key property, it must be present and its value
980 must not collide with other key strings or a ValueError is raised.
982 Any other properties on this class that are missing from the
983 'propvalues' dictionary are set to None.
985 If an id in a link or multilink property does not refer to a valid
986 node, an IndexError is raised.
987 '''
988 self.fireAuditors('create', None, propvalues)
989 newid = self.create_inner(**propvalues)
990 self.fireReactors('create', newid, None)
991 return newid
993 def create_inner(self, **propvalues):
994 ''' Called by create, in-between the audit and react calls.
995 '''
996 if propvalues.has_key('id'):
997 raise KeyError, '"id" is reserved'
999 if self.db.journaltag is None:
1000 raise DatabaseError, 'Database open read-only'
1002 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1003 raise KeyError, '"creation" and "activity" are reserved'
1005 # new node's id
1006 newid = self.db.newid(self.classname)
1008 # validate propvalues
1009 num_re = re.compile('^\d+$')
1010 for key, value in propvalues.items():
1011 if key == self.key:
1012 try:
1013 self.lookup(value)
1014 except KeyError:
1015 pass
1016 else:
1017 raise ValueError, 'node with key "%s" exists'%value
1019 # try to handle this property
1020 try:
1021 prop = self.properties[key]
1022 except KeyError:
1023 raise KeyError, '"%s" has no property "%s"'%(self.classname,
1024 key)
1026 if value is not None and isinstance(prop, Link):
1027 if type(value) != type(''):
1028 raise ValueError, 'link value must be String'
1029 link_class = self.properties[key].classname
1030 # if it isn't a number, it's a key
1031 if not num_re.match(value):
1032 try:
1033 value = self.db.classes[link_class].lookup(value)
1034 except (TypeError, KeyError):
1035 raise IndexError, 'new property "%s": %s not a %s'%(
1036 key, value, link_class)
1037 elif not self.db.getclass(link_class).hasnode(value):
1038 raise IndexError, '%s has no node %s'%(link_class, value)
1040 # save off the value
1041 propvalues[key] = value
1043 # register the link with the newly linked node
1044 if self.do_journal and self.properties[key].do_journal:
1045 self.db.addjournal(link_class, value, 'link',
1046 (self.classname, newid, key))
1048 elif isinstance(prop, Multilink):
1049 if type(value) != type([]):
1050 raise TypeError, 'new property "%s" not a list of ids'%key
1052 # clean up and validate the list of links
1053 link_class = self.properties[key].classname
1054 l = []
1055 for entry in value:
1056 if type(entry) != type(''):
1057 raise ValueError, '"%s" multilink value (%r) '\
1058 'must contain Strings'%(key, value)
1059 # if it isn't a number, it's a key
1060 if not num_re.match(entry):
1061 try:
1062 entry = self.db.classes[link_class].lookup(entry)
1063 except (TypeError, KeyError):
1064 raise IndexError, 'new property "%s": %s not a %s'%(
1065 key, entry, self.properties[key].classname)
1066 l.append(entry)
1067 value = l
1068 propvalues[key] = value
1070 # handle additions
1071 for nodeid in value:
1072 if not self.db.getclass(link_class).hasnode(nodeid):
1073 raise IndexError, '%s has no node %s'%(link_class,
1074 nodeid)
1075 # register the link with the newly linked node
1076 if self.do_journal and self.properties[key].do_journal:
1077 self.db.addjournal(link_class, nodeid, 'link',
1078 (self.classname, newid, key))
1080 elif isinstance(prop, String):
1081 if type(value) != type('') and type(value) != type(u''):
1082 raise TypeError, 'new property "%s" not a string'%key
1084 elif isinstance(prop, Password):
1085 if not isinstance(value, password.Password):
1086 raise TypeError, 'new property "%s" not a Password'%key
1088 elif isinstance(prop, Date):
1089 if value is not None and not isinstance(value, date.Date):
1090 raise TypeError, 'new property "%s" not a Date'%key
1092 elif isinstance(prop, Interval):
1093 if value is not None and not isinstance(value, date.Interval):
1094 raise TypeError, 'new property "%s" not an Interval'%key
1096 elif value is not None and isinstance(prop, Number):
1097 try:
1098 float(value)
1099 except ValueError:
1100 raise TypeError, 'new property "%s" not numeric'%key
1102 elif value is not None and isinstance(prop, Boolean):
1103 try:
1104 int(value)
1105 except ValueError:
1106 raise TypeError, 'new property "%s" not boolean'%key
1108 # make sure there's data where there needs to be
1109 for key, prop in self.properties.items():
1110 if propvalues.has_key(key):
1111 continue
1112 if key == self.key:
1113 raise ValueError, 'key property "%s" is required'%key
1114 if isinstance(prop, Multilink):
1115 propvalues[key] = []
1116 else:
1117 propvalues[key] = None
1119 # done
1120 self.db.addnode(self.classname, newid, propvalues)
1121 if self.do_journal:
1122 self.db.addjournal(self.classname, newid, 'create', {})
1124 return newid
1126 def export_list(self, propnames, nodeid):
1127 ''' Export a node - generate a list of CSV-able data in the order
1128 specified by propnames for the given node.
1129 '''
1130 properties = self.getprops()
1131 l = []
1132 for prop in propnames:
1133 proptype = properties[prop]
1134 value = self.get(nodeid, prop)
1135 # "marshal" data where needed
1136 if value is None:
1137 pass
1138 elif isinstance(proptype, hyperdb.Date):
1139 value = value.get_tuple()
1140 elif isinstance(proptype, hyperdb.Interval):
1141 value = value.get_tuple()
1142 elif isinstance(proptype, hyperdb.Password):
1143 value = str(value)
1144 l.append(repr(value))
1145 return l
1147 def import_list(self, propnames, proplist):
1148 ''' Import a node - all information including "id" is present and
1149 should not be sanity checked. Triggers are not triggered. The
1150 journal should be initialised using the "creator" and "created"
1151 information.
1153 Return the nodeid of the node imported.
1154 '''
1155 if self.db.journaltag is None:
1156 raise DatabaseError, 'Database open read-only'
1157 properties = self.getprops()
1159 # make the new node's property map
1160 d = {}
1161 for i in range(len(propnames)):
1162 # Use eval to reverse the repr() used to output the CSV
1163 value = eval(proplist[i])
1165 # Figure the property for this column
1166 propname = propnames[i]
1167 prop = properties[propname]
1169 # "unmarshal" where necessary
1170 if propname == 'id':
1171 newid = value
1172 continue
1173 elif value is None:
1174 # don't set Nones
1175 continue
1176 elif isinstance(prop, hyperdb.Date):
1177 value = date.Date(value)
1178 elif isinstance(prop, hyperdb.Interval):
1179 value = date.Interval(value)
1180 elif isinstance(prop, hyperdb.Password):
1181 pwd = password.Password()
1182 pwd.unpack(value)
1183 value = pwd
1184 d[propname] = value
1186 # add the node and journal
1187 self.db.addnode(self.classname, newid, d)
1189 # extract the extraneous journalling gumpf and nuke it
1190 if d.has_key('creator'):
1191 creator = d['creator']
1192 del d['creator']
1193 else:
1194 creator = None
1195 if d.has_key('creation'):
1196 creation = d['creation']
1197 del d['creation']
1198 else:
1199 creation = None
1200 if d.has_key('activity'):
1201 del d['activity']
1202 self.db.addjournal(self.classname, newid, 'create', {}, creator,
1203 creation)
1204 return newid
1206 _marker = []
1207 def get(self, nodeid, propname, default=_marker, cache=1):
1208 '''Get the value of a property on an existing node of this class.
1210 'nodeid' must be the id of an existing node of this class or an
1211 IndexError is raised. 'propname' must be the name of a property
1212 of this class or a KeyError is raised.
1214 'cache' indicates whether the transaction cache should be queried
1215 for the node. If the node has been modified and you need to
1216 determine what its values prior to modification are, you need to
1217 set cache=0.
1218 '''
1219 if propname == 'id':
1220 return nodeid
1222 # get the node's dict
1223 d = self.db.getnode(self.classname, nodeid)
1225 if propname == 'creation':
1226 if d.has_key('creation'):
1227 return d['creation']
1228 else:
1229 return date.Date()
1230 if propname == 'activity':
1231 if d.has_key('activity'):
1232 return d['activity']
1233 else:
1234 return date.Date()
1235 if propname == 'creator':
1236 if d.has_key('creator'):
1237 return d['creator']
1238 else:
1239 return self.db.curuserid
1241 # get the property (raises KeyErorr if invalid)
1242 prop = self.properties[propname]
1244 if not d.has_key(propname):
1245 if default is self._marker:
1246 if isinstance(prop, Multilink):
1247 return []
1248 else:
1249 return None
1250 else:
1251 return default
1253 # don't pass our list to other code
1254 if isinstance(prop, Multilink):
1255 return d[propname][:]
1257 return d[propname]
1259 def getnode(self, nodeid, cache=1):
1260 ''' Return a convenience wrapper for the node.
1262 'nodeid' must be the id of an existing node of this class or an
1263 IndexError is raised.
1265 'cache' indicates whether the transaction cache should be queried
1266 for the node. If the node has been modified and you need to
1267 determine what its values prior to modification are, you need to
1268 set cache=0.
1269 '''
1270 return Node(self, nodeid, cache=cache)
1272 def set(self, nodeid, **propvalues):
1273 '''Modify a property on an existing node of this class.
1275 'nodeid' must be the id of an existing node of this class or an
1276 IndexError is raised.
1278 Each key in 'propvalues' must be the name of a property of this
1279 class or a KeyError is raised.
1281 All values in 'propvalues' must be acceptable types for their
1282 corresponding properties or a TypeError is raised.
1284 If the value of the key property is set, it must not collide with
1285 other key strings or a ValueError is raised.
1287 If the value of a Link or Multilink property contains an invalid
1288 node id, a ValueError is raised.
1289 '''
1290 if not propvalues:
1291 return propvalues
1293 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1294 raise KeyError, '"creation" and "activity" are reserved'
1296 if propvalues.has_key('id'):
1297 raise KeyError, '"id" is reserved'
1299 if self.db.journaltag is None:
1300 raise DatabaseError, 'Database open read-only'
1302 self.fireAuditors('set', nodeid, propvalues)
1303 # Take a copy of the node dict so that the subsequent set
1304 # operation doesn't modify the oldvalues structure.
1305 # XXX used to try the cache here first
1306 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1308 node = self.db.getnode(self.classname, nodeid)
1309 if self.is_retired(nodeid):
1310 raise IndexError, 'Requested item is retired'
1311 num_re = re.compile('^\d+$')
1313 # if the journal value is to be different, store it in here
1314 journalvalues = {}
1316 # remember the add/remove stuff for multilinks, making it easier
1317 # for the Database layer to do its stuff
1318 multilink_changes = {}
1320 for propname, value in propvalues.items():
1321 # check to make sure we're not duplicating an existing key
1322 if propname == self.key and node[propname] != value:
1323 try:
1324 self.lookup(value)
1325 except KeyError:
1326 pass
1327 else:
1328 raise ValueError, 'node with key "%s" exists'%value
1330 # this will raise the KeyError if the property isn't valid
1331 # ... we don't use getprops() here because we only care about
1332 # the writeable properties.
1333 try:
1334 prop = self.properties[propname]
1335 except KeyError:
1336 raise KeyError, '"%s" has no property named "%s"'%(
1337 self.classname, propname)
1339 # if the value's the same as the existing value, no sense in
1340 # doing anything
1341 current = node.get(propname, None)
1342 if value == current:
1343 del propvalues[propname]
1344 continue
1345 journalvalues[propname] = current
1347 # do stuff based on the prop type
1348 if isinstance(prop, Link):
1349 link_class = prop.classname
1350 # if it isn't a number, it's a key
1351 if value is not None and not isinstance(value, type('')):
1352 raise ValueError, 'property "%s" link value be a string'%(
1353 propname)
1354 if isinstance(value, type('')) and not num_re.match(value):
1355 try:
1356 value = self.db.classes[link_class].lookup(value)
1357 except (TypeError, KeyError):
1358 raise IndexError, 'new property "%s": %s not a %s'%(
1359 propname, value, prop.classname)
1361 if (value is not None and
1362 not self.db.getclass(link_class).hasnode(value)):
1363 raise IndexError, '%s has no node %s'%(link_class, value)
1365 if self.do_journal and prop.do_journal:
1366 # register the unlink with the old linked node
1367 if node[propname] is not None:
1368 self.db.addjournal(link_class, node[propname], 'unlink',
1369 (self.classname, nodeid, propname))
1371 # register the link with the newly linked node
1372 if value is not None:
1373 self.db.addjournal(link_class, value, 'link',
1374 (self.classname, nodeid, propname))
1376 elif isinstance(prop, Multilink):
1377 if type(value) != type([]):
1378 raise TypeError, 'new property "%s" not a list of'\
1379 ' ids'%propname
1380 link_class = self.properties[propname].classname
1381 l = []
1382 for entry in value:
1383 # if it isn't a number, it's a key
1384 if type(entry) != type(''):
1385 raise ValueError, 'new property "%s" link value ' \
1386 'must be a string'%propname
1387 if not num_re.match(entry):
1388 try:
1389 entry = self.db.classes[link_class].lookup(entry)
1390 except (TypeError, KeyError):
1391 raise IndexError, 'new property "%s": %s not a %s'%(
1392 propname, entry,
1393 self.properties[propname].classname)
1394 l.append(entry)
1395 value = l
1396 propvalues[propname] = value
1398 # figure the journal entry for this property
1399 add = []
1400 remove = []
1402 # handle removals
1403 if node.has_key(propname):
1404 l = node[propname]
1405 else:
1406 l = []
1407 for id in l[:]:
1408 if id in value:
1409 continue
1410 # register the unlink with the old linked node
1411 if self.do_journal and self.properties[propname].do_journal:
1412 self.db.addjournal(link_class, id, 'unlink',
1413 (self.classname, nodeid, propname))
1414 l.remove(id)
1415 remove.append(id)
1417 # handle additions
1418 for id in value:
1419 if not self.db.getclass(link_class).hasnode(id):
1420 raise IndexError, '%s has no node %s'%(link_class, id)
1421 if id in l:
1422 continue
1423 # register the link with the newly linked node
1424 if self.do_journal and self.properties[propname].do_journal:
1425 self.db.addjournal(link_class, id, 'link',
1426 (self.classname, nodeid, propname))
1427 l.append(id)
1428 add.append(id)
1430 # figure the journal entry
1431 l = []
1432 if add:
1433 l.append(('+', add))
1434 if remove:
1435 l.append(('-', remove))
1436 multilink_changes[propname] = (add, remove)
1437 if l:
1438 journalvalues[propname] = tuple(l)
1440 elif isinstance(prop, String):
1441 if value is not None and type(value) != type('') and type(value) != type(u''):
1442 raise TypeError, 'new property "%s" not a string'%propname
1444 elif isinstance(prop, Password):
1445 if not isinstance(value, password.Password):
1446 raise TypeError, 'new property "%s" not a Password'%propname
1447 propvalues[propname] = value
1449 elif value is not None and isinstance(prop, Date):
1450 if not isinstance(value, date.Date):
1451 raise TypeError, 'new property "%s" not a Date'% propname
1452 propvalues[propname] = value
1454 elif value is not None and isinstance(prop, Interval):
1455 if not isinstance(value, date.Interval):
1456 raise TypeError, 'new property "%s" not an '\
1457 'Interval'%propname
1458 propvalues[propname] = value
1460 elif value is not None and isinstance(prop, Number):
1461 try:
1462 float(value)
1463 except ValueError:
1464 raise TypeError, 'new property "%s" not numeric'%propname
1466 elif value is not None and isinstance(prop, Boolean):
1467 try:
1468 int(value)
1469 except ValueError:
1470 raise TypeError, 'new property "%s" not boolean'%propname
1472 # nothing to do?
1473 if not propvalues:
1474 return propvalues
1476 # do the set, and journal it
1477 self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
1479 if self.do_journal:
1480 self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
1482 self.fireReactors('set', nodeid, oldvalues)
1484 return propvalues
1486 def retire(self, nodeid):
1487 '''Retire a node.
1489 The properties on the node remain available from the get() method,
1490 and the node's id is never reused.
1492 Retired nodes are not returned by the find(), list(), or lookup()
1493 methods, and other nodes may reuse the values of their key properties.
1494 '''
1495 if self.db.journaltag is None:
1496 raise DatabaseError, 'Database open read-only'
1498 self.fireAuditors('retire', nodeid, None)
1500 # use the arg for __retired__ to cope with any odd database type
1501 # conversion (hello, sqlite)
1502 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1503 self.db.arg, self.db.arg)
1504 if __debug__:
1505 print >>hyperdb.DEBUG, 'retire', (self, sql, nodeid)
1506 self.db.cursor.execute(sql, (1, nodeid))
1508 self.fireReactors('retire', nodeid, None)
1510 def is_retired(self, nodeid):
1511 '''Return true if the node is rerired
1512 '''
1513 sql = 'select __retired__ from _%s where id=%s'%(self.classname,
1514 self.db.arg)
1515 if __debug__:
1516 print >>hyperdb.DEBUG, 'is_retired', (self, sql, nodeid)
1517 self.db.cursor.execute(sql, (nodeid,))
1518 return int(self.db.sql_fetchone()[0])
1520 def destroy(self, nodeid):
1521 '''Destroy a node.
1523 WARNING: this method should never be used except in extremely rare
1524 situations where there could never be links to the node being
1525 deleted
1526 WARNING: use retire() instead
1527 WARNING: the properties of this node will not be available ever again
1528 WARNING: really, use retire() instead
1530 Well, I think that's enough warnings. This method exists mostly to
1531 support the session storage of the cgi interface.
1533 The node is completely removed from the hyperdb, including all journal
1534 entries. It will no longer be available, and will generally break code
1535 if there are any references to the node.
1536 '''
1537 if self.db.journaltag is None:
1538 raise DatabaseError, 'Database open read-only'
1539 self.db.destroynode(self.classname, nodeid)
1541 def history(self, nodeid):
1542 '''Retrieve the journal of edits on a particular node.
1544 'nodeid' must be the id of an existing node of this class or an
1545 IndexError is raised.
1547 The returned list contains tuples of the form
1549 (nodeid, date, tag, action, params)
1551 'date' is a Timestamp object specifying the time of the change and
1552 'tag' is the journaltag specified when the database was opened.
1553 '''
1554 if not self.do_journal:
1555 raise ValueError, 'Journalling is disabled for this class'
1556 return self.db.getjournal(self.classname, nodeid)
1558 # Locating nodes:
1559 def hasnode(self, nodeid):
1560 '''Determine if the given nodeid actually exists
1561 '''
1562 return self.db.hasnode(self.classname, nodeid)
1564 def setkey(self, propname):
1565 '''Select a String property of this class to be the key property.
1567 'propname' must be the name of a String property of this class or
1568 None, or a TypeError is raised. The values of the key property on
1569 all existing nodes must be unique or a ValueError is raised.
1570 '''
1571 # XXX create an index on the key prop column
1572 prop = self.getprops()[propname]
1573 if not isinstance(prop, String):
1574 raise TypeError, 'key properties must be String'
1575 self.key = propname
1577 def getkey(self):
1578 '''Return the name of the key property for this class or None.'''
1579 return self.key
1581 def labelprop(self, default_to_id=0):
1582 ''' Return the property name for a label for the given node.
1584 This method attempts to generate a consistent label for the node.
1585 It tries the following in order:
1586 1. key property
1587 2. "name" property
1588 3. "title" property
1589 4. first property from the sorted property name list
1590 '''
1591 k = self.getkey()
1592 if k:
1593 return k
1594 props = self.getprops()
1595 if props.has_key('name'):
1596 return 'name'
1597 elif props.has_key('title'):
1598 return 'title'
1599 if default_to_id:
1600 return 'id'
1601 props = props.keys()
1602 props.sort()
1603 return props[0]
1605 def lookup(self, keyvalue):
1606 '''Locate a particular node by its key property and return its id.
1608 If this class has no key property, a TypeError is raised. If the
1609 'keyvalue' matches one of the values for the key property among
1610 the nodes in this class, the matching node's id is returned;
1611 otherwise a KeyError is raised.
1612 '''
1613 if not self.key:
1614 raise TypeError, 'No key property set for class %s'%self.classname
1616 # use the arg to handle any odd database type conversion (hello,
1617 # sqlite)
1618 sql = "select id from _%s where _%s=%s and __retired__ <> %s"%(
1619 self.classname, self.key, self.db.arg, self.db.arg)
1620 self.db.sql(sql, (keyvalue, 1))
1622 # see if there was a result that's not retired
1623 row = self.db.sql_fetchone()
1624 if not row:
1625 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1626 keyvalue, self.classname)
1628 # return the id
1629 return row[0]
1631 def find(self, **propspec):
1632 '''Get the ids of nodes in this class which link to the given nodes.
1634 'propspec' consists of keyword args propname=nodeid or
1635 propname={nodeid:1, }
1636 'propname' must be the name of a property in this class, or a
1637 KeyError is raised. That property must be a Link or Multilink
1638 property, or a TypeError is raised.
1640 Any node in this class whose 'propname' property links to any of the
1641 nodeids will be returned. Used by the full text indexing, which knows
1642 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1643 issues:
1645 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1646 '''
1647 if __debug__:
1648 print >>hyperdb.DEBUG, 'find', (self, propspec)
1650 # shortcut
1651 if not propspec:
1652 return []
1654 # validate the args
1655 props = self.getprops()
1656 propspec = propspec.items()
1657 for propname, nodeids in propspec:
1658 # check the prop is OK
1659 prop = props[propname]
1660 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1661 raise TypeError, "'%s' not a Link/Multilink property"%propname
1663 # first, links
1664 where = []
1665 allvalues = ()
1666 a = self.db.arg
1667 for prop, values in propspec:
1668 if not isinstance(props[prop], hyperdb.Link):
1669 continue
1670 if type(values) is type(''):
1671 allvalues += (values,)
1672 where.append('_%s = %s'%(prop, a))
1673 else:
1674 allvalues += tuple(values.keys())
1675 where.append('_%s in (%s)'%(prop, ','.join([a]*len(values))))
1676 tables = []
1677 if where:
1678 tables.append('select id as nodeid from _%s where %s'%(
1679 self.classname, ' and '.join(where)))
1681 # now multilinks
1682 for prop, values in propspec:
1683 if not isinstance(props[prop], hyperdb.Multilink):
1684 continue
1685 if type(values) is type(''):
1686 allvalues += (values,)
1687 s = a
1688 else:
1689 allvalues += tuple(values.keys())
1690 s = ','.join([a]*len(values))
1691 tables.append('select nodeid from %s_%s where linkid in (%s)'%(
1692 self.classname, prop, s))
1693 sql = '\nunion\n'.join(tables)
1694 self.db.sql(sql, allvalues)
1695 l = [x[0] for x in self.db.sql_fetchall()]
1696 if __debug__:
1697 print >>hyperdb.DEBUG, 'find ... ', l
1698 return l
1700 def stringFind(self, **requirements):
1701 '''Locate a particular node by matching a set of its String
1702 properties in a caseless search.
1704 If the property is not a String property, a TypeError is raised.
1706 The return is a list of the id of all nodes that match.
1707 '''
1708 where = []
1709 args = []
1710 for propname in requirements.keys():
1711 prop = self.properties[propname]
1712 if isinstance(not prop, String):
1713 raise TypeError, "'%s' not a String property"%propname
1714 where.append(propname)
1715 args.append(requirements[propname].lower())
1717 # generate the where clause
1718 s = ' and '.join(['_%s=%s'%(col, self.db.arg) for col in where])
1719 sql = 'select id from _%s where %s'%(self.classname, s)
1720 self.db.sql(sql, tuple(args))
1721 l = [x[0] for x in self.db.sql_fetchall()]
1722 if __debug__:
1723 print >>hyperdb.DEBUG, 'find ... ', l
1724 return l
1726 def list(self):
1727 ''' Return a list of the ids of the active nodes in this class.
1728 '''
1729 return self.db.getnodeids(self.classname, retired=0)
1731 def filter(self, search_matches, filterspec, sort=(None,None),
1732 group=(None,None)):
1733 ''' Return a list of the ids of the active nodes in this class that
1734 match the 'filter' spec, sorted by the group spec and then the
1735 sort spec
1737 "filterspec" is {propname: value(s)}
1738 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1739 and prop is a prop name or None
1740 "search_matches" is {nodeid: marker}
1742 The filter must match all properties specificed - but if the
1743 property value to match is a list, any one of the values in the
1744 list may match for that property to match.
1745 '''
1746 # just don't bother if the full-text search matched diddly
1747 if search_matches == {}:
1748 return []
1750 cn = self.classname
1752 # figure the WHERE clause from the filterspec
1753 props = self.getprops()
1754 frum = ['_'+cn]
1755 where = []
1756 args = []
1757 a = self.db.arg
1758 for k, v in filterspec.items():
1759 propclass = props[k]
1760 # now do other where clause stuff
1761 if isinstance(propclass, Multilink):
1762 tn = '%s_%s'%(cn, k)
1763 frum.append(tn)
1764 if isinstance(v, type([])):
1765 s = ','.join([a for x in v])
1766 where.append('id=%s.nodeid and %s.linkid in (%s)'%(tn,tn,s))
1767 args = args + v
1768 else:
1769 where.append('id=%s.nodeid and %s.linkid = %s'%(tn, tn, a))
1770 args.append(v)
1771 elif k == 'id':
1772 if isinstance(v, type([])):
1773 s = ','.join([a for x in v])
1774 where.append('%s in (%s)'%(k, s))
1775 args = args + v
1776 else:
1777 where.append('%s=%s'%(k, a))
1778 args.append(v)
1779 elif isinstance(propclass, String):
1780 if not isinstance(v, type([])):
1781 v = [v]
1783 # Quote the bits in the string that need it and then embed
1784 # in a "substring" search. Note - need to quote the '%' so
1785 # they make it through the python layer happily
1786 v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
1788 # now add to the where clause
1789 where.append(' or '.join(["_%s LIKE '%s'"%(k, s) for s in v]))
1790 # note: args are embedded in the query string now
1791 elif isinstance(propclass, Link):
1792 if isinstance(v, type([])):
1793 if '-1' in v:
1794 v.remove('-1')
1795 xtra = ' or _%s is NULL'%k
1796 else:
1797 xtra = ''
1798 if v:
1799 s = ','.join([a for x in v])
1800 where.append('(_%s in (%s)%s)'%(k, s, xtra))
1801 args = args + v
1802 else:
1803 where.append('_%s is NULL'%k)
1804 else:
1805 if v == '-1':
1806 v = None
1807 where.append('_%s is NULL'%k)
1808 else:
1809 where.append('_%s=%s'%(k, a))
1810 args.append(v)
1811 elif isinstance(propclass, Date):
1812 if isinstance(v, type([])):
1813 s = ','.join([a for x in v])
1814 where.append('_%s in (%s)'%(k, s))
1815 args = args + [date.Date(x).serialise() for x in v]
1816 else:
1817 where.append('_%s=%s'%(k, a))
1818 args.append(date.Date(v).serialise())
1819 elif isinstance(propclass, Interval):
1820 if isinstance(v, type([])):
1821 s = ','.join([a for x in v])
1822 where.append('_%s in (%s)'%(k, s))
1823 args = args + [date.Interval(x).serialise() for x in v]
1824 else:
1825 where.append('_%s=%s'%(k, a))
1826 args.append(date.Interval(v).serialise())
1827 else:
1828 if isinstance(v, type([])):
1829 s = ','.join([a for x in v])
1830 where.append('_%s in (%s)'%(k, s))
1831 args = args + v
1832 else:
1833 where.append('_%s=%s'%(k, a))
1834 args.append(v)
1836 # add results of full text search
1837 if search_matches is not None:
1838 v = search_matches.keys()
1839 s = ','.join([a for x in v])
1840 where.append('id in (%s)'%s)
1841 args = args + v
1843 # "grouping" is just the first-order sorting in the SQL fetch
1844 # can modify it...)
1845 orderby = []
1846 ordercols = []
1847 if group[0] is not None and group[1] is not None:
1848 if group[0] != '-':
1849 orderby.append('_'+group[1])
1850 ordercols.append('_'+group[1])
1851 else:
1852 orderby.append('_'+group[1]+' desc')
1853 ordercols.append('_'+group[1])
1855 # now add in the sorting
1856 group = ''
1857 if sort[0] is not None and sort[1] is not None:
1858 direction, colname = sort
1859 if direction != '-':
1860 if colname == 'id':
1861 orderby.append(colname)
1862 else:
1863 orderby.append('_'+colname)
1864 ordercols.append('_'+colname)
1865 else:
1866 if colname == 'id':
1867 orderby.append(colname+' desc')
1868 ordercols.append(colname)
1869 else:
1870 orderby.append('_'+colname+' desc')
1871 ordercols.append('_'+colname)
1873 # construct the SQL
1874 frum = ','.join(frum)
1875 if where:
1876 where = ' where ' + (' and '.join(where))
1877 else:
1878 where = ''
1879 cols = ['id']
1880 if orderby:
1881 cols = cols + ordercols
1882 order = ' order by %s'%(','.join(orderby))
1883 else:
1884 order = ''
1885 cols = ','.join(cols)
1886 sql = 'select %s from %s %s%s%s'%(cols, frum, where, group, order)
1887 args = tuple(args)
1888 if __debug__:
1889 print >>hyperdb.DEBUG, 'filter', (self, sql, args)
1890 self.db.cursor.execute(sql, args)
1891 l = self.db.cursor.fetchall()
1893 # return the IDs (the first column)
1894 # XXX The filter(None, l) bit is sqlite-specific... if there's _NO_
1895 # XXX matches to a fetch, it returns NULL instead of nothing!?!
1896 return filter(None, [row[0] for row in l])
1898 def count(self):
1899 '''Get the number of nodes in this class.
1901 If the returned integer is 'numnodes', the ids of all the nodes
1902 in this class run from 1 to numnodes, and numnodes+1 will be the
1903 id of the next node to be created in this class.
1904 '''
1905 return self.db.countnodes(self.classname)
1907 # Manipulating properties:
1908 def getprops(self, protected=1):
1909 '''Return a dictionary mapping property names to property objects.
1910 If the "protected" flag is true, we include protected properties -
1911 those which may not be modified.
1912 '''
1913 d = self.properties.copy()
1914 if protected:
1915 d['id'] = String()
1916 d['creation'] = hyperdb.Date()
1917 d['activity'] = hyperdb.Date()
1918 d['creator'] = hyperdb.Link('user')
1919 return d
1921 def addprop(self, **properties):
1922 '''Add properties to this class.
1924 The keyword arguments in 'properties' must map names to property
1925 objects, or a TypeError is raised. None of the keys in 'properties'
1926 may collide with the names of existing properties, or a ValueError
1927 is raised before any properties have been added.
1928 '''
1929 for key in properties.keys():
1930 if self.properties.has_key(key):
1931 raise ValueError, key
1932 self.properties.update(properties)
1934 def index(self, nodeid):
1935 '''Add (or refresh) the node to search indexes
1936 '''
1937 # find all the String properties that have indexme
1938 for prop, propclass in self.getprops().items():
1939 if isinstance(propclass, String) and propclass.indexme:
1940 try:
1941 value = str(self.get(nodeid, prop))
1942 except IndexError:
1943 # node no longer exists - entry should be removed
1944 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1945 else:
1946 # and index them under (classname, nodeid, property)
1947 self.db.indexer.add_text((self.classname, nodeid, prop),
1948 value)
1951 #
1952 # Detector interface
1953 #
1954 def audit(self, event, detector):
1955 '''Register a detector
1956 '''
1957 l = self.auditors[event]
1958 if detector not in l:
1959 self.auditors[event].append(detector)
1961 def fireAuditors(self, action, nodeid, newvalues):
1962 '''Fire all registered auditors.
1963 '''
1964 for audit in self.auditors[action]:
1965 audit(self.db, self, nodeid, newvalues)
1967 def react(self, event, detector):
1968 '''Register a detector
1969 '''
1970 l = self.reactors[event]
1971 if detector not in l:
1972 self.reactors[event].append(detector)
1974 def fireReactors(self, action, nodeid, oldvalues):
1975 '''Fire all registered reactors.
1976 '''
1977 for react in self.reactors[action]:
1978 react(self.db, self, nodeid, oldvalues)
1980 class FileClass(Class, hyperdb.FileClass):
1981 '''This class defines a large chunk of data. To support this, it has a
1982 mandatory String property "content" which is typically saved off
1983 externally to the hyperdb.
1985 The default MIME type of this data is defined by the
1986 "default_mime_type" class attribute, which may be overridden by each
1987 node if the class defines a "type" String property.
1988 '''
1989 default_mime_type = 'text/plain'
1991 def create(self, **propvalues):
1992 ''' snaffle the file propvalue and store in a file
1993 '''
1994 # we need to fire the auditors now, or the content property won't
1995 # be in propvalues for the auditors to play with
1996 self.fireAuditors('create', None, propvalues)
1998 # now remove the content property so it's not stored in the db
1999 content = propvalues['content']
2000 del propvalues['content']
2002 # do the database create
2003 newid = Class.create_inner(self, **propvalues)
2005 # fire reactors
2006 self.fireReactors('create', newid, None)
2008 # store off the content as a file
2009 self.db.storefile(self.classname, newid, None, content)
2010 return newid
2012 def import_list(self, propnames, proplist):
2013 ''' Trap the "content" property...
2014 '''
2015 # dupe this list so we don't affect others
2016 propnames = propnames[:]
2018 # extract the "content" property from the proplist
2019 i = propnames.index('content')
2020 content = eval(proplist[i])
2021 del propnames[i]
2022 del proplist[i]
2024 # do the normal import
2025 newid = Class.import_list(self, propnames, proplist)
2027 # save off the "content" file
2028 self.db.storefile(self.classname, newid, None, content)
2029 return newid
2031 _marker = []
2032 def get(self, nodeid, propname, default=_marker, cache=1):
2033 ''' trap the content propname and get it from the file
2034 '''
2035 poss_msg = 'Possibly a access right configuration problem.'
2036 if propname == 'content':
2037 try:
2038 return self.db.getfile(self.classname, nodeid, None)
2039 except IOError, (strerror):
2040 # BUG: by catching this we donot see an error in the log.
2041 return 'ERROR reading file: %s%s\n%s\n%s'%(
2042 self.classname, nodeid, poss_msg, strerror)
2043 if default is not self._marker:
2044 return Class.get(self, nodeid, propname, default, cache=cache)
2045 else:
2046 return Class.get(self, nodeid, propname, cache=cache)
2048 def getprops(self, protected=1):
2049 ''' In addition to the actual properties on the node, these methods
2050 provide the "content" property. If the "protected" flag is true,
2051 we include protected properties - those which may not be
2052 modified.
2053 '''
2054 d = Class.getprops(self, protected=protected).copy()
2055 d['content'] = hyperdb.String()
2056 return d
2058 def index(self, nodeid):
2059 ''' Index the node in the search index.
2061 We want to index the content in addition to the normal String
2062 property indexing.
2063 '''
2064 # perform normal indexing
2065 Class.index(self, nodeid)
2067 # get the content to index
2068 content = self.get(nodeid, 'content')
2070 # figure the mime type
2071 if self.properties.has_key('type'):
2072 mime_type = self.get(nodeid, 'type')
2073 else:
2074 mime_type = self.default_mime_type
2076 # and index!
2077 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
2078 mime_type)
2080 # XXX deviation from spec - was called ItemClass
2081 class IssueClass(Class, roundupdb.IssueClass):
2082 # Overridden methods:
2083 def __init__(self, db, classname, **properties):
2084 '''The newly-created class automatically includes the "messages",
2085 "files", "nosy", and "superseder" properties. If the 'properties'
2086 dictionary attempts to specify any of these properties or a
2087 "creation" or "activity" property, a ValueError is raised.
2088 '''
2089 if not properties.has_key('title'):
2090 properties['title'] = hyperdb.String(indexme='yes')
2091 if not properties.has_key('messages'):
2092 properties['messages'] = hyperdb.Multilink("msg")
2093 if not properties.has_key('files'):
2094 properties['files'] = hyperdb.Multilink("file")
2095 if not properties.has_key('nosy'):
2096 # note: journalling is turned off as it really just wastes
2097 # space. this behaviour may be overridden in an instance
2098 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2099 if not properties.has_key('superseder'):
2100 properties['superseder'] = hyperdb.Multilink(classname)
2101 Class.__init__(self, db, classname, **properties)