1 # $Id: rdbms_common.py,v 1.55 2003-04-22 20:53:54 kedder Exp $
2 ''' Relational database (SQL) backend common code.
4 Basics:
6 - map roundup classes to relational tables
7 - automatically detect schema changes and modify the table schemas
8 appropriately (we store the "database version" of the schema in the
9 database itself as the only row of the "schema" table)
10 - multilinks (which represent a many-to-many relationship) are handled through
11 intermediate tables
12 - journals are stored adjunct to the per-class tables
13 - table names and columns have "_" prepended so the names can't clash with
14 restricted names (like "order")
15 - retirement is determined by the __retired__ column being true
17 Database-specific changes may generally be pushed out to the overridable
18 sql_* methods, since everything else should be fairly generic. There's
19 probably a bit of work to be done if a database is used that actually
20 honors column typing, since the initial databases don't (sqlite stores
21 everything as a string.)
22 '''
24 # standard python modules
25 import sys, os, time, re, errno, weakref, copy
27 # roundup modules
28 from roundup import hyperdb, date, password, roundupdb, security
29 from roundup.hyperdb import String, Password, Date, Interval, Link, \
30 Multilink, DatabaseError, Boolean, Number, Node
31 from roundup.backends import locking
33 # support
34 from blobfiles import FileStorage
35 from roundup.indexer import Indexer
36 from sessions import Sessions, OneTimeKeys
37 from roundup.date import Range
39 # number of rows to keep in memory
40 ROW_CACHE_SIZE = 100
42 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
43 ''' Wrapper around an SQL database that presents a hyperdb interface.
45 - some functionality is specific to the actual SQL database, hence
46 the sql_* methods that are NotImplemented
47 - we keep a cache of the latest ROW_CACHE_SIZE row fetches.
48 '''
49 def __init__(self, config, journaltag=None):
50 ''' Open the database and load the schema from it.
51 '''
52 self.config, self.journaltag = config, journaltag
53 self.dir = config.DATABASE
54 self.classes = {}
55 self.indexer = Indexer(self.dir)
56 self.sessions = Sessions(self.config)
57 self.otks = OneTimeKeys(self.config)
58 self.security = security.Security(self)
60 # additional transaction support for external files and the like
61 self.transactions = []
63 # keep a cache of the N most recently retrieved rows of any kind
64 # (classname, nodeid) = row
65 self.cache = {}
66 self.cache_lru = []
68 # database lock
69 self.lockfile = None
71 # open a connection to the database, creating the "conn" attribute
72 self.open_connection()
74 def clearCache(self):
75 self.cache = {}
76 self.cache_lru = []
78 def open_connection(self):
79 ''' Open a connection to the database, creating it if necessary
80 '''
81 raise NotImplemented
83 def sql(self, sql, args=None):
84 ''' Execute the sql with the optional args.
85 '''
86 if __debug__:
87 print >>hyperdb.DEBUG, (self, sql, args)
88 if args:
89 self.cursor.execute(sql, args)
90 else:
91 self.cursor.execute(sql)
93 def sql_fetchone(self):
94 ''' Fetch a single row. If there's nothing to fetch, return None.
95 '''
96 raise NotImplemented
98 def sql_stringquote(self, value):
99 ''' Quote the string so it's safe to put in the 'sql quotes'
100 '''
101 return re.sub("'", "''", str(value))
103 def save_dbschema(self, schema):
104 ''' Save the schema definition that the database currently implements
105 '''
106 raise NotImplemented
108 def load_dbschema(self):
109 ''' Load the schema definition that the database currently implements
110 '''
111 raise NotImplemented
113 def post_init(self):
114 ''' Called once the schema initialisation has finished.
116 We should now confirm that the schema defined by our "classes"
117 attribute actually matches the schema in the database.
118 '''
119 # now detect changes in the schema
120 save = 0
121 for classname, spec in self.classes.items():
122 if self.database_schema.has_key(classname):
123 dbspec = self.database_schema[classname]
124 if self.update_class(spec, dbspec):
125 self.database_schema[classname] = spec.schema()
126 save = 1
127 else:
128 self.create_class(spec)
129 self.database_schema[classname] = spec.schema()
130 save = 1
132 for classname in self.database_schema.keys():
133 if not self.classes.has_key(classname):
134 self.drop_class(classname)
136 # update the database version of the schema
137 if save:
138 self.sql('delete from schema')
139 self.save_dbschema(self.database_schema)
141 # reindex the db if necessary
142 if self.indexer.should_reindex():
143 self.reindex()
145 # commit
146 self.conn.commit()
148 # figure the "curuserid"
149 if self.journaltag is None:
150 self.curuserid = None
151 elif self.journaltag == 'admin':
152 # admin user may not exist, but always has ID 1
153 self.curuserid = '1'
154 else:
155 self.curuserid = self.user.lookup(self.journaltag)
157 def reindex(self):
158 for klass in self.classes.values():
159 for nodeid in klass.list():
160 klass.index(nodeid)
161 self.indexer.save_index()
163 def determine_columns(self, properties):
164 ''' Figure the column names and multilink properties from the spec
166 "properties" is a list of (name, prop) where prop may be an
167 instance of a hyperdb "type" _or_ a string repr of that type.
168 '''
169 cols = ['_activity', '_creator', '_creation']
170 mls = []
171 # add the multilinks separately
172 for col, prop in properties:
173 if isinstance(prop, Multilink):
174 mls.append(col)
175 elif isinstance(prop, type('')) and prop.find('Multilink') != -1:
176 mls.append(col)
177 else:
178 cols.append('_'+col)
179 cols.sort()
180 return cols, mls
182 def update_class(self, spec, old_spec):
183 ''' Determine the differences between the current spec and the
184 database version of the spec, and update where necessary
185 '''
186 new_spec = spec
187 new_has = new_spec.properties.has_key
189 new_spec = new_spec.schema()
190 new_spec[1].sort()
191 old_spec[1].sort()
192 if new_spec == old_spec:
193 # no changes
194 return 0
196 if __debug__:
197 print >>hyperdb.DEBUG, 'update_class FIRING'
199 # key property changed?
200 if old_spec[0] != new_spec[0]:
201 if __debug__:
202 print >>hyperdb.DEBUG, 'update_class setting keyprop', `spec[0]`
203 # XXX turn on indexing for the key property
205 # detect multilinks that have been removed, and drop their table
206 old_has = {}
207 for name,prop in old_spec[1]:
208 old_has[name] = 1
209 if not new_has(name) and isinstance(prop, Multilink):
210 # it's a multilink, and it's been removed - drop the old
211 # table
212 sql = 'drop table %s_%s'%(spec.classname, prop)
213 if __debug__:
214 print >>hyperdb.DEBUG, 'update_class', (self, sql)
215 self.cursor.execute(sql)
216 continue
217 old_has = old_has.has_key
219 # now figure how we populate the new table
220 fetch = ['_activity', '_creation', '_creator']
221 properties = spec.getprops()
222 for propname,x in new_spec[1]:
223 prop = properties[propname]
224 if isinstance(prop, Multilink):
225 if not old_has(propname):
226 # we need to create the new table
227 self.create_multilink_table(spec, propname)
228 elif old_has(propname):
229 # we copy this col over from the old table
230 fetch.append('_'+propname)
232 # select the data out of the old table
233 fetch.append('id')
234 fetch.append('__retired__')
235 fetchcols = ','.join(fetch)
236 cn = spec.classname
237 sql = 'select %s from _%s'%(fetchcols, cn)
238 if __debug__:
239 print >>hyperdb.DEBUG, 'update_class', (self, sql)
240 self.cursor.execute(sql)
241 olddata = self.cursor.fetchall()
243 # drop the old table
244 self.cursor.execute('drop table _%s'%cn)
246 # create the new table
247 self.create_class_table(spec)
249 if olddata:
250 # do the insert
251 args = ','.join([self.arg for x in fetch])
252 sql = 'insert into _%s (%s) values (%s)'%(cn, fetchcols, args)
253 if __debug__:
254 print >>hyperdb.DEBUG, 'update_class', (self, sql, olddata[0])
255 for entry in olddata:
256 self.cursor.execute(sql, tuple(entry))
258 return 1
260 def create_class_table(self, spec):
261 ''' create the class table for the given spec
262 '''
263 cols, mls = self.determine_columns(spec.properties.items())
265 # add on our special columns
266 cols.append('id')
267 cols.append('__retired__')
269 # create the base table
270 scols = ','.join(['%s varchar'%x for x in cols])
271 sql = 'create table _%s (%s)'%(spec.classname, scols)
272 if __debug__:
273 print >>hyperdb.DEBUG, 'create_class', (self, sql)
274 self.cursor.execute(sql)
276 return cols, mls
278 def create_journal_table(self, spec):
279 ''' create the journal table for a class given the spec and
280 already-determined cols
281 '''
282 # journal table
283 cols = ','.join(['%s varchar'%x
284 for x in 'nodeid date tag action params'.split()])
285 sql = 'create table %s__journal (%s)'%(spec.classname, cols)
286 if __debug__:
287 print >>hyperdb.DEBUG, 'create_class', (self, sql)
288 self.cursor.execute(sql)
290 def create_multilink_table(self, spec, ml):
291 ''' Create a multilink table for the "ml" property of the class
292 given by the spec
293 '''
294 sql = 'create table %s_%s (linkid varchar, nodeid varchar)'%(
295 spec.classname, ml)
296 if __debug__:
297 print >>hyperdb.DEBUG, 'create_class', (self, sql)
298 self.cursor.execute(sql)
300 def create_class(self, spec):
301 ''' Create a database table according to the given spec.
302 '''
303 cols, mls = self.create_class_table(spec)
304 self.create_journal_table(spec)
306 # now create the multilink tables
307 for ml in mls:
308 self.create_multilink_table(spec, ml)
310 # ID counter
311 sql = 'insert into ids (name, num) values (%s,%s)'%(self.arg, self.arg)
312 vals = (spec.classname, 1)
313 if __debug__:
314 print >>hyperdb.DEBUG, 'create_class', (self, sql, vals)
315 self.cursor.execute(sql, vals)
317 def drop_class(self, spec):
318 ''' Drop the given table from the database.
320 Drop the journal and multilink tables too.
321 '''
322 # figure the multilinks
323 mls = []
324 for col, prop in spec.properties.items():
325 if isinstance(prop, Multilink):
326 mls.append(col)
328 sql = 'drop table _%s'%spec.classname
329 if __debug__:
330 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
331 self.cursor.execute(sql)
333 sql = 'drop table %s__journal'%spec.classname
334 if __debug__:
335 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
336 self.cursor.execute(sql)
338 for ml in mls:
339 sql = 'drop table %s_%s'%(spec.classname, ml)
340 if __debug__:
341 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
342 self.cursor.execute(sql)
344 #
345 # Classes
346 #
347 def __getattr__(self, classname):
348 ''' A convenient way of calling self.getclass(classname).
349 '''
350 if self.classes.has_key(classname):
351 if __debug__:
352 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
353 return self.classes[classname]
354 raise AttributeError, classname
356 def addclass(self, cl):
357 ''' Add a Class to the hyperdatabase.
358 '''
359 if __debug__:
360 print >>hyperdb.DEBUG, 'addclass', (self, cl)
361 cn = cl.classname
362 if self.classes.has_key(cn):
363 raise ValueError, cn
364 self.classes[cn] = cl
366 def getclasses(self):
367 ''' Return a list of the names of all existing classes.
368 '''
369 if __debug__:
370 print >>hyperdb.DEBUG, 'getclasses', (self,)
371 l = self.classes.keys()
372 l.sort()
373 return l
375 def getclass(self, classname):
376 '''Get the Class object representing a particular class.
378 If 'classname' is not a valid class name, a KeyError is raised.
379 '''
380 if __debug__:
381 print >>hyperdb.DEBUG, 'getclass', (self, classname)
382 try:
383 return self.classes[classname]
384 except KeyError:
385 raise KeyError, 'There is no class called "%s"'%classname
387 def clear(self):
388 ''' Delete all database contents.
390 Note: I don't commit here, which is different behaviour to the
391 "nuke from orbit" behaviour in the *dbms.
392 '''
393 if __debug__:
394 print >>hyperdb.DEBUG, 'clear', (self,)
395 for cn in self.classes.keys():
396 sql = 'delete from _%s'%cn
397 if __debug__:
398 print >>hyperdb.DEBUG, 'clear', (self, sql)
399 self.cursor.execute(sql)
401 #
402 # Node IDs
403 #
404 def newid(self, classname):
405 ''' Generate a new id for the given class
406 '''
407 # get the next ID
408 sql = 'select num from ids where name=%s'%self.arg
409 if __debug__:
410 print >>hyperdb.DEBUG, 'newid', (self, sql, classname)
411 self.cursor.execute(sql, (classname, ))
412 newid = self.cursor.fetchone()[0]
414 # update the counter
415 sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
416 vals = (int(newid)+1, classname)
417 if __debug__:
418 print >>hyperdb.DEBUG, 'newid', (self, sql, vals)
419 self.cursor.execute(sql, vals)
421 # return as string
422 return str(newid)
424 def setid(self, classname, setid):
425 ''' Set the id counter: used during import of database
426 '''
427 sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
428 vals = (setid, classname)
429 if __debug__:
430 print >>hyperdb.DEBUG, 'setid', (self, sql, vals)
431 self.cursor.execute(sql, vals)
433 #
434 # Nodes
435 #
436 def addnode(self, classname, nodeid, node):
437 ''' Add the specified node to its class's db.
438 '''
439 if __debug__:
440 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
442 # determine the column definitions and multilink tables
443 cl = self.classes[classname]
444 cols, mls = self.determine_columns(cl.properties.items())
446 # we'll be supplied these props if we're doing an import
447 if not node.has_key('creator'):
448 # add in the "calculated" properties (dupe so we don't affect
449 # calling code's node assumptions)
450 node = node.copy()
451 node['creation'] = node['activity'] = date.Date()
452 node['creator'] = self.curuserid
454 # default the non-multilink columns
455 for col, prop in cl.properties.items():
456 if not node.has_key(col):
457 if isinstance(prop, Multilink):
458 node[col] = []
459 else:
460 node[col] = None
462 # clear this node out of the cache if it's in there
463 key = (classname, nodeid)
464 if self.cache.has_key(key):
465 del self.cache[key]
466 self.cache_lru.remove(key)
468 # make the node data safe for the DB
469 node = self.serialise(classname, node)
471 # make sure the ordering is correct for column name -> column value
472 vals = tuple([node[col[1:]] for col in cols]) + (nodeid, 0)
473 s = ','.join([self.arg for x in cols]) + ',%s,%s'%(self.arg, self.arg)
474 cols = ','.join(cols) + ',id,__retired__'
476 # perform the inserts
477 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
478 if __debug__:
479 print >>hyperdb.DEBUG, 'addnode', (self, sql, vals)
480 self.cursor.execute(sql, vals)
482 # insert the multilink rows
483 for col in mls:
484 t = '%s_%s'%(classname, col)
485 for entry in node[col]:
486 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
487 self.arg, self.arg)
488 self.sql(sql, (entry, nodeid))
490 # make sure we do the commit-time extra stuff for this node
491 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
493 def setnode(self, classname, nodeid, values, multilink_changes):
494 ''' Change the specified node.
495 '''
496 if __debug__:
497 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, values)
499 # clear this node out of the cache if it's in there
500 key = (classname, nodeid)
501 if self.cache.has_key(key):
502 del self.cache[key]
503 self.cache_lru.remove(key)
505 # add the special props
506 values = values.copy()
507 values['activity'] = date.Date()
509 # make db-friendly
510 values = self.serialise(classname, values)
512 cl = self.classes[classname]
513 cols = []
514 mls = []
515 # add the multilinks separately
516 props = cl.getprops()
517 for col in values.keys():
518 prop = props[col]
519 if isinstance(prop, Multilink):
520 mls.append(col)
521 else:
522 cols.append('_'+col)
523 cols.sort()
525 # if there's any updates to regular columns, do them
526 if cols:
527 # make sure the ordering is correct for column name -> column value
528 sqlvals = tuple([values[col[1:]] for col in cols]) + (nodeid,)
529 s = ','.join(['%s=%s'%(x, self.arg) for x in cols])
530 cols = ','.join(cols)
532 # perform the update
533 sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
534 if __debug__:
535 print >>hyperdb.DEBUG, 'setnode', (self, sql, sqlvals)
536 self.cursor.execute(sql, sqlvals)
538 # now the fun bit, updating the multilinks ;)
539 for col, (add, remove) in multilink_changes.items():
540 tn = '%s_%s'%(classname, col)
541 if add:
542 sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
543 self.arg, self.arg)
544 for addid in add:
545 self.sql(sql, (nodeid, addid))
546 if remove:
547 sql = 'delete from %s where nodeid=%s and linkid=%s'%(tn,
548 self.arg, self.arg)
549 for removeid in remove:
550 self.sql(sql, (nodeid, removeid))
552 # make sure we do the commit-time extra stuff for this node
553 self.transactions.append((self.doSaveNode, (classname, nodeid, values)))
555 def getnode(self, classname, nodeid):
556 ''' Get a node from the database.
557 '''
558 if __debug__:
559 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid)
561 # see if we have this node cached
562 key = (classname, nodeid)
563 if self.cache.has_key(key):
564 # push us back to the top of the LRU
565 self.cache_lru.remove(key)
566 self.cache_lru.insert(0, key)
567 # return the cached information
568 return self.cache[key]
570 # figure the columns we're fetching
571 cl = self.classes[classname]
572 cols, mls = self.determine_columns(cl.properties.items())
573 scols = ','.join(cols)
575 # perform the basic property fetch
576 sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
577 self.sql(sql, (nodeid,))
579 values = self.sql_fetchone()
580 if values is None:
581 raise IndexError, 'no such %s node %s'%(classname, nodeid)
583 # make up the node
584 node = {}
585 for col in range(len(cols)):
586 node[cols[col][1:]] = values[col]
588 # now the multilinks
589 for col in mls:
590 # get the link ids
591 sql = 'select linkid from %s_%s where nodeid=%s'%(classname, col,
592 self.arg)
593 self.cursor.execute(sql, (nodeid,))
594 # extract the first column from the result
595 node[col] = [x[0] for x in self.cursor.fetchall()]
597 # un-dbificate the node data
598 node = self.unserialise(classname, node)
600 # save off in the cache
601 key = (classname, nodeid)
602 self.cache[key] = node
603 # update the LRU
604 self.cache_lru.insert(0, key)
605 if len(self.cache_lru) > ROW_CACHE_SIZE:
606 del self.cache[self.cache_lru.pop()]
608 return node
610 def destroynode(self, classname, nodeid):
611 '''Remove a node from the database. Called exclusively by the
612 destroy() method on Class.
613 '''
614 if __debug__:
615 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
617 # make sure the node exists
618 if not self.hasnode(classname, nodeid):
619 raise IndexError, '%s has no node %s'%(classname, nodeid)
621 # see if we have this node cached
622 if self.cache.has_key((classname, nodeid)):
623 del self.cache[(classname, nodeid)]
625 # see if there's any obvious commit actions that we should get rid of
626 for entry in self.transactions[:]:
627 if entry[1][:2] == (classname, nodeid):
628 self.transactions.remove(entry)
630 # now do the SQL
631 sql = 'delete from _%s where id=%s'%(classname, self.arg)
632 self.sql(sql, (nodeid,))
634 # remove from multilnks
635 cl = self.getclass(classname)
636 x, mls = self.determine_columns(cl.properties.items())
637 for col in mls:
638 # get the link ids
639 sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
640 self.cursor.execute(sql, (nodeid,))
642 # remove journal entries
643 sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
644 self.sql(sql, (nodeid,))
646 def serialise(self, classname, node):
647 '''Copy the node contents, converting non-marshallable data into
648 marshallable data.
649 '''
650 if __debug__:
651 print >>hyperdb.DEBUG, 'serialise', classname, node
652 properties = self.getclass(classname).getprops()
653 d = {}
654 for k, v in node.items():
655 # if the property doesn't exist, or is the "retired" flag then
656 # it won't be in the properties dict
657 if not properties.has_key(k):
658 d[k] = v
659 continue
661 # get the property spec
662 prop = properties[k]
664 if isinstance(prop, Password) and v is not None:
665 d[k] = str(v)
666 elif isinstance(prop, Date) and v is not None:
667 d[k] = v.serialise()
668 elif isinstance(prop, Interval) and v is not None:
669 d[k] = v.serialise()
670 else:
671 d[k] = v
672 return d
674 def unserialise(self, classname, node):
675 '''Decode the marshalled node data
676 '''
677 if __debug__:
678 print >>hyperdb.DEBUG, 'unserialise', classname, node
679 properties = self.getclass(classname).getprops()
680 d = {}
681 for k, v in node.items():
682 # if the property doesn't exist, or is the "retired" flag then
683 # it won't be in the properties dict
684 if not properties.has_key(k):
685 d[k] = v
686 continue
688 # get the property spec
689 prop = properties[k]
691 if isinstance(prop, Date) and v is not None:
692 d[k] = date.Date(v)
693 elif isinstance(prop, Interval) and v is not None:
694 d[k] = date.Interval(v)
695 elif isinstance(prop, Password) and v is not None:
696 p = password.Password()
697 p.unpack(v)
698 d[k] = p
699 elif (isinstance(prop, Boolean) or isinstance(prop, Number)) and v is not None:
700 d[k]=float(v)
701 else:
702 d[k] = v
703 return d
705 def hasnode(self, classname, nodeid):
706 ''' Determine if the database has a given node.
707 '''
708 sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
709 if __debug__:
710 print >>hyperdb.DEBUG, 'hasnode', (self, sql, nodeid)
711 self.cursor.execute(sql, (nodeid,))
712 return int(self.cursor.fetchone()[0])
714 def countnodes(self, classname):
715 ''' Count the number of nodes that exist for a particular Class.
716 '''
717 sql = 'select count(*) from _%s'%classname
718 if __debug__:
719 print >>hyperdb.DEBUG, 'countnodes', (self, sql)
720 self.cursor.execute(sql)
721 return self.cursor.fetchone()[0]
723 def addjournal(self, classname, nodeid, action, params, creator=None,
724 creation=None):
725 ''' Journal the Action
726 'action' may be:
728 'create' or 'set' -- 'params' is a dictionary of property values
729 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
730 'retire' -- 'params' is None
731 '''
732 # serialise the parameters now if necessary
733 if isinstance(params, type({})):
734 if action in ('set', 'create'):
735 params = self.serialise(classname, params)
737 # handle supply of the special journalling parameters (usually
738 # supplied on importing an existing database)
739 if creator:
740 journaltag = creator
741 else:
742 journaltag = self.curuserid
743 if creation:
744 journaldate = creation.serialise()
745 else:
746 journaldate = date.Date().serialise()
748 # create the journal entry
749 cols = ','.join('nodeid date tag action params'.split())
751 if __debug__:
752 print >>hyperdb.DEBUG, 'addjournal', (nodeid, journaldate,
753 journaltag, action, params)
755 self.save_journal(classname, cols, nodeid, journaldate,
756 journaltag, action, params)
758 def save_journal(self, classname, cols, nodeid, journaldate,
759 journaltag, action, params):
760 ''' Save the journal entry to the database
761 '''
762 raise NotImplemented
764 def getjournal(self, classname, nodeid):
765 ''' get the journal for id
766 '''
767 # make sure the node exists
768 if not self.hasnode(classname, nodeid):
769 raise IndexError, '%s has no node %s'%(classname, nodeid)
771 cols = ','.join('nodeid date tag action params'.split())
772 return self.load_journal(classname, cols, nodeid)
774 def load_journal(self, classname, cols, nodeid):
775 ''' Load the journal from the database
776 '''
777 raise NotImplemented
779 def pack(self, pack_before):
780 ''' Delete all journal entries except "create" before 'pack_before'.
781 '''
782 # get a 'yyyymmddhhmmss' version of the date
783 date_stamp = pack_before.serialise()
785 # do the delete
786 for classname in self.classes.keys():
787 sql = "delete from %s__journal where date<%s and "\
788 "action<>'create'"%(classname, self.arg)
789 if __debug__:
790 print >>hyperdb.DEBUG, 'pack', (self, sql, date_stamp)
791 self.cursor.execute(sql, (date_stamp,))
793 def sql_commit(self):
794 ''' Actually commit to the database.
795 '''
796 self.conn.commit()
798 def commit(self):
799 ''' Commit the current transactions.
801 Save all data changed since the database was opened or since the
802 last commit() or rollback().
803 '''
804 if __debug__:
805 print >>hyperdb.DEBUG, 'commit', (self,)
807 # commit the database
808 self.sql_commit()
810 # now, do all the other transaction stuff
811 reindex = {}
812 for method, args in self.transactions:
813 reindex[method(*args)] = 1
815 # reindex the nodes that request it
816 for classname, nodeid in filter(None, reindex.keys()):
817 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
818 self.getclass(classname).index(nodeid)
820 # save the indexer state
821 self.indexer.save_index()
823 # clear out the transactions
824 self.transactions = []
826 def rollback(self):
827 ''' Reverse all actions from the current transaction.
829 Undo all the changes made since the database was opened or the last
830 commit() or rollback() was performed.
831 '''
832 if __debug__:
833 print >>hyperdb.DEBUG, 'rollback', (self,)
835 # roll back
836 self.conn.rollback()
838 # roll back "other" transaction stuff
839 for method, args in self.transactions:
840 # delete temporary files
841 if method == self.doStoreFile:
842 self.rollbackStoreFile(*args)
843 self.transactions = []
845 # clear the cache
846 self.clearCache()
848 def doSaveNode(self, classname, nodeid, node):
849 ''' dummy that just generates a reindex event
850 '''
851 # return the classname, nodeid so we reindex this content
852 return (classname, nodeid)
854 def close(self):
855 ''' Close off the connection.
856 '''
857 self.conn.close()
858 if self.lockfile is not None:
859 locking.release_lock(self.lockfile)
860 if self.lockfile is not None:
861 self.lockfile.close()
862 self.lockfile = None
864 #
865 # The base Class class
866 #
867 class Class(hyperdb.Class):
868 ''' The handle to a particular class of nodes in a hyperdatabase.
870 All methods except __repr__ and getnode must be implemented by a
871 concrete backend Class.
872 '''
874 def __init__(self, db, classname, **properties):
875 '''Create a new class with a given name and property specification.
877 'classname' must not collide with the name of an existing class,
878 or a ValueError is raised. The keyword arguments in 'properties'
879 must map names to property objects, or a TypeError is raised.
880 '''
881 if (properties.has_key('creation') or properties.has_key('activity')
882 or properties.has_key('creator')):
883 raise ValueError, '"creation", "activity" and "creator" are '\
884 'reserved'
886 self.classname = classname
887 self.properties = properties
888 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
889 self.key = ''
891 # should we journal changes (default yes)
892 self.do_journal = 1
894 # do the db-related init stuff
895 db.addclass(self)
897 self.auditors = {'create': [], 'set': [], 'retire': [], 'restore': []}
898 self.reactors = {'create': [], 'set': [], 'retire': [], 'restore': []}
900 def schema(self):
901 ''' A dumpable version of the schema that we can store in the
902 database
903 '''
904 return (self.key, [(x, repr(y)) for x,y in self.properties.items()])
906 def enableJournalling(self):
907 '''Turn journalling on for this class
908 '''
909 self.do_journal = 1
911 def disableJournalling(self):
912 '''Turn journalling off for this class
913 '''
914 self.do_journal = 0
916 # Editing nodes:
917 def create(self, **propvalues):
918 ''' Create a new node of this class and return its id.
920 The keyword arguments in 'propvalues' map property names to values.
922 The values of arguments must be acceptable for the types of their
923 corresponding properties or a TypeError is raised.
925 If this class has a key property, it must be present and its value
926 must not collide with other key strings or a ValueError is raised.
928 Any other properties on this class that are missing from the
929 'propvalues' dictionary are set to None.
931 If an id in a link or multilink property does not refer to a valid
932 node, an IndexError is raised.
933 '''
934 self.fireAuditors('create', None, propvalues)
935 newid = self.create_inner(**propvalues)
936 self.fireReactors('create', newid, None)
937 return newid
939 def create_inner(self, **propvalues):
940 ''' Called by create, in-between the audit and react calls.
941 '''
942 if propvalues.has_key('id'):
943 raise KeyError, '"id" is reserved'
945 if self.db.journaltag is None:
946 raise DatabaseError, 'Database open read-only'
948 if propvalues.has_key('creation') or propvalues.has_key('activity'):
949 raise KeyError, '"creation" and "activity" are reserved'
951 # new node's id
952 newid = self.db.newid(self.classname)
954 # validate propvalues
955 num_re = re.compile('^\d+$')
956 for key, value in propvalues.items():
957 if key == self.key:
958 try:
959 self.lookup(value)
960 except KeyError:
961 pass
962 else:
963 raise ValueError, 'node with key "%s" exists'%value
965 # try to handle this property
966 try:
967 prop = self.properties[key]
968 except KeyError:
969 raise KeyError, '"%s" has no property "%s"'%(self.classname,
970 key)
972 if value is not None and isinstance(prop, Link):
973 if type(value) != type(''):
974 raise ValueError, 'link value must be String'
975 link_class = self.properties[key].classname
976 # if it isn't a number, it's a key
977 if not num_re.match(value):
978 try:
979 value = self.db.classes[link_class].lookup(value)
980 except (TypeError, KeyError):
981 raise IndexError, 'new property "%s": %s not a %s'%(
982 key, value, link_class)
983 elif not self.db.getclass(link_class).hasnode(value):
984 raise IndexError, '%s has no node %s'%(link_class, value)
986 # save off the value
987 propvalues[key] = value
989 # register the link with the newly linked node
990 if self.do_journal and self.properties[key].do_journal:
991 self.db.addjournal(link_class, value, 'link',
992 (self.classname, newid, key))
994 elif isinstance(prop, Multilink):
995 if type(value) != type([]):
996 raise TypeError, 'new property "%s" not a list of ids'%key
998 # clean up and validate the list of links
999 link_class = self.properties[key].classname
1000 l = []
1001 for entry in value:
1002 if type(entry) != type(''):
1003 raise ValueError, '"%s" multilink value (%r) '\
1004 'must contain Strings'%(key, value)
1005 # if it isn't a number, it's a key
1006 if not num_re.match(entry):
1007 try:
1008 entry = self.db.classes[link_class].lookup(entry)
1009 except (TypeError, KeyError):
1010 raise IndexError, 'new property "%s": %s not a %s'%(
1011 key, entry, self.properties[key].classname)
1012 l.append(entry)
1013 value = l
1014 propvalues[key] = value
1016 # handle additions
1017 for nodeid in value:
1018 if not self.db.getclass(link_class).hasnode(nodeid):
1019 raise IndexError, '%s has no node %s'%(link_class,
1020 nodeid)
1021 # register the link with the newly linked node
1022 if self.do_journal and self.properties[key].do_journal:
1023 self.db.addjournal(link_class, nodeid, 'link',
1024 (self.classname, newid, key))
1026 elif isinstance(prop, String):
1027 if type(value) != type('') and type(value) != type(u''):
1028 raise TypeError, 'new property "%s" not a string'%key
1030 elif isinstance(prop, Password):
1031 if not isinstance(value, password.Password):
1032 raise TypeError, 'new property "%s" not a Password'%key
1034 elif isinstance(prop, Date):
1035 if value is not None and not isinstance(value, date.Date):
1036 raise TypeError, 'new property "%s" not a Date'%key
1038 elif isinstance(prop, Interval):
1039 if value is not None and not isinstance(value, date.Interval):
1040 raise TypeError, 'new property "%s" not an Interval'%key
1042 elif value is not None and isinstance(prop, Number):
1043 try:
1044 float(value)
1045 except ValueError:
1046 raise TypeError, 'new property "%s" not numeric'%key
1048 elif value is not None and isinstance(prop, Boolean):
1049 try:
1050 int(value)
1051 except ValueError:
1052 raise TypeError, 'new property "%s" not boolean'%key
1054 # make sure there's data where there needs to be
1055 for key, prop in self.properties.items():
1056 if propvalues.has_key(key):
1057 continue
1058 if key == self.key:
1059 raise ValueError, 'key property "%s" is required'%key
1060 if isinstance(prop, Multilink):
1061 propvalues[key] = []
1062 else:
1063 propvalues[key] = None
1065 # done
1066 self.db.addnode(self.classname, newid, propvalues)
1067 if self.do_journal:
1068 self.db.addjournal(self.classname, newid, 'create', {})
1070 return newid
1072 def export_list(self, propnames, nodeid):
1073 ''' Export a node - generate a list of CSV-able data in the order
1074 specified by propnames for the given node.
1075 '''
1076 properties = self.getprops()
1077 l = []
1078 for prop in propnames:
1079 proptype = properties[prop]
1080 value = self.get(nodeid, prop)
1081 # "marshal" data where needed
1082 if value is None:
1083 pass
1084 elif isinstance(proptype, hyperdb.Date):
1085 value = value.get_tuple()
1086 elif isinstance(proptype, hyperdb.Interval):
1087 value = value.get_tuple()
1088 elif isinstance(proptype, hyperdb.Password):
1089 value = str(value)
1090 l.append(repr(value))
1091 l.append(self.is_retired(nodeid))
1092 return l
1094 def import_list(self, propnames, proplist):
1095 ''' Import a node - all information including "id" is present and
1096 should not be sanity checked. Triggers are not triggered. The
1097 journal should be initialised using the "creator" and "created"
1098 information.
1100 Return the nodeid of the node imported.
1101 '''
1102 if self.db.journaltag is None:
1103 raise DatabaseError, 'Database open read-only'
1104 properties = self.getprops()
1106 # make the new node's property map
1107 d = {}
1108 retire = 0
1109 newid = None
1110 for i in range(len(propnames)):
1111 # Use eval to reverse the repr() used to output the CSV
1112 value = eval(proplist[i])
1114 # Figure the property for this column
1115 propname = propnames[i]
1117 # "unmarshal" where necessary
1118 if propname == 'id':
1119 newid = value
1120 continue
1121 elif propname == 'is retired':
1122 # is the item retired?
1123 if int(value):
1124 retire = 1
1125 continue
1127 prop = properties[propname]
1128 if value is None:
1129 # don't set Nones
1130 continue
1131 elif isinstance(prop, hyperdb.Date):
1132 value = date.Date(value)
1133 elif isinstance(prop, hyperdb.Interval):
1134 value = date.Interval(value)
1135 elif isinstance(prop, hyperdb.Password):
1136 pwd = password.Password()
1137 pwd.unpack(value)
1138 value = pwd
1139 d[propname] = value
1141 # get a new id if necessary
1142 if newid is None:
1143 newid = self.db.newid(self.classname)
1145 # retire?
1146 if retire:
1147 # use the arg for __retired__ to cope with any odd database type
1148 # conversion (hello, sqlite)
1149 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1150 self.db.arg, self.db.arg)
1151 if __debug__:
1152 print >>hyperdb.DEBUG, 'retire', (self, sql, newid)
1153 self.db.cursor.execute(sql, (1, newid))
1155 # add the node and journal
1156 self.db.addnode(self.classname, newid, d)
1158 # extract the extraneous journalling gumpf and nuke it
1159 if d.has_key('creator'):
1160 creator = d['creator']
1161 del d['creator']
1162 else:
1163 creator = None
1164 if d.has_key('creation'):
1165 creation = d['creation']
1166 del d['creation']
1167 else:
1168 creation = None
1169 if d.has_key('activity'):
1170 del d['activity']
1171 self.db.addjournal(self.classname, newid, 'create', {}, creator,
1172 creation)
1173 return newid
1175 _marker = []
1176 def get(self, nodeid, propname, default=_marker, cache=1):
1177 '''Get the value of a property on an existing node of this class.
1179 'nodeid' must be the id of an existing node of this class or an
1180 IndexError is raised. 'propname' must be the name of a property
1181 of this class or a KeyError is raised.
1183 'cache' indicates whether the transaction cache should be queried
1184 for the node. If the node has been modified and you need to
1185 determine what its values prior to modification are, you need to
1186 set cache=0.
1187 '''
1188 if propname == 'id':
1189 return nodeid
1191 # get the node's dict
1192 d = self.db.getnode(self.classname, nodeid)
1194 if propname == 'creation':
1195 if d.has_key('creation'):
1196 return d['creation']
1197 else:
1198 return date.Date()
1199 if propname == 'activity':
1200 if d.has_key('activity'):
1201 return d['activity']
1202 else:
1203 return date.Date()
1204 if propname == 'creator':
1205 if d.has_key('creator'):
1206 return d['creator']
1207 else:
1208 return self.db.curuserid
1210 # get the property (raises KeyErorr if invalid)
1211 prop = self.properties[propname]
1213 if not d.has_key(propname):
1214 if default is self._marker:
1215 if isinstance(prop, Multilink):
1216 return []
1217 else:
1218 return None
1219 else:
1220 return default
1222 # don't pass our list to other code
1223 if isinstance(prop, Multilink):
1224 return d[propname][:]
1226 return d[propname]
1228 def getnode(self, nodeid, cache=1):
1229 ''' Return a convenience wrapper for the node.
1231 'nodeid' must be the id of an existing node of this class or an
1232 IndexError is raised.
1234 'cache' indicates whether the transaction cache should be queried
1235 for the node. If the node has been modified and you need to
1236 determine what its values prior to modification are, you need to
1237 set cache=0.
1238 '''
1239 return Node(self, nodeid, cache=cache)
1241 def set(self, nodeid, **propvalues):
1242 '''Modify a property on an existing node of this class.
1244 'nodeid' must be the id of an existing node of this class or an
1245 IndexError is raised.
1247 Each key in 'propvalues' must be the name of a property of this
1248 class or a KeyError is raised.
1250 All values in 'propvalues' must be acceptable types for their
1251 corresponding properties or a TypeError is raised.
1253 If the value of the key property is set, it must not collide with
1254 other key strings or a ValueError is raised.
1256 If the value of a Link or Multilink property contains an invalid
1257 node id, a ValueError is raised.
1258 '''
1259 if not propvalues:
1260 return propvalues
1262 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1263 raise KeyError, '"creation" and "activity" are reserved'
1265 if propvalues.has_key('id'):
1266 raise KeyError, '"id" is reserved'
1268 if self.db.journaltag is None:
1269 raise DatabaseError, 'Database open read-only'
1271 self.fireAuditors('set', nodeid, propvalues)
1272 # Take a copy of the node dict so that the subsequent set
1273 # operation doesn't modify the oldvalues structure.
1274 # XXX used to try the cache here first
1275 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1277 node = self.db.getnode(self.classname, nodeid)
1278 if self.is_retired(nodeid):
1279 raise IndexError, 'Requested item is retired'
1280 num_re = re.compile('^\d+$')
1282 # if the journal value is to be different, store it in here
1283 journalvalues = {}
1285 # remember the add/remove stuff for multilinks, making it easier
1286 # for the Database layer to do its stuff
1287 multilink_changes = {}
1289 for propname, value in propvalues.items():
1290 # check to make sure we're not duplicating an existing key
1291 if propname == self.key and node[propname] != value:
1292 try:
1293 self.lookup(value)
1294 except KeyError:
1295 pass
1296 else:
1297 raise ValueError, 'node with key "%s" exists'%value
1299 # this will raise the KeyError if the property isn't valid
1300 # ... we don't use getprops() here because we only care about
1301 # the writeable properties.
1302 try:
1303 prop = self.properties[propname]
1304 except KeyError:
1305 raise KeyError, '"%s" has no property named "%s"'%(
1306 self.classname, propname)
1308 # if the value's the same as the existing value, no sense in
1309 # doing anything
1310 current = node.get(propname, None)
1311 if value == current:
1312 del propvalues[propname]
1313 continue
1314 journalvalues[propname] = current
1316 # do stuff based on the prop type
1317 if isinstance(prop, Link):
1318 link_class = prop.classname
1319 # if it isn't a number, it's a key
1320 if value is not None and not isinstance(value, type('')):
1321 raise ValueError, 'property "%s" link value be a string'%(
1322 propname)
1323 if isinstance(value, type('')) and not num_re.match(value):
1324 try:
1325 value = self.db.classes[link_class].lookup(value)
1326 except (TypeError, KeyError):
1327 raise IndexError, 'new property "%s": %s not a %s'%(
1328 propname, value, prop.classname)
1330 if (value is not None and
1331 not self.db.getclass(link_class).hasnode(value)):
1332 raise IndexError, '%s has no node %s'%(link_class, value)
1334 if self.do_journal and prop.do_journal:
1335 # register the unlink with the old linked node
1336 if node[propname] is not None:
1337 self.db.addjournal(link_class, node[propname], 'unlink',
1338 (self.classname, nodeid, propname))
1340 # register the link with the newly linked node
1341 if value is not None:
1342 self.db.addjournal(link_class, value, 'link',
1343 (self.classname, nodeid, propname))
1345 elif isinstance(prop, Multilink):
1346 if type(value) != type([]):
1347 raise TypeError, 'new property "%s" not a list of'\
1348 ' ids'%propname
1349 link_class = self.properties[propname].classname
1350 l = []
1351 for entry in value:
1352 # if it isn't a number, it's a key
1353 if type(entry) != type(''):
1354 raise ValueError, 'new property "%s" link value ' \
1355 'must be a string'%propname
1356 if not num_re.match(entry):
1357 try:
1358 entry = self.db.classes[link_class].lookup(entry)
1359 except (TypeError, KeyError):
1360 raise IndexError, 'new property "%s": %s not a %s'%(
1361 propname, entry,
1362 self.properties[propname].classname)
1363 l.append(entry)
1364 value = l
1365 propvalues[propname] = value
1367 # figure the journal entry for this property
1368 add = []
1369 remove = []
1371 # handle removals
1372 if node.has_key(propname):
1373 l = node[propname]
1374 else:
1375 l = []
1376 for id in l[:]:
1377 if id in value:
1378 continue
1379 # register the unlink with the old linked node
1380 if self.do_journal and self.properties[propname].do_journal:
1381 self.db.addjournal(link_class, id, 'unlink',
1382 (self.classname, nodeid, propname))
1383 l.remove(id)
1384 remove.append(id)
1386 # handle additions
1387 for id in value:
1388 if not self.db.getclass(link_class).hasnode(id):
1389 raise IndexError, '%s has no node %s'%(link_class, id)
1390 if id in l:
1391 continue
1392 # register the link with the newly linked node
1393 if self.do_journal and self.properties[propname].do_journal:
1394 self.db.addjournal(link_class, id, 'link',
1395 (self.classname, nodeid, propname))
1396 l.append(id)
1397 add.append(id)
1399 # figure the journal entry
1400 l = []
1401 if add:
1402 l.append(('+', add))
1403 if remove:
1404 l.append(('-', remove))
1405 multilink_changes[propname] = (add, remove)
1406 if l:
1407 journalvalues[propname] = tuple(l)
1409 elif isinstance(prop, String):
1410 if value is not None and type(value) != type('') and type(value) != type(u''):
1411 raise TypeError, 'new property "%s" not a string'%propname
1413 elif isinstance(prop, Password):
1414 if not isinstance(value, password.Password):
1415 raise TypeError, 'new property "%s" not a Password'%propname
1416 propvalues[propname] = value
1418 elif value is not None and isinstance(prop, Date):
1419 if not isinstance(value, date.Date):
1420 raise TypeError, 'new property "%s" not a Date'% propname
1421 propvalues[propname] = value
1423 elif value is not None and isinstance(prop, Interval):
1424 if not isinstance(value, date.Interval):
1425 raise TypeError, 'new property "%s" not an '\
1426 'Interval'%propname
1427 propvalues[propname] = value
1429 elif value is not None and isinstance(prop, Number):
1430 try:
1431 float(value)
1432 except ValueError:
1433 raise TypeError, 'new property "%s" not numeric'%propname
1435 elif value is not None and isinstance(prop, Boolean):
1436 try:
1437 int(value)
1438 except ValueError:
1439 raise TypeError, 'new property "%s" not boolean'%propname
1441 # nothing to do?
1442 if not propvalues:
1443 return propvalues
1445 # do the set, and journal it
1446 self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
1448 if self.do_journal:
1449 self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
1451 self.fireReactors('set', nodeid, oldvalues)
1453 return propvalues
1455 def retire(self, nodeid):
1456 '''Retire a node.
1458 The properties on the node remain available from the get() method,
1459 and the node's id is never reused.
1461 Retired nodes are not returned by the find(), list(), or lookup()
1462 methods, and other nodes may reuse the values of their key properties.
1463 '''
1464 if self.db.journaltag is None:
1465 raise DatabaseError, 'Database open read-only'
1467 self.fireAuditors('retire', nodeid, None)
1469 # use the arg for __retired__ to cope with any odd database type
1470 # conversion (hello, sqlite)
1471 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1472 self.db.arg, self.db.arg)
1473 if __debug__:
1474 print >>hyperdb.DEBUG, 'retire', (self, sql, nodeid)
1475 self.db.cursor.execute(sql, (1, nodeid))
1476 if self.do_journal:
1477 self.db.addjournal(self.classname, nodeid, 'retired', None)
1479 self.fireReactors('retire', nodeid, None)
1481 def restore(self, nodeid):
1482 '''Restpre a retired node.
1484 Make node available for all operations like it was before retirement.
1485 '''
1486 if self.db.journaltag is None:
1487 raise DatabaseError, 'Database open read-only'
1489 node = self.db.getnode(self.classname, nodeid)
1490 # check if key property was overrided
1491 key = self.getkey()
1492 try:
1493 id = self.lookup(node[key])
1494 except KeyError:
1495 pass
1496 else:
1497 raise KeyError, "Key property (%s) of retired node clashes with \
1498 existing one (%s)" % (key, node[key])
1500 self.fireAuditors('restore', nodeid, None)
1501 # use the arg for __retired__ to cope with any odd database type
1502 # conversion (hello, sqlite)
1503 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1504 self.db.arg, self.db.arg)
1505 if __debug__:
1506 print >>hyperdb.DEBUG, 'restore', (self, sql, nodeid)
1507 self.db.cursor.execute(sql, (0, nodeid))
1508 if self.do_journal:
1509 self.db.addjournal(self.classname, nodeid, 'restored', None)
1511 self.fireReactors('restore', nodeid, None)
1513 def is_retired(self, nodeid):
1514 '''Return true if the node is rerired
1515 '''
1516 sql = 'select __retired__ from _%s where id=%s'%(self.classname,
1517 self.db.arg)
1518 if __debug__:
1519 print >>hyperdb.DEBUG, 'is_retired', (self, sql, nodeid)
1520 self.db.cursor.execute(sql, (nodeid,))
1521 return int(self.db.sql_fetchone()[0])
1523 def destroy(self, nodeid):
1524 '''Destroy a node.
1526 WARNING: this method should never be used except in extremely rare
1527 situations where there could never be links to the node being
1528 deleted
1529 WARNING: use retire() instead
1530 WARNING: the properties of this node will not be available ever again
1531 WARNING: really, use retire() instead
1533 Well, I think that's enough warnings. This method exists mostly to
1534 support the session storage of the cgi interface.
1536 The node is completely removed from the hyperdb, including all journal
1537 entries. It will no longer be available, and will generally break code
1538 if there are any references to the node.
1539 '''
1540 if self.db.journaltag is None:
1541 raise DatabaseError, 'Database open read-only'
1542 self.db.destroynode(self.classname, nodeid)
1544 def history(self, nodeid):
1545 '''Retrieve the journal of edits on a particular node.
1547 'nodeid' must be the id of an existing node of this class or an
1548 IndexError is raised.
1550 The returned list contains tuples of the form
1552 (nodeid, date, tag, action, params)
1554 'date' is a Timestamp object specifying the time of the change and
1555 'tag' is the journaltag specified when the database was opened.
1556 '''
1557 if not self.do_journal:
1558 raise ValueError, 'Journalling is disabled for this class'
1559 return self.db.getjournal(self.classname, nodeid)
1561 # Locating nodes:
1562 def hasnode(self, nodeid):
1563 '''Determine if the given nodeid actually exists
1564 '''
1565 return self.db.hasnode(self.classname, nodeid)
1567 def setkey(self, propname):
1568 '''Select a String property of this class to be the key property.
1570 'propname' must be the name of a String property of this class or
1571 None, or a TypeError is raised. The values of the key property on
1572 all existing nodes must be unique or a ValueError is raised.
1573 '''
1574 # XXX create an index on the key prop column
1575 prop = self.getprops()[propname]
1576 if not isinstance(prop, String):
1577 raise TypeError, 'key properties must be String'
1578 self.key = propname
1580 def getkey(self):
1581 '''Return the name of the key property for this class or None.'''
1582 return self.key
1584 def labelprop(self, default_to_id=0):
1585 ''' Return the property name for a label for the given node.
1587 This method attempts to generate a consistent label for the node.
1588 It tries the following in order:
1589 1. key property
1590 2. "name" property
1591 3. "title" property
1592 4. first property from the sorted property name list
1593 '''
1594 k = self.getkey()
1595 if k:
1596 return k
1597 props = self.getprops()
1598 if props.has_key('name'):
1599 return 'name'
1600 elif props.has_key('title'):
1601 return 'title'
1602 if default_to_id:
1603 return 'id'
1604 props = props.keys()
1605 props.sort()
1606 return props[0]
1608 def lookup(self, keyvalue):
1609 '''Locate a particular node by its key property and return its id.
1611 If this class has no key property, a TypeError is raised. If the
1612 'keyvalue' matches one of the values for the key property among
1613 the nodes in this class, the matching node's id is returned;
1614 otherwise a KeyError is raised.
1615 '''
1616 if not self.key:
1617 raise TypeError, 'No key property set for class %s'%self.classname
1619 # use the arg to handle any odd database type conversion (hello,
1620 # sqlite)
1621 sql = "select id from _%s where _%s=%s and __retired__ <> %s"%(
1622 self.classname, self.key, self.db.arg, self.db.arg)
1623 self.db.sql(sql, (keyvalue, 1))
1625 # see if there was a result that's not retired
1626 row = self.db.sql_fetchone()
1627 if not row:
1628 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1629 keyvalue, self.classname)
1631 # return the id
1632 return row[0]
1634 def find(self, **propspec):
1635 '''Get the ids of nodes in this class which link to the given nodes.
1637 'propspec' consists of keyword args propname=nodeid or
1638 propname={nodeid:1, }
1639 'propname' must be the name of a property in this class, or a
1640 KeyError is raised. That property must be a Link or Multilink
1641 property, or a TypeError is raised.
1643 Any node in this class whose 'propname' property links to any of the
1644 nodeids will be returned. Used by the full text indexing, which knows
1645 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1646 issues:
1648 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1649 '''
1650 if __debug__:
1651 print >>hyperdb.DEBUG, 'find', (self, propspec)
1653 # shortcut
1654 if not propspec:
1655 return []
1657 # validate the args
1658 props = self.getprops()
1659 propspec = propspec.items()
1660 for propname, nodeids in propspec:
1661 # check the prop is OK
1662 prop = props[propname]
1663 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1664 raise TypeError, "'%s' not a Link/Multilink property"%propname
1666 # first, links
1667 where = []
1668 allvalues = ()
1669 a = self.db.arg
1670 for prop, values in propspec:
1671 if not isinstance(props[prop], hyperdb.Link):
1672 continue
1673 if type(values) is type(''):
1674 allvalues += (values,)
1675 where.append('_%s = %s'%(prop, a))
1676 elif values is None:
1677 where.append('_%s is NULL'%prop)
1678 else:
1679 allvalues += tuple(values.keys())
1680 where.append('_%s in (%s)'%(prop, ','.join([a]*len(values))))
1681 tables = []
1682 if where:
1683 tables.append('select id as nodeid from _%s where %s'%(
1684 self.classname, ' and '.join(where)))
1686 # now multilinks
1687 for prop, values in propspec:
1688 if not isinstance(props[prop], hyperdb.Multilink):
1689 continue
1690 if type(values) is type(''):
1691 allvalues += (values,)
1692 s = a
1693 else:
1694 allvalues += tuple(values.keys())
1695 s = ','.join([a]*len(values))
1696 tables.append('select nodeid from %s_%s where linkid in (%s)'%(
1697 self.classname, prop, s))
1698 sql = '\nunion\n'.join(tables)
1699 self.db.sql(sql, allvalues)
1700 l = [x[0] for x in self.db.sql_fetchall()]
1701 if __debug__:
1702 print >>hyperdb.DEBUG, 'find ... ', l
1703 return l
1705 def stringFind(self, **requirements):
1706 '''Locate a particular node by matching a set of its String
1707 properties in a caseless search.
1709 If the property is not a String property, a TypeError is raised.
1711 The return is a list of the id of all nodes that match.
1712 '''
1713 where = []
1714 args = []
1715 for propname in requirements.keys():
1716 prop = self.properties[propname]
1717 if isinstance(not prop, String):
1718 raise TypeError, "'%s' not a String property"%propname
1719 where.append(propname)
1720 args.append(requirements[propname].lower())
1722 # generate the where clause
1723 s = ' and '.join(['lower(_%s)=%s'%(col, self.db.arg) for col in where])
1724 sql = 'select id from _%s where %s'%(self.classname, s)
1725 self.db.sql(sql, tuple(args))
1726 l = [x[0] for x in self.db.sql_fetchall()]
1727 if __debug__:
1728 print >>hyperdb.DEBUG, 'find ... ', l
1729 return l
1731 def list(self):
1732 ''' Return a list of the ids of the active nodes in this class.
1733 '''
1734 return self.getnodeids(retired=0)
1736 def getnodeids(self, retired=None):
1737 ''' Retrieve all the ids of the nodes for a particular Class.
1739 Set retired=None to get all nodes. Otherwise it'll get all the
1740 retired or non-retired nodes, depending on the flag.
1741 '''
1742 # flip the sense of the flag if we don't want all of them
1743 if retired is not None:
1744 retired = not retired
1745 args = (retired, )
1746 sql = 'select id from _%s where __retired__ <> %s'%(self.classname,
1747 self.db.arg)
1748 else:
1749 args = ()
1750 sql = 'select id from _%s'%self.classname
1751 if __debug__:
1752 print >>hyperdb.DEBUG, 'getnodeids', (self, sql, retired)
1753 self.db.cursor.execute(sql, args)
1754 ids = [x[0] for x in self.db.cursor.fetchall()]
1755 return ids
1757 def filter(self, search_matches, filterspec, sort=(None,None),
1758 group=(None,None)):
1759 ''' Return a list of the ids of the active nodes in this class that
1760 match the 'filter' spec, sorted by the group spec and then the
1761 sort spec
1763 "filterspec" is {propname: value(s)}
1764 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1765 and prop is a prop name or None
1766 "search_matches" is {nodeid: marker}
1768 The filter must match all properties specificed - but if the
1769 property value to match is a list, any one of the values in the
1770 list may match for that property to match.
1771 '''
1772 # just don't bother if the full-text search matched diddly
1773 if search_matches == {}:
1774 return []
1776 cn = self.classname
1778 timezone = self.db.getUserTimezone()
1780 # figure the WHERE clause from the filterspec
1781 props = self.getprops()
1782 frum = ['_'+cn]
1783 where = []
1784 args = []
1785 a = self.db.arg
1786 for k, v in filterspec.items():
1787 propclass = props[k]
1788 # now do other where clause stuff
1789 if isinstance(propclass, Multilink):
1790 tn = '%s_%s'%(cn, k)
1791 if v in ('-1', ['-1']):
1792 # only match rows that have count(linkid)=0 in the
1793 # corresponding multilink table)
1794 where.append('id not in (select nodeid from %s)'%tn)
1795 elif isinstance(v, type([])):
1796 frum.append(tn)
1797 s = ','.join([a for x in v])
1798 where.append('id=%s.nodeid and %s.linkid in (%s)'%(tn,tn,s))
1799 args = args + v
1800 else:
1801 frum.append(tn)
1802 where.append('id=%s.nodeid and %s.linkid=%s'%(tn, tn, a))
1803 args.append(v)
1804 elif k == 'id':
1805 if isinstance(v, type([])):
1806 s = ','.join([a for x in v])
1807 where.append('%s in (%s)'%(k, s))
1808 args = args + v
1809 else:
1810 where.append('%s=%s'%(k, a))
1811 args.append(v)
1812 elif isinstance(propclass, String):
1813 if not isinstance(v, type([])):
1814 v = [v]
1816 # Quote the bits in the string that need it and then embed
1817 # in a "substring" search. Note - need to quote the '%' so
1818 # they make it through the python layer happily
1819 v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
1821 # now add to the where clause
1822 where.append(' or '.join(["_%s LIKE '%s'"%(k, s) for s in v]))
1823 # note: args are embedded in the query string now
1824 elif isinstance(propclass, Link):
1825 if isinstance(v, type([])):
1826 if '-1' in v:
1827 v = v[:]
1828 v.remove('-1')
1829 xtra = ' or _%s is NULL'%k
1830 else:
1831 xtra = ''
1832 if v:
1833 s = ','.join([a for x in v])
1834 where.append('(_%s in (%s)%s)'%(k, s, xtra))
1835 args = args + v
1836 else:
1837 where.append('_%s is NULL'%k)
1838 else:
1839 if v == '-1':
1840 v = None
1841 where.append('_%s is NULL'%k)
1842 else:
1843 where.append('_%s=%s'%(k, a))
1844 args.append(v)
1845 elif isinstance(propclass, Date):
1846 if isinstance(v, type([])):
1847 s = ','.join([a for x in v])
1848 where.append('_%s in (%s)'%(k, s))
1849 args = args + [date.Date(x).serialise() for x in v]
1850 else:
1851 try:
1852 # Try to filter on range of dates
1853 date_rng = Range(v, date.Date, offset=timezone)
1854 if (date_rng.from_value):
1855 where.append('_%s >= %s'%(k, a))
1856 args.append(date_rng.from_value.serialise())
1857 if (date_rng.to_value):
1858 where.append('_%s <= %s'%(k, a))
1859 args.append(date_rng.to_value.serialise())
1860 except ValueError:
1861 # If range creation fails - ignore that search parameter
1862 pass
1863 elif isinstance(propclass, Interval):
1864 if isinstance(v, type([])):
1865 s = ','.join([a for x in v])
1866 where.append('_%s in (%s)'%(k, s))
1867 args = args + [date.Interval(x).serialise() for x in v]
1868 else:
1869 try:
1870 # Try to filter on range of intervals
1871 date_rng = Range(v, date.Interval)
1872 if (date_rng.from_value):
1873 where.append('_%s >= %s'%(k, a))
1874 args.append(date_rng.from_value.serialise())
1875 if (date_rng.to_value):
1876 where.append('_%s <= %s'%(k, a))
1877 args.append(date_rng.to_value.serialise())
1878 except ValueError:
1879 # If range creation fails - ignore that search parameter
1880 pass
1881 #where.append('_%s=%s'%(k, a))
1882 #args.append(date.Interval(v).serialise())
1883 else:
1884 if isinstance(v, type([])):
1885 s = ','.join([a for x in v])
1886 where.append('_%s in (%s)'%(k, s))
1887 args = args + v
1888 else:
1889 where.append('_%s=%s'%(k, a))
1890 args.append(v)
1892 # add results of full text search
1893 if search_matches is not None:
1894 v = search_matches.keys()
1895 s = ','.join([a for x in v])
1896 where.append('id in (%s)'%s)
1897 args = args + v
1899 # "grouping" is just the first-order sorting in the SQL fetch
1900 # can modify it...)
1901 orderby = []
1902 ordercols = []
1903 if group[0] is not None and group[1] is not None:
1904 if group[0] != '-':
1905 orderby.append('_'+group[1])
1906 ordercols.append('_'+group[1])
1907 else:
1908 orderby.append('_'+group[1]+' desc')
1909 ordercols.append('_'+group[1])
1911 # now add in the sorting
1912 group = ''
1913 if sort[0] is not None and sort[1] is not None:
1914 direction, colname = sort
1915 if direction != '-':
1916 if colname == 'id':
1917 orderby.append(colname)
1918 else:
1919 orderby.append('_'+colname)
1920 ordercols.append('_'+colname)
1921 else:
1922 if colname == 'id':
1923 orderby.append(colname+' desc')
1924 ordercols.append(colname)
1925 else:
1926 orderby.append('_'+colname+' desc')
1927 ordercols.append('_'+colname)
1929 # construct the SQL
1930 frum = ','.join(frum)
1931 if where:
1932 where = ' where ' + (' and '.join(where))
1933 else:
1934 where = ''
1935 cols = ['id']
1936 if orderby:
1937 cols = cols + ordercols
1938 order = ' order by %s'%(','.join(orderby))
1939 else:
1940 order = ''
1941 cols = ','.join(cols)
1942 sql = 'select %s from %s %s%s%s'%(cols, frum, where, group, order)
1943 args = tuple(args)
1944 if __debug__:
1945 print >>hyperdb.DEBUG, 'filter', (self, sql, args)
1946 self.db.cursor.execute(sql, args)
1947 l = self.db.cursor.fetchall()
1949 # return the IDs (the first column)
1950 # XXX The filter(None, l) bit is sqlite-specific... if there's _NO_
1951 # XXX matches to a fetch, it returns NULL instead of nothing!?!
1952 return filter(None, [row[0] for row in l])
1954 def count(self):
1955 '''Get the number of nodes in this class.
1957 If the returned integer is 'numnodes', the ids of all the nodes
1958 in this class run from 1 to numnodes, and numnodes+1 will be the
1959 id of the next node to be created in this class.
1960 '''
1961 return self.db.countnodes(self.classname)
1963 # Manipulating properties:
1964 def getprops(self, protected=1):
1965 '''Return a dictionary mapping property names to property objects.
1966 If the "protected" flag is true, we include protected properties -
1967 those which may not be modified.
1968 '''
1969 d = self.properties.copy()
1970 if protected:
1971 d['id'] = String()
1972 d['creation'] = hyperdb.Date()
1973 d['activity'] = hyperdb.Date()
1974 d['creator'] = hyperdb.Link('user')
1975 return d
1977 def addprop(self, **properties):
1978 '''Add properties to this class.
1980 The keyword arguments in 'properties' must map names to property
1981 objects, or a TypeError is raised. None of the keys in 'properties'
1982 may collide with the names of existing properties, or a ValueError
1983 is raised before any properties have been added.
1984 '''
1985 for key in properties.keys():
1986 if self.properties.has_key(key):
1987 raise ValueError, key
1988 self.properties.update(properties)
1990 def index(self, nodeid):
1991 '''Add (or refresh) the node to search indexes
1992 '''
1993 # find all the String properties that have indexme
1994 for prop, propclass in self.getprops().items():
1995 if isinstance(propclass, String) and propclass.indexme:
1996 try:
1997 value = str(self.get(nodeid, prop))
1998 except IndexError:
1999 # node no longer exists - entry should be removed
2000 self.db.indexer.purge_entry((self.classname, nodeid, prop))
2001 else:
2002 # and index them under (classname, nodeid, property)
2003 self.db.indexer.add_text((self.classname, nodeid, prop),
2004 value)
2007 #
2008 # Detector interface
2009 #
2010 def audit(self, event, detector):
2011 '''Register a detector
2012 '''
2013 l = self.auditors[event]
2014 if detector not in l:
2015 self.auditors[event].append(detector)
2017 def fireAuditors(self, action, nodeid, newvalues):
2018 '''Fire all registered auditors.
2019 '''
2020 for audit in self.auditors[action]:
2021 audit(self.db, self, nodeid, newvalues)
2023 def react(self, event, detector):
2024 '''Register a detector
2025 '''
2026 l = self.reactors[event]
2027 if detector not in l:
2028 self.reactors[event].append(detector)
2030 def fireReactors(self, action, nodeid, oldvalues):
2031 '''Fire all registered reactors.
2032 '''
2033 for react in self.reactors[action]:
2034 react(self.db, self, nodeid, oldvalues)
2036 class FileClass(Class, hyperdb.FileClass):
2037 '''This class defines a large chunk of data. To support this, it has a
2038 mandatory String property "content" which is typically saved off
2039 externally to the hyperdb.
2041 The default MIME type of this data is defined by the
2042 "default_mime_type" class attribute, which may be overridden by each
2043 node if the class defines a "type" String property.
2044 '''
2045 default_mime_type = 'text/plain'
2047 def create(self, **propvalues):
2048 ''' snaffle the file propvalue and store in a file
2049 '''
2050 # we need to fire the auditors now, or the content property won't
2051 # be in propvalues for the auditors to play with
2052 self.fireAuditors('create', None, propvalues)
2054 # now remove the content property so it's not stored in the db
2055 content = propvalues['content']
2056 del propvalues['content']
2058 # do the database create
2059 newid = Class.create_inner(self, **propvalues)
2061 # fire reactors
2062 self.fireReactors('create', newid, None)
2064 # store off the content as a file
2065 self.db.storefile(self.classname, newid, None, content)
2066 return newid
2068 def import_list(self, propnames, proplist):
2069 ''' Trap the "content" property...
2070 '''
2071 # dupe this list so we don't affect others
2072 propnames = propnames[:]
2074 # extract the "content" property from the proplist
2075 i = propnames.index('content')
2076 content = eval(proplist[i])
2077 del propnames[i]
2078 del proplist[i]
2080 # do the normal import
2081 newid = Class.import_list(self, propnames, proplist)
2083 # save off the "content" file
2084 self.db.storefile(self.classname, newid, None, content)
2085 return newid
2087 _marker = []
2088 def get(self, nodeid, propname, default=_marker, cache=1):
2089 ''' trap the content propname and get it from the file
2090 '''
2091 poss_msg = 'Possibly a access right configuration problem.'
2092 if propname == 'content':
2093 try:
2094 return self.db.getfile(self.classname, nodeid, None)
2095 except IOError, (strerror):
2096 # BUG: by catching this we donot see an error in the log.
2097 return 'ERROR reading file: %s%s\n%s\n%s'%(
2098 self.classname, nodeid, poss_msg, strerror)
2099 if default is not self._marker:
2100 return Class.get(self, nodeid, propname, default, cache=cache)
2101 else:
2102 return Class.get(self, nodeid, propname, cache=cache)
2104 def getprops(self, protected=1):
2105 ''' In addition to the actual properties on the node, these methods
2106 provide the "content" property. If the "protected" flag is true,
2107 we include protected properties - those which may not be
2108 modified.
2109 '''
2110 d = Class.getprops(self, protected=protected).copy()
2111 d['content'] = hyperdb.String()
2112 return d
2114 def index(self, nodeid):
2115 ''' Index the node in the search index.
2117 We want to index the content in addition to the normal String
2118 property indexing.
2119 '''
2120 # perform normal indexing
2121 Class.index(self, nodeid)
2123 # get the content to index
2124 content = self.get(nodeid, 'content')
2126 # figure the mime type
2127 if self.properties.has_key('type'):
2128 mime_type = self.get(nodeid, 'type')
2129 else:
2130 mime_type = self.default_mime_type
2132 # and index!
2133 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
2134 mime_type)
2136 # XXX deviation from spec - was called ItemClass
2137 class IssueClass(Class, roundupdb.IssueClass):
2138 # Overridden methods:
2139 def __init__(self, db, classname, **properties):
2140 '''The newly-created class automatically includes the "messages",
2141 "files", "nosy", and "superseder" properties. If the 'properties'
2142 dictionary attempts to specify any of these properties or a
2143 "creation" or "activity" property, a ValueError is raised.
2144 '''
2145 if not properties.has_key('title'):
2146 properties['title'] = hyperdb.String(indexme='yes')
2147 if not properties.has_key('messages'):
2148 properties['messages'] = hyperdb.Multilink("msg")
2149 if not properties.has_key('files'):
2150 properties['files'] = hyperdb.Multilink("file")
2151 if not properties.has_key('nosy'):
2152 # note: journalling is turned off as it really just wastes
2153 # space. this behaviour may be overridden in an instance
2154 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2155 if not properties.has_key('superseder'):
2156 properties['superseder'] = hyperdb.Multilink(classname)
2157 Class.__init__(self, db, classname, **properties)