1 # $Id: rdbms_common.py,v 1.40 2003-03-06 07:33:29 richard Exp $
2 ''' Relational database (SQL) backend common code.
4 Basics:
6 - map roundup classes to relational tables
7 - automatically detect schema changes and modify the table schemas
8 appropriately (we store the "database version" of the schema in the
9 database itself as the only row of the "schema" table)
10 - multilinks (which represent a many-to-many relationship) are handled through
11 intermediate tables
12 - journals are stored adjunct to the per-class tables
13 - table names and columns have "_" prepended so the names can't clash with
14 restricted names (like "order")
15 - retirement is determined by the __retired__ column being true
17 Database-specific changes may generally be pushed out to the overridable
18 sql_* methods, since everything else should be fairly generic. There's
19 probably a bit of work to be done if a database is used that actually
20 honors column typing, since the initial databases don't (sqlite stores
21 everything as a string, and gadfly stores anything that's marsallable).
22 '''
24 # standard python modules
25 import sys, os, time, re, errno, weakref, copy
27 # roundup modules
28 from roundup import hyperdb, date, password, roundupdb, security
29 from roundup.hyperdb import String, Password, Date, Interval, Link, \
30 Multilink, DatabaseError, Boolean, Number, Node
31 from roundup.backends import locking
33 # support
34 from blobfiles import FileStorage
35 from roundup.indexer import Indexer
36 from sessions import Sessions, OneTimeKeys
38 # number of rows to keep in memory
39 ROW_CACHE_SIZE = 100
41 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
42 ''' Wrapper around an SQL database that presents a hyperdb interface.
44 - some functionality is specific to the actual SQL database, hence
45 the sql_* methods that are NotImplemented
46 - we keep a cache of the latest ROW_CACHE_SIZE row fetches.
47 '''
48 def __init__(self, config, journaltag=None):
49 ''' Open the database and load the schema from it.
50 '''
51 self.config, self.journaltag = config, journaltag
52 self.dir = config.DATABASE
53 self.classes = {}
54 self.indexer = Indexer(self.dir)
55 self.sessions = Sessions(self.config)
56 self.otks = OneTimeKeys(self.config)
57 self.security = security.Security(self)
59 # additional transaction support for external files and the like
60 self.transactions = []
62 # keep a cache of the N most recently retrieved rows of any kind
63 # (classname, nodeid) = row
64 self.cache = {}
65 self.cache_lru = []
67 # database lock
68 self.lockfile = None
70 # open a connection to the database, creating the "conn" attribute
71 self.open_connection()
73 def clearCache(self):
74 self.cache = {}
75 self.cache_lru = []
77 def open_connection(self):
78 ''' Open a connection to the database, creating it if necessary
79 '''
80 raise NotImplemented
82 def sql(self, sql, args=None):
83 ''' Execute the sql with the optional args.
84 '''
85 if __debug__:
86 print >>hyperdb.DEBUG, (self, sql, args)
87 if args:
88 self.cursor.execute(sql, args)
89 else:
90 self.cursor.execute(sql)
92 def sql_fetchone(self):
93 ''' Fetch a single row. If there's nothing to fetch, return None.
94 '''
95 raise NotImplemented
97 def sql_stringquote(self, value):
98 ''' Quote the string so it's safe to put in the 'sql quotes'
99 '''
100 return re.sub("'", "''", str(value))
102 def save_dbschema(self, schema):
103 ''' Save the schema definition that the database currently implements
104 '''
105 raise NotImplemented
107 def load_dbschema(self):
108 ''' Load the schema definition that the database currently implements
109 '''
110 raise NotImplemented
112 def post_init(self):
113 ''' Called once the schema initialisation has finished.
115 We should now confirm that the schema defined by our "classes"
116 attribute actually matches the schema in the database.
117 '''
118 # now detect changes in the schema
119 save = 0
120 for classname, spec in self.classes.items():
121 if self.database_schema.has_key(classname):
122 dbspec = self.database_schema[classname]
123 if self.update_class(spec, dbspec):
124 self.database_schema[classname] = spec.schema()
125 save = 1
126 else:
127 self.create_class(spec)
128 self.database_schema[classname] = spec.schema()
129 save = 1
131 for classname in self.database_schema.keys():
132 if not self.classes.has_key(classname):
133 self.drop_class(classname)
135 # update the database version of the schema
136 if save:
137 self.sql('delete from schema')
138 self.save_dbschema(self.database_schema)
140 # reindex the db if necessary
141 if self.indexer.should_reindex():
142 self.reindex()
144 # commit
145 self.conn.commit()
147 # figure the "curuserid"
148 if self.journaltag is None:
149 self.curuserid = None
150 elif self.journaltag == 'admin':
151 # admin user may not exist, but always has ID 1
152 self.curuserid = '1'
153 else:
154 self.curuserid = self.user.lookup(self.journaltag)
156 def reindex(self):
157 for klass in self.classes.values():
158 for nodeid in klass.list():
159 klass.index(nodeid)
160 self.indexer.save_index()
162 def determine_columns(self, properties):
163 ''' Figure the column names and multilink properties from the spec
165 "properties" is a list of (name, prop) where prop may be an
166 instance of a hyperdb "type" _or_ a string repr of that type.
167 '''
168 cols = ['_activity', '_creator', '_creation']
169 mls = []
170 # add the multilinks separately
171 for col, prop in properties:
172 if isinstance(prop, Multilink):
173 mls.append(col)
174 elif isinstance(prop, type('')) and prop.find('Multilink') != -1:
175 mls.append(col)
176 else:
177 cols.append('_'+col)
178 cols.sort()
179 return cols, mls
181 def update_class(self, spec, old_spec):
182 ''' Determine the differences between the current spec and the
183 database version of the spec, and update where necessary
184 '''
185 new_spec = spec
186 new_has = new_spec.properties.has_key
188 new_spec = new_spec.schema()
189 if new_spec == old_spec:
190 # no changes
191 return 0
193 if __debug__:
194 print >>hyperdb.DEBUG, 'update_class FIRING'
196 # key property changed?
197 if old_spec[0] != new_spec[0]:
198 if __debug__:
199 print >>hyperdb.DEBUG, 'update_class setting keyprop', `spec[0]`
200 # XXX turn on indexing for the key property
202 # detect multilinks that have been removed, and drop their table
203 old_has = {}
204 for name,prop in old_spec[1]:
205 old_has[name] = 1
206 if not new_has(name) and isinstance(prop, Multilink):
207 # it's a multilink, and it's been removed - drop the old
208 # table
209 sql = 'drop table %s_%s'%(spec.classname, prop)
210 if __debug__:
211 print >>hyperdb.DEBUG, 'update_class', (self, sql)
212 self.cursor.execute(sql)
213 continue
214 old_has = old_has.has_key
216 # now figure how we populate the new table
217 fetch = [] # fetch these from the old table
218 properties = spec.getprops()
219 for propname,x in new_spec[1]:
220 prop = properties[propname]
221 if isinstance(prop, Multilink):
222 if not old_has(propname):
223 # we need to create the new table
224 self.create_multilink_table(spec, propname)
225 elif old_has(propname):
226 # we copy this col over from the old table
227 fetch.append('_'+propname)
229 # select the data out of the old table
230 fetch.append('id')
231 fetch.append('__retired__')
232 fetchcols = ','.join(fetch)
233 cn = spec.classname
234 sql = 'select %s from _%s'%(fetchcols, cn)
235 if __debug__:
236 print >>hyperdb.DEBUG, 'update_class', (self, sql)
237 self.cursor.execute(sql)
238 olddata = self.cursor.fetchall()
240 # drop the old table
241 self.cursor.execute('drop table _%s'%cn)
243 # create the new table
244 self.create_class_table(spec)
246 if olddata:
247 # do the insert
248 args = ','.join([self.arg for x in fetch])
249 sql = 'insert into _%s (%s) values (%s)'%(cn, fetchcols, args)
250 if __debug__:
251 print >>hyperdb.DEBUG, 'update_class', (self, sql, olddata[0])
252 for entry in olddata:
253 self.cursor.execute(sql, *entry)
255 return 1
257 def create_class_table(self, spec):
258 ''' create the class table for the given spec
259 '''
260 cols, mls = self.determine_columns(spec.properties.items())
262 # add on our special columns
263 cols.append('id')
264 cols.append('__retired__')
266 # create the base table
267 scols = ','.join(['%s varchar'%x for x in cols])
268 sql = 'create table _%s (%s)'%(spec.classname, scols)
269 if __debug__:
270 print >>hyperdb.DEBUG, 'create_class', (self, sql)
271 self.cursor.execute(sql)
273 return cols, mls
275 def create_journal_table(self, spec):
276 ''' create the journal table for a class given the spec and
277 already-determined cols
278 '''
279 # journal table
280 cols = ','.join(['%s varchar'%x
281 for x in 'nodeid date tag action params'.split()])
282 sql = 'create table %s__journal (%s)'%(spec.classname, cols)
283 if __debug__:
284 print >>hyperdb.DEBUG, 'create_class', (self, sql)
285 self.cursor.execute(sql)
287 def create_multilink_table(self, spec, ml):
288 ''' Create a multilink table for the "ml" property of the class
289 given by the spec
290 '''
291 sql = 'create table %s_%s (linkid varchar, nodeid varchar)'%(
292 spec.classname, ml)
293 if __debug__:
294 print >>hyperdb.DEBUG, 'create_class', (self, sql)
295 self.cursor.execute(sql)
297 def create_class(self, spec):
298 ''' Create a database table according to the given spec.
299 '''
300 cols, mls = self.create_class_table(spec)
301 self.create_journal_table(spec)
303 # now create the multilink tables
304 for ml in mls:
305 self.create_multilink_table(spec, ml)
307 # ID counter
308 sql = 'insert into ids (name, num) values (%s,%s)'%(self.arg, self.arg)
309 vals = (spec.classname, 1)
310 if __debug__:
311 print >>hyperdb.DEBUG, 'create_class', (self, sql, vals)
312 self.cursor.execute(sql, vals)
314 def drop_class(self, spec):
315 ''' Drop the given table from the database.
317 Drop the journal and multilink tables too.
318 '''
319 # figure the multilinks
320 mls = []
321 for col, prop in spec.properties.items():
322 if isinstance(prop, Multilink):
323 mls.append(col)
325 sql = 'drop table _%s'%spec.classname
326 if __debug__:
327 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
328 self.cursor.execute(sql)
330 sql = 'drop table %s__journal'%spec.classname
331 if __debug__:
332 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
333 self.cursor.execute(sql)
335 for ml in mls:
336 sql = 'drop table %s_%s'%(spec.classname, ml)
337 if __debug__:
338 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
339 self.cursor.execute(sql)
341 #
342 # Classes
343 #
344 def __getattr__(self, classname):
345 ''' A convenient way of calling self.getclass(classname).
346 '''
347 if self.classes.has_key(classname):
348 if __debug__:
349 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
350 return self.classes[classname]
351 raise AttributeError, classname
353 def addclass(self, cl):
354 ''' Add a Class to the hyperdatabase.
355 '''
356 if __debug__:
357 print >>hyperdb.DEBUG, 'addclass', (self, cl)
358 cn = cl.classname
359 if self.classes.has_key(cn):
360 raise ValueError, cn
361 self.classes[cn] = cl
363 def getclasses(self):
364 ''' Return a list of the names of all existing classes.
365 '''
366 if __debug__:
367 print >>hyperdb.DEBUG, 'getclasses', (self,)
368 l = self.classes.keys()
369 l.sort()
370 return l
372 def getclass(self, classname):
373 '''Get the Class object representing a particular class.
375 If 'classname' is not a valid class name, a KeyError is raised.
376 '''
377 if __debug__:
378 print >>hyperdb.DEBUG, 'getclass', (self, classname)
379 try:
380 return self.classes[classname]
381 except KeyError:
382 raise KeyError, 'There is no class called "%s"'%classname
384 def clear(self):
385 ''' Delete all database contents.
387 Note: I don't commit here, which is different behaviour to the
388 "nuke from orbit" behaviour in the *dbms.
389 '''
390 if __debug__:
391 print >>hyperdb.DEBUG, 'clear', (self,)
392 for cn in self.classes.keys():
393 sql = 'delete from _%s'%cn
394 if __debug__:
395 print >>hyperdb.DEBUG, 'clear', (self, sql)
396 self.cursor.execute(sql)
398 #
399 # Node IDs
400 #
401 def newid(self, classname):
402 ''' Generate a new id for the given class
403 '''
404 # get the next ID
405 sql = 'select num from ids where name=%s'%self.arg
406 if __debug__:
407 print >>hyperdb.DEBUG, 'newid', (self, sql, classname)
408 self.cursor.execute(sql, (classname, ))
409 newid = self.cursor.fetchone()[0]
411 # update the counter
412 sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
413 vals = (int(newid)+1, classname)
414 if __debug__:
415 print >>hyperdb.DEBUG, 'newid', (self, sql, vals)
416 self.cursor.execute(sql, vals)
418 # return as string
419 return str(newid)
421 def setid(self, classname, setid):
422 ''' Set the id counter: used during import of database
423 '''
424 sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
425 vals = (setid, classname)
426 if __debug__:
427 print >>hyperdb.DEBUG, 'setid', (self, sql, vals)
428 self.cursor.execute(sql, vals)
430 #
431 # Nodes
432 #
434 def addnode(self, classname, nodeid, node):
435 ''' Add the specified node to its class's db.
436 '''
437 if __debug__:
438 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
439 # gadfly requires values for all non-multilink columns
440 cl = self.classes[classname]
441 cols, mls = self.determine_columns(cl.properties.items())
443 # we'll be supplied these props if we're doing an import
444 if not node.has_key('creator'):
445 # add in the "calculated" properties (dupe so we don't affect
446 # calling code's node assumptions)
447 node = node.copy()
448 node['creation'] = node['activity'] = date.Date()
449 node['creator'] = self.curuserid
451 # default the non-multilink columns
452 for col, prop in cl.properties.items():
453 if not node.has_key(col):
454 if isinstance(prop, Multilink):
455 node[col] = []
456 else:
457 node[col] = None
459 # clear this node out of the cache if it's in there
460 key = (classname, nodeid)
461 if self.cache.has_key(key):
462 del self.cache[key]
463 self.cache_lru.remove(key)
465 # make the node data safe for the DB
466 node = self.serialise(classname, node)
468 # make sure the ordering is correct for column name -> column value
469 vals = tuple([node[col[1:]] for col in cols]) + (nodeid, 0)
470 s = ','.join([self.arg for x in cols]) + ',%s,%s'%(self.arg, self.arg)
471 cols = ','.join(cols) + ',id,__retired__'
473 # perform the inserts
474 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
475 if __debug__:
476 print >>hyperdb.DEBUG, 'addnode', (self, sql, vals)
477 self.cursor.execute(sql, vals)
479 # insert the multilink rows
480 for col in mls:
481 t = '%s_%s'%(classname, col)
482 for entry in node[col]:
483 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
484 self.arg, self.arg)
485 self.sql(sql, (entry, nodeid))
487 # make sure we do the commit-time extra stuff for this node
488 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
490 def setnode(self, classname, nodeid, values, multilink_changes):
491 ''' Change the specified node.
492 '''
493 if __debug__:
494 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, values)
496 # clear this node out of the cache if it's in there
497 key = (classname, nodeid)
498 if self.cache.has_key(key):
499 del self.cache[key]
500 self.cache_lru.remove(key)
502 # add the special props
503 values = values.copy()
504 values['activity'] = date.Date()
506 # make db-friendly
507 values = self.serialise(classname, values)
509 cl = self.classes[classname]
510 cols = []
511 mls = []
512 # add the multilinks separately
513 props = cl.getprops()
514 for col in values.keys():
515 prop = props[col]
516 if isinstance(prop, Multilink):
517 mls.append(col)
518 else:
519 cols.append('_'+col)
520 cols.sort()
522 # if there's any updates to regular columns, do them
523 if cols:
524 # make sure the ordering is correct for column name -> column value
525 sqlvals = tuple([values[col[1:]] for col in cols]) + (nodeid,)
526 s = ','.join(['%s=%s'%(x, self.arg) for x in cols])
527 cols = ','.join(cols)
529 # perform the update
530 sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
531 if __debug__:
532 print >>hyperdb.DEBUG, 'setnode', (self, sql, sqlvals)
533 self.cursor.execute(sql, sqlvals)
535 # now the fun bit, updating the multilinks ;)
536 for col, (add, remove) in multilink_changes.items():
537 tn = '%s_%s'%(classname, col)
538 if add:
539 sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
540 self.arg, self.arg)
541 for addid in add:
542 self.sql(sql, (nodeid, addid))
543 if remove:
544 sql = 'delete from %s where nodeid=%s and linkid=%s'%(tn,
545 self.arg, self.arg)
546 for removeid in remove:
547 self.sql(sql, (nodeid, removeid))
549 # make sure we do the commit-time extra stuff for this node
550 self.transactions.append((self.doSaveNode, (classname, nodeid, values)))
552 def getnode(self, classname, nodeid):
553 ''' Get a node from the database.
554 '''
555 if __debug__:
556 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid)
558 # see if we have this node cached
559 key = (classname, nodeid)
560 if self.cache.has_key(key):
561 # push us back to the top of the LRU
562 self.cache_lru.remove(key)
563 self.cache_lru.insert(0, key)
564 # return the cached information
565 return self.cache[key]
567 # figure the columns we're fetching
568 cl = self.classes[classname]
569 cols, mls = self.determine_columns(cl.properties.items())
570 scols = ','.join(cols)
572 # perform the basic property fetch
573 sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
574 self.sql(sql, (nodeid,))
576 values = self.sql_fetchone()
577 if values is None:
578 raise IndexError, 'no such %s node %s'%(classname, nodeid)
580 # make up the node
581 node = {}
582 for col in range(len(cols)):
583 node[cols[col][1:]] = values[col]
585 # now the multilinks
586 for col in mls:
587 # get the link ids
588 sql = 'select linkid from %s_%s where nodeid=%s'%(classname, col,
589 self.arg)
590 self.cursor.execute(sql, (nodeid,))
591 # extract the first column from the result
592 node[col] = [x[0] for x in self.cursor.fetchall()]
594 # un-dbificate the node data
595 node = self.unserialise(classname, node)
597 # save off in the cache
598 key = (classname, nodeid)
599 self.cache[key] = node
600 # update the LRU
601 self.cache_lru.insert(0, key)
602 if len(self.cache_lru) > ROW_CACHE_SIZE:
603 del self.cache[self.cache_lru.pop()]
605 return node
607 def destroynode(self, classname, nodeid):
608 '''Remove a node from the database. Called exclusively by the
609 destroy() method on Class.
610 '''
611 if __debug__:
612 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
614 # make sure the node exists
615 if not self.hasnode(classname, nodeid):
616 raise IndexError, '%s has no node %s'%(classname, nodeid)
618 # see if we have this node cached
619 if self.cache.has_key((classname, nodeid)):
620 del self.cache[(classname, nodeid)]
622 # see if there's any obvious commit actions that we should get rid of
623 for entry in self.transactions[:]:
624 if entry[1][:2] == (classname, nodeid):
625 self.transactions.remove(entry)
627 # now do the SQL
628 sql = 'delete from _%s where id=%s'%(classname, self.arg)
629 self.sql(sql, (nodeid,))
631 # remove from multilnks
632 cl = self.getclass(classname)
633 x, mls = self.determine_columns(cl.properties.items())
634 for col in mls:
635 # get the link ids
636 sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
637 self.cursor.execute(sql, (nodeid,))
639 # remove journal entries
640 sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
641 self.sql(sql, (nodeid,))
643 def serialise(self, classname, node):
644 '''Copy the node contents, converting non-marshallable data into
645 marshallable data.
646 '''
647 if __debug__:
648 print >>hyperdb.DEBUG, 'serialise', classname, node
649 properties = self.getclass(classname).getprops()
650 d = {}
651 for k, v in node.items():
652 # if the property doesn't exist, or is the "retired" flag then
653 # it won't be in the properties dict
654 if not properties.has_key(k):
655 d[k] = v
656 continue
658 # get the property spec
659 prop = properties[k]
661 if isinstance(prop, Password) and v is not None:
662 d[k] = str(v)
663 elif isinstance(prop, Date) and v is not None:
664 d[k] = v.serialise()
665 elif isinstance(prop, Interval) and v is not None:
666 d[k] = v.serialise()
667 else:
668 d[k] = v
669 return d
671 def unserialise(self, classname, node):
672 '''Decode the marshalled node data
673 '''
674 if __debug__:
675 print >>hyperdb.DEBUG, 'unserialise', classname, node
676 properties = self.getclass(classname).getprops()
677 d = {}
678 for k, v in node.items():
679 # if the property doesn't exist, or is the "retired" flag then
680 # it won't be in the properties dict
681 if not properties.has_key(k):
682 d[k] = v
683 continue
685 # get the property spec
686 prop = properties[k]
688 if isinstance(prop, Date) and v is not None:
689 d[k] = date.Date(v)
690 elif isinstance(prop, Interval) and v is not None:
691 d[k] = date.Interval(v)
692 elif isinstance(prop, Password) and v is not None:
693 p = password.Password()
694 p.unpack(v)
695 d[k] = p
696 elif (isinstance(prop, Boolean) or isinstance(prop, Number)) and v is not None:
697 d[k]=float(v)
698 else:
699 d[k] = v
700 return d
702 def hasnode(self, classname, nodeid):
703 ''' Determine if the database has a given node.
704 '''
705 sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
706 if __debug__:
707 print >>hyperdb.DEBUG, 'hasnode', (self, sql, nodeid)
708 self.cursor.execute(sql, (nodeid,))
709 return int(self.cursor.fetchone()[0])
711 def countnodes(self, classname):
712 ''' Count the number of nodes that exist for a particular Class.
713 '''
714 sql = 'select count(*) from _%s'%classname
715 if __debug__:
716 print >>hyperdb.DEBUG, 'countnodes', (self, sql)
717 self.cursor.execute(sql)
718 return self.cursor.fetchone()[0]
720 def addjournal(self, classname, nodeid, action, params, creator=None,
721 creation=None):
722 ''' Journal the Action
723 'action' may be:
725 'create' or 'set' -- 'params' is a dictionary of property values
726 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
727 'retire' -- 'params' is None
728 '''
729 # serialise the parameters now if necessary
730 if isinstance(params, type({})):
731 if action in ('set', 'create'):
732 params = self.serialise(classname, params)
734 # handle supply of the special journalling parameters (usually
735 # supplied on importing an existing database)
736 if creator:
737 journaltag = creator
738 else:
739 journaltag = self.curuserid
740 if creation:
741 journaldate = creation.serialise()
742 else:
743 journaldate = date.Date().serialise()
745 # create the journal entry
746 cols = ','.join('nodeid date tag action params'.split())
748 if __debug__:
749 print >>hyperdb.DEBUG, 'addjournal', (nodeid, journaldate,
750 journaltag, action, params)
752 self.save_journal(classname, cols, nodeid, journaldate,
753 journaltag, action, params)
755 def save_journal(self, classname, cols, nodeid, journaldate,
756 journaltag, action, params):
757 ''' Save the journal entry to the database
758 '''
759 raise NotImplemented
761 def getjournal(self, classname, nodeid):
762 ''' get the journal for id
763 '''
764 # make sure the node exists
765 if not self.hasnode(classname, nodeid):
766 raise IndexError, '%s has no node %s'%(classname, nodeid)
768 cols = ','.join('nodeid date tag action params'.split())
769 return self.load_journal(classname, cols, nodeid)
771 def load_journal(self, classname, cols, nodeid):
772 ''' Load the journal from the database
773 '''
774 raise NotImplemented
776 def pack(self, pack_before):
777 ''' Delete all journal entries except "create" before 'pack_before'.
778 '''
779 # get a 'yyyymmddhhmmss' version of the date
780 date_stamp = pack_before.serialise()
782 # do the delete
783 for classname in self.classes.keys():
784 sql = "delete from %s__journal where date<%s and "\
785 "action<>'create'"%(classname, self.arg)
786 if __debug__:
787 print >>hyperdb.DEBUG, 'pack', (self, sql, date_stamp)
788 self.cursor.execute(sql, (date_stamp,))
790 def sql_commit(self):
791 ''' Actually commit to the database.
792 '''
793 self.conn.commit()
795 def commit(self):
796 ''' Commit the current transactions.
798 Save all data changed since the database was opened or since the
799 last commit() or rollback().
800 '''
801 if __debug__:
802 print >>hyperdb.DEBUG, 'commit', (self,)
804 # commit the database
805 self.sql_commit()
807 # now, do all the other transaction stuff
808 reindex = {}
809 for method, args in self.transactions:
810 reindex[method(*args)] = 1
812 # reindex the nodes that request it
813 for classname, nodeid in filter(None, reindex.keys()):
814 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
815 self.getclass(classname).index(nodeid)
817 # save the indexer state
818 self.indexer.save_index()
820 # clear out the transactions
821 self.transactions = []
823 def rollback(self):
824 ''' Reverse all actions from the current transaction.
826 Undo all the changes made since the database was opened or the last
827 commit() or rollback() was performed.
828 '''
829 if __debug__:
830 print >>hyperdb.DEBUG, 'rollback', (self,)
832 # roll back
833 self.conn.rollback()
835 # roll back "other" transaction stuff
836 for method, args in self.transactions:
837 # delete temporary files
838 if method == self.doStoreFile:
839 self.rollbackStoreFile(*args)
840 self.transactions = []
842 # clear the cache
843 self.clearCache()
845 def doSaveNode(self, classname, nodeid, node):
846 ''' dummy that just generates a reindex event
847 '''
848 # return the classname, nodeid so we reindex this content
849 return (classname, nodeid)
851 def close(self):
852 ''' Close off the connection.
853 '''
854 self.conn.close()
855 if self.lockfile is not None:
856 locking.release_lock(self.lockfile)
857 if self.lockfile is not None:
858 self.lockfile.close()
859 self.lockfile = None
861 #
862 # The base Class class
863 #
864 class Class(hyperdb.Class):
865 ''' The handle to a particular class of nodes in a hyperdatabase.
867 All methods except __repr__ and getnode must be implemented by a
868 concrete backend Class.
869 '''
871 def __init__(self, db, classname, **properties):
872 '''Create a new class with a given name and property specification.
874 'classname' must not collide with the name of an existing class,
875 or a ValueError is raised. The keyword arguments in 'properties'
876 must map names to property objects, or a TypeError is raised.
877 '''
878 if (properties.has_key('creation') or properties.has_key('activity')
879 or properties.has_key('creator')):
880 raise ValueError, '"creation", "activity" and "creator" are '\
881 'reserved'
883 self.classname = classname
884 self.properties = properties
885 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
886 self.key = ''
888 # should we journal changes (default yes)
889 self.do_journal = 1
891 # do the db-related init stuff
892 db.addclass(self)
894 self.auditors = {'create': [], 'set': [], 'retire': []}
895 self.reactors = {'create': [], 'set': [], 'retire': []}
897 def schema(self):
898 ''' A dumpable version of the schema that we can store in the
899 database
900 '''
901 return (self.key, [(x, repr(y)) for x,y in self.properties.items()])
903 def enableJournalling(self):
904 '''Turn journalling on for this class
905 '''
906 self.do_journal = 1
908 def disableJournalling(self):
909 '''Turn journalling off for this class
910 '''
911 self.do_journal = 0
913 # Editing nodes:
914 def create(self, **propvalues):
915 ''' Create a new node of this class and return its id.
917 The keyword arguments in 'propvalues' map property names to values.
919 The values of arguments must be acceptable for the types of their
920 corresponding properties or a TypeError is raised.
922 If this class has a key property, it must be present and its value
923 must not collide with other key strings or a ValueError is raised.
925 Any other properties on this class that are missing from the
926 'propvalues' dictionary are set to None.
928 If an id in a link or multilink property does not refer to a valid
929 node, an IndexError is raised.
930 '''
931 self.fireAuditors('create', None, propvalues)
932 newid = self.create_inner(**propvalues)
933 self.fireReactors('create', newid, None)
934 return newid
936 def create_inner(self, **propvalues):
937 ''' Called by create, in-between the audit and react calls.
938 '''
939 if propvalues.has_key('id'):
940 raise KeyError, '"id" is reserved'
942 if self.db.journaltag is None:
943 raise DatabaseError, 'Database open read-only'
945 if propvalues.has_key('creation') or propvalues.has_key('activity'):
946 raise KeyError, '"creation" and "activity" are reserved'
948 # new node's id
949 newid = self.db.newid(self.classname)
951 # validate propvalues
952 num_re = re.compile('^\d+$')
953 for key, value in propvalues.items():
954 if key == self.key:
955 try:
956 self.lookup(value)
957 except KeyError:
958 pass
959 else:
960 raise ValueError, 'node with key "%s" exists'%value
962 # try to handle this property
963 try:
964 prop = self.properties[key]
965 except KeyError:
966 raise KeyError, '"%s" has no property "%s"'%(self.classname,
967 key)
969 if value is not None and isinstance(prop, Link):
970 if type(value) != type(''):
971 raise ValueError, 'link value must be String'
972 link_class = self.properties[key].classname
973 # if it isn't a number, it's a key
974 if not num_re.match(value):
975 try:
976 value = self.db.classes[link_class].lookup(value)
977 except (TypeError, KeyError):
978 raise IndexError, 'new property "%s": %s not a %s'%(
979 key, value, link_class)
980 elif not self.db.getclass(link_class).hasnode(value):
981 raise IndexError, '%s has no node %s'%(link_class, value)
983 # save off the value
984 propvalues[key] = value
986 # register the link with the newly linked node
987 if self.do_journal and self.properties[key].do_journal:
988 self.db.addjournal(link_class, value, 'link',
989 (self.classname, newid, key))
991 elif isinstance(prop, Multilink):
992 if type(value) != type([]):
993 raise TypeError, 'new property "%s" not a list of ids'%key
995 # clean up and validate the list of links
996 link_class = self.properties[key].classname
997 l = []
998 for entry in value:
999 if type(entry) != type(''):
1000 raise ValueError, '"%s" multilink value (%r) '\
1001 'must contain Strings'%(key, value)
1002 # if it isn't a number, it's a key
1003 if not num_re.match(entry):
1004 try:
1005 entry = self.db.classes[link_class].lookup(entry)
1006 except (TypeError, KeyError):
1007 raise IndexError, 'new property "%s": %s not a %s'%(
1008 key, entry, self.properties[key].classname)
1009 l.append(entry)
1010 value = l
1011 propvalues[key] = value
1013 # handle additions
1014 for nodeid in value:
1015 if not self.db.getclass(link_class).hasnode(nodeid):
1016 raise IndexError, '%s has no node %s'%(link_class,
1017 nodeid)
1018 # register the link with the newly linked node
1019 if self.do_journal and self.properties[key].do_journal:
1020 self.db.addjournal(link_class, nodeid, 'link',
1021 (self.classname, newid, key))
1023 elif isinstance(prop, String):
1024 if type(value) != type('') and type(value) != type(u''):
1025 raise TypeError, 'new property "%s" not a string'%key
1027 elif isinstance(prop, Password):
1028 if not isinstance(value, password.Password):
1029 raise TypeError, 'new property "%s" not a Password'%key
1031 elif isinstance(prop, Date):
1032 if value is not None and not isinstance(value, date.Date):
1033 raise TypeError, 'new property "%s" not a Date'%key
1035 elif isinstance(prop, Interval):
1036 if value is not None and not isinstance(value, date.Interval):
1037 raise TypeError, 'new property "%s" not an Interval'%key
1039 elif value is not None and isinstance(prop, Number):
1040 try:
1041 float(value)
1042 except ValueError:
1043 raise TypeError, 'new property "%s" not numeric'%key
1045 elif value is not None and isinstance(prop, Boolean):
1046 try:
1047 int(value)
1048 except ValueError:
1049 raise TypeError, 'new property "%s" not boolean'%key
1051 # make sure there's data where there needs to be
1052 for key, prop in self.properties.items():
1053 if propvalues.has_key(key):
1054 continue
1055 if key == self.key:
1056 raise ValueError, 'key property "%s" is required'%key
1057 if isinstance(prop, Multilink):
1058 propvalues[key] = []
1059 else:
1060 propvalues[key] = None
1062 # done
1063 self.db.addnode(self.classname, newid, propvalues)
1064 if self.do_journal:
1065 self.db.addjournal(self.classname, newid, 'create', {})
1067 return newid
1069 def export_list(self, propnames, nodeid):
1070 ''' Export a node - generate a list of CSV-able data in the order
1071 specified by propnames for the given node.
1072 '''
1073 properties = self.getprops()
1074 l = []
1075 for prop in propnames:
1076 proptype = properties[prop]
1077 value = self.get(nodeid, prop)
1078 # "marshal" data where needed
1079 if value is None:
1080 pass
1081 elif isinstance(proptype, hyperdb.Date):
1082 value = value.get_tuple()
1083 elif isinstance(proptype, hyperdb.Interval):
1084 value = value.get_tuple()
1085 elif isinstance(proptype, hyperdb.Password):
1086 value = str(value)
1087 l.append(repr(value))
1088 l.append(self.is_retired(nodeid))
1089 return l
1091 def import_list(self, propnames, proplist):
1092 ''' Import a node - all information including "id" is present and
1093 should not be sanity checked. Triggers are not triggered. The
1094 journal should be initialised using the "creator" and "created"
1095 information.
1097 Return the nodeid of the node imported.
1098 '''
1099 if self.db.journaltag is None:
1100 raise DatabaseError, 'Database open read-only'
1101 properties = self.getprops()
1103 # make the new node's property map
1104 d = {}
1105 retire = 0
1106 newid = None
1107 for i in range(len(propnames)):
1108 # Use eval to reverse the repr() used to output the CSV
1109 value = eval(proplist[i])
1111 # Figure the property for this column
1112 propname = propnames[i]
1114 # "unmarshal" where necessary
1115 if propname == 'id':
1116 newid = value
1117 continue
1118 elif propname == 'is retired':
1119 # is the item retired?
1120 if int(value):
1121 retire = 1
1122 continue
1124 prop = properties[propname]
1125 if value is None:
1126 # don't set Nones
1127 continue
1128 elif isinstance(prop, hyperdb.Date):
1129 value = date.Date(value)
1130 elif isinstance(prop, hyperdb.Interval):
1131 value = date.Interval(value)
1132 elif isinstance(prop, hyperdb.Password):
1133 pwd = password.Password()
1134 pwd.unpack(value)
1135 value = pwd
1136 d[propname] = value
1138 # get a new id if necessary
1139 if newid is None:
1140 newid = self.db.newid(self.classname)
1142 # retire?
1143 if retire:
1144 # use the arg for __retired__ to cope with any odd database type
1145 # conversion (hello, sqlite)
1146 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1147 self.db.arg, self.db.arg)
1148 if __debug__:
1149 print >>hyperdb.DEBUG, 'retire', (self, sql, newid)
1150 self.db.cursor.execute(sql, (1, newid))
1152 # add the node and journal
1153 self.db.addnode(self.classname, newid, d)
1155 # extract the extraneous journalling gumpf and nuke it
1156 if d.has_key('creator'):
1157 creator = d['creator']
1158 del d['creator']
1159 else:
1160 creator = None
1161 if d.has_key('creation'):
1162 creation = d['creation']
1163 del d['creation']
1164 else:
1165 creation = None
1166 if d.has_key('activity'):
1167 del d['activity']
1168 self.db.addjournal(self.classname, newid, 'create', {}, creator,
1169 creation)
1170 return newid
1172 _marker = []
1173 def get(self, nodeid, propname, default=_marker, cache=1):
1174 '''Get the value of a property on an existing node of this class.
1176 'nodeid' must be the id of an existing node of this class or an
1177 IndexError is raised. 'propname' must be the name of a property
1178 of this class or a KeyError is raised.
1180 'cache' indicates whether the transaction cache should be queried
1181 for the node. If the node has been modified and you need to
1182 determine what its values prior to modification are, you need to
1183 set cache=0.
1184 '''
1185 if propname == 'id':
1186 return nodeid
1188 # get the node's dict
1189 d = self.db.getnode(self.classname, nodeid)
1191 if propname == 'creation':
1192 if d.has_key('creation'):
1193 return d['creation']
1194 else:
1195 return date.Date()
1196 if propname == 'activity':
1197 if d.has_key('activity'):
1198 return d['activity']
1199 else:
1200 return date.Date()
1201 if propname == 'creator':
1202 if d.has_key('creator'):
1203 return d['creator']
1204 else:
1205 return self.db.curuserid
1207 # get the property (raises KeyErorr if invalid)
1208 prop = self.properties[propname]
1210 if not d.has_key(propname):
1211 if default is self._marker:
1212 if isinstance(prop, Multilink):
1213 return []
1214 else:
1215 return None
1216 else:
1217 return default
1219 # don't pass our list to other code
1220 if isinstance(prop, Multilink):
1221 return d[propname][:]
1223 return d[propname]
1225 def getnode(self, nodeid, cache=1):
1226 ''' Return a convenience wrapper for the node.
1228 'nodeid' must be the id of an existing node of this class or an
1229 IndexError is raised.
1231 'cache' indicates whether the transaction cache should be queried
1232 for the node. If the node has been modified and you need to
1233 determine what its values prior to modification are, you need to
1234 set cache=0.
1235 '''
1236 return Node(self, nodeid, cache=cache)
1238 def set(self, nodeid, **propvalues):
1239 '''Modify a property on an existing node of this class.
1241 'nodeid' must be the id of an existing node of this class or an
1242 IndexError is raised.
1244 Each key in 'propvalues' must be the name of a property of this
1245 class or a KeyError is raised.
1247 All values in 'propvalues' must be acceptable types for their
1248 corresponding properties or a TypeError is raised.
1250 If the value of the key property is set, it must not collide with
1251 other key strings or a ValueError is raised.
1253 If the value of a Link or Multilink property contains an invalid
1254 node id, a ValueError is raised.
1255 '''
1256 if not propvalues:
1257 return propvalues
1259 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1260 raise KeyError, '"creation" and "activity" are reserved'
1262 if propvalues.has_key('id'):
1263 raise KeyError, '"id" is reserved'
1265 if self.db.journaltag is None:
1266 raise DatabaseError, 'Database open read-only'
1268 self.fireAuditors('set', nodeid, propvalues)
1269 # Take a copy of the node dict so that the subsequent set
1270 # operation doesn't modify the oldvalues structure.
1271 # XXX used to try the cache here first
1272 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1274 node = self.db.getnode(self.classname, nodeid)
1275 if self.is_retired(nodeid):
1276 raise IndexError, 'Requested item is retired'
1277 num_re = re.compile('^\d+$')
1279 # if the journal value is to be different, store it in here
1280 journalvalues = {}
1282 # remember the add/remove stuff for multilinks, making it easier
1283 # for the Database layer to do its stuff
1284 multilink_changes = {}
1286 for propname, value in propvalues.items():
1287 # check to make sure we're not duplicating an existing key
1288 if propname == self.key and node[propname] != value:
1289 try:
1290 self.lookup(value)
1291 except KeyError:
1292 pass
1293 else:
1294 raise ValueError, 'node with key "%s" exists'%value
1296 # this will raise the KeyError if the property isn't valid
1297 # ... we don't use getprops() here because we only care about
1298 # the writeable properties.
1299 try:
1300 prop = self.properties[propname]
1301 except KeyError:
1302 raise KeyError, '"%s" has no property named "%s"'%(
1303 self.classname, propname)
1305 # if the value's the same as the existing value, no sense in
1306 # doing anything
1307 current = node.get(propname, None)
1308 if value == current:
1309 del propvalues[propname]
1310 continue
1311 journalvalues[propname] = current
1313 # do stuff based on the prop type
1314 if isinstance(prop, Link):
1315 link_class = prop.classname
1316 # if it isn't a number, it's a key
1317 if value is not None and not isinstance(value, type('')):
1318 raise ValueError, 'property "%s" link value be a string'%(
1319 propname)
1320 if isinstance(value, type('')) and not num_re.match(value):
1321 try:
1322 value = self.db.classes[link_class].lookup(value)
1323 except (TypeError, KeyError):
1324 raise IndexError, 'new property "%s": %s not a %s'%(
1325 propname, value, prop.classname)
1327 if (value is not None and
1328 not self.db.getclass(link_class).hasnode(value)):
1329 raise IndexError, '%s has no node %s'%(link_class, value)
1331 if self.do_journal and prop.do_journal:
1332 # register the unlink with the old linked node
1333 if node[propname] is not None:
1334 self.db.addjournal(link_class, node[propname], 'unlink',
1335 (self.classname, nodeid, propname))
1337 # register the link with the newly linked node
1338 if value is not None:
1339 self.db.addjournal(link_class, value, 'link',
1340 (self.classname, nodeid, propname))
1342 elif isinstance(prop, Multilink):
1343 if type(value) != type([]):
1344 raise TypeError, 'new property "%s" not a list of'\
1345 ' ids'%propname
1346 link_class = self.properties[propname].classname
1347 l = []
1348 for entry in value:
1349 # if it isn't a number, it's a key
1350 if type(entry) != type(''):
1351 raise ValueError, 'new property "%s" link value ' \
1352 'must be a string'%propname
1353 if not num_re.match(entry):
1354 try:
1355 entry = self.db.classes[link_class].lookup(entry)
1356 except (TypeError, KeyError):
1357 raise IndexError, 'new property "%s": %s not a %s'%(
1358 propname, entry,
1359 self.properties[propname].classname)
1360 l.append(entry)
1361 value = l
1362 propvalues[propname] = value
1364 # figure the journal entry for this property
1365 add = []
1366 remove = []
1368 # handle removals
1369 if node.has_key(propname):
1370 l = node[propname]
1371 else:
1372 l = []
1373 for id in l[:]:
1374 if id in value:
1375 continue
1376 # register the unlink with the old linked node
1377 if self.do_journal and self.properties[propname].do_journal:
1378 self.db.addjournal(link_class, id, 'unlink',
1379 (self.classname, nodeid, propname))
1380 l.remove(id)
1381 remove.append(id)
1383 # handle additions
1384 for id in value:
1385 if not self.db.getclass(link_class).hasnode(id):
1386 raise IndexError, '%s has no node %s'%(link_class, id)
1387 if id in l:
1388 continue
1389 # register the link with the newly linked node
1390 if self.do_journal and self.properties[propname].do_journal:
1391 self.db.addjournal(link_class, id, 'link',
1392 (self.classname, nodeid, propname))
1393 l.append(id)
1394 add.append(id)
1396 # figure the journal entry
1397 l = []
1398 if add:
1399 l.append(('+', add))
1400 if remove:
1401 l.append(('-', remove))
1402 multilink_changes[propname] = (add, remove)
1403 if l:
1404 journalvalues[propname] = tuple(l)
1406 elif isinstance(prop, String):
1407 if value is not None and type(value) != type('') and type(value) != type(u''):
1408 raise TypeError, 'new property "%s" not a string'%propname
1410 elif isinstance(prop, Password):
1411 if not isinstance(value, password.Password):
1412 raise TypeError, 'new property "%s" not a Password'%propname
1413 propvalues[propname] = value
1415 elif value is not None and isinstance(prop, Date):
1416 if not isinstance(value, date.Date):
1417 raise TypeError, 'new property "%s" not a Date'% propname
1418 propvalues[propname] = value
1420 elif value is not None and isinstance(prop, Interval):
1421 if not isinstance(value, date.Interval):
1422 raise TypeError, 'new property "%s" not an '\
1423 'Interval'%propname
1424 propvalues[propname] = value
1426 elif value is not None and isinstance(prop, Number):
1427 try:
1428 float(value)
1429 except ValueError:
1430 raise TypeError, 'new property "%s" not numeric'%propname
1432 elif value is not None and isinstance(prop, Boolean):
1433 try:
1434 int(value)
1435 except ValueError:
1436 raise TypeError, 'new property "%s" not boolean'%propname
1438 # nothing to do?
1439 if not propvalues:
1440 return propvalues
1442 # do the set, and journal it
1443 self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
1445 if self.do_journal:
1446 self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
1448 self.fireReactors('set', nodeid, oldvalues)
1450 return propvalues
1452 def retire(self, nodeid):
1453 '''Retire a node.
1455 The properties on the node remain available from the get() method,
1456 and the node's id is never reused.
1458 Retired nodes are not returned by the find(), list(), or lookup()
1459 methods, and other nodes may reuse the values of their key properties.
1460 '''
1461 if self.db.journaltag is None:
1462 raise DatabaseError, 'Database open read-only'
1464 self.fireAuditors('retire', nodeid, None)
1466 # use the arg for __retired__ to cope with any odd database type
1467 # conversion (hello, sqlite)
1468 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1469 self.db.arg, self.db.arg)
1470 if __debug__:
1471 print >>hyperdb.DEBUG, 'retire', (self, sql, nodeid)
1472 self.db.cursor.execute(sql, (1, nodeid))
1474 self.fireReactors('retire', nodeid, None)
1476 def is_retired(self, nodeid):
1477 '''Return true if the node is rerired
1478 '''
1479 sql = 'select __retired__ from _%s where id=%s'%(self.classname,
1480 self.db.arg)
1481 if __debug__:
1482 print >>hyperdb.DEBUG, 'is_retired', (self, sql, nodeid)
1483 self.db.cursor.execute(sql, (nodeid,))
1484 return int(self.db.sql_fetchone()[0])
1486 def destroy(self, nodeid):
1487 '''Destroy a node.
1489 WARNING: this method should never be used except in extremely rare
1490 situations where there could never be links to the node being
1491 deleted
1492 WARNING: use retire() instead
1493 WARNING: the properties of this node will not be available ever again
1494 WARNING: really, use retire() instead
1496 Well, I think that's enough warnings. This method exists mostly to
1497 support the session storage of the cgi interface.
1499 The node is completely removed from the hyperdb, including all journal
1500 entries. It will no longer be available, and will generally break code
1501 if there are any references to the node.
1502 '''
1503 if self.db.journaltag is None:
1504 raise DatabaseError, 'Database open read-only'
1505 self.db.destroynode(self.classname, nodeid)
1507 def history(self, nodeid):
1508 '''Retrieve the journal of edits on a particular node.
1510 'nodeid' must be the id of an existing node of this class or an
1511 IndexError is raised.
1513 The returned list contains tuples of the form
1515 (nodeid, date, tag, action, params)
1517 'date' is a Timestamp object specifying the time of the change and
1518 'tag' is the journaltag specified when the database was opened.
1519 '''
1520 if not self.do_journal:
1521 raise ValueError, 'Journalling is disabled for this class'
1522 return self.db.getjournal(self.classname, nodeid)
1524 # Locating nodes:
1525 def hasnode(self, nodeid):
1526 '''Determine if the given nodeid actually exists
1527 '''
1528 return self.db.hasnode(self.classname, nodeid)
1530 def setkey(self, propname):
1531 '''Select a String property of this class to be the key property.
1533 'propname' must be the name of a String property of this class or
1534 None, or a TypeError is raised. The values of the key property on
1535 all existing nodes must be unique or a ValueError is raised.
1536 '''
1537 # XXX create an index on the key prop column
1538 prop = self.getprops()[propname]
1539 if not isinstance(prop, String):
1540 raise TypeError, 'key properties must be String'
1541 self.key = propname
1543 def getkey(self):
1544 '''Return the name of the key property for this class or None.'''
1545 return self.key
1547 def labelprop(self, default_to_id=0):
1548 ''' Return the property name for a label for the given node.
1550 This method attempts to generate a consistent label for the node.
1551 It tries the following in order:
1552 1. key property
1553 2. "name" property
1554 3. "title" property
1555 4. first property from the sorted property name list
1556 '''
1557 k = self.getkey()
1558 if k:
1559 return k
1560 props = self.getprops()
1561 if props.has_key('name'):
1562 return 'name'
1563 elif props.has_key('title'):
1564 return 'title'
1565 if default_to_id:
1566 return 'id'
1567 props = props.keys()
1568 props.sort()
1569 return props[0]
1571 def lookup(self, keyvalue):
1572 '''Locate a particular node by its key property and return its id.
1574 If this class has no key property, a TypeError is raised. If the
1575 'keyvalue' matches one of the values for the key property among
1576 the nodes in this class, the matching node's id is returned;
1577 otherwise a KeyError is raised.
1578 '''
1579 if not self.key:
1580 raise TypeError, 'No key property set for class %s'%self.classname
1582 # use the arg to handle any odd database type conversion (hello,
1583 # sqlite)
1584 sql = "select id from _%s where _%s=%s and __retired__ <> %s"%(
1585 self.classname, self.key, self.db.arg, self.db.arg)
1586 self.db.sql(sql, (keyvalue, 1))
1588 # see if there was a result that's not retired
1589 row = self.db.sql_fetchone()
1590 if not row:
1591 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1592 keyvalue, self.classname)
1594 # return the id
1595 return row[0]
1597 def find(self, **propspec):
1598 '''Get the ids of nodes in this class which link to the given nodes.
1600 'propspec' consists of keyword args propname=nodeid or
1601 propname={nodeid:1, }
1602 'propname' must be the name of a property in this class, or a
1603 KeyError is raised. That property must be a Link or Multilink
1604 property, or a TypeError is raised.
1606 Any node in this class whose 'propname' property links to any of the
1607 nodeids will be returned. Used by the full text indexing, which knows
1608 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1609 issues:
1611 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1612 '''
1613 if __debug__:
1614 print >>hyperdb.DEBUG, 'find', (self, propspec)
1616 # shortcut
1617 if not propspec:
1618 return []
1620 # validate the args
1621 props = self.getprops()
1622 propspec = propspec.items()
1623 for propname, nodeids in propspec:
1624 # check the prop is OK
1625 prop = props[propname]
1626 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1627 raise TypeError, "'%s' not a Link/Multilink property"%propname
1629 # first, links
1630 where = []
1631 allvalues = ()
1632 a = self.db.arg
1633 for prop, values in propspec:
1634 if not isinstance(props[prop], hyperdb.Link):
1635 continue
1636 if type(values) is type(''):
1637 allvalues += (values,)
1638 where.append('_%s = %s'%(prop, a))
1639 else:
1640 allvalues += tuple(values.keys())
1641 where.append('_%s in (%s)'%(prop, ','.join([a]*len(values))))
1642 tables = []
1643 if where:
1644 tables.append('select id as nodeid from _%s where %s'%(
1645 self.classname, ' and '.join(where)))
1647 # now multilinks
1648 for prop, values in propspec:
1649 if not isinstance(props[prop], hyperdb.Multilink):
1650 continue
1651 if type(values) is type(''):
1652 allvalues += (values,)
1653 s = a
1654 else:
1655 allvalues += tuple(values.keys())
1656 s = ','.join([a]*len(values))
1657 tables.append('select nodeid from %s_%s where linkid in (%s)'%(
1658 self.classname, prop, s))
1659 sql = '\nunion\n'.join(tables)
1660 self.db.sql(sql, allvalues)
1661 l = [x[0] for x in self.db.sql_fetchall()]
1662 if __debug__:
1663 print >>hyperdb.DEBUG, 'find ... ', l
1664 return l
1666 def stringFind(self, **requirements):
1667 '''Locate a particular node by matching a set of its String
1668 properties in a caseless search.
1670 If the property is not a String property, a TypeError is raised.
1672 The return is a list of the id of all nodes that match.
1673 '''
1674 where = []
1675 args = []
1676 for propname in requirements.keys():
1677 prop = self.properties[propname]
1678 if isinstance(not prop, String):
1679 raise TypeError, "'%s' not a String property"%propname
1680 where.append(propname)
1681 args.append(requirements[propname].lower())
1683 # generate the where clause
1684 s = ' and '.join(['_%s=%s'%(col, self.db.arg) for col in where])
1685 sql = 'select id from _%s where %s'%(self.classname, s)
1686 self.db.sql(sql, tuple(args))
1687 l = [x[0] for x in self.db.sql_fetchall()]
1688 if __debug__:
1689 print >>hyperdb.DEBUG, 'find ... ', l
1690 return l
1692 def list(self):
1693 ''' Return a list of the ids of the active nodes in this class.
1694 '''
1695 return self.getnodeids(retired=0)
1697 def getnodeids(self, retired=None):
1698 ''' Retrieve all the ids of the nodes for a particular Class.
1700 Set retired=None to get all nodes. Otherwise it'll get all the
1701 retired or non-retired nodes, depending on the flag.
1702 '''
1703 # flip the sense of the flag if we don't want all of them
1704 if retired is not None:
1705 retired = not retired
1706 sql = 'select id from _%s where __retired__ <> %s'%(self.classname,
1707 self.db.arg)
1708 if __debug__:
1709 print >>hyperdb.DEBUG, 'getnodeids', (self, sql, retired)
1710 self.db.cursor.execute(sql, (retired,))
1711 return [x[0] for x in self.db.cursor.fetchall()]
1713 def filter(self, search_matches, filterspec, sort=(None,None),
1714 group=(None,None)):
1715 ''' Return a list of the ids of the active nodes in this class that
1716 match the 'filter' spec, sorted by the group spec and then the
1717 sort spec
1719 "filterspec" is {propname: value(s)}
1720 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1721 and prop is a prop name or None
1722 "search_matches" is {nodeid: marker}
1724 The filter must match all properties specificed - but if the
1725 property value to match is a list, any one of the values in the
1726 list may match for that property to match.
1727 '''
1728 # just don't bother if the full-text search matched diddly
1729 if search_matches == {}:
1730 return []
1732 cn = self.classname
1734 # figure the WHERE clause from the filterspec
1735 props = self.getprops()
1736 frum = ['_'+cn]
1737 where = []
1738 args = []
1739 a = self.db.arg
1740 for k, v in filterspec.items():
1741 propclass = props[k]
1742 # now do other where clause stuff
1743 if isinstance(propclass, Multilink):
1744 tn = '%s_%s'%(cn, k)
1745 frum.append(tn)
1746 if isinstance(v, type([])):
1747 s = ','.join([a for x in v])
1748 where.append('id=%s.nodeid and %s.linkid in (%s)'%(tn,tn,s))
1749 args = args + v
1750 else:
1751 where.append('id=%s.nodeid and %s.linkid = %s'%(tn, tn, a))
1752 args.append(v)
1753 elif k == 'id':
1754 if isinstance(v, type([])):
1755 s = ','.join([a for x in v])
1756 where.append('%s in (%s)'%(k, s))
1757 args = args + v
1758 else:
1759 where.append('%s=%s'%(k, a))
1760 args.append(v)
1761 elif isinstance(propclass, String):
1762 if not isinstance(v, type([])):
1763 v = [v]
1765 # Quote the bits in the string that need it and then embed
1766 # in a "substring" search. Note - need to quote the '%' so
1767 # they make it through the python layer happily
1768 v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
1770 # now add to the where clause
1771 where.append(' or '.join(["_%s LIKE '%s'"%(k, s) for s in v]))
1772 # note: args are embedded in the query string now
1773 elif isinstance(propclass, Link):
1774 if isinstance(v, type([])):
1775 if '-1' in v:
1776 v.remove('-1')
1777 xtra = ' or _%s is NULL'%k
1778 else:
1779 xtra = ''
1780 if v:
1781 s = ','.join([a for x in v])
1782 where.append('(_%s in (%s)%s)'%(k, s, xtra))
1783 args = args + v
1784 else:
1785 where.append('_%s is NULL'%k)
1786 else:
1787 if v == '-1':
1788 v = None
1789 where.append('_%s is NULL'%k)
1790 else:
1791 where.append('_%s=%s'%(k, a))
1792 args.append(v)
1793 elif isinstance(propclass, Date):
1794 if isinstance(v, type([])):
1795 s = ','.join([a for x in v])
1796 where.append('_%s in (%s)'%(k, s))
1797 args = args + [date.Date(x).serialise() for x in v]
1798 else:
1799 where.append('_%s=%s'%(k, a))
1800 args.append(date.Date(v).serialise())
1801 elif isinstance(propclass, Interval):
1802 if isinstance(v, type([])):
1803 s = ','.join([a for x in v])
1804 where.append('_%s in (%s)'%(k, s))
1805 args = args + [date.Interval(x).serialise() for x in v]
1806 else:
1807 where.append('_%s=%s'%(k, a))
1808 args.append(date.Interval(v).serialise())
1809 else:
1810 if isinstance(v, type([])):
1811 s = ','.join([a for x in v])
1812 where.append('_%s in (%s)'%(k, s))
1813 args = args + v
1814 else:
1815 where.append('_%s=%s'%(k, a))
1816 args.append(v)
1818 # add results of full text search
1819 if search_matches is not None:
1820 v = search_matches.keys()
1821 s = ','.join([a for x in v])
1822 where.append('id in (%s)'%s)
1823 args = args + v
1825 # "grouping" is just the first-order sorting in the SQL fetch
1826 # can modify it...)
1827 orderby = []
1828 ordercols = []
1829 if group[0] is not None and group[1] is not None:
1830 if group[0] != '-':
1831 orderby.append('_'+group[1])
1832 ordercols.append('_'+group[1])
1833 else:
1834 orderby.append('_'+group[1]+' desc')
1835 ordercols.append('_'+group[1])
1837 # now add in the sorting
1838 group = ''
1839 if sort[0] is not None and sort[1] is not None:
1840 direction, colname = sort
1841 if direction != '-':
1842 if colname == 'id':
1843 orderby.append(colname)
1844 else:
1845 orderby.append('_'+colname)
1846 ordercols.append('_'+colname)
1847 else:
1848 if colname == 'id':
1849 orderby.append(colname+' desc')
1850 ordercols.append(colname)
1851 else:
1852 orderby.append('_'+colname+' desc')
1853 ordercols.append('_'+colname)
1855 # construct the SQL
1856 frum = ','.join(frum)
1857 if where:
1858 where = ' where ' + (' and '.join(where))
1859 else:
1860 where = ''
1861 cols = ['id']
1862 if orderby:
1863 cols = cols + ordercols
1864 order = ' order by %s'%(','.join(orderby))
1865 else:
1866 order = ''
1867 cols = ','.join(cols)
1868 sql = 'select %s from %s %s%s%s'%(cols, frum, where, group, order)
1869 args = tuple(args)
1870 if __debug__:
1871 print >>hyperdb.DEBUG, 'filter', (self, sql, args)
1872 self.db.cursor.execute(sql, args)
1873 l = self.db.cursor.fetchall()
1875 # return the IDs (the first column)
1876 # XXX The filter(None, l) bit is sqlite-specific... if there's _NO_
1877 # XXX matches to a fetch, it returns NULL instead of nothing!?!
1878 return filter(None, [row[0] for row in l])
1880 def count(self):
1881 '''Get the number of nodes in this class.
1883 If the returned integer is 'numnodes', the ids of all the nodes
1884 in this class run from 1 to numnodes, and numnodes+1 will be the
1885 id of the next node to be created in this class.
1886 '''
1887 return self.db.countnodes(self.classname)
1889 # Manipulating properties:
1890 def getprops(self, protected=1):
1891 '''Return a dictionary mapping property names to property objects.
1892 If the "protected" flag is true, we include protected properties -
1893 those which may not be modified.
1894 '''
1895 d = self.properties.copy()
1896 if protected:
1897 d['id'] = String()
1898 d['creation'] = hyperdb.Date()
1899 d['activity'] = hyperdb.Date()
1900 d['creator'] = hyperdb.Link('user')
1901 return d
1903 def addprop(self, **properties):
1904 '''Add properties to this class.
1906 The keyword arguments in 'properties' must map names to property
1907 objects, or a TypeError is raised. None of the keys in 'properties'
1908 may collide with the names of existing properties, or a ValueError
1909 is raised before any properties have been added.
1910 '''
1911 for key in properties.keys():
1912 if self.properties.has_key(key):
1913 raise ValueError, key
1914 self.properties.update(properties)
1916 def index(self, nodeid):
1917 '''Add (or refresh) the node to search indexes
1918 '''
1919 # find all the String properties that have indexme
1920 for prop, propclass in self.getprops().items():
1921 if isinstance(propclass, String) and propclass.indexme:
1922 try:
1923 value = str(self.get(nodeid, prop))
1924 except IndexError:
1925 # node no longer exists - entry should be removed
1926 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1927 else:
1928 # and index them under (classname, nodeid, property)
1929 self.db.indexer.add_text((self.classname, nodeid, prop),
1930 value)
1933 #
1934 # Detector interface
1935 #
1936 def audit(self, event, detector):
1937 '''Register a detector
1938 '''
1939 l = self.auditors[event]
1940 if detector not in l:
1941 self.auditors[event].append(detector)
1943 def fireAuditors(self, action, nodeid, newvalues):
1944 '''Fire all registered auditors.
1945 '''
1946 for audit in self.auditors[action]:
1947 audit(self.db, self, nodeid, newvalues)
1949 def react(self, event, detector):
1950 '''Register a detector
1951 '''
1952 l = self.reactors[event]
1953 if detector not in l:
1954 self.reactors[event].append(detector)
1956 def fireReactors(self, action, nodeid, oldvalues):
1957 '''Fire all registered reactors.
1958 '''
1959 for react in self.reactors[action]:
1960 react(self.db, self, nodeid, oldvalues)
1962 class FileClass(Class, hyperdb.FileClass):
1963 '''This class defines a large chunk of data. To support this, it has a
1964 mandatory String property "content" which is typically saved off
1965 externally to the hyperdb.
1967 The default MIME type of this data is defined by the
1968 "default_mime_type" class attribute, which may be overridden by each
1969 node if the class defines a "type" String property.
1970 '''
1971 default_mime_type = 'text/plain'
1973 def create(self, **propvalues):
1974 ''' snaffle the file propvalue and store in a file
1975 '''
1976 # we need to fire the auditors now, or the content property won't
1977 # be in propvalues for the auditors to play with
1978 self.fireAuditors('create', None, propvalues)
1980 # now remove the content property so it's not stored in the db
1981 content = propvalues['content']
1982 del propvalues['content']
1984 # do the database create
1985 newid = Class.create_inner(self, **propvalues)
1987 # fire reactors
1988 self.fireReactors('create', newid, None)
1990 # store off the content as a file
1991 self.db.storefile(self.classname, newid, None, content)
1992 return newid
1994 def import_list(self, propnames, proplist):
1995 ''' Trap the "content" property...
1996 '''
1997 # dupe this list so we don't affect others
1998 propnames = propnames[:]
2000 # extract the "content" property from the proplist
2001 i = propnames.index('content')
2002 content = eval(proplist[i])
2003 del propnames[i]
2004 del proplist[i]
2006 # do the normal import
2007 newid = Class.import_list(self, propnames, proplist)
2009 # save off the "content" file
2010 self.db.storefile(self.classname, newid, None, content)
2011 return newid
2013 _marker = []
2014 def get(self, nodeid, propname, default=_marker, cache=1):
2015 ''' trap the content propname and get it from the file
2016 '''
2017 poss_msg = 'Possibly a access right configuration problem.'
2018 if propname == 'content':
2019 try:
2020 return self.db.getfile(self.classname, nodeid, None)
2021 except IOError, (strerror):
2022 # BUG: by catching this we donot see an error in the log.
2023 return 'ERROR reading file: %s%s\n%s\n%s'%(
2024 self.classname, nodeid, poss_msg, strerror)
2025 if default is not self._marker:
2026 return Class.get(self, nodeid, propname, default, cache=cache)
2027 else:
2028 return Class.get(self, nodeid, propname, cache=cache)
2030 def getprops(self, protected=1):
2031 ''' In addition to the actual properties on the node, these methods
2032 provide the "content" property. If the "protected" flag is true,
2033 we include protected properties - those which may not be
2034 modified.
2035 '''
2036 d = Class.getprops(self, protected=protected).copy()
2037 d['content'] = hyperdb.String()
2038 return d
2040 def index(self, nodeid):
2041 ''' Index the node in the search index.
2043 We want to index the content in addition to the normal String
2044 property indexing.
2045 '''
2046 # perform normal indexing
2047 Class.index(self, nodeid)
2049 # get the content to index
2050 content = self.get(nodeid, 'content')
2052 # figure the mime type
2053 if self.properties.has_key('type'):
2054 mime_type = self.get(nodeid, 'type')
2055 else:
2056 mime_type = self.default_mime_type
2058 # and index!
2059 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
2060 mime_type)
2062 # XXX deviation from spec - was called ItemClass
2063 class IssueClass(Class, roundupdb.IssueClass):
2064 # Overridden methods:
2065 def __init__(self, db, classname, **properties):
2066 '''The newly-created class automatically includes the "messages",
2067 "files", "nosy", and "superseder" properties. If the 'properties'
2068 dictionary attempts to specify any of these properties or a
2069 "creation" or "activity" property, a ValueError is raised.
2070 '''
2071 if not properties.has_key('title'):
2072 properties['title'] = hyperdb.String(indexme='yes')
2073 if not properties.has_key('messages'):
2074 properties['messages'] = hyperdb.Multilink("msg")
2075 if not properties.has_key('files'):
2076 properties['files'] = hyperdb.Multilink("file")
2077 if not properties.has_key('nosy'):
2078 # note: journalling is turned off as it really just wastes
2079 # space. this behaviour may be overridden in an instance
2080 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2081 if not properties.has_key('superseder'):
2082 properties['superseder'] = hyperdb.Multilink(classname)
2083 Class.__init__(self, db, classname, **properties)