ce6455c02838bed48e505207fcffa315fc49817c
1 # $Id: rdbms_common.py,v 1.41 2003-03-08 20:41:45 kedder Exp $
2 ''' Relational database (SQL) backend common code.
4 Basics:
6 - map roundup classes to relational tables
7 - automatically detect schema changes and modify the table schemas
8 appropriately (we store the "database version" of the schema in the
9 database itself as the only row of the "schema" table)
10 - multilinks (which represent a many-to-many relationship) are handled through
11 intermediate tables
12 - journals are stored adjunct to the per-class tables
13 - table names and columns have "_" prepended so the names can't clash with
14 restricted names (like "order")
15 - retirement is determined by the __retired__ column being true
17 Database-specific changes may generally be pushed out to the overridable
18 sql_* methods, since everything else should be fairly generic. There's
19 probably a bit of work to be done if a database is used that actually
20 honors column typing, since the initial databases don't (sqlite stores
21 everything as a string, and gadfly stores anything that's marsallable).
22 '''
24 # standard python modules
25 import sys, os, time, re, errno, weakref, copy
27 # roundup modules
28 from roundup import hyperdb, date, password, roundupdb, security
29 from roundup.hyperdb import String, Password, Date, Interval, Link, \
30 Multilink, DatabaseError, Boolean, Number, Node
31 from roundup.backends import locking
33 # support
34 from blobfiles import FileStorage
35 from roundup.indexer import Indexer
36 from sessions import Sessions, OneTimeKeys
37 from roundup.date import Range
39 # number of rows to keep in memory
40 ROW_CACHE_SIZE = 100
42 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
43 ''' Wrapper around an SQL database that presents a hyperdb interface.
45 - some functionality is specific to the actual SQL database, hence
46 the sql_* methods that are NotImplemented
47 - we keep a cache of the latest ROW_CACHE_SIZE row fetches.
48 '''
49 def __init__(self, config, journaltag=None):
50 ''' Open the database and load the schema from it.
51 '''
52 self.config, self.journaltag = config, journaltag
53 self.dir = config.DATABASE
54 self.classes = {}
55 self.indexer = Indexer(self.dir)
56 self.sessions = Sessions(self.config)
57 self.otks = OneTimeKeys(self.config)
58 self.security = security.Security(self)
60 # additional transaction support for external files and the like
61 self.transactions = []
63 # keep a cache of the N most recently retrieved rows of any kind
64 # (classname, nodeid) = row
65 self.cache = {}
66 self.cache_lru = []
68 # database lock
69 self.lockfile = None
71 # open a connection to the database, creating the "conn" attribute
72 self.open_connection()
74 def clearCache(self):
75 self.cache = {}
76 self.cache_lru = []
78 def open_connection(self):
79 ''' Open a connection to the database, creating it if necessary
80 '''
81 raise NotImplemented
83 def sql(self, sql, args=None):
84 ''' Execute the sql with the optional args.
85 '''
86 if __debug__:
87 print >>hyperdb.DEBUG, (self, sql, args)
88 if args:
89 self.cursor.execute(sql, args)
90 else:
91 self.cursor.execute(sql)
93 def sql_fetchone(self):
94 ''' Fetch a single row. If there's nothing to fetch, return None.
95 '''
96 raise NotImplemented
98 def sql_stringquote(self, value):
99 ''' Quote the string so it's safe to put in the 'sql quotes'
100 '''
101 return re.sub("'", "''", str(value))
103 def save_dbschema(self, schema):
104 ''' Save the schema definition that the database currently implements
105 '''
106 raise NotImplemented
108 def load_dbschema(self):
109 ''' Load the schema definition that the database currently implements
110 '''
111 raise NotImplemented
113 def post_init(self):
114 ''' Called once the schema initialisation has finished.
116 We should now confirm that the schema defined by our "classes"
117 attribute actually matches the schema in the database.
118 '''
119 # now detect changes in the schema
120 save = 0
121 for classname, spec in self.classes.items():
122 if self.database_schema.has_key(classname):
123 dbspec = self.database_schema[classname]
124 if self.update_class(spec, dbspec):
125 self.database_schema[classname] = spec.schema()
126 save = 1
127 else:
128 self.create_class(spec)
129 self.database_schema[classname] = spec.schema()
130 save = 1
132 for classname in self.database_schema.keys():
133 if not self.classes.has_key(classname):
134 self.drop_class(classname)
136 # update the database version of the schema
137 if save:
138 self.sql('delete from schema')
139 self.save_dbschema(self.database_schema)
141 # reindex the db if necessary
142 if self.indexer.should_reindex():
143 self.reindex()
145 # commit
146 self.conn.commit()
148 # figure the "curuserid"
149 if self.journaltag is None:
150 self.curuserid = None
151 elif self.journaltag == 'admin':
152 # admin user may not exist, but always has ID 1
153 self.curuserid = '1'
154 else:
155 self.curuserid = self.user.lookup(self.journaltag)
157 def reindex(self):
158 for klass in self.classes.values():
159 for nodeid in klass.list():
160 klass.index(nodeid)
161 self.indexer.save_index()
163 def determine_columns(self, properties):
164 ''' Figure the column names and multilink properties from the spec
166 "properties" is a list of (name, prop) where prop may be an
167 instance of a hyperdb "type" _or_ a string repr of that type.
168 '''
169 cols = ['_activity', '_creator', '_creation']
170 mls = []
171 # add the multilinks separately
172 for col, prop in properties:
173 if isinstance(prop, Multilink):
174 mls.append(col)
175 elif isinstance(prop, type('')) and prop.find('Multilink') != -1:
176 mls.append(col)
177 else:
178 cols.append('_'+col)
179 cols.sort()
180 return cols, mls
182 def update_class(self, spec, old_spec):
183 ''' Determine the differences between the current spec and the
184 database version of the spec, and update where necessary
185 '''
186 new_spec = spec
187 new_has = new_spec.properties.has_key
189 new_spec = new_spec.schema()
190 if new_spec == old_spec:
191 # no changes
192 return 0
194 if __debug__:
195 print >>hyperdb.DEBUG, 'update_class FIRING'
197 # key property changed?
198 if old_spec[0] != new_spec[0]:
199 if __debug__:
200 print >>hyperdb.DEBUG, 'update_class setting keyprop', `spec[0]`
201 # XXX turn on indexing for the key property
203 # detect multilinks that have been removed, and drop their table
204 old_has = {}
205 for name,prop in old_spec[1]:
206 old_has[name] = 1
207 if not new_has(name) and isinstance(prop, Multilink):
208 # it's a multilink, and it's been removed - drop the old
209 # table
210 sql = 'drop table %s_%s'%(spec.classname, prop)
211 if __debug__:
212 print >>hyperdb.DEBUG, 'update_class', (self, sql)
213 self.cursor.execute(sql)
214 continue
215 old_has = old_has.has_key
217 # now figure how we populate the new table
218 fetch = [] # fetch these from the old table
219 properties = spec.getprops()
220 for propname,x in new_spec[1]:
221 prop = properties[propname]
222 if isinstance(prop, Multilink):
223 if not old_has(propname):
224 # we need to create the new table
225 self.create_multilink_table(spec, propname)
226 elif old_has(propname):
227 # we copy this col over from the old table
228 fetch.append('_'+propname)
230 # select the data out of the old table
231 fetch.append('id')
232 fetch.append('__retired__')
233 fetchcols = ','.join(fetch)
234 cn = spec.classname
235 sql = 'select %s from _%s'%(fetchcols, cn)
236 if __debug__:
237 print >>hyperdb.DEBUG, 'update_class', (self, sql)
238 self.cursor.execute(sql)
239 olddata = self.cursor.fetchall()
241 # drop the old table
242 self.cursor.execute('drop table _%s'%cn)
244 # create the new table
245 self.create_class_table(spec)
247 if olddata:
248 # do the insert
249 args = ','.join([self.arg for x in fetch])
250 sql = 'insert into _%s (%s) values (%s)'%(cn, fetchcols, args)
251 if __debug__:
252 print >>hyperdb.DEBUG, 'update_class', (self, sql, olddata[0])
253 for entry in olddata:
254 self.cursor.execute(sql, *entry)
256 return 1
258 def create_class_table(self, spec):
259 ''' create the class table for the given spec
260 '''
261 cols, mls = self.determine_columns(spec.properties.items())
263 # add on our special columns
264 cols.append('id')
265 cols.append('__retired__')
267 # create the base table
268 scols = ','.join(['%s varchar'%x for x in cols])
269 sql = 'create table _%s (%s)'%(spec.classname, scols)
270 if __debug__:
271 print >>hyperdb.DEBUG, 'create_class', (self, sql)
272 self.cursor.execute(sql)
274 return cols, mls
276 def create_journal_table(self, spec):
277 ''' create the journal table for a class given the spec and
278 already-determined cols
279 '''
280 # journal table
281 cols = ','.join(['%s varchar'%x
282 for x in 'nodeid date tag action params'.split()])
283 sql = 'create table %s__journal (%s)'%(spec.classname, cols)
284 if __debug__:
285 print >>hyperdb.DEBUG, 'create_class', (self, sql)
286 self.cursor.execute(sql)
288 def create_multilink_table(self, spec, ml):
289 ''' Create a multilink table for the "ml" property of the class
290 given by the spec
291 '''
292 sql = 'create table %s_%s (linkid varchar, nodeid varchar)'%(
293 spec.classname, ml)
294 if __debug__:
295 print >>hyperdb.DEBUG, 'create_class', (self, sql)
296 self.cursor.execute(sql)
298 def create_class(self, spec):
299 ''' Create a database table according to the given spec.
300 '''
301 cols, mls = self.create_class_table(spec)
302 self.create_journal_table(spec)
304 # now create the multilink tables
305 for ml in mls:
306 self.create_multilink_table(spec, ml)
308 # ID counter
309 sql = 'insert into ids (name, num) values (%s,%s)'%(self.arg, self.arg)
310 vals = (spec.classname, 1)
311 if __debug__:
312 print >>hyperdb.DEBUG, 'create_class', (self, sql, vals)
313 self.cursor.execute(sql, vals)
315 def drop_class(self, spec):
316 ''' Drop the given table from the database.
318 Drop the journal and multilink tables too.
319 '''
320 # figure the multilinks
321 mls = []
322 for col, prop in spec.properties.items():
323 if isinstance(prop, Multilink):
324 mls.append(col)
326 sql = 'drop table _%s'%spec.classname
327 if __debug__:
328 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
329 self.cursor.execute(sql)
331 sql = 'drop table %s__journal'%spec.classname
332 if __debug__:
333 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
334 self.cursor.execute(sql)
336 for ml in mls:
337 sql = 'drop table %s_%s'%(spec.classname, ml)
338 if __debug__:
339 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
340 self.cursor.execute(sql)
342 #
343 # Classes
344 #
345 def __getattr__(self, classname):
346 ''' A convenient way of calling self.getclass(classname).
347 '''
348 if self.classes.has_key(classname):
349 if __debug__:
350 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
351 return self.classes[classname]
352 raise AttributeError, classname
354 def addclass(self, cl):
355 ''' Add a Class to the hyperdatabase.
356 '''
357 if __debug__:
358 print >>hyperdb.DEBUG, 'addclass', (self, cl)
359 cn = cl.classname
360 if self.classes.has_key(cn):
361 raise ValueError, cn
362 self.classes[cn] = cl
364 def getclasses(self):
365 ''' Return a list of the names of all existing classes.
366 '''
367 if __debug__:
368 print >>hyperdb.DEBUG, 'getclasses', (self,)
369 l = self.classes.keys()
370 l.sort()
371 return l
373 def getclass(self, classname):
374 '''Get the Class object representing a particular class.
376 If 'classname' is not a valid class name, a KeyError is raised.
377 '''
378 if __debug__:
379 print >>hyperdb.DEBUG, 'getclass', (self, classname)
380 try:
381 return self.classes[classname]
382 except KeyError:
383 raise KeyError, 'There is no class called "%s"'%classname
385 def clear(self):
386 ''' Delete all database contents.
388 Note: I don't commit here, which is different behaviour to the
389 "nuke from orbit" behaviour in the *dbms.
390 '''
391 if __debug__:
392 print >>hyperdb.DEBUG, 'clear', (self,)
393 for cn in self.classes.keys():
394 sql = 'delete from _%s'%cn
395 if __debug__:
396 print >>hyperdb.DEBUG, 'clear', (self, sql)
397 self.cursor.execute(sql)
399 #
400 # Node IDs
401 #
402 def newid(self, classname):
403 ''' Generate a new id for the given class
404 '''
405 # get the next ID
406 sql = 'select num from ids where name=%s'%self.arg
407 if __debug__:
408 print >>hyperdb.DEBUG, 'newid', (self, sql, classname)
409 self.cursor.execute(sql, (classname, ))
410 newid = self.cursor.fetchone()[0]
412 # update the counter
413 sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
414 vals = (int(newid)+1, classname)
415 if __debug__:
416 print >>hyperdb.DEBUG, 'newid', (self, sql, vals)
417 self.cursor.execute(sql, vals)
419 # return as string
420 return str(newid)
422 def setid(self, classname, setid):
423 ''' Set the id counter: used during import of database
424 '''
425 sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
426 vals = (setid, classname)
427 if __debug__:
428 print >>hyperdb.DEBUG, 'setid', (self, sql, vals)
429 self.cursor.execute(sql, vals)
431 #
432 # Nodes
433 #
435 def addnode(self, classname, nodeid, node):
436 ''' Add the specified node to its class's db.
437 '''
438 if __debug__:
439 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
440 # gadfly requires values for all non-multilink columns
441 cl = self.classes[classname]
442 cols, mls = self.determine_columns(cl.properties.items())
444 # we'll be supplied these props if we're doing an import
445 if not node.has_key('creator'):
446 # add in the "calculated" properties (dupe so we don't affect
447 # calling code's node assumptions)
448 node = node.copy()
449 node['creation'] = node['activity'] = date.Date()
450 node['creator'] = self.curuserid
452 # default the non-multilink columns
453 for col, prop in cl.properties.items():
454 if not node.has_key(col):
455 if isinstance(prop, Multilink):
456 node[col] = []
457 else:
458 node[col] = None
460 # clear this node out of the cache if it's in there
461 key = (classname, nodeid)
462 if self.cache.has_key(key):
463 del self.cache[key]
464 self.cache_lru.remove(key)
466 # make the node data safe for the DB
467 node = self.serialise(classname, node)
469 # make sure the ordering is correct for column name -> column value
470 vals = tuple([node[col[1:]] for col in cols]) + (nodeid, 0)
471 s = ','.join([self.arg for x in cols]) + ',%s,%s'%(self.arg, self.arg)
472 cols = ','.join(cols) + ',id,__retired__'
474 # perform the inserts
475 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
476 if __debug__:
477 print >>hyperdb.DEBUG, 'addnode', (self, sql, vals)
478 self.cursor.execute(sql, vals)
480 # insert the multilink rows
481 for col in mls:
482 t = '%s_%s'%(classname, col)
483 for entry in node[col]:
484 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
485 self.arg, self.arg)
486 self.sql(sql, (entry, nodeid))
488 # make sure we do the commit-time extra stuff for this node
489 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
491 def setnode(self, classname, nodeid, values, multilink_changes):
492 ''' Change the specified node.
493 '''
494 if __debug__:
495 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, values)
497 # clear this node out of the cache if it's in there
498 key = (classname, nodeid)
499 if self.cache.has_key(key):
500 del self.cache[key]
501 self.cache_lru.remove(key)
503 # add the special props
504 values = values.copy()
505 values['activity'] = date.Date()
507 # make db-friendly
508 values = self.serialise(classname, values)
510 cl = self.classes[classname]
511 cols = []
512 mls = []
513 # add the multilinks separately
514 props = cl.getprops()
515 for col in values.keys():
516 prop = props[col]
517 if isinstance(prop, Multilink):
518 mls.append(col)
519 else:
520 cols.append('_'+col)
521 cols.sort()
523 # if there's any updates to regular columns, do them
524 if cols:
525 # make sure the ordering is correct for column name -> column value
526 sqlvals = tuple([values[col[1:]] for col in cols]) + (nodeid,)
527 s = ','.join(['%s=%s'%(x, self.arg) for x in cols])
528 cols = ','.join(cols)
530 # perform the update
531 sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
532 if __debug__:
533 print >>hyperdb.DEBUG, 'setnode', (self, sql, sqlvals)
534 self.cursor.execute(sql, sqlvals)
536 # now the fun bit, updating the multilinks ;)
537 for col, (add, remove) in multilink_changes.items():
538 tn = '%s_%s'%(classname, col)
539 if add:
540 sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
541 self.arg, self.arg)
542 for addid in add:
543 self.sql(sql, (nodeid, addid))
544 if remove:
545 sql = 'delete from %s where nodeid=%s and linkid=%s'%(tn,
546 self.arg, self.arg)
547 for removeid in remove:
548 self.sql(sql, (nodeid, removeid))
550 # make sure we do the commit-time extra stuff for this node
551 self.transactions.append((self.doSaveNode, (classname, nodeid, values)))
553 def getnode(self, classname, nodeid):
554 ''' Get a node from the database.
555 '''
556 if __debug__:
557 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid)
559 # see if we have this node cached
560 key = (classname, nodeid)
561 if self.cache.has_key(key):
562 # push us back to the top of the LRU
563 self.cache_lru.remove(key)
564 self.cache_lru.insert(0, key)
565 # return the cached information
566 return self.cache[key]
568 # figure the columns we're fetching
569 cl = self.classes[classname]
570 cols, mls = self.determine_columns(cl.properties.items())
571 scols = ','.join(cols)
573 # perform the basic property fetch
574 sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
575 self.sql(sql, (nodeid,))
577 values = self.sql_fetchone()
578 if values is None:
579 raise IndexError, 'no such %s node %s'%(classname, nodeid)
581 # make up the node
582 node = {}
583 for col in range(len(cols)):
584 node[cols[col][1:]] = values[col]
586 # now the multilinks
587 for col in mls:
588 # get the link ids
589 sql = 'select linkid from %s_%s where nodeid=%s'%(classname, col,
590 self.arg)
591 self.cursor.execute(sql, (nodeid,))
592 # extract the first column from the result
593 node[col] = [x[0] for x in self.cursor.fetchall()]
595 # un-dbificate the node data
596 node = self.unserialise(classname, node)
598 # save off in the cache
599 key = (classname, nodeid)
600 self.cache[key] = node
601 # update the LRU
602 self.cache_lru.insert(0, key)
603 if len(self.cache_lru) > ROW_CACHE_SIZE:
604 del self.cache[self.cache_lru.pop()]
606 return node
608 def destroynode(self, classname, nodeid):
609 '''Remove a node from the database. Called exclusively by the
610 destroy() method on Class.
611 '''
612 if __debug__:
613 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
615 # make sure the node exists
616 if not self.hasnode(classname, nodeid):
617 raise IndexError, '%s has no node %s'%(classname, nodeid)
619 # see if we have this node cached
620 if self.cache.has_key((classname, nodeid)):
621 del self.cache[(classname, nodeid)]
623 # see if there's any obvious commit actions that we should get rid of
624 for entry in self.transactions[:]:
625 if entry[1][:2] == (classname, nodeid):
626 self.transactions.remove(entry)
628 # now do the SQL
629 sql = 'delete from _%s where id=%s'%(classname, self.arg)
630 self.sql(sql, (nodeid,))
632 # remove from multilnks
633 cl = self.getclass(classname)
634 x, mls = self.determine_columns(cl.properties.items())
635 for col in mls:
636 # get the link ids
637 sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
638 self.cursor.execute(sql, (nodeid,))
640 # remove journal entries
641 sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
642 self.sql(sql, (nodeid,))
644 def serialise(self, classname, node):
645 '''Copy the node contents, converting non-marshallable data into
646 marshallable data.
647 '''
648 if __debug__:
649 print >>hyperdb.DEBUG, 'serialise', classname, node
650 properties = self.getclass(classname).getprops()
651 d = {}
652 for k, v in node.items():
653 # if the property doesn't exist, or is the "retired" flag then
654 # it won't be in the properties dict
655 if not properties.has_key(k):
656 d[k] = v
657 continue
659 # get the property spec
660 prop = properties[k]
662 if isinstance(prop, Password) and v is not None:
663 d[k] = str(v)
664 elif isinstance(prop, Date) and v is not None:
665 d[k] = v.serialise()
666 elif isinstance(prop, Interval) and v is not None:
667 d[k] = v.serialise()
668 else:
669 d[k] = v
670 return d
672 def unserialise(self, classname, node):
673 '''Decode the marshalled node data
674 '''
675 if __debug__:
676 print >>hyperdb.DEBUG, 'unserialise', classname, node
677 properties = self.getclass(classname).getprops()
678 d = {}
679 for k, v in node.items():
680 # if the property doesn't exist, or is the "retired" flag then
681 # it won't be in the properties dict
682 if not properties.has_key(k):
683 d[k] = v
684 continue
686 # get the property spec
687 prop = properties[k]
689 if isinstance(prop, Date) and v is not None:
690 d[k] = date.Date(v)
691 elif isinstance(prop, Interval) and v is not None:
692 d[k] = date.Interval(v)
693 elif isinstance(prop, Password) and v is not None:
694 p = password.Password()
695 p.unpack(v)
696 d[k] = p
697 elif (isinstance(prop, Boolean) or isinstance(prop, Number)) and v is not None:
698 d[k]=float(v)
699 else:
700 d[k] = v
701 return d
703 def hasnode(self, classname, nodeid):
704 ''' Determine if the database has a given node.
705 '''
706 sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
707 if __debug__:
708 print >>hyperdb.DEBUG, 'hasnode', (self, sql, nodeid)
709 self.cursor.execute(sql, (nodeid,))
710 return int(self.cursor.fetchone()[0])
712 def countnodes(self, classname):
713 ''' Count the number of nodes that exist for a particular Class.
714 '''
715 sql = 'select count(*) from _%s'%classname
716 if __debug__:
717 print >>hyperdb.DEBUG, 'countnodes', (self, sql)
718 self.cursor.execute(sql)
719 return self.cursor.fetchone()[0]
721 def addjournal(self, classname, nodeid, action, params, creator=None,
722 creation=None):
723 ''' Journal the Action
724 'action' may be:
726 'create' or 'set' -- 'params' is a dictionary of property values
727 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
728 'retire' -- 'params' is None
729 '''
730 # serialise the parameters now if necessary
731 if isinstance(params, type({})):
732 if action in ('set', 'create'):
733 params = self.serialise(classname, params)
735 # handle supply of the special journalling parameters (usually
736 # supplied on importing an existing database)
737 if creator:
738 journaltag = creator
739 else:
740 journaltag = self.curuserid
741 if creation:
742 journaldate = creation.serialise()
743 else:
744 journaldate = date.Date().serialise()
746 # create the journal entry
747 cols = ','.join('nodeid date tag action params'.split())
749 if __debug__:
750 print >>hyperdb.DEBUG, 'addjournal', (nodeid, journaldate,
751 journaltag, action, params)
753 self.save_journal(classname, cols, nodeid, journaldate,
754 journaltag, action, params)
756 def save_journal(self, classname, cols, nodeid, journaldate,
757 journaltag, action, params):
758 ''' Save the journal entry to the database
759 '''
760 raise NotImplemented
762 def getjournal(self, classname, nodeid):
763 ''' get the journal for id
764 '''
765 # make sure the node exists
766 if not self.hasnode(classname, nodeid):
767 raise IndexError, '%s has no node %s'%(classname, nodeid)
769 cols = ','.join('nodeid date tag action params'.split())
770 return self.load_journal(classname, cols, nodeid)
772 def load_journal(self, classname, cols, nodeid):
773 ''' Load the journal from the database
774 '''
775 raise NotImplemented
777 def pack(self, pack_before):
778 ''' Delete all journal entries except "create" before 'pack_before'.
779 '''
780 # get a 'yyyymmddhhmmss' version of the date
781 date_stamp = pack_before.serialise()
783 # do the delete
784 for classname in self.classes.keys():
785 sql = "delete from %s__journal where date<%s and "\
786 "action<>'create'"%(classname, self.arg)
787 if __debug__:
788 print >>hyperdb.DEBUG, 'pack', (self, sql, date_stamp)
789 self.cursor.execute(sql, (date_stamp,))
791 def sql_commit(self):
792 ''' Actually commit to the database.
793 '''
794 self.conn.commit()
796 def commit(self):
797 ''' Commit the current transactions.
799 Save all data changed since the database was opened or since the
800 last commit() or rollback().
801 '''
802 if __debug__:
803 print >>hyperdb.DEBUG, 'commit', (self,)
805 # commit the database
806 self.sql_commit()
808 # now, do all the other transaction stuff
809 reindex = {}
810 for method, args in self.transactions:
811 reindex[method(*args)] = 1
813 # reindex the nodes that request it
814 for classname, nodeid in filter(None, reindex.keys()):
815 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
816 self.getclass(classname).index(nodeid)
818 # save the indexer state
819 self.indexer.save_index()
821 # clear out the transactions
822 self.transactions = []
824 def rollback(self):
825 ''' Reverse all actions from the current transaction.
827 Undo all the changes made since the database was opened or the last
828 commit() or rollback() was performed.
829 '''
830 if __debug__:
831 print >>hyperdb.DEBUG, 'rollback', (self,)
833 # roll back
834 self.conn.rollback()
836 # roll back "other" transaction stuff
837 for method, args in self.transactions:
838 # delete temporary files
839 if method == self.doStoreFile:
840 self.rollbackStoreFile(*args)
841 self.transactions = []
843 # clear the cache
844 self.clearCache()
846 def doSaveNode(self, classname, nodeid, node):
847 ''' dummy that just generates a reindex event
848 '''
849 # return the classname, nodeid so we reindex this content
850 return (classname, nodeid)
852 def close(self):
853 ''' Close off the connection.
854 '''
855 self.conn.close()
856 if self.lockfile is not None:
857 locking.release_lock(self.lockfile)
858 if self.lockfile is not None:
859 self.lockfile.close()
860 self.lockfile = None
862 #
863 # The base Class class
864 #
865 class Class(hyperdb.Class):
866 ''' The handle to a particular class of nodes in a hyperdatabase.
868 All methods except __repr__ and getnode must be implemented by a
869 concrete backend Class.
870 '''
872 def __init__(self, db, classname, **properties):
873 '''Create a new class with a given name and property specification.
875 'classname' must not collide with the name of an existing class,
876 or a ValueError is raised. The keyword arguments in 'properties'
877 must map names to property objects, or a TypeError is raised.
878 '''
879 if (properties.has_key('creation') or properties.has_key('activity')
880 or properties.has_key('creator')):
881 raise ValueError, '"creation", "activity" and "creator" are '\
882 'reserved'
884 self.classname = classname
885 self.properties = properties
886 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
887 self.key = ''
889 # should we journal changes (default yes)
890 self.do_journal = 1
892 # do the db-related init stuff
893 db.addclass(self)
895 self.auditors = {'create': [], 'set': [], 'retire': []}
896 self.reactors = {'create': [], 'set': [], 'retire': []}
898 def schema(self):
899 ''' A dumpable version of the schema that we can store in the
900 database
901 '''
902 return (self.key, [(x, repr(y)) for x,y in self.properties.items()])
904 def enableJournalling(self):
905 '''Turn journalling on for this class
906 '''
907 self.do_journal = 1
909 def disableJournalling(self):
910 '''Turn journalling off for this class
911 '''
912 self.do_journal = 0
914 # Editing nodes:
915 def create(self, **propvalues):
916 ''' Create a new node of this class and return its id.
918 The keyword arguments in 'propvalues' map property names to values.
920 The values of arguments must be acceptable for the types of their
921 corresponding properties or a TypeError is raised.
923 If this class has a key property, it must be present and its value
924 must not collide with other key strings or a ValueError is raised.
926 Any other properties on this class that are missing from the
927 'propvalues' dictionary are set to None.
929 If an id in a link or multilink property does not refer to a valid
930 node, an IndexError is raised.
931 '''
932 self.fireAuditors('create', None, propvalues)
933 newid = self.create_inner(**propvalues)
934 self.fireReactors('create', newid, None)
935 return newid
937 def create_inner(self, **propvalues):
938 ''' Called by create, in-between the audit and react calls.
939 '''
940 if propvalues.has_key('id'):
941 raise KeyError, '"id" is reserved'
943 if self.db.journaltag is None:
944 raise DatabaseError, 'Database open read-only'
946 if propvalues.has_key('creation') or propvalues.has_key('activity'):
947 raise KeyError, '"creation" and "activity" are reserved'
949 # new node's id
950 newid = self.db.newid(self.classname)
952 # validate propvalues
953 num_re = re.compile('^\d+$')
954 for key, value in propvalues.items():
955 if key == self.key:
956 try:
957 self.lookup(value)
958 except KeyError:
959 pass
960 else:
961 raise ValueError, 'node with key "%s" exists'%value
963 # try to handle this property
964 try:
965 prop = self.properties[key]
966 except KeyError:
967 raise KeyError, '"%s" has no property "%s"'%(self.classname,
968 key)
970 if value is not None and isinstance(prop, Link):
971 if type(value) != type(''):
972 raise ValueError, 'link value must be String'
973 link_class = self.properties[key].classname
974 # if it isn't a number, it's a key
975 if not num_re.match(value):
976 try:
977 value = self.db.classes[link_class].lookup(value)
978 except (TypeError, KeyError):
979 raise IndexError, 'new property "%s": %s not a %s'%(
980 key, value, link_class)
981 elif not self.db.getclass(link_class).hasnode(value):
982 raise IndexError, '%s has no node %s'%(link_class, value)
984 # save off the value
985 propvalues[key] = value
987 # register the link with the newly linked node
988 if self.do_journal and self.properties[key].do_journal:
989 self.db.addjournal(link_class, value, 'link',
990 (self.classname, newid, key))
992 elif isinstance(prop, Multilink):
993 if type(value) != type([]):
994 raise TypeError, 'new property "%s" not a list of ids'%key
996 # clean up and validate the list of links
997 link_class = self.properties[key].classname
998 l = []
999 for entry in value:
1000 if type(entry) != type(''):
1001 raise ValueError, '"%s" multilink value (%r) '\
1002 'must contain Strings'%(key, value)
1003 # if it isn't a number, it's a key
1004 if not num_re.match(entry):
1005 try:
1006 entry = self.db.classes[link_class].lookup(entry)
1007 except (TypeError, KeyError):
1008 raise IndexError, 'new property "%s": %s not a %s'%(
1009 key, entry, self.properties[key].classname)
1010 l.append(entry)
1011 value = l
1012 propvalues[key] = value
1014 # handle additions
1015 for nodeid in value:
1016 if not self.db.getclass(link_class).hasnode(nodeid):
1017 raise IndexError, '%s has no node %s'%(link_class,
1018 nodeid)
1019 # register the link with the newly linked node
1020 if self.do_journal and self.properties[key].do_journal:
1021 self.db.addjournal(link_class, nodeid, 'link',
1022 (self.classname, newid, key))
1024 elif isinstance(prop, String):
1025 if type(value) != type('') and type(value) != type(u''):
1026 raise TypeError, 'new property "%s" not a string'%key
1028 elif isinstance(prop, Password):
1029 if not isinstance(value, password.Password):
1030 raise TypeError, 'new property "%s" not a Password'%key
1032 elif isinstance(prop, Date):
1033 if value is not None and not isinstance(value, date.Date):
1034 raise TypeError, 'new property "%s" not a Date'%key
1036 elif isinstance(prop, Interval):
1037 if value is not None and not isinstance(value, date.Interval):
1038 raise TypeError, 'new property "%s" not an Interval'%key
1040 elif value is not None and isinstance(prop, Number):
1041 try:
1042 float(value)
1043 except ValueError:
1044 raise TypeError, 'new property "%s" not numeric'%key
1046 elif value is not None and isinstance(prop, Boolean):
1047 try:
1048 int(value)
1049 except ValueError:
1050 raise TypeError, 'new property "%s" not boolean'%key
1052 # make sure there's data where there needs to be
1053 for key, prop in self.properties.items():
1054 if propvalues.has_key(key):
1055 continue
1056 if key == self.key:
1057 raise ValueError, 'key property "%s" is required'%key
1058 if isinstance(prop, Multilink):
1059 propvalues[key] = []
1060 else:
1061 propvalues[key] = None
1063 # done
1064 self.db.addnode(self.classname, newid, propvalues)
1065 if self.do_journal:
1066 self.db.addjournal(self.classname, newid, 'create', {})
1068 return newid
1070 def export_list(self, propnames, nodeid):
1071 ''' Export a node - generate a list of CSV-able data in the order
1072 specified by propnames for the given node.
1073 '''
1074 properties = self.getprops()
1075 l = []
1076 for prop in propnames:
1077 proptype = properties[prop]
1078 value = self.get(nodeid, prop)
1079 # "marshal" data where needed
1080 if value is None:
1081 pass
1082 elif isinstance(proptype, hyperdb.Date):
1083 value = value.get_tuple()
1084 elif isinstance(proptype, hyperdb.Interval):
1085 value = value.get_tuple()
1086 elif isinstance(proptype, hyperdb.Password):
1087 value = str(value)
1088 l.append(repr(value))
1089 l.append(self.is_retired(nodeid))
1090 return l
1092 def import_list(self, propnames, proplist):
1093 ''' Import a node - all information including "id" is present and
1094 should not be sanity checked. Triggers are not triggered. The
1095 journal should be initialised using the "creator" and "created"
1096 information.
1098 Return the nodeid of the node imported.
1099 '''
1100 if self.db.journaltag is None:
1101 raise DatabaseError, 'Database open read-only'
1102 properties = self.getprops()
1104 # make the new node's property map
1105 d = {}
1106 retire = 0
1107 newid = None
1108 for i in range(len(propnames)):
1109 # Use eval to reverse the repr() used to output the CSV
1110 value = eval(proplist[i])
1112 # Figure the property for this column
1113 propname = propnames[i]
1115 # "unmarshal" where necessary
1116 if propname == 'id':
1117 newid = value
1118 continue
1119 elif propname == 'is retired':
1120 # is the item retired?
1121 if int(value):
1122 retire = 1
1123 continue
1125 prop = properties[propname]
1126 if value is None:
1127 # don't set Nones
1128 continue
1129 elif isinstance(prop, hyperdb.Date):
1130 value = date.Date(value)
1131 elif isinstance(prop, hyperdb.Interval):
1132 value = date.Interval(value)
1133 elif isinstance(prop, hyperdb.Password):
1134 pwd = password.Password()
1135 pwd.unpack(value)
1136 value = pwd
1137 d[propname] = value
1139 # get a new id if necessary
1140 if newid is None:
1141 newid = self.db.newid(self.classname)
1143 # retire?
1144 if retire:
1145 # use the arg for __retired__ to cope with any odd database type
1146 # conversion (hello, sqlite)
1147 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1148 self.db.arg, self.db.arg)
1149 if __debug__:
1150 print >>hyperdb.DEBUG, 'retire', (self, sql, newid)
1151 self.db.cursor.execute(sql, (1, newid))
1153 # add the node and journal
1154 self.db.addnode(self.classname, newid, d)
1156 # extract the extraneous journalling gumpf and nuke it
1157 if d.has_key('creator'):
1158 creator = d['creator']
1159 del d['creator']
1160 else:
1161 creator = None
1162 if d.has_key('creation'):
1163 creation = d['creation']
1164 del d['creation']
1165 else:
1166 creation = None
1167 if d.has_key('activity'):
1168 del d['activity']
1169 self.db.addjournal(self.classname, newid, 'create', {}, creator,
1170 creation)
1171 return newid
1173 _marker = []
1174 def get(self, nodeid, propname, default=_marker, cache=1):
1175 '''Get the value of a property on an existing node of this class.
1177 'nodeid' must be the id of an existing node of this class or an
1178 IndexError is raised. 'propname' must be the name of a property
1179 of this class or a KeyError is raised.
1181 'cache' indicates whether the transaction cache should be queried
1182 for the node. If the node has been modified and you need to
1183 determine what its values prior to modification are, you need to
1184 set cache=0.
1185 '''
1186 if propname == 'id':
1187 return nodeid
1189 # get the node's dict
1190 d = self.db.getnode(self.classname, nodeid)
1192 if propname == 'creation':
1193 if d.has_key('creation'):
1194 return d['creation']
1195 else:
1196 return date.Date()
1197 if propname == 'activity':
1198 if d.has_key('activity'):
1199 return d['activity']
1200 else:
1201 return date.Date()
1202 if propname == 'creator':
1203 if d.has_key('creator'):
1204 return d['creator']
1205 else:
1206 return self.db.curuserid
1208 # get the property (raises KeyErorr if invalid)
1209 prop = self.properties[propname]
1211 if not d.has_key(propname):
1212 if default is self._marker:
1213 if isinstance(prop, Multilink):
1214 return []
1215 else:
1216 return None
1217 else:
1218 return default
1220 # don't pass our list to other code
1221 if isinstance(prop, Multilink):
1222 return d[propname][:]
1224 return d[propname]
1226 def getnode(self, nodeid, cache=1):
1227 ''' Return a convenience wrapper for the node.
1229 'nodeid' must be the id of an existing node of this class or an
1230 IndexError is raised.
1232 'cache' indicates whether the transaction cache should be queried
1233 for the node. If the node has been modified and you need to
1234 determine what its values prior to modification are, you need to
1235 set cache=0.
1236 '''
1237 return Node(self, nodeid, cache=cache)
1239 def set(self, nodeid, **propvalues):
1240 '''Modify a property on an existing node of this class.
1242 'nodeid' must be the id of an existing node of this class or an
1243 IndexError is raised.
1245 Each key in 'propvalues' must be the name of a property of this
1246 class or a KeyError is raised.
1248 All values in 'propvalues' must be acceptable types for their
1249 corresponding properties or a TypeError is raised.
1251 If the value of the key property is set, it must not collide with
1252 other key strings or a ValueError is raised.
1254 If the value of a Link or Multilink property contains an invalid
1255 node id, a ValueError is raised.
1256 '''
1257 if not propvalues:
1258 return propvalues
1260 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1261 raise KeyError, '"creation" and "activity" are reserved'
1263 if propvalues.has_key('id'):
1264 raise KeyError, '"id" is reserved'
1266 if self.db.journaltag is None:
1267 raise DatabaseError, 'Database open read-only'
1269 self.fireAuditors('set', nodeid, propvalues)
1270 # Take a copy of the node dict so that the subsequent set
1271 # operation doesn't modify the oldvalues structure.
1272 # XXX used to try the cache here first
1273 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1275 node = self.db.getnode(self.classname, nodeid)
1276 if self.is_retired(nodeid):
1277 raise IndexError, 'Requested item is retired'
1278 num_re = re.compile('^\d+$')
1280 # if the journal value is to be different, store it in here
1281 journalvalues = {}
1283 # remember the add/remove stuff for multilinks, making it easier
1284 # for the Database layer to do its stuff
1285 multilink_changes = {}
1287 for propname, value in propvalues.items():
1288 # check to make sure we're not duplicating an existing key
1289 if propname == self.key and node[propname] != value:
1290 try:
1291 self.lookup(value)
1292 except KeyError:
1293 pass
1294 else:
1295 raise ValueError, 'node with key "%s" exists'%value
1297 # this will raise the KeyError if the property isn't valid
1298 # ... we don't use getprops() here because we only care about
1299 # the writeable properties.
1300 try:
1301 prop = self.properties[propname]
1302 except KeyError:
1303 raise KeyError, '"%s" has no property named "%s"'%(
1304 self.classname, propname)
1306 # if the value's the same as the existing value, no sense in
1307 # doing anything
1308 current = node.get(propname, None)
1309 if value == current:
1310 del propvalues[propname]
1311 continue
1312 journalvalues[propname] = current
1314 # do stuff based on the prop type
1315 if isinstance(prop, Link):
1316 link_class = prop.classname
1317 # if it isn't a number, it's a key
1318 if value is not None and not isinstance(value, type('')):
1319 raise ValueError, 'property "%s" link value be a string'%(
1320 propname)
1321 if isinstance(value, type('')) and not num_re.match(value):
1322 try:
1323 value = self.db.classes[link_class].lookup(value)
1324 except (TypeError, KeyError):
1325 raise IndexError, 'new property "%s": %s not a %s'%(
1326 propname, value, prop.classname)
1328 if (value is not None and
1329 not self.db.getclass(link_class).hasnode(value)):
1330 raise IndexError, '%s has no node %s'%(link_class, value)
1332 if self.do_journal and prop.do_journal:
1333 # register the unlink with the old linked node
1334 if node[propname] is not None:
1335 self.db.addjournal(link_class, node[propname], 'unlink',
1336 (self.classname, nodeid, propname))
1338 # register the link with the newly linked node
1339 if value is not None:
1340 self.db.addjournal(link_class, value, 'link',
1341 (self.classname, nodeid, propname))
1343 elif isinstance(prop, Multilink):
1344 if type(value) != type([]):
1345 raise TypeError, 'new property "%s" not a list of'\
1346 ' ids'%propname
1347 link_class = self.properties[propname].classname
1348 l = []
1349 for entry in value:
1350 # if it isn't a number, it's a key
1351 if type(entry) != type(''):
1352 raise ValueError, 'new property "%s" link value ' \
1353 'must be a string'%propname
1354 if not num_re.match(entry):
1355 try:
1356 entry = self.db.classes[link_class].lookup(entry)
1357 except (TypeError, KeyError):
1358 raise IndexError, 'new property "%s": %s not a %s'%(
1359 propname, entry,
1360 self.properties[propname].classname)
1361 l.append(entry)
1362 value = l
1363 propvalues[propname] = value
1365 # figure the journal entry for this property
1366 add = []
1367 remove = []
1369 # handle removals
1370 if node.has_key(propname):
1371 l = node[propname]
1372 else:
1373 l = []
1374 for id in l[:]:
1375 if id in value:
1376 continue
1377 # register the unlink with the old linked node
1378 if self.do_journal and self.properties[propname].do_journal:
1379 self.db.addjournal(link_class, id, 'unlink',
1380 (self.classname, nodeid, propname))
1381 l.remove(id)
1382 remove.append(id)
1384 # handle additions
1385 for id in value:
1386 if not self.db.getclass(link_class).hasnode(id):
1387 raise IndexError, '%s has no node %s'%(link_class, id)
1388 if id in l:
1389 continue
1390 # register the link with the newly linked node
1391 if self.do_journal and self.properties[propname].do_journal:
1392 self.db.addjournal(link_class, id, 'link',
1393 (self.classname, nodeid, propname))
1394 l.append(id)
1395 add.append(id)
1397 # figure the journal entry
1398 l = []
1399 if add:
1400 l.append(('+', add))
1401 if remove:
1402 l.append(('-', remove))
1403 multilink_changes[propname] = (add, remove)
1404 if l:
1405 journalvalues[propname] = tuple(l)
1407 elif isinstance(prop, String):
1408 if value is not None and type(value) != type('') and type(value) != type(u''):
1409 raise TypeError, 'new property "%s" not a string'%propname
1411 elif isinstance(prop, Password):
1412 if not isinstance(value, password.Password):
1413 raise TypeError, 'new property "%s" not a Password'%propname
1414 propvalues[propname] = value
1416 elif value is not None and isinstance(prop, Date):
1417 if not isinstance(value, date.Date):
1418 raise TypeError, 'new property "%s" not a Date'% propname
1419 propvalues[propname] = value
1421 elif value is not None and isinstance(prop, Interval):
1422 if not isinstance(value, date.Interval):
1423 raise TypeError, 'new property "%s" not an '\
1424 'Interval'%propname
1425 propvalues[propname] = value
1427 elif value is not None and isinstance(prop, Number):
1428 try:
1429 float(value)
1430 except ValueError:
1431 raise TypeError, 'new property "%s" not numeric'%propname
1433 elif value is not None and isinstance(prop, Boolean):
1434 try:
1435 int(value)
1436 except ValueError:
1437 raise TypeError, 'new property "%s" not boolean'%propname
1439 # nothing to do?
1440 if not propvalues:
1441 return propvalues
1443 # do the set, and journal it
1444 self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
1446 if self.do_journal:
1447 self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
1449 self.fireReactors('set', nodeid, oldvalues)
1451 return propvalues
1453 def retire(self, nodeid):
1454 '''Retire a node.
1456 The properties on the node remain available from the get() method,
1457 and the node's id is never reused.
1459 Retired nodes are not returned by the find(), list(), or lookup()
1460 methods, and other nodes may reuse the values of their key properties.
1461 '''
1462 if self.db.journaltag is None:
1463 raise DatabaseError, 'Database open read-only'
1465 self.fireAuditors('retire', nodeid, None)
1467 # use the arg for __retired__ to cope with any odd database type
1468 # conversion (hello, sqlite)
1469 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1470 self.db.arg, self.db.arg)
1471 if __debug__:
1472 print >>hyperdb.DEBUG, 'retire', (self, sql, nodeid)
1473 self.db.cursor.execute(sql, (1, nodeid))
1475 self.fireReactors('retire', nodeid, None)
1477 def is_retired(self, nodeid):
1478 '''Return true if the node is rerired
1479 '''
1480 sql = 'select __retired__ from _%s where id=%s'%(self.classname,
1481 self.db.arg)
1482 if __debug__:
1483 print >>hyperdb.DEBUG, 'is_retired', (self, sql, nodeid)
1484 self.db.cursor.execute(sql, (nodeid,))
1485 return int(self.db.sql_fetchone()[0])
1487 def destroy(self, nodeid):
1488 '''Destroy a node.
1490 WARNING: this method should never be used except in extremely rare
1491 situations where there could never be links to the node being
1492 deleted
1493 WARNING: use retire() instead
1494 WARNING: the properties of this node will not be available ever again
1495 WARNING: really, use retire() instead
1497 Well, I think that's enough warnings. This method exists mostly to
1498 support the session storage of the cgi interface.
1500 The node is completely removed from the hyperdb, including all journal
1501 entries. It will no longer be available, and will generally break code
1502 if there are any references to the node.
1503 '''
1504 if self.db.journaltag is None:
1505 raise DatabaseError, 'Database open read-only'
1506 self.db.destroynode(self.classname, nodeid)
1508 def history(self, nodeid):
1509 '''Retrieve the journal of edits on a particular node.
1511 'nodeid' must be the id of an existing node of this class or an
1512 IndexError is raised.
1514 The returned list contains tuples of the form
1516 (nodeid, date, tag, action, params)
1518 'date' is a Timestamp object specifying the time of the change and
1519 'tag' is the journaltag specified when the database was opened.
1520 '''
1521 if not self.do_journal:
1522 raise ValueError, 'Journalling is disabled for this class'
1523 return self.db.getjournal(self.classname, nodeid)
1525 # Locating nodes:
1526 def hasnode(self, nodeid):
1527 '''Determine if the given nodeid actually exists
1528 '''
1529 return self.db.hasnode(self.classname, nodeid)
1531 def setkey(self, propname):
1532 '''Select a String property of this class to be the key property.
1534 'propname' must be the name of a String property of this class or
1535 None, or a TypeError is raised. The values of the key property on
1536 all existing nodes must be unique or a ValueError is raised.
1537 '''
1538 # XXX create an index on the key prop column
1539 prop = self.getprops()[propname]
1540 if not isinstance(prop, String):
1541 raise TypeError, 'key properties must be String'
1542 self.key = propname
1544 def getkey(self):
1545 '''Return the name of the key property for this class or None.'''
1546 return self.key
1548 def labelprop(self, default_to_id=0):
1549 ''' Return the property name for a label for the given node.
1551 This method attempts to generate a consistent label for the node.
1552 It tries the following in order:
1553 1. key property
1554 2. "name" property
1555 3. "title" property
1556 4. first property from the sorted property name list
1557 '''
1558 k = self.getkey()
1559 if k:
1560 return k
1561 props = self.getprops()
1562 if props.has_key('name'):
1563 return 'name'
1564 elif props.has_key('title'):
1565 return 'title'
1566 if default_to_id:
1567 return 'id'
1568 props = props.keys()
1569 props.sort()
1570 return props[0]
1572 def lookup(self, keyvalue):
1573 '''Locate a particular node by its key property and return its id.
1575 If this class has no key property, a TypeError is raised. If the
1576 'keyvalue' matches one of the values for the key property among
1577 the nodes in this class, the matching node's id is returned;
1578 otherwise a KeyError is raised.
1579 '''
1580 if not self.key:
1581 raise TypeError, 'No key property set for class %s'%self.classname
1583 # use the arg to handle any odd database type conversion (hello,
1584 # sqlite)
1585 sql = "select id from _%s where _%s=%s and __retired__ <> %s"%(
1586 self.classname, self.key, self.db.arg, self.db.arg)
1587 self.db.sql(sql, (keyvalue, 1))
1589 # see if there was a result that's not retired
1590 row = self.db.sql_fetchone()
1591 if not row:
1592 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1593 keyvalue, self.classname)
1595 # return the id
1596 return row[0]
1598 def find(self, **propspec):
1599 '''Get the ids of nodes in this class which link to the given nodes.
1601 'propspec' consists of keyword args propname=nodeid or
1602 propname={nodeid:1, }
1603 'propname' must be the name of a property in this class, or a
1604 KeyError is raised. That property must be a Link or Multilink
1605 property, or a TypeError is raised.
1607 Any node in this class whose 'propname' property links to any of the
1608 nodeids will be returned. Used by the full text indexing, which knows
1609 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1610 issues:
1612 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1613 '''
1614 if __debug__:
1615 print >>hyperdb.DEBUG, 'find', (self, propspec)
1617 # shortcut
1618 if not propspec:
1619 return []
1621 # validate the args
1622 props = self.getprops()
1623 propspec = propspec.items()
1624 for propname, nodeids in propspec:
1625 # check the prop is OK
1626 prop = props[propname]
1627 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1628 raise TypeError, "'%s' not a Link/Multilink property"%propname
1630 # first, links
1631 where = []
1632 allvalues = ()
1633 a = self.db.arg
1634 for prop, values in propspec:
1635 if not isinstance(props[prop], hyperdb.Link):
1636 continue
1637 if type(values) is type(''):
1638 allvalues += (values,)
1639 where.append('_%s = %s'%(prop, a))
1640 else:
1641 allvalues += tuple(values.keys())
1642 where.append('_%s in (%s)'%(prop, ','.join([a]*len(values))))
1643 tables = []
1644 if where:
1645 tables.append('select id as nodeid from _%s where %s'%(
1646 self.classname, ' and '.join(where)))
1648 # now multilinks
1649 for prop, values in propspec:
1650 if not isinstance(props[prop], hyperdb.Multilink):
1651 continue
1652 if type(values) is type(''):
1653 allvalues += (values,)
1654 s = a
1655 else:
1656 allvalues += tuple(values.keys())
1657 s = ','.join([a]*len(values))
1658 tables.append('select nodeid from %s_%s where linkid in (%s)'%(
1659 self.classname, prop, s))
1660 sql = '\nunion\n'.join(tables)
1661 self.db.sql(sql, allvalues)
1662 l = [x[0] for x in self.db.sql_fetchall()]
1663 if __debug__:
1664 print >>hyperdb.DEBUG, 'find ... ', l
1665 return l
1667 def stringFind(self, **requirements):
1668 '''Locate a particular node by matching a set of its String
1669 properties in a caseless search.
1671 If the property is not a String property, a TypeError is raised.
1673 The return is a list of the id of all nodes that match.
1674 '''
1675 where = []
1676 args = []
1677 for propname in requirements.keys():
1678 prop = self.properties[propname]
1679 if isinstance(not prop, String):
1680 raise TypeError, "'%s' not a String property"%propname
1681 where.append(propname)
1682 args.append(requirements[propname].lower())
1684 # generate the where clause
1685 s = ' and '.join(['_%s=%s'%(col, self.db.arg) for col in where])
1686 sql = 'select id from _%s where %s'%(self.classname, s)
1687 self.db.sql(sql, tuple(args))
1688 l = [x[0] for x in self.db.sql_fetchall()]
1689 if __debug__:
1690 print >>hyperdb.DEBUG, 'find ... ', l
1691 return l
1693 def list(self):
1694 ''' Return a list of the ids of the active nodes in this class.
1695 '''
1696 return self.getnodeids(retired=0)
1698 def getnodeids(self, retired=None):
1699 ''' Retrieve all the ids of the nodes for a particular Class.
1701 Set retired=None to get all nodes. Otherwise it'll get all the
1702 retired or non-retired nodes, depending on the flag.
1703 '''
1704 # flip the sense of the flag if we don't want all of them
1705 if retired is not None:
1706 retired = not retired
1707 sql = 'select id from _%s where __retired__ <> %s'%(self.classname,
1708 self.db.arg)
1709 if __debug__:
1710 print >>hyperdb.DEBUG, 'getnodeids', (self, sql, retired)
1711 self.db.cursor.execute(sql, (retired,))
1712 return [x[0] for x in self.db.cursor.fetchall()]
1714 def filter(self, search_matches, filterspec, sort=(None,None),
1715 group=(None,None)):
1716 ''' Return a list of the ids of the active nodes in this class that
1717 match the 'filter' spec, sorted by the group spec and then the
1718 sort spec
1720 "filterspec" is {propname: value(s)}
1721 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1722 and prop is a prop name or None
1723 "search_matches" is {nodeid: marker}
1725 The filter must match all properties specificed - but if the
1726 property value to match is a list, any one of the values in the
1727 list may match for that property to match.
1728 '''
1729 # just don't bother if the full-text search matched diddly
1730 if search_matches == {}:
1731 return []
1733 cn = self.classname
1735 timezone = self.db.getUserTimezone()
1737 # figure the WHERE clause from the filterspec
1738 props = self.getprops()
1739 frum = ['_'+cn]
1740 where = []
1741 args = []
1742 a = self.db.arg
1743 for k, v in filterspec.items():
1744 propclass = props[k]
1745 # now do other where clause stuff
1746 if isinstance(propclass, Multilink):
1747 tn = '%s_%s'%(cn, k)
1748 frum.append(tn)
1749 if isinstance(v, type([])):
1750 s = ','.join([a for x in v])
1751 where.append('id=%s.nodeid and %s.linkid in (%s)'%(tn,tn,s))
1752 args = args + v
1753 else:
1754 where.append('id=%s.nodeid and %s.linkid = %s'%(tn, tn, a))
1755 args.append(v)
1756 elif k == 'id':
1757 if isinstance(v, type([])):
1758 s = ','.join([a for x in v])
1759 where.append('%s in (%s)'%(k, s))
1760 args = args + v
1761 else:
1762 where.append('%s=%s'%(k, a))
1763 args.append(v)
1764 elif isinstance(propclass, String):
1765 if not isinstance(v, type([])):
1766 v = [v]
1768 # Quote the bits in the string that need it and then embed
1769 # in a "substring" search. Note - need to quote the '%' so
1770 # they make it through the python layer happily
1771 v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
1773 # now add to the where clause
1774 where.append(' or '.join(["_%s LIKE '%s'"%(k, s) for s in v]))
1775 # note: args are embedded in the query string now
1776 elif isinstance(propclass, Link):
1777 if isinstance(v, type([])):
1778 if '-1' in v:
1779 v.remove('-1')
1780 xtra = ' or _%s is NULL'%k
1781 else:
1782 xtra = ''
1783 if v:
1784 s = ','.join([a for x in v])
1785 where.append('(_%s in (%s)%s)'%(k, s, xtra))
1786 args = args + v
1787 else:
1788 where.append('_%s is NULL'%k)
1789 else:
1790 if v == '-1':
1791 v = None
1792 where.append('_%s is NULL'%k)
1793 else:
1794 where.append('_%s=%s'%(k, a))
1795 args.append(v)
1796 elif isinstance(propclass, Date):
1797 if isinstance(v, type([])):
1798 s = ','.join([a for x in v])
1799 where.append('_%s in (%s)'%(k, s))
1800 args = args + [date.Date(x).serialise() for x in v]
1801 else:
1802 try:
1803 # Try to filter on range of dates
1804 date_rng = Range(v, date.Date, offset=timezone)
1805 if (date_rng.from_value):
1806 where.append('_%s > %s'%(k, a))
1807 args.append(date_rng.from_value.serialise())
1808 if (date_rng.to_value):
1809 where.append('_%s < %s'%(k, a))
1810 args.append(date_rng.to_value.serialise())
1811 except ValueError:
1812 # If range creation fails - ignore that search parameter
1813 pass
1814 elif isinstance(propclass, Interval):
1815 if isinstance(v, type([])):
1816 s = ','.join([a for x in v])
1817 where.append('_%s in (%s)'%(k, s))
1818 args = args + [date.Interval(x).serialise() for x in v]
1819 else:
1820 where.append('_%s=%s'%(k, a))
1821 args.append(date.Interval(v).serialise())
1822 else:
1823 if isinstance(v, type([])):
1824 s = ','.join([a for x in v])
1825 where.append('_%s in (%s)'%(k, s))
1826 args = args + v
1827 else:
1828 where.append('_%s=%s'%(k, a))
1829 args.append(v)
1831 # add results of full text search
1832 if search_matches is not None:
1833 v = search_matches.keys()
1834 s = ','.join([a for x in v])
1835 where.append('id in (%s)'%s)
1836 args = args + v
1838 # "grouping" is just the first-order sorting in the SQL fetch
1839 # can modify it...)
1840 orderby = []
1841 ordercols = []
1842 if group[0] is not None and group[1] is not None:
1843 if group[0] != '-':
1844 orderby.append('_'+group[1])
1845 ordercols.append('_'+group[1])
1846 else:
1847 orderby.append('_'+group[1]+' desc')
1848 ordercols.append('_'+group[1])
1850 # now add in the sorting
1851 group = ''
1852 if sort[0] is not None and sort[1] is not None:
1853 direction, colname = sort
1854 if direction != '-':
1855 if colname == 'id':
1856 orderby.append(colname)
1857 else:
1858 orderby.append('_'+colname)
1859 ordercols.append('_'+colname)
1860 else:
1861 if colname == 'id':
1862 orderby.append(colname+' desc')
1863 ordercols.append(colname)
1864 else:
1865 orderby.append('_'+colname+' desc')
1866 ordercols.append('_'+colname)
1868 # construct the SQL
1869 frum = ','.join(frum)
1870 if where:
1871 where = ' where ' + (' and '.join(where))
1872 else:
1873 where = ''
1874 cols = ['id']
1875 if orderby:
1876 cols = cols + ordercols
1877 order = ' order by %s'%(','.join(orderby))
1878 else:
1879 order = ''
1880 cols = ','.join(cols)
1881 sql = 'select %s from %s %s%s%s'%(cols, frum, where, group, order)
1882 args = tuple(args)
1883 if __debug__:
1884 print >>hyperdb.DEBUG, 'filter', (self, sql, args)
1885 self.db.cursor.execute(sql, args)
1886 l = self.db.cursor.fetchall()
1888 # return the IDs (the first column)
1889 # XXX The filter(None, l) bit is sqlite-specific... if there's _NO_
1890 # XXX matches to a fetch, it returns NULL instead of nothing!?!
1891 return filter(None, [row[0] for row in l])
1893 def count(self):
1894 '''Get the number of nodes in this class.
1896 If the returned integer is 'numnodes', the ids of all the nodes
1897 in this class run from 1 to numnodes, and numnodes+1 will be the
1898 id of the next node to be created in this class.
1899 '''
1900 return self.db.countnodes(self.classname)
1902 # Manipulating properties:
1903 def getprops(self, protected=1):
1904 '''Return a dictionary mapping property names to property objects.
1905 If the "protected" flag is true, we include protected properties -
1906 those which may not be modified.
1907 '''
1908 d = self.properties.copy()
1909 if protected:
1910 d['id'] = String()
1911 d['creation'] = hyperdb.Date()
1912 d['activity'] = hyperdb.Date()
1913 d['creator'] = hyperdb.Link('user')
1914 return d
1916 def addprop(self, **properties):
1917 '''Add properties to this class.
1919 The keyword arguments in 'properties' must map names to property
1920 objects, or a TypeError is raised. None of the keys in 'properties'
1921 may collide with the names of existing properties, or a ValueError
1922 is raised before any properties have been added.
1923 '''
1924 for key in properties.keys():
1925 if self.properties.has_key(key):
1926 raise ValueError, key
1927 self.properties.update(properties)
1929 def index(self, nodeid):
1930 '''Add (or refresh) the node to search indexes
1931 '''
1932 # find all the String properties that have indexme
1933 for prop, propclass in self.getprops().items():
1934 if isinstance(propclass, String) and propclass.indexme:
1935 try:
1936 value = str(self.get(nodeid, prop))
1937 except IndexError:
1938 # node no longer exists - entry should be removed
1939 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1940 else:
1941 # and index them under (classname, nodeid, property)
1942 self.db.indexer.add_text((self.classname, nodeid, prop),
1943 value)
1946 #
1947 # Detector interface
1948 #
1949 def audit(self, event, detector):
1950 '''Register a detector
1951 '''
1952 l = self.auditors[event]
1953 if detector not in l:
1954 self.auditors[event].append(detector)
1956 def fireAuditors(self, action, nodeid, newvalues):
1957 '''Fire all registered auditors.
1958 '''
1959 for audit in self.auditors[action]:
1960 audit(self.db, self, nodeid, newvalues)
1962 def react(self, event, detector):
1963 '''Register a detector
1964 '''
1965 l = self.reactors[event]
1966 if detector not in l:
1967 self.reactors[event].append(detector)
1969 def fireReactors(self, action, nodeid, oldvalues):
1970 '''Fire all registered reactors.
1971 '''
1972 for react in self.reactors[action]:
1973 react(self.db, self, nodeid, oldvalues)
1975 class FileClass(Class, hyperdb.FileClass):
1976 '''This class defines a large chunk of data. To support this, it has a
1977 mandatory String property "content" which is typically saved off
1978 externally to the hyperdb.
1980 The default MIME type of this data is defined by the
1981 "default_mime_type" class attribute, which may be overridden by each
1982 node if the class defines a "type" String property.
1983 '''
1984 default_mime_type = 'text/plain'
1986 def create(self, **propvalues):
1987 ''' snaffle the file propvalue and store in a file
1988 '''
1989 # we need to fire the auditors now, or the content property won't
1990 # be in propvalues for the auditors to play with
1991 self.fireAuditors('create', None, propvalues)
1993 # now remove the content property so it's not stored in the db
1994 content = propvalues['content']
1995 del propvalues['content']
1997 # do the database create
1998 newid = Class.create_inner(self, **propvalues)
2000 # fire reactors
2001 self.fireReactors('create', newid, None)
2003 # store off the content as a file
2004 self.db.storefile(self.classname, newid, None, content)
2005 return newid
2007 def import_list(self, propnames, proplist):
2008 ''' Trap the "content" property...
2009 '''
2010 # dupe this list so we don't affect others
2011 propnames = propnames[:]
2013 # extract the "content" property from the proplist
2014 i = propnames.index('content')
2015 content = eval(proplist[i])
2016 del propnames[i]
2017 del proplist[i]
2019 # do the normal import
2020 newid = Class.import_list(self, propnames, proplist)
2022 # save off the "content" file
2023 self.db.storefile(self.classname, newid, None, content)
2024 return newid
2026 _marker = []
2027 def get(self, nodeid, propname, default=_marker, cache=1):
2028 ''' trap the content propname and get it from the file
2029 '''
2030 poss_msg = 'Possibly a access right configuration problem.'
2031 if propname == 'content':
2032 try:
2033 return self.db.getfile(self.classname, nodeid, None)
2034 except IOError, (strerror):
2035 # BUG: by catching this we donot see an error in the log.
2036 return 'ERROR reading file: %s%s\n%s\n%s'%(
2037 self.classname, nodeid, poss_msg, strerror)
2038 if default is not self._marker:
2039 return Class.get(self, nodeid, propname, default, cache=cache)
2040 else:
2041 return Class.get(self, nodeid, propname, cache=cache)
2043 def getprops(self, protected=1):
2044 ''' In addition to the actual properties on the node, these methods
2045 provide the "content" property. If the "protected" flag is true,
2046 we include protected properties - those which may not be
2047 modified.
2048 '''
2049 d = Class.getprops(self, protected=protected).copy()
2050 d['content'] = hyperdb.String()
2051 return d
2053 def index(self, nodeid):
2054 ''' Index the node in the search index.
2056 We want to index the content in addition to the normal String
2057 property indexing.
2058 '''
2059 # perform normal indexing
2060 Class.index(self, nodeid)
2062 # get the content to index
2063 content = self.get(nodeid, 'content')
2065 # figure the mime type
2066 if self.properties.has_key('type'):
2067 mime_type = self.get(nodeid, 'type')
2068 else:
2069 mime_type = self.default_mime_type
2071 # and index!
2072 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
2073 mime_type)
2075 # XXX deviation from spec - was called ItemClass
2076 class IssueClass(Class, roundupdb.IssueClass):
2077 # Overridden methods:
2078 def __init__(self, db, classname, **properties):
2079 '''The newly-created class automatically includes the "messages",
2080 "files", "nosy", and "superseder" properties. If the 'properties'
2081 dictionary attempts to specify any of these properties or a
2082 "creation" or "activity" property, a ValueError is raised.
2083 '''
2084 if not properties.has_key('title'):
2085 properties['title'] = hyperdb.String(indexme='yes')
2086 if not properties.has_key('messages'):
2087 properties['messages'] = hyperdb.Multilink("msg")
2088 if not properties.has_key('files'):
2089 properties['files'] = hyperdb.Multilink("file")
2090 if not properties.has_key('nosy'):
2091 # note: journalling is turned off as it really just wastes
2092 # space. this behaviour may be overridden in an instance
2093 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2094 if not properties.has_key('superseder'):
2095 properties['superseder'] = hyperdb.Multilink(classname)
2096 Class.__init__(self, db, classname, **properties)