1 # $Id: rdbms_common.py,v 1.15 2002-09-24 01:59:28 richard Exp $
3 # standard python modules
4 import sys, os, time, re, errno, weakref, copy
6 # roundup modules
7 from roundup import hyperdb, date, password, roundupdb, security
8 from roundup.hyperdb import String, Password, Date, Interval, Link, \
9 Multilink, DatabaseError, Boolean, Number
11 # support
12 from blobfiles import FileStorage
13 from roundup.indexer import Indexer
14 from sessions import Sessions
16 # number of rows to keep in memory
17 ROW_CACHE_SIZE = 100
19 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
20 ''' Wrapper around an SQL database that presents a hyperdb interface.
22 - some functionality is specific to the actual SQL database, hence
23 the sql_* methods that are NotImplemented
24 - we keep a cache of the latest ROW_CACHE_SIZE row fetches.
25 '''
26 def __init__(self, config, journaltag=None):
27 ''' Open the database and load the schema from it.
28 '''
29 self.config, self.journaltag = config, journaltag
30 self.dir = config.DATABASE
31 self.classes = {}
32 self.indexer = Indexer(self.dir)
33 self.sessions = Sessions(self.config)
34 self.security = security.Security(self)
36 # additional transaction support for external files and the like
37 self.transactions = []
39 # keep a cache of the N most recently retrieved rows of any kind
40 # (classname, nodeid) = row
41 self.cache = {}
42 self.cache_lru = []
44 # open a connection to the database, creating the "conn" attribute
45 self.open_connection()
47 def clearCache(self):
48 self.cache = {}
49 self.cache_lru = []
51 def open_connection(self):
52 ''' Open a connection to the database, creating it if necessary
53 '''
54 raise NotImplemented
56 def sql(self, sql, args=None):
57 ''' Execute the sql with the optional args.
58 '''
59 if __debug__:
60 print >>hyperdb.DEBUG, (self, sql, args)
61 if args:
62 self.cursor.execute(sql, args)
63 else:
64 self.cursor.execute(sql)
66 def sql_fetchone(self):
67 ''' Fetch a single row. If there's nothing to fetch, return None.
68 '''
69 raise NotImplemented
71 def sql_stringquote(self, value):
72 ''' Quote the string so it's safe to put in the 'sql quotes'
73 '''
74 return re.sub("'", "''", str(value))
76 def save_dbschema(self, schema):
77 ''' Save the schema definition that the database currently implements
78 '''
79 raise NotImplemented
81 def load_dbschema(self):
82 ''' Load the schema definition that the database currently implements
83 '''
84 raise NotImplemented
86 def post_init(self):
87 ''' Called once the schema initialisation has finished.
89 We should now confirm that the schema defined by our "classes"
90 attribute actually matches the schema in the database.
91 '''
92 # now detect changes in the schema
93 save = 0
94 for classname, spec in self.classes.items():
95 if self.database_schema.has_key(classname):
96 dbspec = self.database_schema[classname]
97 if self.update_class(spec, dbspec):
98 self.database_schema[classname] = spec.schema()
99 save = 1
100 else:
101 self.create_class(spec)
102 self.database_schema[classname] = spec.schema()
103 save = 1
105 for classname in self.database_schema.keys():
106 if not self.classes.has_key(classname):
107 self.drop_class(classname)
109 # update the database version of the schema
110 if save:
111 self.sql('delete from schema')
112 self.save_dbschema(self.database_schema)
114 # reindex the db if necessary
115 if self.indexer.should_reindex():
116 self.reindex()
118 # commit
119 self.conn.commit()
121 # figure the "curuserid"
122 if self.journaltag is None:
123 self.curuserid = None
124 elif self.journaltag == 'admin':
125 # admin user may not exist, but always has ID 1
126 self.curuserid = '1'
127 else:
128 self.curuserid = self.user.lookup(self.journaltag)
130 def reindex(self):
131 for klass in self.classes.values():
132 for nodeid in klass.list():
133 klass.index(nodeid)
134 self.indexer.save_index()
136 def determine_columns(self, properties):
137 ''' Figure the column names and multilink properties from the spec
139 "properties" is a list of (name, prop) where prop may be an
140 instance of a hyperdb "type" _or_ a string repr of that type.
141 '''
142 cols = ['_activity', '_creator', '_creation']
143 mls = []
144 # add the multilinks separately
145 for col, prop in properties:
146 if isinstance(prop, Multilink):
147 mls.append(col)
148 elif isinstance(prop, type('')) and prop.find('Multilink') != -1:
149 mls.append(col)
150 else:
151 cols.append('_'+col)
152 cols.sort()
153 return cols, mls
155 def update_class(self, spec, dbspec):
156 ''' Determine the differences between the current spec and the
157 database version of the spec, and update where necessary
158 '''
159 spec_schema = spec.schema()
160 if spec_schema == dbspec:
161 # no save needed for this one
162 return 0
163 if __debug__:
164 print >>hyperdb.DEBUG, 'update_class FIRING'
166 # key property changed?
167 if dbspec[0] != spec_schema[0]:
168 if __debug__:
169 print >>hyperdb.DEBUG, 'update_class setting keyprop', `spec[0]`
170 # XXX turn on indexing for the key property
172 # dict 'em up
173 spec_propnames,spec_props = [],{}
174 for propname,prop in spec_schema[1]:
175 spec_propnames.append(propname)
176 spec_props[propname] = prop
177 dbspec_propnames,dbspec_props = [],{}
178 for propname,prop in dbspec[1]:
179 dbspec_propnames.append(propname)
180 dbspec_props[propname] = prop
182 # now compare
183 for propname in spec_propnames:
184 prop = spec_props[propname]
185 if dbspec_props.has_key(propname) and prop==dbspec_props[propname]:
186 continue
187 if __debug__:
188 print >>hyperdb.DEBUG, 'update_class ADD', (propname, prop)
190 if not dbspec_props.has_key(propname):
191 # add the property
192 if isinstance(prop, Multilink):
193 # all we have to do here is create a new table, easy!
194 self.create_multilink_table(spec, propname)
195 continue
197 # no ALTER TABLE, so we:
198 # 1. pull out the data, including an extra None column
199 oldcols, x = self.determine_columns(dbspec[1])
200 oldcols.append('id')
201 oldcols.append('__retired__')
202 cn = spec.classname
203 sql = 'select %s,%s from _%s'%(','.join(oldcols), self.arg, cn)
204 if __debug__:
205 print >>hyperdb.DEBUG, 'update_class', (self, sql, None)
206 self.cursor.execute(sql, (None,))
207 olddata = self.cursor.fetchall()
209 # 2. drop the old table
210 self.cursor.execute('drop table _%s'%cn)
212 # 3. create the new table
213 cols, mls = self.create_class_table(spec)
214 # ensure the new column is last
215 cols.remove('_'+propname)
216 assert oldcols == cols, "Column lists don't match!"
217 cols.append('_'+propname)
219 # 4. populate with the data from step one
220 s = ','.join([self.arg for x in cols])
221 scols = ','.join(cols)
222 sql = 'insert into _%s (%s) values (%s)'%(cn, scols, s)
224 # GAH, nothing had better go wrong from here on in... but
225 # we have to commit the drop...
226 # XXX this isn't necessary in sqlite :(
227 self.conn.commit()
229 # do the insert
230 for row in olddata:
231 self.sql(sql, tuple(row))
233 else:
234 # modify the property
235 if __debug__:
236 print >>hyperdb.DEBUG, 'update_class NOOP'
237 pass # NOOP in gadfly
239 # and the other way - only worry about deletions here
240 for propname in dbspec_propnames:
241 prop = dbspec_props[propname]
242 if spec_props.has_key(propname):
243 continue
244 if __debug__:
245 print >>hyperdb.DEBUG, 'update_class REMOVE', `prop`
247 # delete the property
248 if isinstance(prop, Multilink):
249 sql = 'drop table %s_%s'%(spec.classname, prop)
250 if __debug__:
251 print >>hyperdb.DEBUG, 'update_class', (self, sql)
252 self.cursor.execute(sql)
253 else:
254 # no ALTER TABLE, so we:
255 # 1. pull out the data, excluding the removed column
256 oldcols, x = self.determine_columns(spec.properties.items())
257 oldcols.append('id')
258 oldcols.append('__retired__')
259 # remove the missing column
260 oldcols.remove('_'+propname)
261 cn = spec.classname
262 sql = 'select %s from _%s'%(','.join(oldcols), cn)
263 self.cursor.execute(sql, (None,))
264 olddata = sql.fetchall()
266 # 2. drop the old table
267 self.cursor.execute('drop table _%s'%cn)
269 # 3. create the new table
270 cols, mls = self.create_class_table(self, spec)
271 assert oldcols != cols, "Column lists don't match!"
273 # 4. populate with the data from step one
274 qs = ','.join([self.arg for x in cols])
275 sql = 'insert into _%s values (%s)'%(cn, s)
276 self.cursor.execute(sql, olddata)
277 return 1
279 def create_class_table(self, spec):
280 ''' create the class table for the given spec
281 '''
282 cols, mls = self.determine_columns(spec.properties.items())
284 # add on our special columns
285 cols.append('id')
286 cols.append('__retired__')
288 # create the base table
289 scols = ','.join(['%s varchar'%x for x in cols])
290 sql = 'create table _%s (%s)'%(spec.classname, scols)
291 if __debug__:
292 print >>hyperdb.DEBUG, 'create_class', (self, sql)
293 self.cursor.execute(sql)
295 return cols, mls
297 def create_journal_table(self, spec):
298 ''' create the journal table for a class given the spec and
299 already-determined cols
300 '''
301 # journal table
302 cols = ','.join(['%s varchar'%x
303 for x in 'nodeid date tag action params'.split()])
304 sql = 'create table %s__journal (%s)'%(spec.classname, cols)
305 if __debug__:
306 print >>hyperdb.DEBUG, 'create_class', (self, sql)
307 self.cursor.execute(sql)
309 def create_multilink_table(self, spec, ml):
310 ''' Create a multilink table for the "ml" property of the class
311 given by the spec
312 '''
313 sql = 'create table %s_%s (linkid varchar, nodeid varchar)'%(
314 spec.classname, ml)
315 if __debug__:
316 print >>hyperdb.DEBUG, 'create_class', (self, sql)
317 self.cursor.execute(sql)
319 def create_class(self, spec):
320 ''' Create a database table according to the given spec.
321 '''
322 cols, mls = self.create_class_table(spec)
323 self.create_journal_table(spec)
325 # now create the multilink tables
326 for ml in mls:
327 self.create_multilink_table(spec, ml)
329 # ID counter
330 sql = 'insert into ids (name, num) values (%s,%s)'%(self.arg, self.arg)
331 vals = (spec.classname, 1)
332 if __debug__:
333 print >>hyperdb.DEBUG, 'create_class', (self, sql, vals)
334 self.cursor.execute(sql, vals)
336 def drop_class(self, spec):
337 ''' Drop the given table from the database.
339 Drop the journal and multilink tables too.
340 '''
341 # figure the multilinks
342 mls = []
343 for col, prop in spec.properties.items():
344 if isinstance(prop, Multilink):
345 mls.append(col)
347 sql = 'drop table _%s'%spec.classname
348 if __debug__:
349 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
350 self.cursor.execute(sql)
352 sql = 'drop table %s__journal'%spec.classname
353 if __debug__:
354 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
355 self.cursor.execute(sql)
357 for ml in mls:
358 sql = 'drop table %s_%s'%(spec.classname, ml)
359 if __debug__:
360 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
361 self.cursor.execute(sql)
363 #
364 # Classes
365 #
366 def __getattr__(self, classname):
367 ''' A convenient way of calling self.getclass(classname).
368 '''
369 if self.classes.has_key(classname):
370 if __debug__:
371 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
372 return self.classes[classname]
373 raise AttributeError, classname
375 def addclass(self, cl):
376 ''' Add a Class to the hyperdatabase.
377 '''
378 if __debug__:
379 print >>hyperdb.DEBUG, 'addclass', (self, cl)
380 cn = cl.classname
381 if self.classes.has_key(cn):
382 raise ValueError, cn
383 self.classes[cn] = cl
385 def getclasses(self):
386 ''' Return a list of the names of all existing classes.
387 '''
388 if __debug__:
389 print >>hyperdb.DEBUG, 'getclasses', (self,)
390 l = self.classes.keys()
391 l.sort()
392 return l
394 def getclass(self, classname):
395 '''Get the Class object representing a particular class.
397 If 'classname' is not a valid class name, a KeyError is raised.
398 '''
399 if __debug__:
400 print >>hyperdb.DEBUG, 'getclass', (self, classname)
401 try:
402 return self.classes[classname]
403 except KeyError:
404 raise KeyError, 'There is no class called "%s"'%classname
406 def clear(self):
407 ''' Delete all database contents.
409 Note: I don't commit here, which is different behaviour to the
410 "nuke from orbit" behaviour in the *dbms.
411 '''
412 if __debug__:
413 print >>hyperdb.DEBUG, 'clear', (self,)
414 for cn in self.classes.keys():
415 sql = 'delete from _%s'%cn
416 if __debug__:
417 print >>hyperdb.DEBUG, 'clear', (self, sql)
418 self.cursor.execute(sql)
420 #
421 # Node IDs
422 #
423 def newid(self, classname):
424 ''' Generate a new id for the given class
425 '''
426 # get the next ID
427 sql = 'select num from ids where name=%s'%self.arg
428 if __debug__:
429 print >>hyperdb.DEBUG, 'newid', (self, sql, classname)
430 self.cursor.execute(sql, (classname, ))
431 newid = self.cursor.fetchone()[0]
433 # update the counter
434 sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
435 vals = (int(newid)+1, classname)
436 if __debug__:
437 print >>hyperdb.DEBUG, 'newid', (self, sql, vals)
438 self.cursor.execute(sql, vals)
440 # return as string
441 return str(newid)
443 def setid(self, classname, setid):
444 ''' Set the id counter: used during import of database
445 '''
446 sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
447 vals = (setid, classname)
448 if __debug__:
449 print >>hyperdb.DEBUG, 'setid', (self, sql, vals)
450 self.cursor.execute(sql, vals)
452 #
453 # Nodes
454 #
456 def addnode(self, classname, nodeid, node):
457 ''' Add the specified node to its class's db.
458 '''
459 if __debug__:
460 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
461 # gadfly requires values for all non-multilink columns
462 cl = self.classes[classname]
463 cols, mls = self.determine_columns(cl.properties.items())
465 # we'll be supplied these props if we're doing an import
466 if not node.has_key('creator'):
467 # add in the "calculated" properties (dupe so we don't affect
468 # calling code's node assumptions)
469 node = node.copy()
470 node['creation'] = node['activity'] = date.Date()
471 node['creator'] = self.curuserid
473 # default the non-multilink columns
474 for col, prop in cl.properties.items():
475 if not isinstance(col, Multilink):
476 if not node.has_key(col):
477 node[col] = None
479 # clear this node out of the cache if it's in there
480 key = (classname, nodeid)
481 if self.cache.has_key(key):
482 del self.cache[key]
483 self.cache_lru.remove(key)
485 # make the node data safe for the DB
486 node = self.serialise(classname, node)
488 # make sure the ordering is correct for column name -> column value
489 vals = tuple([node[col[1:]] for col in cols]) + (nodeid, 0)
490 s = ','.join([self.arg for x in cols]) + ',%s,%s'%(self.arg, self.arg)
491 cols = ','.join(cols) + ',id,__retired__'
493 # perform the inserts
494 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
495 if __debug__:
496 print >>hyperdb.DEBUG, 'addnode', (self, sql, vals)
497 self.cursor.execute(sql, vals)
499 # insert the multilink rows
500 for col in mls:
501 t = '%s_%s'%(classname, col)
502 for entry in node[col]:
503 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
504 self.arg, self.arg)
505 self.sql(sql, (entry, nodeid))
507 # make sure we do the commit-time extra stuff for this node
508 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
510 def setnode(self, classname, nodeid, values, multilink_changes):
511 ''' Change the specified node.
512 '''
513 if __debug__:
514 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, values)
516 # clear this node out of the cache if it's in there
517 key = (classname, nodeid)
518 if self.cache.has_key(key):
519 del self.cache[key]
520 self.cache_lru.remove(key)
522 # add the special props
523 values = values.copy()
524 values['activity'] = date.Date()
526 # make db-friendly
527 values = self.serialise(classname, values)
529 cl = self.classes[classname]
530 cols = []
531 mls = []
532 # add the multilinks separately
533 props = cl.getprops()
534 for col in values.keys():
535 prop = props[col]
536 if isinstance(prop, Multilink):
537 mls.append(col)
538 else:
539 cols.append('_'+col)
540 cols.sort()
542 # if there's any updates to regular columns, do them
543 if cols:
544 # make sure the ordering is correct for column name -> column value
545 sqlvals = tuple([values[col[1:]] for col in cols]) + (nodeid,)
546 s = ','.join(['%s=%s'%(x, self.arg) for x in cols])
547 cols = ','.join(cols)
549 # perform the update
550 sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
551 if __debug__:
552 print >>hyperdb.DEBUG, 'setnode', (self, sql, sqlvals)
553 self.cursor.execute(sql, sqlvals)
555 # now the fun bit, updating the multilinks ;)
556 for col, (add, remove) in multilink_changes.items():
557 tn = '%s_%s'%(classname, col)
558 if add:
559 sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
560 self.arg, self.arg)
561 for addid in add:
562 self.sql(sql, (nodeid, addid))
563 if remove:
564 sql = 'delete from %s where nodeid=%s and linkid=%s'%(tn,
565 self.arg, self.arg)
566 for removeid in remove:
567 self.sql(sql, (nodeid, removeid))
569 # make sure we do the commit-time extra stuff for this node
570 self.transactions.append((self.doSaveNode, (classname, nodeid, values)))
572 def getnode(self, classname, nodeid):
573 ''' Get a node from the database.
574 '''
575 if __debug__:
576 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid)
578 # see if we have this node cached
579 key = (classname, nodeid)
580 if self.cache.has_key(key):
581 # push us back to the top of the LRU
582 self.cache_lru.remove(key)
583 self.cache_lru.insert(0, key)
584 # return the cached information
585 return self.cache[key]
587 # figure the columns we're fetching
588 cl = self.classes[classname]
589 cols, mls = self.determine_columns(cl.properties.items())
590 scols = ','.join(cols)
592 # perform the basic property fetch
593 sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
594 self.sql(sql, (nodeid,))
596 values = self.sql_fetchone()
597 if values is None:
598 raise IndexError, 'no such %s node %s'%(classname, nodeid)
600 # make up the node
601 node = {}
602 for col in range(len(cols)):
603 node[cols[col][1:]] = values[col]
605 # now the multilinks
606 for col in mls:
607 # get the link ids
608 sql = 'select linkid from %s_%s where nodeid=%s'%(classname, col,
609 self.arg)
610 self.cursor.execute(sql, (nodeid,))
611 # extract the first column from the result
612 node[col] = [x[0] for x in self.cursor.fetchall()]
614 # un-dbificate the node data
615 node = self.unserialise(classname, node)
617 # save off in the cache
618 key = (classname, nodeid)
619 self.cache[key] = node
620 # update the LRU
621 self.cache_lru.insert(0, key)
622 if len(self.cache_lru) > ROW_CACHE_SIZE:
623 del self.cache[self.cache_lru.pop()]
625 return node
627 def destroynode(self, classname, nodeid):
628 '''Remove a node from the database. Called exclusively by the
629 destroy() method on Class.
630 '''
631 if __debug__:
632 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
634 # make sure the node exists
635 if not self.hasnode(classname, nodeid):
636 raise IndexError, '%s has no node %s'%(classname, nodeid)
638 # see if we have this node cached
639 if self.cache.has_key((classname, nodeid)):
640 del self.cache[(classname, nodeid)]
642 # see if there's any obvious commit actions that we should get rid of
643 for entry in self.transactions[:]:
644 if entry[1][:2] == (classname, nodeid):
645 self.transactions.remove(entry)
647 # now do the SQL
648 sql = 'delete from _%s where id=%s'%(classname, self.arg)
649 self.sql(sql, (nodeid,))
651 # remove from multilnks
652 cl = self.getclass(classname)
653 x, mls = self.determine_columns(cl.properties.items())
654 for col in mls:
655 # get the link ids
656 sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
657 self.cursor.execute(sql, (nodeid,))
659 # remove journal entries
660 sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
661 self.sql(sql, (nodeid,))
663 def serialise(self, classname, node):
664 '''Copy the node contents, converting non-marshallable data into
665 marshallable data.
666 '''
667 if __debug__:
668 print >>hyperdb.DEBUG, 'serialise', classname, node
669 properties = self.getclass(classname).getprops()
670 d = {}
671 for k, v in node.items():
672 # if the property doesn't exist, or is the "retired" flag then
673 # it won't be in the properties dict
674 if not properties.has_key(k):
675 d[k] = v
676 continue
678 # get the property spec
679 prop = properties[k]
681 if isinstance(prop, Password):
682 d[k] = str(v)
683 elif isinstance(prop, Date) and v is not None:
684 d[k] = v.serialise()
685 elif isinstance(prop, Interval) and v is not None:
686 d[k] = v.serialise()
687 else:
688 d[k] = v
689 return d
691 def unserialise(self, classname, node):
692 '''Decode the marshalled node data
693 '''
694 if __debug__:
695 print >>hyperdb.DEBUG, 'unserialise', classname, node
696 properties = self.getclass(classname).getprops()
697 d = {}
698 for k, v in node.items():
699 # if the property doesn't exist, or is the "retired" flag then
700 # it won't be in the properties dict
701 if not properties.has_key(k):
702 d[k] = v
703 continue
705 # get the property spec
706 prop = properties[k]
708 if isinstance(prop, Date) and v is not None:
709 d[k] = date.Date(v)
710 elif isinstance(prop, Interval) and v is not None:
711 d[k] = date.Interval(v)
712 elif isinstance(prop, Password):
713 p = password.Password()
714 p.unpack(v)
715 d[k] = p
716 else:
717 d[k] = v
718 return d
720 def hasnode(self, classname, nodeid):
721 ''' Determine if the database has a given node.
722 '''
723 sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
724 if __debug__:
725 print >>hyperdb.DEBUG, 'hasnode', (self, sql, nodeid)
726 self.cursor.execute(sql, (nodeid,))
727 return int(self.cursor.fetchone()[0])
729 def countnodes(self, classname):
730 ''' Count the number of nodes that exist for a particular Class.
731 '''
732 sql = 'select count(*) from _%s'%classname
733 if __debug__:
734 print >>hyperdb.DEBUG, 'countnodes', (self, sql)
735 self.cursor.execute(sql)
736 return self.cursor.fetchone()[0]
738 def getnodeids(self, classname, retired=0):
739 ''' Retrieve all the ids of the nodes for a particular Class.
741 Set retired=None to get all nodes. Otherwise it'll get all the
742 retired or non-retired nodes, depending on the flag.
743 '''
744 # flip the sense of the flag if we don't want all of them
745 if retired is not None:
746 retired = not retired
747 sql = 'select id from _%s where __retired__ <> %s'%(classname, self.arg)
748 if __debug__:
749 print >>hyperdb.DEBUG, 'getnodeids', (self, sql, retired)
750 self.cursor.execute(sql, (retired,))
751 return [x[0] for x in self.cursor.fetchall()]
753 def addjournal(self, classname, nodeid, action, params, creator=None,
754 creation=None):
755 ''' Journal the Action
756 'action' may be:
758 'create' or 'set' -- 'params' is a dictionary of property values
759 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
760 'retire' -- 'params' is None
761 '''
762 # serialise the parameters now if necessary
763 if isinstance(params, type({})):
764 if action in ('set', 'create'):
765 params = self.serialise(classname, params)
767 # handle supply of the special journalling parameters (usually
768 # supplied on importing an existing database)
769 if creator:
770 journaltag = creator
771 else:
772 journaltag = self.curuserid
773 if creation:
774 journaldate = creation.serialise()
775 else:
776 journaldate = date.Date().serialise()
778 # create the journal entry
779 cols = ','.join('nodeid date tag action params'.split())
781 if __debug__:
782 print >>hyperdb.DEBUG, 'addjournal', (nodeid, journaldate,
783 journaltag, action, params)
785 self.save_journal(classname, cols, nodeid, journaldate,
786 journaltag, action, params)
788 def save_journal(self, classname, cols, nodeid, journaldate,
789 journaltag, action, params):
790 ''' Save the journal entry to the database
791 '''
792 raise NotImplemented
794 def getjournal(self, classname, nodeid):
795 ''' get the journal for id
796 '''
797 # make sure the node exists
798 if not self.hasnode(classname, nodeid):
799 raise IndexError, '%s has no node %s'%(classname, nodeid)
801 cols = ','.join('nodeid date tag action params'.split())
802 return self.load_journal(classname, cols, nodeid)
804 def load_journal(self, classname, cols, nodeid):
805 ''' Load the journal from the database
806 '''
807 raise NotImplemented
809 def pack(self, pack_before):
810 ''' Delete all journal entries except "create" before 'pack_before'.
811 '''
812 # get a 'yyyymmddhhmmss' version of the date
813 date_stamp = pack_before.serialise()
815 # do the delete
816 for classname in self.classes.keys():
817 sql = "delete from %s__journal where date<%s and "\
818 "action<>'create'"%(classname, self.arg)
819 if __debug__:
820 print >>hyperdb.DEBUG, 'pack', (self, sql, date_stamp)
821 self.cursor.execute(sql, (date_stamp,))
823 def sql_commit(self):
824 ''' Actually commit to the database.
825 '''
826 self.conn.commit()
828 def commit(self):
829 ''' Commit the current transactions.
831 Save all data changed since the database was opened or since the
832 last commit() or rollback().
833 '''
834 if __debug__:
835 print >>hyperdb.DEBUG, 'commit', (self,)
837 # commit the database
838 self.sql_commit()
840 # now, do all the other transaction stuff
841 reindex = {}
842 for method, args in self.transactions:
843 reindex[method(*args)] = 1
845 # reindex the nodes that request it
846 for classname, nodeid in filter(None, reindex.keys()):
847 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
848 self.getclass(classname).index(nodeid)
850 # save the indexer state
851 self.indexer.save_index()
853 # clear out the transactions
854 self.transactions = []
856 def rollback(self):
857 ''' Reverse all actions from the current transaction.
859 Undo all the changes made since the database was opened or the last
860 commit() or rollback() was performed.
861 '''
862 if __debug__:
863 print >>hyperdb.DEBUG, 'rollback', (self,)
865 # roll back
866 self.conn.rollback()
868 # roll back "other" transaction stuff
869 for method, args in self.transactions:
870 # delete temporary files
871 if method == self.doStoreFile:
872 self.rollbackStoreFile(*args)
873 self.transactions = []
875 def doSaveNode(self, classname, nodeid, node):
876 ''' dummy that just generates a reindex event
877 '''
878 # return the classname, nodeid so we reindex this content
879 return (classname, nodeid)
881 def close(self):
882 ''' Close off the connection.
883 '''
884 self.conn.close()
886 #
887 # The base Class class
888 #
889 class Class(hyperdb.Class):
890 ''' The handle to a particular class of nodes in a hyperdatabase.
892 All methods except __repr__ and getnode must be implemented by a
893 concrete backend Class.
894 '''
896 def __init__(self, db, classname, **properties):
897 '''Create a new class with a given name and property specification.
899 'classname' must not collide with the name of an existing class,
900 or a ValueError is raised. The keyword arguments in 'properties'
901 must map names to property objects, or a TypeError is raised.
902 '''
903 if (properties.has_key('creation') or properties.has_key('activity')
904 or properties.has_key('creator')):
905 raise ValueError, '"creation", "activity" and "creator" are '\
906 'reserved'
908 self.classname = classname
909 self.properties = properties
910 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
911 self.key = ''
913 # should we journal changes (default yes)
914 self.do_journal = 1
916 # do the db-related init stuff
917 db.addclass(self)
919 self.auditors = {'create': [], 'set': [], 'retire': []}
920 self.reactors = {'create': [], 'set': [], 'retire': []}
922 def schema(self):
923 ''' A dumpable version of the schema that we can store in the
924 database
925 '''
926 return (self.key, [(x, repr(y)) for x,y in self.properties.items()])
928 def enableJournalling(self):
929 '''Turn journalling on for this class
930 '''
931 self.do_journal = 1
933 def disableJournalling(self):
934 '''Turn journalling off for this class
935 '''
936 self.do_journal = 0
938 # Editing nodes:
939 def create(self, **propvalues):
940 ''' Create a new node of this class and return its id.
942 The keyword arguments in 'propvalues' map property names to values.
944 The values of arguments must be acceptable for the types of their
945 corresponding properties or a TypeError is raised.
947 If this class has a key property, it must be present and its value
948 must not collide with other key strings or a ValueError is raised.
950 Any other properties on this class that are missing from the
951 'propvalues' dictionary are set to None.
953 If an id in a link or multilink property does not refer to a valid
954 node, an IndexError is raised.
955 '''
956 if propvalues.has_key('id'):
957 raise KeyError, '"id" is reserved'
959 if self.db.journaltag is None:
960 raise DatabaseError, 'Database open read-only'
962 if propvalues.has_key('creation') or propvalues.has_key('activity'):
963 raise KeyError, '"creation" and "activity" are reserved'
965 self.fireAuditors('create', None, propvalues)
967 # new node's id
968 newid = self.db.newid(self.classname)
970 # validate propvalues
971 num_re = re.compile('^\d+$')
972 for key, value in propvalues.items():
973 if key == self.key:
974 try:
975 self.lookup(value)
976 except KeyError:
977 pass
978 else:
979 raise ValueError, 'node with key "%s" exists'%value
981 # try to handle this property
982 try:
983 prop = self.properties[key]
984 except KeyError:
985 raise KeyError, '"%s" has no property "%s"'%(self.classname,
986 key)
988 if value is not None and isinstance(prop, Link):
989 if type(value) != type(''):
990 raise ValueError, 'link value must be String'
991 link_class = self.properties[key].classname
992 # if it isn't a number, it's a key
993 if not num_re.match(value):
994 try:
995 value = self.db.classes[link_class].lookup(value)
996 except (TypeError, KeyError):
997 raise IndexError, 'new property "%s": %s not a %s'%(
998 key, value, link_class)
999 elif not self.db.getclass(link_class).hasnode(value):
1000 raise IndexError, '%s has no node %s'%(link_class, value)
1002 # save off the value
1003 propvalues[key] = value
1005 # register the link with the newly linked node
1006 if self.do_journal and self.properties[key].do_journal:
1007 self.db.addjournal(link_class, value, 'link',
1008 (self.classname, newid, key))
1010 elif isinstance(prop, Multilink):
1011 if type(value) != type([]):
1012 raise TypeError, 'new property "%s" not a list of ids'%key
1014 # clean up and validate the list of links
1015 link_class = self.properties[key].classname
1016 l = []
1017 for entry in value:
1018 if type(entry) != type(''):
1019 raise ValueError, '"%s" multilink value (%r) '\
1020 'must contain Strings'%(key, value)
1021 # if it isn't a number, it's a key
1022 if not num_re.match(entry):
1023 try:
1024 entry = self.db.classes[link_class].lookup(entry)
1025 except (TypeError, KeyError):
1026 raise IndexError, 'new property "%s": %s not a %s'%(
1027 key, entry, self.properties[key].classname)
1028 l.append(entry)
1029 value = l
1030 propvalues[key] = value
1032 # handle additions
1033 for nodeid in value:
1034 if not self.db.getclass(link_class).hasnode(nodeid):
1035 raise IndexError, '%s has no node %s'%(link_class,
1036 nodeid)
1037 # register the link with the newly linked node
1038 if self.do_journal and self.properties[key].do_journal:
1039 self.db.addjournal(link_class, nodeid, 'link',
1040 (self.classname, newid, key))
1042 elif isinstance(prop, String):
1043 if type(value) != type(''):
1044 raise TypeError, 'new property "%s" not a string'%key
1046 elif isinstance(prop, Password):
1047 if not isinstance(value, password.Password):
1048 raise TypeError, 'new property "%s" not a Password'%key
1050 elif isinstance(prop, Date):
1051 if value is not None and not isinstance(value, date.Date):
1052 raise TypeError, 'new property "%s" not a Date'%key
1054 elif isinstance(prop, Interval):
1055 if value is not None and not isinstance(value, date.Interval):
1056 raise TypeError, 'new property "%s" not an Interval'%key
1058 elif value is not None and isinstance(prop, Number):
1059 try:
1060 float(value)
1061 except ValueError:
1062 raise TypeError, 'new property "%s" not numeric'%key
1064 elif value is not None and isinstance(prop, Boolean):
1065 try:
1066 int(value)
1067 except ValueError:
1068 raise TypeError, 'new property "%s" not boolean'%key
1070 # make sure there's data where there needs to be
1071 for key, prop in self.properties.items():
1072 if propvalues.has_key(key):
1073 continue
1074 if key == self.key:
1075 raise ValueError, 'key property "%s" is required'%key
1076 if isinstance(prop, Multilink):
1077 propvalues[key] = []
1078 else:
1079 propvalues[key] = None
1081 # done
1082 self.db.addnode(self.classname, newid, propvalues)
1083 if self.do_journal:
1084 self.db.addjournal(self.classname, newid, 'create', propvalues)
1086 self.fireReactors('create', newid, None)
1088 return newid
1090 def export_list(self, propnames, nodeid):
1091 ''' Export a node - generate a list of CSV-able data in the order
1092 specified by propnames for the given node.
1093 '''
1094 properties = self.getprops()
1095 l = []
1096 for prop in propnames:
1097 proptype = properties[prop]
1098 value = self.get(nodeid, prop)
1099 # "marshal" data where needed
1100 if value is None:
1101 pass
1102 elif isinstance(proptype, hyperdb.Date):
1103 value = value.get_tuple()
1104 elif isinstance(proptype, hyperdb.Interval):
1105 value = value.get_tuple()
1106 elif isinstance(proptype, hyperdb.Password):
1107 value = str(value)
1108 l.append(repr(value))
1109 return l
1111 def import_list(self, propnames, proplist):
1112 ''' Import a node - all information including "id" is present and
1113 should not be sanity checked. Triggers are not triggered. The
1114 journal should be initialised using the "creator" and "created"
1115 information.
1117 Return the nodeid of the node imported.
1118 '''
1119 if self.db.journaltag is None:
1120 raise DatabaseError, 'Database open read-only'
1121 properties = self.getprops()
1123 # make the new node's property map
1124 d = {}
1125 for i in range(len(propnames)):
1126 # Use eval to reverse the repr() used to output the CSV
1127 value = eval(proplist[i])
1129 # Figure the property for this column
1130 propname = propnames[i]
1131 prop = properties[propname]
1133 # "unmarshal" where necessary
1134 if propname == 'id':
1135 newid = value
1136 continue
1137 elif value is None:
1138 # don't set Nones
1139 continue
1140 elif isinstance(prop, hyperdb.Date):
1141 value = date.Date(value)
1142 elif isinstance(prop, hyperdb.Interval):
1143 value = date.Interval(value)
1144 elif isinstance(prop, hyperdb.Password):
1145 pwd = password.Password()
1146 pwd.unpack(value)
1147 value = pwd
1148 d[propname] = value
1150 # add the node and journal
1151 self.db.addnode(self.classname, newid, d)
1153 # extract the extraneous journalling gumpf and nuke it
1154 if d.has_key('creator'):
1155 creator = d['creator']
1156 del d['creator']
1157 else:
1158 creator = None
1159 if d.has_key('creation'):
1160 creation = d['creation']
1161 del d['creation']
1162 else:
1163 creation = None
1164 if d.has_key('activity'):
1165 del d['activity']
1166 self.db.addjournal(self.classname, newid, 'create', d, creator,
1167 creation)
1168 return newid
1170 _marker = []
1171 def get(self, nodeid, propname, default=_marker, cache=1):
1172 '''Get the value of a property on an existing node of this class.
1174 'nodeid' must be the id of an existing node of this class or an
1175 IndexError is raised. 'propname' must be the name of a property
1176 of this class or a KeyError is raised.
1178 'cache' indicates whether the transaction cache should be queried
1179 for the node. If the node has been modified and you need to
1180 determine what its values prior to modification are, you need to
1181 set cache=0.
1182 '''
1183 if propname == 'id':
1184 return nodeid
1186 # get the node's dict
1187 d = self.db.getnode(self.classname, nodeid)
1189 if propname == 'creation':
1190 if d.has_key('creation'):
1191 return d['creation']
1192 else:
1193 return date.Date()
1194 if propname == 'activity':
1195 if d.has_key('activity'):
1196 return d['activity']
1197 else:
1198 return date.Date()
1199 if propname == 'creator':
1200 if d.has_key('creator'):
1201 return d['creator']
1202 else:
1203 return self.db.curuserid
1205 # get the property (raises KeyErorr if invalid)
1206 prop = self.properties[propname]
1208 if not d.has_key(propname):
1209 if default is self._marker:
1210 if isinstance(prop, Multilink):
1211 return []
1212 else:
1213 return None
1214 else:
1215 return default
1217 # don't pass our list to other code
1218 if isinstance(prop, Multilink):
1219 return d[propname][:]
1221 return d[propname]
1223 def getnode(self, nodeid, cache=1):
1224 ''' Return a convenience wrapper for the node.
1226 'nodeid' must be the id of an existing node of this class or an
1227 IndexError is raised.
1229 'cache' indicates whether the transaction cache should be queried
1230 for the node. If the node has been modified and you need to
1231 determine what its values prior to modification are, you need to
1232 set cache=0.
1233 '''
1234 return Node(self, nodeid, cache=cache)
1236 def set(self, nodeid, **propvalues):
1237 '''Modify a property on an existing node of this class.
1239 'nodeid' must be the id of an existing node of this class or an
1240 IndexError is raised.
1242 Each key in 'propvalues' must be the name of a property of this
1243 class or a KeyError is raised.
1245 All values in 'propvalues' must be acceptable types for their
1246 corresponding properties or a TypeError is raised.
1248 If the value of the key property is set, it must not collide with
1249 other key strings or a ValueError is raised.
1251 If the value of a Link or Multilink property contains an invalid
1252 node id, a ValueError is raised.
1253 '''
1254 if not propvalues:
1255 return propvalues
1257 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1258 raise KeyError, '"creation" and "activity" are reserved'
1260 if propvalues.has_key('id'):
1261 raise KeyError, '"id" is reserved'
1263 if self.db.journaltag is None:
1264 raise DatabaseError, 'Database open read-only'
1266 self.fireAuditors('set', nodeid, propvalues)
1267 # Take a copy of the node dict so that the subsequent set
1268 # operation doesn't modify the oldvalues structure.
1269 # XXX used to try the cache here first
1270 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1272 node = self.db.getnode(self.classname, nodeid)
1273 if self.is_retired(nodeid):
1274 raise IndexError, 'Requested item is retired'
1275 num_re = re.compile('^\d+$')
1277 # if the journal value is to be different, store it in here
1278 journalvalues = {}
1280 # remember the add/remove stuff for multilinks, making it easier
1281 # for the Database layer to do its stuff
1282 multilink_changes = {}
1284 for propname, value in propvalues.items():
1285 # check to make sure we're not duplicating an existing key
1286 if propname == self.key and node[propname] != value:
1287 try:
1288 self.lookup(value)
1289 except KeyError:
1290 pass
1291 else:
1292 raise ValueError, 'node with key "%s" exists'%value
1294 # this will raise the KeyError if the property isn't valid
1295 # ... we don't use getprops() here because we only care about
1296 # the writeable properties.
1297 try:
1298 prop = self.properties[propname]
1299 except KeyError:
1300 raise KeyError, '"%s" has no property named "%s"'%(
1301 self.classname, propname)
1303 # if the value's the same as the existing value, no sense in
1304 # doing anything
1305 if node.has_key(propname) and value == node[propname]:
1306 del propvalues[propname]
1307 continue
1309 # do stuff based on the prop type
1310 if isinstance(prop, Link):
1311 link_class = prop.classname
1312 # if it isn't a number, it's a key
1313 if value is not None and not isinstance(value, type('')):
1314 raise ValueError, 'property "%s" link value be a string'%(
1315 propname)
1316 if isinstance(value, type('')) and not num_re.match(value):
1317 try:
1318 value = self.db.classes[link_class].lookup(value)
1319 except (TypeError, KeyError):
1320 raise IndexError, 'new property "%s": %s not a %s'%(
1321 propname, value, prop.classname)
1323 if (value is not None and
1324 not self.db.getclass(link_class).hasnode(value)):
1325 raise IndexError, '%s has no node %s'%(link_class, value)
1327 if self.do_journal and prop.do_journal:
1328 # register the unlink with the old linked node
1329 if node[propname] is not None:
1330 self.db.addjournal(link_class, node[propname], 'unlink',
1331 (self.classname, nodeid, propname))
1333 # register the link with the newly linked node
1334 if value is not None:
1335 self.db.addjournal(link_class, value, 'link',
1336 (self.classname, nodeid, propname))
1338 elif isinstance(prop, Multilink):
1339 if type(value) != type([]):
1340 raise TypeError, 'new property "%s" not a list of'\
1341 ' ids'%propname
1342 link_class = self.properties[propname].classname
1343 l = []
1344 for entry in value:
1345 # if it isn't a number, it's a key
1346 if type(entry) != type(''):
1347 raise ValueError, 'new property "%s" link value ' \
1348 'must be a string'%propname
1349 if not num_re.match(entry):
1350 try:
1351 entry = self.db.classes[link_class].lookup(entry)
1352 except (TypeError, KeyError):
1353 raise IndexError, 'new property "%s": %s not a %s'%(
1354 propname, entry,
1355 self.properties[propname].classname)
1356 l.append(entry)
1357 value = l
1358 propvalues[propname] = value
1360 # figure the journal entry for this property
1361 add = []
1362 remove = []
1364 # handle removals
1365 if node.has_key(propname):
1366 l = node[propname]
1367 else:
1368 l = []
1369 for id in l[:]:
1370 if id in value:
1371 continue
1372 # register the unlink with the old linked node
1373 if self.do_journal and self.properties[propname].do_journal:
1374 self.db.addjournal(link_class, id, 'unlink',
1375 (self.classname, nodeid, propname))
1376 l.remove(id)
1377 remove.append(id)
1379 # handle additions
1380 for id in value:
1381 if not self.db.getclass(link_class).hasnode(id):
1382 raise IndexError, '%s has no node %s'%(link_class, id)
1383 if id in l:
1384 continue
1385 # register the link with the newly linked node
1386 if self.do_journal and self.properties[propname].do_journal:
1387 self.db.addjournal(link_class, id, 'link',
1388 (self.classname, nodeid, propname))
1389 l.append(id)
1390 add.append(id)
1392 # figure the journal entry
1393 l = []
1394 if add:
1395 l.append(('+', add))
1396 if remove:
1397 l.append(('-', remove))
1398 multilink_changes[propname] = (add, remove)
1399 if l:
1400 journalvalues[propname] = tuple(l)
1402 elif isinstance(prop, String):
1403 if value is not None and type(value) != type(''):
1404 raise TypeError, 'new property "%s" not a string'%propname
1406 elif isinstance(prop, Password):
1407 if not isinstance(value, password.Password):
1408 raise TypeError, 'new property "%s" not a Password'%propname
1409 propvalues[propname] = value
1411 elif value is not None and isinstance(prop, Date):
1412 if not isinstance(value, date.Date):
1413 raise TypeError, 'new property "%s" not a Date'% propname
1414 propvalues[propname] = value
1416 elif value is not None and isinstance(prop, Interval):
1417 if not isinstance(value, date.Interval):
1418 raise TypeError, 'new property "%s" not an '\
1419 'Interval'%propname
1420 propvalues[propname] = value
1422 elif value is not None and isinstance(prop, Number):
1423 try:
1424 float(value)
1425 except ValueError:
1426 raise TypeError, 'new property "%s" not numeric'%propname
1428 elif value is not None and isinstance(prop, Boolean):
1429 try:
1430 int(value)
1431 except ValueError:
1432 raise TypeError, 'new property "%s" not boolean'%propname
1434 # nothing to do?
1435 if not propvalues:
1436 return propvalues
1438 # do the set, and journal it
1439 self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
1441 if self.do_journal:
1442 propvalues.update(journalvalues)
1443 self.db.addjournal(self.classname, nodeid, 'set', propvalues)
1445 self.fireReactors('set', nodeid, oldvalues)
1447 return propvalues
1449 def retire(self, nodeid):
1450 '''Retire a node.
1452 The properties on the node remain available from the get() method,
1453 and the node's id is never reused.
1455 Retired nodes are not returned by the find(), list(), or lookup()
1456 methods, and other nodes may reuse the values of their key properties.
1457 '''
1458 if self.db.journaltag is None:
1459 raise DatabaseError, 'Database open read-only'
1461 sql = 'update _%s set __retired__=1 where id=%s'%(self.classname,
1462 self.db.arg)
1463 if __debug__:
1464 print >>hyperdb.DEBUG, 'retire', (self, sql, nodeid)
1465 self.db.cursor.execute(sql, (nodeid,))
1467 def is_retired(self, nodeid):
1468 '''Return true if the node is rerired
1469 '''
1470 sql = 'select __retired__ from _%s where id=%s'%(self.classname,
1471 self.db.arg)
1472 if __debug__:
1473 print >>hyperdb.DEBUG, 'is_retired', (self, sql, nodeid)
1474 self.db.cursor.execute(sql, (nodeid,))
1475 return int(self.db.sql_fetchone()[0])
1477 def destroy(self, nodeid):
1478 '''Destroy a node.
1480 WARNING: this method should never be used except in extremely rare
1481 situations where there could never be links to the node being
1482 deleted
1483 WARNING: use retire() instead
1484 WARNING: the properties of this node will not be available ever again
1485 WARNING: really, use retire() instead
1487 Well, I think that's enough warnings. This method exists mostly to
1488 support the session storage of the cgi interface.
1490 The node is completely removed from the hyperdb, including all journal
1491 entries. It will no longer be available, and will generally break code
1492 if there are any references to the node.
1493 '''
1494 if self.db.journaltag is None:
1495 raise DatabaseError, 'Database open read-only'
1496 self.db.destroynode(self.classname, nodeid)
1498 def history(self, nodeid):
1499 '''Retrieve the journal of edits on a particular node.
1501 'nodeid' must be the id of an existing node of this class or an
1502 IndexError is raised.
1504 The returned list contains tuples of the form
1506 (date, tag, action, params)
1508 'date' is a Timestamp object specifying the time of the change and
1509 'tag' is the journaltag specified when the database was opened.
1510 '''
1511 if not self.do_journal:
1512 raise ValueError, 'Journalling is disabled for this class'
1513 return self.db.getjournal(self.classname, nodeid)
1515 # Locating nodes:
1516 def hasnode(self, nodeid):
1517 '''Determine if the given nodeid actually exists
1518 '''
1519 return self.db.hasnode(self.classname, nodeid)
1521 def setkey(self, propname):
1522 '''Select a String property of this class to be the key property.
1524 'propname' must be the name of a String property of this class or
1525 None, or a TypeError is raised. The values of the key property on
1526 all existing nodes must be unique or a ValueError is raised.
1527 '''
1528 # XXX create an index on the key prop column
1529 prop = self.getprops()[propname]
1530 if not isinstance(prop, String):
1531 raise TypeError, 'key properties must be String'
1532 self.key = propname
1534 def getkey(self):
1535 '''Return the name of the key property for this class or None.'''
1536 return self.key
1538 def labelprop(self, default_to_id=0):
1539 ''' Return the property name for a label for the given node.
1541 This method attempts to generate a consistent label for the node.
1542 It tries the following in order:
1543 1. key property
1544 2. "name" property
1545 3. "title" property
1546 4. first property from the sorted property name list
1547 '''
1548 k = self.getkey()
1549 if k:
1550 return k
1551 props = self.getprops()
1552 if props.has_key('name'):
1553 return 'name'
1554 elif props.has_key('title'):
1555 return 'title'
1556 if default_to_id:
1557 return 'id'
1558 props = props.keys()
1559 props.sort()
1560 return props[0]
1562 def lookup(self, keyvalue):
1563 '''Locate a particular node by its key property and return its id.
1565 If this class has no key property, a TypeError is raised. If the
1566 'keyvalue' matches one of the values for the key property among
1567 the nodes in this class, the matching node's id is returned;
1568 otherwise a KeyError is raised.
1569 '''
1570 if not self.key:
1571 raise TypeError, 'No key property set for class %s'%self.classname
1573 sql = 'select id,__retired__ from _%s where _%s=%s'%(self.classname,
1574 self.key, self.db.arg)
1575 self.db.sql(sql, (keyvalue,))
1577 # see if there was a result that's not retired
1578 l = self.db.cursor.fetchall()
1579 if not l or int(l[0][1]):
1580 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1581 keyvalue, self.classname)
1583 # return the id
1584 return l[0][0]
1586 def find(self, **propspec):
1587 '''Get the ids of nodes in this class which link to the given nodes.
1589 'propspec' consists of keyword args propname={nodeid:1,}
1590 'propname' must be the name of a property in this class, or a
1591 KeyError is raised. That property must be a Link or Multilink
1592 property, or a TypeError is raised.
1594 Any node in this class whose 'propname' property links to any of the
1595 nodeids will be returned. Used by the full text indexing, which knows
1596 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1597 issues:
1599 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1600 '''
1601 if __debug__:
1602 print >>hyperdb.DEBUG, 'find', (self, propspec)
1603 if not propspec:
1604 return []
1605 queries = []
1606 tables = []
1607 allvalues = ()
1608 for prop, values in propspec.items():
1609 allvalues += tuple(values.keys())
1610 a = self.db.arg
1611 tables.append('select nodeid from %s_%s where linkid in (%s)'%(
1612 self.classname, prop, ','.join([a for x in values.keys()])))
1613 sql = '\nintersect\n'.join(tables)
1614 self.db.sql(sql, allvalues)
1615 l = [x[0] for x in self.db.sql_fetchall()]
1616 if __debug__:
1617 print >>hyperdb.DEBUG, 'find ... ', l
1618 return l
1620 def stringFind(self, **requirements):
1621 '''Locate a particular node by matching a set of its String
1622 properties in a caseless search.
1624 If the property is not a String property, a TypeError is raised.
1626 The return is a list of the id of all nodes that match.
1627 '''
1628 where = []
1629 args = []
1630 for propname in requirements.keys():
1631 prop = self.properties[propname]
1632 if isinstance(not prop, String):
1633 raise TypeError, "'%s' not a String property"%propname
1634 where.append(propname)
1635 args.append(requirements[propname].lower())
1637 # generate the where clause
1638 s = ' and '.join(['_%s=%s'%(col, self.db.arg) for col in where])
1639 sql = 'select id from _%s where %s'%(self.classname, s)
1640 self.db.sql(sql, tuple(args))
1641 l = [x[0] for x in self.db.sql_fetchall()]
1642 if __debug__:
1643 print >>hyperdb.DEBUG, 'find ... ', l
1644 return l
1646 def list(self):
1647 ''' Return a list of the ids of the active nodes in this class.
1648 '''
1649 return self.db.getnodeids(self.classname, retired=0)
1651 def filter(self, search_matches, filterspec, sort, group):
1652 ''' Return a list of the ids of the active nodes in this class that
1653 match the 'filter' spec, sorted by the group spec and then the
1654 sort spec
1656 "filterspec" is {propname: value(s)}
1657 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1658 and prop is a prop name or None
1659 "search_matches" is {nodeid: marker}
1661 The filter must match all properties specificed - but if the
1662 property value to match is a list, any one of the values in the
1663 list may match for that property to match.
1664 '''
1665 cn = self.classname
1667 # figure the WHERE clause from the filterspec
1668 props = self.getprops()
1669 frum = ['_'+cn]
1670 where = []
1671 args = []
1672 a = self.db.arg
1673 for k, v in filterspec.items():
1674 propclass = props[k]
1675 # now do other where clause stuff
1676 if isinstance(propclass, Multilink):
1677 tn = '%s_%s'%(cn, k)
1678 frum.append(tn)
1679 if isinstance(v, type([])):
1680 s = ','.join([a for x in v])
1681 where.append('id=%s.nodeid and %s.linkid in (%s)'%(tn,tn,s))
1682 args = args + v
1683 else:
1684 where.append('id=%s.nodeid and %s.linkid = %s'%(tn, tn, a))
1685 args.append(v)
1686 elif isinstance(propclass, String):
1687 if not isinstance(v, type([])):
1688 v = [v]
1690 # Quote the bits in the string that need it and then embed
1691 # in a "substring" search. Note - need to quote the '%' so
1692 # they make it through the python layer happily
1693 v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
1695 # now add to the where clause
1696 where.append(' or '.join(["_%s LIKE '%s'"%(k, s) for s in v]))
1697 # note: args are embedded in the query string now
1698 elif isinstance(propclass, Link):
1699 if isinstance(v, type([])):
1700 if '-1' in v:
1701 v.remove('-1')
1702 xtra = ' or _%s is NULL'%k
1703 else:
1704 xtra = ''
1705 s = ','.join([a for x in v])
1706 where.append('(_%s in (%s)%s)'%(k, s, xtra))
1707 args = args + v
1708 else:
1709 if v == '-1':
1710 v = None
1711 where.append('_%s is NULL'%k)
1712 else:
1713 where.append('_%s=%s'%(k, a))
1714 args.append(v)
1715 else:
1716 if isinstance(v, type([])):
1717 s = ','.join([a for x in v])
1718 where.append('_%s in (%s)'%(k, s))
1719 args = args + v
1720 else:
1721 where.append('_%s=%s'%(k, a))
1722 args.append(v)
1724 # add results of full text search
1725 if search_matches is not None:
1726 v = search_matches.keys()
1727 s = ','.join([a for x in v])
1728 where.append('id in (%s)'%s)
1729 args = args + v
1731 # "grouping" is just the first-order sorting in the SQL fetch
1732 # can modify it...)
1733 orderby = []
1734 ordercols = []
1735 if group[0] is not None and group[1] is not None:
1736 if group[0] != '-':
1737 orderby.append('_'+group[1])
1738 ordercols.append('_'+group[1])
1739 else:
1740 orderby.append('_'+group[1]+' desc')
1741 ordercols.append('_'+group[1])
1743 # now add in the sorting
1744 group = ''
1745 if sort[0] is not None and sort[1] is not None:
1746 direction, colname = sort
1747 if direction != '-':
1748 if colname == 'id':
1749 orderby.append(colname)
1750 else:
1751 orderby.append('_'+colname)
1752 ordercols.append('_'+colname)
1753 else:
1754 if colname == 'id':
1755 orderby.append(colname+' desc')
1756 ordercols.append(colname)
1757 else:
1758 orderby.append('_'+colname+' desc')
1759 ordercols.append('_'+colname)
1761 # construct the SQL
1762 frum = ','.join(frum)
1763 if where:
1764 where = ' where ' + (' and '.join(where))
1765 else:
1766 where = ''
1767 cols = ['id']
1768 if orderby:
1769 cols = cols + ordercols
1770 order = ' order by %s'%(','.join(orderby))
1771 else:
1772 order = ''
1773 cols = ','.join(cols)
1774 sql = 'select %s from %s %s%s%s'%(cols, frum, where, group, order)
1775 args = tuple(args)
1776 if __debug__:
1777 print >>hyperdb.DEBUG, 'filter', (self, sql, args)
1778 self.db.cursor.execute(sql, args)
1779 l = self.db.cursor.fetchall()
1781 # return the IDs (the first column)
1782 # XXX The filter(None, l) bit is sqlite-specific... if there's _NO_
1783 # XXX matches to a fetch, it returns NULL instead of nothing!?!
1784 return filter(None, [row[0] for row in l])
1786 def count(self):
1787 '''Get the number of nodes in this class.
1789 If the returned integer is 'numnodes', the ids of all the nodes
1790 in this class run from 1 to numnodes, and numnodes+1 will be the
1791 id of the next node to be created in this class.
1792 '''
1793 return self.db.countnodes(self.classname)
1795 # Manipulating properties:
1796 def getprops(self, protected=1):
1797 '''Return a dictionary mapping property names to property objects.
1798 If the "protected" flag is true, we include protected properties -
1799 those which may not be modified.
1800 '''
1801 d = self.properties.copy()
1802 if protected:
1803 d['id'] = String()
1804 d['creation'] = hyperdb.Date()
1805 d['activity'] = hyperdb.Date()
1806 d['creator'] = hyperdb.Link('user')
1807 return d
1809 def addprop(self, **properties):
1810 '''Add properties to this class.
1812 The keyword arguments in 'properties' must map names to property
1813 objects, or a TypeError is raised. None of the keys in 'properties'
1814 may collide with the names of existing properties, or a ValueError
1815 is raised before any properties have been added.
1816 '''
1817 for key in properties.keys():
1818 if self.properties.has_key(key):
1819 raise ValueError, key
1820 self.properties.update(properties)
1822 def index(self, nodeid):
1823 '''Add (or refresh) the node to search indexes
1824 '''
1825 # find all the String properties that have indexme
1826 for prop, propclass in self.getprops().items():
1827 if isinstance(propclass, String) and propclass.indexme:
1828 try:
1829 value = str(self.get(nodeid, prop))
1830 except IndexError:
1831 # node no longer exists - entry should be removed
1832 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1833 else:
1834 # and index them under (classname, nodeid, property)
1835 self.db.indexer.add_text((self.classname, nodeid, prop),
1836 value)
1839 #
1840 # Detector interface
1841 #
1842 def audit(self, event, detector):
1843 '''Register a detector
1844 '''
1845 l = self.auditors[event]
1846 if detector not in l:
1847 self.auditors[event].append(detector)
1849 def fireAuditors(self, action, nodeid, newvalues):
1850 '''Fire all registered auditors.
1851 '''
1852 for audit in self.auditors[action]:
1853 audit(self.db, self, nodeid, newvalues)
1855 def react(self, event, detector):
1856 '''Register a detector
1857 '''
1858 l = self.reactors[event]
1859 if detector not in l:
1860 self.reactors[event].append(detector)
1862 def fireReactors(self, action, nodeid, oldvalues):
1863 '''Fire all registered reactors.
1864 '''
1865 for react in self.reactors[action]:
1866 react(self.db, self, nodeid, oldvalues)
1868 class FileClass(Class):
1869 '''This class defines a large chunk of data. To support this, it has a
1870 mandatory String property "content" which is typically saved off
1871 externally to the hyperdb.
1873 The default MIME type of this data is defined by the
1874 "default_mime_type" class attribute, which may be overridden by each
1875 node if the class defines a "type" String property.
1876 '''
1877 default_mime_type = 'text/plain'
1879 def create(self, **propvalues):
1880 ''' snaffle the file propvalue and store in a file
1881 '''
1882 content = propvalues['content']
1883 del propvalues['content']
1884 newid = Class.create(self, **propvalues)
1885 self.db.storefile(self.classname, newid, None, content)
1886 return newid
1888 def import_list(self, propnames, proplist):
1889 ''' Trap the "content" property...
1890 '''
1891 # dupe this list so we don't affect others
1892 propnames = propnames[:]
1894 # extract the "content" property from the proplist
1895 i = propnames.index('content')
1896 content = eval(proplist[i])
1897 del propnames[i]
1898 del proplist[i]
1900 # do the normal import
1901 newid = Class.import_list(self, propnames, proplist)
1903 # save off the "content" file
1904 self.db.storefile(self.classname, newid, None, content)
1905 return newid
1907 _marker = []
1908 def get(self, nodeid, propname, default=_marker, cache=1):
1909 ''' trap the content propname and get it from the file
1910 '''
1912 poss_msg = 'Possibly a access right configuration problem.'
1913 if propname == 'content':
1914 try:
1915 return self.db.getfile(self.classname, nodeid, None)
1916 except IOError, (strerror):
1917 # BUG: by catching this we donot see an error in the log.
1918 return 'ERROR reading file: %s%s\n%s\n%s'%(
1919 self.classname, nodeid, poss_msg, strerror)
1920 if default is not self._marker:
1921 return Class.get(self, nodeid, propname, default, cache=cache)
1922 else:
1923 return Class.get(self, nodeid, propname, cache=cache)
1925 def getprops(self, protected=1):
1926 ''' In addition to the actual properties on the node, these methods
1927 provide the "content" property. If the "protected" flag is true,
1928 we include protected properties - those which may not be
1929 modified.
1930 '''
1931 d = Class.getprops(self, protected=protected).copy()
1932 d['content'] = hyperdb.String()
1933 return d
1935 def index(self, nodeid):
1936 ''' Index the node in the search index.
1938 We want to index the content in addition to the normal String
1939 property indexing.
1940 '''
1941 # perform normal indexing
1942 Class.index(self, nodeid)
1944 # get the content to index
1945 content = self.get(nodeid, 'content')
1947 # figure the mime type
1948 if self.properties.has_key('type'):
1949 mime_type = self.get(nodeid, 'type')
1950 else:
1951 mime_type = self.default_mime_type
1953 # and index!
1954 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
1955 mime_type)
1957 # XXX deviation from spec - was called ItemClass
1958 class IssueClass(Class, roundupdb.IssueClass):
1959 # Overridden methods:
1960 def __init__(self, db, classname, **properties):
1961 '''The newly-created class automatically includes the "messages",
1962 "files", "nosy", and "superseder" properties. If the 'properties'
1963 dictionary attempts to specify any of these properties or a
1964 "creation" or "activity" property, a ValueError is raised.
1965 '''
1966 if not properties.has_key('title'):
1967 properties['title'] = hyperdb.String(indexme='yes')
1968 if not properties.has_key('messages'):
1969 properties['messages'] = hyperdb.Multilink("msg")
1970 if not properties.has_key('files'):
1971 properties['files'] = hyperdb.Multilink("file")
1972 if not properties.has_key('nosy'):
1973 # note: journalling is turned off as it really just wastes
1974 # space. this behaviour may be overridden in an instance
1975 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
1976 if not properties.has_key('superseder'):
1977 properties['superseder'] = hyperdb.Multilink(classname)
1978 Class.__init__(self, db, classname, **properties)