1 # $Id: rdbms_common.py,v 1.9 2002-09-20 05:08:00 richard Exp $
3 # standard python modules
4 import sys, os, time, re, errno, weakref, copy
6 # roundup modules
7 from roundup import hyperdb, date, password, roundupdb, security
8 from roundup.hyperdb import String, Password, Date, Interval, Link, \
9 Multilink, DatabaseError, Boolean, Number
11 # support
12 from blobfiles import FileStorage
13 from roundup.indexer import Indexer
14 from sessions import Sessions
16 # number of rows to keep in memory
17 ROW_CACHE_SIZE = 100
19 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
20 ''' Wrapper around an SQL database that presents a hyperdb interface.
22 - some functionality is specific to the actual SQL database, hence
23 the sql_* methods that are NotImplemented
24 - we keep a cache of the latest ROW_CACHE_SIZE row fetches.
25 '''
26 def __init__(self, config, journaltag=None):
27 ''' Open the database and load the schema from it.
28 '''
29 self.config, self.journaltag = config, journaltag
30 self.dir = config.DATABASE
31 self.classes = {}
32 self.indexer = Indexer(self.dir)
33 self.sessions = Sessions(self.config)
34 self.security = security.Security(self)
36 # additional transaction support for external files and the like
37 self.transactions = []
39 # keep a cache of the N most recently retrieved rows of any kind
40 # (classname, nodeid) = row
41 self.cache = {}
42 self.cache_lru = []
44 # open a connection to the database, creating the "conn" attribute
45 self.open_connection()
47 def open_connection(self):
48 ''' Open a connection to the database, creating it if necessary
49 '''
50 raise NotImplemented
52 def sql(self, cursor, sql, args=None):
53 ''' Execute the sql with the optional args.
54 '''
55 if __debug__:
56 print >>hyperdb.DEBUG, (self, sql, args)
57 if args:
58 cursor.execute(sql, args)
59 else:
60 cursor.execute(sql)
62 def sql_fetchone(self, cursor):
63 ''' Fetch a single row. If there's nothing to fetch, return None.
64 '''
65 raise NotImplemented
67 def sql_stringquote(self, value):
68 ''' Quote the string so it's safe to put in the 'sql quotes'
69 '''
70 return re.sub("'", "''", str(value))
72 def save_dbschema(self, cursor, schema):
73 ''' Save the schema definition that the database currently implements
74 '''
75 raise NotImplemented
77 def load_dbschema(self, cursor):
78 ''' Load the schema definition that the database currently implements
79 '''
80 raise NotImplemented
82 def post_init(self):
83 ''' Called once the schema initialisation has finished.
85 We should now confirm that the schema defined by our "classes"
86 attribute actually matches the schema in the database.
87 '''
88 # now detect changes in the schema
89 save = 0
90 for classname, spec in self.classes.items():
91 if self.database_schema.has_key(classname):
92 dbspec = self.database_schema[classname]
93 if self.update_class(spec, dbspec):
94 self.database_schema[classname] = spec.schema()
95 save = 1
96 else:
97 self.create_class(spec)
98 self.database_schema[classname] = spec.schema()
99 save = 1
101 for classname in self.database_schema.keys():
102 if not self.classes.has_key(classname):
103 self.drop_class(classname)
105 # update the database version of the schema
106 if save:
107 cursor = self.conn.cursor()
108 self.sql(cursor, 'delete from schema')
109 self.save_dbschema(cursor, self.database_schema)
111 # reindex the db if necessary
112 if self.indexer.should_reindex():
113 self.reindex()
115 # commit
116 self.conn.commit()
118 # figure the "curuserid"
119 if self.journaltag is None:
120 self.curuserid = None
121 elif self.journaltag == 'admin':
122 # admin user may not exist, but always has ID 1
123 self.curuserid = '1'
124 else:
125 self.curuserid = self.user.lookup(self.journaltag)
127 def reindex(self):
128 for klass in self.classes.values():
129 for nodeid in klass.list():
130 klass.index(nodeid)
131 self.indexer.save_index()
133 def determine_columns(self, properties):
134 ''' Figure the column names and multilink properties from the spec
136 "properties" is a list of (name, prop) where prop may be an
137 instance of a hyperdb "type" _or_ a string repr of that type.
138 '''
139 cols = ['_activity', '_creator', '_creation']
140 mls = []
141 # add the multilinks separately
142 for col, prop in properties:
143 if isinstance(prop, Multilink):
144 mls.append(col)
145 elif isinstance(prop, type('')) and prop.find('Multilink') != -1:
146 mls.append(col)
147 else:
148 cols.append('_'+col)
149 cols.sort()
150 return cols, mls
152 def update_class(self, spec, dbspec):
153 ''' Determine the differences between the current spec and the
154 database version of the spec, and update where necessary
155 '''
156 spec_schema = spec.schema()
157 if spec_schema == dbspec:
158 # no save needed for this one
159 return 0
160 if __debug__:
161 print >>hyperdb.DEBUG, 'update_class FIRING'
163 # key property changed?
164 if dbspec[0] != spec_schema[0]:
165 if __debug__:
166 print >>hyperdb.DEBUG, 'update_class setting keyprop', `spec[0]`
167 # XXX turn on indexing for the key property
169 # dict 'em up
170 spec_propnames,spec_props = [],{}
171 for propname,prop in spec_schema[1]:
172 spec_propnames.append(propname)
173 spec_props[propname] = prop
174 dbspec_propnames,dbspec_props = [],{}
175 for propname,prop in dbspec[1]:
176 dbspec_propnames.append(propname)
177 dbspec_props[propname] = prop
179 # we're going to need one of these
180 cursor = self.conn.cursor()
182 # now compare
183 for propname in spec_propnames:
184 prop = spec_props[propname]
185 if dbspec_props.has_key(propname) and prop==dbspec_props[propname]:
186 continue
187 if __debug__:
188 print >>hyperdb.DEBUG, 'update_class ADD', (propname, prop)
190 if not dbspec_props.has_key(propname):
191 # add the property
192 if isinstance(prop, Multilink):
193 # all we have to do here is create a new table, easy!
194 self.create_multilink_table(cursor, spec, propname)
195 continue
197 # no ALTER TABLE, so we:
198 # 1. pull out the data, including an extra None column
199 oldcols, x = self.determine_columns(dbspec[1])
200 oldcols.append('id')
201 oldcols.append('__retired__')
202 cn = spec.classname
203 sql = 'select %s,%s from _%s'%(','.join(oldcols), self.arg, cn)
204 if __debug__:
205 print >>hyperdb.DEBUG, 'update_class', (self, sql, None)
206 cursor.execute(sql, (None,))
207 olddata = cursor.fetchall()
209 # 2. drop the old table
210 cursor.execute('drop table _%s'%cn)
212 # 3. create the new table
213 cols, mls = self.create_class_table(cursor, spec)
214 # ensure the new column is last
215 cols.remove('_'+propname)
216 assert oldcols == cols, "Column lists don't match!"
217 cols.append('_'+propname)
219 # 4. populate with the data from step one
220 s = ','.join([self.arg for x in cols])
221 scols = ','.join(cols)
222 sql = 'insert into _%s (%s) values (%s)'%(cn, scols, s)
224 # GAH, nothing had better go wrong from here on in... but
225 # we have to commit the drop...
226 # XXX this isn't necessary in sqlite :(
227 self.conn.commit()
229 # do the insert
230 for row in olddata:
231 self.sql(cursor, sql, tuple(row))
233 else:
234 # modify the property
235 if __debug__:
236 print >>hyperdb.DEBUG, 'update_class NOOP'
237 pass # NOOP in gadfly
239 # and the other way - only worry about deletions here
240 for propname in dbspec_propnames:
241 prop = dbspec_props[propname]
242 if spec_props.has_key(propname):
243 continue
244 if __debug__:
245 print >>hyperdb.DEBUG, 'update_class REMOVE', `prop`
247 # delete the property
248 if isinstance(prop, Multilink):
249 sql = 'drop table %s_%s'%(spec.classname, prop)
250 if __debug__:
251 print >>hyperdb.DEBUG, 'update_class', (self, sql)
252 cursor.execute(sql)
253 else:
254 # no ALTER TABLE, so we:
255 # 1. pull out the data, excluding the removed column
256 oldcols, x = self.determine_columns(spec.properties.items())
257 oldcols.append('id')
258 oldcols.append('__retired__')
259 # remove the missing column
260 oldcols.remove('_'+propname)
261 cn = spec.classname
262 sql = 'select %s from _%s'%(','.join(oldcols), cn)
263 cursor.execute(sql, (None,))
264 olddata = sql.fetchall()
266 # 2. drop the old table
267 cursor.execute('drop table _%s'%cn)
269 # 3. create the new table
270 cols, mls = self.create_class_table(self, cursor, spec)
271 assert oldcols != cols, "Column lists don't match!"
273 # 4. populate with the data from step one
274 qs = ','.join([self.arg for x in cols])
275 sql = 'insert into _%s values (%s)'%(cn, s)
276 cursor.execute(sql, olddata)
277 return 1
279 def create_class_table(self, cursor, spec):
280 ''' create the class table for the given spec
281 '''
282 cols, mls = self.determine_columns(spec.properties.items())
284 # add on our special columns
285 cols.append('id')
286 cols.append('__retired__')
288 # create the base table
289 scols = ','.join(['%s varchar'%x for x in cols])
290 sql = 'create table _%s (%s)'%(spec.classname, scols)
291 if __debug__:
292 print >>hyperdb.DEBUG, 'create_class', (self, sql)
293 cursor.execute(sql)
295 return cols, mls
297 def create_journal_table(self, cursor, spec):
298 ''' create the journal table for a class given the spec and
299 already-determined cols
300 '''
301 # journal table
302 cols = ','.join(['%s varchar'%x
303 for x in 'nodeid date tag action params'.split()])
304 sql = 'create table %s__journal (%s)'%(spec.classname, cols)
305 if __debug__:
306 print >>hyperdb.DEBUG, 'create_class', (self, sql)
307 cursor.execute(sql)
309 def create_multilink_table(self, cursor, spec, ml):
310 ''' Create a multilink table for the "ml" property of the class
311 given by the spec
312 '''
313 sql = 'create table %s_%s (linkid varchar, nodeid varchar)'%(
314 spec.classname, ml)
315 if __debug__:
316 print >>hyperdb.DEBUG, 'create_class', (self, sql)
317 cursor.execute(sql)
319 def create_class(self, spec):
320 ''' Create a database table according to the given spec.
321 '''
322 cursor = self.conn.cursor()
323 cols, mls = self.create_class_table(cursor, spec)
324 self.create_journal_table(cursor, spec)
326 # now create the multilink tables
327 for ml in mls:
328 self.create_multilink_table(cursor, spec, ml)
330 # ID counter
331 sql = 'insert into ids (name, num) values (%s,%s)'%(self.arg, self.arg)
332 vals = (spec.classname, 1)
333 if __debug__:
334 print >>hyperdb.DEBUG, 'create_class', (self, sql, vals)
335 cursor.execute(sql, vals)
337 def drop_class(self, spec):
338 ''' Drop the given table from the database.
340 Drop the journal and multilink tables too.
341 '''
342 # figure the multilinks
343 mls = []
344 for col, prop in spec.properties.items():
345 if isinstance(prop, Multilink):
346 mls.append(col)
347 cursor = self.conn.cursor()
349 sql = 'drop table _%s'%spec.classname
350 if __debug__:
351 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
352 cursor.execute(sql)
354 sql = 'drop table %s__journal'%spec.classname
355 if __debug__:
356 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
357 cursor.execute(sql)
359 for ml in mls:
360 sql = 'drop table %s_%s'%(spec.classname, ml)
361 if __debug__:
362 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
363 cursor.execute(sql)
365 #
366 # Classes
367 #
368 def __getattr__(self, classname):
369 ''' A convenient way of calling self.getclass(classname).
370 '''
371 if self.classes.has_key(classname):
372 if __debug__:
373 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
374 return self.classes[classname]
375 raise AttributeError, classname
377 def addclass(self, cl):
378 ''' Add a Class to the hyperdatabase.
379 '''
380 if __debug__:
381 print >>hyperdb.DEBUG, 'addclass', (self, cl)
382 cn = cl.classname
383 if self.classes.has_key(cn):
384 raise ValueError, cn
385 self.classes[cn] = cl
387 def getclasses(self):
388 ''' Return a list of the names of all existing classes.
389 '''
390 if __debug__:
391 print >>hyperdb.DEBUG, 'getclasses', (self,)
392 l = self.classes.keys()
393 l.sort()
394 return l
396 def getclass(self, classname):
397 '''Get the Class object representing a particular class.
399 If 'classname' is not a valid class name, a KeyError is raised.
400 '''
401 if __debug__:
402 print >>hyperdb.DEBUG, 'getclass', (self, classname)
403 try:
404 return self.classes[classname]
405 except KeyError:
406 raise KeyError, 'There is no class called "%s"'%classname
408 def clear(self):
409 ''' Delete all database contents.
411 Note: I don't commit here, which is different behaviour to the
412 "nuke from orbit" behaviour in the *dbms.
413 '''
414 if __debug__:
415 print >>hyperdb.DEBUG, 'clear', (self,)
416 cursor = self.conn.cursor()
417 for cn in self.classes.keys():
418 sql = 'delete from _%s'%cn
419 if __debug__:
420 print >>hyperdb.DEBUG, 'clear', (self, sql)
421 cursor.execute(sql)
423 #
424 # Node IDs
425 #
426 def newid(self, classname):
427 ''' Generate a new id for the given class
428 '''
429 # get the next ID
430 cursor = self.conn.cursor()
431 sql = 'select num from ids where name=%s'%self.arg
432 if __debug__:
433 print >>hyperdb.DEBUG, 'newid', (self, sql, classname)
434 cursor.execute(sql, (classname, ))
435 newid = cursor.fetchone()[0]
437 # update the counter
438 sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
439 vals = (int(newid)+1, classname)
440 if __debug__:
441 print >>hyperdb.DEBUG, 'newid', (self, sql, vals)
442 cursor.execute(sql, vals)
444 # return as string
445 return str(newid)
447 def setid(self, classname, setid):
448 ''' Set the id counter: used during import of database
449 '''
450 cursor = self.conn.cursor()
451 sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
452 vals = (setid, classname)
453 if __debug__:
454 print >>hyperdb.DEBUG, 'setid', (self, sql, vals)
455 cursor.execute(sql, vals)
457 #
458 # Nodes
459 #
461 def addnode(self, classname, nodeid, node):
462 ''' Add the specified node to its class's db.
463 '''
464 if __debug__:
465 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
466 # gadfly requires values for all non-multilink columns
467 cl = self.classes[classname]
468 cols, mls = self.determine_columns(cl.properties.items())
470 # add the special props
471 node = node.copy()
472 node['creation'] = node['activity'] = date.Date()
473 node['creator'] = self.curuserid
475 # default the non-multilink columns
476 for col, prop in cl.properties.items():
477 if not isinstance(col, Multilink):
478 if not node.has_key(col):
479 node[col] = None
481 # clear this node out of the cache if it's in there
482 key = (classname, nodeid)
483 if self.cache.has_key(key):
484 del self.cache[key]
485 self.cache_lru.remove(key)
487 # make the node data safe for the DB
488 node = self.serialise(classname, node)
490 # make sure the ordering is correct for column name -> column value
491 vals = tuple([node[col[1:]] for col in cols]) + (nodeid, 0)
492 s = ','.join([self.arg for x in cols]) + ',%s,%s'%(self.arg, self.arg)
493 cols = ','.join(cols) + ',id,__retired__'
495 # perform the inserts
496 cursor = self.conn.cursor()
497 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
498 if __debug__:
499 print >>hyperdb.DEBUG, 'addnode', (self, sql, vals)
500 cursor.execute(sql, vals)
502 # insert the multilink rows
503 for col in mls:
504 t = '%s_%s'%(classname, col)
505 for entry in node[col]:
506 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
507 self.arg, self.arg)
508 self.sql(cursor, sql, (entry, nodeid))
510 # make sure we do the commit-time extra stuff for this node
511 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
513 def setnode(self, classname, nodeid, values, multilink_changes):
514 ''' Change the specified node.
515 '''
516 if __debug__:
517 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, values)
519 # clear this node out of the cache if it's in there
520 key = (classname, nodeid)
521 if self.cache.has_key(key):
522 del self.cache[key]
523 self.cache_lru.remove(key)
525 # add the special props
526 values = values.copy()
527 values['activity'] = date.Date()
529 # make db-friendly
530 values = self.serialise(classname, values)
532 cl = self.classes[classname]
533 cols = []
534 mls = []
535 # add the multilinks separately
536 props = cl.getprops()
537 for col in values.keys():
538 prop = props[col]
539 if isinstance(prop, Multilink):
540 mls.append(col)
541 else:
542 cols.append('_'+col)
543 cols.sort()
545 cursor = self.conn.cursor()
547 # if there's any updates to regular columns, do them
548 if cols:
549 # make sure the ordering is correct for column name -> column value
550 sqlvals = tuple([values[col[1:]] for col in cols]) + (nodeid,)
551 s = ','.join(['%s=%s'%(x, self.arg) for x in cols])
552 cols = ','.join(cols)
554 # perform the update
555 sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
556 if __debug__:
557 print >>hyperdb.DEBUG, 'setnode', (self, sql, sqlvals)
558 cursor.execute(sql, sqlvals)
560 # now the fun bit, updating the multilinks ;)
561 for col, (add, remove) in multilink_changes.items():
562 tn = '%s_%s'%(classname, col)
563 if add:
564 sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
565 self.arg, self.arg)
566 for addid in add:
567 self.sql(cursor, sql, (nodeid, addid))
568 if remove:
569 sql = 'delete from %s where nodeid=%s and linkid=%s'%(tn,
570 self.arg, self.arg)
571 for removeid in remove:
572 self.sql(cursor, sql, (nodeid, removeid))
574 # make sure we do the commit-time extra stuff for this node
575 self.transactions.append((self.doSaveNode, (classname, nodeid, values)))
577 def getnode(self, classname, nodeid):
578 ''' Get a node from the database.
579 '''
580 if __debug__:
581 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid)
583 # see if we have this node cached
584 key = (classname, nodeid)
585 if self.cache.has_key(key):
586 # push us back to the top of the LRU
587 self.cache_lru.remove(key)
588 self.cache_lry.insert(0, key)
589 # return the cached information
590 return self.cache[key]
592 # figure the columns we're fetching
593 cl = self.classes[classname]
594 cols, mls = self.determine_columns(cl.properties.items())
595 scols = ','.join(cols)
597 # perform the basic property fetch
598 cursor = self.conn.cursor()
599 sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
600 self.sql(cursor, sql, (nodeid,))
602 values = self.sql_fetchone(cursor)
603 if values is None:
604 raise IndexError, 'no such %s node %s'%(classname, nodeid)
606 # make up the node
607 node = {}
608 for col in range(len(cols)):
609 node[cols[col][1:]] = values[col]
611 # now the multilinks
612 for col in mls:
613 # get the link ids
614 sql = 'select linkid from %s_%s where nodeid=%s'%(classname, col,
615 self.arg)
616 cursor.execute(sql, (nodeid,))
617 # extract the first column from the result
618 node[col] = [x[0] for x in cursor.fetchall()]
620 # un-dbificate the node data
621 node = self.unserialise(classname, node)
623 # save off in the cache
624 key = (classname, nodeid)
625 self.cache[key] = node
626 # update the LRU
627 self.cache_lru.insert(0, key)
628 del self.cache[self.cache_lru.pop()]
630 return node
632 def destroynode(self, classname, nodeid):
633 '''Remove a node from the database. Called exclusively by the
634 destroy() method on Class.
635 '''
636 if __debug__:
637 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
639 # make sure the node exists
640 if not self.hasnode(classname, nodeid):
641 raise IndexError, '%s has no node %s'%(classname, nodeid)
643 # see if we have this node cached
644 if self.cache.has_key((classname, nodeid)):
645 del self.cache[(classname, nodeid)]
647 # see if there's any obvious commit actions that we should get rid of
648 for entry in self.transactions[:]:
649 if entry[1][:2] == (classname, nodeid):
650 self.transactions.remove(entry)
652 # now do the SQL
653 cursor = self.conn.cursor()
654 sql = 'delete from _%s where id=%s'%(classname, self.arg)
655 self.sql(cursor, sql, (nodeid,))
657 # remove from multilnks
658 cl = self.getclass(classname)
659 x, mls = self.determine_columns(cl.properties.items())
660 for col in mls:
661 # get the link ids
662 sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
663 cursor.execute(sql, (nodeid,))
665 # remove journal entries
666 sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
667 self.sql(cursor, sql, (nodeid,))
669 def serialise(self, classname, node):
670 '''Copy the node contents, converting non-marshallable data into
671 marshallable data.
672 '''
673 if __debug__:
674 print >>hyperdb.DEBUG, 'serialise', classname, node
675 properties = self.getclass(classname).getprops()
676 d = {}
677 for k, v in node.items():
678 # if the property doesn't exist, or is the "retired" flag then
679 # it won't be in the properties dict
680 if not properties.has_key(k):
681 d[k] = v
682 continue
684 # get the property spec
685 prop = properties[k]
687 if isinstance(prop, Password):
688 d[k] = str(v)
689 elif isinstance(prop, Date) and v is not None:
690 d[k] = v.serialise()
691 elif isinstance(prop, Interval) and v is not None:
692 d[k] = v.serialise()
693 else:
694 d[k] = v
695 return d
697 def unserialise(self, classname, node):
698 '''Decode the marshalled node data
699 '''
700 if __debug__:
701 print >>hyperdb.DEBUG, 'unserialise', classname, node
702 properties = self.getclass(classname).getprops()
703 d = {}
704 for k, v in node.items():
705 # if the property doesn't exist, or is the "retired" flag then
706 # it won't be in the properties dict
707 if not properties.has_key(k):
708 d[k] = v
709 continue
711 # get the property spec
712 prop = properties[k]
714 if isinstance(prop, Date) and v is not None:
715 d[k] = date.Date(v)
716 elif isinstance(prop, Interval) and v is not None:
717 d[k] = date.Interval(v)
718 elif isinstance(prop, Password):
719 p = password.Password()
720 p.unpack(v)
721 d[k] = p
722 else:
723 d[k] = v
724 return d
726 def hasnode(self, classname, nodeid):
727 ''' Determine if the database has a given node.
728 '''
729 cursor = self.conn.cursor()
730 sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
731 if __debug__:
732 print >>hyperdb.DEBUG, 'hasnode', (self, sql, nodeid)
733 cursor.execute(sql, (nodeid,))
734 return int(cursor.fetchone()[0])
736 def countnodes(self, classname):
737 ''' Count the number of nodes that exist for a particular Class.
738 '''
739 cursor = self.conn.cursor()
740 sql = 'select count(*) from _%s'%classname
741 if __debug__:
742 print >>hyperdb.DEBUG, 'countnodes', (self, sql)
743 cursor.execute(sql)
744 return cursor.fetchone()[0]
746 def getnodeids(self, classname, retired=0):
747 ''' Retrieve all the ids of the nodes for a particular Class.
749 Set retired=None to get all nodes. Otherwise it'll get all the
750 retired or non-retired nodes, depending on the flag.
751 '''
752 cursor = self.conn.cursor()
753 # flip the sense of the flag if we don't want all of them
754 if retired is not None:
755 retired = not retired
756 sql = 'select id from _%s where __retired__ <> %s'%(classname, self.arg)
757 if __debug__:
758 print >>hyperdb.DEBUG, 'getnodeids', (self, sql, retired)
759 cursor.execute(sql, (retired,))
760 return [x[0] for x in cursor.fetchall()]
762 def addjournal(self, classname, nodeid, action, params, creator=None,
763 creation=None):
764 ''' Journal the Action
765 'action' may be:
767 'create' or 'set' -- 'params' is a dictionary of property values
768 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
769 'retire' -- 'params' is None
770 '''
771 # serialise the parameters now if necessary
772 if isinstance(params, type({})):
773 if action in ('set', 'create'):
774 params = self.serialise(classname, params)
776 # handle supply of the special journalling parameters (usually
777 # supplied on importing an existing database)
778 if creator:
779 journaltag = creator
780 else:
781 journaltag = self.curuserid
782 if creation:
783 journaldate = creation.serialise()
784 else:
785 journaldate = date.Date().serialise()
787 # create the journal entry
788 cols = ','.join('nodeid date tag action params'.split())
790 if __debug__:
791 print >>hyperdb.DEBUG, 'addjournal', (nodeid, journaldate,
792 journaltag, action, params)
794 cursor = self.conn.cursor()
795 self.save_journal(cursor, classname, cols, nodeid, journaldate,
796 journaltag, action, params)
798 def save_journal(self, cursor, classname, cols, nodeid, journaldate,
799 journaltag, action, params):
800 ''' Save the journal entry to the database
801 '''
802 raise NotImplemented
804 def getjournal(self, classname, nodeid):
805 ''' get the journal for id
806 '''
807 # make sure the node exists
808 if not self.hasnode(classname, nodeid):
809 raise IndexError, '%s has no node %s'%(classname, nodeid)
811 cursor = self.conn.cursor()
812 cols = ','.join('nodeid date tag action params'.split())
813 return self.load_journal(cursor, classname, cols, nodeid)
815 def load_journal(self, cursor, classname, cols, nodeid):
816 ''' Load the journal from the database
817 '''
818 raise NotImplemented
820 def pack(self, pack_before):
821 ''' Delete all journal entries except "create" before 'pack_before'.
822 '''
823 # get a 'yyyymmddhhmmss' version of the date
824 date_stamp = pack_before.serialise()
826 # do the delete
827 cursor = self.conn.cursor()
828 for classname in self.classes.keys():
829 sql = "delete from %s__journal where date<%s and "\
830 "action<>'create'"%(classname, self.arg)
831 if __debug__:
832 print >>hyperdb.DEBUG, 'pack', (self, sql, date_stamp)
833 cursor.execute(sql, (date_stamp,))
835 def sql_commit(self):
836 ''' Actually commit to the database.
837 '''
838 self.conn.commit()
840 def commit(self):
841 ''' Commit the current transactions.
843 Save all data changed since the database was opened or since the
844 last commit() or rollback().
845 '''
846 if __debug__:
847 print >>hyperdb.DEBUG, 'commit', (self,)
849 # commit the database
850 self.sql_commit()
852 # now, do all the other transaction stuff
853 reindex = {}
854 for method, args in self.transactions:
855 reindex[method(*args)] = 1
857 # reindex the nodes that request it
858 for classname, nodeid in filter(None, reindex.keys()):
859 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
860 self.getclass(classname).index(nodeid)
862 # save the indexer state
863 self.indexer.save_index()
865 # clear out the transactions
866 self.transactions = []
868 def rollback(self):
869 ''' Reverse all actions from the current transaction.
871 Undo all the changes made since the database was opened or the last
872 commit() or rollback() was performed.
873 '''
874 if __debug__:
875 print >>hyperdb.DEBUG, 'rollback', (self,)
877 # roll back
878 self.conn.rollback()
880 # roll back "other" transaction stuff
881 for method, args in self.transactions:
882 # delete temporary files
883 if method == self.doStoreFile:
884 self.rollbackStoreFile(*args)
885 self.transactions = []
887 def doSaveNode(self, classname, nodeid, node):
888 ''' dummy that just generates a reindex event
889 '''
890 # return the classname, nodeid so we reindex this content
891 return (classname, nodeid)
893 def close(self):
894 ''' Close off the connection.
895 '''
896 self.conn.close()
898 #
899 # The base Class class
900 #
901 class Class(hyperdb.Class):
902 ''' The handle to a particular class of nodes in a hyperdatabase.
904 All methods except __repr__ and getnode must be implemented by a
905 concrete backend Class.
906 '''
908 def __init__(self, db, classname, **properties):
909 '''Create a new class with a given name and property specification.
911 'classname' must not collide with the name of an existing class,
912 or a ValueError is raised. The keyword arguments in 'properties'
913 must map names to property objects, or a TypeError is raised.
914 '''
915 if (properties.has_key('creation') or properties.has_key('activity')
916 or properties.has_key('creator')):
917 raise ValueError, '"creation", "activity" and "creator" are '\
918 'reserved'
920 self.classname = classname
921 self.properties = properties
922 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
923 self.key = ''
925 # should we journal changes (default yes)
926 self.do_journal = 1
928 # do the db-related init stuff
929 db.addclass(self)
931 self.auditors = {'create': [], 'set': [], 'retire': []}
932 self.reactors = {'create': [], 'set': [], 'retire': []}
934 def schema(self):
935 ''' A dumpable version of the schema that we can store in the
936 database
937 '''
938 return (self.key, [(x, repr(y)) for x,y in self.properties.items()])
940 def enableJournalling(self):
941 '''Turn journalling on for this class
942 '''
943 self.do_journal = 1
945 def disableJournalling(self):
946 '''Turn journalling off for this class
947 '''
948 self.do_journal = 0
950 # Editing nodes:
951 def create(self, **propvalues):
952 ''' Create a new node of this class and return its id.
954 The keyword arguments in 'propvalues' map property names to values.
956 The values of arguments must be acceptable for the types of their
957 corresponding properties or a TypeError is raised.
959 If this class has a key property, it must be present and its value
960 must not collide with other key strings or a ValueError is raised.
962 Any other properties on this class that are missing from the
963 'propvalues' dictionary are set to None.
965 If an id in a link or multilink property does not refer to a valid
966 node, an IndexError is raised.
967 '''
968 if propvalues.has_key('id'):
969 raise KeyError, '"id" is reserved'
971 if self.db.journaltag is None:
972 raise DatabaseError, 'Database open read-only'
974 if propvalues.has_key('creation') or propvalues.has_key('activity'):
975 raise KeyError, '"creation" and "activity" are reserved'
977 self.fireAuditors('create', None, propvalues)
979 # new node's id
980 newid = self.db.newid(self.classname)
982 # validate propvalues
983 num_re = re.compile('^\d+$')
984 for key, value in propvalues.items():
985 if key == self.key:
986 try:
987 self.lookup(value)
988 except KeyError:
989 pass
990 else:
991 raise ValueError, 'node with key "%s" exists'%value
993 # try to handle this property
994 try:
995 prop = self.properties[key]
996 except KeyError:
997 raise KeyError, '"%s" has no property "%s"'%(self.classname,
998 key)
1000 if value is not None and isinstance(prop, Link):
1001 if type(value) != type(''):
1002 raise ValueError, 'link value must be String'
1003 link_class = self.properties[key].classname
1004 # if it isn't a number, it's a key
1005 if not num_re.match(value):
1006 try:
1007 value = self.db.classes[link_class].lookup(value)
1008 except (TypeError, KeyError):
1009 raise IndexError, 'new property "%s": %s not a %s'%(
1010 key, value, link_class)
1011 elif not self.db.getclass(link_class).hasnode(value):
1012 raise IndexError, '%s has no node %s'%(link_class, value)
1014 # save off the value
1015 propvalues[key] = value
1017 # register the link with the newly linked node
1018 if self.do_journal and self.properties[key].do_journal:
1019 self.db.addjournal(link_class, value, 'link',
1020 (self.classname, newid, key))
1022 elif isinstance(prop, Multilink):
1023 if type(value) != type([]):
1024 raise TypeError, 'new property "%s" not a list of ids'%key
1026 # clean up and validate the list of links
1027 link_class = self.properties[key].classname
1028 l = []
1029 for entry in value:
1030 if type(entry) != type(''):
1031 raise ValueError, '"%s" multilink value (%r) '\
1032 'must contain Strings'%(key, value)
1033 # if it isn't a number, it's a key
1034 if not num_re.match(entry):
1035 try:
1036 entry = self.db.classes[link_class].lookup(entry)
1037 except (TypeError, KeyError):
1038 raise IndexError, 'new property "%s": %s not a %s'%(
1039 key, entry, self.properties[key].classname)
1040 l.append(entry)
1041 value = l
1042 propvalues[key] = value
1044 # handle additions
1045 for nodeid in value:
1046 if not self.db.getclass(link_class).hasnode(nodeid):
1047 raise IndexError, '%s has no node %s'%(link_class,
1048 nodeid)
1049 # register the link with the newly linked node
1050 if self.do_journal and self.properties[key].do_journal:
1051 self.db.addjournal(link_class, nodeid, 'link',
1052 (self.classname, newid, key))
1054 elif isinstance(prop, String):
1055 if type(value) != type(''):
1056 raise TypeError, 'new property "%s" not a string'%key
1058 elif isinstance(prop, Password):
1059 if not isinstance(value, password.Password):
1060 raise TypeError, 'new property "%s" not a Password'%key
1062 elif isinstance(prop, Date):
1063 if value is not None and not isinstance(value, date.Date):
1064 raise TypeError, 'new property "%s" not a Date'%key
1066 elif isinstance(prop, Interval):
1067 if value is not None and not isinstance(value, date.Interval):
1068 raise TypeError, 'new property "%s" not an Interval'%key
1070 elif value is not None and isinstance(prop, Number):
1071 try:
1072 float(value)
1073 except ValueError:
1074 raise TypeError, 'new property "%s" not numeric'%key
1076 elif value is not None and isinstance(prop, Boolean):
1077 try:
1078 int(value)
1079 except ValueError:
1080 raise TypeError, 'new property "%s" not boolean'%key
1082 # make sure there's data where there needs to be
1083 for key, prop in self.properties.items():
1084 if propvalues.has_key(key):
1085 continue
1086 if key == self.key:
1087 raise ValueError, 'key property "%s" is required'%key
1088 if isinstance(prop, Multilink):
1089 propvalues[key] = []
1090 else:
1091 propvalues[key] = None
1093 # done
1094 self.db.addnode(self.classname, newid, propvalues)
1095 if self.do_journal:
1096 self.db.addjournal(self.classname, newid, 'create', propvalues)
1098 self.fireReactors('create', newid, None)
1100 return newid
1102 def export_list(self, propnames, nodeid):
1103 ''' Export a node - generate a list of CSV-able data in the order
1104 specified by propnames for the given node.
1105 '''
1106 properties = self.getprops()
1107 l = []
1108 for prop in propnames:
1109 proptype = properties[prop]
1110 value = self.get(nodeid, prop)
1111 # "marshal" data where needed
1112 if value is None:
1113 pass
1114 elif isinstance(proptype, hyperdb.Date):
1115 value = value.get_tuple()
1116 elif isinstance(proptype, hyperdb.Interval):
1117 value = value.get_tuple()
1118 elif isinstance(proptype, hyperdb.Password):
1119 value = str(value)
1120 l.append(repr(value))
1121 return l
1123 def import_list(self, propnames, proplist):
1124 ''' Import a node - all information including "id" is present and
1125 should not be sanity checked. Triggers are not triggered. The
1126 journal should be initialised using the "creator" and "created"
1127 information.
1129 Return the nodeid of the node imported.
1130 '''
1131 if self.db.journaltag is None:
1132 raise DatabaseError, 'Database open read-only'
1133 properties = self.getprops()
1135 # make the new node's property map
1136 d = {}
1137 for i in range(len(propnames)):
1138 # Use eval to reverse the repr() used to output the CSV
1139 value = eval(proplist[i])
1141 # Figure the property for this column
1142 propname = propnames[i]
1143 prop = properties[propname]
1145 # "unmarshal" where necessary
1146 if propname == 'id':
1147 newid = value
1148 continue
1149 elif value is None:
1150 # don't set Nones
1151 continue
1152 elif isinstance(prop, hyperdb.Date):
1153 value = date.Date(value)
1154 elif isinstance(prop, hyperdb.Interval):
1155 value = date.Interval(value)
1156 elif isinstance(prop, hyperdb.Password):
1157 pwd = password.Password()
1158 pwd.unpack(value)
1159 value = pwd
1160 d[propname] = value
1162 # extract the extraneous journalling gumpf and nuke it
1163 if d.has_key('creator'):
1164 creator = d['creator']
1165 del d['creator']
1166 if d.has_key('creation'):
1167 creation = d['creation']
1168 del d['creation']
1169 if d.has_key('activity'):
1170 del d['activity']
1172 # add the node and journal
1173 self.db.addnode(self.classname, newid, d)
1174 self.db.addjournal(self.classname, newid, 'create', d, creator,
1175 creation)
1176 return newid
1178 _marker = []
1179 def get(self, nodeid, propname, default=_marker, cache=1):
1180 '''Get the value of a property on an existing node of this class.
1182 'nodeid' must be the id of an existing node of this class or an
1183 IndexError is raised. 'propname' must be the name of a property
1184 of this class or a KeyError is raised.
1186 'cache' indicates whether the transaction cache should be queried
1187 for the node. If the node has been modified and you need to
1188 determine what its values prior to modification are, you need to
1189 set cache=0.
1190 '''
1191 if propname == 'id':
1192 return nodeid
1194 # get the node's dict
1195 d = self.db.getnode(self.classname, nodeid)
1197 if propname == 'creation':
1198 if d.has_key('creation'):
1199 return d['creation']
1200 else:
1201 return date.Date()
1202 if propname == 'activity':
1203 if d.has_key('activity'):
1204 return d['activity']
1205 else:
1206 return date.Date()
1207 if propname == 'creator':
1208 if d.has_key('creator'):
1209 return d['creator']
1210 else:
1211 return self.db.curuserid
1213 # get the property (raises KeyErorr if invalid)
1214 prop = self.properties[propname]
1216 if not d.has_key(propname):
1217 if default is self._marker:
1218 if isinstance(prop, Multilink):
1219 return []
1220 else:
1221 return None
1222 else:
1223 return default
1225 # don't pass our list to other code
1226 if isinstance(prop, Multilink):
1227 return d[propname][:]
1229 return d[propname]
1231 def getnode(self, nodeid, cache=1):
1232 ''' Return a convenience wrapper for the node.
1234 'nodeid' must be the id of an existing node of this class or an
1235 IndexError is raised.
1237 'cache' indicates whether the transaction cache should be queried
1238 for the node. If the node has been modified and you need to
1239 determine what its values prior to modification are, you need to
1240 set cache=0.
1241 '''
1242 return Node(self, nodeid, cache=cache)
1244 def set(self, nodeid, **propvalues):
1245 '''Modify a property on an existing node of this class.
1247 'nodeid' must be the id of an existing node of this class or an
1248 IndexError is raised.
1250 Each key in 'propvalues' must be the name of a property of this
1251 class or a KeyError is raised.
1253 All values in 'propvalues' must be acceptable types for their
1254 corresponding properties or a TypeError is raised.
1256 If the value of the key property is set, it must not collide with
1257 other key strings or a ValueError is raised.
1259 If the value of a Link or Multilink property contains an invalid
1260 node id, a ValueError is raised.
1261 '''
1262 if not propvalues:
1263 return propvalues
1265 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1266 raise KeyError, '"creation" and "activity" are reserved'
1268 if propvalues.has_key('id'):
1269 raise KeyError, '"id" is reserved'
1271 if self.db.journaltag is None:
1272 raise DatabaseError, 'Database open read-only'
1274 self.fireAuditors('set', nodeid, propvalues)
1275 # Take a copy of the node dict so that the subsequent set
1276 # operation doesn't modify the oldvalues structure.
1277 # XXX used to try the cache here first
1278 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1280 node = self.db.getnode(self.classname, nodeid)
1281 if self.is_retired(nodeid):
1282 raise IndexError, 'Requested item is retired'
1283 num_re = re.compile('^\d+$')
1285 # if the journal value is to be different, store it in here
1286 journalvalues = {}
1288 # remember the add/remove stuff for multilinks, making it easier
1289 # for the Database layer to do its stuff
1290 multilink_changes = {}
1292 for propname, value in propvalues.items():
1293 # check to make sure we're not duplicating an existing key
1294 if propname == self.key and node[propname] != value:
1295 try:
1296 self.lookup(value)
1297 except KeyError:
1298 pass
1299 else:
1300 raise ValueError, 'node with key "%s" exists'%value
1302 # this will raise the KeyError if the property isn't valid
1303 # ... we don't use getprops() here because we only care about
1304 # the writeable properties.
1305 try:
1306 prop = self.properties[propname]
1307 except KeyError:
1308 raise KeyError, '"%s" has no property named "%s"'%(
1309 self.classname, propname)
1311 # if the value's the same as the existing value, no sense in
1312 # doing anything
1313 if node.has_key(propname) and value == node[propname]:
1314 del propvalues[propname]
1315 continue
1317 # do stuff based on the prop type
1318 if isinstance(prop, Link):
1319 link_class = prop.classname
1320 # if it isn't a number, it's a key
1321 if value is not None and not isinstance(value, type('')):
1322 raise ValueError, 'property "%s" link value be a string'%(
1323 propname)
1324 if isinstance(value, type('')) and not num_re.match(value):
1325 try:
1326 value = self.db.classes[link_class].lookup(value)
1327 except (TypeError, KeyError):
1328 raise IndexError, 'new property "%s": %s not a %s'%(
1329 propname, value, prop.classname)
1331 if (value is not None and
1332 not self.db.getclass(link_class).hasnode(value)):
1333 raise IndexError, '%s has no node %s'%(link_class, value)
1335 if self.do_journal and prop.do_journal:
1336 # register the unlink with the old linked node
1337 if node[propname] is not None:
1338 self.db.addjournal(link_class, node[propname], 'unlink',
1339 (self.classname, nodeid, propname))
1341 # register the link with the newly linked node
1342 if value is not None:
1343 self.db.addjournal(link_class, value, 'link',
1344 (self.classname, nodeid, propname))
1346 elif isinstance(prop, Multilink):
1347 if type(value) != type([]):
1348 raise TypeError, 'new property "%s" not a list of'\
1349 ' ids'%propname
1350 link_class = self.properties[propname].classname
1351 l = []
1352 for entry in value:
1353 # if it isn't a number, it's a key
1354 if type(entry) != type(''):
1355 raise ValueError, 'new property "%s" link value ' \
1356 'must be a string'%propname
1357 if not num_re.match(entry):
1358 try:
1359 entry = self.db.classes[link_class].lookup(entry)
1360 except (TypeError, KeyError):
1361 raise IndexError, 'new property "%s": %s not a %s'%(
1362 propname, entry,
1363 self.properties[propname].classname)
1364 l.append(entry)
1365 value = l
1366 propvalues[propname] = value
1368 # figure the journal entry for this property
1369 add = []
1370 remove = []
1372 # handle removals
1373 if node.has_key(propname):
1374 l = node[propname]
1375 else:
1376 l = []
1377 for id in l[:]:
1378 if id in value:
1379 continue
1380 # register the unlink with the old linked node
1381 if self.do_journal and self.properties[propname].do_journal:
1382 self.db.addjournal(link_class, id, 'unlink',
1383 (self.classname, nodeid, propname))
1384 l.remove(id)
1385 remove.append(id)
1387 # handle additions
1388 for id in value:
1389 if not self.db.getclass(link_class).hasnode(id):
1390 raise IndexError, '%s has no node %s'%(link_class, id)
1391 if id in l:
1392 continue
1393 # register the link with the newly linked node
1394 if self.do_journal and self.properties[propname].do_journal:
1395 self.db.addjournal(link_class, id, 'link',
1396 (self.classname, nodeid, propname))
1397 l.append(id)
1398 add.append(id)
1400 # figure the journal entry
1401 l = []
1402 if add:
1403 l.append(('+', add))
1404 if remove:
1405 l.append(('-', remove))
1406 multilink_changes[propname] = (add, remove)
1407 if l:
1408 journalvalues[propname] = tuple(l)
1410 elif isinstance(prop, String):
1411 if value is not None and type(value) != type(''):
1412 raise TypeError, 'new property "%s" not a string'%propname
1414 elif isinstance(prop, Password):
1415 if not isinstance(value, password.Password):
1416 raise TypeError, 'new property "%s" not a Password'%propname
1417 propvalues[propname] = value
1419 elif value is not None and isinstance(prop, Date):
1420 if not isinstance(value, date.Date):
1421 raise TypeError, 'new property "%s" not a Date'% propname
1422 propvalues[propname] = value
1424 elif value is not None and isinstance(prop, Interval):
1425 if not isinstance(value, date.Interval):
1426 raise TypeError, 'new property "%s" not an '\
1427 'Interval'%propname
1428 propvalues[propname] = value
1430 elif value is not None and isinstance(prop, Number):
1431 try:
1432 float(value)
1433 except ValueError:
1434 raise TypeError, 'new property "%s" not numeric'%propname
1436 elif value is not None and isinstance(prop, Boolean):
1437 try:
1438 int(value)
1439 except ValueError:
1440 raise TypeError, 'new property "%s" not boolean'%propname
1442 # nothing to do?
1443 if not propvalues:
1444 return propvalues
1446 # do the set, and journal it
1447 self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
1449 if self.do_journal:
1450 propvalues.update(journalvalues)
1451 self.db.addjournal(self.classname, nodeid, 'set', propvalues)
1453 self.fireReactors('set', nodeid, oldvalues)
1455 return propvalues
1457 def retire(self, nodeid):
1458 '''Retire a node.
1460 The properties on the node remain available from the get() method,
1461 and the node's id is never reused.
1463 Retired nodes are not returned by the find(), list(), or lookup()
1464 methods, and other nodes may reuse the values of their key properties.
1465 '''
1466 if self.db.journaltag is None:
1467 raise DatabaseError, 'Database open read-only'
1469 cursor = self.db.conn.cursor()
1470 sql = 'update _%s set __retired__=1 where id=%s'%(self.classname,
1471 self.db.arg)
1472 if __debug__:
1473 print >>hyperdb.DEBUG, 'retire', (self, sql, nodeid)
1474 cursor.execute(sql, (nodeid,))
1476 def is_retired(self, nodeid):
1477 '''Return true if the node is rerired
1478 '''
1479 cursor = self.db.conn.cursor()
1480 sql = 'select __retired__ from _%s where id=%s'%(self.classname,
1481 self.db.arg)
1482 if __debug__:
1483 print >>hyperdb.DEBUG, 'is_retired', (self, sql, nodeid)
1484 cursor.execute(sql, (nodeid,))
1485 return int(cursor.fetchone()[0])
1487 def destroy(self, nodeid):
1488 '''Destroy a node.
1490 WARNING: this method should never be used except in extremely rare
1491 situations where there could never be links to the node being
1492 deleted
1493 WARNING: use retire() instead
1494 WARNING: the properties of this node will not be available ever again
1495 WARNING: really, use retire() instead
1497 Well, I think that's enough warnings. This method exists mostly to
1498 support the session storage of the cgi interface.
1500 The node is completely removed from the hyperdb, including all journal
1501 entries. It will no longer be available, and will generally break code
1502 if there are any references to the node.
1503 '''
1504 if self.db.journaltag is None:
1505 raise DatabaseError, 'Database open read-only'
1506 self.db.destroynode(self.classname, nodeid)
1508 def history(self, nodeid):
1509 '''Retrieve the journal of edits on a particular node.
1511 'nodeid' must be the id of an existing node of this class or an
1512 IndexError is raised.
1514 The returned list contains tuples of the form
1516 (date, tag, action, params)
1518 'date' is a Timestamp object specifying the time of the change and
1519 'tag' is the journaltag specified when the database was opened.
1520 '''
1521 if not self.do_journal:
1522 raise ValueError, 'Journalling is disabled for this class'
1523 return self.db.getjournal(self.classname, nodeid)
1525 # Locating nodes:
1526 def hasnode(self, nodeid):
1527 '''Determine if the given nodeid actually exists
1528 '''
1529 return self.db.hasnode(self.classname, nodeid)
1531 def setkey(self, propname):
1532 '''Select a String property of this class to be the key property.
1534 'propname' must be the name of a String property of this class or
1535 None, or a TypeError is raised. The values of the key property on
1536 all existing nodes must be unique or a ValueError is raised.
1537 '''
1538 # XXX create an index on the key prop column
1539 prop = self.getprops()[propname]
1540 if not isinstance(prop, String):
1541 raise TypeError, 'key properties must be String'
1542 self.key = propname
1544 def getkey(self):
1545 '''Return the name of the key property for this class or None.'''
1546 return self.key
1548 def labelprop(self, default_to_id=0):
1549 ''' Return the property name for a label for the given node.
1551 This method attempts to generate a consistent label for the node.
1552 It tries the following in order:
1553 1. key property
1554 2. "name" property
1555 3. "title" property
1556 4. first property from the sorted property name list
1557 '''
1558 k = self.getkey()
1559 if k:
1560 return k
1561 props = self.getprops()
1562 if props.has_key('name'):
1563 return 'name'
1564 elif props.has_key('title'):
1565 return 'title'
1566 if default_to_id:
1567 return 'id'
1568 props = props.keys()
1569 props.sort()
1570 return props[0]
1572 def lookup(self, keyvalue):
1573 '''Locate a particular node by its key property and return its id.
1575 If this class has no key property, a TypeError is raised. If the
1576 'keyvalue' matches one of the values for the key property among
1577 the nodes in this class, the matching node's id is returned;
1578 otherwise a KeyError is raised.
1579 '''
1580 if not self.key:
1581 raise TypeError, 'No key property set for class %s'%self.classname
1583 cursor = self.db.conn.cursor()
1584 sql = 'select id,__retired__ from _%s where _%s=%s'%(self.classname,
1585 self.key, self.db.arg)
1586 self.db.sql(cursor, sql, (keyvalue,))
1588 # see if there was a result that's not retired
1589 l = cursor.fetchall()
1590 if not l or int(l[0][1]):
1591 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1592 keyvalue, self.classname)
1594 # return the id
1595 return l[0][0]
1597 def find(self, **propspec):
1598 '''Get the ids of nodes in this class which link to the given nodes.
1600 'propspec' consists of keyword args propname={nodeid:1,}
1601 'propname' must be the name of a property in this class, or a
1602 KeyError is raised. That property must be a Link or Multilink
1603 property, or a TypeError is raised.
1605 Any node in this class whose 'propname' property links to any of the
1606 nodeids will be returned. Used by the full text indexing, which knows
1607 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1608 issues:
1610 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1611 '''
1612 if __debug__:
1613 print >>hyperdb.DEBUG, 'find', (self, propspec)
1614 if not propspec:
1615 return []
1616 queries = []
1617 tables = []
1618 allvalues = ()
1619 for prop, values in propspec.items():
1620 allvalues += tuple(values.keys())
1621 a = self.db.arg
1622 tables.append('select nodeid from %s_%s where linkid in (%s)'%(
1623 self.classname, prop, ','.join([a for x in values.keys()])))
1624 sql = '\nintersect\n'.join(tables)
1625 if __debug__:
1626 print >>hyperdb.DEBUG, 'find', (self, sql, allvalues)
1627 cursor = self.db.conn.cursor()
1628 cursor.execute(sql, allvalues)
1629 try:
1630 l = [x[0] for x in cursor.fetchall()]
1631 except gadfly.database.error, message:
1632 if message == 'no more results':
1633 l = []
1634 raise
1635 if __debug__:
1636 print >>hyperdb.DEBUG, 'find ... ', l
1637 return l
1639 def list(self):
1640 ''' Return a list of the ids of the active nodes in this class.
1641 '''
1642 return self.db.getnodeids(self.classname, retired=0)
1644 def filter(self, search_matches, filterspec, sort, group):
1645 ''' Return a list of the ids of the active nodes in this class that
1646 match the 'filter' spec, sorted by the group spec and then the
1647 sort spec
1649 "filterspec" is {propname: value(s)}
1650 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1651 and prop is a prop name or None
1652 "search_matches" is {nodeid: marker}
1654 The filter must match all properties specificed - but if the
1655 property value to match is a list, any one of the values in the
1656 list may match for that property to match.
1657 '''
1658 cn = self.classname
1660 # figure the WHERE clause from the filterspec
1661 props = self.getprops()
1662 frum = ['_'+cn]
1663 where = []
1664 args = []
1665 a = self.db.arg
1666 for k, v in filterspec.items():
1667 propclass = props[k]
1668 # now do other where clause stuff
1669 if isinstance(propclass, Multilink):
1670 tn = '%s_%s'%(cn, k)
1671 frum.append(tn)
1672 if isinstance(v, type([])):
1673 s = ','.join([a for x in v])
1674 where.append('id=%s.nodeid and %s.linkid in (%s)'%(tn,tn,s))
1675 args = args + v
1676 else:
1677 where.append('id=%s.nodeid and %s.linkid = %s'%(tn, tn, a))
1678 args.append(v)
1679 elif isinstance(propclass, String):
1680 if not isinstance(v, type([])):
1681 v = [v]
1683 # Quote the bits in the string that need it and then embed
1684 # in a "substring" search. Note - need to quote the '%' so
1685 # they make it through the python layer happily
1686 v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
1688 # now add to the where clause
1689 where.append(' or '.join(["_%s LIKE '%s'"%(k, s) for s in v]))
1690 # note: args are embedded in the query string now
1691 elif isinstance(propclass, Link):
1692 if isinstance(v, type([])):
1693 if '-1' in v:
1694 v.remove('-1')
1695 xtra = ' or _%s is NULL'%k
1696 else:
1697 xtra = ''
1698 s = ','.join([a for x in v])
1699 where.append('(_%s in (%s)%s)'%(k, s, xtra))
1700 args = args + v
1701 else:
1702 if v == '-1':
1703 v = None
1704 where.append('_%s is NULL'%k)
1705 else:
1706 where.append('_%s=%s'%(k, a))
1707 args.append(v)
1708 else:
1709 if isinstance(v, type([])):
1710 s = ','.join([a for x in v])
1711 where.append('_%s in (%s)'%(k, s))
1712 args = args + v
1713 else:
1714 where.append('_%s=%s'%(k, a))
1715 args.append(v)
1717 # add results of full text search
1718 if search_matches is not None:
1719 v = search_matches.keys()
1720 s = ','.join([a for x in v])
1721 where.append('id in (%s)'%s)
1722 args = args + v
1724 # "grouping" is just the first-order sorting in the SQL fetch
1725 # can modify it...)
1726 orderby = []
1727 ordercols = []
1728 if group[0] is not None and group[1] is not None:
1729 if group[0] != '-':
1730 orderby.append('_'+group[1])
1731 ordercols.append('_'+group[1])
1732 else:
1733 orderby.append('_'+group[1]+' desc')
1734 ordercols.append('_'+group[1])
1736 # now add in the sorting
1737 group = ''
1738 if sort[0] is not None and sort[1] is not None:
1739 direction, colname = sort
1740 if direction != '-':
1741 if colname == 'id':
1742 orderby.append(colname)
1743 else:
1744 orderby.append('_'+colname)
1745 ordercols.append('_'+colname)
1746 else:
1747 if colname == 'id':
1748 orderby.append(colname+' desc')
1749 ordercols.append(colname)
1750 else:
1751 orderby.append('_'+colname+' desc')
1752 ordercols.append('_'+colname)
1754 # construct the SQL
1755 frum = ','.join(frum)
1756 if where:
1757 where = ' where ' + (' and '.join(where))
1758 else:
1759 where = ''
1760 cols = ['id']
1761 if orderby:
1762 cols = cols + ordercols
1763 order = ' order by %s'%(','.join(orderby))
1764 else:
1765 order = ''
1766 cols = ','.join(cols)
1767 sql = 'select %s from %s %s%s%s'%(cols, frum, where, group, order)
1768 args = tuple(args)
1769 if __debug__:
1770 print >>hyperdb.DEBUG, 'filter', (self, sql, args)
1771 cursor = self.db.conn.cursor()
1772 cursor.execute(sql, args)
1773 l = cursor.fetchall()
1775 # return the IDs (the first column)
1776 # XXX The filter(None, l) bit is sqlite-specific... if there's _NO_
1777 # XXX matches to a fetch, it returns NULL instead of nothing!?!
1778 return filter(None, [row[0] for row in l])
1780 def count(self):
1781 '''Get the number of nodes in this class.
1783 If the returned integer is 'numnodes', the ids of all the nodes
1784 in this class run from 1 to numnodes, and numnodes+1 will be the
1785 id of the next node to be created in this class.
1786 '''
1787 return self.db.countnodes(self.classname)
1789 # Manipulating properties:
1790 def getprops(self, protected=1):
1791 '''Return a dictionary mapping property names to property objects.
1792 If the "protected" flag is true, we include protected properties -
1793 those which may not be modified.
1794 '''
1795 d = self.properties.copy()
1796 if protected:
1797 d['id'] = String()
1798 d['creation'] = hyperdb.Date()
1799 d['activity'] = hyperdb.Date()
1800 d['creator'] = hyperdb.Link('user')
1801 return d
1803 def addprop(self, **properties):
1804 '''Add properties to this class.
1806 The keyword arguments in 'properties' must map names to property
1807 objects, or a TypeError is raised. None of the keys in 'properties'
1808 may collide with the names of existing properties, or a ValueError
1809 is raised before any properties have been added.
1810 '''
1811 for key in properties.keys():
1812 if self.properties.has_key(key):
1813 raise ValueError, key
1814 self.properties.update(properties)
1816 def index(self, nodeid):
1817 '''Add (or refresh) the node to search indexes
1818 '''
1819 # find all the String properties that have indexme
1820 for prop, propclass in self.getprops().items():
1821 if isinstance(propclass, String) and propclass.indexme:
1822 try:
1823 value = str(self.get(nodeid, prop))
1824 except IndexError:
1825 # node no longer exists - entry should be removed
1826 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1827 else:
1828 # and index them under (classname, nodeid, property)
1829 self.db.indexer.add_text((self.classname, nodeid, prop),
1830 value)
1833 #
1834 # Detector interface
1835 #
1836 def audit(self, event, detector):
1837 '''Register a detector
1838 '''
1839 l = self.auditors[event]
1840 if detector not in l:
1841 self.auditors[event].append(detector)
1843 def fireAuditors(self, action, nodeid, newvalues):
1844 '''Fire all registered auditors.
1845 '''
1846 for audit in self.auditors[action]:
1847 audit(self.db, self, nodeid, newvalues)
1849 def react(self, event, detector):
1850 '''Register a detector
1851 '''
1852 l = self.reactors[event]
1853 if detector not in l:
1854 self.reactors[event].append(detector)
1856 def fireReactors(self, action, nodeid, oldvalues):
1857 '''Fire all registered reactors.
1858 '''
1859 for react in self.reactors[action]:
1860 react(self.db, self, nodeid, oldvalues)
1862 class FileClass(Class):
1863 '''This class defines a large chunk of data. To support this, it has a
1864 mandatory String property "content" which is typically saved off
1865 externally to the hyperdb.
1867 The default MIME type of this data is defined by the
1868 "default_mime_type" class attribute, which may be overridden by each
1869 node if the class defines a "type" String property.
1870 '''
1871 default_mime_type = 'text/plain'
1873 def create(self, **propvalues):
1874 ''' snaffle the file propvalue and store in a file
1875 '''
1876 content = propvalues['content']
1877 del propvalues['content']
1878 newid = Class.create(self, **propvalues)
1879 self.db.storefile(self.classname, newid, None, content)
1880 return newid
1882 def import_list(self, propnames, proplist):
1883 ''' Trap the "content" property...
1884 '''
1885 # dupe this list so we don't affect others
1886 propnames = propnames[:]
1888 # extract the "content" property from the proplist
1889 i = propnames.index('content')
1890 content = eval(proplist[i])
1891 del propnames[i]
1892 del proplist[i]
1894 # do the normal import
1895 newid = Class.import_list(self, propnames, proplist)
1897 # save off the "content" file
1898 self.db.storefile(self.classname, newid, None, content)
1899 return newid
1901 _marker = []
1902 def get(self, nodeid, propname, default=_marker, cache=1):
1903 ''' trap the content propname and get it from the file
1904 '''
1906 poss_msg = 'Possibly a access right configuration problem.'
1907 if propname == 'content':
1908 try:
1909 return self.db.getfile(self.classname, nodeid, None)
1910 except IOError, (strerror):
1911 # BUG: by catching this we donot see an error in the log.
1912 return 'ERROR reading file: %s%s\n%s\n%s'%(
1913 self.classname, nodeid, poss_msg, strerror)
1914 if default is not self._marker:
1915 return Class.get(self, nodeid, propname, default, cache=cache)
1916 else:
1917 return Class.get(self, nodeid, propname, cache=cache)
1919 def getprops(self, protected=1):
1920 ''' In addition to the actual properties on the node, these methods
1921 provide the "content" property. If the "protected" flag is true,
1922 we include protected properties - those which may not be
1923 modified.
1924 '''
1925 d = Class.getprops(self, protected=protected).copy()
1926 d['content'] = hyperdb.String()
1927 return d
1929 def index(self, nodeid):
1930 ''' Index the node in the search index.
1932 We want to index the content in addition to the normal String
1933 property indexing.
1934 '''
1935 # perform normal indexing
1936 Class.index(self, nodeid)
1938 # get the content to index
1939 content = self.get(nodeid, 'content')
1941 # figure the mime type
1942 if self.properties.has_key('type'):
1943 mime_type = self.get(nodeid, 'type')
1944 else:
1945 mime_type = self.default_mime_type
1947 # and index!
1948 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
1949 mime_type)
1951 # XXX deviation from spec - was called ItemClass
1952 class IssueClass(Class, roundupdb.IssueClass):
1953 # Overridden methods:
1954 def __init__(self, db, classname, **properties):
1955 '''The newly-created class automatically includes the "messages",
1956 "files", "nosy", and "superseder" properties. If the 'properties'
1957 dictionary attempts to specify any of these properties or a
1958 "creation" or "activity" property, a ValueError is raised.
1959 '''
1960 if not properties.has_key('title'):
1961 properties['title'] = hyperdb.String(indexme='yes')
1962 if not properties.has_key('messages'):
1963 properties['messages'] = hyperdb.Multilink("msg")
1964 if not properties.has_key('files'):
1965 properties['files'] = hyperdb.Multilink("file")
1966 if not properties.has_key('nosy'):
1967 # note: journalling is turned off as it really just wastes
1968 # space. this behaviour may be overridden in an instance
1969 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
1970 if not properties.has_key('superseder'):
1971 properties['superseder'] = hyperdb.Multilink(classname)
1972 Class.__init__(self, db, classname, **properties)