d264a581b8d0f0d4c4b83ca1c19c6695fcb51afd
1 # $Id: rdbms_common.py,v 1.1 2002-09-18 05:07:47 richard Exp $
3 # standard python modules
4 import sys, os, time, re, errno, weakref, copy
6 # roundup modules
7 from roundup import hyperdb, date, password, roundupdb, security
8 from roundup.hyperdb import String, Password, Date, Interval, Link, \
9 Multilink, DatabaseError, Boolean, Number
11 # support
12 from blobfiles import FileStorage
13 from roundup.indexer import Indexer
14 from sessions import Sessions
16 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
17 # flag to set on retired entries
18 RETIRED_FLAG = '__hyperdb_retired'
20 def __init__(self, config, journaltag=None):
21 ''' Open the database and load the schema from it.
22 '''
23 self.config, self.journaltag = config, journaltag
24 self.dir = config.DATABASE
25 self.classes = {}
26 self.indexer = Indexer(self.dir)
27 self.sessions = Sessions(self.config)
28 self.security = security.Security(self)
30 # additional transaction support for external files and the like
31 self.transactions = []
33 # open a connection to the database, creating the "conn" attribute
34 self.open_connection()
36 def open_connection(self):
37 ''' Open a connection to the database, creating it if necessary
38 '''
39 raise NotImplemented
41 def sql(self, cursor, sql, args=None):
42 ''' Execute the sql with the optional args.
43 '''
44 if __debug__:
45 print >>hyperdb.DEBUG, (self, sql, args)
46 if args:
47 cursor.execute(sql, args)
48 else:
49 cursor.execute(sql)
51 def sql_fetchone(self, cursor):
52 ''' Fetch a single row. If there's nothing to fetch, return None.
53 '''
54 raise NotImplemented
56 def save_dbschema(self, cursor, schema):
57 ''' Save the schema definition that the database currently implements
58 '''
59 raise NotImplemented
61 def load_dbschema(self, cursor):
62 ''' Load the schema definition that the database currently implements
63 '''
64 raise NotImplemented
66 def post_init(self):
67 ''' Called once the schema initialisation has finished.
69 We should now confirm that the schema defined by our "classes"
70 attribute actually matches the schema in the database.
71 '''
72 # now detect changes in the schema
73 for classname, spec in self.classes.items():
74 if self.database_schema.has_key(classname):
75 dbspec = self.database_schema[classname]
76 self.update_class(spec, dbspec)
77 self.database_schema[classname] = spec.schema()
78 else:
79 self.create_class(spec)
80 self.database_schema[classname] = spec.schema()
82 for classname in self.database_schema.keys():
83 if not self.classes.has_key(classname):
84 self.drop_class(classname)
86 # update the database version of the schema
87 cursor = self.conn.cursor()
88 self.sql(cursor, 'delete from schema')
89 self.save_dbschema(cursor, self.database_schema)
91 # reindex the db if necessary
92 if self.indexer.should_reindex():
93 self.reindex()
95 # commit
96 self.conn.commit()
98 def reindex(self):
99 for klass in self.classes.values():
100 for nodeid in klass.list():
101 klass.index(nodeid)
102 self.indexer.save_index()
104 def determine_columns(self, properties):
105 ''' Figure the column names and multilink properties from the spec
107 "properties" is a list of (name, prop) where prop may be an
108 instance of a hyperdb "type" _or_ a string repr of that type.
109 '''
110 cols = []
111 mls = []
112 # add the multilinks separately
113 for col, prop in properties:
114 if isinstance(prop, Multilink):
115 mls.append(col)
116 elif isinstance(prop, type('')) and prop.find('Multilink') != -1:
117 mls.append(col)
118 else:
119 cols.append('_'+col)
120 cols.sort()
121 return cols, mls
123 def update_class(self, spec, dbspec):
124 ''' Determine the differences between the current spec and the
125 database version of the spec, and update where necessary
126 '''
127 spec_schema = spec.schema()
128 if spec_schema == dbspec:
129 return
130 if __debug__:
131 print >>hyperdb.DEBUG, 'update_class FIRING'
133 # key property changed?
134 if dbspec[0] != spec_schema[0]:
135 if __debug__:
136 print >>hyperdb.DEBUG, 'update_class setting keyprop', `spec[0]`
137 # XXX turn on indexing for the key property
139 # dict 'em up
140 spec_propnames,spec_props = [],{}
141 for propname,prop in spec_schema[1]:
142 spec_propnames.append(propname)
143 spec_props[propname] = prop
144 dbspec_propnames,dbspec_props = [],{}
145 for propname,prop in dbspec[1]:
146 dbspec_propnames.append(propname)
147 dbspec_props[propname] = prop
149 # we're going to need one of these
150 cursor = self.conn.cursor()
152 # now compare
153 for propname in spec_propnames:
154 prop = spec_props[propname]
155 if dbspec_props.has_key(propname) and prop==dbspec_props[propname]:
156 continue
157 if __debug__:
158 print >>hyperdb.DEBUG, 'update_class ADD', (propname, prop)
160 if not dbspec_props.has_key(propname):
161 # add the property
162 if isinstance(prop, Multilink):
163 # all we have to do here is create a new table, easy!
164 self.create_multilink_table(cursor, spec, propname)
165 continue
167 # no ALTER TABLE, so we:
168 # 1. pull out the data, including an extra None column
169 oldcols, x = self.determine_columns(dbspec[1])
170 oldcols.append('id')
171 oldcols.append('__retired__')
172 cn = spec.classname
173 sql = 'select %s,%s from _%s'%(','.join(oldcols), self.arg, cn)
174 if __debug__:
175 print >>hyperdb.DEBUG, 'update_class', (self, sql, None)
176 cursor.execute(sql, (None,))
177 olddata = cursor.fetchall()
179 # 2. drop the old table
180 cursor.execute('drop table _%s'%cn)
182 # 3. create the new table
183 cols, mls = self.create_class_table(cursor, spec)
184 # ensure the new column is last
185 cols.remove('_'+propname)
186 assert oldcols == cols, "Column lists don't match!"
187 cols.append('_'+propname)
189 # 4. populate with the data from step one
190 s = ','.join([self.arg for x in cols])
191 scols = ','.join(cols)
192 sql = 'insert into _%s (%s) values (%s)'%(cn, scols, s)
194 # GAH, nothing had better go wrong from here on in... but
195 # we have to commit the drop...
196 # XXX this isn't necessary in sqlite :(
197 self.conn.commit()
199 # do the insert
200 for row in olddata:
201 self.sql(cursor, sql, tuple(row))
203 else:
204 # modify the property
205 if __debug__:
206 print >>hyperdb.DEBUG, 'update_class NOOP'
207 pass # NOOP in gadfly
209 # and the other way - only worry about deletions here
210 for propname in dbspec_propnames:
211 prop = dbspec_props[propname]
212 if spec_props.has_key(propname):
213 continue
214 if __debug__:
215 print >>hyperdb.DEBUG, 'update_class REMOVE', `prop`
217 # delete the property
218 if isinstance(prop, Multilink):
219 sql = 'drop table %s_%s'%(spec.classname, prop)
220 if __debug__:
221 print >>hyperdb.DEBUG, 'update_class', (self, sql)
222 cursor.execute(sql)
223 else:
224 # no ALTER TABLE, so we:
225 # 1. pull out the data, excluding the removed column
226 oldcols, x = self.determine_columns(spec.properties.items())
227 oldcols.append('id')
228 oldcols.append('__retired__')
229 # remove the missing column
230 oldcols.remove('_'+propname)
231 cn = spec.classname
232 sql = 'select %s from _%s'%(','.join(oldcols), cn)
233 cursor.execute(sql, (None,))
234 olddata = sql.fetchall()
236 # 2. drop the old table
237 cursor.execute('drop table _%s'%cn)
239 # 3. create the new table
240 cols, mls = self.create_class_table(self, cursor, spec)
241 assert oldcols != cols, "Column lists don't match!"
243 # 4. populate with the data from step one
244 qs = ','.join([self.arg for x in cols])
245 sql = 'insert into _%s values (%s)'%(cn, s)
246 cursor.execute(sql, olddata)
248 def create_class_table(self, cursor, spec):
249 ''' create the class table for the given spec
250 '''
251 cols, mls = self.determine_columns(spec.properties.items())
253 # add on our special columns
254 cols.append('id')
255 cols.append('__retired__')
257 # create the base table
258 scols = ','.join(['%s varchar'%x for x in cols])
259 sql = 'create table _%s (%s)'%(spec.classname, scols)
260 if __debug__:
261 print >>hyperdb.DEBUG, 'create_class', (self, sql)
262 cursor.execute(sql)
264 return cols, mls
266 def create_journal_table(self, cursor, spec):
267 ''' create the journal table for a class given the spec and
268 already-determined cols
269 '''
270 # journal table
271 cols = ','.join(['%s varchar'%x
272 for x in 'nodeid date tag action params'.split()])
273 sql = 'create table %s__journal (%s)'%(spec.classname, cols)
274 if __debug__:
275 print >>hyperdb.DEBUG, 'create_class', (self, sql)
276 cursor.execute(sql)
278 def create_multilink_table(self, cursor, spec, ml):
279 ''' Create a multilink table for the "ml" property of the class
280 given by the spec
281 '''
282 sql = 'create table %s_%s (linkid varchar, nodeid varchar)'%(
283 spec.classname, ml)
284 if __debug__:
285 print >>hyperdb.DEBUG, 'create_class', (self, sql)
286 cursor.execute(sql)
288 def create_class(self, spec):
289 ''' Create a database table according to the given spec.
290 '''
291 cursor = self.conn.cursor()
292 cols, mls = self.create_class_table(cursor, spec)
293 self.create_journal_table(cursor, spec)
295 # now create the multilink tables
296 for ml in mls:
297 self.create_multilink_table(cursor, spec, ml)
299 # ID counter
300 sql = 'insert into ids (name, num) values (%s,%s)'%(self.arg, self.arg)
301 vals = (spec.classname, 1)
302 if __debug__:
303 print >>hyperdb.DEBUG, 'create_class', (self, sql, vals)
304 cursor.execute(sql, vals)
306 def drop_class(self, spec):
307 ''' Drop the given table from the database.
309 Drop the journal and multilink tables too.
310 '''
311 # figure the multilinks
312 mls = []
313 for col, prop in spec.properties.items():
314 if isinstance(prop, Multilink):
315 mls.append(col)
316 cursor = self.conn.cursor()
318 sql = 'drop table _%s'%spec.classname
319 if __debug__:
320 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
321 cursor.execute(sql)
323 sql = 'drop table %s__journal'%spec.classname
324 if __debug__:
325 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
326 cursor.execute(sql)
328 for ml in mls:
329 sql = 'drop table %s_%s'%(spec.classname, ml)
330 if __debug__:
331 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
332 cursor.execute(sql)
334 #
335 # Classes
336 #
337 def __getattr__(self, classname):
338 ''' A convenient way of calling self.getclass(classname).
339 '''
340 if self.classes.has_key(classname):
341 if __debug__:
342 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
343 return self.classes[classname]
344 raise AttributeError, classname
346 def addclass(self, cl):
347 ''' Add a Class to the hyperdatabase.
348 '''
349 if __debug__:
350 print >>hyperdb.DEBUG, 'addclass', (self, cl)
351 cn = cl.classname
352 if self.classes.has_key(cn):
353 raise ValueError, cn
354 self.classes[cn] = cl
356 def getclasses(self):
357 ''' Return a list of the names of all existing classes.
358 '''
359 if __debug__:
360 print >>hyperdb.DEBUG, 'getclasses', (self,)
361 l = self.classes.keys()
362 l.sort()
363 return l
365 def getclass(self, classname):
366 '''Get the Class object representing a particular class.
368 If 'classname' is not a valid class name, a KeyError is raised.
369 '''
370 if __debug__:
371 print >>hyperdb.DEBUG, 'getclass', (self, classname)
372 try:
373 return self.classes[classname]
374 except KeyError:
375 raise KeyError, 'There is no class called "%s"'%classname
377 def clear(self):
378 ''' Delete all database contents.
380 Note: I don't commit here, which is different behaviour to the
381 "nuke from orbit" behaviour in the *dbms.
382 '''
383 if __debug__:
384 print >>hyperdb.DEBUG, 'clear', (self,)
385 cursor = self.conn.cursor()
386 for cn in self.classes.keys():
387 sql = 'delete from _%s'%cn
388 if __debug__:
389 print >>hyperdb.DEBUG, 'clear', (self, sql)
390 cursor.execute(sql)
392 #
393 # Node IDs
394 #
395 def newid(self, classname):
396 ''' Generate a new id for the given class
397 '''
398 # get the next ID
399 cursor = self.conn.cursor()
400 sql = 'select num from ids where name=%s'%self.arg
401 if __debug__:
402 print >>hyperdb.DEBUG, 'newid', (self, sql, classname)
403 cursor.execute(sql, (classname, ))
404 newid = cursor.fetchone()[0]
406 # update the counter
407 sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
408 vals = (int(newid)+1, classname)
409 if __debug__:
410 print >>hyperdb.DEBUG, 'newid', (self, sql, vals)
411 cursor.execute(sql, vals)
413 # return as string
414 return str(newid)
416 def setid(self, classname, setid):
417 ''' Set the id counter: used during import of database
418 '''
419 cursor = self.conn.cursor()
420 sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
421 vals = (setid, spec.classname)
422 if __debug__:
423 print >>hyperdb.DEBUG, 'setid', (self, sql, vals)
424 cursor.execute(sql, vals)
426 #
427 # Nodes
428 #
430 def addnode(self, classname, nodeid, node):
431 ''' Add the specified node to its class's db.
432 '''
433 if __debug__:
434 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
435 # gadfly requires values for all non-multilink columns
436 cl = self.classes[classname]
437 cols, mls = self.determine_columns(cl.properties.items())
439 # default the non-multilink columns
440 for col, prop in cl.properties.items():
441 if not isinstance(col, Multilink):
442 if not node.has_key(col):
443 node[col] = None
445 node = self.serialise(classname, node)
447 # make sure the ordering is correct for column name -> column value
448 vals = tuple([node[col[1:]] for col in cols]) + (nodeid, 0)
449 s = ','.join([self.arg for x in cols]) + ',%s,%s'%(self.arg, self.arg)
450 cols = ','.join(cols) + ',id,__retired__'
452 # perform the inserts
453 cursor = self.conn.cursor()
454 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
455 if __debug__:
456 print >>hyperdb.DEBUG, 'addnode', (self, sql, vals)
457 cursor.execute(sql, vals)
459 # insert the multilink rows
460 for col in mls:
461 t = '%s_%s'%(classname, col)
462 for entry in node[col]:
463 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
464 self.arg, self.arg)
465 self.sql(cursor, sql, (entry, nodeid))
467 # make sure we do the commit-time extra stuff for this node
468 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
470 def setnode(self, classname, nodeid, node, multilink_changes):
471 ''' Change the specified node.
472 '''
473 if __debug__:
474 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
475 node = self.serialise(classname, node)
477 cl = self.classes[classname]
478 cols = []
479 mls = []
480 # add the multilinks separately
481 for col in node.keys():
482 prop = cl.properties[col]
483 if isinstance(prop, Multilink):
484 mls.append(col)
485 else:
486 cols.append('_'+col)
487 cols.sort()
489 # make sure the ordering is correct for column name -> column value
490 vals = tuple([node[col[1:]] for col in cols])
491 s = ','.join(['%s=%s'%(x, self.arg) for x in cols])
492 cols = ','.join(cols)
494 # perform the update
495 cursor = self.conn.cursor()
496 sql = 'update _%s set %s'%(classname, s)
497 if __debug__:
498 print >>hyperdb.DEBUG, 'setnode', (self, sql, vals)
499 cursor.execute(sql, vals)
501 # now the fun bit, updating the multilinks ;)
502 for col, (add, remove) in multilink_changes.items():
503 tn = '%s_%s'%(classname, col)
504 if add:
505 sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
506 self.arg, self.arg)
507 for addid in add:
508 self.sql(cursor, sql, (nodeid, addid))
509 if remove:
510 sql = 'delete from %s where nodeid=%s and linkid=%s'%(tn,
511 self.arg, self.arg)
512 for removeid in remove:
513 self.sql(cursor, sql, (nodeid, removeid))
515 # make sure we do the commit-time extra stuff for this node
516 self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
518 def getnode(self, classname, nodeid):
519 ''' Get a node from the database.
520 '''
521 if __debug__:
522 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid)
523 # figure the columns we're fetching
524 cl = self.classes[classname]
525 cols, mls = self.determine_columns(cl.properties.items())
526 scols = ','.join(cols)
528 # perform the basic property fetch
529 cursor = self.conn.cursor()
530 sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
531 self.sql(cursor, sql, (nodeid,))
533 values = self.sql_fetchone(cursor)
534 if values is None:
535 raise IndexError, 'no such %s node %s'%(classname, nodeid)
537 # make up the node
538 node = {}
539 for col in range(len(cols)):
540 node[cols[col][1:]] = values[col]
542 # now the multilinks
543 for col in mls:
544 # get the link ids
545 sql = 'select linkid from %s_%s where nodeid=%s'%(classname, col,
546 self.arg)
547 cursor.execute(sql, (nodeid,))
548 # extract the first column from the result
549 node[col] = [x[0] for x in cursor.fetchall()]
551 return self.unserialise(classname, node)
553 def destroynode(self, classname, nodeid):
554 '''Remove a node from the database. Called exclusively by the
555 destroy() method on Class.
556 '''
557 if __debug__:
558 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
560 # make sure the node exists
561 if not self.hasnode(classname, nodeid):
562 raise IndexError, '%s has no node %s'%(classname, nodeid)
564 # see if there's any obvious commit actions that we should get rid of
565 for entry in self.transactions[:]:
566 if entry[1][:2] == (classname, nodeid):
567 self.transactions.remove(entry)
569 # now do the SQL
570 cursor = self.conn.cursor()
571 sql = 'delete from _%s where id=%s'%(classname, self.arg)
572 self.sql(cursor, sql, (nodeid,))
574 # remove from multilnks
575 cl = self.getclass(classname)
576 x, mls = self.determine_columns(cl.properties.items())
577 for col in mls:
578 # get the link ids
579 sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
580 cursor.execute(sql, (nodeid,))
582 # remove journal entries
583 sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
584 self.sql(cursor, sql, (nodeid,))
586 def serialise(self, classname, node):
587 '''Copy the node contents, converting non-marshallable data into
588 marshallable data.
589 '''
590 if __debug__:
591 print >>hyperdb.DEBUG, 'serialise', classname, node
592 properties = self.getclass(classname).getprops()
593 d = {}
594 for k, v in node.items():
595 # if the property doesn't exist, or is the "retired" flag then
596 # it won't be in the properties dict
597 if not properties.has_key(k):
598 d[k] = v
599 continue
601 # get the property spec
602 prop = properties[k]
604 if isinstance(prop, Password):
605 d[k] = str(v)
606 elif isinstance(prop, Date) and v is not None:
607 d[k] = v.serialise()
608 elif isinstance(prop, Interval) and v is not None:
609 d[k] = v.serialise()
610 else:
611 d[k] = v
612 return d
614 def unserialise(self, classname, node):
615 '''Decode the marshalled node data
616 '''
617 if __debug__:
618 print >>hyperdb.DEBUG, 'unserialise', classname, node
619 properties = self.getclass(classname).getprops()
620 d = {}
621 for k, v in node.items():
622 # if the property doesn't exist, or is the "retired" flag then
623 # it won't be in the properties dict
624 if not properties.has_key(k):
625 d[k] = v
626 continue
628 # get the property spec
629 prop = properties[k]
631 if isinstance(prop, Date) and v is not None:
632 d[k] = date.Date(v)
633 elif isinstance(prop, Interval) and v is not None:
634 d[k] = date.Interval(v)
635 elif isinstance(prop, Password):
636 p = password.Password()
637 p.unpack(v)
638 d[k] = p
639 else:
640 d[k] = v
641 return d
643 def hasnode(self, classname, nodeid):
644 ''' Determine if the database has a given node.
645 '''
646 cursor = self.conn.cursor()
647 sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
648 if __debug__:
649 print >>hyperdb.DEBUG, 'hasnode', (self, sql, nodeid)
650 cursor.execute(sql, (nodeid,))
651 return int(cursor.fetchone()[0])
653 def countnodes(self, classname):
654 ''' Count the number of nodes that exist for a particular Class.
655 '''
656 cursor = self.conn.cursor()
657 sql = 'select count(*) from _%s'%classname
658 if __debug__:
659 print >>hyperdb.DEBUG, 'countnodes', (self, sql)
660 cursor.execute(sql)
661 return cursor.fetchone()[0]
663 def getnodeids(self, classname, retired=0):
664 ''' Retrieve all the ids of the nodes for a particular Class.
666 Set retired=None to get all nodes. Otherwise it'll get all the
667 retired or non-retired nodes, depending on the flag.
668 '''
669 cursor = self.conn.cursor()
670 # flip the sense of the flag if we don't want all of them
671 if retired is not None:
672 retired = not retired
673 sql = 'select id from _%s where __retired__ <> %s'%(classname, self.arg)
674 if __debug__:
675 print >>hyperdb.DEBUG, 'getnodeids', (self, sql, retired)
676 cursor.execute(sql, (retired,))
677 return [x[0] for x in cursor.fetchall()]
679 def addjournal(self, classname, nodeid, action, params, creator=None,
680 creation=None):
681 ''' Journal the Action
682 'action' may be:
684 'create' or 'set' -- 'params' is a dictionary of property values
685 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
686 'retire' -- 'params' is None
687 '''
688 # serialise the parameters now if necessary
689 if isinstance(params, type({})):
690 if action in ('set', 'create'):
691 params = self.serialise(classname, params)
693 # handle supply of the special journalling parameters (usually
694 # supplied on importing an existing database)
695 if creator:
696 journaltag = creator
697 else:
698 journaltag = self.journaltag
699 if creation:
700 journaldate = creation.serialise()
701 else:
702 journaldate = date.Date().serialise()
704 # create the journal entry
705 cols = ','.join('nodeid date tag action params'.split())
707 if __debug__:
708 print >>hyperdb.DEBUG, 'addjournal', (nodeid, journaldate,
709 journaltag, action, params)
711 cursor = self.conn.cursor()
712 self.save_journal(cursor, classname, cols, nodeid, journaldate,
713 journaltag, action, params)
715 def save_journal(self, cursor, classname, cols, nodeid, journaldate,
716 journaltag, action, params):
717 ''' Save the journal entry to the database
718 '''
719 raise NotImplemented
721 def getjournal(self, classname, nodeid):
722 ''' get the journal for id
723 '''
724 # make sure the node exists
725 if not self.hasnode(classname, nodeid):
726 raise IndexError, '%s has no node %s'%(classname, nodeid)
728 cursor = self.conn.cursor()
729 cols = ','.join('nodeid date tag action params'.split())
730 return self.load_journal(cursor, classname, cols, nodeid)
732 def load_journal(self, cursor, classname, cols, nodeid):
733 ''' Load the journal from the database
734 '''
735 raise NotImplemented
737 def pack(self, pack_before):
738 ''' Delete all journal entries except "create" before 'pack_before'.
739 '''
740 # get a 'yyyymmddhhmmss' version of the date
741 date_stamp = pack_before.serialise()
743 # do the delete
744 cursor = self.conn.cursor()
745 for classname in self.classes.keys():
746 sql = "delete from %s__journal where date<%s and "\
747 "action<>'create'"%(classname, self.arg)
748 if __debug__:
749 print >>hyperdb.DEBUG, 'pack', (self, sql, date_stamp)
750 cursor.execute(sql, (date_stamp,))
752 def sql_commit(self):
753 ''' Actually commit to the database.
754 '''
755 self.conn.commit()
757 def commit(self):
758 ''' Commit the current transactions.
760 Save all data changed since the database was opened or since the
761 last commit() or rollback().
762 '''
763 if __debug__:
764 print >>hyperdb.DEBUG, 'commit', (self,)
766 # commit the database
767 self.sql_commit()
769 # now, do all the other transaction stuff
770 reindex = {}
771 for method, args in self.transactions:
772 reindex[method(*args)] = 1
774 # reindex the nodes that request it
775 for classname, nodeid in filter(None, reindex.keys()):
776 print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
777 self.getclass(classname).index(nodeid)
779 # save the indexer state
780 self.indexer.save_index()
782 # clear out the transactions
783 self.transactions = []
785 def rollback(self):
786 ''' Reverse all actions from the current transaction.
788 Undo all the changes made since the database was opened or the last
789 commit() or rollback() was performed.
790 '''
791 if __debug__:
792 print >>hyperdb.DEBUG, 'rollback', (self,)
794 # roll back
795 self.conn.rollback()
797 # roll back "other" transaction stuff
798 for method, args in self.transactions:
799 # delete temporary files
800 if method == self.doStoreFile:
801 self.rollbackStoreFile(*args)
802 self.transactions = []
804 def doSaveNode(self, classname, nodeid, node):
805 ''' dummy that just generates a reindex event
806 '''
807 # return the classname, nodeid so we reindex this content
808 return (classname, nodeid)
810 def close(self):
811 ''' Close off the connection.
812 '''
813 self.conn.close()
815 #
816 # The base Class class
817 #
818 class Class(hyperdb.Class):
819 ''' The handle to a particular class of nodes in a hyperdatabase.
821 All methods except __repr__ and getnode must be implemented by a
822 concrete backend Class.
823 '''
825 def __init__(self, db, classname, **properties):
826 '''Create a new class with a given name and property specification.
828 'classname' must not collide with the name of an existing class,
829 or a ValueError is raised. The keyword arguments in 'properties'
830 must map names to property objects, or a TypeError is raised.
831 '''
832 if (properties.has_key('creation') or properties.has_key('activity')
833 or properties.has_key('creator')):
834 raise ValueError, '"creation", "activity" and "creator" are '\
835 'reserved'
837 self.classname = classname
838 self.properties = properties
839 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
840 self.key = ''
842 # should we journal changes (default yes)
843 self.do_journal = 1
845 # do the db-related init stuff
846 db.addclass(self)
848 self.auditors = {'create': [], 'set': [], 'retire': []}
849 self.reactors = {'create': [], 'set': [], 'retire': []}
851 def schema(self):
852 ''' A dumpable version of the schema that we can store in the
853 database
854 '''
855 return (self.key, [(x, repr(y)) for x,y in self.properties.items()])
857 def enableJournalling(self):
858 '''Turn journalling on for this class
859 '''
860 self.do_journal = 1
862 def disableJournalling(self):
863 '''Turn journalling off for this class
864 '''
865 self.do_journal = 0
867 # Editing nodes:
868 def create(self, **propvalues):
869 ''' Create a new node of this class and return its id.
871 The keyword arguments in 'propvalues' map property names to values.
873 The values of arguments must be acceptable for the types of their
874 corresponding properties or a TypeError is raised.
876 If this class has a key property, it must be present and its value
877 must not collide with other key strings or a ValueError is raised.
879 Any other properties on this class that are missing from the
880 'propvalues' dictionary are set to None.
882 If an id in a link or multilink property does not refer to a valid
883 node, an IndexError is raised.
884 '''
885 if propvalues.has_key('id'):
886 raise KeyError, '"id" is reserved'
888 if self.db.journaltag is None:
889 raise DatabaseError, 'Database open read-only'
891 if propvalues.has_key('creation') or propvalues.has_key('activity'):
892 raise KeyError, '"creation" and "activity" are reserved'
894 self.fireAuditors('create', None, propvalues)
896 # new node's id
897 newid = self.db.newid(self.classname)
899 # validate propvalues
900 num_re = re.compile('^\d+$')
901 for key, value in propvalues.items():
902 if key == self.key:
903 try:
904 self.lookup(value)
905 except KeyError:
906 pass
907 else:
908 raise ValueError, 'node with key "%s" exists'%value
910 # try to handle this property
911 try:
912 prop = self.properties[key]
913 except KeyError:
914 raise KeyError, '"%s" has no property "%s"'%(self.classname,
915 key)
917 if value is not None and isinstance(prop, Link):
918 if type(value) != type(''):
919 raise ValueError, 'link value must be String'
920 link_class = self.properties[key].classname
921 # if it isn't a number, it's a key
922 if not num_re.match(value):
923 try:
924 value = self.db.classes[link_class].lookup(value)
925 except (TypeError, KeyError):
926 raise IndexError, 'new property "%s": %s not a %s'%(
927 key, value, link_class)
928 elif not self.db.getclass(link_class).hasnode(value):
929 raise IndexError, '%s has no node %s'%(link_class, value)
931 # save off the value
932 propvalues[key] = value
934 # register the link with the newly linked node
935 if self.do_journal and self.properties[key].do_journal:
936 self.db.addjournal(link_class, value, 'link',
937 (self.classname, newid, key))
939 elif isinstance(prop, Multilink):
940 if type(value) != type([]):
941 raise TypeError, 'new property "%s" not a list of ids'%key
943 # clean up and validate the list of links
944 link_class = self.properties[key].classname
945 l = []
946 for entry in value:
947 if type(entry) != type(''):
948 raise ValueError, '"%s" multilink value (%r) '\
949 'must contain Strings'%(key, value)
950 # if it isn't a number, it's a key
951 if not num_re.match(entry):
952 try:
953 entry = self.db.classes[link_class].lookup(entry)
954 except (TypeError, KeyError):
955 raise IndexError, 'new property "%s": %s not a %s'%(
956 key, entry, self.properties[key].classname)
957 l.append(entry)
958 value = l
959 propvalues[key] = value
961 # handle additions
962 for nodeid in value:
963 if not self.db.getclass(link_class).hasnode(nodeid):
964 raise IndexError, '%s has no node %s'%(link_class,
965 nodeid)
966 # register the link with the newly linked node
967 if self.do_journal and self.properties[key].do_journal:
968 self.db.addjournal(link_class, nodeid, 'link',
969 (self.classname, newid, key))
971 elif isinstance(prop, String):
972 if type(value) != type(''):
973 raise TypeError, 'new property "%s" not a string'%key
975 elif isinstance(prop, Password):
976 if not isinstance(value, password.Password):
977 raise TypeError, 'new property "%s" not a Password'%key
979 elif isinstance(prop, Date):
980 if value is not None and not isinstance(value, date.Date):
981 raise TypeError, 'new property "%s" not a Date'%key
983 elif isinstance(prop, Interval):
984 if value is not None and not isinstance(value, date.Interval):
985 raise TypeError, 'new property "%s" not an Interval'%key
987 elif value is not None and isinstance(prop, Number):
988 try:
989 float(value)
990 except ValueError:
991 raise TypeError, 'new property "%s" not numeric'%key
993 elif value is not None and isinstance(prop, Boolean):
994 try:
995 int(value)
996 except ValueError:
997 raise TypeError, 'new property "%s" not boolean'%key
999 # make sure there's data where there needs to be
1000 for key, prop in self.properties.items():
1001 if propvalues.has_key(key):
1002 continue
1003 if key == self.key:
1004 raise ValueError, 'key property "%s" is required'%key
1005 if isinstance(prop, Multilink):
1006 propvalues[key] = []
1007 else:
1008 propvalues[key] = None
1010 # done
1011 self.db.addnode(self.classname, newid, propvalues)
1012 if self.do_journal:
1013 self.db.addjournal(self.classname, newid, 'create', propvalues)
1015 self.fireReactors('create', newid, None)
1017 return newid
1019 def export_list(self, propnames, nodeid):
1020 ''' Export a node - generate a list of CSV-able data in the order
1021 specified by propnames for the given node.
1022 '''
1023 properties = self.getprops()
1024 l = []
1025 for prop in propnames:
1026 proptype = properties[prop]
1027 value = self.get(nodeid, prop)
1028 # "marshal" data where needed
1029 if value is None:
1030 pass
1031 elif isinstance(proptype, hyperdb.Date):
1032 value = value.get_tuple()
1033 elif isinstance(proptype, hyperdb.Interval):
1034 value = value.get_tuple()
1035 elif isinstance(proptype, hyperdb.Password):
1036 value = str(value)
1037 l.append(repr(value))
1038 return l
1040 def import_list(self, propnames, proplist):
1041 ''' Import a node - all information including "id" is present and
1042 should not be sanity checked. Triggers are not triggered. The
1043 journal should be initialised using the "creator" and "created"
1044 information.
1046 Return the nodeid of the node imported.
1047 '''
1048 if self.db.journaltag is None:
1049 raise DatabaseError, 'Database open read-only'
1050 properties = self.getprops()
1052 # make the new node's property map
1053 d = {}
1054 for i in range(len(propnames)):
1055 # Use eval to reverse the repr() used to output the CSV
1056 value = eval(proplist[i])
1058 # Figure the property for this column
1059 propname = propnames[i]
1060 prop = properties[propname]
1062 # "unmarshal" where necessary
1063 if propname == 'id':
1064 newid = value
1065 continue
1066 elif value is None:
1067 # don't set Nones
1068 continue
1069 elif isinstance(prop, hyperdb.Date):
1070 value = date.Date(value)
1071 elif isinstance(prop, hyperdb.Interval):
1072 value = date.Interval(value)
1073 elif isinstance(prop, hyperdb.Password):
1074 pwd = password.Password()
1075 pwd.unpack(value)
1076 value = pwd
1077 d[propname] = value
1079 # extract the extraneous journalling gumpf and nuke it
1080 if d.has_key('creator'):
1081 creator = d['creator']
1082 del d['creator']
1083 if d.has_key('creation'):
1084 creation = d['creation']
1085 del d['creation']
1086 if d.has_key('activity'):
1087 del d['activity']
1089 # add the node and journal
1090 self.db.addnode(self.classname, newid, d)
1091 self.db.addjournal(self.classname, newid, 'create', d, creator,
1092 creation)
1093 return newid
1095 _marker = []
1096 def get(self, nodeid, propname, default=_marker, cache=1):
1097 '''Get the value of a property on an existing node of this class.
1099 'nodeid' must be the id of an existing node of this class or an
1100 IndexError is raised. 'propname' must be the name of a property
1101 of this class or a KeyError is raised.
1103 'cache' indicates whether the transaction cache should be queried
1104 for the node. If the node has been modified and you need to
1105 determine what its values prior to modification are, you need to
1106 set cache=0.
1107 '''
1108 if propname == 'id':
1109 return nodeid
1111 if propname == 'creation':
1112 if not self.do_journal:
1113 raise ValueError, 'Journalling is disabled for this class'
1114 journal = self.db.getjournal(self.classname, nodeid)
1115 if journal:
1116 return self.db.getjournal(self.classname, nodeid)[0][1]
1117 else:
1118 # on the strange chance that there's no journal
1119 return date.Date()
1120 if propname == 'activity':
1121 if not self.do_journal:
1122 raise ValueError, 'Journalling is disabled for this class'
1123 journal = self.db.getjournal(self.classname, nodeid)
1124 if journal:
1125 return self.db.getjournal(self.classname, nodeid)[-1][1]
1126 else:
1127 # on the strange chance that there's no journal
1128 return date.Date()
1129 if propname == 'creator':
1130 if not self.do_journal:
1131 raise ValueError, 'Journalling is disabled for this class'
1132 journal = self.db.getjournal(self.classname, nodeid)
1133 if journal:
1134 name = self.db.getjournal(self.classname, nodeid)[0][2]
1135 else:
1136 return None
1137 try:
1138 return self.db.user.lookup(name)
1139 except KeyError:
1140 # the journaltag user doesn't exist any more
1141 return None
1143 # get the property (raises KeyErorr if invalid)
1144 prop = self.properties[propname]
1146 # get the node's dict
1147 d = self.db.getnode(self.classname, nodeid) #, cache=cache)
1149 if not d.has_key(propname):
1150 if default is self._marker:
1151 if isinstance(prop, Multilink):
1152 return []
1153 else:
1154 return None
1155 else:
1156 return default
1158 # don't pass our list to other code
1159 if isinstance(prop, Multilink):
1160 return d[propname][:]
1162 return d[propname]
1164 def getnode(self, nodeid, cache=1):
1165 ''' Return a convenience wrapper for the node.
1167 'nodeid' must be the id of an existing node of this class or an
1168 IndexError is raised.
1170 'cache' indicates whether the transaction cache should be queried
1171 for the node. If the node has been modified and you need to
1172 determine what its values prior to modification are, you need to
1173 set cache=0.
1174 '''
1175 return Node(self, nodeid, cache=cache)
1177 def set(self, nodeid, **propvalues):
1178 '''Modify a property on an existing node of this class.
1180 'nodeid' must be the id of an existing node of this class or an
1181 IndexError is raised.
1183 Each key in 'propvalues' must be the name of a property of this
1184 class or a KeyError is raised.
1186 All values in 'propvalues' must be acceptable types for their
1187 corresponding properties or a TypeError is raised.
1189 If the value of the key property is set, it must not collide with
1190 other key strings or a ValueError is raised.
1192 If the value of a Link or Multilink property contains an invalid
1193 node id, a ValueError is raised.
1194 '''
1195 if not propvalues:
1196 return propvalues
1198 if propvalues.has_key('creation') or propvalues.has_key('activity'):
1199 raise KeyError, '"creation" and "activity" are reserved'
1201 if propvalues.has_key('id'):
1202 raise KeyError, '"id" is reserved'
1204 if self.db.journaltag is None:
1205 raise DatabaseError, 'Database open read-only'
1207 self.fireAuditors('set', nodeid, propvalues)
1208 # Take a copy of the node dict so that the subsequent set
1209 # operation doesn't modify the oldvalues structure.
1210 # XXX used to try the cache here first
1211 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1213 node = self.db.getnode(self.classname, nodeid)
1214 if self.is_retired(nodeid):
1215 raise IndexError, 'Requested item is retired'
1216 num_re = re.compile('^\d+$')
1218 # if the journal value is to be different, store it in here
1219 journalvalues = {}
1221 # remember the add/remove stuff for multilinks, making it easier
1222 # for the Database layer to do its stuff
1223 multilink_changes = {}
1225 for propname, value in propvalues.items():
1226 # check to make sure we're not duplicating an existing key
1227 if propname == self.key and node[propname] != value:
1228 try:
1229 self.lookup(value)
1230 except KeyError:
1231 pass
1232 else:
1233 raise ValueError, 'node with key "%s" exists'%value
1235 # this will raise the KeyError if the property isn't valid
1236 # ... we don't use getprops() here because we only care about
1237 # the writeable properties.
1238 prop = self.properties[propname]
1240 # if the value's the same as the existing value, no sense in
1241 # doing anything
1242 if node.has_key(propname) and value == node[propname]:
1243 del propvalues[propname]
1244 continue
1246 # do stuff based on the prop type
1247 if isinstance(prop, Link):
1248 link_class = prop.classname
1249 # if it isn't a number, it's a key
1250 if value is not None and not isinstance(value, type('')):
1251 raise ValueError, 'property "%s" link value be a string'%(
1252 propname)
1253 if isinstance(value, type('')) and not num_re.match(value):
1254 try:
1255 value = self.db.classes[link_class].lookup(value)
1256 except (TypeError, KeyError):
1257 raise IndexError, 'new property "%s": %s not a %s'%(
1258 propname, value, prop.classname)
1260 if (value is not None and
1261 not self.db.getclass(link_class).hasnode(value)):
1262 raise IndexError, '%s has no node %s'%(link_class, value)
1264 if self.do_journal and prop.do_journal:
1265 # register the unlink with the old linked node
1266 if node[propname] is not None:
1267 self.db.addjournal(link_class, node[propname], 'unlink',
1268 (self.classname, nodeid, propname))
1270 # register the link with the newly linked node
1271 if value is not None:
1272 self.db.addjournal(link_class, value, 'link',
1273 (self.classname, nodeid, propname))
1275 elif isinstance(prop, Multilink):
1276 if type(value) != type([]):
1277 raise TypeError, 'new property "%s" not a list of'\
1278 ' ids'%propname
1279 link_class = self.properties[propname].classname
1280 l = []
1281 for entry in value:
1282 # if it isn't a number, it's a key
1283 if type(entry) != type(''):
1284 raise ValueError, 'new property "%s" link value ' \
1285 'must be a string'%propname
1286 if not num_re.match(entry):
1287 try:
1288 entry = self.db.classes[link_class].lookup(entry)
1289 except (TypeError, KeyError):
1290 raise IndexError, 'new property "%s": %s not a %s'%(
1291 propname, entry,
1292 self.properties[propname].classname)
1293 l.append(entry)
1294 value = l
1295 propvalues[propname] = value
1297 # figure the journal entry for this property
1298 add = []
1299 remove = []
1301 # handle removals
1302 if node.has_key(propname):
1303 l = node[propname]
1304 else:
1305 l = []
1306 for id in l[:]:
1307 if id in value:
1308 continue
1309 # register the unlink with the old linked node
1310 if self.do_journal and self.properties[propname].do_journal:
1311 self.db.addjournal(link_class, id, 'unlink',
1312 (self.classname, nodeid, propname))
1313 l.remove(id)
1314 remove.append(id)
1316 # handle additions
1317 for id in value:
1318 if not self.db.getclass(link_class).hasnode(id):
1319 raise IndexError, '%s has no node %s'%(link_class, id)
1320 if id in l:
1321 continue
1322 # register the link with the newly linked node
1323 if self.do_journal and self.properties[propname].do_journal:
1324 self.db.addjournal(link_class, id, 'link',
1325 (self.classname, nodeid, propname))
1326 l.append(id)
1327 add.append(id)
1329 # figure the journal entry
1330 l = []
1331 if add:
1332 l.append(('+', add))
1333 if remove:
1334 l.append(('-', remove))
1335 multilink_changes[propname] = (add, remove)
1336 if l:
1337 journalvalues[propname] = tuple(l)
1339 elif isinstance(prop, String):
1340 if value is not None and type(value) != type(''):
1341 raise TypeError, 'new property "%s" not a string'%propname
1343 elif isinstance(prop, Password):
1344 if not isinstance(value, password.Password):
1345 raise TypeError, 'new property "%s" not a Password'%propname
1346 propvalues[propname] = value
1348 elif value is not None and isinstance(prop, Date):
1349 if not isinstance(value, date.Date):
1350 raise TypeError, 'new property "%s" not a Date'% propname
1351 propvalues[propname] = value
1353 elif value is not None and isinstance(prop, Interval):
1354 if not isinstance(value, date.Interval):
1355 raise TypeError, 'new property "%s" not an '\
1356 'Interval'%propname
1357 propvalues[propname] = value
1359 elif value is not None and isinstance(prop, Number):
1360 try:
1361 float(value)
1362 except ValueError:
1363 raise TypeError, 'new property "%s" not numeric'%propname
1365 elif value is not None and isinstance(prop, Boolean):
1366 try:
1367 int(value)
1368 except ValueError:
1369 raise TypeError, 'new property "%s" not boolean'%propname
1371 node[propname] = value
1373 # nothing to do?
1374 if not propvalues:
1375 return propvalues
1377 # do the set, and journal it
1378 self.db.setnode(self.classname, nodeid, node, multilink_changes)
1380 if self.do_journal:
1381 propvalues.update(journalvalues)
1382 self.db.addjournal(self.classname, nodeid, 'set', propvalues)
1384 self.fireReactors('set', nodeid, oldvalues)
1386 return propvalues
1388 def retire(self, nodeid):
1389 '''Retire a node.
1391 The properties on the node remain available from the get() method,
1392 and the node's id is never reused.
1394 Retired nodes are not returned by the find(), list(), or lookup()
1395 methods, and other nodes may reuse the values of their key properties.
1396 '''
1397 if self.db.journaltag is None:
1398 raise DatabaseError, 'Database open read-only'
1400 cursor = self.db.conn.cursor()
1401 sql = 'update _%s set __retired__=1 where id=%s'%(self.classname,
1402 self.db.arg)
1403 if __debug__:
1404 print >>hyperdb.DEBUG, 'retire', (self, sql, nodeid)
1405 cursor.execute(sql, (nodeid,))
1407 def is_retired(self, nodeid):
1408 '''Return true if the node is rerired
1409 '''
1410 cursor = self.db.conn.cursor()
1411 sql = 'select __retired__ from _%s where id=%s'%(self.classname,
1412 self.db.arg)
1413 if __debug__:
1414 print >>hyperdb.DEBUG, 'is_retired', (self, sql, nodeid)
1415 cursor.execute(sql, (nodeid,))
1416 return int(cursor.fetchone()[0])
1418 def destroy(self, nodeid):
1419 '''Destroy a node.
1421 WARNING: this method should never be used except in extremely rare
1422 situations where there could never be links to the node being
1423 deleted
1424 WARNING: use retire() instead
1425 WARNING: the properties of this node will not be available ever again
1426 WARNING: really, use retire() instead
1428 Well, I think that's enough warnings. This method exists mostly to
1429 support the session storage of the cgi interface.
1431 The node is completely removed from the hyperdb, including all journal
1432 entries. It will no longer be available, and will generally break code
1433 if there are any references to the node.
1434 '''
1435 if self.db.journaltag is None:
1436 raise DatabaseError, 'Database open read-only'
1437 self.db.destroynode(self.classname, nodeid)
1439 def history(self, nodeid):
1440 '''Retrieve the journal of edits on a particular node.
1442 'nodeid' must be the id of an existing node of this class or an
1443 IndexError is raised.
1445 The returned list contains tuples of the form
1447 (date, tag, action, params)
1449 'date' is a Timestamp object specifying the time of the change and
1450 'tag' is the journaltag specified when the database was opened.
1451 '''
1452 if not self.do_journal:
1453 raise ValueError, 'Journalling is disabled for this class'
1454 return self.db.getjournal(self.classname, nodeid)
1456 # Locating nodes:
1457 def hasnode(self, nodeid):
1458 '''Determine if the given nodeid actually exists
1459 '''
1460 return self.db.hasnode(self.classname, nodeid)
1462 def setkey(self, propname):
1463 '''Select a String property of this class to be the key property.
1465 'propname' must be the name of a String property of this class or
1466 None, or a TypeError is raised. The values of the key property on
1467 all existing nodes must be unique or a ValueError is raised.
1468 '''
1469 # XXX create an index on the key prop column
1470 prop = self.getprops()[propname]
1471 if not isinstance(prop, String):
1472 raise TypeError, 'key properties must be String'
1473 self.key = propname
1475 def getkey(self):
1476 '''Return the name of the key property for this class or None.'''
1477 return self.key
1479 def labelprop(self, default_to_id=0):
1480 ''' Return the property name for a label for the given node.
1482 This method attempts to generate a consistent label for the node.
1483 It tries the following in order:
1484 1. key property
1485 2. "name" property
1486 3. "title" property
1487 4. first property from the sorted property name list
1488 '''
1489 k = self.getkey()
1490 if k:
1491 return k
1492 props = self.getprops()
1493 if props.has_key('name'):
1494 return 'name'
1495 elif props.has_key('title'):
1496 return 'title'
1497 if default_to_id:
1498 return 'id'
1499 props = props.keys()
1500 props.sort()
1501 return props[0]
1503 def lookup(self, keyvalue):
1504 '''Locate a particular node by its key property and return its id.
1506 If this class has no key property, a TypeError is raised. If the
1507 'keyvalue' matches one of the values for the key property among
1508 the nodes in this class, the matching node's id is returned;
1509 otherwise a KeyError is raised.
1510 '''
1511 if not self.key:
1512 raise TypeError, 'No key property set for class %s'%self.classname
1514 cursor = self.db.conn.cursor()
1515 sql = 'select id from _%s where _%s=%s'%(self.classname, self.key,
1516 self.db.arg)
1517 if __debug__:
1518 print >>hyperdb.DEBUG, 'lookup', (self, sql, keyvalue)
1519 cursor.execute(sql, (keyvalue,))
1521 # see if there was a result
1522 l = cursor.fetchall()
1523 if not l:
1524 raise KeyError, keyvalue
1526 # return the id
1527 return l[0][0]
1529 def find(self, **propspec):
1530 '''Get the ids of nodes in this class which link to the given nodes.
1532 'propspec' consists of keyword args propname={nodeid:1,}
1533 'propname' must be the name of a property in this class, or a
1534 KeyError is raised. That property must be a Link or Multilink
1535 property, or a TypeError is raised.
1537 Any node in this class whose 'propname' property links to any of the
1538 nodeids will be returned. Used by the full text indexing, which knows
1539 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1540 issues:
1542 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1543 '''
1544 if __debug__:
1545 print >>hyperdb.DEBUG, 'find', (self, propspec)
1546 if not propspec:
1547 return []
1548 queries = []
1549 tables = []
1550 allvalues = ()
1551 for prop, values in propspec.items():
1552 allvalues += tuple(values.keys())
1553 a = self.db.arg
1554 tables.append('select nodeid from %s_%s where linkid in (%s)'%(
1555 self.classname, prop, ','.join([a for x in values.keys()])))
1556 sql = '\nintersect\n'.join(tables)
1557 if __debug__:
1558 print >>hyperdb.DEBUG, 'find', (self, sql, allvalues)
1559 cursor = self.db.conn.cursor()
1560 cursor.execute(sql, allvalues)
1561 try:
1562 l = [x[0] for x in cursor.fetchall()]
1563 except gadfly.database.error, message:
1564 if message == 'no more results':
1565 l = []
1566 raise
1567 if __debug__:
1568 print >>hyperdb.DEBUG, 'find ... ', l
1569 return l
1571 def list(self):
1572 ''' Return a list of the ids of the active nodes in this class.
1573 '''
1574 return self.db.getnodeids(self.classname, retired=0)
1576 def filter(self, search_matches, filterspec, sort, group):
1577 ''' Return a list of the ids of the active nodes in this class that
1578 match the 'filter' spec, sorted by the group spec and then the
1579 sort spec
1581 "filterspec" is {propname: value(s)}
1582 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1583 and prop is a prop name or None
1584 "search_matches" is {nodeid: marker}
1585 '''
1586 cn = self.classname
1588 # figure the WHERE clause from the filterspec
1589 props = self.getprops()
1590 frum = ['_'+cn]
1591 where = []
1592 args = []
1593 for k, v in filterspec.items():
1594 propclass = props[k]
1595 if isinstance(propclass, Multilink):
1596 tn = '%s_%s'%(cn, k)
1597 frum.append(tn)
1598 if isinstance(v, type([])):
1599 s = ','.join([self.arg for x in v])
1600 where.append('id=%s.nodeid and %s.linkid in (%s)'%(tn,tn,s))
1601 args = args + v
1602 else:
1603 where.append('id=%s.nodeid and %s.linkid = %s'%(tn, tn,
1604 self.arg))
1605 args.append(v)
1606 else:
1607 if isinstance(v, type([])):
1608 s = ','.join([self.arg for x in v])
1609 where.append('_%s in (%s)'%(k, s))
1610 args = args + v
1611 else:
1612 where.append('_%s=%s'%(k, self.arg))
1613 args.append(v)
1615 # add results of full text search
1616 if search_matches is not None:
1617 v = search_matches.keys()
1618 s = ','.join([self.arg for x in v])
1619 where.append('id in (%s)'%s)
1620 args = args + v
1622 # figure the order by clause
1623 orderby = []
1624 ordercols = []
1625 if sort[0] is not None and sort[1] is not None:
1626 if sort[0] != '-':
1627 orderby.append('_'+sort[1])
1628 ordercols.append(sort[1])
1629 else:
1630 orderby.append('_'+sort[1]+' desc')
1631 ordercols.append(sort[1])
1633 # figure the group by clause
1634 groupby = []
1635 groupcols = []
1636 if group[0] is not None and group[1] is not None:
1637 if group[0] != '-':
1638 groupby.append('_'+group[1])
1639 groupcols.append(group[1])
1640 else:
1641 groupby.append('_'+group[1]+' desc')
1642 groupcols.append(group[1])
1644 # construct the SQL
1645 frum = ','.join(frum)
1646 where = ' and '.join(where)
1647 cols = ['id']
1648 if orderby:
1649 cols = cols + ordercols
1650 order = ' order by %s'%(','.join(orderby))
1651 else:
1652 order = ''
1653 if groupby:
1654 cols = cols + groupcols
1655 group = ' group by %s'%(','.join(groupby))
1656 else:
1657 group = ''
1658 cols = ','.join(cols)
1659 sql = 'select %s from %s where %s%s%s'%(cols, frum, where, order,
1660 group)
1661 args = tuple(args)
1662 if __debug__:
1663 print >>hyperdb.DEBUG, 'find', (self, sql, args)
1664 cursor = self.db.conn.cursor()
1665 cursor.execute(sql, args)
1667 def count(self):
1668 '''Get the number of nodes in this class.
1670 If the returned integer is 'numnodes', the ids of all the nodes
1671 in this class run from 1 to numnodes, and numnodes+1 will be the
1672 id of the next node to be created in this class.
1673 '''
1674 return self.db.countnodes(self.classname)
1676 # Manipulating properties:
1677 def getprops(self, protected=1):
1678 '''Return a dictionary mapping property names to property objects.
1679 If the "protected" flag is true, we include protected properties -
1680 those which may not be modified.
1681 '''
1682 d = self.properties.copy()
1683 if protected:
1684 d['id'] = String()
1685 d['creation'] = hyperdb.Date()
1686 d['activity'] = hyperdb.Date()
1687 d['creator'] = hyperdb.Link("user")
1688 return d
1690 def addprop(self, **properties):
1691 '''Add properties to this class.
1693 The keyword arguments in 'properties' must map names to property
1694 objects, or a TypeError is raised. None of the keys in 'properties'
1695 may collide with the names of existing properties, or a ValueError
1696 is raised before any properties have been added.
1697 '''
1698 for key in properties.keys():
1699 if self.properties.has_key(key):
1700 raise ValueError, key
1701 self.properties.update(properties)
1703 def index(self, nodeid):
1704 '''Add (or refresh) the node to search indexes
1705 '''
1706 # find all the String properties that have indexme
1707 for prop, propclass in self.getprops().items():
1708 if isinstance(propclass, String) and propclass.indexme:
1709 try:
1710 value = str(self.get(nodeid, prop))
1711 except IndexError:
1712 # node no longer exists - entry should be removed
1713 self.db.indexer.purge_entry((self.classname, nodeid, prop))
1714 else:
1715 # and index them under (classname, nodeid, property)
1716 self.db.indexer.add_text((self.classname, nodeid, prop),
1717 value)
1720 #
1721 # Detector interface
1722 #
1723 def audit(self, event, detector):
1724 '''Register a detector
1725 '''
1726 l = self.auditors[event]
1727 if detector not in l:
1728 self.auditors[event].append(detector)
1730 def fireAuditors(self, action, nodeid, newvalues):
1731 '''Fire all registered auditors.
1732 '''
1733 for audit in self.auditors[action]:
1734 audit(self.db, self, nodeid, newvalues)
1736 def react(self, event, detector):
1737 '''Register a detector
1738 '''
1739 l = self.reactors[event]
1740 if detector not in l:
1741 self.reactors[event].append(detector)
1743 def fireReactors(self, action, nodeid, oldvalues):
1744 '''Fire all registered reactors.
1745 '''
1746 for react in self.reactors[action]:
1747 react(self.db, self, nodeid, oldvalues)
1749 class FileClass(Class):
1750 '''This class defines a large chunk of data. To support this, it has a
1751 mandatory String property "content" which is typically saved off
1752 externally to the hyperdb.
1754 The default MIME type of this data is defined by the
1755 "default_mime_type" class attribute, which may be overridden by each
1756 node if the class defines a "type" String property.
1757 '''
1758 default_mime_type = 'text/plain'
1760 def create(self, **propvalues):
1761 ''' snaffle the file propvalue and store in a file
1762 '''
1763 content = propvalues['content']
1764 del propvalues['content']
1765 newid = Class.create(self, **propvalues)
1766 self.db.storefile(self.classname, newid, None, content)
1767 return newid
1769 def import_list(self, propnames, proplist):
1770 ''' Trap the "content" property...
1771 '''
1772 # dupe this list so we don't affect others
1773 propnames = propnames[:]
1775 # extract the "content" property from the proplist
1776 i = propnames.index('content')
1777 content = eval(proplist[i])
1778 del propnames[i]
1779 del proplist[i]
1781 # do the normal import
1782 newid = Class.import_list(self, propnames, proplist)
1784 # save off the "content" file
1785 self.db.storefile(self.classname, newid, None, content)
1786 return newid
1788 _marker = []
1789 def get(self, nodeid, propname, default=_marker, cache=1):
1790 ''' trap the content propname and get it from the file
1791 '''
1793 poss_msg = 'Possibly a access right configuration problem.'
1794 if propname == 'content':
1795 try:
1796 return self.db.getfile(self.classname, nodeid, None)
1797 except IOError, (strerror):
1798 # BUG: by catching this we donot see an error in the log.
1799 return 'ERROR reading file: %s%s\n%s\n%s'%(
1800 self.classname, nodeid, poss_msg, strerror)
1801 if default is not self._marker:
1802 return Class.get(self, nodeid, propname, default, cache=cache)
1803 else:
1804 return Class.get(self, nodeid, propname, cache=cache)
1806 def getprops(self, protected=1):
1807 ''' In addition to the actual properties on the node, these methods
1808 provide the "content" property. If the "protected" flag is true,
1809 we include protected properties - those which may not be
1810 modified.
1811 '''
1812 d = Class.getprops(self, protected=protected).copy()
1813 d['content'] = hyperdb.String()
1814 return d
1816 def index(self, nodeid):
1817 ''' Index the node in the search index.
1819 We want to index the content in addition to the normal String
1820 property indexing.
1821 '''
1822 # perform normal indexing
1823 Class.index(self, nodeid)
1825 # get the content to index
1826 content = self.get(nodeid, 'content')
1828 # figure the mime type
1829 if self.properties.has_key('type'):
1830 mime_type = self.get(nodeid, 'type')
1831 else:
1832 mime_type = self.default_mime_type
1834 # and index!
1835 self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
1836 mime_type)
1838 # XXX deviation from spec - was called ItemClass
1839 class IssueClass(Class, roundupdb.IssueClass):
1840 # Overridden methods:
1841 def __init__(self, db, classname, **properties):
1842 '''The newly-created class automatically includes the "messages",
1843 "files", "nosy", and "superseder" properties. If the 'properties'
1844 dictionary attempts to specify any of these properties or a
1845 "creation" or "activity" property, a ValueError is raised.
1846 '''
1847 if not properties.has_key('title'):
1848 properties['title'] = hyperdb.String(indexme='yes')
1849 if not properties.has_key('messages'):
1850 properties['messages'] = hyperdb.Multilink("msg")
1851 if not properties.has_key('files'):
1852 properties['files'] = hyperdb.Multilink("file")
1853 if not properties.has_key('nosy'):
1854 # note: journalling is turned off as it really just wastes
1855 # space. this behaviour may be overridden in an instance
1856 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
1857 if not properties.has_key('superseder'):
1858 properties['superseder'] = hyperdb.Multilink(classname)
1859 Class.__init__(self, db, classname, **properties)