1 # $Id: rdbms_common.py,v 1.89 2004-04-05 07:13:10 richard Exp $
2 ''' Relational database (SQL) backend common code.
4 Basics:
6 - map roundup classes to relational tables
7 - automatically detect schema changes and modify the table schemas
8 appropriately (we store the "database version" of the schema in the
9 database itself as the only row of the "schema" table)
10 - multilinks (which represent a many-to-many relationship) are handled through
11 intermediate tables
12 - journals are stored adjunct to the per-class tables
13 - table names and columns have "_" prepended so the names can't clash with
14 restricted names (like "order")
15 - retirement is determined by the __retired__ column being true
17 Database-specific changes may generally be pushed out to the overridable
18 sql_* methods, since everything else should be fairly generic. There's
19 probably a bit of work to be done if a database is used that actually
20 honors column typing, since the initial databases don't (sqlite stores
21 everything as a string.)
23 The schema of the hyperdb being mapped to the database is stored in the
24 database itself as a repr()'ed dictionary of information about each Class
25 that maps to a table. If that information differs from the hyperdb schema,
26 then we update it. We also store in the schema dict a version which
27 allows us to upgrade the database schema when necessary. See upgrade_db().
28 '''
29 __docformat__ = 'restructuredtext'
31 # standard python modules
32 import sys, os, time, re, errno, weakref, copy
34 # roundup modules
35 from roundup import hyperdb, date, password, roundupdb, security
36 from roundup.hyperdb import String, Password, Date, Interval, Link, \
37 Multilink, DatabaseError, Boolean, Number, Node
38 from roundup.backends import locking
40 # support
41 from blobfiles import FileStorage
42 from indexer_rdbms import Indexer
43 from sessions_rdbms import Sessions, OneTimeKeys
44 from roundup.date import Range
46 # number of rows to keep in memory
47 ROW_CACHE_SIZE = 100
49 def _num_cvt(num):
50 num = str(num)
51 try:
52 return int(num)
53 except:
54 return float(num)
56 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
57 ''' Wrapper around an SQL database that presents a hyperdb interface.
59 - some functionality is specific to the actual SQL database, hence
60 the sql_* methods that are NotImplemented
61 - we keep a cache of the latest ROW_CACHE_SIZE row fetches.
62 '''
63 def __init__(self, config, journaltag=None):
64 ''' Open the database and load the schema from it.
65 '''
66 self.config, self.journaltag = config, journaltag
67 self.dir = config.DATABASE
68 self.classes = {}
69 self.indexer = Indexer(self)
70 self.security = security.Security(self)
72 # additional transaction support for external files and the like
73 self.transactions = []
75 # keep a cache of the N most recently retrieved rows of any kind
76 # (classname, nodeid) = row
77 self.cache = {}
78 self.cache_lru = []
80 # database lock
81 self.lockfile = None
83 # open a connection to the database, creating the "conn" attribute
84 self.open_connection()
86 def clearCache(self):
87 self.cache = {}
88 self.cache_lru = []
90 def getSessionManager(self):
91 return Sessions(self)
93 def getOTKManager(self):
94 return OneTimeKeys(self)
96 def open_connection(self):
97 ''' Open a connection to the database, creating it if necessary.
99 Must call self.load_dbschema()
100 '''
101 raise NotImplemented
103 def sql(self, sql, args=None):
104 ''' Execute the sql with the optional args.
105 '''
106 if __debug__:
107 print >>hyperdb.DEBUG, (self, sql, args)
108 if args:
109 self.cursor.execute(sql, args)
110 else:
111 self.cursor.execute(sql)
113 def sql_fetchone(self):
114 ''' Fetch a single row. If there's nothing to fetch, return None.
115 '''
116 return self.cursor.fetchone()
118 def sql_fetchall(self):
119 ''' Fetch all rows. If there's nothing to fetch, return [].
120 '''
121 return self.cursor.fetchall()
123 def sql_stringquote(self, value):
124 ''' Quote the string so it's safe to put in the 'sql quotes'
125 '''
126 return re.sub("'", "''", str(value))
128 def init_dbschema(self):
129 self.database_schema = {
130 'version': self.current_db_version,
131 'tables': {}
132 }
134 def load_dbschema(self):
135 ''' Load the schema definition that the database currently implements
136 '''
137 self.cursor.execute('select schema from schema')
138 schema = self.cursor.fetchone()
139 if schema:
140 self.database_schema = eval(schema[0])
141 else:
142 self.database_schema = {}
144 def save_dbschema(self, schema):
145 ''' Save the schema definition that the database currently implements
146 '''
147 s = repr(self.database_schema)
148 self.sql('insert into schema values (%s)', (s,))
150 def post_init(self):
151 ''' Called once the schema initialisation has finished.
153 We should now confirm that the schema defined by our "classes"
154 attribute actually matches the schema in the database.
155 '''
156 save = self.upgrade_db()
158 # now detect changes in the schema
159 tables = self.database_schema['tables']
160 for classname, spec in self.classes.items():
161 if tables.has_key(classname):
162 dbspec = tables[classname]
163 if self.update_class(spec, dbspec):
164 tables[classname] = spec.schema()
165 save = 1
166 else:
167 self.create_class(spec)
168 tables[classname] = spec.schema()
169 save = 1
171 for classname, spec in tables.items():
172 if not self.classes.has_key(classname):
173 self.drop_class(classname, tables[classname])
174 del tables[classname]
175 save = 1
177 # update the database version of the schema
178 if save:
179 self.sql('delete from schema')
180 self.save_dbschema(self.database_schema)
182 # reindex the db if necessary
183 if self.indexer.should_reindex():
184 self.reindex()
186 # commit
187 self.sql_commit()
189 # update this number when we need to make changes to the SQL structure
190 # of the backen database
191 current_db_version = 2
192 def upgrade_db(self):
193 ''' Update the SQL database to reflect changes in the backend code.
195 Return boolean whether we need to save the schema.
196 '''
197 version = self.database_schema.get('version', 1)
198 if version == self.current_db_version:
199 # nothing to do
200 return 0
202 if version == 1:
203 # change the schema structure
204 self.database_schema = {'tables': self.database_schema}
206 # version 1 didn't have the actor column (note that in
207 # MySQL this will also transition the tables to typed columns)
208 self.add_actor_column()
210 # version 1 doesn't have the OTK, session and indexing in the
211 # database
212 self.create_version_2_tables()
214 self.database_schema['version'] = self.current_db_version
215 return 1
218 def refresh_database(self):
219 self.post_init()
221 def reindex(self):
222 for klass in self.classes.values():
223 for nodeid in klass.list():
224 klass.index(nodeid)
225 self.indexer.save_index()
228 hyperdb_to_sql_datatypes = {
229 hyperdb.String : 'VARCHAR(255)',
230 hyperdb.Date : 'TIMESTAMP',
231 hyperdb.Link : 'INTEGER',
232 hyperdb.Interval : 'VARCHAR(255)',
233 hyperdb.Password : 'VARCHAR(255)',
234 hyperdb.Boolean : 'BOOLEAN',
235 hyperdb.Number : 'REAL',
236 }
237 def determine_columns(self, properties):
238 ''' Figure the column names and multilink properties from the spec
240 "properties" is a list of (name, prop) where prop may be an
241 instance of a hyperdb "type" _or_ a string repr of that type.
242 '''
243 cols = [
244 ('_actor', self.hyperdb_to_sql_datatypes[hyperdb.Link]),
245 ('_activity', self.hyperdb_to_sql_datatypes[hyperdb.Date]),
246 ('_creator', self.hyperdb_to_sql_datatypes[hyperdb.Link]),
247 ('_creation', self.hyperdb_to_sql_datatypes[hyperdb.Date]),
248 ]
249 mls = []
250 # add the multilinks separately
251 for col, prop in properties:
252 if isinstance(prop, Multilink):
253 mls.append(col)
254 continue
256 if isinstance(prop, type('')):
257 raise ValueError, "string property spec!"
258 #and prop.find('Multilink') != -1:
259 #mls.append(col)
261 datatype = self.hyperdb_to_sql_datatypes[prop.__class__]
262 cols.append(('_'+col, datatype))
264 cols.sort()
265 return cols, mls
267 def update_class(self, spec, old_spec, force=0):
268 ''' Determine the differences between the current spec and the
269 database version of the spec, and update where necessary.
271 If 'force' is true, update the database anyway.
272 '''
273 new_has = spec.properties.has_key
274 new_spec = spec.schema()
275 new_spec[1].sort()
276 old_spec[1].sort()
277 if not force and new_spec == old_spec:
278 # no changes
279 return 0
281 if __debug__:
282 print >>hyperdb.DEBUG, 'update_class FIRING'
284 # detect key prop change for potential index change
285 keyprop_changes = {}
286 if new_spec[0] != old_spec[0]:
287 keyprop_changes = {'remove': old_spec[0], 'add': new_spec[0]}
289 # detect multilinks that have been removed, and drop their table
290 old_has = {}
291 for name, prop in old_spec[1]:
292 old_has[name] = 1
293 if new_has(name):
294 continue
296 if prop.find('Multilink to') != -1:
297 # first drop indexes.
298 self.drop_multilink_table_indexes(spec.classname, name)
300 # now the multilink table itself
301 sql = 'drop table %s_%s'%(spec.classname, name)
302 else:
303 # if this is the key prop, drop the index first
304 if old_spec[0] == prop:
305 self.drop_class_table_key_index(spec.classname, name)
306 del keyprop_changes['remove']
308 # drop the column
309 sql = 'alter table _%s drop column _%s'%(spec.classname, name)
311 if __debug__:
312 print >>hyperdb.DEBUG, 'update_class', (self, sql)
313 self.cursor.execute(sql)
314 old_has = old_has.has_key
316 # if we didn't remove the key prop just then, but the key prop has
317 # changed, we still need to remove the old index
318 if keyprop_changes.has_key('remove'):
319 self.drop_class_table_key_index(spec.classname,
320 keyprop_changes['remove'])
322 # add new columns
323 for propname, x in new_spec[1]:
324 if old_has(propname):
325 continue
326 sql = 'alter table _%s add column _%s varchar(255)'%(
327 spec.classname, propname)
328 if __debug__:
329 print >>hyperdb.DEBUG, 'update_class', (self, sql)
330 self.cursor.execute(sql)
332 # if the new column is a key prop, we need an index!
333 if new_spec[0] == propname:
334 self.create_class_table_key_index(spec.classname, propname)
335 del keyprop_changes['add']
337 # if we didn't add the key prop just then, but the key prop has
338 # changed, we still need to add the new index
339 if keyprop_changes.has_key('add'):
340 self.create_class_table_key_index(spec.classname,
341 keyprop_changes['add'])
343 return 1
345 def create_class_table(self, spec):
346 '''Create the class table for the given Class "spec". Creates the
347 indexes too.'''
348 cols, mls = self.determine_columns(spec.properties.items())
350 # add on our special columns
351 cols.append(('id', 'INTEGER PRIMARY KEY'))
352 cols.append(('__retired__', 'INTEGER DEFAULT 0'))
354 # create the base table
355 scols = ','.join(['%s %s'%x for x in cols])
356 sql = 'create table _%s (%s)'%(spec.classname, scols)
357 if __debug__:
358 print >>hyperdb.DEBUG, 'create_class', (self, sql)
359 self.cursor.execute(sql)
361 self.create_class_table_indexes(spec)
363 return cols, mls
365 def create_class_table_indexes(self, spec):
366 ''' create the class table for the given spec
367 '''
368 # create __retired__ index
369 index_sql2 = 'create index _%s_retired_idx on _%s(__retired__)'%(
370 spec.classname, spec.classname)
371 if __debug__:
372 print >>hyperdb.DEBUG, 'create_index', (self, index_sql2)
373 self.cursor.execute(index_sql2)
375 # create index for key property
376 if spec.key:
377 if __debug__:
378 print >>hyperdb.DEBUG, 'update_class setting keyprop %r'% \
379 spec.key
380 index_sql3 = 'create index _%s_%s_idx on _%s(_%s)'%(
381 spec.classname, spec.key,
382 spec.classname, spec.key)
383 if __debug__:
384 print >>hyperdb.DEBUG, 'create_index', (self, index_sql3)
385 self.cursor.execute(index_sql3)
387 def drop_class_table_indexes(self, cn, key):
388 # drop the old table indexes first
389 l = ['_%s_id_idx'%cn, '_%s_retired_idx'%cn]
390 if key:
391 l.append('_%s_%s_idx'%(cn, key))
393 table_name = '_%s'%cn
394 for index_name in l:
395 if not self.sql_index_exists(table_name, index_name):
396 continue
397 index_sql = 'drop index '+index_name
398 if __debug__:
399 print >>hyperdb.DEBUG, 'drop_index', (self, index_sql)
400 self.cursor.execute(index_sql)
402 def create_class_table_key_index(self, cn, key):
403 ''' create the class table for the given spec
404 '''
405 sql = 'create index _%s_%s_idx on _%s(_%s)'%(cn, key, cn, key)
406 if __debug__:
407 print >>hyperdb.DEBUG, 'create_class_tab_key_index', (self, sql)
408 self.cursor.execute(sql)
410 def drop_class_table_key_index(self, cn, key):
411 table_name = '_%s'%cn
412 index_name = '_%s_%s_idx'%(cn, key)
413 if not self.sql_index_exists(table_name, index_name):
414 return
415 sql = 'drop index '+index_name
416 if __debug__:
417 print >>hyperdb.DEBUG, 'drop_class_tab_key_index', (self, sql)
418 self.cursor.execute(sql)
420 def create_journal_table(self, spec):
421 ''' create the journal table for a class given the spec and
422 already-determined cols
423 '''
424 # journal table
425 cols = ','.join(['%s varchar'%x
426 for x in 'nodeid date tag action params'.split()])
427 sql = '''create table %s__journal (
428 nodeid integer, date timestamp, tag varchar(255),
429 action varchar(255), params varchar(25))'''%spec.classname
430 if __debug__:
431 print >>hyperdb.DEBUG, 'create_journal_table', (self, sql)
432 self.cursor.execute(sql)
433 self.create_journal_table_indexes(spec)
435 def create_journal_table_indexes(self, spec):
436 # index on nodeid
437 sql = 'create index %s_journ_idx on %s__journal(nodeid)'%(
438 spec.classname, spec.classname)
439 if __debug__:
440 print >>hyperdb.DEBUG, 'create_index', (self, sql)
441 self.cursor.execute(sql)
443 def drop_journal_table_indexes(self, classname):
444 index_name = '%s_journ_idx'%classname
445 if not self.sql_index_exists('%s__journal'%classname, index_name):
446 return
447 index_sql = 'drop index '+index_name
448 if __debug__:
449 print >>hyperdb.DEBUG, 'drop_index', (self, index_sql)
450 self.cursor.execute(index_sql)
452 def create_multilink_table(self, spec, ml):
453 ''' Create a multilink table for the "ml" property of the class
454 given by the spec
455 '''
456 # create the table
457 sql = 'create table %s_%s (linkid varchar, nodeid varchar)'%(
458 spec.classname, ml)
459 if __debug__:
460 print >>hyperdb.DEBUG, 'create_class', (self, sql)
461 self.cursor.execute(sql)
462 self.create_multilink_table_indexes(spec, ml)
464 def create_multilink_table_indexes(self, spec, ml):
465 # create index on linkid
466 index_sql = 'create index %s_%s_l_idx on %s_%s(linkid)'%(
467 spec.classname, ml, spec.classname, ml)
468 if __debug__:
469 print >>hyperdb.DEBUG, 'create_index', (self, index_sql)
470 self.cursor.execute(index_sql)
472 # create index on nodeid
473 index_sql = 'create index %s_%s_n_idx on %s_%s(nodeid)'%(
474 spec.classname, ml, spec.classname, ml)
475 if __debug__:
476 print >>hyperdb.DEBUG, 'create_index', (self, index_sql)
477 self.cursor.execute(index_sql)
479 def drop_multilink_table_indexes(self, classname, ml):
480 l = [
481 '%s_%s_l_idx'%(classname, ml),
482 '%s_%s_n_idx'%(classname, ml)
483 ]
484 table_name = '%s_%s'%(classname, ml)
485 for index_name in l:
486 if not self.sql_index_exists(table_name, index_name):
487 continue
488 index_sql = 'drop index %s'%index_name
489 if __debug__:
490 print >>hyperdb.DEBUG, 'drop_index', (self, index_sql)
491 self.cursor.execute(index_sql)
493 def create_class(self, spec):
494 ''' Create a database table according to the given spec.
495 '''
496 cols, mls = self.create_class_table(spec)
497 self.create_journal_table(spec)
499 # now create the multilink tables
500 for ml in mls:
501 self.create_multilink_table(spec, ml)
503 def drop_class(self, cn, spec):
504 ''' Drop the given table from the database.
506 Drop the journal and multilink tables too.
507 '''
508 properties = spec[1]
509 # figure the multilinks
510 mls = []
511 for propanme, prop in properties:
512 if isinstance(prop, Multilink):
513 mls.append(propname)
515 # drop class table and indexes
516 self.drop_class_table_indexes(cn, spec[0])
518 self.drop_class_table(cn)
520 # drop journal table and indexes
521 self.drop_journal_table_indexes(cn)
522 sql = 'drop table %s__journal'%cn
523 if __debug__:
524 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
525 self.cursor.execute(sql)
527 for ml in mls:
528 # drop multilink table and indexes
529 self.drop_multilink_table_indexes(cn, ml)
530 sql = 'drop table %s_%s'%(spec.classname, ml)
531 if __debug__:
532 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
533 self.cursor.execute(sql)
535 def drop_class_table(self, cn):
536 sql = 'drop table _%s'%cn
537 if __debug__:
538 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
539 self.cursor.execute(sql)
541 #
542 # Classes
543 #
544 def __getattr__(self, classname):
545 ''' A convenient way of calling self.getclass(classname).
546 '''
547 if self.classes.has_key(classname):
548 if __debug__:
549 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
550 return self.classes[classname]
551 raise AttributeError, classname
553 def addclass(self, cl):
554 ''' Add a Class to the hyperdatabase.
555 '''
556 if __debug__:
557 print >>hyperdb.DEBUG, 'addclass', (self, cl)
558 cn = cl.classname
559 if self.classes.has_key(cn):
560 raise ValueError, cn
561 self.classes[cn] = cl
563 # add default Edit and View permissions
564 self.security.addPermission(name="Edit", klass=cn,
565 description="User is allowed to edit "+cn)
566 self.security.addPermission(name="View", klass=cn,
567 description="User is allowed to access "+cn)
569 def getclasses(self):
570 ''' Return a list of the names of all existing classes.
571 '''
572 if __debug__:
573 print >>hyperdb.DEBUG, 'getclasses', (self,)
574 l = self.classes.keys()
575 l.sort()
576 return l
578 def getclass(self, classname):
579 '''Get the Class object representing a particular class.
581 If 'classname' is not a valid class name, a KeyError is raised.
582 '''
583 if __debug__:
584 print >>hyperdb.DEBUG, 'getclass', (self, classname)
585 try:
586 return self.classes[classname]
587 except KeyError:
588 raise KeyError, 'There is no class called "%s"'%classname
590 def clear(self):
591 '''Delete all database contents.
593 Note: I don't commit here, which is different behaviour to the
594 "nuke from orbit" behaviour in the dbs.
595 '''
596 if __debug__:
597 print >>hyperdb.DEBUG, 'clear', (self,)
598 for cn in self.classes.keys():
599 sql = 'delete from _%s'%cn
600 if __debug__:
601 print >>hyperdb.DEBUG, 'clear', (self, sql)
602 self.cursor.execute(sql)
604 #
605 # Nodes
606 #
608 hyperdb_to_sql_value = {
609 hyperdb.String : str,
610 hyperdb.Date : lambda x: x.formal(sep=' ', sec='%.3f'),
611 hyperdb.Link : int,
612 hyperdb.Interval : lambda x: x.serialise(),
613 hyperdb.Password : str,
614 hyperdb.Boolean : lambda x: x and 'TRUE' or 'FALSE',
615 hyperdb.Number : lambda x: x,
616 }
617 def addnode(self, classname, nodeid, node):
618 ''' Add the specified node to its class's db.
619 '''
620 if __debug__:
621 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
623 # determine the column definitions and multilink tables
624 cl = self.classes[classname]
625 cols, mls = self.determine_columns(cl.properties.items())
627 # we'll be supplied these props if we're doing an import
628 values = node.copy()
629 if not values.has_key('creator'):
630 # add in the "calculated" properties (dupe so we don't affect
631 # calling code's node assumptions)
632 values['creation'] = values['activity'] = date.Date()
633 values['actor'] = values['creator'] = self.getuid()
635 cl = self.classes[classname]
636 props = cl.getprops(protected=1)
637 del props['id']
639 # default the non-multilink columns
640 for col, prop in props.items():
641 if not values.has_key(col):
642 if isinstance(prop, Multilink):
643 values[col] = []
644 else:
645 values[col] = None
647 # clear this node out of the cache if it's in there
648 key = (classname, nodeid)
649 if self.cache.has_key(key):
650 del self.cache[key]
651 self.cache_lru.remove(key)
653 # figure the values to insert
654 vals = []
655 for col,dt in cols:
656 prop = props[col[1:]]
657 value = values[col[1:]]
658 if value:
659 value = self.hyperdb_to_sql_value[prop.__class__](value)
660 vals.append(value)
661 vals.append(nodeid)
662 vals = tuple(vals)
664 # make sure the ordering is correct for column name -> column value
665 s = ','.join([self.arg for x in cols]) + ',%s'%self.arg
666 cols = ','.join([col for col,dt in cols]) + ',id'
668 # perform the inserts
669 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
670 if __debug__:
671 print >>hyperdb.DEBUG, 'addnode', (self, sql, vals)
672 self.cursor.execute(sql, vals)
674 # insert the multilink rows
675 for col in mls:
676 t = '%s_%s'%(classname, col)
677 for entry in node[col]:
678 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
679 self.arg, self.arg)
680 self.sql(sql, (entry, nodeid))
682 def setnode(self, classname, nodeid, values, multilink_changes={}):
683 ''' Change the specified node.
684 '''
685 if __debug__:
686 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, values)
688 # clear this node out of the cache if it's in there
689 key = (classname, nodeid)
690 if self.cache.has_key(key):
691 del self.cache[key]
692 self.cache_lru.remove(key)
694 # add the special props
695 values = values.copy()
696 values['activity'] = date.Date()
697 values['actor'] = self.getuid()
699 cl = self.classes[classname]
700 props = cl.getprops()
702 cols = []
703 mls = []
704 # add the multilinks separately
705 for col in values.keys():
706 prop = props[col]
707 if isinstance(prop, Multilink):
708 mls.append(col)
709 else:
710 cols.append(col)
711 cols.sort()
713 # figure the values to insert
714 vals = []
715 for col in cols:
716 prop = props[col]
717 value = values[col]
718 if value is not None:
719 value = self.hyperdb_to_sql_value[prop.__class__](value)
720 vals.append(value)
721 vals.append(int(nodeid))
722 vals = tuple(vals)
724 # if there's any updates to regular columns, do them
725 if cols:
726 # make sure the ordering is correct for column name -> column value
727 s = ','.join(['_%s=%s'%(x, self.arg) for x in cols])
728 cols = ','.join(cols)
730 # perform the update
731 sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
732 if __debug__:
733 print >>hyperdb.DEBUG, 'setnode', (self, sql, vals)
734 self.cursor.execute(sql, vals)
736 # we're probably coming from an import, not a change
737 if not multilink_changes:
738 for name in mls:
739 prop = props[name]
740 value = values[name]
742 t = '%s_%s'%(classname, name)
744 # clear out previous values for this node
745 # XXX numeric ids
746 self.sql('delete from %s where nodeid=%s'%(t, self.arg),
747 (nodeid,))
749 # insert the values for this node
750 for entry in values[name]:
751 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
752 self.arg, self.arg)
753 # XXX numeric ids
754 self.sql(sql, (entry, nodeid))
756 # we have multilink changes to apply
757 for col, (add, remove) in multilink_changes.items():
758 tn = '%s_%s'%(classname, col)
759 if add:
760 sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
761 self.arg, self.arg)
762 for addid in add:
763 # XXX numeric ids
764 self.sql(sql, (int(nodeid), int(addid)))
765 if remove:
766 sql = 'delete from %s where nodeid=%s and linkid=%s'%(tn,
767 self.arg, self.arg)
768 for removeid in remove:
769 # XXX numeric ids
770 self.sql(sql, (int(nodeid), int(removeid)))
772 sql_to_hyperdb_value = {
773 hyperdb.String : str,
774 hyperdb.Date : lambda x:date.Date(str(x).replace(' ', '.')),
775 # hyperdb.Link : int, # XXX numeric ids
776 hyperdb.Link : str,
777 hyperdb.Interval : date.Interval,
778 hyperdb.Password : lambda x: password.Password(encrypted=x),
779 hyperdb.Boolean : int,
780 hyperdb.Number : _num_cvt,
781 }
782 def getnode(self, classname, nodeid):
783 ''' Get a node from the database.
784 '''
785 if __debug__:
786 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid)
788 # see if we have this node cached
789 key = (classname, nodeid)
790 if self.cache.has_key(key):
791 # push us back to the top of the LRU
792 self.cache_lru.remove(key)
793 self.cache_lru.insert(0, key)
794 # return the cached information
795 return self.cache[key]
797 # figure the columns we're fetching
798 cl = self.classes[classname]
799 cols, mls = self.determine_columns(cl.properties.items())
800 scols = ','.join([col for col,dt in cols])
802 # perform the basic property fetch
803 sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
804 self.sql(sql, (nodeid,))
806 values = self.sql_fetchone()
807 if values is None:
808 raise IndexError, 'no such %s node %s'%(classname, nodeid)
810 # make up the node
811 node = {}
812 props = cl.getprops(protected=1)
813 for col in range(len(cols)):
814 name = cols[col][0][1:]
815 value = values[col]
816 if value is not None:
817 value = self.sql_to_hyperdb_value[props[name].__class__](value)
818 node[name] = value
821 # now the multilinks
822 for col in mls:
823 # get the link ids
824 sql = 'select linkid from %s_%s where nodeid=%s'%(classname, col,
825 self.arg)
826 self.cursor.execute(sql, (nodeid,))
827 # extract the first column from the result
828 # XXX numeric ids
829 node[col] = [str(x[0]) for x in self.cursor.fetchall()]
831 # save off in the cache
832 key = (classname, nodeid)
833 self.cache[key] = node
834 # update the LRU
835 self.cache_lru.insert(0, key)
836 if len(self.cache_lru) > ROW_CACHE_SIZE:
837 del self.cache[self.cache_lru.pop()]
839 return node
841 def destroynode(self, classname, nodeid):
842 '''Remove a node from the database. Called exclusively by the
843 destroy() method on Class.
844 '''
845 if __debug__:
846 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
848 # make sure the node exists
849 if not self.hasnode(classname, nodeid):
850 raise IndexError, '%s has no node %s'%(classname, nodeid)
852 # see if we have this node cached
853 if self.cache.has_key((classname, nodeid)):
854 del self.cache[(classname, nodeid)]
856 # see if there's any obvious commit actions that we should get rid of
857 for entry in self.transactions[:]:
858 if entry[1][:2] == (classname, nodeid):
859 self.transactions.remove(entry)
861 # now do the SQL
862 sql = 'delete from _%s where id=%s'%(classname, self.arg)
863 self.sql(sql, (nodeid,))
865 # remove from multilnks
866 cl = self.getclass(classname)
867 x, mls = self.determine_columns(cl.properties.items())
868 for col in mls:
869 # get the link ids
870 sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
871 self.sql(sql, (nodeid,))
873 # remove journal entries
874 sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
875 self.sql(sql, (nodeid,))
877 def hasnode(self, classname, nodeid):
878 ''' Determine if the database has a given node.
879 '''
880 sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
881 if __debug__:
882 print >>hyperdb.DEBUG, 'hasnode', (self, sql, nodeid)
883 self.cursor.execute(sql, (nodeid,))
884 return int(self.cursor.fetchone()[0])
886 def countnodes(self, classname):
887 ''' Count the number of nodes that exist for a particular Class.
888 '''
889 sql = 'select count(*) from _%s'%classname
890 if __debug__:
891 print >>hyperdb.DEBUG, 'countnodes', (self, sql)
892 self.cursor.execute(sql)
893 return self.cursor.fetchone()[0]
895 def addjournal(self, classname, nodeid, action, params, creator=None,
896 creation=None):
897 ''' Journal the Action
898 'action' may be:
900 'create' or 'set' -- 'params' is a dictionary of property values
901 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
902 'retire' -- 'params' is None
903 '''
904 # handle supply of the special journalling parameters (usually
905 # supplied on importing an existing database)
906 if creator:
907 journaltag = creator
908 else:
909 journaltag = self.getuid()
910 if creation:
911 journaldate = creation
912 else:
913 journaldate = date.Date()
915 # create the journal entry
916 cols = 'nodeid,date,tag,action,params'
918 if __debug__:
919 print >>hyperdb.DEBUG, 'addjournal', (nodeid, journaldate,
920 journaltag, action, params)
922 self.save_journal(classname, cols, nodeid, journaldate,
923 journaltag, action, params)
925 def setjournal(self, classname, nodeid, journal):
926 '''Set the journal to the "journal" list.'''
927 # clear out any existing entries
928 self.sql('delete from %s__journal where nodeid=%s'%(classname,
929 self.arg), (nodeid,))
931 # create the journal entry
932 cols = 'nodeid,date,tag,action,params'
934 for nodeid, journaldate, journaltag, action, params in journal:
935 if __debug__:
936 print >>hyperdb.DEBUG, 'setjournal', (nodeid, journaldate,
937 journaltag, action, params)
938 self.save_journal(classname, cols, nodeid, journaldate,
939 journaltag, action, params)
941 def getjournal(self, classname, nodeid):
942 ''' get the journal for id
943 '''
944 # make sure the node exists
945 if not self.hasnode(classname, nodeid):
946 raise IndexError, '%s has no node %s'%(classname, nodeid)
948 cols = ','.join('nodeid date tag action params'.split())
949 return self.load_journal(classname, cols, nodeid)
951 def save_journal(self, classname, cols, nodeid, journaldate,
952 journaltag, action, params):
953 ''' Save the journal entry to the database
954 '''
955 # make the params db-friendly
956 params = repr(params)
957 dc = self.hyperdb_to_sql_value[hyperdb.Date]
958 entry = (nodeid, dc(journaldate), journaltag, action, params)
960 # do the insert
961 a = self.arg
962 sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(
963 classname, cols, a, a, a, a, a)
964 if __debug__:
965 print >>hyperdb.DEBUG, 'save_journal', (self, sql, entry)
966 self.cursor.execute(sql, entry)
968 def load_journal(self, classname, cols, nodeid):
969 ''' Load the journal from the database
970 '''
971 # now get the journal entries
972 sql = 'select %s from %s__journal where nodeid=%s order by date'%(
973 cols, classname, self.arg)
974 if __debug__:
975 print >>hyperdb.DEBUG, 'load_journal', (self, sql, nodeid)
976 self.cursor.execute(sql, (nodeid,))
977 res = []
978 dc = self.sql_to_hyperdb_value[hyperdb.Date]
979 for nodeid, date_stamp, user, action, params in self.cursor.fetchall():
980 params = eval(params)
981 # XXX numeric ids
982 res.append((str(nodeid), dc(date_stamp), user, action, params))
983 return res
985 def pack(self, pack_before):
986 ''' Delete all journal entries except "create" before 'pack_before'.
987 '''
988 # get a 'yyyymmddhhmmss' version of the date
989 date_stamp = pack_before.serialise()
991 # do the delete
992 for classname in self.classes.keys():
993 sql = "delete from %s__journal where date<%s and "\
994 "action<>'create'"%(classname, self.arg)
995 if __debug__:
996 print >>hyperdb.DEBUG, 'pack', (self, sql, date_stamp)
997 self.cursor.execute(sql, (date_stamp,))
999 def sql_commit(self):
1000 ''' Actually commit to the database.
1001 '''
1002 if __debug__:
1003 print >>hyperdb.DEBUG, '+++ commit database connection +++'
1004 self.conn.commit()
1006 def commit(self):
1007 ''' Commit the current transactions.
1009 Save all data changed since the database was opened or since the
1010 last commit() or rollback().
1011 '''
1012 if __debug__:
1013 print >>hyperdb.DEBUG, 'commit', (self,)
1015 # commit the database
1016 self.sql_commit()
1018 # now, do all the other transaction stuff
1019 for method, args in self.transactions:
1020 method(*args)
1022 # save the indexer state
1023 self.indexer.save_index()
1025 # clear out the transactions
1026 self.transactions = []
1028 def sql_rollback(self):
1029 self.conn.rollback()
1031 def rollback(self):
1032 ''' Reverse all actions from the current transaction.
1034 Undo all the changes made since the database was opened or the last
1035 commit() or rollback() was performed.
1036 '''
1037 if __debug__:
1038 print >>hyperdb.DEBUG, 'rollback', (self,)
1040 self.sql_rollback()
1042 # roll back "other" transaction stuff
1043 for method, args in self.transactions:
1044 # delete temporary files
1045 if method == self.doStoreFile:
1046 self.rollbackStoreFile(*args)
1047 self.transactions = []
1049 # clear the cache
1050 self.clearCache()
1052 def sql_close(self):
1053 if __debug__:
1054 print >>hyperdb.DEBUG, '+++ close database connection +++'
1055 self.conn.close()
1057 def close(self):
1058 ''' Close off the connection.
1059 '''
1060 self.indexer.close()
1061 self.sql_close()
1063 #
1064 # The base Class class
1065 #
1066 class Class(hyperdb.Class):
1067 ''' The handle to a particular class of nodes in a hyperdatabase.
1069 All methods except __repr__ and getnode must be implemented by a
1070 concrete backend Class.
1071 '''
1073 def __init__(self, db, classname, **properties):
1074 '''Create a new class with a given name and property specification.
1076 'classname' must not collide with the name of an existing class,
1077 or a ValueError is raised. The keyword arguments in 'properties'
1078 must map names to property objects, or a TypeError is raised.
1079 '''
1080 for name in 'creation activity creator actor'.split():
1081 if properties.has_key(name):
1082 raise ValueError, '"creation", "activity", "creator" and '\
1083 '"actor" are reserved'
1085 self.classname = classname
1086 self.properties = properties
1087 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
1088 self.key = ''
1090 # should we journal changes (default yes)
1091 self.do_journal = 1
1093 # do the db-related init stuff
1094 db.addclass(self)
1096 self.auditors = {'create': [], 'set': [], 'retire': [], 'restore': []}
1097 self.reactors = {'create': [], 'set': [], 'retire': [], 'restore': []}
1099 def schema(self):
1100 ''' A dumpable version of the schema that we can store in the
1101 database
1102 '''
1103 return (self.key, [(x, repr(y)) for x,y in self.properties.items()])
1105 def enableJournalling(self):
1106 '''Turn journalling on for this class
1107 '''
1108 self.do_journal = 1
1110 def disableJournalling(self):
1111 '''Turn journalling off for this class
1112 '''
1113 self.do_journal = 0
1115 # Editing nodes:
1116 def create(self, **propvalues):
1117 ''' Create a new node of this class and return its id.
1119 The keyword arguments in 'propvalues' map property names to values.
1121 The values of arguments must be acceptable for the types of their
1122 corresponding properties or a TypeError is raised.
1124 If this class has a key property, it must be present and its value
1125 must not collide with other key strings or a ValueError is raised.
1127 Any other properties on this class that are missing from the
1128 'propvalues' dictionary are set to None.
1130 If an id in a link or multilink property does not refer to a valid
1131 node, an IndexError is raised.
1132 '''
1133 self.fireAuditors('create', None, propvalues)
1134 newid = self.create_inner(**propvalues)
1135 self.fireReactors('create', newid, None)
1136 return newid
1138 def create_inner(self, **propvalues):
1139 ''' Called by create, in-between the audit and react calls.
1140 '''
1141 if propvalues.has_key('id'):
1142 raise KeyError, '"id" is reserved'
1144 if self.db.journaltag is None:
1145 raise DatabaseError, 'Database open read-only'
1147 if propvalues.has_key('creator') or propvalues.has_key('actor') or \
1148 propvalues.has_key('creation') or propvalues.has_key('activity'):
1149 raise KeyError, '"creator", "actor", "creation" and '\
1150 '"activity" are reserved'
1152 # new node's id
1153 newid = self.db.newid(self.classname)
1155 # validate propvalues
1156 num_re = re.compile('^\d+$')
1157 for key, value in propvalues.items():
1158 if key == self.key:
1159 try:
1160 self.lookup(value)
1161 except KeyError:
1162 pass
1163 else:
1164 raise ValueError, 'node with key "%s" exists'%value
1166 # try to handle this property
1167 try:
1168 prop = self.properties[key]
1169 except KeyError:
1170 raise KeyError, '"%s" has no property "%s"'%(self.classname,
1171 key)
1173 if value is not None and isinstance(prop, Link):
1174 if type(value) != type(''):
1175 raise ValueError, 'link value must be String'
1176 link_class = self.properties[key].classname
1177 # if it isn't a number, it's a key
1178 if not num_re.match(value):
1179 try:
1180 value = self.db.classes[link_class].lookup(value)
1181 except (TypeError, KeyError):
1182 raise IndexError, 'new property "%s": %s not a %s'%(
1183 key, value, link_class)
1184 elif not self.db.getclass(link_class).hasnode(value):
1185 raise IndexError, '%s has no node %s'%(link_class, value)
1187 # save off the value
1188 propvalues[key] = value
1190 # register the link with the newly linked node
1191 if self.do_journal and self.properties[key].do_journal:
1192 self.db.addjournal(link_class, value, 'link',
1193 (self.classname, newid, key))
1195 elif isinstance(prop, Multilink):
1196 if type(value) != type([]):
1197 raise TypeError, 'new property "%s" not a list of ids'%key
1199 # clean up and validate the list of links
1200 link_class = self.properties[key].classname
1201 l = []
1202 for entry in value:
1203 if type(entry) != type(''):
1204 raise ValueError, '"%s" multilink value (%r) '\
1205 'must contain Strings'%(key, value)
1206 # if it isn't a number, it's a key
1207 if not num_re.match(entry):
1208 try:
1209 entry = self.db.classes[link_class].lookup(entry)
1210 except (TypeError, KeyError):
1211 raise IndexError, 'new property "%s": %s not a %s'%(
1212 key, entry, self.properties[key].classname)
1213 l.append(entry)
1214 value = l
1215 propvalues[key] = value
1217 # handle additions
1218 for nodeid in value:
1219 if not self.db.getclass(link_class).hasnode(nodeid):
1220 raise IndexError, '%s has no node %s'%(link_class,
1221 nodeid)
1222 # register the link with the newly linked node
1223 if self.do_journal and self.properties[key].do_journal:
1224 self.db.addjournal(link_class, nodeid, 'link',
1225 (self.classname, newid, key))
1227 elif isinstance(prop, String):
1228 if type(value) != type('') and type(value) != type(u''):
1229 raise TypeError, 'new property "%s" not a string'%key
1230 self.db.indexer.add_text((self.classname, newid, key), value)
1232 elif isinstance(prop, Password):
1233 if not isinstance(value, password.Password):
1234 raise TypeError, 'new property "%s" not a Password'%key
1236 elif isinstance(prop, Date):
1237 if value is not None and not isinstance(value, date.Date):
1238 raise TypeError, 'new property "%s" not a Date'%key
1240 elif isinstance(prop, Interval):
1241 if value is not None and not isinstance(value, date.Interval):
1242 raise TypeError, 'new property "%s" not an Interval'%key
1244 elif value is not None and isinstance(prop, Number):
1245 try:
1246 float(value)
1247 except ValueError:
1248 raise TypeError, 'new property "%s" not numeric'%key
1250 elif value is not None and isinstance(prop, Boolean):
1251 try:
1252 int(value)
1253 except ValueError:
1254 raise TypeError, 'new property "%s" not boolean'%key
1256 # make sure there's data where there needs to be
1257 for key, prop in self.properties.items():
1258 if propvalues.has_key(key):
1259 continue
1260 if key == self.key:
1261 raise ValueError, 'key property "%s" is required'%key
1262 if isinstance(prop, Multilink):
1263 propvalues[key] = []
1264 else:
1265 propvalues[key] = None
1267 # done
1268 self.db.addnode(self.classname, newid, propvalues)
1269 if self.do_journal:
1270 self.db.addjournal(self.classname, newid, 'create', {})
1272 # XXX numeric ids
1273 return str(newid)
1275 _marker = []
1276 def get(self, nodeid, propname, default=_marker, cache=1):
1277 '''Get the value of a property on an existing node of this class.
1279 'nodeid' must be the id of an existing node of this class or an
1280 IndexError is raised. 'propname' must be the name of a property
1281 of this class or a KeyError is raised.
1283 'cache' exists for backwards compatibility, and is not used.
1284 '''
1285 if propname == 'id':
1286 return nodeid
1288 # get the node's dict
1289 d = self.db.getnode(self.classname, nodeid)
1291 if propname == 'creation':
1292 if d.has_key('creation'):
1293 return d['creation']
1294 else:
1295 return date.Date()
1296 if propname == 'activity':
1297 if d.has_key('activity'):
1298 return d['activity']
1299 else:
1300 return date.Date()
1301 if propname == 'creator':
1302 if d.has_key('creator'):
1303 return d['creator']
1304 else:
1305 return self.db.getuid()
1306 if propname == 'actor':
1307 if d.has_key('actor'):
1308 return d['actor']
1309 else:
1310 return self.db.getuid()
1312 # get the property (raises KeyErorr if invalid)
1313 prop = self.properties[propname]
1315 if not d.has_key(propname):
1316 if default is self._marker:
1317 if isinstance(prop, Multilink):
1318 return []
1319 else:
1320 return None
1321 else:
1322 return default
1324 # don't pass our list to other code
1325 if isinstance(prop, Multilink):
1326 return d[propname][:]
1328 return d[propname]
1330 def set(self, nodeid, **propvalues):
1331 '''Modify a property on an existing node of this class.
1333 'nodeid' must be the id of an existing node of this class or an
1334 IndexError is raised.
1336 Each key in 'propvalues' must be the name of a property of this
1337 class or a KeyError is raised.
1339 All values in 'propvalues' must be acceptable types for their
1340 corresponding properties or a TypeError is raised.
1342 If the value of the key property is set, it must not collide with
1343 other key strings or a ValueError is raised.
1345 If the value of a Link or Multilink property contains an invalid
1346 node id, a ValueError is raised.
1347 '''
1348 self.fireAuditors('set', nodeid, propvalues)
1349 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1350 propvalues = self.set_inner(nodeid, **propvalues)
1351 self.fireReactors('set', nodeid, oldvalues)
1352 return propvalues
1354 def set_inner(self, nodeid, **propvalues):
1355 ''' Called by set, in-between the audit and react calls.
1356 '''
1357 if not propvalues:
1358 return propvalues
1360 if propvalues.has_key('creation') or propvalues.has_key('creator') or \
1361 propvalues.has_key('actor') or propvalues.has_key('activity'):
1362 raise KeyError, '"creation", "creator", "actor" and '\
1363 '"activity" are reserved'
1365 if propvalues.has_key('id'):
1366 raise KeyError, '"id" is reserved'
1368 if self.db.journaltag is None:
1369 raise DatabaseError, 'Database open read-only'
1371 node = self.db.getnode(self.classname, nodeid)
1372 if self.is_retired(nodeid):
1373 raise IndexError, 'Requested item is retired'
1374 num_re = re.compile('^\d+$')
1376 # if the journal value is to be different, store it in here
1377 journalvalues = {}
1379 # remember the add/remove stuff for multilinks, making it easier
1380 # for the Database layer to do its stuff
1381 multilink_changes = {}
1383 for propname, value in propvalues.items():
1384 # check to make sure we're not duplicating an existing key
1385 if propname == self.key and node[propname] != value:
1386 try:
1387 self.lookup(value)
1388 except KeyError:
1389 pass
1390 else:
1391 raise ValueError, 'node with key "%s" exists'%value
1393 # this will raise the KeyError if the property isn't valid
1394 # ... we don't use getprops() here because we only care about
1395 # the writeable properties.
1396 try:
1397 prop = self.properties[propname]
1398 except KeyError:
1399 raise KeyError, '"%s" has no property named "%s"'%(
1400 self.classname, propname)
1402 # if the value's the same as the existing value, no sense in
1403 # doing anything
1404 current = node.get(propname, None)
1405 if value == current:
1406 del propvalues[propname]
1407 continue
1408 journalvalues[propname] = current
1410 # do stuff based on the prop type
1411 if isinstance(prop, Link):
1412 link_class = prop.classname
1413 # if it isn't a number, it's a key
1414 if value is not None and not isinstance(value, type('')):
1415 raise ValueError, 'property "%s" link value be a string'%(
1416 propname)
1417 if isinstance(value, type('')) and not num_re.match(value):
1418 try:
1419 value = self.db.classes[link_class].lookup(value)
1420 except (TypeError, KeyError):
1421 raise IndexError, 'new property "%s": %s not a %s'%(
1422 propname, value, prop.classname)
1424 if (value is not None and
1425 not self.db.getclass(link_class).hasnode(value)):
1426 raise IndexError, '%s has no node %s'%(link_class, value)
1428 if self.do_journal and prop.do_journal:
1429 # register the unlink with the old linked node
1430 if node[propname] is not None:
1431 self.db.addjournal(link_class, node[propname], 'unlink',
1432 (self.classname, nodeid, propname))
1434 # register the link with the newly linked node
1435 if value is not None:
1436 self.db.addjournal(link_class, value, 'link',
1437 (self.classname, nodeid, propname))
1439 elif isinstance(prop, Multilink):
1440 if type(value) != type([]):
1441 raise TypeError, 'new property "%s" not a list of'\
1442 ' ids'%propname
1443 link_class = self.properties[propname].classname
1444 l = []
1445 for entry in value:
1446 # if it isn't a number, it's a key
1447 if type(entry) != type(''):
1448 raise ValueError, 'new property "%s" link value ' \
1449 'must be a string'%propname
1450 if not num_re.match(entry):
1451 try:
1452 entry = self.db.classes[link_class].lookup(entry)
1453 except (TypeError, KeyError):
1454 raise IndexError, 'new property "%s": %s not a %s'%(
1455 propname, entry,
1456 self.properties[propname].classname)
1457 l.append(entry)
1458 value = l
1459 propvalues[propname] = value
1461 # figure the journal entry for this property
1462 add = []
1463 remove = []
1465 # handle removals
1466 if node.has_key(propname):
1467 l = node[propname]
1468 else:
1469 l = []
1470 for id in l[:]:
1471 if id in value:
1472 continue
1473 # register the unlink with the old linked node
1474 if self.do_journal and self.properties[propname].do_journal:
1475 self.db.addjournal(link_class, id, 'unlink',
1476 (self.classname, nodeid, propname))
1477 l.remove(id)
1478 remove.append(id)
1480 # handle additions
1481 for id in value:
1482 if not self.db.getclass(link_class).hasnode(id):
1483 raise IndexError, '%s has no node %s'%(link_class, id)
1484 if id in l:
1485 continue
1486 # register the link with the newly linked node
1487 if self.do_journal and self.properties[propname].do_journal:
1488 self.db.addjournal(link_class, id, 'link',
1489 (self.classname, nodeid, propname))
1490 l.append(id)
1491 add.append(id)
1493 # figure the journal entry
1494 l = []
1495 if add:
1496 l.append(('+', add))
1497 if remove:
1498 l.append(('-', remove))
1499 multilink_changes[propname] = (add, remove)
1500 if l:
1501 journalvalues[propname] = tuple(l)
1503 elif isinstance(prop, String):
1504 if value is not None and type(value) != type('') and type(value) != type(u''):
1505 raise TypeError, 'new property "%s" not a string'%propname
1506 self.db.indexer.add_text((self.classname, nodeid, propname),
1507 value)
1509 elif isinstance(prop, Password):
1510 if not isinstance(value, password.Password):
1511 raise TypeError, 'new property "%s" not a Password'%propname
1512 propvalues[propname] = value
1514 elif value is not None and isinstance(prop, Date):
1515 if not isinstance(value, date.Date):
1516 raise TypeError, 'new property "%s" not a Date'% propname
1517 propvalues[propname] = value
1519 elif value is not None and isinstance(prop, Interval):
1520 if not isinstance(value, date.Interval):
1521 raise TypeError, 'new property "%s" not an '\
1522 'Interval'%propname
1523 propvalues[propname] = value
1525 elif value is not None and isinstance(prop, Number):
1526 try:
1527 float(value)
1528 except ValueError:
1529 raise TypeError, 'new property "%s" not numeric'%propname
1531 elif value is not None and isinstance(prop, Boolean):
1532 try:
1533 int(value)
1534 except ValueError:
1535 raise TypeError, 'new property "%s" not boolean'%propname
1537 # nothing to do?
1538 if not propvalues:
1539 return propvalues
1541 # do the set, and journal it
1542 self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
1544 if self.do_journal:
1545 self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
1547 return propvalues
1549 def retire(self, nodeid):
1550 '''Retire a node.
1552 The properties on the node remain available from the get() method,
1553 and the node's id is never reused.
1555 Retired nodes are not returned by the find(), list(), or lookup()
1556 methods, and other nodes may reuse the values of their key properties.
1557 '''
1558 if self.db.journaltag is None:
1559 raise DatabaseError, 'Database open read-only'
1561 self.fireAuditors('retire', nodeid, None)
1563 # use the arg for __retired__ to cope with any odd database type
1564 # conversion (hello, sqlite)
1565 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1566 self.db.arg, self.db.arg)
1567 if __debug__:
1568 print >>hyperdb.DEBUG, 'retire', (self, sql, nodeid)
1569 self.db.cursor.execute(sql, (1, nodeid))
1570 if self.do_journal:
1571 self.db.addjournal(self.classname, nodeid, 'retired', None)
1573 self.fireReactors('retire', nodeid, None)
1575 def restore(self, nodeid):
1576 '''Restore a retired node.
1578 Make node available for all operations like it was before retirement.
1579 '''
1580 if self.db.journaltag is None:
1581 raise DatabaseError, 'Database open read-only'
1583 node = self.db.getnode(self.classname, nodeid)
1584 # check if key property was overrided
1585 key = self.getkey()
1586 try:
1587 id = self.lookup(node[key])
1588 except KeyError:
1589 pass
1590 else:
1591 raise KeyError, "Key property (%s) of retired node clashes with \
1592 existing one (%s)" % (key, node[key])
1594 self.fireAuditors('restore', nodeid, None)
1595 # use the arg for __retired__ to cope with any odd database type
1596 # conversion (hello, sqlite)
1597 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1598 self.db.arg, self.db.arg)
1599 if __debug__:
1600 print >>hyperdb.DEBUG, 'restore', (self, sql, nodeid)
1601 self.db.cursor.execute(sql, (0, nodeid))
1602 if self.do_journal:
1603 self.db.addjournal(self.classname, nodeid, 'restored', None)
1605 self.fireReactors('restore', nodeid, None)
1607 def is_retired(self, nodeid):
1608 '''Return true if the node is rerired
1609 '''
1610 sql = 'select __retired__ from _%s where id=%s'%(self.classname,
1611 self.db.arg)
1612 if __debug__:
1613 print >>hyperdb.DEBUG, 'is_retired', (self, sql, nodeid)
1614 self.db.cursor.execute(sql, (nodeid,))
1615 return int(self.db.sql_fetchone()[0])
1617 def destroy(self, nodeid):
1618 '''Destroy a node.
1620 WARNING: this method should never be used except in extremely rare
1621 situations where there could never be links to the node being
1622 deleted
1624 WARNING: use retire() instead
1626 WARNING: the properties of this node will not be available ever again
1628 WARNING: really, use retire() instead
1630 Well, I think that's enough warnings. This method exists mostly to
1631 support the session storage of the cgi interface.
1633 The node is completely removed from the hyperdb, including all journal
1634 entries. It will no longer be available, and will generally break code
1635 if there are any references to the node.
1636 '''
1637 if self.db.journaltag is None:
1638 raise DatabaseError, 'Database open read-only'
1639 self.db.destroynode(self.classname, nodeid)
1641 def history(self, nodeid):
1642 '''Retrieve the journal of edits on a particular node.
1644 'nodeid' must be the id of an existing node of this class or an
1645 IndexError is raised.
1647 The returned list contains tuples of the form
1649 (nodeid, date, tag, action, params)
1651 'date' is a Timestamp object specifying the time of the change and
1652 'tag' is the journaltag specified when the database was opened.
1653 '''
1654 if not self.do_journal:
1655 raise ValueError, 'Journalling is disabled for this class'
1656 return self.db.getjournal(self.classname, nodeid)
1658 # Locating nodes:
1659 def hasnode(self, nodeid):
1660 '''Determine if the given nodeid actually exists
1661 '''
1662 return self.db.hasnode(self.classname, nodeid)
1664 def setkey(self, propname):
1665 '''Select a String property of this class to be the key property.
1667 'propname' must be the name of a String property of this class or
1668 None, or a TypeError is raised. The values of the key property on
1669 all existing nodes must be unique or a ValueError is raised.
1670 '''
1671 # XXX create an index on the key prop column. We should also
1672 # record that we've created this index in the schema somewhere.
1673 prop = self.getprops()[propname]
1674 if not isinstance(prop, String):
1675 raise TypeError, 'key properties must be String'
1676 self.key = propname
1678 def getkey(self):
1679 '''Return the name of the key property for this class or None.'''
1680 return self.key
1682 def labelprop(self, default_to_id=0):
1683 '''Return the property name for a label for the given node.
1685 This method attempts to generate a consistent label for the node.
1686 It tries the following in order:
1688 1. key property
1689 2. "name" property
1690 3. "title" property
1691 4. first property from the sorted property name list
1692 '''
1693 k = self.getkey()
1694 if k:
1695 return k
1696 props = self.getprops()
1697 if props.has_key('name'):
1698 return 'name'
1699 elif props.has_key('title'):
1700 return 'title'
1701 if default_to_id:
1702 return 'id'
1703 props = props.keys()
1704 props.sort()
1705 return props[0]
1707 def lookup(self, keyvalue):
1708 '''Locate a particular node by its key property and return its id.
1710 If this class has no key property, a TypeError is raised. If the
1711 'keyvalue' matches one of the values for the key property among
1712 the nodes in this class, the matching node's id is returned;
1713 otherwise a KeyError is raised.
1714 '''
1715 if not self.key:
1716 raise TypeError, 'No key property set for class %s'%self.classname
1718 # use the arg to handle any odd database type conversion (hello,
1719 # sqlite)
1720 sql = "select id from _%s where _%s=%s and __retired__ <> %s"%(
1721 self.classname, self.key, self.db.arg, self.db.arg)
1722 self.db.sql(sql, (keyvalue, 1))
1724 # see if there was a result that's not retired
1725 row = self.db.sql_fetchone()
1726 if not row:
1727 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1728 keyvalue, self.classname)
1730 # return the id
1731 # XXX numeric ids
1732 return str(row[0])
1734 def find(self, **propspec):
1735 '''Get the ids of nodes in this class which link to the given nodes.
1737 'propspec' consists of keyword args propname=nodeid or
1738 propname={nodeid:1, }
1739 'propname' must be the name of a property in this class, or a
1740 KeyError is raised. That property must be a Link or
1741 Multilink property, or a TypeError is raised.
1743 Any node in this class whose 'propname' property links to any of the
1744 nodeids will be returned. Used by the full text indexing, which knows
1745 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1746 issues:
1748 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1749 '''
1750 if __debug__:
1751 print >>hyperdb.DEBUG, 'find', (self, propspec)
1753 # shortcut
1754 if not propspec:
1755 return []
1757 # validate the args
1758 props = self.getprops()
1759 propspec = propspec.items()
1760 for propname, nodeids in propspec:
1761 # check the prop is OK
1762 prop = props[propname]
1763 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1764 raise TypeError, "'%s' not a Link/Multilink property"%propname
1766 # first, links
1767 a = self.db.arg
1768 allvalues = (1,)
1769 o = []
1770 where = []
1771 for prop, values in propspec:
1772 if not isinstance(props[prop], hyperdb.Link):
1773 continue
1774 if type(values) is type({}) and len(values) == 1:
1775 values = values.keys()[0]
1776 if type(values) is type(''):
1777 allvalues += (values,)
1778 where.append('_%s = %s'%(prop, a))
1779 elif values is None:
1780 where.append('_%s is NULL'%prop)
1781 else:
1782 allvalues += tuple(values.keys())
1783 where.append('_%s in (%s)'%(prop, ','.join([a]*len(values))))
1784 tables = ['_%s'%self.classname]
1785 if where:
1786 o.append('(' + ' and '.join(where) + ')')
1788 # now multilinks
1789 for prop, values in propspec:
1790 if not isinstance(props[prop], hyperdb.Multilink):
1791 continue
1792 if not values:
1793 continue
1794 if type(values) is type(''):
1795 allvalues += (values,)
1796 s = a
1797 else:
1798 allvalues += tuple(values.keys())
1799 s = ','.join([a]*len(values))
1800 tn = '%s_%s'%(self.classname, prop)
1801 tables.append(tn)
1802 o.append('(id=%s.nodeid and %s.linkid in (%s))'%(tn, tn, s))
1804 if not o:
1805 return []
1806 elif len(o) > 1:
1807 o = '(' + ' or '.join(['(%s)'%i for i in o]) + ')'
1808 else:
1809 o = o[0]
1810 t = ', '.join(tables)
1811 sql = 'select distinct(id) from %s where __retired__ <> %s and %s'%(
1812 t, a, o)
1813 self.db.sql(sql, allvalues)
1814 # XXX numeric ids
1815 l = [str(x[0]) for x in self.db.sql_fetchall()]
1816 if __debug__:
1817 print >>hyperdb.DEBUG, 'find ... ', l
1818 return l
1820 def stringFind(self, **requirements):
1821 '''Locate a particular node by matching a set of its String
1822 properties in a caseless search.
1824 If the property is not a String property, a TypeError is raised.
1826 The return is a list of the id of all nodes that match.
1827 '''
1828 where = []
1829 args = []
1830 for propname in requirements.keys():
1831 prop = self.properties[propname]
1832 if not isinstance(prop, String):
1833 raise TypeError, "'%s' not a String property"%propname
1834 where.append(propname)
1835 args.append(requirements[propname].lower())
1837 # generate the where clause
1838 s = ' and '.join(['lower(_%s)=%s'%(col, self.db.arg) for col in where])
1839 sql = 'select id from _%s where %s and __retired__=%s'%(self.classname,
1840 s, self.db.arg)
1841 args.append(0)
1842 self.db.sql(sql, tuple(args))
1843 # XXX numeric ids
1844 l = [str(x[0]) for x in self.db.sql_fetchall()]
1845 if __debug__:
1846 print >>hyperdb.DEBUG, 'find ... ', l
1847 return l
1849 def list(self):
1850 ''' Return a list of the ids of the active nodes in this class.
1851 '''
1852 return self.getnodeids(retired=0)
1854 def getnodeids(self, retired=None):
1855 ''' Retrieve all the ids of the nodes for a particular Class.
1857 Set retired=None to get all nodes. Otherwise it'll get all the
1858 retired or non-retired nodes, depending on the flag.
1859 '''
1860 # flip the sense of the 'retired' flag if we don't want all of them
1861 if retired is not None:
1862 if retired:
1863 args = (0, )
1864 else:
1865 args = (1, )
1866 sql = 'select id from _%s where __retired__ <> %s'%(self.classname,
1867 self.db.arg)
1868 else:
1869 args = ()
1870 sql = 'select id from _%s'%self.classname
1871 if __debug__:
1872 print >>hyperdb.DEBUG, 'getnodeids', (self, sql, retired)
1873 self.db.cursor.execute(sql, args)
1874 # XXX numeric ids
1875 ids = [str(x[0]) for x in self.db.cursor.fetchall()]
1876 return ids
1878 def filter(self, search_matches, filterspec, sort=(None,None),
1879 group=(None,None)):
1880 '''Return a list of the ids of the active nodes in this class that
1881 match the 'filter' spec, sorted by the group spec and then the
1882 sort spec
1884 "filterspec" is {propname: value(s)}
1886 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1887 and prop is a prop name or None
1889 "search_matches" is {nodeid: marker}
1891 The filter must match all properties specificed - but if the
1892 property value to match is a list, any one of the values in the
1893 list may match for that property to match.
1894 '''
1895 # just don't bother if the full-text search matched diddly
1896 if search_matches == {}:
1897 return []
1899 cn = self.classname
1901 timezone = self.db.getUserTimezone()
1903 # figure the WHERE clause from the filterspec
1904 props = self.getprops()
1905 frum = ['_'+cn]
1906 where = []
1907 args = []
1908 a = self.db.arg
1909 for k, v in filterspec.items():
1910 propclass = props[k]
1911 # now do other where clause stuff
1912 if isinstance(propclass, Multilink):
1913 tn = '%s_%s'%(cn, k)
1914 if v in ('-1', ['-1']):
1915 # only match rows that have count(linkid)=0 in the
1916 # corresponding multilink table)
1917 where.append('id not in (select nodeid from %s)'%tn)
1918 elif isinstance(v, type([])):
1919 frum.append(tn)
1920 s = ','.join([a for x in v])
1921 where.append('id=%s.nodeid and %s.linkid in (%s)'%(tn,tn,s))
1922 args = args + v
1923 else:
1924 frum.append(tn)
1925 where.append('id=%s.nodeid and %s.linkid=%s'%(tn, tn, a))
1926 args.append(v)
1927 elif k == 'id':
1928 if isinstance(v, type([])):
1929 s = ','.join([a for x in v])
1930 where.append('%s in (%s)'%(k, s))
1931 args = args + v
1932 else:
1933 where.append('%s=%s'%(k, a))
1934 args.append(v)
1935 elif isinstance(propclass, String):
1936 if not isinstance(v, type([])):
1937 v = [v]
1939 # Quote the bits in the string that need it and then embed
1940 # in a "substring" search. Note - need to quote the '%' so
1941 # they make it through the python layer happily
1942 v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
1944 # now add to the where clause
1945 where.append(' or '.join(["_%s LIKE '%s'"%(k, s) for s in v]))
1946 # note: args are embedded in the query string now
1947 elif isinstance(propclass, Link):
1948 if isinstance(v, type([])):
1949 if '-1' in v:
1950 v = v[:]
1951 v.remove('-1')
1952 xtra = ' or _%s is NULL'%k
1953 else:
1954 xtra = ''
1955 if v:
1956 s = ','.join([a for x in v])
1957 where.append('(_%s in (%s)%s)'%(k, s, xtra))
1958 args = args + v
1959 else:
1960 where.append('_%s is NULL'%k)
1961 else:
1962 if v == '-1':
1963 v = None
1964 where.append('_%s is NULL'%k)
1965 else:
1966 where.append('_%s=%s'%(k, a))
1967 args.append(v)
1968 elif isinstance(propclass, Date):
1969 dc = self.db.hyperdb_to_sql_value[hyperdb.Date]
1970 if isinstance(v, type([])):
1971 s = ','.join([a for x in v])
1972 where.append('_%s in (%s)'%(k, s))
1973 args = args + [dc(date.Date(v)) for x in v]
1974 else:
1975 try:
1976 # Try to filter on range of dates
1977 date_rng = Range(v, date.Date, offset=timezone)
1978 if date_rng.from_value:
1979 where.append('_%s >= %s'%(k, a))
1980 args.append(dc(date_rng.from_value))
1981 if date_rng.to_value:
1982 where.append('_%s <= %s'%(k, a))
1983 args.append(dc(date_rng.to_value))
1984 except ValueError:
1985 # If range creation fails - ignore that search parameter
1986 pass
1987 elif isinstance(propclass, Interval):
1988 if isinstance(v, type([])):
1989 s = ','.join([a for x in v])
1990 where.append('_%s in (%s)'%(k, s))
1991 args = args + [date.Interval(x).serialise() for x in v]
1992 else:
1993 try:
1994 # Try to filter on range of intervals
1995 date_rng = Range(v, date.Interval)
1996 if date_rng.from_value:
1997 where.append('_%s >= %s'%(k, a))
1998 args.append(date_rng.from_value.serialise())
1999 if date_rng.to_value:
2000 where.append('_%s <= %s'%(k, a))
2001 args.append(date_rng.to_value.serialise())
2002 except ValueError:
2003 # If range creation fails - ignore that search parameter
2004 pass
2005 #where.append('_%s=%s'%(k, a))
2006 #args.append(date.Interval(v).serialise())
2007 else:
2008 if isinstance(v, type([])):
2009 s = ','.join([a for x in v])
2010 where.append('_%s in (%s)'%(k, s))
2011 args = args + v
2012 else:
2013 where.append('_%s=%s'%(k, a))
2014 args.append(v)
2016 # don't match retired nodes
2017 where.append('__retired__ <> 1')
2019 # add results of full text search
2020 if search_matches is not None:
2021 v = search_matches.keys()
2022 s = ','.join([a for x in v])
2023 where.append('id in (%s)'%s)
2024 args = args + v
2026 # "grouping" is just the first-order sorting in the SQL fetch
2027 orderby = []
2028 ordercols = []
2029 mlsort = []
2030 for sortby in group, sort:
2031 sdir, prop = sortby
2032 if sdir and prop:
2033 if isinstance(props[prop], Multilink):
2034 mlsort.append(sortby)
2035 continue
2036 elif prop == 'id':
2037 o = 'id'
2038 else:
2039 o = '_'+prop
2040 ordercols.append(o)
2041 if sdir == '-':
2042 o += ' desc'
2043 orderby.append(o)
2045 # construct the SQL
2046 frum = ','.join(frum)
2047 if where:
2048 where = ' where ' + (' and '.join(where))
2049 else:
2050 where = ''
2051 cols = ['distinct(id)']
2052 if orderby:
2053 cols = cols + ordercols
2054 order = ' order by %s'%(','.join(orderby))
2055 else:
2056 order = ''
2057 cols = ','.join(cols)
2058 sql = 'select %s from %s %s%s'%(cols, frum, where, order)
2059 args = tuple(args)
2060 if __debug__:
2061 print >>hyperdb.DEBUG, 'filter', (self, sql, args)
2062 if args:
2063 self.db.cursor.execute(sql, args)
2064 else:
2065 # psycopg doesn't like empty args
2066 self.db.cursor.execute(sql)
2067 l = self.db.sql_fetchall()
2069 # return the IDs (the first column)
2070 # XXX numeric ids
2071 l = [str(row[0]) for row in l]
2073 if not mlsort:
2074 return l
2076 # ergh. someone wants to sort by a multilink.
2077 r = []
2078 for id in l:
2079 m = []
2080 for ml in mlsort:
2081 m.append(self.get(id, ml[1]))
2082 r.append((id, m))
2083 i = 0
2084 for sortby in mlsort:
2085 def sortfun(a, b, dir=sortby[i]):
2086 if dir == '-':
2087 return cmp(b[1][i], a[1][i])
2088 else:
2089 return cmp(a[1][i], b[1][i])
2090 r.sort(sortfun)
2091 i += 1
2092 return [i[0] for i in r]
2094 def count(self):
2095 '''Get the number of nodes in this class.
2097 If the returned integer is 'numnodes', the ids of all the nodes
2098 in this class run from 1 to numnodes, and numnodes+1 will be the
2099 id of the next node to be created in this class.
2100 '''
2101 return self.db.countnodes(self.classname)
2103 # Manipulating properties:
2104 def getprops(self, protected=1):
2105 '''Return a dictionary mapping property names to property objects.
2106 If the "protected" flag is true, we include protected properties -
2107 those which may not be modified.
2108 '''
2109 d = self.properties.copy()
2110 if protected:
2111 d['id'] = String()
2112 d['creation'] = hyperdb.Date()
2113 d['activity'] = hyperdb.Date()
2114 d['creator'] = hyperdb.Link('user')
2115 d['actor'] = hyperdb.Link('user')
2116 return d
2118 def addprop(self, **properties):
2119 '''Add properties to this class.
2121 The keyword arguments in 'properties' must map names to property
2122 objects, or a TypeError is raised. None of the keys in 'properties'
2123 may collide with the names of existing properties, or a ValueError
2124 is raised before any properties have been added.
2125 '''
2126 for key in properties.keys():
2127 if self.properties.has_key(key):
2128 raise ValueError, key
2129 self.properties.update(properties)
2131 def index(self, nodeid):
2132 '''Add (or refresh) the node to search indexes
2133 '''
2134 # find all the String properties that have indexme
2135 for prop, propclass in self.getprops().items():
2136 if isinstance(propclass, String) and propclass.indexme:
2137 self.db.indexer.add_text((self.classname, nodeid, prop),
2138 str(self.get(nodeid, prop)))
2141 #
2142 # Detector interface
2143 #
2144 def audit(self, event, detector):
2145 '''Register a detector
2146 '''
2147 l = self.auditors[event]
2148 if detector not in l:
2149 self.auditors[event].append(detector)
2151 def fireAuditors(self, action, nodeid, newvalues):
2152 '''Fire all registered auditors.
2153 '''
2154 for audit in self.auditors[action]:
2155 audit(self.db, self, nodeid, newvalues)
2157 def react(self, event, detector):
2158 '''Register a detector
2159 '''
2160 l = self.reactors[event]
2161 if detector not in l:
2162 self.reactors[event].append(detector)
2164 def fireReactors(self, action, nodeid, oldvalues):
2165 '''Fire all registered reactors.
2166 '''
2167 for react in self.reactors[action]:
2168 react(self.db, self, nodeid, oldvalues)
2170 #
2171 # import / export support
2172 #
2173 def export_list(self, propnames, nodeid):
2174 ''' Export a node - generate a list of CSV-able data in the order
2175 specified by propnames for the given node.
2176 '''
2177 properties = self.getprops()
2178 l = []
2179 for prop in propnames:
2180 proptype = properties[prop]
2181 value = self.get(nodeid, prop)
2182 # "marshal" data where needed
2183 if value is None:
2184 pass
2185 elif isinstance(proptype, hyperdb.Date):
2186 value = value.get_tuple()
2187 elif isinstance(proptype, hyperdb.Interval):
2188 value = value.get_tuple()
2189 elif isinstance(proptype, hyperdb.Password):
2190 value = str(value)
2191 l.append(repr(value))
2192 l.append(repr(self.is_retired(nodeid)))
2193 return l
2195 def import_list(self, propnames, proplist):
2196 ''' Import a node - all information including "id" is present and
2197 should not be sanity checked. Triggers are not triggered. The
2198 journal should be initialised using the "creator" and "created"
2199 information.
2201 Return the nodeid of the node imported.
2202 '''
2203 if self.db.journaltag is None:
2204 raise DatabaseError, 'Database open read-only'
2205 properties = self.getprops()
2207 # make the new node's property map
2208 d = {}
2209 retire = 0
2210 newid = None
2211 for i in range(len(propnames)):
2212 # Use eval to reverse the repr() used to output the CSV
2213 value = eval(proplist[i])
2215 # Figure the property for this column
2216 propname = propnames[i]
2218 # "unmarshal" where necessary
2219 if propname == 'id':
2220 newid = value
2221 continue
2222 elif propname == 'is retired':
2223 # is the item retired?
2224 if int(value):
2225 retire = 1
2226 continue
2227 elif value is None:
2228 d[propname] = None
2229 continue
2231 prop = properties[propname]
2232 if value is None:
2233 # don't set Nones
2234 continue
2235 elif isinstance(prop, hyperdb.Date):
2236 value = date.Date(value)
2237 elif isinstance(prop, hyperdb.Interval):
2238 value = date.Interval(value)
2239 elif isinstance(prop, hyperdb.Password):
2240 pwd = password.Password()
2241 pwd.unpack(value)
2242 value = pwd
2243 d[propname] = value
2245 # get a new id if necessary
2246 if newid is None or not self.hasnode(newid):
2247 newid = self.db.newid(self.classname)
2248 self.db.addnode(self.classname, newid, d)
2249 else:
2250 # update
2251 self.db.setnode(self.classname, newid, d)
2253 # retire?
2254 if retire:
2255 # use the arg for __retired__ to cope with any odd database type
2256 # conversion (hello, sqlite)
2257 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
2258 self.db.arg, self.db.arg)
2259 if __debug__:
2260 print >>hyperdb.DEBUG, 'retire', (self, sql, newid)
2261 self.db.cursor.execute(sql, (1, newid))
2262 return newid
2264 def export_journals(self):
2265 '''Export a class's journal - generate a list of lists of
2266 CSV-able data:
2268 nodeid, date, user, action, params
2270 No heading here - the columns are fixed.
2271 '''
2272 properties = self.getprops()
2273 r = []
2274 for nodeid in self.getnodeids():
2275 for nodeid, date, user, action, params in self.history(nodeid):
2276 date = date.get_tuple()
2277 if action == 'set':
2278 for propname, value in params.items():
2279 prop = properties[propname]
2280 # make sure the params are eval()'able
2281 if value is None:
2282 pass
2283 elif isinstance(prop, Date):
2284 value = value.get_tuple()
2285 elif isinstance(prop, Interval):
2286 value = value.get_tuple()
2287 elif isinstance(prop, Password):
2288 value = str(value)
2289 params[propname] = value
2290 l = [nodeid, date, user, action, params]
2291 r.append(map(repr, l))
2292 return r
2294 def import_journals(self, entries):
2295 '''Import a class's journal.
2297 Uses setjournal() to set the journal for each item.'''
2298 properties = self.getprops()
2299 d = {}
2300 for l in entries:
2301 l = map(eval, l)
2302 nodeid, jdate, user, action, params = l
2303 r = d.setdefault(nodeid, [])
2304 if action == 'set':
2305 for propname, value in params.items():
2306 prop = properties[propname]
2307 if value is None:
2308 pass
2309 elif isinstance(prop, Date):
2310 value = date.Date(value)
2311 elif isinstance(prop, Interval):
2312 value = date.Interval(value)
2313 elif isinstance(prop, Password):
2314 pwd = password.Password()
2315 pwd.unpack(value)
2316 value = pwd
2317 params[propname] = value
2318 r.append((nodeid, date.Date(jdate), user, action, params))
2320 for nodeid, l in d.items():
2321 self.db.setjournal(self.classname, nodeid, l)
2323 class FileClass(Class, hyperdb.FileClass):
2324 '''This class defines a large chunk of data. To support this, it has a
2325 mandatory String property "content" which is typically saved off
2326 externally to the hyperdb.
2328 The default MIME type of this data is defined by the
2329 "default_mime_type" class attribute, which may be overridden by each
2330 node if the class defines a "type" String property.
2331 '''
2332 default_mime_type = 'text/plain'
2334 def create(self, **propvalues):
2335 ''' snaffle the file propvalue and store in a file
2336 '''
2337 # we need to fire the auditors now, or the content property won't
2338 # be in propvalues for the auditors to play with
2339 self.fireAuditors('create', None, propvalues)
2341 # now remove the content property so it's not stored in the db
2342 content = propvalues['content']
2343 del propvalues['content']
2345 # do the database create
2346 newid = self.create_inner(**propvalues)
2348 # figure the mime type
2349 mime_type = propvalues.get('type', self.default_mime_type)
2351 # and index!
2352 self.db.indexer.add_text((self.classname, newid, 'content'), content,
2353 mime_type)
2355 # fire reactors
2356 self.fireReactors('create', newid, None)
2358 # store off the content as a file
2359 self.db.storefile(self.classname, newid, None, content)
2360 return newid
2362 def import_list(self, propnames, proplist):
2363 ''' Trap the "content" property...
2364 '''
2365 # dupe this list so we don't affect others
2366 propnames = propnames[:]
2368 # extract the "content" property from the proplist
2369 i = propnames.index('content')
2370 content = eval(proplist[i])
2371 del propnames[i]
2372 del proplist[i]
2374 # do the normal import
2375 newid = Class.import_list(self, propnames, proplist)
2377 # save off the "content" file
2378 self.db.storefile(self.classname, newid, None, content)
2379 return newid
2381 _marker = []
2382 def get(self, nodeid, propname, default=_marker, cache=1):
2383 ''' Trap the content propname and get it from the file
2385 'cache' exists for backwards compatibility, and is not used.
2386 '''
2387 poss_msg = 'Possibly a access right configuration problem.'
2388 if propname == 'content':
2389 try:
2390 return self.db.getfile(self.classname, nodeid, None)
2391 except IOError, (strerror):
2392 # BUG: by catching this we donot see an error in the log.
2393 return 'ERROR reading file: %s%s\n%s\n%s'%(
2394 self.classname, nodeid, poss_msg, strerror)
2395 if default is not self._marker:
2396 return Class.get(self, nodeid, propname, default)
2397 else:
2398 return Class.get(self, nodeid, propname)
2400 def getprops(self, protected=1):
2401 ''' In addition to the actual properties on the node, these methods
2402 provide the "content" property. If the "protected" flag is true,
2403 we include protected properties - those which may not be
2404 modified.
2405 '''
2406 d = Class.getprops(self, protected=protected).copy()
2407 d['content'] = hyperdb.String()
2408 return d
2410 def set(self, itemid, **propvalues):
2411 ''' Snarf the "content" propvalue and update it in a file
2412 '''
2413 self.fireAuditors('set', itemid, propvalues)
2414 oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid))
2416 # now remove the content property so it's not stored in the db
2417 content = None
2418 if propvalues.has_key('content'):
2419 content = propvalues['content']
2420 del propvalues['content']
2422 # do the database create
2423 propvalues = self.set_inner(itemid, **propvalues)
2425 # do content?
2426 if content:
2427 # store and index
2428 self.db.storefile(self.classname, itemid, None, content)
2429 mime_type = propvalues.get('type', self.get(itemid, 'type'))
2430 if not mime_type:
2431 mime_type = self.default_mime_type
2432 self.db.indexer.add_text((self.classname, itemid, 'content'),
2433 content, mime_type)
2435 # fire reactors
2436 self.fireReactors('set', itemid, oldvalues)
2437 return propvalues
2439 # XXX deviation from spec - was called ItemClass
2440 class IssueClass(Class, roundupdb.IssueClass):
2441 # Overridden methods:
2442 def __init__(self, db, classname, **properties):
2443 '''The newly-created class automatically includes the "messages",
2444 "files", "nosy", and "superseder" properties. If the 'properties'
2445 dictionary attempts to specify any of these properties or a
2446 "creation", "creator", "activity" or "actor" property, a ValueError
2447 is raised.
2448 '''
2449 if not properties.has_key('title'):
2450 properties['title'] = hyperdb.String(indexme='yes')
2451 if not properties.has_key('messages'):
2452 properties['messages'] = hyperdb.Multilink("msg")
2453 if not properties.has_key('files'):
2454 properties['files'] = hyperdb.Multilink("file")
2455 if not properties.has_key('nosy'):
2456 # note: journalling is turned off as it really just wastes
2457 # space. this behaviour may be overridden in an instance
2458 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2459 if not properties.has_key('superseder'):
2460 properties['superseder'] = hyperdb.Multilink(classname)
2461 Class.__init__(self, db, classname, **properties)