e468a0f2b4e56e3faae4c4d15465b60eccb0136e
1 # $Id: rdbms_common.py,v 1.90 2004-04-08 00:40:20 richard Exp $
2 ''' Relational database (SQL) backend common code.
4 Basics:
6 - map roundup classes to relational tables
7 - automatically detect schema changes and modify the table schemas
8 appropriately (we store the "database version" of the schema in the
9 database itself as the only row of the "schema" table)
10 - multilinks (which represent a many-to-many relationship) are handled through
11 intermediate tables
12 - journals are stored adjunct to the per-class tables
13 - table names and columns have "_" prepended so the names can't clash with
14 restricted names (like "order")
15 - retirement is determined by the __retired__ column being true
17 Database-specific changes may generally be pushed out to the overridable
18 sql_* methods, since everything else should be fairly generic. There's
19 probably a bit of work to be done if a database is used that actually
20 honors column typing, since the initial databases don't (sqlite stores
21 everything as a string.)
23 The schema of the hyperdb being mapped to the database is stored in the
24 database itself as a repr()'ed dictionary of information about each Class
25 that maps to a table. If that information differs from the hyperdb schema,
26 then we update it. We also store in the schema dict a version which
27 allows us to upgrade the database schema when necessary. See upgrade_db().
28 '''
29 __docformat__ = 'restructuredtext'
31 # standard python modules
32 import sys, os, time, re, errno, weakref, copy
34 # roundup modules
35 from roundup import hyperdb, date, password, roundupdb, security
36 from roundup.hyperdb import String, Password, Date, Interval, Link, \
37 Multilink, DatabaseError, Boolean, Number, Node
38 from roundup.backends import locking
40 # support
41 from blobfiles import FileStorage
42 from indexer_rdbms import Indexer
43 from sessions_rdbms import Sessions, OneTimeKeys
44 from roundup.date import Range
46 # number of rows to keep in memory
47 ROW_CACHE_SIZE = 100
49 def _num_cvt(num):
50 num = str(num)
51 try:
52 return int(num)
53 except:
54 return float(num)
56 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
57 ''' Wrapper around an SQL database that presents a hyperdb interface.
59 - some functionality is specific to the actual SQL database, hence
60 the sql_* methods that are NotImplemented
61 - we keep a cache of the latest ROW_CACHE_SIZE row fetches.
62 '''
63 def __init__(self, config, journaltag=None):
64 ''' Open the database and load the schema from it.
65 '''
66 self.config, self.journaltag = config, journaltag
67 self.dir = config.DATABASE
68 self.classes = {}
69 self.indexer = Indexer(self)
70 self.security = security.Security(self)
72 # additional transaction support for external files and the like
73 self.transactions = []
75 # keep a cache of the N most recently retrieved rows of any kind
76 # (classname, nodeid) = row
77 self.cache = {}
78 self.cache_lru = []
80 # database lock
81 self.lockfile = None
83 # open a connection to the database, creating the "conn" attribute
84 self.open_connection()
86 def clearCache(self):
87 self.cache = {}
88 self.cache_lru = []
90 def getSessionManager(self):
91 return Sessions(self)
93 def getOTKManager(self):
94 return OneTimeKeys(self)
96 def open_connection(self):
97 ''' Open a connection to the database, creating it if necessary.
99 Must call self.load_dbschema()
100 '''
101 raise NotImplemented
103 def sql(self, sql, args=None):
104 ''' Execute the sql with the optional args.
105 '''
106 if __debug__:
107 print >>hyperdb.DEBUG, (self, sql, args)
108 if args:
109 self.cursor.execute(sql, args)
110 else:
111 self.cursor.execute(sql)
113 def sql_fetchone(self):
114 ''' Fetch a single row. If there's nothing to fetch, return None.
115 '''
116 return self.cursor.fetchone()
118 def sql_fetchall(self):
119 ''' Fetch all rows. If there's nothing to fetch, return [].
120 '''
121 return self.cursor.fetchall()
123 def sql_stringquote(self, value):
124 ''' Quote the string so it's safe to put in the 'sql quotes'
125 '''
126 return re.sub("'", "''", str(value))
128 def init_dbschema(self):
129 self.database_schema = {
130 'version': self.current_db_version,
131 'tables': {}
132 }
134 def load_dbschema(self):
135 ''' Load the schema definition that the database currently implements
136 '''
137 self.cursor.execute('select schema from schema')
138 schema = self.cursor.fetchone()
139 if schema:
140 self.database_schema = eval(schema[0])
141 else:
142 self.database_schema = {}
144 def save_dbschema(self, schema):
145 ''' Save the schema definition that the database currently implements
146 '''
147 s = repr(self.database_schema)
148 self.sql('insert into schema values (%s)', (s,))
150 def post_init(self):
151 ''' Called once the schema initialisation has finished.
153 We should now confirm that the schema defined by our "classes"
154 attribute actually matches the schema in the database.
155 '''
156 save = self.upgrade_db()
158 # now detect changes in the schema
159 tables = self.database_schema['tables']
160 for classname, spec in self.classes.items():
161 if tables.has_key(classname):
162 dbspec = tables[classname]
163 if self.update_class(spec, dbspec):
164 tables[classname] = spec.schema()
165 save = 1
166 else:
167 self.create_class(spec)
168 tables[classname] = spec.schema()
169 save = 1
171 for classname, spec in tables.items():
172 if not self.classes.has_key(classname):
173 self.drop_class(classname, tables[classname])
174 del tables[classname]
175 save = 1
177 # update the database version of the schema
178 if save:
179 self.sql('delete from schema')
180 self.save_dbschema(self.database_schema)
182 # reindex the db if necessary
183 if self.indexer.should_reindex():
184 self.reindex()
186 # commit
187 self.sql_commit()
189 # update this number when we need to make changes to the SQL structure
190 # of the backen database
191 current_db_version = 2
192 def upgrade_db(self):
193 ''' Update the SQL database to reflect changes in the backend code.
195 Return boolean whether we need to save the schema.
196 '''
197 version = self.database_schema.get('version', 1)
198 if version == self.current_db_version:
199 # nothing to do
200 return 0
202 if version == 1:
203 # change the schema structure
204 self.database_schema = {'tables': self.database_schema}
206 # version 1 didn't have the actor column (note that in
207 # MySQL this will also transition the tables to typed columns)
208 self.add_actor_column()
210 # version 1 doesn't have the OTK, session and indexing in the
211 # database
212 self.create_version_2_tables()
214 self.database_schema['version'] = self.current_db_version
215 return 1
218 def refresh_database(self):
219 self.post_init()
221 def reindex(self):
222 for klass in self.classes.values():
223 for nodeid in klass.list():
224 klass.index(nodeid)
225 self.indexer.save_index()
228 hyperdb_to_sql_datatypes = {
229 hyperdb.String : 'VARCHAR(255)',
230 hyperdb.Date : 'TIMESTAMP',
231 hyperdb.Link : 'INTEGER',
232 hyperdb.Interval : 'VARCHAR(255)',
233 hyperdb.Password : 'VARCHAR(255)',
234 hyperdb.Boolean : 'BOOLEAN',
235 hyperdb.Number : 'REAL',
236 }
237 def determine_columns(self, properties):
238 ''' Figure the column names and multilink properties from the spec
240 "properties" is a list of (name, prop) where prop may be an
241 instance of a hyperdb "type" _or_ a string repr of that type.
242 '''
243 cols = [
244 ('_actor', self.hyperdb_to_sql_datatypes[hyperdb.Link]),
245 ('_activity', self.hyperdb_to_sql_datatypes[hyperdb.Date]),
246 ('_creator', self.hyperdb_to_sql_datatypes[hyperdb.Link]),
247 ('_creation', self.hyperdb_to_sql_datatypes[hyperdb.Date]),
248 ]
249 mls = []
250 # add the multilinks separately
251 for col, prop in properties:
252 if isinstance(prop, Multilink):
253 mls.append(col)
254 continue
256 if isinstance(prop, type('')):
257 raise ValueError, "string property spec!"
258 #and prop.find('Multilink') != -1:
259 #mls.append(col)
261 datatype = self.hyperdb_to_sql_datatypes[prop.__class__]
262 cols.append(('_'+col, datatype))
264 cols.sort()
265 return cols, mls
267 def update_class(self, spec, old_spec, force=0):
268 ''' Determine the differences between the current spec and the
269 database version of the spec, and update where necessary.
271 If 'force' is true, update the database anyway.
272 '''
273 new_has = spec.properties.has_key
274 new_spec = spec.schema()
275 new_spec[1].sort()
276 old_spec[1].sort()
277 if not force and new_spec == old_spec:
278 # no changes
279 return 0
281 if __debug__:
282 print >>hyperdb.DEBUG, 'update_class FIRING'
284 # detect key prop change for potential index change
285 keyprop_changes = {}
286 if new_spec[0] != old_spec[0]:
287 keyprop_changes = {'remove': old_spec[0], 'add': new_spec[0]}
289 # detect multilinks that have been removed, and drop their table
290 old_has = {}
291 for name, prop in old_spec[1]:
292 old_has[name] = 1
293 if new_has(name):
294 continue
296 if prop.find('Multilink to') != -1:
297 # first drop indexes.
298 self.drop_multilink_table_indexes(spec.classname, name)
300 # now the multilink table itself
301 sql = 'drop table %s_%s'%(spec.classname, name)
302 else:
303 # if this is the key prop, drop the index first
304 if old_spec[0] == prop:
305 self.drop_class_table_key_index(spec.classname, name)
306 del keyprop_changes['remove']
308 # drop the column
309 sql = 'alter table _%s drop column _%s'%(spec.classname, name)
311 if __debug__:
312 print >>hyperdb.DEBUG, 'update_class', (self, sql)
313 self.cursor.execute(sql)
314 old_has = old_has.has_key
316 # if we didn't remove the key prop just then, but the key prop has
317 # changed, we still need to remove the old index
318 if keyprop_changes.has_key('remove'):
319 self.drop_class_table_key_index(spec.classname,
320 keyprop_changes['remove'])
322 # add new columns
323 for propname, prop in new_spec[1]:
324 if old_has(propname):
325 continue
326 prop = spec.properties[propname]
327 if isinstance(prop, Multilink):
328 self.create_multilink_table(spec, propname)
329 else:
330 sql = 'alter table _%s add column _%s varchar(255)'%(
331 spec.classname, propname)
332 if __debug__:
333 print >>hyperdb.DEBUG, 'update_class', (self, sql)
334 self.cursor.execute(sql)
336 # if the new column is a key prop, we need an index!
337 if new_spec[0] == propname:
338 self.create_class_table_key_index(spec.classname, propname)
339 del keyprop_changes['add']
341 # if we didn't add the key prop just then, but the key prop has
342 # changed, we still need to add the new index
343 if keyprop_changes.has_key('add'):
344 self.create_class_table_key_index(spec.classname,
345 keyprop_changes['add'])
347 return 1
349 def create_class_table(self, spec):
350 '''Create the class table for the given Class "spec". Creates the
351 indexes too.'''
352 cols, mls = self.determine_columns(spec.properties.items())
354 # add on our special columns
355 cols.append(('id', 'INTEGER PRIMARY KEY'))
356 cols.append(('__retired__', 'INTEGER DEFAULT 0'))
358 # create the base table
359 scols = ','.join(['%s %s'%x for x in cols])
360 sql = 'create table _%s (%s)'%(spec.classname, scols)
361 if __debug__:
362 print >>hyperdb.DEBUG, 'create_class', (self, sql)
363 self.cursor.execute(sql)
365 self.create_class_table_indexes(spec)
367 return cols, mls
369 def create_class_table_indexes(self, spec):
370 ''' create the class table for the given spec
371 '''
372 # create __retired__ index
373 index_sql2 = 'create index _%s_retired_idx on _%s(__retired__)'%(
374 spec.classname, spec.classname)
375 if __debug__:
376 print >>hyperdb.DEBUG, 'create_index', (self, index_sql2)
377 self.cursor.execute(index_sql2)
379 # create index for key property
380 if spec.key:
381 if __debug__:
382 print >>hyperdb.DEBUG, 'update_class setting keyprop %r'% \
383 spec.key
384 index_sql3 = 'create index _%s_%s_idx on _%s(_%s)'%(
385 spec.classname, spec.key,
386 spec.classname, spec.key)
387 if __debug__:
388 print >>hyperdb.DEBUG, 'create_index', (self, index_sql3)
389 self.cursor.execute(index_sql3)
391 def drop_class_table_indexes(self, cn, key):
392 # drop the old table indexes first
393 l = ['_%s_id_idx'%cn, '_%s_retired_idx'%cn]
394 if key:
395 l.append('_%s_%s_idx'%(cn, key))
397 table_name = '_%s'%cn
398 for index_name in l:
399 if not self.sql_index_exists(table_name, index_name):
400 continue
401 index_sql = 'drop index '+index_name
402 if __debug__:
403 print >>hyperdb.DEBUG, 'drop_index', (self, index_sql)
404 self.cursor.execute(index_sql)
406 def create_class_table_key_index(self, cn, key):
407 ''' create the class table for the given spec
408 '''
409 sql = 'create index _%s_%s_idx on _%s(_%s)'%(cn, key, cn, key)
410 if __debug__:
411 print >>hyperdb.DEBUG, 'create_class_tab_key_index', (self, sql)
412 self.cursor.execute(sql)
414 def drop_class_table_key_index(self, cn, key):
415 table_name = '_%s'%cn
416 index_name = '_%s_%s_idx'%(cn, key)
417 if not self.sql_index_exists(table_name, index_name):
418 return
419 sql = 'drop index '+index_name
420 if __debug__:
421 print >>hyperdb.DEBUG, 'drop_class_tab_key_index', (self, sql)
422 self.cursor.execute(sql)
424 def create_journal_table(self, spec):
425 ''' create the journal table for a class given the spec and
426 already-determined cols
427 '''
428 # journal table
429 cols = ','.join(['%s varchar'%x
430 for x in 'nodeid date tag action params'.split()])
431 sql = '''create table %s__journal (
432 nodeid integer, date timestamp, tag varchar(255),
433 action varchar(255), params varchar(25))'''%spec.classname
434 if __debug__:
435 print >>hyperdb.DEBUG, 'create_journal_table', (self, sql)
436 self.cursor.execute(sql)
437 self.create_journal_table_indexes(spec)
439 def create_journal_table_indexes(self, spec):
440 # index on nodeid
441 sql = 'create index %s_journ_idx on %s__journal(nodeid)'%(
442 spec.classname, spec.classname)
443 if __debug__:
444 print >>hyperdb.DEBUG, 'create_index', (self, sql)
445 self.cursor.execute(sql)
447 def drop_journal_table_indexes(self, classname):
448 index_name = '%s_journ_idx'%classname
449 if not self.sql_index_exists('%s__journal'%classname, index_name):
450 return
451 index_sql = 'drop index '+index_name
452 if __debug__:
453 print >>hyperdb.DEBUG, 'drop_index', (self, index_sql)
454 self.cursor.execute(index_sql)
456 def create_multilink_table(self, spec, ml):
457 ''' Create a multilink table for the "ml" property of the class
458 given by the spec
459 '''
460 # create the table
461 sql = 'create table %s_%s (linkid varchar, nodeid varchar)'%(
462 spec.classname, ml)
463 if __debug__:
464 print >>hyperdb.DEBUG, 'create_class', (self, sql)
465 self.cursor.execute(sql)
466 self.create_multilink_table_indexes(spec, ml)
468 def create_multilink_table_indexes(self, spec, ml):
469 # create index on linkid
470 index_sql = 'create index %s_%s_l_idx on %s_%s(linkid)'%(
471 spec.classname, ml, spec.classname, ml)
472 if __debug__:
473 print >>hyperdb.DEBUG, 'create_index', (self, index_sql)
474 self.cursor.execute(index_sql)
476 # create index on nodeid
477 index_sql = 'create index %s_%s_n_idx on %s_%s(nodeid)'%(
478 spec.classname, ml, spec.classname, ml)
479 if __debug__:
480 print >>hyperdb.DEBUG, 'create_index', (self, index_sql)
481 self.cursor.execute(index_sql)
483 def drop_multilink_table_indexes(self, classname, ml):
484 l = [
485 '%s_%s_l_idx'%(classname, ml),
486 '%s_%s_n_idx'%(classname, ml)
487 ]
488 table_name = '%s_%s'%(classname, ml)
489 for index_name in l:
490 if not self.sql_index_exists(table_name, index_name):
491 continue
492 index_sql = 'drop index %s'%index_name
493 if __debug__:
494 print >>hyperdb.DEBUG, 'drop_index', (self, index_sql)
495 self.cursor.execute(index_sql)
497 def create_class(self, spec):
498 ''' Create a database table according to the given spec.
499 '''
500 cols, mls = self.create_class_table(spec)
501 self.create_journal_table(spec)
503 # now create the multilink tables
504 for ml in mls:
505 self.create_multilink_table(spec, ml)
507 def drop_class(self, cn, spec):
508 ''' Drop the given table from the database.
510 Drop the journal and multilink tables too.
511 '''
512 properties = spec[1]
513 # figure the multilinks
514 mls = []
515 for propanme, prop in properties:
516 if isinstance(prop, Multilink):
517 mls.append(propname)
519 # drop class table and indexes
520 self.drop_class_table_indexes(cn, spec[0])
522 self.drop_class_table(cn)
524 # drop journal table and indexes
525 self.drop_journal_table_indexes(cn)
526 sql = 'drop table %s__journal'%cn
527 if __debug__:
528 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
529 self.cursor.execute(sql)
531 for ml in mls:
532 # drop multilink table and indexes
533 self.drop_multilink_table_indexes(cn, ml)
534 sql = 'drop table %s_%s'%(spec.classname, ml)
535 if __debug__:
536 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
537 self.cursor.execute(sql)
539 def drop_class_table(self, cn):
540 sql = 'drop table _%s'%cn
541 if __debug__:
542 print >>hyperdb.DEBUG, 'drop_class', (self, sql)
543 self.cursor.execute(sql)
545 #
546 # Classes
547 #
548 def __getattr__(self, classname):
549 ''' A convenient way of calling self.getclass(classname).
550 '''
551 if self.classes.has_key(classname):
552 if __debug__:
553 print >>hyperdb.DEBUG, '__getattr__', (self, classname)
554 return self.classes[classname]
555 raise AttributeError, classname
557 def addclass(self, cl):
558 ''' Add a Class to the hyperdatabase.
559 '''
560 if __debug__:
561 print >>hyperdb.DEBUG, 'addclass', (self, cl)
562 cn = cl.classname
563 if self.classes.has_key(cn):
564 raise ValueError, cn
565 self.classes[cn] = cl
567 # add default Edit and View permissions
568 self.security.addPermission(name="Edit", klass=cn,
569 description="User is allowed to edit "+cn)
570 self.security.addPermission(name="View", klass=cn,
571 description="User is allowed to access "+cn)
573 def getclasses(self):
574 ''' Return a list of the names of all existing classes.
575 '''
576 if __debug__:
577 print >>hyperdb.DEBUG, 'getclasses', (self,)
578 l = self.classes.keys()
579 l.sort()
580 return l
582 def getclass(self, classname):
583 '''Get the Class object representing a particular class.
585 If 'classname' is not a valid class name, a KeyError is raised.
586 '''
587 if __debug__:
588 print >>hyperdb.DEBUG, 'getclass', (self, classname)
589 try:
590 return self.classes[classname]
591 except KeyError:
592 raise KeyError, 'There is no class called "%s"'%classname
594 def clear(self):
595 '''Delete all database contents.
597 Note: I don't commit here, which is different behaviour to the
598 "nuke from orbit" behaviour in the dbs.
599 '''
600 if __debug__:
601 print >>hyperdb.DEBUG, 'clear', (self,)
602 for cn in self.classes.keys():
603 sql = 'delete from _%s'%cn
604 if __debug__:
605 print >>hyperdb.DEBUG, 'clear', (self, sql)
606 self.cursor.execute(sql)
608 #
609 # Nodes
610 #
612 hyperdb_to_sql_value = {
613 hyperdb.String : str,
614 hyperdb.Date : lambda x: x.formal(sep=' ', sec='%.3f'),
615 hyperdb.Link : int,
616 hyperdb.Interval : lambda x: x.serialise(),
617 hyperdb.Password : str,
618 hyperdb.Boolean : lambda x: x and 'TRUE' or 'FALSE',
619 hyperdb.Number : lambda x: x,
620 }
621 def addnode(self, classname, nodeid, node):
622 ''' Add the specified node to its class's db.
623 '''
624 if __debug__:
625 print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
627 # determine the column definitions and multilink tables
628 cl = self.classes[classname]
629 cols, mls = self.determine_columns(cl.properties.items())
631 # we'll be supplied these props if we're doing an import
632 values = node.copy()
633 if not values.has_key('creator'):
634 # add in the "calculated" properties (dupe so we don't affect
635 # calling code's node assumptions)
636 values['creation'] = values['activity'] = date.Date()
637 values['actor'] = values['creator'] = self.getuid()
639 cl = self.classes[classname]
640 props = cl.getprops(protected=1)
641 del props['id']
643 # default the non-multilink columns
644 for col, prop in props.items():
645 if not values.has_key(col):
646 if isinstance(prop, Multilink):
647 values[col] = []
648 else:
649 values[col] = None
651 # clear this node out of the cache if it's in there
652 key = (classname, nodeid)
653 if self.cache.has_key(key):
654 del self.cache[key]
655 self.cache_lru.remove(key)
657 # figure the values to insert
658 vals = []
659 for col,dt in cols:
660 prop = props[col[1:]]
661 value = values[col[1:]]
662 if value:
663 value = self.hyperdb_to_sql_value[prop.__class__](value)
664 vals.append(value)
665 vals.append(nodeid)
666 vals = tuple(vals)
668 # make sure the ordering is correct for column name -> column value
669 s = ','.join([self.arg for x in cols]) + ',%s'%self.arg
670 cols = ','.join([col for col,dt in cols]) + ',id'
672 # perform the inserts
673 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
674 if __debug__:
675 print >>hyperdb.DEBUG, 'addnode', (self, sql, vals)
676 self.cursor.execute(sql, vals)
678 # insert the multilink rows
679 for col in mls:
680 t = '%s_%s'%(classname, col)
681 for entry in node[col]:
682 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
683 self.arg, self.arg)
684 self.sql(sql, (entry, nodeid))
686 def setnode(self, classname, nodeid, values, multilink_changes={}):
687 ''' Change the specified node.
688 '''
689 if __debug__:
690 print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, values)
692 # clear this node out of the cache if it's in there
693 key = (classname, nodeid)
694 if self.cache.has_key(key):
695 del self.cache[key]
696 self.cache_lru.remove(key)
698 # add the special props
699 values = values.copy()
700 values['activity'] = date.Date()
701 values['actor'] = self.getuid()
703 cl = self.classes[classname]
704 props = cl.getprops()
706 cols = []
707 mls = []
708 # add the multilinks separately
709 for col in values.keys():
710 prop = props[col]
711 if isinstance(prop, Multilink):
712 mls.append(col)
713 else:
714 cols.append(col)
715 cols.sort()
717 # figure the values to insert
718 vals = []
719 for col in cols:
720 prop = props[col]
721 value = values[col]
722 if value is not None:
723 value = self.hyperdb_to_sql_value[prop.__class__](value)
724 vals.append(value)
725 vals.append(int(nodeid))
726 vals = tuple(vals)
728 # if there's any updates to regular columns, do them
729 if cols:
730 # make sure the ordering is correct for column name -> column value
731 s = ','.join(['_%s=%s'%(x, self.arg) for x in cols])
732 cols = ','.join(cols)
734 # perform the update
735 sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
736 if __debug__:
737 print >>hyperdb.DEBUG, 'setnode', (self, sql, vals)
738 self.cursor.execute(sql, vals)
740 # we're probably coming from an import, not a change
741 if not multilink_changes:
742 for name in mls:
743 prop = props[name]
744 value = values[name]
746 t = '%s_%s'%(classname, name)
748 # clear out previous values for this node
749 # XXX numeric ids
750 self.sql('delete from %s where nodeid=%s'%(t, self.arg),
751 (nodeid,))
753 # insert the values for this node
754 for entry in values[name]:
755 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
756 self.arg, self.arg)
757 # XXX numeric ids
758 self.sql(sql, (entry, nodeid))
760 # we have multilink changes to apply
761 for col, (add, remove) in multilink_changes.items():
762 tn = '%s_%s'%(classname, col)
763 if add:
764 sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
765 self.arg, self.arg)
766 for addid in add:
767 # XXX numeric ids
768 self.sql(sql, (int(nodeid), int(addid)))
769 if remove:
770 sql = 'delete from %s where nodeid=%s and linkid=%s'%(tn,
771 self.arg, self.arg)
772 for removeid in remove:
773 # XXX numeric ids
774 self.sql(sql, (int(nodeid), int(removeid)))
776 sql_to_hyperdb_value = {
777 hyperdb.String : str,
778 hyperdb.Date : lambda x:date.Date(str(x).replace(' ', '.')),
779 # hyperdb.Link : int, # XXX numeric ids
780 hyperdb.Link : str,
781 hyperdb.Interval : date.Interval,
782 hyperdb.Password : lambda x: password.Password(encrypted=x),
783 hyperdb.Boolean : int,
784 hyperdb.Number : _num_cvt,
785 }
786 def getnode(self, classname, nodeid):
787 ''' Get a node from the database.
788 '''
789 if __debug__:
790 print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid)
792 # see if we have this node cached
793 key = (classname, nodeid)
794 if self.cache.has_key(key):
795 # push us back to the top of the LRU
796 self.cache_lru.remove(key)
797 self.cache_lru.insert(0, key)
798 # return the cached information
799 return self.cache[key]
801 # figure the columns we're fetching
802 cl = self.classes[classname]
803 cols, mls = self.determine_columns(cl.properties.items())
804 scols = ','.join([col for col,dt in cols])
806 # perform the basic property fetch
807 sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
808 self.sql(sql, (nodeid,))
810 values = self.sql_fetchone()
811 if values is None:
812 raise IndexError, 'no such %s node %s'%(classname, nodeid)
814 # make up the node
815 node = {}
816 props = cl.getprops(protected=1)
817 for col in range(len(cols)):
818 name = cols[col][0][1:]
819 value = values[col]
820 if value is not None:
821 value = self.sql_to_hyperdb_value[props[name].__class__](value)
822 node[name] = value
825 # now the multilinks
826 for col in mls:
827 # get the link ids
828 sql = 'select linkid from %s_%s where nodeid=%s'%(classname, col,
829 self.arg)
830 self.cursor.execute(sql, (nodeid,))
831 # extract the first column from the result
832 # XXX numeric ids
833 node[col] = [str(x[0]) for x in self.cursor.fetchall()]
835 # save off in the cache
836 key = (classname, nodeid)
837 self.cache[key] = node
838 # update the LRU
839 self.cache_lru.insert(0, key)
840 if len(self.cache_lru) > ROW_CACHE_SIZE:
841 del self.cache[self.cache_lru.pop()]
843 return node
845 def destroynode(self, classname, nodeid):
846 '''Remove a node from the database. Called exclusively by the
847 destroy() method on Class.
848 '''
849 if __debug__:
850 print >>hyperdb.DEBUG, 'destroynode', (self, classname, nodeid)
852 # make sure the node exists
853 if not self.hasnode(classname, nodeid):
854 raise IndexError, '%s has no node %s'%(classname, nodeid)
856 # see if we have this node cached
857 if self.cache.has_key((classname, nodeid)):
858 del self.cache[(classname, nodeid)]
860 # see if there's any obvious commit actions that we should get rid of
861 for entry in self.transactions[:]:
862 if entry[1][:2] == (classname, nodeid):
863 self.transactions.remove(entry)
865 # now do the SQL
866 sql = 'delete from _%s where id=%s'%(classname, self.arg)
867 self.sql(sql, (nodeid,))
869 # remove from multilnks
870 cl = self.getclass(classname)
871 x, mls = self.determine_columns(cl.properties.items())
872 for col in mls:
873 # get the link ids
874 sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
875 self.sql(sql, (nodeid,))
877 # remove journal entries
878 sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
879 self.sql(sql, (nodeid,))
881 def hasnode(self, classname, nodeid):
882 ''' Determine if the database has a given node.
883 '''
884 sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
885 if __debug__:
886 print >>hyperdb.DEBUG, 'hasnode', (self, sql, nodeid)
887 self.cursor.execute(sql, (nodeid,))
888 return int(self.cursor.fetchone()[0])
890 def countnodes(self, classname):
891 ''' Count the number of nodes that exist for a particular Class.
892 '''
893 sql = 'select count(*) from _%s'%classname
894 if __debug__:
895 print >>hyperdb.DEBUG, 'countnodes', (self, sql)
896 self.cursor.execute(sql)
897 return self.cursor.fetchone()[0]
899 def addjournal(self, classname, nodeid, action, params, creator=None,
900 creation=None):
901 ''' Journal the Action
902 'action' may be:
904 'create' or 'set' -- 'params' is a dictionary of property values
905 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
906 'retire' -- 'params' is None
907 '''
908 # handle supply of the special journalling parameters (usually
909 # supplied on importing an existing database)
910 if creator:
911 journaltag = creator
912 else:
913 journaltag = self.getuid()
914 if creation:
915 journaldate = creation
916 else:
917 journaldate = date.Date()
919 # create the journal entry
920 cols = 'nodeid,date,tag,action,params'
922 if __debug__:
923 print >>hyperdb.DEBUG, 'addjournal', (nodeid, journaldate,
924 journaltag, action, params)
926 self.save_journal(classname, cols, nodeid, journaldate,
927 journaltag, action, params)
929 def setjournal(self, classname, nodeid, journal):
930 '''Set the journal to the "journal" list.'''
931 # clear out any existing entries
932 self.sql('delete from %s__journal where nodeid=%s'%(classname,
933 self.arg), (nodeid,))
935 # create the journal entry
936 cols = 'nodeid,date,tag,action,params'
938 for nodeid, journaldate, journaltag, action, params in journal:
939 if __debug__:
940 print >>hyperdb.DEBUG, 'setjournal', (nodeid, journaldate,
941 journaltag, action, params)
942 self.save_journal(classname, cols, nodeid, journaldate,
943 journaltag, action, params)
945 def getjournal(self, classname, nodeid):
946 ''' get the journal for id
947 '''
948 # make sure the node exists
949 if not self.hasnode(classname, nodeid):
950 raise IndexError, '%s has no node %s'%(classname, nodeid)
952 cols = ','.join('nodeid date tag action params'.split())
953 return self.load_journal(classname, cols, nodeid)
955 def save_journal(self, classname, cols, nodeid, journaldate,
956 journaltag, action, params):
957 ''' Save the journal entry to the database
958 '''
959 # make the params db-friendly
960 params = repr(params)
961 dc = self.hyperdb_to_sql_value[hyperdb.Date]
962 entry = (nodeid, dc(journaldate), journaltag, action, params)
964 # do the insert
965 a = self.arg
966 sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(
967 classname, cols, a, a, a, a, a)
968 if __debug__:
969 print >>hyperdb.DEBUG, 'save_journal', (self, sql, entry)
970 self.cursor.execute(sql, entry)
972 def load_journal(self, classname, cols, nodeid):
973 ''' Load the journal from the database
974 '''
975 # now get the journal entries
976 sql = 'select %s from %s__journal where nodeid=%s order by date'%(
977 cols, classname, self.arg)
978 if __debug__:
979 print >>hyperdb.DEBUG, 'load_journal', (self, sql, nodeid)
980 self.cursor.execute(sql, (nodeid,))
981 res = []
982 dc = self.sql_to_hyperdb_value[hyperdb.Date]
983 for nodeid, date_stamp, user, action, params in self.cursor.fetchall():
984 params = eval(params)
985 # XXX numeric ids
986 res.append((str(nodeid), dc(date_stamp), user, action, params))
987 return res
989 def pack(self, pack_before):
990 ''' Delete all journal entries except "create" before 'pack_before'.
991 '''
992 # get a 'yyyymmddhhmmss' version of the date
993 date_stamp = pack_before.serialise()
995 # do the delete
996 for classname in self.classes.keys():
997 sql = "delete from %s__journal where date<%s and "\
998 "action<>'create'"%(classname, self.arg)
999 if __debug__:
1000 print >>hyperdb.DEBUG, 'pack', (self, sql, date_stamp)
1001 self.cursor.execute(sql, (date_stamp,))
1003 def sql_commit(self):
1004 ''' Actually commit to the database.
1005 '''
1006 if __debug__:
1007 print >>hyperdb.DEBUG, '+++ commit database connection +++'
1008 self.conn.commit()
1010 def commit(self):
1011 ''' Commit the current transactions.
1013 Save all data changed since the database was opened or since the
1014 last commit() or rollback().
1015 '''
1016 if __debug__:
1017 print >>hyperdb.DEBUG, 'commit', (self,)
1019 # commit the database
1020 self.sql_commit()
1022 # now, do all the other transaction stuff
1023 for method, args in self.transactions:
1024 method(*args)
1026 # save the indexer state
1027 self.indexer.save_index()
1029 # clear out the transactions
1030 self.transactions = []
1032 def sql_rollback(self):
1033 self.conn.rollback()
1035 def rollback(self):
1036 ''' Reverse all actions from the current transaction.
1038 Undo all the changes made since the database was opened or the last
1039 commit() or rollback() was performed.
1040 '''
1041 if __debug__:
1042 print >>hyperdb.DEBUG, 'rollback', (self,)
1044 self.sql_rollback()
1046 # roll back "other" transaction stuff
1047 for method, args in self.transactions:
1048 # delete temporary files
1049 if method == self.doStoreFile:
1050 self.rollbackStoreFile(*args)
1051 self.transactions = []
1053 # clear the cache
1054 self.clearCache()
1056 def sql_close(self):
1057 if __debug__:
1058 print >>hyperdb.DEBUG, '+++ close database connection +++'
1059 self.conn.close()
1061 def close(self):
1062 ''' Close off the connection.
1063 '''
1064 self.indexer.close()
1065 self.sql_close()
1067 #
1068 # The base Class class
1069 #
1070 class Class(hyperdb.Class):
1071 ''' The handle to a particular class of nodes in a hyperdatabase.
1073 All methods except __repr__ and getnode must be implemented by a
1074 concrete backend Class.
1075 '''
1077 def __init__(self, db, classname, **properties):
1078 '''Create a new class with a given name and property specification.
1080 'classname' must not collide with the name of an existing class,
1081 or a ValueError is raised. The keyword arguments in 'properties'
1082 must map names to property objects, or a TypeError is raised.
1083 '''
1084 for name in 'creation activity creator actor'.split():
1085 if properties.has_key(name):
1086 raise ValueError, '"creation", "activity", "creator" and '\
1087 '"actor" are reserved'
1089 self.classname = classname
1090 self.properties = properties
1091 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
1092 self.key = ''
1094 # should we journal changes (default yes)
1095 self.do_journal = 1
1097 # do the db-related init stuff
1098 db.addclass(self)
1100 self.auditors = {'create': [], 'set': [], 'retire': [], 'restore': []}
1101 self.reactors = {'create': [], 'set': [], 'retire': [], 'restore': []}
1103 def schema(self):
1104 ''' A dumpable version of the schema that we can store in the
1105 database
1106 '''
1107 return (self.key, [(x, repr(y)) for x,y in self.properties.items()])
1109 def enableJournalling(self):
1110 '''Turn journalling on for this class
1111 '''
1112 self.do_journal = 1
1114 def disableJournalling(self):
1115 '''Turn journalling off for this class
1116 '''
1117 self.do_journal = 0
1119 # Editing nodes:
1120 def create(self, **propvalues):
1121 ''' Create a new node of this class and return its id.
1123 The keyword arguments in 'propvalues' map property names to values.
1125 The values of arguments must be acceptable for the types of their
1126 corresponding properties or a TypeError is raised.
1128 If this class has a key property, it must be present and its value
1129 must not collide with other key strings or a ValueError is raised.
1131 Any other properties on this class that are missing from the
1132 'propvalues' dictionary are set to None.
1134 If an id in a link or multilink property does not refer to a valid
1135 node, an IndexError is raised.
1136 '''
1137 self.fireAuditors('create', None, propvalues)
1138 newid = self.create_inner(**propvalues)
1139 self.fireReactors('create', newid, None)
1140 return newid
1142 def create_inner(self, **propvalues):
1143 ''' Called by create, in-between the audit and react calls.
1144 '''
1145 if propvalues.has_key('id'):
1146 raise KeyError, '"id" is reserved'
1148 if self.db.journaltag is None:
1149 raise DatabaseError, 'Database open read-only'
1151 if propvalues.has_key('creator') or propvalues.has_key('actor') or \
1152 propvalues.has_key('creation') or propvalues.has_key('activity'):
1153 raise KeyError, '"creator", "actor", "creation" and '\
1154 '"activity" are reserved'
1156 # new node's id
1157 newid = self.db.newid(self.classname)
1159 # validate propvalues
1160 num_re = re.compile('^\d+$')
1161 for key, value in propvalues.items():
1162 if key == self.key:
1163 try:
1164 self.lookup(value)
1165 except KeyError:
1166 pass
1167 else:
1168 raise ValueError, 'node with key "%s" exists'%value
1170 # try to handle this property
1171 try:
1172 prop = self.properties[key]
1173 except KeyError:
1174 raise KeyError, '"%s" has no property "%s"'%(self.classname,
1175 key)
1177 if value is not None and isinstance(prop, Link):
1178 if type(value) != type(''):
1179 raise ValueError, 'link value must be String'
1180 link_class = self.properties[key].classname
1181 # if it isn't a number, it's a key
1182 if not num_re.match(value):
1183 try:
1184 value = self.db.classes[link_class].lookup(value)
1185 except (TypeError, KeyError):
1186 raise IndexError, 'new property "%s": %s not a %s'%(
1187 key, value, link_class)
1188 elif not self.db.getclass(link_class).hasnode(value):
1189 raise IndexError, '%s has no node %s'%(link_class, value)
1191 # save off the value
1192 propvalues[key] = value
1194 # register the link with the newly linked node
1195 if self.do_journal and self.properties[key].do_journal:
1196 self.db.addjournal(link_class, value, 'link',
1197 (self.classname, newid, key))
1199 elif isinstance(prop, Multilink):
1200 if type(value) != type([]):
1201 raise TypeError, 'new property "%s" not a list of ids'%key
1203 # clean up and validate the list of links
1204 link_class = self.properties[key].classname
1205 l = []
1206 for entry in value:
1207 if type(entry) != type(''):
1208 raise ValueError, '"%s" multilink value (%r) '\
1209 'must contain Strings'%(key, value)
1210 # if it isn't a number, it's a key
1211 if not num_re.match(entry):
1212 try:
1213 entry = self.db.classes[link_class].lookup(entry)
1214 except (TypeError, KeyError):
1215 raise IndexError, 'new property "%s": %s not a %s'%(
1216 key, entry, self.properties[key].classname)
1217 l.append(entry)
1218 value = l
1219 propvalues[key] = value
1221 # handle additions
1222 for nodeid in value:
1223 if not self.db.getclass(link_class).hasnode(nodeid):
1224 raise IndexError, '%s has no node %s'%(link_class,
1225 nodeid)
1226 # register the link with the newly linked node
1227 if self.do_journal and self.properties[key].do_journal:
1228 self.db.addjournal(link_class, nodeid, 'link',
1229 (self.classname, newid, key))
1231 elif isinstance(prop, String):
1232 if type(value) != type('') and type(value) != type(u''):
1233 raise TypeError, 'new property "%s" not a string'%key
1234 self.db.indexer.add_text((self.classname, newid, key), value)
1236 elif isinstance(prop, Password):
1237 if not isinstance(value, password.Password):
1238 raise TypeError, 'new property "%s" not a Password'%key
1240 elif isinstance(prop, Date):
1241 if value is not None and not isinstance(value, date.Date):
1242 raise TypeError, 'new property "%s" not a Date'%key
1244 elif isinstance(prop, Interval):
1245 if value is not None and not isinstance(value, date.Interval):
1246 raise TypeError, 'new property "%s" not an Interval'%key
1248 elif value is not None and isinstance(prop, Number):
1249 try:
1250 float(value)
1251 except ValueError:
1252 raise TypeError, 'new property "%s" not numeric'%key
1254 elif value is not None and isinstance(prop, Boolean):
1255 try:
1256 int(value)
1257 except ValueError:
1258 raise TypeError, 'new property "%s" not boolean'%key
1260 # make sure there's data where there needs to be
1261 for key, prop in self.properties.items():
1262 if propvalues.has_key(key):
1263 continue
1264 if key == self.key:
1265 raise ValueError, 'key property "%s" is required'%key
1266 if isinstance(prop, Multilink):
1267 propvalues[key] = []
1268 else:
1269 propvalues[key] = None
1271 # done
1272 self.db.addnode(self.classname, newid, propvalues)
1273 if self.do_journal:
1274 self.db.addjournal(self.classname, newid, 'create', {})
1276 # XXX numeric ids
1277 return str(newid)
1279 _marker = []
1280 def get(self, nodeid, propname, default=_marker, cache=1):
1281 '''Get the value of a property on an existing node of this class.
1283 'nodeid' must be the id of an existing node of this class or an
1284 IndexError is raised. 'propname' must be the name of a property
1285 of this class or a KeyError is raised.
1287 'cache' exists for backwards compatibility, and is not used.
1288 '''
1289 if propname == 'id':
1290 return nodeid
1292 # get the node's dict
1293 d = self.db.getnode(self.classname, nodeid)
1295 if propname == 'creation':
1296 if d.has_key('creation'):
1297 return d['creation']
1298 else:
1299 return date.Date()
1300 if propname == 'activity':
1301 if d.has_key('activity'):
1302 return d['activity']
1303 else:
1304 return date.Date()
1305 if propname == 'creator':
1306 if d.has_key('creator'):
1307 return d['creator']
1308 else:
1309 return self.db.getuid()
1310 if propname == 'actor':
1311 if d.has_key('actor'):
1312 return d['actor']
1313 else:
1314 return self.db.getuid()
1316 # get the property (raises KeyErorr if invalid)
1317 prop = self.properties[propname]
1319 if not d.has_key(propname):
1320 if default is self._marker:
1321 if isinstance(prop, Multilink):
1322 return []
1323 else:
1324 return None
1325 else:
1326 return default
1328 # don't pass our list to other code
1329 if isinstance(prop, Multilink):
1330 return d[propname][:]
1332 return d[propname]
1334 def set(self, nodeid, **propvalues):
1335 '''Modify a property on an existing node of this class.
1337 'nodeid' must be the id of an existing node of this class or an
1338 IndexError is raised.
1340 Each key in 'propvalues' must be the name of a property of this
1341 class or a KeyError is raised.
1343 All values in 'propvalues' must be acceptable types for their
1344 corresponding properties or a TypeError is raised.
1346 If the value of the key property is set, it must not collide with
1347 other key strings or a ValueError is raised.
1349 If the value of a Link or Multilink property contains an invalid
1350 node id, a ValueError is raised.
1351 '''
1352 self.fireAuditors('set', nodeid, propvalues)
1353 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1354 propvalues = self.set_inner(nodeid, **propvalues)
1355 self.fireReactors('set', nodeid, oldvalues)
1356 return propvalues
1358 def set_inner(self, nodeid, **propvalues):
1359 ''' Called by set, in-between the audit and react calls.
1360 '''
1361 if not propvalues:
1362 return propvalues
1364 if propvalues.has_key('creation') or propvalues.has_key('creator') or \
1365 propvalues.has_key('actor') or propvalues.has_key('activity'):
1366 raise KeyError, '"creation", "creator", "actor" and '\
1367 '"activity" are reserved'
1369 if propvalues.has_key('id'):
1370 raise KeyError, '"id" is reserved'
1372 if self.db.journaltag is None:
1373 raise DatabaseError, 'Database open read-only'
1375 node = self.db.getnode(self.classname, nodeid)
1376 if self.is_retired(nodeid):
1377 raise IndexError, 'Requested item is retired'
1378 num_re = re.compile('^\d+$')
1380 # if the journal value is to be different, store it in here
1381 journalvalues = {}
1383 # remember the add/remove stuff for multilinks, making it easier
1384 # for the Database layer to do its stuff
1385 multilink_changes = {}
1387 for propname, value in propvalues.items():
1388 # check to make sure we're not duplicating an existing key
1389 if propname == self.key and node[propname] != value:
1390 try:
1391 self.lookup(value)
1392 except KeyError:
1393 pass
1394 else:
1395 raise ValueError, 'node with key "%s" exists'%value
1397 # this will raise the KeyError if the property isn't valid
1398 # ... we don't use getprops() here because we only care about
1399 # the writeable properties.
1400 try:
1401 prop = self.properties[propname]
1402 except KeyError:
1403 raise KeyError, '"%s" has no property named "%s"'%(
1404 self.classname, propname)
1406 # if the value's the same as the existing value, no sense in
1407 # doing anything
1408 current = node.get(propname, None)
1409 if value == current:
1410 del propvalues[propname]
1411 continue
1412 journalvalues[propname] = current
1414 # do stuff based on the prop type
1415 if isinstance(prop, Link):
1416 link_class = prop.classname
1417 # if it isn't a number, it's a key
1418 if value is not None and not isinstance(value, type('')):
1419 raise ValueError, 'property "%s" link value be a string'%(
1420 propname)
1421 if isinstance(value, type('')) and not num_re.match(value):
1422 try:
1423 value = self.db.classes[link_class].lookup(value)
1424 except (TypeError, KeyError):
1425 raise IndexError, 'new property "%s": %s not a %s'%(
1426 propname, value, prop.classname)
1428 if (value is not None and
1429 not self.db.getclass(link_class).hasnode(value)):
1430 raise IndexError, '%s has no node %s'%(link_class, value)
1432 if self.do_journal and prop.do_journal:
1433 # register the unlink with the old linked node
1434 if node[propname] is not None:
1435 self.db.addjournal(link_class, node[propname], 'unlink',
1436 (self.classname, nodeid, propname))
1438 # register the link with the newly linked node
1439 if value is not None:
1440 self.db.addjournal(link_class, value, 'link',
1441 (self.classname, nodeid, propname))
1443 elif isinstance(prop, Multilink):
1444 if type(value) != type([]):
1445 raise TypeError, 'new property "%s" not a list of'\
1446 ' ids'%propname
1447 link_class = self.properties[propname].classname
1448 l = []
1449 for entry in value:
1450 # if it isn't a number, it's a key
1451 if type(entry) != type(''):
1452 raise ValueError, 'new property "%s" link value ' \
1453 'must be a string'%propname
1454 if not num_re.match(entry):
1455 try:
1456 entry = self.db.classes[link_class].lookup(entry)
1457 except (TypeError, KeyError):
1458 raise IndexError, 'new property "%s": %s not a %s'%(
1459 propname, entry,
1460 self.properties[propname].classname)
1461 l.append(entry)
1462 value = l
1463 propvalues[propname] = value
1465 # figure the journal entry for this property
1466 add = []
1467 remove = []
1469 # handle removals
1470 if node.has_key(propname):
1471 l = node[propname]
1472 else:
1473 l = []
1474 for id in l[:]:
1475 if id in value:
1476 continue
1477 # register the unlink with the old linked node
1478 if self.do_journal and self.properties[propname].do_journal:
1479 self.db.addjournal(link_class, id, 'unlink',
1480 (self.classname, nodeid, propname))
1481 l.remove(id)
1482 remove.append(id)
1484 # handle additions
1485 for id in value:
1486 if not self.db.getclass(link_class).hasnode(id):
1487 raise IndexError, '%s has no node %s'%(link_class, id)
1488 if id in l:
1489 continue
1490 # register the link with the newly linked node
1491 if self.do_journal and self.properties[propname].do_journal:
1492 self.db.addjournal(link_class, id, 'link',
1493 (self.classname, nodeid, propname))
1494 l.append(id)
1495 add.append(id)
1497 # figure the journal entry
1498 l = []
1499 if add:
1500 l.append(('+', add))
1501 if remove:
1502 l.append(('-', remove))
1503 multilink_changes[propname] = (add, remove)
1504 if l:
1505 journalvalues[propname] = tuple(l)
1507 elif isinstance(prop, String):
1508 if value is not None and type(value) != type('') and type(value) != type(u''):
1509 raise TypeError, 'new property "%s" not a string'%propname
1510 self.db.indexer.add_text((self.classname, nodeid, propname),
1511 value)
1513 elif isinstance(prop, Password):
1514 if not isinstance(value, password.Password):
1515 raise TypeError, 'new property "%s" not a Password'%propname
1516 propvalues[propname] = value
1518 elif value is not None and isinstance(prop, Date):
1519 if not isinstance(value, date.Date):
1520 raise TypeError, 'new property "%s" not a Date'% propname
1521 propvalues[propname] = value
1523 elif value is not None and isinstance(prop, Interval):
1524 if not isinstance(value, date.Interval):
1525 raise TypeError, 'new property "%s" not an '\
1526 'Interval'%propname
1527 propvalues[propname] = value
1529 elif value is not None and isinstance(prop, Number):
1530 try:
1531 float(value)
1532 except ValueError:
1533 raise TypeError, 'new property "%s" not numeric'%propname
1535 elif value is not None and isinstance(prop, Boolean):
1536 try:
1537 int(value)
1538 except ValueError:
1539 raise TypeError, 'new property "%s" not boolean'%propname
1541 # nothing to do?
1542 if not propvalues:
1543 return propvalues
1545 # do the set, and journal it
1546 self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
1548 if self.do_journal:
1549 self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
1551 return propvalues
1553 def retire(self, nodeid):
1554 '''Retire a node.
1556 The properties on the node remain available from the get() method,
1557 and the node's id is never reused.
1559 Retired nodes are not returned by the find(), list(), or lookup()
1560 methods, and other nodes may reuse the values of their key properties.
1561 '''
1562 if self.db.journaltag is None:
1563 raise DatabaseError, 'Database open read-only'
1565 self.fireAuditors('retire', nodeid, None)
1567 # use the arg for __retired__ to cope with any odd database type
1568 # conversion (hello, sqlite)
1569 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1570 self.db.arg, self.db.arg)
1571 if __debug__:
1572 print >>hyperdb.DEBUG, 'retire', (self, sql, nodeid)
1573 self.db.cursor.execute(sql, (1, nodeid))
1574 if self.do_journal:
1575 self.db.addjournal(self.classname, nodeid, 'retired', None)
1577 self.fireReactors('retire', nodeid, None)
1579 def restore(self, nodeid):
1580 '''Restore a retired node.
1582 Make node available for all operations like it was before retirement.
1583 '''
1584 if self.db.journaltag is None:
1585 raise DatabaseError, 'Database open read-only'
1587 node = self.db.getnode(self.classname, nodeid)
1588 # check if key property was overrided
1589 key = self.getkey()
1590 try:
1591 id = self.lookup(node[key])
1592 except KeyError:
1593 pass
1594 else:
1595 raise KeyError, "Key property (%s) of retired node clashes with \
1596 existing one (%s)" % (key, node[key])
1598 self.fireAuditors('restore', nodeid, None)
1599 # use the arg for __retired__ to cope with any odd database type
1600 # conversion (hello, sqlite)
1601 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1602 self.db.arg, self.db.arg)
1603 if __debug__:
1604 print >>hyperdb.DEBUG, 'restore', (self, sql, nodeid)
1605 self.db.cursor.execute(sql, (0, nodeid))
1606 if self.do_journal:
1607 self.db.addjournal(self.classname, nodeid, 'restored', None)
1609 self.fireReactors('restore', nodeid, None)
1611 def is_retired(self, nodeid):
1612 '''Return true if the node is rerired
1613 '''
1614 sql = 'select __retired__ from _%s where id=%s'%(self.classname,
1615 self.db.arg)
1616 if __debug__:
1617 print >>hyperdb.DEBUG, 'is_retired', (self, sql, nodeid)
1618 self.db.cursor.execute(sql, (nodeid,))
1619 return int(self.db.sql_fetchone()[0])
1621 def destroy(self, nodeid):
1622 '''Destroy a node.
1624 WARNING: this method should never be used except in extremely rare
1625 situations where there could never be links to the node being
1626 deleted
1628 WARNING: use retire() instead
1630 WARNING: the properties of this node will not be available ever again
1632 WARNING: really, use retire() instead
1634 Well, I think that's enough warnings. This method exists mostly to
1635 support the session storage of the cgi interface.
1637 The node is completely removed from the hyperdb, including all journal
1638 entries. It will no longer be available, and will generally break code
1639 if there are any references to the node.
1640 '''
1641 if self.db.journaltag is None:
1642 raise DatabaseError, 'Database open read-only'
1643 self.db.destroynode(self.classname, nodeid)
1645 def history(self, nodeid):
1646 '''Retrieve the journal of edits on a particular node.
1648 'nodeid' must be the id of an existing node of this class or an
1649 IndexError is raised.
1651 The returned list contains tuples of the form
1653 (nodeid, date, tag, action, params)
1655 'date' is a Timestamp object specifying the time of the change and
1656 'tag' is the journaltag specified when the database was opened.
1657 '''
1658 if not self.do_journal:
1659 raise ValueError, 'Journalling is disabled for this class'
1660 return self.db.getjournal(self.classname, nodeid)
1662 # Locating nodes:
1663 def hasnode(self, nodeid):
1664 '''Determine if the given nodeid actually exists
1665 '''
1666 return self.db.hasnode(self.classname, nodeid)
1668 def setkey(self, propname):
1669 '''Select a String property of this class to be the key property.
1671 'propname' must be the name of a String property of this class or
1672 None, or a TypeError is raised. The values of the key property on
1673 all existing nodes must be unique or a ValueError is raised.
1674 '''
1675 # XXX create an index on the key prop column. We should also
1676 # record that we've created this index in the schema somewhere.
1677 prop = self.getprops()[propname]
1678 if not isinstance(prop, String):
1679 raise TypeError, 'key properties must be String'
1680 self.key = propname
1682 def getkey(self):
1683 '''Return the name of the key property for this class or None.'''
1684 return self.key
1686 def labelprop(self, default_to_id=0):
1687 '''Return the property name for a label for the given node.
1689 This method attempts to generate a consistent label for the node.
1690 It tries the following in order:
1692 1. key property
1693 2. "name" property
1694 3. "title" property
1695 4. first property from the sorted property name list
1696 '''
1697 k = self.getkey()
1698 if k:
1699 return k
1700 props = self.getprops()
1701 if props.has_key('name'):
1702 return 'name'
1703 elif props.has_key('title'):
1704 return 'title'
1705 if default_to_id:
1706 return 'id'
1707 props = props.keys()
1708 props.sort()
1709 return props[0]
1711 def lookup(self, keyvalue):
1712 '''Locate a particular node by its key property and return its id.
1714 If this class has no key property, a TypeError is raised. If the
1715 'keyvalue' matches one of the values for the key property among
1716 the nodes in this class, the matching node's id is returned;
1717 otherwise a KeyError is raised.
1718 '''
1719 if not self.key:
1720 raise TypeError, 'No key property set for class %s'%self.classname
1722 # use the arg to handle any odd database type conversion (hello,
1723 # sqlite)
1724 sql = "select id from _%s where _%s=%s and __retired__ <> %s"%(
1725 self.classname, self.key, self.db.arg, self.db.arg)
1726 self.db.sql(sql, (keyvalue, 1))
1728 # see if there was a result that's not retired
1729 row = self.db.sql_fetchone()
1730 if not row:
1731 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1732 keyvalue, self.classname)
1734 # return the id
1735 # XXX numeric ids
1736 return str(row[0])
1738 def find(self, **propspec):
1739 '''Get the ids of nodes in this class which link to the given nodes.
1741 'propspec' consists of keyword args propname=nodeid or
1742 propname={nodeid:1, }
1743 'propname' must be the name of a property in this class, or a
1744 KeyError is raised. That property must be a Link or
1745 Multilink property, or a TypeError is raised.
1747 Any node in this class whose 'propname' property links to any of the
1748 nodeids will be returned. Used by the full text indexing, which knows
1749 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1750 issues:
1752 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1753 '''
1754 if __debug__:
1755 print >>hyperdb.DEBUG, 'find', (self, propspec)
1757 # shortcut
1758 if not propspec:
1759 return []
1761 # validate the args
1762 props = self.getprops()
1763 propspec = propspec.items()
1764 for propname, nodeids in propspec:
1765 # check the prop is OK
1766 prop = props[propname]
1767 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1768 raise TypeError, "'%s' not a Link/Multilink property"%propname
1770 # first, links
1771 a = self.db.arg
1772 allvalues = (1,)
1773 o = []
1774 where = []
1775 for prop, values in propspec:
1776 if not isinstance(props[prop], hyperdb.Link):
1777 continue
1778 if type(values) is type({}) and len(values) == 1:
1779 values = values.keys()[0]
1780 if type(values) is type(''):
1781 allvalues += (values,)
1782 where.append('_%s = %s'%(prop, a))
1783 elif values is None:
1784 where.append('_%s is NULL'%prop)
1785 else:
1786 allvalues += tuple(values.keys())
1787 where.append('_%s in (%s)'%(prop, ','.join([a]*len(values))))
1788 tables = ['_%s'%self.classname]
1789 if where:
1790 o.append('(' + ' and '.join(where) + ')')
1792 # now multilinks
1793 for prop, values in propspec:
1794 if not isinstance(props[prop], hyperdb.Multilink):
1795 continue
1796 if not values:
1797 continue
1798 if type(values) is type(''):
1799 allvalues += (values,)
1800 s = a
1801 else:
1802 allvalues += tuple(values.keys())
1803 s = ','.join([a]*len(values))
1804 tn = '%s_%s'%(self.classname, prop)
1805 tables.append(tn)
1806 o.append('(id=%s.nodeid and %s.linkid in (%s))'%(tn, tn, s))
1808 if not o:
1809 return []
1810 elif len(o) > 1:
1811 o = '(' + ' or '.join(['(%s)'%i for i in o]) + ')'
1812 else:
1813 o = o[0]
1814 t = ', '.join(tables)
1815 sql = 'select distinct(id) from %s where __retired__ <> %s and %s'%(
1816 t, a, o)
1817 self.db.sql(sql, allvalues)
1818 # XXX numeric ids
1819 l = [str(x[0]) for x in self.db.sql_fetchall()]
1820 if __debug__:
1821 print >>hyperdb.DEBUG, 'find ... ', l
1822 return l
1824 def stringFind(self, **requirements):
1825 '''Locate a particular node by matching a set of its String
1826 properties in a caseless search.
1828 If the property is not a String property, a TypeError is raised.
1830 The return is a list of the id of all nodes that match.
1831 '''
1832 where = []
1833 args = []
1834 for propname in requirements.keys():
1835 prop = self.properties[propname]
1836 if not isinstance(prop, String):
1837 raise TypeError, "'%s' not a String property"%propname
1838 where.append(propname)
1839 args.append(requirements[propname].lower())
1841 # generate the where clause
1842 s = ' and '.join(['lower(_%s)=%s'%(col, self.db.arg) for col in where])
1843 sql = 'select id from _%s where %s and __retired__=%s'%(self.classname,
1844 s, self.db.arg)
1845 args.append(0)
1846 self.db.sql(sql, tuple(args))
1847 # XXX numeric ids
1848 l = [str(x[0]) for x in self.db.sql_fetchall()]
1849 if __debug__:
1850 print >>hyperdb.DEBUG, 'find ... ', l
1851 return l
1853 def list(self):
1854 ''' Return a list of the ids of the active nodes in this class.
1855 '''
1856 return self.getnodeids(retired=0)
1858 def getnodeids(self, retired=None):
1859 ''' Retrieve all the ids of the nodes for a particular Class.
1861 Set retired=None to get all nodes. Otherwise it'll get all the
1862 retired or non-retired nodes, depending on the flag.
1863 '''
1864 # flip the sense of the 'retired' flag if we don't want all of them
1865 if retired is not None:
1866 if retired:
1867 args = (0, )
1868 else:
1869 args = (1, )
1870 sql = 'select id from _%s where __retired__ <> %s'%(self.classname,
1871 self.db.arg)
1872 else:
1873 args = ()
1874 sql = 'select id from _%s'%self.classname
1875 if __debug__:
1876 print >>hyperdb.DEBUG, 'getnodeids', (self, sql, retired)
1877 self.db.cursor.execute(sql, args)
1878 # XXX numeric ids
1879 ids = [str(x[0]) for x in self.db.cursor.fetchall()]
1880 return ids
1882 def filter(self, search_matches, filterspec, sort=(None,None),
1883 group=(None,None)):
1884 '''Return a list of the ids of the active nodes in this class that
1885 match the 'filter' spec, sorted by the group spec and then the
1886 sort spec
1888 "filterspec" is {propname: value(s)}
1890 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
1891 and prop is a prop name or None
1893 "search_matches" is {nodeid: marker}
1895 The filter must match all properties specificed - but if the
1896 property value to match is a list, any one of the values in the
1897 list may match for that property to match.
1898 '''
1899 # just don't bother if the full-text search matched diddly
1900 if search_matches == {}:
1901 return []
1903 cn = self.classname
1905 timezone = self.db.getUserTimezone()
1907 # figure the WHERE clause from the filterspec
1908 props = self.getprops()
1909 frum = ['_'+cn]
1910 where = []
1911 args = []
1912 a = self.db.arg
1913 for k, v in filterspec.items():
1914 propclass = props[k]
1915 # now do other where clause stuff
1916 if isinstance(propclass, Multilink):
1917 tn = '%s_%s'%(cn, k)
1918 if v in ('-1', ['-1']):
1919 # only match rows that have count(linkid)=0 in the
1920 # corresponding multilink table)
1921 where.append('id not in (select nodeid from %s)'%tn)
1922 elif isinstance(v, type([])):
1923 frum.append(tn)
1924 s = ','.join([a for x in v])
1925 where.append('id=%s.nodeid and %s.linkid in (%s)'%(tn,tn,s))
1926 args = args + v
1927 else:
1928 frum.append(tn)
1929 where.append('id=%s.nodeid and %s.linkid=%s'%(tn, tn, a))
1930 args.append(v)
1931 elif k == 'id':
1932 if isinstance(v, type([])):
1933 s = ','.join([a for x in v])
1934 where.append('%s in (%s)'%(k, s))
1935 args = args + v
1936 else:
1937 where.append('%s=%s'%(k, a))
1938 args.append(v)
1939 elif isinstance(propclass, String):
1940 if not isinstance(v, type([])):
1941 v = [v]
1943 # Quote the bits in the string that need it and then embed
1944 # in a "substring" search. Note - need to quote the '%' so
1945 # they make it through the python layer happily
1946 v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
1948 # now add to the where clause
1949 where.append(' or '.join(["_%s LIKE '%s'"%(k, s) for s in v]))
1950 # note: args are embedded in the query string now
1951 elif isinstance(propclass, Link):
1952 if isinstance(v, type([])):
1953 if '-1' in v:
1954 v = v[:]
1955 v.remove('-1')
1956 xtra = ' or _%s is NULL'%k
1957 else:
1958 xtra = ''
1959 if v:
1960 s = ','.join([a for x in v])
1961 where.append('(_%s in (%s)%s)'%(k, s, xtra))
1962 args = args + v
1963 else:
1964 where.append('_%s is NULL'%k)
1965 else:
1966 if v == '-1':
1967 v = None
1968 where.append('_%s is NULL'%k)
1969 else:
1970 where.append('_%s=%s'%(k, a))
1971 args.append(v)
1972 elif isinstance(propclass, Date):
1973 dc = self.db.hyperdb_to_sql_value[hyperdb.Date]
1974 if isinstance(v, type([])):
1975 s = ','.join([a for x in v])
1976 where.append('_%s in (%s)'%(k, s))
1977 args = args + [dc(date.Date(v)) for x in v]
1978 else:
1979 try:
1980 # Try to filter on range of dates
1981 date_rng = Range(v, date.Date, offset=timezone)
1982 if date_rng.from_value:
1983 where.append('_%s >= %s'%(k, a))
1984 args.append(dc(date_rng.from_value))
1985 if date_rng.to_value:
1986 where.append('_%s <= %s'%(k, a))
1987 args.append(dc(date_rng.to_value))
1988 except ValueError:
1989 # If range creation fails - ignore that search parameter
1990 pass
1991 elif isinstance(propclass, Interval):
1992 if isinstance(v, type([])):
1993 s = ','.join([a for x in v])
1994 where.append('_%s in (%s)'%(k, s))
1995 args = args + [date.Interval(x).serialise() for x in v]
1996 else:
1997 try:
1998 # Try to filter on range of intervals
1999 date_rng = Range(v, date.Interval)
2000 if date_rng.from_value:
2001 where.append('_%s >= %s'%(k, a))
2002 args.append(date_rng.from_value.serialise())
2003 if date_rng.to_value:
2004 where.append('_%s <= %s'%(k, a))
2005 args.append(date_rng.to_value.serialise())
2006 except ValueError:
2007 # If range creation fails - ignore that search parameter
2008 pass
2009 #where.append('_%s=%s'%(k, a))
2010 #args.append(date.Interval(v).serialise())
2011 else:
2012 if isinstance(v, type([])):
2013 s = ','.join([a for x in v])
2014 where.append('_%s in (%s)'%(k, s))
2015 args = args + v
2016 else:
2017 where.append('_%s=%s'%(k, a))
2018 args.append(v)
2020 # don't match retired nodes
2021 where.append('__retired__ <> 1')
2023 # add results of full text search
2024 if search_matches is not None:
2025 v = search_matches.keys()
2026 s = ','.join([a for x in v])
2027 where.append('id in (%s)'%s)
2028 args = args + v
2030 # "grouping" is just the first-order sorting in the SQL fetch
2031 orderby = []
2032 ordercols = []
2033 mlsort = []
2034 for sortby in group, sort:
2035 sdir, prop = sortby
2036 if sdir and prop:
2037 if isinstance(props[prop], Multilink):
2038 mlsort.append(sortby)
2039 continue
2040 elif prop == 'id':
2041 o = 'id'
2042 else:
2043 o = '_'+prop
2044 ordercols.append(o)
2045 if sdir == '-':
2046 o += ' desc'
2047 orderby.append(o)
2049 # construct the SQL
2050 frum = ','.join(frum)
2051 if where:
2052 where = ' where ' + (' and '.join(where))
2053 else:
2054 where = ''
2055 cols = ['distinct(id)']
2056 if orderby:
2057 cols = cols + ordercols
2058 order = ' order by %s'%(','.join(orderby))
2059 else:
2060 order = ''
2061 cols = ','.join(cols)
2062 sql = 'select %s from %s %s%s'%(cols, frum, where, order)
2063 args = tuple(args)
2064 if __debug__:
2065 print >>hyperdb.DEBUG, 'filter', (self, sql, args)
2066 if args:
2067 self.db.cursor.execute(sql, args)
2068 else:
2069 # psycopg doesn't like empty args
2070 self.db.cursor.execute(sql)
2071 l = self.db.sql_fetchall()
2073 # return the IDs (the first column)
2074 # XXX numeric ids
2075 l = [str(row[0]) for row in l]
2077 if not mlsort:
2078 return l
2080 # ergh. someone wants to sort by a multilink.
2081 r = []
2082 for id in l:
2083 m = []
2084 for ml in mlsort:
2085 m.append(self.get(id, ml[1]))
2086 r.append((id, m))
2087 i = 0
2088 for sortby in mlsort:
2089 def sortfun(a, b, dir=sortby[i]):
2090 if dir == '-':
2091 return cmp(b[1][i], a[1][i])
2092 else:
2093 return cmp(a[1][i], b[1][i])
2094 r.sort(sortfun)
2095 i += 1
2096 return [i[0] for i in r]
2098 def count(self):
2099 '''Get the number of nodes in this class.
2101 If the returned integer is 'numnodes', the ids of all the nodes
2102 in this class run from 1 to numnodes, and numnodes+1 will be the
2103 id of the next node to be created in this class.
2104 '''
2105 return self.db.countnodes(self.classname)
2107 # Manipulating properties:
2108 def getprops(self, protected=1):
2109 '''Return a dictionary mapping property names to property objects.
2110 If the "protected" flag is true, we include protected properties -
2111 those which may not be modified.
2112 '''
2113 d = self.properties.copy()
2114 if protected:
2115 d['id'] = String()
2116 d['creation'] = hyperdb.Date()
2117 d['activity'] = hyperdb.Date()
2118 d['creator'] = hyperdb.Link('user')
2119 d['actor'] = hyperdb.Link('user')
2120 return d
2122 def addprop(self, **properties):
2123 '''Add properties to this class.
2125 The keyword arguments in 'properties' must map names to property
2126 objects, or a TypeError is raised. None of the keys in 'properties'
2127 may collide with the names of existing properties, or a ValueError
2128 is raised before any properties have been added.
2129 '''
2130 for key in properties.keys():
2131 if self.properties.has_key(key):
2132 raise ValueError, key
2133 self.properties.update(properties)
2135 def index(self, nodeid):
2136 '''Add (or refresh) the node to search indexes
2137 '''
2138 # find all the String properties that have indexme
2139 for prop, propclass in self.getprops().items():
2140 if isinstance(propclass, String) and propclass.indexme:
2141 self.db.indexer.add_text((self.classname, nodeid, prop),
2142 str(self.get(nodeid, prop)))
2145 #
2146 # Detector interface
2147 #
2148 def audit(self, event, detector):
2149 '''Register a detector
2150 '''
2151 l = self.auditors[event]
2152 if detector not in l:
2153 self.auditors[event].append(detector)
2155 def fireAuditors(self, action, nodeid, newvalues):
2156 '''Fire all registered auditors.
2157 '''
2158 for audit in self.auditors[action]:
2159 audit(self.db, self, nodeid, newvalues)
2161 def react(self, event, detector):
2162 '''Register a detector
2163 '''
2164 l = self.reactors[event]
2165 if detector not in l:
2166 self.reactors[event].append(detector)
2168 def fireReactors(self, action, nodeid, oldvalues):
2169 '''Fire all registered reactors.
2170 '''
2171 for react in self.reactors[action]:
2172 react(self.db, self, nodeid, oldvalues)
2174 #
2175 # import / export support
2176 #
2177 def export_list(self, propnames, nodeid):
2178 ''' Export a node - generate a list of CSV-able data in the order
2179 specified by propnames for the given node.
2180 '''
2181 properties = self.getprops()
2182 l = []
2183 for prop in propnames:
2184 proptype = properties[prop]
2185 value = self.get(nodeid, prop)
2186 # "marshal" data where needed
2187 if value is None:
2188 pass
2189 elif isinstance(proptype, hyperdb.Date):
2190 value = value.get_tuple()
2191 elif isinstance(proptype, hyperdb.Interval):
2192 value = value.get_tuple()
2193 elif isinstance(proptype, hyperdb.Password):
2194 value = str(value)
2195 l.append(repr(value))
2196 l.append(repr(self.is_retired(nodeid)))
2197 return l
2199 def import_list(self, propnames, proplist):
2200 ''' Import a node - all information including "id" is present and
2201 should not be sanity checked. Triggers are not triggered. The
2202 journal should be initialised using the "creator" and "created"
2203 information.
2205 Return the nodeid of the node imported.
2206 '''
2207 if self.db.journaltag is None:
2208 raise DatabaseError, 'Database open read-only'
2209 properties = self.getprops()
2211 # make the new node's property map
2212 d = {}
2213 retire = 0
2214 newid = None
2215 for i in range(len(propnames)):
2216 # Use eval to reverse the repr() used to output the CSV
2217 value = eval(proplist[i])
2219 # Figure the property for this column
2220 propname = propnames[i]
2222 # "unmarshal" where necessary
2223 if propname == 'id':
2224 newid = value
2225 continue
2226 elif propname == 'is retired':
2227 # is the item retired?
2228 if int(value):
2229 retire = 1
2230 continue
2231 elif value is None:
2232 d[propname] = None
2233 continue
2235 prop = properties[propname]
2236 if value is None:
2237 # don't set Nones
2238 continue
2239 elif isinstance(prop, hyperdb.Date):
2240 value = date.Date(value)
2241 elif isinstance(prop, hyperdb.Interval):
2242 value = date.Interval(value)
2243 elif isinstance(prop, hyperdb.Password):
2244 pwd = password.Password()
2245 pwd.unpack(value)
2246 value = pwd
2247 d[propname] = value
2249 # get a new id if necessary
2250 if newid is None or not self.hasnode(newid):
2251 newid = self.db.newid(self.classname)
2252 self.db.addnode(self.classname, newid, d)
2253 else:
2254 # update
2255 self.db.setnode(self.classname, newid, d)
2257 # retire?
2258 if retire:
2259 # use the arg for __retired__ to cope with any odd database type
2260 # conversion (hello, sqlite)
2261 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
2262 self.db.arg, self.db.arg)
2263 if __debug__:
2264 print >>hyperdb.DEBUG, 'retire', (self, sql, newid)
2265 self.db.cursor.execute(sql, (1, newid))
2266 return newid
2268 def export_journals(self):
2269 '''Export a class's journal - generate a list of lists of
2270 CSV-able data:
2272 nodeid, date, user, action, params
2274 No heading here - the columns are fixed.
2275 '''
2276 properties = self.getprops()
2277 r = []
2278 for nodeid in self.getnodeids():
2279 for nodeid, date, user, action, params in self.history(nodeid):
2280 date = date.get_tuple()
2281 if action == 'set':
2282 for propname, value in params.items():
2283 prop = properties[propname]
2284 # make sure the params are eval()'able
2285 if value is None:
2286 pass
2287 elif isinstance(prop, Date):
2288 value = value.get_tuple()
2289 elif isinstance(prop, Interval):
2290 value = value.get_tuple()
2291 elif isinstance(prop, Password):
2292 value = str(value)
2293 params[propname] = value
2294 l = [nodeid, date, user, action, params]
2295 r.append(map(repr, l))
2296 return r
2298 def import_journals(self, entries):
2299 '''Import a class's journal.
2301 Uses setjournal() to set the journal for each item.'''
2302 properties = self.getprops()
2303 d = {}
2304 for l in entries:
2305 l = map(eval, l)
2306 nodeid, jdate, user, action, params = l
2307 r = d.setdefault(nodeid, [])
2308 if action == 'set':
2309 for propname, value in params.items():
2310 prop = properties[propname]
2311 if value is None:
2312 pass
2313 elif isinstance(prop, Date):
2314 value = date.Date(value)
2315 elif isinstance(prop, Interval):
2316 value = date.Interval(value)
2317 elif isinstance(prop, Password):
2318 pwd = password.Password()
2319 pwd.unpack(value)
2320 value = pwd
2321 params[propname] = value
2322 r.append((nodeid, date.Date(jdate), user, action, params))
2324 for nodeid, l in d.items():
2325 self.db.setjournal(self.classname, nodeid, l)
2327 class FileClass(Class, hyperdb.FileClass):
2328 '''This class defines a large chunk of data. To support this, it has a
2329 mandatory String property "content" which is typically saved off
2330 externally to the hyperdb.
2332 The default MIME type of this data is defined by the
2333 "default_mime_type" class attribute, which may be overridden by each
2334 node if the class defines a "type" String property.
2335 '''
2336 default_mime_type = 'text/plain'
2338 def create(self, **propvalues):
2339 ''' snaffle the file propvalue and store in a file
2340 '''
2341 # we need to fire the auditors now, or the content property won't
2342 # be in propvalues for the auditors to play with
2343 self.fireAuditors('create', None, propvalues)
2345 # now remove the content property so it's not stored in the db
2346 content = propvalues['content']
2347 del propvalues['content']
2349 # do the database create
2350 newid = self.create_inner(**propvalues)
2352 # figure the mime type
2353 mime_type = propvalues.get('type', self.default_mime_type)
2355 # and index!
2356 self.db.indexer.add_text((self.classname, newid, 'content'), content,
2357 mime_type)
2359 # fire reactors
2360 self.fireReactors('create', newid, None)
2362 # store off the content as a file
2363 self.db.storefile(self.classname, newid, None, content)
2364 return newid
2366 def import_list(self, propnames, proplist):
2367 ''' Trap the "content" property...
2368 '''
2369 # dupe this list so we don't affect others
2370 propnames = propnames[:]
2372 # extract the "content" property from the proplist
2373 i = propnames.index('content')
2374 content = eval(proplist[i])
2375 del propnames[i]
2376 del proplist[i]
2378 # do the normal import
2379 newid = Class.import_list(self, propnames, proplist)
2381 # save off the "content" file
2382 self.db.storefile(self.classname, newid, None, content)
2383 return newid
2385 _marker = []
2386 def get(self, nodeid, propname, default=_marker, cache=1):
2387 ''' Trap the content propname and get it from the file
2389 'cache' exists for backwards compatibility, and is not used.
2390 '''
2391 poss_msg = 'Possibly a access right configuration problem.'
2392 if propname == 'content':
2393 try:
2394 return self.db.getfile(self.classname, nodeid, None)
2395 except IOError, (strerror):
2396 # BUG: by catching this we donot see an error in the log.
2397 return 'ERROR reading file: %s%s\n%s\n%s'%(
2398 self.classname, nodeid, poss_msg, strerror)
2399 if default is not self._marker:
2400 return Class.get(self, nodeid, propname, default)
2401 else:
2402 return Class.get(self, nodeid, propname)
2404 def getprops(self, protected=1):
2405 ''' In addition to the actual properties on the node, these methods
2406 provide the "content" property. If the "protected" flag is true,
2407 we include protected properties - those which may not be
2408 modified.
2409 '''
2410 d = Class.getprops(self, protected=protected).copy()
2411 d['content'] = hyperdb.String()
2412 return d
2414 def set(self, itemid, **propvalues):
2415 ''' Snarf the "content" propvalue and update it in a file
2416 '''
2417 self.fireAuditors('set', itemid, propvalues)
2418 oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid))
2420 # now remove the content property so it's not stored in the db
2421 content = None
2422 if propvalues.has_key('content'):
2423 content = propvalues['content']
2424 del propvalues['content']
2426 # do the database create
2427 propvalues = self.set_inner(itemid, **propvalues)
2429 # do content?
2430 if content:
2431 # store and index
2432 self.db.storefile(self.classname, itemid, None, content)
2433 mime_type = propvalues.get('type', self.get(itemid, 'type'))
2434 if not mime_type:
2435 mime_type = self.default_mime_type
2436 self.db.indexer.add_text((self.classname, itemid, 'content'),
2437 content, mime_type)
2439 # fire reactors
2440 self.fireReactors('set', itemid, oldvalues)
2441 return propvalues
2443 # XXX deviation from spec - was called ItemClass
2444 class IssueClass(Class, roundupdb.IssueClass):
2445 # Overridden methods:
2446 def __init__(self, db, classname, **properties):
2447 '''The newly-created class automatically includes the "messages",
2448 "files", "nosy", and "superseder" properties. If the 'properties'
2449 dictionary attempts to specify any of these properties or a
2450 "creation", "creator", "activity" or "actor" property, a ValueError
2451 is raised.
2452 '''
2453 if not properties.has_key('title'):
2454 properties['title'] = hyperdb.String(indexme='yes')
2455 if not properties.has_key('messages'):
2456 properties['messages'] = hyperdb.Multilink("msg")
2457 if not properties.has_key('files'):
2458 properties['files'] = hyperdb.Multilink("file")
2459 if not properties.has_key('nosy'):
2460 # note: journalling is turned off as it really just wastes
2461 # space. this behaviour may be overridden in an instance
2462 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2463 if not properties.has_key('superseder'):
2464 properties['superseder'] = hyperdb.Multilink(classname)
2465 Class.__init__(self, db, classname, **properties)