1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 """ Relational database (SQL) backend common code.
20 Basics:
22 - map roundup classes to relational tables
23 - automatically detect schema changes and modify the table schemas
24 appropriately (we store the "database version" of the schema in the
25 database itself as the only row of the "schema" table)
26 - multilinks (which represent a many-to-many relationship) are handled through
27 intermediate tables
28 - journals are stored adjunct to the per-class tables
29 - table names and columns have "_" prepended so the names can't clash with
30 restricted names (like "order")
31 - retirement is determined by the __retired__ column being > 0
33 Database-specific changes may generally be pushed out to the overridable
34 sql_* methods, since everything else should be fairly generic. There's
35 probably a bit of work to be done if a database is used that actually
36 honors column typing, since the initial databases don't (sqlite stores
37 everything as a string.)
39 The schema of the hyperdb being mapped to the database is stored in the
40 database itself as a repr()'ed dictionary of information about each Class
41 that maps to a table. If that information differs from the hyperdb schema,
42 then we update it. We also store in the schema dict a version which
43 allows us to upgrade the database schema when necessary. See upgrade_db().
45 To force a unqiueness constraint on the key properties we put the item
46 id into the __retired__ column duing retirement (so it's 0 for "active"
47 items) and place a unqiueness constraint on key + __retired__. This is
48 particularly important for the users class where multiple users may
49 try to have the same username, with potentially many retired users with
50 the same name.
51 """
52 __docformat__ = 'restructuredtext'
54 # standard python modules
55 import sys, os, time, re, errno, weakref, copy, logging, datetime
57 # roundup modules
58 from roundup import hyperdb, date, password, roundupdb, security, support
59 from roundup.hyperdb import String, Password, Date, Interval, Link, \
60 Multilink, DatabaseError, Boolean, Number, Node
61 from roundup.backends import locking
62 from roundup.support import reversed
63 from roundup.i18n import _
66 # support
67 from roundup.backends.blobfiles import FileStorage
68 try:
69 from roundup.backends.indexer_xapian import Indexer
70 except ImportError:
71 from roundup.backends.indexer_rdbms import Indexer
72 from roundup.backends.sessions_rdbms import Sessions, OneTimeKeys
73 from roundup.date import Range
75 from roundup.backends.back_anydbm import compile_expression
78 # dummy value meaning "argument not passed"
79 _marker = []
81 def _num_cvt(num):
82 num = str(num)
83 try:
84 return int(num)
85 except:
86 return float(num)
88 def _bool_cvt(value):
89 if value in ('TRUE', 'FALSE'):
90 return {'TRUE': 1, 'FALSE': 0}[value]
91 # assume it's a number returned from the db API
92 return int(value)
94 def date_to_hyperdb_value(d):
95 """ convert date d to a roundup date """
96 if isinstance (d, datetime.datetime):
97 return date.Date(d)
98 return date.Date (str(d).replace(' ', '.'))
101 def connection_dict(config, dbnamestr=None):
102 """ Used by Postgresql and MySQL to detemine the keyword args for
103 opening the database connection."""
104 d = { }
105 if dbnamestr:
106 d[dbnamestr] = config.RDBMS_NAME
107 for name in ('host', 'port', 'password', 'user', 'read_default_group',
108 'read_default_file'):
109 cvar = 'RDBMS_'+name.upper()
110 if config[cvar] is not None:
111 d[name] = config[cvar]
112 return d
115 class IdListOptimizer:
116 """ To prevent flooding the SQL parser of the underlaying
117 db engine with "x IN (1, 2, 3, ..., <large number>)" collapses
118 these cases to "x BETWEEN 1 AND <large number>".
119 """
121 def __init__(self):
122 self.ranges = []
123 self.singles = []
125 def append(self, nid):
126 """ Invariant: nid are ordered ascending """
127 if self.ranges:
128 last = self.ranges[-1]
129 if last[1] == nid-1:
130 last[1] = nid
131 return
132 if self.singles:
133 last = self.singles[-1]
134 if last == nid-1:
135 self.singles.pop()
136 self.ranges.append([last, nid])
137 return
138 self.singles.append(nid)
140 def where(self, field, placeholder):
141 ranges = self.ranges
142 singles = self.singles
144 if not singles and not ranges: return "(1=0)", []
146 if ranges:
147 between = '%s BETWEEN %s AND %s' % (
148 field, placeholder, placeholder)
149 stmnt = [between] * len(ranges)
150 else:
151 stmnt = []
152 if singles:
153 stmnt.append('%s in (%s)' % (
154 field, ','.join([placeholder]*len(singles))))
156 return '(%s)' % ' OR '.join(stmnt), sum(ranges, []) + singles
158 def __str__(self):
159 return "ranges: %r / singles: %r" % (self.ranges, self.singles)
162 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
163 """ Wrapper around an SQL database that presents a hyperdb interface.
165 - some functionality is specific to the actual SQL database, hence
166 the sql_* methods that are NotImplemented
167 - we keep a cache of the latest N row fetches (where N is configurable).
168 """
169 def __init__(self, config, journaltag=None):
170 """ Open the database and load the schema from it.
171 """
172 FileStorage.__init__(self, config.UMASK)
173 self.config, self.journaltag = config, journaltag
174 self.dir = config.DATABASE
175 self.classes = {}
176 self.indexer = Indexer(self)
177 self.security = security.Security(self)
179 # additional transaction support for external files and the like
180 self.transactions = []
182 # keep a cache of the N most recently retrieved rows of any kind
183 # (classname, nodeid) = row
184 self.cache_size = config.RDBMS_CACHE_SIZE
185 self.clearCache()
186 self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0,
187 'filtering': 0}
189 # database lock
190 self.lockfile = None
192 # open a connection to the database, creating the "conn" attribute
193 self.open_connection()
195 def clearCache(self):
196 self.cache = {}
197 self.cache_lru = []
199 def getSessionManager(self):
200 return Sessions(self)
202 def getOTKManager(self):
203 return OneTimeKeys(self)
205 def open_connection(self):
206 """ Open a connection to the database, creating it if necessary.
208 Must call self.load_dbschema()
209 """
210 raise NotImplemented
212 def sql(self, sql, args=None, cursor=None):
213 """ Execute the sql with the optional args.
214 """
215 self.log_debug('SQL %r %r'%(sql, args))
216 if not cursor:
217 cursor = self.cursor
218 if args:
219 cursor.execute(sql, args)
220 else:
221 cursor.execute(sql)
223 def sql_fetchone(self):
224 """ Fetch a single row. If there's nothing to fetch, return None.
225 """
226 return self.cursor.fetchone()
228 def sql_fetchall(self):
229 """ Fetch all rows. If there's nothing to fetch, return [].
230 """
231 return self.cursor.fetchall()
233 def sql_fetchiter(self):
234 """ Fetch all row as a generator
235 """
236 while True:
237 row = self.cursor.fetchone()
238 if not row: break
239 yield row
241 def sql_stringquote(self, value):
242 """ Quote the string so it's safe to put in the 'sql quotes'
243 """
244 return re.sub("'", "''", str(value))
246 def init_dbschema(self):
247 self.database_schema = {
248 'version': self.current_db_version,
249 'tables': {}
250 }
252 def load_dbschema(self):
253 """ Load the schema definition that the database currently implements
254 """
255 self.cursor.execute('select schema from schema')
256 schema = self.cursor.fetchone()
257 if schema:
258 self.database_schema = eval(schema[0])
259 else:
260 self.database_schema = {}
262 def save_dbschema(self):
263 """ Save the schema definition that the database currently implements
264 """
265 s = repr(self.database_schema)
266 self.sql('delete from schema')
267 self.sql('insert into schema values (%s)'%self.arg, (s,))
269 def post_init(self):
270 """ Called once the schema initialisation has finished.
272 We should now confirm that the schema defined by our "classes"
273 attribute actually matches the schema in the database.
274 """
275 save = 0
277 # handle changes in the schema
278 tables = self.database_schema['tables']
279 for classname, spec in self.classes.iteritems():
280 if classname in tables:
281 dbspec = tables[classname]
282 if self.update_class(spec, dbspec):
283 tables[classname] = spec.schema()
284 save = 1
285 else:
286 self.create_class(spec)
287 tables[classname] = spec.schema()
288 save = 1
290 for classname, spec in list(tables.items()):
291 if classname not in self.classes:
292 self.drop_class(classname, tables[classname])
293 del tables[classname]
294 save = 1
296 # now upgrade the database for column type changes, new internal
297 # tables, etc.
298 save = save | self.upgrade_db()
300 # update the database version of the schema
301 if save:
302 self.save_dbschema()
304 # reindex the db if necessary
305 if self.indexer.should_reindex():
306 self.reindex()
308 # commit
309 self.sql_commit()
311 # update this number when we need to make changes to the SQL structure
312 # of the backen database
313 current_db_version = 5
314 db_version_updated = False
315 def upgrade_db(self):
316 """ Update the SQL database to reflect changes in the backend code.
318 Return boolean whether we need to save the schema.
319 """
320 version = self.database_schema.get('version', 1)
321 if version > self.current_db_version:
322 raise DatabaseError('attempting to run rev %d DATABASE with rev '
323 '%d CODE!'%(version, self.current_db_version))
324 if version == self.current_db_version:
325 # nothing to do
326 return 0
328 if version < 2:
329 self.log_info('upgrade to version 2')
330 # change the schema structure
331 self.database_schema = {'tables': self.database_schema}
333 # version 1 didn't have the actor column (note that in
334 # MySQL this will also transition the tables to typed columns)
335 self.add_new_columns_v2()
337 # version 1 doesn't have the OTK, session and indexing in the
338 # database
339 self.create_version_2_tables()
341 if version < 3:
342 self.log_info('upgrade to version 3')
343 self.fix_version_2_tables()
345 if version < 4:
346 self.fix_version_3_tables()
348 if version < 5:
349 self.fix_version_4_tables()
351 self.database_schema['version'] = self.current_db_version
352 self.db_version_updated = True
353 return 1
355 def fix_version_3_tables(self):
356 # drop the shorter VARCHAR OTK column and add a new TEXT one
357 for name in ('otk', 'session'):
358 self.sql('DELETE FROM %ss'%name)
359 self.sql('ALTER TABLE %ss DROP %s_value'%(name, name))
360 self.sql('ALTER TABLE %ss ADD %s_value TEXT'%(name, name))
362 def fix_version_2_tables(self):
363 # Default (used by sqlite): NOOP
364 pass
366 def fix_version_4_tables(self):
367 # note this is an explicit call now
368 c = self.cursor
369 for cn, klass in self.classes.iteritems():
370 c.execute('select id from _%s where __retired__<>0'%(cn,))
371 for (id,) in c.fetchall():
372 c.execute('update _%s set __retired__=%s where id=%s'%(cn,
373 self.arg, self.arg), (id, id))
375 if klass.key:
376 self.add_class_key_required_unique_constraint(cn, klass.key)
378 def _convert_journal_tables(self):
379 """Get current journal table contents, drop the table and re-create"""
380 c = self.cursor
381 cols = ','.join('nodeid date tag action params'.split())
382 for klass in self.classes.itervalues():
383 # slurp and drop
384 sql = 'select %s from %s__journal order by date'%(cols,
385 klass.classname)
386 c.execute(sql)
387 contents = c.fetchall()
388 self.drop_journal_table_indexes(klass.classname)
389 c.execute('drop table %s__journal'%klass.classname)
391 # re-create and re-populate
392 self.create_journal_table(klass)
393 a = self.arg
394 sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(
395 klass.classname, cols, a, a, a, a, a)
396 for row in contents:
397 # no data conversion needed
398 self.cursor.execute(sql, row)
400 def _convert_string_properties(self):
401 """Get current Class tables that contain String properties, and
402 convert the VARCHAR columns to TEXT"""
403 c = self.cursor
404 for klass in self.classes.itervalues():
405 # slurp and drop
406 cols, mls = self.determine_columns(list(klass.properties.iteritems()))
407 scols = ','.join([i[0] for i in cols])
408 sql = 'select id,%s from _%s'%(scols, klass.classname)
409 c.execute(sql)
410 contents = c.fetchall()
411 self.drop_class_table_indexes(klass.classname, klass.getkey())
412 c.execute('drop table _%s'%klass.classname)
414 # re-create and re-populate
415 self.create_class_table(klass, create_sequence=0)
416 a = ','.join([self.arg for i in range(len(cols)+1)])
417 sql = 'insert into _%s (id,%s) values (%s)'%(klass.classname,
418 scols, a)
419 for row in contents:
420 l = []
421 for entry in row:
422 # mysql will already be a string - psql needs "help"
423 if entry is not None and not isinstance(entry, type('')):
424 entry = str(entry)
425 l.append(entry)
426 self.cursor.execute(sql, l)
428 def refresh_database(self):
429 self.post_init()
432 def reindex(self, classname=None, show_progress=False):
433 if classname:
434 classes = [self.getclass(classname)]
435 else:
436 classes = list(self.classes.itervalues())
437 for klass in classes:
438 if show_progress:
439 for nodeid in support.Progress('Reindex %s'%klass.classname,
440 klass.list()):
441 klass.index(nodeid)
442 else:
443 for nodeid in klass.list():
444 klass.index(nodeid)
445 self.indexer.save_index()
447 hyperdb_to_sql_datatypes = {
448 hyperdb.String : 'TEXT',
449 hyperdb.Date : 'TIMESTAMP',
450 hyperdb.Link : 'INTEGER',
451 hyperdb.Interval : 'VARCHAR(255)',
452 hyperdb.Password : 'VARCHAR(255)',
453 hyperdb.Boolean : 'BOOLEAN',
454 hyperdb.Number : 'REAL',
455 }
457 def hyperdb_to_sql_datatype(self, propclass):
459 datatype = self.hyperdb_to_sql_datatypes.get(propclass)
460 if datatype:
461 return datatype
463 for k, v in self.hyperdb_to_sql_datatypes.iteritems():
464 if issubclass(propclass, k):
465 return v
467 raise ValueError('%r is not a hyperdb property class' % propclass)
469 def determine_columns(self, properties):
470 """ Figure the column names and multilink properties from the spec
472 "properties" is a list of (name, prop) where prop may be an
473 instance of a hyperdb "type" _or_ a string repr of that type.
474 """
475 cols = [
476 ('_actor', self.hyperdb_to_sql_datatype(hyperdb.Link)),
477 ('_activity', self.hyperdb_to_sql_datatype(hyperdb.Date)),
478 ('_creator', self.hyperdb_to_sql_datatype(hyperdb.Link)),
479 ('_creation', self.hyperdb_to_sql_datatype(hyperdb.Date)),
480 ]
481 mls = []
482 # add the multilinks separately
483 for col, prop in properties:
484 if isinstance(prop, Multilink):
485 mls.append(col)
486 continue
488 if isinstance(prop, type('')):
489 raise ValueError("string property spec!")
490 #and prop.find('Multilink') != -1:
491 #mls.append(col)
493 datatype = self.hyperdb_to_sql_datatype(prop.__class__)
494 cols.append(('_'+col, datatype))
496 # Intervals stored as two columns
497 if isinstance(prop, Interval):
498 cols.append(('__'+col+'_int__', 'BIGINT'))
500 cols.sort()
501 return cols, mls
503 def update_class(self, spec, old_spec, force=0):
504 """ Determine the differences between the current spec and the
505 database version of the spec, and update where necessary.
507 If 'force' is true, update the database anyway.
508 """
509 new_spec = spec.schema()
510 new_spec[1].sort()
511 old_spec[1].sort()
512 if not force and new_spec == old_spec:
513 # no changes
514 return 0
516 if not self.config.RDBMS_ALLOW_ALTER:
517 raise DatabaseError(_('ALTER operation disallowed: %r -> %r.'%(old_spec, new_spec)))
519 logger = logging.getLogger('roundup.hyperdb')
520 logger.info('update_class %s'%spec.classname)
522 logger.debug('old_spec %r'%(old_spec,))
523 logger.debug('new_spec %r'%(new_spec,))
525 # detect key prop change for potential index change
526 keyprop_changes = {}
527 if new_spec[0] != old_spec[0]:
528 if old_spec[0]:
529 keyprop_changes['remove'] = old_spec[0]
530 if new_spec[0]:
531 keyprop_changes['add'] = new_spec[0]
533 # detect multilinks that have been removed, and drop their table
534 old_has = {}
535 for name, prop in old_spec[1]:
536 old_has[name] = 1
537 if name in spec.properties:
538 continue
540 if prop.find('Multilink to') != -1:
541 # first drop indexes.
542 self.drop_multilink_table_indexes(spec.classname, name)
544 # now the multilink table itself
545 sql = 'drop table %s_%s'%(spec.classname, name)
546 else:
547 # if this is the key prop, drop the index first
548 if old_spec[0] == prop:
549 self.drop_class_table_key_index(spec.classname, name)
550 del keyprop_changes['remove']
552 # drop the column
553 sql = 'alter table _%s drop column _%s'%(spec.classname, name)
555 self.sql(sql)
557 # if we didn't remove the key prop just then, but the key prop has
558 # changed, we still need to remove the old index
559 if 'remove' in keyprop_changes:
560 self.drop_class_table_key_index(spec.classname,
561 keyprop_changes['remove'])
563 # add new columns
564 for propname, prop in new_spec[1]:
565 if propname in old_has:
566 continue
567 prop = spec.properties[propname]
568 if isinstance(prop, Multilink):
569 self.create_multilink_table(spec, propname)
570 else:
571 # add the column
572 coltype = self.hyperdb_to_sql_datatype(prop.__class__)
573 sql = 'alter table _%s add column _%s %s'%(
574 spec.classname, propname, coltype)
575 self.sql(sql)
577 # extra Interval column
578 if isinstance(prop, Interval):
579 sql = 'alter table _%s add column __%s_int__ BIGINT'%(
580 spec.classname, propname)
581 self.sql(sql)
583 # if the new column is a key prop, we need an index!
584 if new_spec[0] == propname:
585 self.create_class_table_key_index(spec.classname, propname)
586 del keyprop_changes['add']
588 # if we didn't add the key prop just then, but the key prop has
589 # changed, we still need to add the new index
590 if 'add' in keyprop_changes:
591 self.create_class_table_key_index(spec.classname,
592 keyprop_changes['add'])
594 return 1
596 def determine_all_columns(self, spec):
597 """Figure out the columns from the spec and also add internal columns
599 """
600 cols, mls = self.determine_columns(list(spec.properties.iteritems()))
602 # add on our special columns
603 cols.append(('id', 'INTEGER PRIMARY KEY'))
604 cols.append(('__retired__', 'INTEGER DEFAULT 0'))
605 return cols, mls
607 def create_class_table(self, spec):
608 """Create the class table for the given Class "spec". Creates the
609 indexes too."""
610 cols, mls = self.determine_all_columns(spec)
612 # create the base table
613 scols = ','.join(['%s %s'%x for x in cols])
614 sql = 'create table _%s (%s)'%(spec.classname, scols)
615 self.sql(sql)
617 self.create_class_table_indexes(spec)
619 return cols, mls
621 def create_class_table_indexes(self, spec):
622 """ create the class table for the given spec
623 """
624 # create __retired__ index
625 index_sql2 = 'create index _%s_retired_idx on _%s(__retired__)'%(
626 spec.classname, spec.classname)
627 self.sql(index_sql2)
629 # create index for key property
630 if spec.key:
631 index_sql3 = 'create index _%s_%s_idx on _%s(_%s)'%(
632 spec.classname, spec.key,
633 spec.classname, spec.key)
634 self.sql(index_sql3)
636 # and the unique index for key / retired(id)
637 self.add_class_key_required_unique_constraint(spec.classname,
638 spec.key)
640 # TODO: create indexes on (selected?) Link property columns, as
641 # they're more likely to be used for lookup
643 def add_class_key_required_unique_constraint(self, cn, key):
644 sql = '''create unique index _%s_key_retired_idx
645 on _%s(__retired__, _%s)'''%(cn, cn, key)
646 self.sql(sql)
648 def drop_class_table_indexes(self, cn, key):
649 # drop the old table indexes first
650 l = ['_%s_id_idx'%cn, '_%s_retired_idx'%cn]
651 if key:
652 l.append('_%s_%s_idx'%(cn, key))
654 table_name = '_%s'%cn
655 for index_name in l:
656 if not self.sql_index_exists(table_name, index_name):
657 continue
658 index_sql = 'drop index '+index_name
659 self.sql(index_sql)
661 def create_class_table_key_index(self, cn, key):
662 """ create the class table for the given spec
663 """
664 sql = 'create index _%s_%s_idx on _%s(_%s)'%(cn, key, cn, key)
665 self.sql(sql)
667 def drop_class_table_key_index(self, cn, key):
668 table_name = '_%s'%cn
669 index_name = '_%s_%s_idx'%(cn, key)
670 if self.sql_index_exists(table_name, index_name):
671 sql = 'drop index '+index_name
672 self.sql(sql)
674 # and now the retired unique index too
675 index_name = '_%s_key_retired_idx'%cn
676 if self.sql_index_exists(table_name, index_name):
677 sql = 'drop index '+index_name
678 self.sql(sql)
680 def create_journal_table(self, spec):
681 """ create the journal table for a class given the spec and
682 already-determined cols
683 """
684 # journal table
685 cols = ','.join(['%s varchar'%x
686 for x in 'nodeid date tag action params'.split()])
687 sql = """create table %s__journal (
688 nodeid integer, date %s, tag varchar(255),
689 action varchar(255), params text)""" % (spec.classname,
690 self.hyperdb_to_sql_datatype(hyperdb.Date))
691 self.sql(sql)
692 self.create_journal_table_indexes(spec)
694 def create_journal_table_indexes(self, spec):
695 # index on nodeid
696 sql = 'create index %s_journ_idx on %s__journal(nodeid)'%(
697 spec.classname, spec.classname)
698 self.sql(sql)
700 def drop_journal_table_indexes(self, classname):
701 index_name = '%s_journ_idx'%classname
702 if not self.sql_index_exists('%s__journal'%classname, index_name):
703 return
704 index_sql = 'drop index '+index_name
705 self.sql(index_sql)
707 def create_multilink_table(self, spec, ml):
708 """ Create a multilink table for the "ml" property of the class
709 given by the spec
710 """
711 # create the table
712 sql = 'create table %s_%s (linkid INTEGER, nodeid INTEGER)'%(
713 spec.classname, ml)
714 self.sql(sql)
715 self.create_multilink_table_indexes(spec, ml)
717 def create_multilink_table_indexes(self, spec, ml):
718 # create index on linkid
719 index_sql = 'create index %s_%s_l_idx on %s_%s(linkid)'%(
720 spec.classname, ml, spec.classname, ml)
721 self.sql(index_sql)
723 # create index on nodeid
724 index_sql = 'create index %s_%s_n_idx on %s_%s(nodeid)'%(
725 spec.classname, ml, spec.classname, ml)
726 self.sql(index_sql)
728 def drop_multilink_table_indexes(self, classname, ml):
729 l = [
730 '%s_%s_l_idx'%(classname, ml),
731 '%s_%s_n_idx'%(classname, ml)
732 ]
733 table_name = '%s_%s'%(classname, ml)
734 for index_name in l:
735 if not self.sql_index_exists(table_name, index_name):
736 continue
737 index_sql = 'drop index %s'%index_name
738 self.sql(index_sql)
740 def create_class(self, spec):
741 """ Create a database table according to the given spec.
742 """
744 if not self.config.RDBMS_ALLOW_CREATE:
745 raise DatabaseError(_('CREATE operation disallowed: "%s".'%spec.classname))
747 cols, mls = self.create_class_table(spec)
748 self.create_journal_table(spec)
750 # now create the multilink tables
751 for ml in mls:
752 self.create_multilink_table(spec, ml)
754 def drop_class(self, cn, spec):
755 """ Drop the given table from the database.
757 Drop the journal and multilink tables too.
758 """
760 if not self.config.RDBMS_ALLOW_DROP:
761 raise DatabaseError(_('DROP operation disallowed: "%s".'%cn))
763 properties = spec[1]
764 # figure the multilinks
765 mls = []
766 for propname, prop in properties:
767 if isinstance(prop, Multilink):
768 mls.append(propname)
770 # drop class table and indexes
771 self.drop_class_table_indexes(cn, spec[0])
773 self.drop_class_table(cn)
775 # drop journal table and indexes
776 self.drop_journal_table_indexes(cn)
777 sql = 'drop table %s__journal'%cn
778 self.sql(sql)
780 for ml in mls:
781 # drop multilink table and indexes
782 self.drop_multilink_table_indexes(cn, ml)
783 sql = 'drop table %s_%s'%(spec.classname, ml)
784 self.sql(sql)
786 def drop_class_table(self, cn):
787 sql = 'drop table _%s'%cn
788 self.sql(sql)
790 #
791 # Classes
792 #
793 def __getattr__(self, classname):
794 """ A convenient way of calling self.getclass(classname).
795 """
796 if classname in self.classes:
797 return self.classes[classname]
798 raise AttributeError(classname)
800 def addclass(self, cl):
801 """ Add a Class to the hyperdatabase.
802 """
803 cn = cl.classname
804 if cn in self.classes:
805 raise ValueError(cn)
806 self.classes[cn] = cl
808 # add default Edit and View permissions
809 self.security.addPermission(name="Create", klass=cn,
810 description="User is allowed to create "+cn)
811 self.security.addPermission(name="Edit", klass=cn,
812 description="User is allowed to edit "+cn)
813 self.security.addPermission(name="View", klass=cn,
814 description="User is allowed to access "+cn)
816 def getclasses(self):
817 """ Return a list of the names of all existing classes.
818 """
819 return sorted(self.classes)
821 def getclass(self, classname):
822 """Get the Class object representing a particular class.
824 If 'classname' is not a valid class name, a KeyError is raised.
825 """
826 try:
827 return self.classes[classname]
828 except KeyError:
829 raise KeyError('There is no class called "%s"'%classname)
831 def clear(self):
832 """Delete all database contents.
834 Note: I don't commit here, which is different behaviour to the
835 "nuke from orbit" behaviour in the dbs.
836 """
837 logging.getLogger('roundup.hyperdb').info('clear')
838 for cn in self.classes:
839 sql = 'delete from _%s'%cn
840 self.sql(sql)
842 #
843 # Nodes
844 #
846 hyperdb_to_sql_value = {
847 hyperdb.String : str,
848 # fractional seconds by default
849 hyperdb.Date : lambda x: x.formal(sep=' ', sec='%06.3f'),
850 hyperdb.Link : int,
851 hyperdb.Interval : str,
852 hyperdb.Password : str,
853 hyperdb.Boolean : lambda x: x and 'TRUE' or 'FALSE',
854 hyperdb.Number : lambda x: x,
855 hyperdb.Multilink : lambda x: x, # used in journal marshalling
856 }
858 def to_sql_value(self, propklass):
860 fn = self.hyperdb_to_sql_value.get(propklass)
861 if fn:
862 return fn
864 for k, v in self.hyperdb_to_sql_value.iteritems():
865 if issubclass(propklass, k):
866 return v
868 raise ValueError('%r is not a hyperdb property class' % propklass)
870 def _cache_del(self, key):
871 del self.cache[key]
872 self.cache_lru.remove(key)
874 def _cache_refresh(self, key):
875 self.cache_lru.remove(key)
876 self.cache_lru.insert(0, key)
878 def _cache_save(self, key, node):
879 self.cache[key] = node
880 # update the LRU
881 self.cache_lru.insert(0, key)
882 if len(self.cache_lru) > self.cache_size:
883 del self.cache[self.cache_lru.pop()]
885 def addnode(self, classname, nodeid, node):
886 """ Add the specified node to its class's db.
887 """
888 self.log_debug('addnode %s%s %r'%(classname,
889 nodeid, node))
891 # determine the column definitions and multilink tables
892 cl = self.classes[classname]
893 cols, mls = self.determine_columns(list(cl.properties.iteritems()))
895 # we'll be supplied these props if we're doing an import
896 values = node.copy()
897 if 'creator' not in values:
898 # add in the "calculated" properties (dupe so we don't affect
899 # calling code's node assumptions)
900 values['creation'] = values['activity'] = date.Date()
901 values['actor'] = values['creator'] = self.getuid()
903 cl = self.classes[classname]
904 props = cl.getprops(protected=1)
905 del props['id']
907 # default the non-multilink columns
908 for col, prop in props.iteritems():
909 if col not in values:
910 if isinstance(prop, Multilink):
911 values[col] = []
912 else:
913 values[col] = None
915 # clear this node out of the cache if it's in there
916 key = (classname, nodeid)
917 if key in self.cache:
918 self._cache_del(key)
920 # figure the values to insert
921 vals = []
922 for col,dt in cols:
923 # this is somewhat dodgy....
924 if col.endswith('_int__'):
925 # XXX eugh, this test suxxors
926 value = values[col[2:-6]]
927 # this is an Interval special "int" column
928 if value is not None:
929 vals.append(value.as_seconds())
930 else:
931 vals.append(value)
932 continue
934 prop = props[col[1:]]
935 value = values[col[1:]]
936 if value is not None:
937 value = self.to_sql_value(prop.__class__)(value)
938 vals.append(value)
939 vals.append(nodeid)
940 vals = tuple(vals)
942 # make sure the ordering is correct for column name -> column value
943 s = ','.join([self.arg for x in cols]) + ',%s'%self.arg
944 cols = ','.join([col for col,dt in cols]) + ',id'
946 # perform the inserts
947 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
948 self.sql(sql, vals)
950 # insert the multilink rows
951 for col in mls:
952 t = '%s_%s'%(classname, col)
953 for entry in node[col]:
954 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
955 self.arg, self.arg)
956 self.sql(sql, (entry, nodeid))
958 def setnode(self, classname, nodeid, values, multilink_changes={}):
959 """ Change the specified node.
960 """
961 self.log_debug('setnode %s%s %r'
962 % (classname, nodeid, values))
964 # clear this node out of the cache if it's in there
965 key = (classname, nodeid)
966 if key in self.cache:
967 self._cache_del(key)
969 cl = self.classes[classname]
970 props = cl.getprops()
972 cols = []
973 mls = []
974 # add the multilinks separately
975 for col in values:
976 prop = props[col]
977 if isinstance(prop, Multilink):
978 mls.append(col)
979 elif isinstance(prop, Interval):
980 # Intervals store the seconds value too
981 cols.append(col)
982 # extra leading '_' added by code below
983 cols.append('_' +col + '_int__')
984 else:
985 cols.append(col)
986 cols.sort()
988 # figure the values to insert
989 vals = []
990 for col in cols:
991 if col.endswith('_int__'):
992 # XXX eugh, this test suxxors
993 # Intervals store the seconds value too
994 col = col[1:-6]
995 prop = props[col]
996 value = values[col]
997 if value is None:
998 vals.append(None)
999 else:
1000 vals.append(value.as_seconds())
1001 else:
1002 prop = props[col]
1003 value = values[col]
1004 if value is None:
1005 e = None
1006 else:
1007 e = self.to_sql_value(prop.__class__)(value)
1008 vals.append(e)
1010 vals.append(int(nodeid))
1011 vals = tuple(vals)
1013 # if there's any updates to regular columns, do them
1014 if cols:
1015 # make sure the ordering is correct for column name -> column value
1016 s = ','.join(['_%s=%s'%(x, self.arg) for x in cols])
1017 cols = ','.join(cols)
1019 # perform the update
1020 sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
1021 self.sql(sql, vals)
1023 # we're probably coming from an import, not a change
1024 if not multilink_changes:
1025 for name in mls:
1026 prop = props[name]
1027 value = values[name]
1029 t = '%s_%s'%(classname, name)
1031 # clear out previous values for this node
1032 # XXX numeric ids
1033 self.sql('delete from %s where nodeid=%s'%(t, self.arg),
1034 (nodeid,))
1036 # insert the values for this node
1037 for entry in values[name]:
1038 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
1039 self.arg, self.arg)
1040 # XXX numeric ids
1041 self.sql(sql, (entry, nodeid))
1043 # we have multilink changes to apply
1044 for col, (add, remove) in multilink_changes.iteritems():
1045 tn = '%s_%s'%(classname, col)
1046 if add:
1047 sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
1048 self.arg, self.arg)
1049 for addid in add:
1050 # XXX numeric ids
1051 self.sql(sql, (int(nodeid), int(addid)))
1052 if remove:
1053 s = ','.join([self.arg]*len(remove))
1054 sql = 'delete from %s where nodeid=%s and linkid in (%s)'%(tn,
1055 self.arg, s)
1056 # XXX numeric ids
1057 self.sql(sql, [int(nodeid)] + remove)
1059 sql_to_hyperdb_value = {
1060 hyperdb.String : str,
1061 hyperdb.Date : date_to_hyperdb_value,
1062 # hyperdb.Link : int, # XXX numeric ids
1063 hyperdb.Link : str,
1064 hyperdb.Interval : date.Interval,
1065 hyperdb.Password : lambda x: password.Password(encrypted=x),
1066 hyperdb.Boolean : _bool_cvt,
1067 hyperdb.Number : _num_cvt,
1068 hyperdb.Multilink : lambda x: x, # used in journal marshalling
1069 }
1071 def to_hyperdb_value(self, propklass):
1073 fn = self.sql_to_hyperdb_value.get(propklass)
1074 if fn:
1075 return fn
1077 for k, v in self.sql_to_hyperdb_value.iteritems():
1078 if issubclass(propklass, k):
1079 return v
1081 raise ValueError('%r is not a hyperdb property class' % propklass)
1083 def getnode(self, classname, nodeid):
1084 """ Get a node from the database.
1085 """
1086 # see if we have this node cached
1087 key = (classname, nodeid)
1088 if key in self.cache:
1089 # push us back to the top of the LRU
1090 self._cache_refresh(key)
1091 if __debug__:
1092 self.stats['cache_hits'] += 1
1093 # return the cached information
1094 return self.cache[key]
1096 if __debug__:
1097 self.stats['cache_misses'] += 1
1098 start_t = time.time()
1100 # figure the columns we're fetching
1101 cl = self.classes[classname]
1102 cols, mls = self.determine_columns(list(cl.properties.iteritems()))
1103 scols = ','.join([col for col,dt in cols])
1105 # perform the basic property fetch
1106 sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
1107 self.sql(sql, (nodeid,))
1109 values = self.sql_fetchone()
1110 if values is None:
1111 raise IndexError('no such %s node %s'%(classname, nodeid))
1113 # make up the node
1114 node = {}
1115 props = cl.getprops(protected=1)
1116 for col in range(len(cols)):
1117 name = cols[col][0][1:]
1118 if name.endswith('_int__'):
1119 # XXX eugh, this test suxxors
1120 # ignore the special Interval-as-seconds column
1121 continue
1122 value = values[col]
1123 if value is not None:
1124 value = self.to_hyperdb_value(props[name].__class__)(value)
1125 node[name] = value
1127 # save off in the cache
1128 key = (classname, nodeid)
1129 self._cache_save(key, node)
1131 if __debug__:
1132 self.stats['get_items'] += (time.time() - start_t)
1134 return node
1136 def destroynode(self, classname, nodeid):
1137 """Remove a node from the database. Called exclusively by the
1138 destroy() method on Class.
1139 """
1140 logging.getLogger('roundup.hyperdb').info('destroynode %s%s'%(
1141 classname, nodeid))
1143 # make sure the node exists
1144 if not self.hasnode(classname, nodeid):
1145 raise IndexError('%s has no node %s'%(classname, nodeid))
1147 # see if we have this node cached
1148 if (classname, nodeid) in self.cache:
1149 del self.cache[(classname, nodeid)]
1151 # see if there's any obvious commit actions that we should get rid of
1152 for entry in self.transactions[:]:
1153 if entry[1][:2] == (classname, nodeid):
1154 self.transactions.remove(entry)
1156 # now do the SQL
1157 sql = 'delete from _%s where id=%s'%(classname, self.arg)
1158 self.sql(sql, (nodeid,))
1160 # remove from multilnks
1161 cl = self.getclass(classname)
1162 x, mls = self.determine_columns(list(cl.properties.iteritems()))
1163 for col in mls:
1164 # get the link ids
1165 sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
1166 self.sql(sql, (nodeid,))
1168 # remove journal entries
1169 sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
1170 self.sql(sql, (nodeid,))
1172 # cleanup any blob filestorage when we commit
1173 self.transactions.append((FileStorage.destroy, (self, classname, nodeid)))
1175 def hasnode(self, classname, nodeid):
1176 """ Determine if the database has a given node.
1177 """
1178 # If this node is in the cache, then we do not need to go to
1179 # the database. (We don't consider this an LRU hit, though.)
1180 if (classname, nodeid) in self.cache:
1181 # Return 1, not True, to match the type of the result of
1182 # the SQL operation below.
1183 return 1
1184 sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
1185 self.sql(sql, (nodeid,))
1186 return int(self.cursor.fetchone()[0])
1188 def countnodes(self, classname):
1189 """ Count the number of nodes that exist for a particular Class.
1190 """
1191 sql = 'select count(*) from _%s'%classname
1192 self.sql(sql)
1193 return self.cursor.fetchone()[0]
1195 def addjournal(self, classname, nodeid, action, params, creator=None,
1196 creation=None):
1197 """ Journal the Action
1198 'action' may be:
1200 'create' or 'set' -- 'params' is a dictionary of property values
1201 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
1202 'retire' -- 'params' is None
1203 """
1204 # handle supply of the special journalling parameters (usually
1205 # supplied on importing an existing database)
1206 if creator:
1207 journaltag = creator
1208 else:
1209 journaltag = self.getuid()
1210 if creation:
1211 journaldate = creation
1212 else:
1213 journaldate = date.Date()
1215 # create the journal entry
1216 cols = 'nodeid,date,tag,action,params'
1218 self.log_debug('addjournal %s%s %r %s %s %r'%(classname,
1219 nodeid, journaldate, journaltag, action, params))
1221 # make the journalled data marshallable
1222 if isinstance(params, type({})):
1223 self._journal_marshal(params, classname)
1225 params = repr(params)
1227 dc = self.to_sql_value(hyperdb.Date)
1228 journaldate = dc(journaldate)
1230 self.save_journal(classname, cols, nodeid, journaldate,
1231 journaltag, action, params)
1233 def setjournal(self, classname, nodeid, journal):
1234 """Set the journal to the "journal" list."""
1235 # clear out any existing entries
1236 self.sql('delete from %s__journal where nodeid=%s'%(classname,
1237 self.arg), (nodeid,))
1239 # create the journal entry
1240 cols = 'nodeid,date,tag,action,params'
1242 dc = self.to_sql_value(hyperdb.Date)
1243 for nodeid, journaldate, journaltag, action, params in journal:
1244 self.log_debug('addjournal %s%s %r %s %s %r'%(
1245 classname, nodeid, journaldate, journaltag, action,
1246 params))
1248 # make the journalled data marshallable
1249 if isinstance(params, type({})):
1250 self._journal_marshal(params, classname)
1251 params = repr(params)
1253 self.save_journal(classname, cols, nodeid, dc(journaldate),
1254 journaltag, action, params)
1256 def _journal_marshal(self, params, classname):
1257 """Convert the journal params values into safely repr'able and
1258 eval'able values."""
1259 properties = self.getclass(classname).getprops()
1260 for param, value in params.iteritems():
1261 if not value:
1262 continue
1263 property = properties[param]
1264 cvt = self.to_sql_value(property.__class__)
1265 if isinstance(property, Password):
1266 params[param] = cvt(value)
1267 elif isinstance(property, Date):
1268 params[param] = cvt(value)
1269 elif isinstance(property, Interval):
1270 params[param] = cvt(value)
1271 elif isinstance(property, Boolean):
1272 params[param] = cvt(value)
1274 def getjournal(self, classname, nodeid):
1275 """ get the journal for id
1276 """
1277 # make sure the node exists
1278 if not self.hasnode(classname, nodeid):
1279 raise IndexError('%s has no node %s'%(classname, nodeid))
1281 cols = ','.join('nodeid date tag action params'.split())
1282 journal = self.load_journal(classname, cols, nodeid)
1284 # now unmarshal the data
1285 dc = self.to_hyperdb_value(hyperdb.Date)
1286 res = []
1287 properties = self.getclass(classname).getprops()
1288 for nodeid, date_stamp, user, action, params in journal:
1289 params = eval(params)
1290 if isinstance(params, type({})):
1291 for param, value in params.iteritems():
1292 if not value:
1293 continue
1294 property = properties.get(param, None)
1295 if property is None:
1296 # deleted property
1297 continue
1298 cvt = self.to_hyperdb_value(property.__class__)
1299 if isinstance(property, Password):
1300 params[param] = cvt(value)
1301 elif isinstance(property, Date):
1302 params[param] = cvt(value)
1303 elif isinstance(property, Interval):
1304 params[param] = cvt(value)
1305 elif isinstance(property, Boolean):
1306 params[param] = cvt(value)
1307 # XXX numeric ids
1308 res.append((str(nodeid), dc(date_stamp), user, action, params))
1309 return res
1311 def save_journal(self, classname, cols, nodeid, journaldate,
1312 journaltag, action, params):
1313 """ Save the journal entry to the database
1314 """
1315 entry = (nodeid, journaldate, journaltag, action, params)
1317 # do the insert
1318 a = self.arg
1319 sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(
1320 classname, cols, a, a, a, a, a)
1321 self.sql(sql, entry)
1323 def load_journal(self, classname, cols, nodeid):
1324 """ Load the journal from the database
1325 """
1326 # now get the journal entries
1327 sql = 'select %s from %s__journal where nodeid=%s order by date'%(
1328 cols, classname, self.arg)
1329 self.sql(sql, (nodeid,))
1330 return self.cursor.fetchall()
1332 def pack(self, pack_before):
1333 """ Delete all journal entries except "create" before 'pack_before'.
1334 """
1335 date_stamp = self.to_sql_value(Date)(pack_before)
1337 # do the delete
1338 for classname in self.classes:
1339 sql = "delete from %s__journal where date<%s and "\
1340 "action<>'create'"%(classname, self.arg)
1341 self.sql(sql, (date_stamp,))
1343 def sql_commit(self, fail_ok=False):
1344 """ Actually commit to the database.
1345 """
1346 logging.getLogger('roundup.hyperdb').info('commit')
1348 self.conn.commit()
1350 # open a new cursor for subsequent work
1351 self.cursor = self.conn.cursor()
1353 def commit(self, fail_ok=False):
1354 """ Commit the current transactions.
1356 Save all data changed since the database was opened or since the
1357 last commit() or rollback().
1359 fail_ok indicates that the commit is allowed to fail. This is used
1360 in the web interface when committing cleaning of the session
1361 database. We don't care if there's a concurrency issue there.
1363 The only backend this seems to affect is postgres.
1364 """
1365 # commit the database
1366 self.sql_commit(fail_ok)
1368 # now, do all the other transaction stuff
1369 for method, args in self.transactions:
1370 method(*args)
1372 # save the indexer
1373 self.indexer.save_index()
1375 # clear out the transactions
1376 self.transactions = []
1378 # clear the cache: Don't carry over cached values from one
1379 # transaction to the next (there may be other changes from other
1380 # transactions)
1381 self.clearCache()
1383 def sql_rollback(self):
1384 self.conn.rollback()
1386 def rollback(self):
1387 """ Reverse all actions from the current transaction.
1389 Undo all the changes made since the database was opened or the last
1390 commit() or rollback() was performed.
1391 """
1392 logging.getLogger('roundup.hyperdb').info('rollback')
1394 self.sql_rollback()
1396 # roll back "other" transaction stuff
1397 for method, args in self.transactions:
1398 # delete temporary files
1399 if method == self.doStoreFile:
1400 self.rollbackStoreFile(*args)
1401 self.transactions = []
1403 # clear the cache
1404 self.clearCache()
1406 def sql_close(self):
1407 logging.getLogger('roundup.hyperdb').info('close')
1408 self.conn.close()
1410 def close(self):
1411 """ Close off the connection.
1412 """
1413 self.indexer.close()
1414 self.sql_close()
1416 #
1417 # The base Class class
1418 #
1419 class Class(hyperdb.Class):
1420 """ The handle to a particular class of nodes in a hyperdatabase.
1422 All methods except __repr__ and getnode must be implemented by a
1423 concrete backend Class.
1424 """
1426 def schema(self):
1427 """ A dumpable version of the schema that we can store in the
1428 database
1429 """
1430 return (self.key, [(x, repr(y)) for x,y in self.properties.iteritems()])
1432 def enableJournalling(self):
1433 """Turn journalling on for this class
1434 """
1435 self.do_journal = 1
1437 def disableJournalling(self):
1438 """Turn journalling off for this class
1439 """
1440 self.do_journal = 0
1442 # Editing nodes:
1443 def create(self, **propvalues):
1444 """ Create a new node of this class and return its id.
1446 The keyword arguments in 'propvalues' map property names to values.
1448 The values of arguments must be acceptable for the types of their
1449 corresponding properties or a TypeError is raised.
1451 If this class has a key property, it must be present and its value
1452 must not collide with other key strings or a ValueError is raised.
1454 Any other properties on this class that are missing from the
1455 'propvalues' dictionary are set to None.
1457 If an id in a link or multilink property does not refer to a valid
1458 node, an IndexError is raised.
1459 """
1460 self.fireAuditors('create', None, propvalues)
1461 newid = self.create_inner(**propvalues)
1462 self.fireReactors('create', newid, None)
1463 return newid
1465 def create_inner(self, **propvalues):
1466 """ Called by create, in-between the audit and react calls.
1467 """
1468 if 'id' in propvalues:
1469 raise KeyError('"id" is reserved')
1471 if self.db.journaltag is None:
1472 raise DatabaseError(_('Database open read-only'))
1474 if ('creator' in propvalues or 'actor' in propvalues or
1475 'creation' in propvalues or 'activity' in propvalues):
1476 raise KeyError('"creator", "actor", "creation" and '
1477 '"activity" are reserved')
1479 # new node's id
1480 newid = self.db.newid(self.classname)
1482 # validate propvalues
1483 num_re = re.compile('^\d+$')
1484 for key, value in propvalues.iteritems():
1485 if key == self.key:
1486 try:
1487 self.lookup(value)
1488 except KeyError:
1489 pass
1490 else:
1491 raise ValueError('node with key "%s" exists'%value)
1493 # try to handle this property
1494 try:
1495 prop = self.properties[key]
1496 except KeyError:
1497 raise KeyError('"%s" has no property "%s"'%(self.classname,
1498 key))
1500 if value is not None and isinstance(prop, Link):
1501 if type(value) != type(''):
1502 raise ValueError('link value must be String')
1503 link_class = self.properties[key].classname
1504 # if it isn't a number, it's a key
1505 if not num_re.match(value):
1506 try:
1507 value = self.db.classes[link_class].lookup(value)
1508 except (TypeError, KeyError):
1509 raise IndexError('new property "%s": %s not a %s'%(
1510 key, value, link_class))
1511 elif not self.db.getclass(link_class).hasnode(value):
1512 raise IndexError('%s has no node %s'%(link_class,
1513 value))
1515 # save off the value
1516 propvalues[key] = value
1518 # register the link with the newly linked node
1519 if self.do_journal and self.properties[key].do_journal:
1520 self.db.addjournal(link_class, value, 'link',
1521 (self.classname, newid, key))
1523 elif isinstance(prop, Multilink):
1524 if value is None:
1525 value = []
1526 if not hasattr(value, '__iter__'):
1527 raise TypeError('new property "%s" not an iterable of ids'%key)
1528 # clean up and validate the list of links
1529 link_class = self.properties[key].classname
1530 l = []
1531 for entry in value:
1532 if type(entry) != type(''):
1533 raise ValueError('"%s" multilink value (%r) '
1534 'must contain Strings'%(key, value))
1535 # if it isn't a number, it's a key
1536 if not num_re.match(entry):
1537 try:
1538 entry = self.db.classes[link_class].lookup(entry)
1539 except (TypeError, KeyError):
1540 raise IndexError('new property "%s": %s not a %s'%(
1541 key, entry, self.properties[key].classname))
1542 l.append(entry)
1543 value = l
1544 propvalues[key] = value
1546 # handle additions
1547 for nodeid in value:
1548 if not self.db.getclass(link_class).hasnode(nodeid):
1549 raise IndexError('%s has no node %s'%(link_class,
1550 nodeid))
1551 # register the link with the newly linked node
1552 if self.do_journal and self.properties[key].do_journal:
1553 self.db.addjournal(link_class, nodeid, 'link',
1554 (self.classname, newid, key))
1556 elif isinstance(prop, String):
1557 if type(value) != type('') and type(value) != type(u''):
1558 raise TypeError('new property "%s" not a string'%key)
1559 if prop.indexme:
1560 self.db.indexer.add_text((self.classname, newid, key),
1561 value)
1563 elif isinstance(prop, Password):
1564 if not isinstance(value, password.Password):
1565 raise TypeError('new property "%s" not a Password'%key)
1567 elif isinstance(prop, Date):
1568 if value is not None and not isinstance(value, date.Date):
1569 raise TypeError('new property "%s" not a Date'%key)
1571 elif isinstance(prop, Interval):
1572 if value is not None and not isinstance(value, date.Interval):
1573 raise TypeError('new property "%s" not an Interval'%key)
1575 elif value is not None and isinstance(prop, Number):
1576 try:
1577 float(value)
1578 except ValueError:
1579 raise TypeError('new property "%s" not numeric'%key)
1581 elif value is not None and isinstance(prop, Boolean):
1582 try:
1583 int(value)
1584 except ValueError:
1585 raise TypeError('new property "%s" not boolean'%key)
1587 # make sure there's data where there needs to be
1588 for key, prop in self.properties.iteritems():
1589 if key in propvalues:
1590 continue
1591 if key == self.key:
1592 raise ValueError('key property "%s" is required'%key)
1593 if isinstance(prop, Multilink):
1594 propvalues[key] = []
1595 else:
1596 propvalues[key] = None
1598 # done
1599 self.db.addnode(self.classname, newid, propvalues)
1600 if self.do_journal:
1601 self.db.addjournal(self.classname, newid, ''"create", {})
1603 # XXX numeric ids
1604 return str(newid)
1606 def get(self, nodeid, propname, default=_marker, cache=1):
1607 """Get the value of a property on an existing node of this class.
1609 'nodeid' must be the id of an existing node of this class or an
1610 IndexError is raised. 'propname' must be the name of a property
1611 of this class or a KeyError is raised.
1613 'cache' exists for backwards compatibility, and is not used.
1614 """
1615 if propname == 'id':
1616 return nodeid
1618 # get the node's dict
1619 d = self.db.getnode(self.classname, nodeid)
1621 if propname == 'creation':
1622 if 'creation' in d:
1623 return d['creation']
1624 else:
1625 return date.Date()
1626 if propname == 'activity':
1627 if 'activity' in d:
1628 return d['activity']
1629 else:
1630 return date.Date()
1631 if propname == 'creator':
1632 if 'creator' in d:
1633 return d['creator']
1634 else:
1635 return self.db.getuid()
1636 if propname == 'actor':
1637 if 'actor' in d:
1638 return d['actor']
1639 else:
1640 return self.db.getuid()
1642 # get the property (raises KeyError if invalid)
1643 prop = self.properties[propname]
1645 # lazy evaluation of Multilink
1646 if propname not in d and isinstance(prop, Multilink):
1647 sql = 'select linkid from %s_%s where nodeid=%s'%(self.classname,
1648 propname, self.db.arg)
1649 self.db.sql(sql, (nodeid,))
1650 # extract the first column from the result
1651 # XXX numeric ids
1652 items = [int(x[0]) for x in self.db.cursor.fetchall()]
1653 items.sort ()
1654 d[propname] = [str(x) for x in items]
1656 # handle there being no value in the table for the property
1657 if propname not in d or d[propname] is None:
1658 if default is _marker:
1659 if isinstance(prop, Multilink):
1660 return []
1661 else:
1662 return None
1663 else:
1664 return default
1666 # don't pass our list to other code
1667 if isinstance(prop, Multilink):
1668 return d[propname][:]
1670 return d[propname]
1672 def set(self, nodeid, **propvalues):
1673 """Modify a property on an existing node of this class.
1675 'nodeid' must be the id of an existing node of this class or an
1676 IndexError is raised.
1678 Each key in 'propvalues' must be the name of a property of this
1679 class or a KeyError is raised.
1681 All values in 'propvalues' must be acceptable types for their
1682 corresponding properties or a TypeError is raised.
1684 If the value of the key property is set, it must not collide with
1685 other key strings or a ValueError is raised.
1687 If the value of a Link or Multilink property contains an invalid
1688 node id, a ValueError is raised.
1689 """
1690 self.fireAuditors('set', nodeid, propvalues)
1691 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1692 propvalues = self.set_inner(nodeid, **propvalues)
1693 self.fireReactors('set', nodeid, oldvalues)
1694 return propvalues
1696 def set_inner(self, nodeid, **propvalues):
1697 """ Called by set, in-between the audit and react calls.
1698 """
1699 if not propvalues:
1700 return propvalues
1702 if ('creator' in propvalues or 'actor' in propvalues or
1703 'creation' in propvalues or 'activity' in propvalues):
1704 raise KeyError('"creator", "actor", "creation" and '
1705 '"activity" are reserved')
1707 if 'id' in propvalues:
1708 raise KeyError('"id" is reserved')
1710 if self.db.journaltag is None:
1711 raise DatabaseError(_('Database open read-only'))
1713 node = self.db.getnode(self.classname, nodeid)
1714 if self.is_retired(nodeid):
1715 raise IndexError('Requested item is retired')
1716 num_re = re.compile('^\d+$')
1718 # make a copy of the values dictionary - we'll modify the contents
1719 propvalues = propvalues.copy()
1721 # if the journal value is to be different, store it in here
1722 journalvalues = {}
1724 # remember the add/remove stuff for multilinks, making it easier
1725 # for the Database layer to do its stuff
1726 multilink_changes = {}
1728 for propname, value in list(propvalues.items()):
1729 # check to make sure we're not duplicating an existing key
1730 if propname == self.key and node[propname] != value:
1731 try:
1732 self.lookup(value)
1733 except KeyError:
1734 pass
1735 else:
1736 raise ValueError('node with key "%s" exists'%value)
1738 # this will raise the KeyError if the property isn't valid
1739 # ... we don't use getprops() here because we only care about
1740 # the writeable properties.
1741 try:
1742 prop = self.properties[propname]
1743 except KeyError:
1744 raise KeyError('"%s" has no property named "%s"'%(
1745 self.classname, propname))
1747 # if the value's the same as the existing value, no sense in
1748 # doing anything
1749 current = node.get(propname, None)
1750 if value == current:
1751 del propvalues[propname]
1752 continue
1753 journalvalues[propname] = current
1755 # do stuff based on the prop type
1756 if isinstance(prop, Link):
1757 link_class = prop.classname
1758 # if it isn't a number, it's a key
1759 if value is not None and not isinstance(value, type('')):
1760 raise ValueError('property "%s" link value be a string'%(
1761 propname))
1762 if isinstance(value, type('')) and not num_re.match(value):
1763 try:
1764 value = self.db.classes[link_class].lookup(value)
1765 except (TypeError, KeyError):
1766 raise IndexError('new property "%s": %s not a %s'%(
1767 propname, value, prop.classname))
1769 if (value is not None and
1770 not self.db.getclass(link_class).hasnode(value)):
1771 raise IndexError('%s has no node %s'%(link_class,
1772 value))
1774 if self.do_journal and prop.do_journal:
1775 # register the unlink with the old linked node
1776 if node[propname] is not None:
1777 self.db.addjournal(link_class, node[propname],
1778 ''"unlink", (self.classname, nodeid, propname))
1780 # register the link with the newly linked node
1781 if value is not None:
1782 self.db.addjournal(link_class, value, ''"link",
1783 (self.classname, nodeid, propname))
1785 elif isinstance(prop, Multilink):
1786 if value is None:
1787 value = []
1788 if not hasattr(value, '__iter__'):
1789 raise TypeError('new property "%s" not an iterable of'
1790 ' ids'%propname)
1791 link_class = self.properties[propname].classname
1792 l = []
1793 for entry in value:
1794 # if it isn't a number, it's a key
1795 if type(entry) != type(''):
1796 raise ValueError('new property "%s" link value '
1797 'must be a string'%propname)
1798 if not num_re.match(entry):
1799 try:
1800 entry = self.db.classes[link_class].lookup(entry)
1801 except (TypeError, KeyError):
1802 raise IndexError('new property "%s": %s not a %s'%(
1803 propname, entry,
1804 self.properties[propname].classname))
1805 l.append(entry)
1806 value = l
1807 propvalues[propname] = value
1809 # figure the journal entry for this property
1810 add = []
1811 remove = []
1813 # handle removals
1814 if propname in node:
1815 l = node[propname]
1816 else:
1817 l = []
1818 for id in l[:]:
1819 if id in value:
1820 continue
1821 # register the unlink with the old linked node
1822 if self.do_journal and self.properties[propname].do_journal:
1823 self.db.addjournal(link_class, id, 'unlink',
1824 (self.classname, nodeid, propname))
1825 l.remove(id)
1826 remove.append(id)
1828 # handle additions
1829 for id in value:
1830 if id in l:
1831 continue
1832 # We can safely check this condition after
1833 # checking that this is an addition to the
1834 # multilink since the condition was checked for
1835 # existing entries at the point they were added to
1836 # the multilink. Since the hasnode call will
1837 # result in a SQL query, it is more efficient to
1838 # avoid the check if possible.
1839 if not self.db.getclass(link_class).hasnode(id):
1840 raise IndexError('%s has no node %s'%(link_class,
1841 id))
1842 # register the link with the newly linked node
1843 if self.do_journal and self.properties[propname].do_journal:
1844 self.db.addjournal(link_class, id, 'link',
1845 (self.classname, nodeid, propname))
1846 l.append(id)
1847 add.append(id)
1849 # figure the journal entry
1850 l = []
1851 if add:
1852 l.append(('+', add))
1853 if remove:
1854 l.append(('-', remove))
1855 multilink_changes[propname] = (add, remove)
1856 if l:
1857 journalvalues[propname] = tuple(l)
1859 elif isinstance(prop, String):
1860 if value is not None and type(value) != type('') and type(value) != type(u''):
1861 raise TypeError('new property "%s" not a string'%propname)
1862 if prop.indexme:
1863 if value is None: value = ''
1864 self.db.indexer.add_text((self.classname, nodeid, propname),
1865 value)
1867 elif isinstance(prop, Password):
1868 if not isinstance(value, password.Password):
1869 raise TypeError('new property "%s" not a Password'%propname)
1870 propvalues[propname] = value
1872 elif value is not None and isinstance(prop, Date):
1873 if not isinstance(value, date.Date):
1874 raise TypeError('new property "%s" not a Date'% propname)
1875 propvalues[propname] = value
1877 elif value is not None and isinstance(prop, Interval):
1878 if not isinstance(value, date.Interval):
1879 raise TypeError('new property "%s" not an '
1880 'Interval'%propname)
1881 propvalues[propname] = value
1883 elif value is not None and isinstance(prop, Number):
1884 try:
1885 float(value)
1886 except ValueError:
1887 raise TypeError('new property "%s" not numeric'%propname)
1889 elif value is not None and isinstance(prop, Boolean):
1890 try:
1891 int(value)
1892 except ValueError:
1893 raise TypeError('new property "%s" not boolean'%propname)
1895 # nothing to do?
1896 if not propvalues:
1897 return propvalues
1899 # update the activity time
1900 propvalues['activity'] = date.Date()
1901 propvalues['actor'] = self.db.getuid()
1903 # do the set
1904 self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
1906 # remove the activity props now they're handled
1907 del propvalues['activity']
1908 del propvalues['actor']
1910 # journal the set
1911 if self.do_journal:
1912 self.db.addjournal(self.classname, nodeid, ''"set", journalvalues)
1914 return propvalues
1916 def retire(self, nodeid):
1917 """Retire a node.
1919 The properties on the node remain available from the get() method,
1920 and the node's id is never reused.
1922 Retired nodes are not returned by the find(), list(), or lookup()
1923 methods, and other nodes may reuse the values of their key properties.
1924 """
1925 if self.db.journaltag is None:
1926 raise DatabaseError(_('Database open read-only'))
1928 self.fireAuditors('retire', nodeid, None)
1930 # use the arg for __retired__ to cope with any odd database type
1931 # conversion (hello, sqlite)
1932 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1933 self.db.arg, self.db.arg)
1934 self.db.sql(sql, (nodeid, nodeid))
1935 if self.do_journal:
1936 self.db.addjournal(self.classname, nodeid, ''"retired", None)
1938 self.fireReactors('retire', nodeid, None)
1940 def restore(self, nodeid):
1941 """Restore a retired node.
1943 Make node available for all operations like it was before retirement.
1944 """
1945 if self.db.journaltag is None:
1946 raise DatabaseError(_('Database open read-only'))
1948 node = self.db.getnode(self.classname, nodeid)
1949 # check if key property was overrided
1950 key = self.getkey()
1951 try:
1952 id = self.lookup(node[key])
1953 except KeyError:
1954 pass
1955 else:
1956 raise KeyError("Key property (%s) of retired node clashes "
1957 "with existing one (%s)" % (key, node[key]))
1959 self.fireAuditors('restore', nodeid, None)
1960 # use the arg for __retired__ to cope with any odd database type
1961 # conversion (hello, sqlite)
1962 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1963 self.db.arg, self.db.arg)
1964 self.db.sql(sql, (0, nodeid))
1965 if self.do_journal:
1966 self.db.addjournal(self.classname, nodeid, ''"restored", None)
1968 self.fireReactors('restore', nodeid, None)
1970 def is_retired(self, nodeid):
1971 """Return true if the node is rerired
1972 """
1973 sql = 'select __retired__ from _%s where id=%s'%(self.classname,
1974 self.db.arg)
1975 self.db.sql(sql, (nodeid,))
1976 return int(self.db.sql_fetchone()[0]) > 0
1978 def destroy(self, nodeid):
1979 """Destroy a node.
1981 WARNING: this method should never be used except in extremely rare
1982 situations where there could never be links to the node being
1983 deleted
1985 WARNING: use retire() instead
1987 WARNING: the properties of this node will not be available ever again
1989 WARNING: really, use retire() instead
1991 Well, I think that's enough warnings. This method exists mostly to
1992 support the session storage of the cgi interface.
1994 The node is completely removed from the hyperdb, including all journal
1995 entries. It will no longer be available, and will generally break code
1996 if there are any references to the node.
1997 """
1998 if self.db.journaltag is None:
1999 raise DatabaseError(_('Database open read-only'))
2000 self.db.destroynode(self.classname, nodeid)
2002 def history(self, nodeid):
2003 """Retrieve the journal of edits on a particular node.
2005 'nodeid' must be the id of an existing node of this class or an
2006 IndexError is raised.
2008 The returned list contains tuples of the form
2010 (nodeid, date, tag, action, params)
2012 'date' is a Timestamp object specifying the time of the change and
2013 'tag' is the journaltag specified when the database was opened.
2014 """
2015 if not self.do_journal:
2016 raise ValueError('Journalling is disabled for this class')
2017 return self.db.getjournal(self.classname, nodeid)
2019 # Locating nodes:
2020 def hasnode(self, nodeid):
2021 """Determine if the given nodeid actually exists
2022 """
2023 return self.db.hasnode(self.classname, nodeid)
2025 def setkey(self, propname):
2026 """Select a String property of this class to be the key property.
2028 'propname' must be the name of a String property of this class or
2029 None, or a TypeError is raised. The values of the key property on
2030 all existing nodes must be unique or a ValueError is raised.
2031 """
2032 prop = self.getprops()[propname]
2033 if not isinstance(prop, String):
2034 raise TypeError('key properties must be String')
2035 self.key = propname
2037 def getkey(self):
2038 """Return the name of the key property for this class or None."""
2039 return self.key
2041 def lookup(self, keyvalue):
2042 """Locate a particular node by its key property and return its id.
2044 If this class has no key property, a TypeError is raised. If the
2045 'keyvalue' matches one of the values for the key property among
2046 the nodes in this class, the matching node's id is returned;
2047 otherwise a KeyError is raised.
2048 """
2049 if not self.key:
2050 raise TypeError('No key property set for class %s'%self.classname)
2052 # use the arg to handle any odd database type conversion (hello,
2053 # sqlite)
2054 sql = "select id from _%s where _%s=%s and __retired__=%s"%(
2055 self.classname, self.key, self.db.arg, self.db.arg)
2056 self.db.sql(sql, (str(keyvalue), 0))
2058 # see if there was a result that's not retired
2059 row = self.db.sql_fetchone()
2060 if not row:
2061 raise KeyError('No key (%s) value "%s" for "%s"'%(self.key,
2062 keyvalue, self.classname))
2064 # return the id
2065 # XXX numeric ids
2066 return str(row[0])
2068 def find(self, **propspec):
2069 """Get the ids of nodes in this class which link to the given nodes.
2071 'propspec' consists of keyword args propname=nodeid or
2072 propname={nodeid:1, }
2073 'propname' must be the name of a property in this class, or a
2074 KeyError is raised. That property must be a Link or
2075 Multilink property, or a TypeError is raised.
2077 Any node in this class whose 'propname' property links to any of
2078 the nodeids will be returned. Examples::
2080 db.issue.find(messages='1')
2081 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
2082 """
2083 # shortcut
2084 if not propspec:
2085 return []
2087 # validate the args
2088 props = self.getprops()
2089 for propname, nodeids in propspec.iteritems():
2090 # check the prop is OK
2091 prop = props[propname]
2092 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
2093 raise TypeError("'%s' not a Link/Multilink property"%propname)
2095 # first, links
2096 a = self.db.arg
2097 allvalues = ()
2098 sql = []
2099 where = []
2100 for prop, values in propspec.iteritems():
2101 if not isinstance(props[prop], hyperdb.Link):
2102 continue
2103 if type(values) is type({}) and len(values) == 1:
2104 values = list(values)[0]
2105 if type(values) is type(''):
2106 allvalues += (values,)
2107 where.append('_%s = %s'%(prop, a))
2108 elif values is None:
2109 where.append('_%s is NULL'%prop)
2110 else:
2111 values = list(values)
2112 s = ''
2113 if None in values:
2114 values.remove(None)
2115 s = '_%s is NULL or '%prop
2116 allvalues += tuple(values)
2117 s += '_%s in (%s)'%(prop, ','.join([a]*len(values)))
2118 where.append('(' + s +')')
2119 if where:
2120 allvalues = (0, ) + allvalues
2121 sql.append("""select id from _%s where __retired__=%s
2122 and %s"""%(self.classname, a, ' and '.join(where)))
2124 # now multilinks
2125 for prop, values in propspec.iteritems():
2126 if not isinstance(props[prop], hyperdb.Multilink):
2127 continue
2128 if not values:
2129 continue
2130 allvalues += (0, )
2131 if type(values) is type(''):
2132 allvalues += (values,)
2133 s = a
2134 else:
2135 allvalues += tuple(values)
2136 s = ','.join([a]*len(values))
2137 tn = '%s_%s'%(self.classname, prop)
2138 sql.append("""select id from _%s, %s where __retired__=%s
2139 and id = %s.nodeid and %s.linkid in (%s)"""%(self.classname,
2140 tn, a, tn, tn, s))
2142 if not sql:
2143 return []
2144 sql = ' union '.join(sql)
2145 self.db.sql(sql, allvalues)
2146 # XXX numeric ids
2147 l = [str(x[0]) for x in self.db.sql_fetchall()]
2148 return l
2150 def stringFind(self, **requirements):
2151 """Locate a particular node by matching a set of its String
2152 properties in a caseless search.
2154 If the property is not a String property, a TypeError is raised.
2156 The return is a list of the id of all nodes that match.
2157 """
2158 where = []
2159 args = []
2160 for propname in requirements:
2161 prop = self.properties[propname]
2162 if not isinstance(prop, String):
2163 raise TypeError("'%s' not a String property"%propname)
2164 where.append(propname)
2165 args.append(requirements[propname].lower())
2167 # generate the where clause
2168 s = ' and '.join(['lower(_%s)=%s'%(col, self.db.arg) for col in where])
2169 sql = 'select id from _%s where %s and __retired__=%s'%(
2170 self.classname, s, self.db.arg)
2171 args.append(0)
2172 self.db.sql(sql, tuple(args))
2173 # XXX numeric ids
2174 l = [str(x[0]) for x in self.db.sql_fetchall()]
2175 return l
2177 def list(self):
2178 """ Return a list of the ids of the active nodes in this class.
2179 """
2180 return self.getnodeids(retired=0)
2182 def getnodeids(self, retired=None):
2183 """ Retrieve all the ids of the nodes for a particular Class.
2185 Set retired=None to get all nodes. Otherwise it'll get all the
2186 retired or non-retired nodes, depending on the flag.
2187 """
2188 # flip the sense of the 'retired' flag if we don't want all of them
2189 if retired is not None:
2190 args = (0, )
2191 if retired:
2192 compare = '>'
2193 else:
2194 compare = '='
2195 sql = 'select id from _%s where __retired__%s%s'%(self.classname,
2196 compare, self.db.arg)
2197 else:
2198 args = ()
2199 sql = 'select id from _%s'%self.classname
2200 self.db.sql(sql, args)
2201 # XXX numeric ids
2202 ids = [str(x[0]) for x in self.db.cursor.fetchall()]
2203 return ids
2205 def _subselect(self, classname, multilink_table):
2206 """Create a subselect. This is factored out because some
2207 databases (hmm only one, so far) doesn't support subselects
2208 look for "I can't believe it's not a toy RDBMS" in the mysql
2209 backend.
2210 """
2211 return '_%s.id not in (select nodeid from %s)'%(classname,
2212 multilink_table)
2214 # Some DBs order NULL values last. Set this variable in the backend
2215 # for prepending an order by clause for each attribute that causes
2216 # correct sort order for NULLs. Examples:
2217 # order_by_null_values = '(%s is not NULL)'
2218 # order_by_null_values = 'notnull(%s)'
2219 # The format parameter is replaced with the attribute.
2220 order_by_null_values = None
2222 def supports_subselects(self):
2223 '''Assuming DBs can do subselects, overwrite if they cannot.
2224 '''
2225 return True
2227 def _filter_multilink_expression_fallback(
2228 self, classname, multilink_table, expr):
2229 '''This is a fallback for database that do not support
2230 subselects.'''
2232 is_valid = expr.evaluate
2234 last_id, kws = None, []
2236 ids = IdListOptimizer()
2237 append = ids.append
2239 # This join and the evaluation in program space
2240 # can be expensive for larger databases!
2241 # TODO: Find a faster way to collect the data needed
2242 # to evalute the expression.
2243 # Moving the expression evaluation into the database
2244 # would be nice but this tricky: Think about the cases
2245 # where the multilink table does not have join values
2246 # needed in evaluation.
2248 stmnt = "SELECT c.id, m.linkid FROM _%s c " \
2249 "LEFT OUTER JOIN %s m " \
2250 "ON c.id = m.nodeid ORDER BY c.id" % (
2251 classname, multilink_table)
2252 self.db.sql(stmnt)
2254 # collect all multilink items for a class item
2255 for nid, kw in self.db.sql_fetchiter():
2256 if nid != last_id:
2257 if last_id is None:
2258 last_id = nid
2259 else:
2260 # we have all multilink items -> evaluate!
2261 if is_valid(kws): append(last_id)
2262 last_id, kws = nid, []
2263 if kw is not None:
2264 kws.append(kw)
2266 if last_id is not None and is_valid(kws):
2267 append(last_id)
2269 # we have ids of the classname table
2270 return ids.where("_%s.id" % classname, self.db.arg)
2272 def _filter_multilink_expression(self, classname, multilink_table, v):
2273 """ Filters out elements of the classname table that do not
2274 match the given expression.
2275 Returns tuple of 'WHERE' introns for the overall filter.
2276 """
2277 try:
2278 opcodes = [int(x) for x in v]
2279 if min(opcodes) >= -1: raise ValueError()
2281 expr = compile_expression(opcodes)
2283 if not self.supports_subselects():
2284 # We heavily rely on subselects. If there is
2285 # no decent support fall back to slower variant.
2286 return self._filter_multilink_expression_fallback(
2287 classname, multilink_table, expr)
2289 atom = \
2290 "%s IN(SELECT linkid FROM %s WHERE nodeid=a.id)" % (
2291 self.db.arg,
2292 multilink_table)
2294 intron = \
2295 "_%(classname)s.id in (SELECT id " \
2296 "FROM _%(classname)s AS a WHERE %(condition)s) " % {
2297 'classname' : classname,
2298 'condition' : expr.generate(lambda n: atom) }
2300 values = []
2301 def collect_values(n): values.append(n.x)
2302 expr.visit(collect_values)
2304 return intron, values
2305 except:
2306 # original behavior
2307 where = "%s.linkid in (%s)" % (
2308 multilink_table, ','.join([self.db.arg] * len(v)))
2309 return where, v, True # True to indicate original
2311 def _filter_sql (self, search_matches, filterspec, srt=[], grp=[], retr=0):
2312 """ Compute the proptree and the SQL/ARGS for a filter.
2313 For argument description see filter below.
2314 We return a 3-tuple, the proptree, the sql and the sql-args
2315 or None if no SQL is necessary.
2316 The flag retr serves to retrieve *all* non-Multilink properties
2317 (for filling the cache during a filter_iter)
2318 """
2319 # we can't match anything if search_matches is empty
2320 if not search_matches and search_matches is not None:
2321 return None
2323 icn = self.classname
2325 # vars to hold the components of the SQL statement
2326 frum = [] # FROM clauses
2327 loj = [] # LEFT OUTER JOIN clauses
2328 where = [] # WHERE clauses
2329 args = [] # *any* positional arguments
2330 a = self.db.arg
2332 # figure the WHERE clause from the filterspec
2333 mlfilt = 0 # are we joining with Multilink tables?
2334 sortattr = self._sortattr (group = grp, sort = srt)
2335 proptree = self._proptree(filterspec, sortattr, retr)
2336 mlseen = 0
2337 for pt in reversed(proptree.sortattr):
2338 p = pt
2339 while p.parent:
2340 if isinstance (p.propclass, Multilink):
2341 mlseen = True
2342 if mlseen:
2343 p.sort_ids_needed = True
2344 p.tree_sort_done = False
2345 p = p.parent
2346 if not mlseen:
2347 pt.attr_sort_done = pt.tree_sort_done = True
2348 proptree.compute_sort_done()
2350 cols = ['_%s.id'%icn]
2351 mlsort = []
2352 rhsnum = 0
2353 for p in proptree:
2354 rc = ac = oc = None
2355 cn = p.classname
2356 ln = p.uniqname
2357 pln = p.parent.uniqname
2358 pcn = p.parent.classname
2359 k = p.name
2360 v = p.val
2361 propclass = p.propclass
2362 if p.parent == proptree and p.name == 'id' \
2363 and 'retrieve' in p.need_for:
2364 p.sql_idx = 0
2365 if 'sort' in p.need_for or 'retrieve' in p.need_for:
2366 rc = oc = ac = '_%s._%s'%(pln, k)
2367 if isinstance(propclass, Multilink):
2368 if 'search' in p.need_for:
2369 mlfilt = 1
2370 tn = '%s_%s'%(pcn, k)
2371 if v in ('-1', ['-1'], []):
2372 # only match rows that have count(linkid)=0 in the
2373 # corresponding multilink table)
2374 where.append(self._subselect(pcn, tn))
2375 else:
2376 frum.append(tn)
2377 gen_join = True
2379 if p.has_values and isinstance(v, type([])):
2380 result = self._filter_multilink_expression(pln, tn, v)
2381 # XXX: We dont need an id join if we used the filter
2382 gen_join = len(result) == 3
2384 if gen_join:
2385 where.append('_%s.id=%s.nodeid'%(pln,tn))
2387 if p.children:
2388 frum.append('_%s as _%s' % (cn, ln))
2389 where.append('%s.linkid=_%s.id'%(tn, ln))
2391 if p.has_values:
2392 if isinstance(v, type([])):
2393 where.append(result[0])
2394 args += result[1]
2395 else:
2396 where.append('%s.linkid=%s'%(tn, a))
2397 args.append(v)
2398 if 'sort' in p.need_for:
2399 assert not p.attr_sort_done and not p.sort_ids_needed
2400 elif k == 'id':
2401 if 'search' in p.need_for:
2402 if isinstance(v, type([])):
2403 # If there are no permitted values, then the
2404 # where clause will always be false, and we
2405 # can optimize the query away.
2406 if not v:
2407 return []
2408 s = ','.join([a for x in v])
2409 where.append('_%s.%s in (%s)'%(pln, k, s))
2410 args = args + v
2411 else:
2412 where.append('_%s.%s=%s'%(pln, k, a))
2413 args.append(v)
2414 if 'sort' in p.need_for or 'retrieve' in p.need_for:
2415 rc = oc = ac = '_%s.id'%pln
2416 elif isinstance(propclass, String):
2417 if 'search' in p.need_for:
2418 if not isinstance(v, type([])):
2419 v = [v]
2421 # Quote the bits in the string that need it and then embed
2422 # in a "substring" search. Note - need to quote the '%' so
2423 # they make it through the python layer happily
2424 v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
2426 # now add to the where clause
2427 where.append('('
2428 +' and '.join(["_%s._%s LIKE '%s'"%(pln, k, s) for s in v])
2429 +')')
2430 # note: args are embedded in the query string now
2431 if 'sort' in p.need_for:
2432 oc = ac = 'lower(_%s._%s)'%(pln, k)
2433 elif isinstance(propclass, Link):
2434 if 'search' in p.need_for:
2435 if p.children:
2436 if 'sort' not in p.need_for:
2437 frum.append('_%s as _%s' % (cn, ln))
2438 where.append('_%s._%s=_%s.id'%(pln, k, ln))
2439 if p.has_values:
2440 if isinstance(v, type([])):
2441 d = {}
2442 for entry in v:
2443 if entry == '-1':
2444 entry = None
2445 d[entry] = entry
2446 l = []
2447 if None in d or not d:
2448 if None in d: del d[None]
2449 l.append('_%s._%s is NULL'%(pln, k))
2450 if d:
2451 v = list(d)
2452 s = ','.join([a for x in v])
2453 l.append('(_%s._%s in (%s))'%(pln, k, s))
2454 args = args + v
2455 if l:
2456 where.append('(' + ' or '.join(l) +')')
2457 else:
2458 if v in ('-1', None):
2459 v = None
2460 where.append('_%s._%s is NULL'%(pln, k))
2461 else:
2462 where.append('_%s._%s=%s'%(pln, k, a))
2463 args.append(v)
2464 if 'sort' in p.need_for:
2465 lp = p.cls.labelprop()
2466 oc = ac = '_%s._%s'%(pln, k)
2467 if lp != 'id':
2468 if p.tree_sort_done:
2469 loj.append(
2470 'LEFT OUTER JOIN _%s as _%s on _%s._%s=_%s.id'%(
2471 cn, ln, pln, k, ln))
2472 oc = '_%s._%s'%(ln, lp)
2473 if 'retrieve' in p.need_for:
2474 rc = '_%s._%s'%(pln, k)
2475 elif isinstance(propclass, Date) and 'search' in p.need_for:
2476 dc = self.db.to_sql_value(hyperdb.Date)
2477 if isinstance(v, type([])):
2478 s = ','.join([a for x in v])
2479 where.append('_%s._%s in (%s)'%(pln, k, s))
2480 args = args + [dc(date.Date(x)) for x in v]
2481 else:
2482 try:
2483 # Try to filter on range of dates
2484 date_rng = propclass.range_from_raw(v, self.db)
2485 if date_rng.from_value:
2486 where.append('_%s._%s >= %s'%(pln, k, a))
2487 args.append(dc(date_rng.from_value))
2488 if date_rng.to_value:
2489 where.append('_%s._%s <= %s'%(pln, k, a))
2490 args.append(dc(date_rng.to_value))
2491 except ValueError:
2492 # If range creation fails - ignore that search parameter
2493 pass
2494 elif isinstance(propclass, Interval):
2495 # filter/sort using the __<prop>_int__ column
2496 if 'search' in p.need_for:
2497 if isinstance(v, type([])):
2498 s = ','.join([a for x in v])
2499 where.append('_%s.__%s_int__ in (%s)'%(pln, k, s))
2500 args = args + [date.Interval(x).as_seconds() for x in v]
2501 else:
2502 try:
2503 # Try to filter on range of intervals
2504 date_rng = Range(v, date.Interval)
2505 if date_rng.from_value:
2506 where.append('_%s.__%s_int__ >= %s'%(pln, k, a))
2507 args.append(date_rng.from_value.as_seconds())
2508 if date_rng.to_value:
2509 where.append('_%s.__%s_int__ <= %s'%(pln, k, a))
2510 args.append(date_rng.to_value.as_seconds())
2511 except ValueError:
2512 # If range creation fails - ignore search parameter
2513 pass
2514 if 'sort' in p.need_for:
2515 oc = ac = '_%s.__%s_int__'%(pln,k)
2516 if 'retrieve' in p.need_for:
2517 rc = '_%s._%s'%(pln,k)
2518 elif isinstance(propclass, Boolean) and 'search' in p.need_for:
2519 if type(v) == type(""):
2520 v = v.split(',')
2521 if type(v) != type([]):
2522 v = [v]
2523 bv = []
2524 for val in v:
2525 if type(val) is type(''):
2526 bv.append(propclass.from_raw (val))
2527 else:
2528 bv.append(bool(val))
2529 if len(bv) == 1:
2530 where.append('_%s._%s=%s'%(pln, k, a))
2531 args = args + bv
2532 else:
2533 s = ','.join([a for x in v])
2534 where.append('_%s._%s in (%s)'%(pln, k, s))
2535 args = args + bv
2536 elif 'search' in p.need_for:
2537 if isinstance(v, type([])):
2538 s = ','.join([a for x in v])
2539 where.append('_%s._%s in (%s)'%(pln, k, s))
2540 args = args + v
2541 else:
2542 where.append('_%s._%s=%s'%(pln, k, a))
2543 args.append(v)
2544 if oc:
2545 if p.sort_ids_needed:
2546 if rc == ac:
2547 p.sql_idx = len(cols)
2548 p.auxcol = len(cols)
2549 cols.append(ac)
2550 if p.tree_sort_done and p.sort_direction:
2551 # Don't select top-level id or multilink twice
2552 if (not p.sort_ids_needed or ac != oc) and (p.name != 'id'
2553 or p.parent != proptree):
2554 if rc == oc:
2555 p.sql_idx = len(cols)
2556 cols.append(oc)
2557 desc = ['', ' desc'][p.sort_direction == '-']
2558 # Some SQL dbs sort NULL values last -- we want them first.
2559 if (self.order_by_null_values and p.name != 'id'):
2560 nv = self.order_by_null_values % oc
2561 cols.append(nv)
2562 p.orderby.append(nv + desc)
2563 p.orderby.append(oc + desc)
2564 if 'retrieve' in p.need_for and p.sql_idx is None:
2565 assert(rc)
2566 p.sql_idx = len(cols)
2567 cols.append (rc)
2569 props = self.getprops()
2571 # don't match retired nodes
2572 where.append('_%s.__retired__=0'%icn)
2574 # add results of full text search
2575 if search_matches is not None:
2576 s = ','.join([a for x in search_matches])
2577 where.append('_%s.id in (%s)'%(icn, s))
2578 args = args + [x for x in search_matches]
2580 # construct the SQL
2581 frum.append('_'+icn)
2582 frum = ','.join(frum)
2583 if where:
2584 where = ' where ' + (' and '.join(where))
2585 else:
2586 where = ''
2587 if mlfilt:
2588 # we're joining tables on the id, so we will get dupes if we
2589 # don't distinct()
2590 cols[0] = 'distinct(_%s.id)'%icn
2592 order = []
2593 # keep correct sequence of order attributes.
2594 for sa in proptree.sortattr:
2595 if not sa.attr_sort_done:
2596 continue
2597 order.extend(sa.orderby)
2598 if order:
2599 order = ' order by %s'%(','.join(order))
2600 else:
2601 order = ''
2603 cols = ','.join(cols)
2604 loj = ' '.join(loj)
2605 sql = 'select %s from %s %s %s%s'%(cols, frum, loj, where, order)
2606 args = tuple(args)
2607 __traceback_info__ = (sql, args)
2608 return proptree, sql, args
2610 def filter(self, search_matches, filterspec, sort=[], group=[]):
2611 """Return a list of the ids of the active nodes in this class that
2612 match the 'filter' spec, sorted by the group spec and then the
2613 sort spec
2615 "filterspec" is {propname: value(s)}
2617 "sort" and "group" are [(dir, prop), ...] where dir is '+', '-'
2618 or None and prop is a prop name or None. Note that for
2619 backward-compatibility reasons a single (dir, prop) tuple is
2620 also allowed.
2622 "search_matches" is a container type or None
2624 The filter must match all properties specificed. If the property
2625 value to match is a list:
2627 1. String properties must match all elements in the list, and
2628 2. Other properties must match any of the elements in the list.
2629 """
2630 if __debug__:
2631 start_t = time.time()
2633 sq = self._filter_sql (search_matches, filterspec, sort, group)
2634 # nothing to match?
2635 if sq is None:
2636 return []
2637 proptree, sql, args = sq
2639 self.db.sql(sql, args)
2640 l = self.db.sql_fetchall()
2642 # Compute values needed for sorting in proptree.sort
2643 for p in proptree:
2644 if hasattr(p, 'auxcol'):
2645 p.sort_ids = p.sort_result = [row[p.auxcol] for row in l]
2646 # return the IDs (the first column)
2647 # XXX numeric ids
2648 l = [str(row[0]) for row in l]
2649 l = proptree.sort (l)
2651 if __debug__:
2652 self.db.stats['filtering'] += (time.time() - start_t)
2653 return l
2655 def filter_iter(self, search_matches, filterspec, sort=[], group=[]):
2656 """Iterator similar to filter above with same args.
2657 Limitation: We don't sort on multilinks.
2658 This uses an optimisation: We put all nodes that are in the
2659 current row into the node cache. Then we return the node id.
2660 That way a fetch of a node won't create another sql-fetch (with
2661 a join) from the database because the nodes are already in the
2662 cache. We're using our own temporary cursor.
2663 """
2664 sq = self._filter_sql(search_matches, filterspec, sort, group, retr=1)
2665 # nothing to match?
2666 if sq is None:
2667 return
2668 proptree, sql, args = sq
2669 cursor = self.db.conn.cursor()
2670 self.db.sql(sql, args, cursor)
2671 classes = {}
2672 for p in proptree:
2673 if 'retrieve' in p.need_for:
2674 cn = p.parent.classname
2675 ptid = p.parent.id # not the nodeid!
2676 key = (cn, ptid)
2677 if key not in classes:
2678 classes[key] = {}
2679 name = p.name
2680 assert (name)
2681 classes[key][name] = p
2682 p.to_hyperdb = self.db.to_hyperdb_value(p.propclass.__class__)
2683 while True:
2684 row = cursor.fetchone()
2685 if not row: break
2686 # populate cache with current items
2687 for (classname, ptid), pt in classes.iteritems():
2688 nodeid = str(row[pt['id'].sql_idx])
2689 key = (classname, nodeid)
2690 if key in self.db.cache:
2691 self.db._cache_refresh(key)
2692 continue
2693 node = {}
2694 for propname, p in pt.iteritems():
2695 value = row[p.sql_idx]
2696 if value is not None:
2697 value = p.to_hyperdb(value)
2698 node[propname] = value
2699 self.db._cache_save(key, node)
2700 yield str(row[0])
2702 def filter_sql(self, sql):
2703 """Return a list of the ids of the items in this class that match
2704 the SQL provided. The SQL is a complete "select" statement.
2706 The SQL select must include the item id as the first column.
2708 This function DOES NOT filter out retired items, add on a where
2709 clause "__retired__=0" if you don't want retired nodes.
2710 """
2711 if __debug__:
2712 start_t = time.time()
2714 self.db.sql(sql)
2715 l = self.db.sql_fetchall()
2717 if __debug__:
2718 self.db.stats['filtering'] += (time.time() - start_t)
2719 return l
2721 def count(self):
2722 """Get the number of nodes in this class.
2724 If the returned integer is 'numnodes', the ids of all the nodes
2725 in this class run from 1 to numnodes, and numnodes+1 will be the
2726 id of the next node to be created in this class.
2727 """
2728 return self.db.countnodes(self.classname)
2730 # Manipulating properties:
2731 def getprops(self, protected=1):
2732 """Return a dictionary mapping property names to property objects.
2733 If the "protected" flag is true, we include protected properties -
2734 those which may not be modified.
2735 """
2736 d = self.properties.copy()
2737 if protected:
2738 d['id'] = String()
2739 d['creation'] = hyperdb.Date()
2740 d['activity'] = hyperdb.Date()
2741 d['creator'] = hyperdb.Link('user')
2742 d['actor'] = hyperdb.Link('user')
2743 return d
2745 def addprop(self, **properties):
2746 """Add properties to this class.
2748 The keyword arguments in 'properties' must map names to property
2749 objects, or a TypeError is raised. None of the keys in 'properties'
2750 may collide with the names of existing properties, or a ValueError
2751 is raised before any properties have been added.
2752 """
2753 for key in properties:
2754 if key in self.properties:
2755 raise ValueError(key)
2756 self.properties.update(properties)
2758 def index(self, nodeid):
2759 """Add (or refresh) the node to search indexes
2760 """
2761 # find all the String properties that have indexme
2762 for prop, propclass in self.getprops().iteritems():
2763 if isinstance(propclass, String) and propclass.indexme:
2764 self.db.indexer.add_text((self.classname, nodeid, prop),
2765 str(self.get(nodeid, prop)))
2767 #
2768 # import / export support
2769 #
2770 def export_list(self, propnames, nodeid):
2771 """ Export a node - generate a list of CSV-able data in the order
2772 specified by propnames for the given node.
2773 """
2774 properties = self.getprops()
2775 l = []
2776 for prop in propnames:
2777 proptype = properties[prop]
2778 value = self.get(nodeid, prop)
2779 # "marshal" data where needed
2780 if value is None:
2781 pass
2782 elif isinstance(proptype, hyperdb.Date):
2783 value = value.get_tuple()
2784 elif isinstance(proptype, hyperdb.Interval):
2785 value = value.get_tuple()
2786 elif isinstance(proptype, hyperdb.Password):
2787 value = str(value)
2788 l.append(repr(value))
2789 l.append(repr(self.is_retired(nodeid)))
2790 return l
2792 def import_list(self, propnames, proplist):
2793 """ Import a node - all information including "id" is present and
2794 should not be sanity checked. Triggers are not triggered. The
2795 journal should be initialised using the "creator" and "created"
2796 information.
2798 Return the nodeid of the node imported.
2799 """
2800 if self.db.journaltag is None:
2801 raise DatabaseError(_('Database open read-only'))
2802 properties = self.getprops()
2804 # make the new node's property map
2805 d = {}
2806 retire = 0
2807 if not "id" in propnames:
2808 newid = self.db.newid(self.classname)
2809 else:
2810 newid = eval(proplist[propnames.index("id")])
2811 for i in range(len(propnames)):
2812 # Use eval to reverse the repr() used to output the CSV
2813 value = eval(proplist[i])
2815 # Figure the property for this column
2816 propname = propnames[i]
2818 # "unmarshal" where necessary
2819 if propname == 'id':
2820 continue
2821 elif propname == 'is retired':
2822 # is the item retired?
2823 if int(value):
2824 retire = 1
2825 continue
2826 elif value is None:
2827 d[propname] = None
2828 continue
2830 prop = properties[propname]
2831 if value is None:
2832 # don't set Nones
2833 continue
2834 elif isinstance(prop, hyperdb.Date):
2835 value = date.Date(value)
2836 elif isinstance(prop, hyperdb.Interval):
2837 value = date.Interval(value)
2838 elif isinstance(prop, hyperdb.Password):
2839 pwd = password.Password()
2840 pwd.unpack(value)
2841 value = pwd
2842 elif isinstance(prop, String):
2843 if isinstance(value, unicode):
2844 value = value.encode('utf8')
2845 if not isinstance(value, str):
2846 raise TypeError('new property "%(propname)s" not a '
2847 'string: %(value)r'%locals())
2848 if prop.indexme:
2849 self.db.indexer.add_text((self.classname, newid, propname),
2850 value)
2851 d[propname] = value
2853 # get a new id if necessary
2854 if newid is None:
2855 newid = self.db.newid(self.classname)
2857 # insert new node or update existing?
2858 if not self.hasnode(newid):
2859 self.db.addnode(self.classname, newid, d) # insert
2860 else:
2861 self.db.setnode(self.classname, newid, d) # update
2863 # retire?
2864 if retire:
2865 # use the arg for __retired__ to cope with any odd database type
2866 # conversion (hello, sqlite)
2867 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
2868 self.db.arg, self.db.arg)
2869 self.db.sql(sql, (newid, newid))
2870 return newid
2872 def export_journals(self):
2873 """Export a class's journal - generate a list of lists of
2874 CSV-able data:
2876 nodeid, date, user, action, params
2878 No heading here - the columns are fixed.
2879 """
2880 properties = self.getprops()
2881 r = []
2882 for nodeid in self.getnodeids():
2883 for nodeid, date, user, action, params in self.history(nodeid):
2884 date = date.get_tuple()
2885 if action == 'set':
2886 export_data = {}
2887 for propname, value in params.iteritems():
2888 if propname not in properties:
2889 # property no longer in the schema
2890 continue
2892 prop = properties[propname]
2893 # make sure the params are eval()'able
2894 if value is None:
2895 pass
2896 elif isinstance(prop, Date):
2897 value = value.get_tuple()
2898 elif isinstance(prop, Interval):
2899 value = value.get_tuple()
2900 elif isinstance(prop, Password):
2901 value = str(value)
2902 export_data[propname] = value
2903 params = export_data
2904 elif action == 'create' and params:
2905 # old tracker with data stored in the create!
2906 params = {}
2907 l = [nodeid, date, user, action, params]
2908 r.append(list(map(repr, l)))
2909 return r
2911 class FileClass(hyperdb.FileClass, Class):
2912 """This class defines a large chunk of data. To support this, it has a
2913 mandatory String property "content" which is typically saved off
2914 externally to the hyperdb.
2916 The default MIME type of this data is defined by the
2917 "default_mime_type" class attribute, which may be overridden by each
2918 node if the class defines a "type" String property.
2919 """
2920 def __init__(self, db, classname, **properties):
2921 """The newly-created class automatically includes the "content"
2922 and "type" properties.
2923 """
2924 if 'content' not in properties:
2925 properties['content'] = hyperdb.String(indexme='yes')
2926 if 'type' not in properties:
2927 properties['type'] = hyperdb.String()
2928 Class.__init__(self, db, classname, **properties)
2930 def create(self, **propvalues):
2931 """ snaffle the file propvalue and store in a file
2932 """
2933 # we need to fire the auditors now, or the content property won't
2934 # be in propvalues for the auditors to play with
2935 self.fireAuditors('create', None, propvalues)
2937 # now remove the content property so it's not stored in the db
2938 content = propvalues['content']
2939 del propvalues['content']
2941 # do the database create
2942 newid = self.create_inner(**propvalues)
2944 # figure the mime type
2945 mime_type = propvalues.get('type', self.default_mime_type)
2947 # and index!
2948 if self.properties['content'].indexme:
2949 self.db.indexer.add_text((self.classname, newid, 'content'),
2950 content, mime_type)
2952 # store off the content as a file
2953 self.db.storefile(self.classname, newid, None, content)
2955 # fire reactors
2956 self.fireReactors('create', newid, None)
2958 return newid
2960 def get(self, nodeid, propname, default=_marker, cache=1):
2961 """ Trap the content propname and get it from the file
2963 'cache' exists for backwards compatibility, and is not used.
2964 """
2965 poss_msg = 'Possibly a access right configuration problem.'
2966 if propname == 'content':
2967 try:
2968 return self.db.getfile(self.classname, nodeid, None)
2969 except IOError, strerror:
2970 # BUG: by catching this we donot see an error in the log.
2971 return 'ERROR reading file: %s%s\n%s\n%s'%(
2972 self.classname, nodeid, poss_msg, strerror)
2973 if default is not _marker:
2974 return Class.get(self, nodeid, propname, default)
2975 else:
2976 return Class.get(self, nodeid, propname)
2978 def set(self, itemid, **propvalues):
2979 """ Snarf the "content" propvalue and update it in a file
2980 """
2981 self.fireAuditors('set', itemid, propvalues)
2982 oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid))
2984 # now remove the content property so it's not stored in the db
2985 content = None
2986 if 'content' in propvalues:
2987 content = propvalues['content']
2988 del propvalues['content']
2990 # do the database create
2991 propvalues = self.set_inner(itemid, **propvalues)
2993 # do content?
2994 if content:
2995 # store and possibly index
2996 self.db.storefile(self.classname, itemid, None, content)
2997 if self.properties['content'].indexme:
2998 mime_type = self.get(itemid, 'type', self.default_mime_type)
2999 self.db.indexer.add_text((self.classname, itemid, 'content'),
3000 content, mime_type)
3001 propvalues['content'] = content
3003 # fire reactors
3004 self.fireReactors('set', itemid, oldvalues)
3005 return propvalues
3007 def index(self, nodeid):
3008 """ Add (or refresh) the node to search indexes.
3010 Use the content-type property for the content property.
3011 """
3012 # find all the String properties that have indexme
3013 for prop, propclass in self.getprops().iteritems():
3014 if prop == 'content' and propclass.indexme:
3015 mime_type = self.get(nodeid, 'type', self.default_mime_type)
3016 self.db.indexer.add_text((self.classname, nodeid, 'content'),
3017 str(self.get(nodeid, 'content')), mime_type)
3018 elif isinstance(propclass, hyperdb.String) and propclass.indexme:
3019 # index them under (classname, nodeid, property)
3020 try:
3021 value = str(self.get(nodeid, prop))
3022 except IndexError:
3023 # node has been destroyed
3024 continue
3025 self.db.indexer.add_text((self.classname, nodeid, prop), value)
3027 # XXX deviation from spec - was called ItemClass
3028 class IssueClass(Class, roundupdb.IssueClass):
3029 # Overridden methods:
3030 def __init__(self, db, classname, **properties):
3031 """The newly-created class automatically includes the "messages",
3032 "files", "nosy", and "superseder" properties. If the 'properties'
3033 dictionary attempts to specify any of these properties or a
3034 "creation", "creator", "activity" or "actor" property, a ValueError
3035 is raised.
3036 """
3037 if 'title' not in properties:
3038 properties['title'] = hyperdb.String(indexme='yes')
3039 if 'messages' not in properties:
3040 properties['messages'] = hyperdb.Multilink("msg")
3041 if 'files' not in properties:
3042 properties['files'] = hyperdb.Multilink("file")
3043 if 'nosy' not in properties:
3044 # note: journalling is turned off as it really just wastes
3045 # space. this behaviour may be overridden in an instance
3046 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
3047 if 'superseder' not in properties:
3048 properties['superseder'] = hyperdb.Multilink(classname)
3049 Class.__init__(self, db, classname, **properties)
3051 # vim: set et sts=4 sw=4 :