1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 """ Relational database (SQL) backend common code.
20 Basics:
22 - map roundup classes to relational tables
23 - automatically detect schema changes and modify the table schemas
24 appropriately (we store the "database version" of the schema in the
25 database itself as the only row of the "schema" table)
26 - multilinks (which represent a many-to-many relationship) are handled through
27 intermediate tables
28 - journals are stored adjunct to the per-class tables
29 - table names and columns have "_" prepended so the names can't clash with
30 restricted names (like "order")
31 - retirement is determined by the __retired__ column being > 0
33 Database-specific changes may generally be pushed out to the overridable
34 sql_* methods, since everything else should be fairly generic. There's
35 probably a bit of work to be done if a database is used that actually
36 honors column typing, since the initial databases don't (sqlite stores
37 everything as a string.)
39 The schema of the hyperdb being mapped to the database is stored in the
40 database itself as a repr()'ed dictionary of information about each Class
41 that maps to a table. If that information differs from the hyperdb schema,
42 then we update it. We also store in the schema dict a version which
43 allows us to upgrade the database schema when necessary. See upgrade_db().
45 To force a unqiueness constraint on the key properties we put the item
46 id into the __retired__ column duing retirement (so it's 0 for "active"
47 items) and place a unqiueness constraint on key + __retired__. This is
48 particularly important for the users class where multiple users may
49 try to have the same username, with potentially many retired users with
50 the same name.
51 """
52 __docformat__ = 'restructuredtext'
54 # standard python modules
55 import sys, os, time, re, errno, weakref, copy, logging, datetime
57 # roundup modules
58 from roundup import hyperdb, date, password, roundupdb, security, support
59 from roundup.hyperdb import String, Password, Date, Interval, Link, \
60 Multilink, DatabaseError, Boolean, Number, Node
61 from roundup.backends import locking
62 from roundup.support import reversed
63 from roundup.i18n import _
66 # support
67 from roundup.backends.blobfiles import FileStorage
68 try:
69 from roundup.backends.indexer_xapian import Indexer
70 except ImportError:
71 from roundup.backends.indexer_rdbms import Indexer
72 from roundup.backends.sessions_rdbms import Sessions, OneTimeKeys
73 from roundup.date import Range
75 from roundup.backends.back_anydbm import compile_expression
78 # dummy value meaning "argument not passed"
79 _marker = []
81 def _num_cvt(num):
82 num = str(num)
83 try:
84 return int(num)
85 except:
86 return float(num)
88 def _bool_cvt(value):
89 if value in ('TRUE', 'FALSE'):
90 return {'TRUE': 1, 'FALSE': 0}[value]
91 # assume it's a number returned from the db API
92 return int(value)
94 def date_to_hyperdb_value(d):
95 """ convert date d to a roundup date """
96 if isinstance (d, datetime.datetime):
97 return date.Date(d)
98 return date.Date (str(d).replace(' ', '.'))
101 def connection_dict(config, dbnamestr=None):
102 """ Used by Postgresql and MySQL to detemine the keyword args for
103 opening the database connection."""
104 d = { }
105 if dbnamestr:
106 d[dbnamestr] = config.RDBMS_NAME
107 for name in ('host', 'port', 'password', 'user', 'read_default_group',
108 'read_default_file'):
109 cvar = 'RDBMS_'+name.upper()
110 if config[cvar] is not None:
111 d[name] = config[cvar]
112 return d
115 class IdListOptimizer:
116 """ To prevent flooding the SQL parser of the underlaying
117 db engine with "x IN (1, 2, 3, ..., <large number>)" collapses
118 these cases to "x BETWEEN 1 AND <large number>".
119 """
121 def __init__(self):
122 self.ranges = []
123 self.singles = []
125 def append(self, nid):
126 """ Invariant: nid are ordered ascending """
127 if self.ranges:
128 last = self.ranges[-1]
129 if last[1] == nid-1:
130 last[1] = nid
131 return
132 if self.singles:
133 last = self.singles[-1]
134 if last == nid-1:
135 self.singles.pop()
136 self.ranges.append([last, nid])
137 return
138 self.singles.append(nid)
140 def where(self, field, placeholder):
141 ranges = self.ranges
142 singles = self.singles
144 if not singles and not ranges: return "(1=0)", []
146 if ranges:
147 between = '%s BETWEEN %s AND %s' % (
148 field, placeholder, placeholder)
149 stmnt = [between] * len(ranges)
150 else:
151 stmnt = []
152 if singles:
153 stmnt.append('%s in (%s)' % (
154 field, ','.join([placeholder]*len(singles))))
156 return '(%s)' % ' OR '.join(stmnt), sum(ranges, []) + singles
158 def __str__(self):
159 return "ranges: %r / singles: %r" % (self.ranges, self.singles)
162 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
163 """ Wrapper around an SQL database that presents a hyperdb interface.
165 - some functionality is specific to the actual SQL database, hence
166 the sql_* methods that are NotImplemented
167 - we keep a cache of the latest N row fetches (where N is configurable).
168 """
169 def __init__(self, config, journaltag=None):
170 """ Open the database and load the schema from it.
171 """
172 FileStorage.__init__(self, config.UMASK)
173 self.config, self.journaltag = config, journaltag
174 self.dir = config.DATABASE
175 self.classes = {}
176 self.indexer = Indexer(self)
177 self.security = security.Security(self)
179 # additional transaction support for external files and the like
180 self.transactions = []
182 # keep a cache of the N most recently retrieved rows of any kind
183 # (classname, nodeid) = row
184 self.cache_size = config.RDBMS_CACHE_SIZE
185 self.clearCache()
186 self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0,
187 'filtering': 0}
189 # database lock
190 self.lockfile = None
192 # open a connection to the database, creating the "conn" attribute
193 self.open_connection()
195 def clearCache(self):
196 self.cache = {}
197 self.cache_lru = []
199 def getSessionManager(self):
200 return Sessions(self)
202 def getOTKManager(self):
203 return OneTimeKeys(self)
205 def open_connection(self):
206 """ Open a connection to the database, creating it if necessary.
208 Must call self.load_dbschema()
209 """
210 raise NotImplemented
212 def sql(self, sql, args=None, cursor=None):
213 """ Execute the sql with the optional args.
214 """
215 self.log_debug('SQL %r %r'%(sql, args))
216 if not cursor:
217 cursor = self.cursor
218 if args:
219 cursor.execute(sql, args)
220 else:
221 cursor.execute(sql)
223 def sql_fetchone(self):
224 """ Fetch a single row. If there's nothing to fetch, return None.
225 """
226 return self.cursor.fetchone()
228 def sql_fetchall(self):
229 """ Fetch all rows. If there's nothing to fetch, return [].
230 """
231 return self.cursor.fetchall()
233 def sql_fetchiter(self):
234 """ Fetch all row as a generator
235 """
236 while True:
237 row = self.cursor.fetchone()
238 if not row: break
239 yield row
241 def sql_stringquote(self, value):
242 """ Quote the string so it's safe to put in the 'sql quotes'
243 """
244 return re.sub("'", "''", str(value))
246 def init_dbschema(self):
247 self.database_schema = {
248 'version': self.current_db_version,
249 'tables': {}
250 }
252 def load_dbschema(self):
253 """ Load the schema definition that the database currently implements
254 """
255 self.cursor.execute('select schema from schema')
256 schema = self.cursor.fetchone()
257 if schema:
258 self.database_schema = eval(schema[0])
259 else:
260 self.database_schema = {}
262 def save_dbschema(self):
263 """ Save the schema definition that the database currently implements
264 """
265 s = repr(self.database_schema)
266 self.sql('delete from schema')
267 self.sql('insert into schema values (%s)'%self.arg, (s,))
269 def post_init(self):
270 """ Called once the schema initialisation has finished.
272 We should now confirm that the schema defined by our "classes"
273 attribute actually matches the schema in the database.
274 """
275 save = 0
277 # handle changes in the schema
278 tables = self.database_schema['tables']
279 for classname, spec in self.classes.iteritems():
280 if classname in tables:
281 dbspec = tables[classname]
282 if self.update_class(spec, dbspec):
283 tables[classname] = spec.schema()
284 save = 1
285 else:
286 self.create_class(spec)
287 tables[classname] = spec.schema()
288 save = 1
290 for classname, spec in list(tables.items()):
291 if classname not in self.classes:
292 self.drop_class(classname, tables[classname])
293 del tables[classname]
294 save = 1
296 # now upgrade the database for column type changes, new internal
297 # tables, etc.
298 save = save | self.upgrade_db()
300 # update the database version of the schema
301 if save:
302 self.save_dbschema()
304 # reindex the db if necessary
305 if self.indexer.should_reindex():
306 self.reindex()
308 # commit
309 self.sql_commit()
311 # update this number when we need to make changes to the SQL structure
312 # of the backen database
313 current_db_version = 5
314 db_version_updated = False
315 def upgrade_db(self):
316 """ Update the SQL database to reflect changes in the backend code.
318 Return boolean whether we need to save the schema.
319 """
320 version = self.database_schema.get('version', 1)
321 if version > self.current_db_version:
322 raise DatabaseError('attempting to run rev %d DATABASE with rev '
323 '%d CODE!'%(version, self.current_db_version))
324 if version == self.current_db_version:
325 # nothing to do
326 return 0
328 if version < 2:
329 self.log_info('upgrade to version 2')
330 # change the schema structure
331 self.database_schema = {'tables': self.database_schema}
333 # version 1 didn't have the actor column (note that in
334 # MySQL this will also transition the tables to typed columns)
335 self.add_new_columns_v2()
337 # version 1 doesn't have the OTK, session and indexing in the
338 # database
339 self.create_version_2_tables()
341 if version < 3:
342 self.log_info('upgrade to version 3')
343 self.fix_version_2_tables()
345 if version < 4:
346 self.fix_version_3_tables()
348 if version < 5:
349 self.fix_version_4_tables()
351 self.database_schema['version'] = self.current_db_version
352 self.db_version_updated = True
353 return 1
355 def fix_version_3_tables(self):
356 # drop the shorter VARCHAR OTK column and add a new TEXT one
357 for name in ('otk', 'session'):
358 self.sql('DELETE FROM %ss'%name)
359 self.sql('ALTER TABLE %ss DROP %s_value'%(name, name))
360 self.sql('ALTER TABLE %ss ADD %s_value TEXT'%(name, name))
362 def fix_version_2_tables(self):
363 # Default (used by sqlite): NOOP
364 pass
366 def fix_version_4_tables(self):
367 # note this is an explicit call now
368 c = self.cursor
369 for cn, klass in self.classes.iteritems():
370 c.execute('select id from _%s where __retired__<>0'%(cn,))
371 for (id,) in c.fetchall():
372 c.execute('update _%s set __retired__=%s where id=%s'%(cn,
373 self.arg, self.arg), (id, id))
375 if klass.key:
376 self.add_class_key_required_unique_constraint(cn, klass.key)
378 def _convert_journal_tables(self):
379 """Get current journal table contents, drop the table and re-create"""
380 c = self.cursor
381 cols = ','.join('nodeid date tag action params'.split())
382 for klass in self.classes.itervalues():
383 # slurp and drop
384 sql = 'select %s from %s__journal order by date'%(cols,
385 klass.classname)
386 c.execute(sql)
387 contents = c.fetchall()
388 self.drop_journal_table_indexes(klass.classname)
389 c.execute('drop table %s__journal'%klass.classname)
391 # re-create and re-populate
392 self.create_journal_table(klass)
393 a = self.arg
394 sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(
395 klass.classname, cols, a, a, a, a, a)
396 for row in contents:
397 # no data conversion needed
398 self.cursor.execute(sql, row)
400 def _convert_string_properties(self):
401 """Get current Class tables that contain String properties, and
402 convert the VARCHAR columns to TEXT"""
403 c = self.cursor
404 for klass in self.classes.itervalues():
405 # slurp and drop
406 cols, mls = self.determine_columns(list(klass.properties.iteritems()))
407 scols = ','.join([i[0] for i in cols])
408 sql = 'select id,%s from _%s'%(scols, klass.classname)
409 c.execute(sql)
410 contents = c.fetchall()
411 self.drop_class_table_indexes(klass.classname, klass.getkey())
412 c.execute('drop table _%s'%klass.classname)
414 # re-create and re-populate
415 self.create_class_table(klass, create_sequence=0)
416 a = ','.join([self.arg for i in range(len(cols)+1)])
417 sql = 'insert into _%s (id,%s) values (%s)'%(klass.classname,
418 scols, a)
419 for row in contents:
420 l = []
421 for entry in row:
422 # mysql will already be a string - psql needs "help"
423 if entry is not None and not isinstance(entry, type('')):
424 entry = str(entry)
425 l.append(entry)
426 self.cursor.execute(sql, l)
428 def refresh_database(self):
429 self.post_init()
432 def reindex(self, classname=None, show_progress=False):
433 if classname:
434 classes = [self.getclass(classname)]
435 else:
436 classes = list(self.classes.itervalues())
437 for klass in classes:
438 if show_progress:
439 for nodeid in support.Progress('Reindex %s'%klass.classname,
440 klass.list()):
441 klass.index(nodeid)
442 else:
443 for nodeid in klass.list():
444 klass.index(nodeid)
445 self.indexer.save_index()
447 hyperdb_to_sql_datatypes = {
448 hyperdb.String : 'TEXT',
449 hyperdb.Date : 'TIMESTAMP',
450 hyperdb.Link : 'INTEGER',
451 hyperdb.Interval : 'VARCHAR(255)',
452 hyperdb.Password : 'VARCHAR(255)',
453 hyperdb.Boolean : 'BOOLEAN',
454 hyperdb.Number : 'REAL',
455 }
457 def hyperdb_to_sql_datatype(self, propclass):
459 datatype = self.hyperdb_to_sql_datatypes.get(propclass)
460 if datatype:
461 return datatype
463 for k, v in self.hyperdb_to_sql_datatypes.iteritems():
464 if issubclass(propclass, k):
465 return v
467 raise ValueError('%r is not a hyperdb property class' % propclass)
469 def determine_columns(self, properties):
470 """ Figure the column names and multilink properties from the spec
472 "properties" is a list of (name, prop) where prop may be an
473 instance of a hyperdb "type" _or_ a string repr of that type.
474 """
475 cols = [
476 ('_actor', self.hyperdb_to_sql_datatype(hyperdb.Link)),
477 ('_activity', self.hyperdb_to_sql_datatype(hyperdb.Date)),
478 ('_creator', self.hyperdb_to_sql_datatype(hyperdb.Link)),
479 ('_creation', self.hyperdb_to_sql_datatype(hyperdb.Date)),
480 ]
481 mls = []
482 # add the multilinks separately
483 for col, prop in properties:
484 if isinstance(prop, Multilink):
485 mls.append(col)
486 continue
488 if isinstance(prop, type('')):
489 raise ValueError("string property spec!")
490 #and prop.find('Multilink') != -1:
491 #mls.append(col)
493 datatype = self.hyperdb_to_sql_datatype(prop.__class__)
494 cols.append(('_'+col, datatype))
496 # Intervals stored as two columns
497 if isinstance(prop, Interval):
498 cols.append(('__'+col+'_int__', 'BIGINT'))
500 cols.sort()
501 return cols, mls
503 def update_class(self, spec, old_spec, force=0):
504 """ Determine the differences between the current spec and the
505 database version of the spec, and update where necessary.
507 If 'force' is true, update the database anyway.
508 """
509 new_spec = spec.schema()
510 new_spec[1].sort()
511 old_spec[1].sort()
512 if not force and new_spec == old_spec:
513 # no changes
514 return 0
516 if not self.config.RDBMS_ALLOW_ALTER:
517 raise DatabaseError(_('ALTER operation disallowed: %r -> %r.'%(old_spec, new_spec)))
519 logger = logging.getLogger('roundup.hyperdb')
520 logger.info('update_class %s'%spec.classname)
522 logger.debug('old_spec %r'%(old_spec,))
523 logger.debug('new_spec %r'%(new_spec,))
525 # detect key prop change for potential index change
526 keyprop_changes = {}
527 if new_spec[0] != old_spec[0]:
528 if old_spec[0]:
529 keyprop_changes['remove'] = old_spec[0]
530 if new_spec[0]:
531 keyprop_changes['add'] = new_spec[0]
533 # detect multilinks that have been removed, and drop their table
534 old_has = {}
535 for name, prop in old_spec[1]:
536 old_has[name] = 1
537 if name in spec.properties:
538 continue
540 if prop.find('Multilink to') != -1:
541 # first drop indexes.
542 self.drop_multilink_table_indexes(spec.classname, name)
544 # now the multilink table itself
545 sql = 'drop table %s_%s'%(spec.classname, name)
546 else:
547 # if this is the key prop, drop the index first
548 if old_spec[0] == prop:
549 self.drop_class_table_key_index(spec.classname, name)
550 del keyprop_changes['remove']
552 # drop the column
553 sql = 'alter table _%s drop column _%s'%(spec.classname, name)
555 self.sql(sql)
557 # if we didn't remove the key prop just then, but the key prop has
558 # changed, we still need to remove the old index
559 if 'remove' in keyprop_changes:
560 self.drop_class_table_key_index(spec.classname,
561 keyprop_changes['remove'])
563 # add new columns
564 for propname, prop in new_spec[1]:
565 if propname in old_has:
566 continue
567 prop = spec.properties[propname]
568 if isinstance(prop, Multilink):
569 self.create_multilink_table(spec, propname)
570 else:
571 # add the column
572 coltype = self.hyperdb_to_sql_datatype(prop.__class__)
573 sql = 'alter table _%s add column _%s %s'%(
574 spec.classname, propname, coltype)
575 self.sql(sql)
577 # extra Interval column
578 if isinstance(prop, Interval):
579 sql = 'alter table _%s add column __%s_int__ BIGINT'%(
580 spec.classname, propname)
581 self.sql(sql)
583 # if the new column is a key prop, we need an index!
584 if new_spec[0] == propname:
585 self.create_class_table_key_index(spec.classname, propname)
586 del keyprop_changes['add']
588 # if we didn't add the key prop just then, but the key prop has
589 # changed, we still need to add the new index
590 if 'add' in keyprop_changes:
591 self.create_class_table_key_index(spec.classname,
592 keyprop_changes['add'])
594 return 1
596 def determine_all_columns(self, spec):
597 """Figure out the columns from the spec and also add internal columns
599 """
600 cols, mls = self.determine_columns(list(spec.properties.iteritems()))
602 # add on our special columns
603 cols.append(('id', 'INTEGER PRIMARY KEY'))
604 cols.append(('__retired__', 'INTEGER DEFAULT 0'))
605 return cols, mls
607 def create_class_table(self, spec):
608 """Create the class table for the given Class "spec". Creates the
609 indexes too."""
610 cols, mls = self.determine_all_columns(spec)
612 # create the base table
613 scols = ','.join(['%s %s'%x for x in cols])
614 sql = 'create table _%s (%s)'%(spec.classname, scols)
615 self.sql(sql)
617 self.create_class_table_indexes(spec)
619 return cols, mls
621 def create_class_table_indexes(self, spec):
622 """ create the class table for the given spec
623 """
624 # create __retired__ index
625 index_sql2 = 'create index _%s_retired_idx on _%s(__retired__)'%(
626 spec.classname, spec.classname)
627 self.sql(index_sql2)
629 # create index for key property
630 if spec.key:
631 index_sql3 = 'create index _%s_%s_idx on _%s(_%s)'%(
632 spec.classname, spec.key,
633 spec.classname, spec.key)
634 self.sql(index_sql3)
636 # and the unique index for key / retired(id)
637 self.add_class_key_required_unique_constraint(spec.classname,
638 spec.key)
640 # TODO: create indexes on (selected?) Link property columns, as
641 # they're more likely to be used for lookup
643 def add_class_key_required_unique_constraint(self, cn, key):
644 sql = '''create unique index _%s_key_retired_idx
645 on _%s(__retired__, _%s)'''%(cn, cn, key)
646 self.sql(sql)
648 def drop_class_table_indexes(self, cn, key):
649 # drop the old table indexes first
650 l = ['_%s_id_idx'%cn, '_%s_retired_idx'%cn]
651 if key:
652 l.append('_%s_%s_idx'%(cn, key))
654 table_name = '_%s'%cn
655 for index_name in l:
656 if not self.sql_index_exists(table_name, index_name):
657 continue
658 index_sql = 'drop index '+index_name
659 self.sql(index_sql)
661 def create_class_table_key_index(self, cn, key):
662 """ create the class table for the given spec
663 """
664 sql = 'create index _%s_%s_idx on _%s(_%s)'%(cn, key, cn, key)
665 self.sql(sql)
667 def drop_class_table_key_index(self, cn, key):
668 table_name = '_%s'%cn
669 index_name = '_%s_%s_idx'%(cn, key)
670 if self.sql_index_exists(table_name, index_name):
671 sql = 'drop index '+index_name
672 self.sql(sql)
674 # and now the retired unique index too
675 index_name = '_%s_key_retired_idx'%cn
676 if self.sql_index_exists(table_name, index_name):
677 sql = 'drop index '+index_name
678 self.sql(sql)
680 def create_journal_table(self, spec):
681 """ create the journal table for a class given the spec and
682 already-determined cols
683 """
684 # journal table
685 cols = ','.join(['%s varchar'%x
686 for x in 'nodeid date tag action params'.split()])
687 sql = """create table %s__journal (
688 nodeid integer, date %s, tag varchar(255),
689 action varchar(255), params text)""" % (spec.classname,
690 self.hyperdb_to_sql_datatype(hyperdb.Date))
691 self.sql(sql)
692 self.create_journal_table_indexes(spec)
694 def create_journal_table_indexes(self, spec):
695 # index on nodeid
696 sql = 'create index %s_journ_idx on %s__journal(nodeid)'%(
697 spec.classname, spec.classname)
698 self.sql(sql)
700 def drop_journal_table_indexes(self, classname):
701 index_name = '%s_journ_idx'%classname
702 if not self.sql_index_exists('%s__journal'%classname, index_name):
703 return
704 index_sql = 'drop index '+index_name
705 self.sql(index_sql)
707 def create_multilink_table(self, spec, ml):
708 """ Create a multilink table for the "ml" property of the class
709 given by the spec
710 """
711 # create the table
712 sql = 'create table %s_%s (linkid INTEGER, nodeid INTEGER)'%(
713 spec.classname, ml)
714 self.sql(sql)
715 self.create_multilink_table_indexes(spec, ml)
717 def create_multilink_table_indexes(self, spec, ml):
718 # create index on linkid
719 index_sql = 'create index %s_%s_l_idx on %s_%s(linkid)'%(
720 spec.classname, ml, spec.classname, ml)
721 self.sql(index_sql)
723 # create index on nodeid
724 index_sql = 'create index %s_%s_n_idx on %s_%s(nodeid)'%(
725 spec.classname, ml, spec.classname, ml)
726 self.sql(index_sql)
728 def drop_multilink_table_indexes(self, classname, ml):
729 l = [
730 '%s_%s_l_idx'%(classname, ml),
731 '%s_%s_n_idx'%(classname, ml)
732 ]
733 table_name = '%s_%s'%(classname, ml)
734 for index_name in l:
735 if not self.sql_index_exists(table_name, index_name):
736 continue
737 index_sql = 'drop index %s'%index_name
738 self.sql(index_sql)
740 def create_class(self, spec):
741 """ Create a database table according to the given spec.
742 """
744 if not self.config.RDBMS_ALLOW_CREATE:
745 raise DatabaseError(_('CREATE operation disallowed: "%s".'%spec.classname))
747 cols, mls = self.create_class_table(spec)
748 self.create_journal_table(spec)
750 # now create the multilink tables
751 for ml in mls:
752 self.create_multilink_table(spec, ml)
754 def drop_class(self, cn, spec):
755 """ Drop the given table from the database.
757 Drop the journal and multilink tables too.
758 """
760 if not self.config.RDBMS_ALLOW_DROP:
761 raise DatabaseError(_('DROP operation disallowed: "%s".'%cn))
763 properties = spec[1]
764 # figure the multilinks
765 mls = []
766 for propname, prop in properties:
767 if isinstance(prop, Multilink):
768 mls.append(propname)
770 # drop class table and indexes
771 self.drop_class_table_indexes(cn, spec[0])
773 self.drop_class_table(cn)
775 # drop journal table and indexes
776 self.drop_journal_table_indexes(cn)
777 sql = 'drop table %s__journal'%cn
778 self.sql(sql)
780 for ml in mls:
781 # drop multilink table and indexes
782 self.drop_multilink_table_indexes(cn, ml)
783 sql = 'drop table %s_%s'%(spec.classname, ml)
784 self.sql(sql)
786 def drop_class_table(self, cn):
787 sql = 'drop table _%s'%cn
788 self.sql(sql)
790 #
791 # Classes
792 #
793 def __getattr__(self, classname):
794 """ A convenient way of calling self.getclass(classname).
795 """
796 if classname in self.classes:
797 return self.classes[classname]
798 raise AttributeError(classname)
800 def addclass(self, cl):
801 """ Add a Class to the hyperdatabase.
802 """
803 cn = cl.classname
804 if cn in self.classes:
805 raise ValueError(cn)
806 self.classes[cn] = cl
808 # add default Edit and View permissions
809 self.security.addPermission(name="Create", klass=cn,
810 description="User is allowed to create "+cn)
811 self.security.addPermission(name="Edit", klass=cn,
812 description="User is allowed to edit "+cn)
813 self.security.addPermission(name="View", klass=cn,
814 description="User is allowed to access "+cn)
816 def getclasses(self):
817 """ Return a list of the names of all existing classes.
818 """
819 return sorted(self.classes)
821 def getclass(self, classname):
822 """Get the Class object representing a particular class.
824 If 'classname' is not a valid class name, a KeyError is raised.
825 """
826 try:
827 return self.classes[classname]
828 except KeyError:
829 raise KeyError('There is no class called "%s"'%classname)
831 def clear(self):
832 """Delete all database contents.
834 Note: I don't commit here, which is different behaviour to the
835 "nuke from orbit" behaviour in the dbs.
836 """
837 logging.getLogger('roundup.hyperdb').info('clear')
838 for cn in self.classes:
839 sql = 'delete from _%s'%cn
840 self.sql(sql)
842 #
843 # Nodes
844 #
846 hyperdb_to_sql_value = {
847 hyperdb.String : str,
848 # fractional seconds by default
849 hyperdb.Date : lambda x: x.formal(sep=' ', sec='%06.3f'),
850 hyperdb.Link : int,
851 hyperdb.Interval : str,
852 hyperdb.Password : str,
853 hyperdb.Boolean : lambda x: x and 'TRUE' or 'FALSE',
854 hyperdb.Number : lambda x: x,
855 hyperdb.Multilink : lambda x: x, # used in journal marshalling
856 }
858 def to_sql_value(self, propklass):
860 fn = self.hyperdb_to_sql_value.get(propklass)
861 if fn:
862 return fn
864 for k, v in self.hyperdb_to_sql_value.iteritems():
865 if issubclass(propklass, k):
866 return v
868 raise ValueError('%r is not a hyperdb property class' % propklass)
870 def _cache_del(self, key):
871 del self.cache[key]
872 self.cache_lru.remove(key)
874 def _cache_refresh(self, key):
875 self.cache_lru.remove(key)
876 self.cache_lru.insert(0, key)
878 def _cache_save(self, key, node):
879 self.cache[key] = node
880 # update the LRU
881 self.cache_lru.insert(0, key)
882 if len(self.cache_lru) > self.cache_size:
883 del self.cache[self.cache_lru.pop()]
885 def addnode(self, classname, nodeid, node):
886 """ Add the specified node to its class's db.
887 """
888 self.log_debug('addnode %s%s %r'%(classname,
889 nodeid, node))
891 # determine the column definitions and multilink tables
892 cl = self.classes[classname]
893 cols, mls = self.determine_columns(list(cl.properties.iteritems()))
895 # we'll be supplied these props if we're doing an import
896 values = node.copy()
897 if 'creator' not in values:
898 # add in the "calculated" properties (dupe so we don't affect
899 # calling code's node assumptions)
900 values['creation'] = values['activity'] = date.Date()
901 values['actor'] = values['creator'] = self.getuid()
903 cl = self.classes[classname]
904 props = cl.getprops(protected=1)
905 del props['id']
907 # default the non-multilink columns
908 for col, prop in props.iteritems():
909 if col not in values:
910 if isinstance(prop, Multilink):
911 values[col] = []
912 else:
913 values[col] = None
915 # clear this node out of the cache if it's in there
916 key = (classname, nodeid)
917 if key in self.cache:
918 self._cache_del(key)
920 # figure the values to insert
921 vals = []
922 for col,dt in cols:
923 # this is somewhat dodgy....
924 if col.endswith('_int__'):
925 # XXX eugh, this test suxxors
926 value = values[col[2:-6]]
927 # this is an Interval special "int" column
928 if value is not None:
929 vals.append(value.as_seconds())
930 else:
931 vals.append(value)
932 continue
934 prop = props[col[1:]]
935 value = values[col[1:]]
936 if value is not None:
937 value = self.to_sql_value(prop.__class__)(value)
938 vals.append(value)
939 vals.append(nodeid)
940 vals = tuple(vals)
942 # make sure the ordering is correct for column name -> column value
943 s = ','.join([self.arg for x in cols]) + ',%s'%self.arg
944 cols = ','.join([col for col,dt in cols]) + ',id'
946 # perform the inserts
947 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
948 self.sql(sql, vals)
950 # insert the multilink rows
951 for col in mls:
952 t = '%s_%s'%(classname, col)
953 for entry in node[col]:
954 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
955 self.arg, self.arg)
956 self.sql(sql, (entry, nodeid))
958 def setnode(self, classname, nodeid, values, multilink_changes={}):
959 """ Change the specified node.
960 """
961 self.log_debug('setnode %s%s %r'
962 % (classname, nodeid, values))
964 # clear this node out of the cache if it's in there
965 key = (classname, nodeid)
966 if key in self.cache:
967 self._cache_del(key)
969 cl = self.classes[classname]
970 props = cl.getprops()
972 cols = []
973 mls = []
974 # add the multilinks separately
975 for col in values:
976 prop = props[col]
977 if isinstance(prop, Multilink):
978 mls.append(col)
979 elif isinstance(prop, Interval):
980 # Intervals store the seconds value too
981 cols.append(col)
982 # extra leading '_' added by code below
983 cols.append('_' +col + '_int__')
984 else:
985 cols.append(col)
986 cols.sort()
988 # figure the values to insert
989 vals = []
990 for col in cols:
991 if col.endswith('_int__'):
992 # XXX eugh, this test suxxors
993 # Intervals store the seconds value too
994 col = col[1:-6]
995 prop = props[col]
996 value = values[col]
997 if value is None:
998 vals.append(None)
999 else:
1000 vals.append(value.as_seconds())
1001 else:
1002 prop = props[col]
1003 value = values[col]
1004 if value is None:
1005 e = None
1006 else:
1007 e = self.to_sql_value(prop.__class__)(value)
1008 vals.append(e)
1010 vals.append(int(nodeid))
1011 vals = tuple(vals)
1013 # if there's any updates to regular columns, do them
1014 if cols:
1015 # make sure the ordering is correct for column name -> column value
1016 s = ','.join(['_%s=%s'%(x, self.arg) for x in cols])
1017 cols = ','.join(cols)
1019 # perform the update
1020 sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
1021 self.sql(sql, vals)
1023 # we're probably coming from an import, not a change
1024 if not multilink_changes:
1025 for name in mls:
1026 prop = props[name]
1027 value = values[name]
1029 t = '%s_%s'%(classname, name)
1031 # clear out previous values for this node
1032 # XXX numeric ids
1033 self.sql('delete from %s where nodeid=%s'%(t, self.arg),
1034 (nodeid,))
1036 # insert the values for this node
1037 for entry in values[name]:
1038 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
1039 self.arg, self.arg)
1040 # XXX numeric ids
1041 self.sql(sql, (entry, nodeid))
1043 # we have multilink changes to apply
1044 for col, (add, remove) in multilink_changes.iteritems():
1045 tn = '%s_%s'%(classname, col)
1046 if add:
1047 sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
1048 self.arg, self.arg)
1049 for addid in add:
1050 # XXX numeric ids
1051 self.sql(sql, (int(nodeid), int(addid)))
1052 if remove:
1053 s = ','.join([self.arg]*len(remove))
1054 sql = 'delete from %s where nodeid=%s and linkid in (%s)'%(tn,
1055 self.arg, s)
1056 # XXX numeric ids
1057 self.sql(sql, [int(nodeid)] + remove)
1059 sql_to_hyperdb_value = {
1060 hyperdb.String : str,
1061 hyperdb.Date : date_to_hyperdb_value,
1062 # hyperdb.Link : int, # XXX numeric ids
1063 hyperdb.Link : str,
1064 hyperdb.Interval : date.Interval,
1065 hyperdb.Password : lambda x: password.Password(encrypted=x),
1066 hyperdb.Boolean : _bool_cvt,
1067 hyperdb.Number : _num_cvt,
1068 hyperdb.Multilink : lambda x: x, # used in journal marshalling
1069 }
1071 def to_hyperdb_value(self, propklass):
1073 fn = self.sql_to_hyperdb_value.get(propklass)
1074 if fn:
1075 return fn
1077 for k, v in self.sql_to_hyperdb_value.iteritems():
1078 if issubclass(propklass, k):
1079 return v
1081 raise ValueError('%r is not a hyperdb property class' % propklass)
1083 def getnode(self, classname, nodeid):
1084 """ Get a node from the database.
1085 """
1086 # see if we have this node cached
1087 key = (classname, nodeid)
1088 if key in self.cache:
1089 # push us back to the top of the LRU
1090 self._cache_refresh(key)
1091 if __debug__:
1092 self.stats['cache_hits'] += 1
1093 # return the cached information
1094 return self.cache[key]
1096 if __debug__:
1097 self.stats['cache_misses'] += 1
1098 start_t = time.time()
1100 # figure the columns we're fetching
1101 cl = self.classes[classname]
1102 cols, mls = self.determine_columns(list(cl.properties.iteritems()))
1103 scols = ','.join([col for col,dt in cols])
1105 # perform the basic property fetch
1106 sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
1107 self.sql(sql, (nodeid,))
1109 values = self.sql_fetchone()
1110 if values is None:
1111 raise IndexError('no such %s node %s'%(classname, nodeid))
1113 # make up the node
1114 node = {}
1115 props = cl.getprops(protected=1)
1116 for col in range(len(cols)):
1117 name = cols[col][0][1:]
1118 if name.endswith('_int__'):
1119 # XXX eugh, this test suxxors
1120 # ignore the special Interval-as-seconds column
1121 continue
1122 value = values[col]
1123 if value is not None:
1124 value = self.to_hyperdb_value(props[name].__class__)(value)
1125 node[name] = value
1127 # save off in the cache
1128 key = (classname, nodeid)
1129 self._cache_save(key, node)
1131 if __debug__:
1132 self.stats['get_items'] += (time.time() - start_t)
1134 return node
1136 def destroynode(self, classname, nodeid):
1137 """Remove a node from the database. Called exclusively by the
1138 destroy() method on Class.
1139 """
1140 logging.getLogger('roundup.hyperdb').info('destroynode %s%s'%(
1141 classname, nodeid))
1143 # make sure the node exists
1144 if not self.hasnode(classname, nodeid):
1145 raise IndexError('%s has no node %s'%(classname, nodeid))
1147 # see if we have this node cached
1148 if (classname, nodeid) in self.cache:
1149 del self.cache[(classname, nodeid)]
1151 # see if there's any obvious commit actions that we should get rid of
1152 for entry in self.transactions[:]:
1153 if entry[1][:2] == (classname, nodeid):
1154 self.transactions.remove(entry)
1156 # now do the SQL
1157 sql = 'delete from _%s where id=%s'%(classname, self.arg)
1158 self.sql(sql, (nodeid,))
1160 # remove from multilnks
1161 cl = self.getclass(classname)
1162 x, mls = self.determine_columns(list(cl.properties.iteritems()))
1163 for col in mls:
1164 # get the link ids
1165 sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
1166 self.sql(sql, (nodeid,))
1168 # remove journal entries
1169 sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
1170 self.sql(sql, (nodeid,))
1172 # cleanup any blob filestorage when we commit
1173 self.transactions.append((FileStorage.destroy, (self, classname, nodeid)))
1175 def hasnode(self, classname, nodeid):
1176 """ Determine if the database has a given node.
1177 """
1178 # If this node is in the cache, then we do not need to go to
1179 # the database. (We don't consider this an LRU hit, though.)
1180 if (classname, nodeid) in self.cache:
1181 # Return 1, not True, to match the type of the result of
1182 # the SQL operation below.
1183 return 1
1184 sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
1185 self.sql(sql, (nodeid,))
1186 return int(self.cursor.fetchone()[0])
1188 def countnodes(self, classname):
1189 """ Count the number of nodes that exist for a particular Class.
1190 """
1191 sql = 'select count(*) from _%s'%classname
1192 self.sql(sql)
1193 return self.cursor.fetchone()[0]
1195 def addjournal(self, classname, nodeid, action, params, creator=None,
1196 creation=None):
1197 """ Journal the Action
1198 'action' may be:
1200 'create' or 'set' -- 'params' is a dictionary of property values
1201 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
1202 'retire' -- 'params' is None
1203 """
1204 # handle supply of the special journalling parameters (usually
1205 # supplied on importing an existing database)
1206 if creator:
1207 journaltag = creator
1208 else:
1209 journaltag = self.getuid()
1210 if creation:
1211 journaldate = creation
1212 else:
1213 journaldate = date.Date()
1215 # create the journal entry
1216 cols = 'nodeid,date,tag,action,params'
1218 self.log_debug('addjournal %s%s %r %s %s %r'%(classname,
1219 nodeid, journaldate, journaltag, action, params))
1221 # make the journalled data marshallable
1222 if isinstance(params, type({})):
1223 self._journal_marshal(params, classname)
1225 params = repr(params)
1227 dc = self.to_sql_value(hyperdb.Date)
1228 journaldate = dc(journaldate)
1230 self.save_journal(classname, cols, nodeid, journaldate,
1231 journaltag, action, params)
1233 def setjournal(self, classname, nodeid, journal):
1234 """Set the journal to the "journal" list."""
1235 # clear out any existing entries
1236 self.sql('delete from %s__journal where nodeid=%s'%(classname,
1237 self.arg), (nodeid,))
1239 # create the journal entry
1240 cols = 'nodeid,date,tag,action,params'
1242 dc = self.to_sql_value(hyperdb.Date)
1243 for nodeid, journaldate, journaltag, action, params in journal:
1244 self.log_debug('addjournal %s%s %r %s %s %r'%(
1245 classname, nodeid, journaldate, journaltag, action,
1246 params))
1248 # make the journalled data marshallable
1249 if isinstance(params, type({})):
1250 self._journal_marshal(params, classname)
1251 params = repr(params)
1253 self.save_journal(classname, cols, nodeid, dc(journaldate),
1254 journaltag, action, params)
1256 def _journal_marshal(self, params, classname):
1257 """Convert the journal params values into safely repr'able and
1258 eval'able values."""
1259 properties = self.getclass(classname).getprops()
1260 for param, value in params.iteritems():
1261 if not value:
1262 continue
1263 property = properties[param]
1264 cvt = self.to_sql_value(property.__class__)
1265 if isinstance(property, Password):
1266 params[param] = cvt(value)
1267 elif isinstance(property, Date):
1268 params[param] = cvt(value)
1269 elif isinstance(property, Interval):
1270 params[param] = cvt(value)
1271 elif isinstance(property, Boolean):
1272 params[param] = cvt(value)
1274 def getjournal(self, classname, nodeid):
1275 """ get the journal for id
1276 """
1277 # make sure the node exists
1278 if not self.hasnode(classname, nodeid):
1279 raise IndexError('%s has no node %s'%(classname, nodeid))
1281 cols = ','.join('nodeid date tag action params'.split())
1282 journal = self.load_journal(classname, cols, nodeid)
1284 # now unmarshal the data
1285 dc = self.to_hyperdb_value(hyperdb.Date)
1286 res = []
1287 properties = self.getclass(classname).getprops()
1288 for nodeid, date_stamp, user, action, params in journal:
1289 params = eval(params)
1290 if isinstance(params, type({})):
1291 for param, value in params.iteritems():
1292 if not value:
1293 continue
1294 property = properties.get(param, None)
1295 if property is None:
1296 # deleted property
1297 continue
1298 cvt = self.to_hyperdb_value(property.__class__)
1299 if isinstance(property, Password):
1300 params[param] = password.JournalPassword(value)
1301 elif isinstance(property, Date):
1302 params[param] = cvt(value)
1303 elif isinstance(property, Interval):
1304 params[param] = cvt(value)
1305 elif isinstance(property, Boolean):
1306 params[param] = cvt(value)
1307 # XXX numeric ids
1308 res.append((str(nodeid), dc(date_stamp), user, action, params))
1309 return res
1311 def save_journal(self, classname, cols, nodeid, journaldate,
1312 journaltag, action, params):
1313 """ Save the journal entry to the database
1314 """
1315 entry = (nodeid, journaldate, journaltag, action, params)
1317 # do the insert
1318 a = self.arg
1319 sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(
1320 classname, cols, a, a, a, a, a)
1321 self.sql(sql, entry)
1323 def load_journal(self, classname, cols, nodeid):
1324 """ Load the journal from the database
1325 """
1326 # now get the journal entries
1327 sql = 'select %s from %s__journal where nodeid=%s order by date'%(
1328 cols, classname, self.arg)
1329 self.sql(sql, (nodeid,))
1330 return self.cursor.fetchall()
1332 def pack(self, pack_before):
1333 """ Delete all journal entries except "create" before 'pack_before'.
1334 """
1335 date_stamp = self.to_sql_value(Date)(pack_before)
1337 # do the delete
1338 for classname in self.classes:
1339 sql = "delete from %s__journal where date<%s and "\
1340 "action<>'create'"%(classname, self.arg)
1341 self.sql(sql, (date_stamp,))
1343 def sql_commit(self, fail_ok=False):
1344 """ Actually commit to the database.
1345 """
1346 logging.getLogger('roundup.hyperdb').info('commit')
1348 self.conn.commit()
1350 # open a new cursor for subsequent work
1351 self.cursor = self.conn.cursor()
1353 def commit(self, fail_ok=False):
1354 """ Commit the current transactions.
1356 Save all data changed since the database was opened or since the
1357 last commit() or rollback().
1359 fail_ok indicates that the commit is allowed to fail. This is used
1360 in the web interface when committing cleaning of the session
1361 database. We don't care if there's a concurrency issue there.
1363 The only backend this seems to affect is postgres.
1364 """
1365 # commit the database
1366 self.sql_commit(fail_ok)
1368 # now, do all the other transaction stuff
1369 for method, args in self.transactions:
1370 method(*args)
1372 # save the indexer
1373 self.indexer.save_index()
1375 # clear out the transactions
1376 self.transactions = []
1378 # clear the cache: Don't carry over cached values from one
1379 # transaction to the next (there may be other changes from other
1380 # transactions)
1381 self.clearCache()
1383 def sql_rollback(self):
1384 self.conn.rollback()
1386 def rollback(self):
1387 """ Reverse all actions from the current transaction.
1389 Undo all the changes made since the database was opened or the last
1390 commit() or rollback() was performed.
1391 """
1392 logging.getLogger('roundup.hyperdb').info('rollback')
1394 self.sql_rollback()
1396 # roll back "other" transaction stuff
1397 for method, args in self.transactions:
1398 # delete temporary files
1399 if method == self.doStoreFile:
1400 self.rollbackStoreFile(*args)
1401 self.transactions = []
1403 # clear the cache
1404 self.clearCache()
1406 def sql_close(self):
1407 logging.getLogger('roundup.hyperdb').info('close')
1408 self.conn.close()
1410 def close(self):
1411 """ Close off the connection.
1412 """
1413 self.indexer.close()
1414 self.sql_close()
1416 #
1417 # The base Class class
1418 #
1419 class Class(hyperdb.Class):
1420 """ The handle to a particular class of nodes in a hyperdatabase.
1422 All methods except __repr__ and getnode must be implemented by a
1423 concrete backend Class.
1424 """
1426 def schema(self):
1427 """ A dumpable version of the schema that we can store in the
1428 database
1429 """
1430 return (self.key, [(x, repr(y)) for x,y in self.properties.iteritems()])
1432 def enableJournalling(self):
1433 """Turn journalling on for this class
1434 """
1435 self.do_journal = 1
1437 def disableJournalling(self):
1438 """Turn journalling off for this class
1439 """
1440 self.do_journal = 0
1442 # Editing nodes:
1443 def create(self, **propvalues):
1444 """ Create a new node of this class and return its id.
1446 The keyword arguments in 'propvalues' map property names to values.
1448 The values of arguments must be acceptable for the types of their
1449 corresponding properties or a TypeError is raised.
1451 If this class has a key property, it must be present and its value
1452 must not collide with other key strings or a ValueError is raised.
1454 Any other properties on this class that are missing from the
1455 'propvalues' dictionary are set to None.
1457 If an id in a link or multilink property does not refer to a valid
1458 node, an IndexError is raised.
1459 """
1460 self.fireAuditors('create', None, propvalues)
1461 newid = self.create_inner(**propvalues)
1462 self.fireReactors('create', newid, None)
1463 return newid
1465 def create_inner(self, **propvalues):
1466 """ Called by create, in-between the audit and react calls.
1467 """
1468 if 'id' in propvalues:
1469 raise KeyError('"id" is reserved')
1471 if self.db.journaltag is None:
1472 raise DatabaseError(_('Database open read-only'))
1474 if ('creator' in propvalues or 'actor' in propvalues or
1475 'creation' in propvalues or 'activity' in propvalues):
1476 raise KeyError('"creator", "actor", "creation" and '
1477 '"activity" are reserved')
1479 # new node's id
1480 newid = self.db.newid(self.classname)
1482 # validate propvalues
1483 num_re = re.compile('^\d+$')
1484 for key, value in propvalues.iteritems():
1485 if key == self.key:
1486 try:
1487 self.lookup(value)
1488 except KeyError:
1489 pass
1490 else:
1491 raise ValueError('node with key "%s" exists'%value)
1493 # try to handle this property
1494 try:
1495 prop = self.properties[key]
1496 except KeyError:
1497 raise KeyError('"%s" has no property "%s"'%(self.classname,
1498 key))
1500 if value is not None and isinstance(prop, Link):
1501 if type(value) != type(''):
1502 raise ValueError('link value must be String')
1503 link_class = self.properties[key].classname
1504 # if it isn't a number, it's a key
1505 if not num_re.match(value):
1506 try:
1507 value = self.db.classes[link_class].lookup(value)
1508 except (TypeError, KeyError):
1509 raise IndexError('new property "%s": %s not a %s'%(
1510 key, value, link_class))
1511 elif not self.db.getclass(link_class).hasnode(value):
1512 raise IndexError('%s has no node %s'%(link_class,
1513 value))
1515 # save off the value
1516 propvalues[key] = value
1518 # register the link with the newly linked node
1519 if self.do_journal and self.properties[key].do_journal:
1520 self.db.addjournal(link_class, value, 'link',
1521 (self.classname, newid, key))
1523 elif isinstance(prop, Multilink):
1524 if value is None:
1525 value = []
1526 if not hasattr(value, '__iter__'):
1527 raise TypeError('new property "%s" not an iterable of ids'%key)
1528 # clean up and validate the list of links
1529 link_class = self.properties[key].classname
1530 l = []
1531 for entry in value:
1532 if type(entry) != type(''):
1533 raise ValueError('"%s" multilink value (%r) '
1534 'must contain Strings'%(key, value))
1535 # if it isn't a number, it's a key
1536 if not num_re.match(entry):
1537 try:
1538 entry = self.db.classes[link_class].lookup(entry)
1539 except (TypeError, KeyError):
1540 raise IndexError('new property "%s": %s not a %s'%(
1541 key, entry, self.properties[key].classname))
1542 l.append(entry)
1543 value = l
1544 propvalues[key] = value
1546 # handle additions
1547 for nodeid in value:
1548 if not self.db.getclass(link_class).hasnode(nodeid):
1549 raise IndexError('%s has no node %s'%(link_class,
1550 nodeid))
1551 # register the link with the newly linked node
1552 if self.do_journal and self.properties[key].do_journal:
1553 self.db.addjournal(link_class, nodeid, 'link',
1554 (self.classname, newid, key))
1556 elif isinstance(prop, String):
1557 if type(value) != type('') and type(value) != type(u''):
1558 raise TypeError('new property "%s" not a string'%key)
1559 if prop.indexme:
1560 self.db.indexer.add_text((self.classname, newid, key),
1561 value)
1563 elif isinstance(prop, Password):
1564 if not isinstance(value, password.Password):
1565 raise TypeError('new property "%s" not a Password'%key)
1567 elif isinstance(prop, Date):
1568 if value is not None and not isinstance(value, date.Date):
1569 raise TypeError('new property "%s" not a Date'%key)
1571 elif isinstance(prop, Interval):
1572 if value is not None and not isinstance(value, date.Interval):
1573 raise TypeError('new property "%s" not an Interval'%key)
1575 elif value is not None and isinstance(prop, Number):
1576 try:
1577 float(value)
1578 except ValueError:
1579 raise TypeError('new property "%s" not numeric'%key)
1581 elif value is not None and isinstance(prop, Boolean):
1582 try:
1583 int(value)
1584 except ValueError:
1585 raise TypeError('new property "%s" not boolean'%key)
1587 # make sure there's data where there needs to be
1588 for key, prop in self.properties.iteritems():
1589 if key in propvalues:
1590 continue
1591 if key == self.key:
1592 raise ValueError('key property "%s" is required'%key)
1593 if isinstance(prop, Multilink):
1594 propvalues[key] = []
1595 else:
1596 propvalues[key] = None
1598 # done
1599 self.db.addnode(self.classname, newid, propvalues)
1600 if self.do_journal:
1601 self.db.addjournal(self.classname, newid, ''"create", {})
1603 # XXX numeric ids
1604 return str(newid)
1606 def get(self, nodeid, propname, default=_marker, cache=1):
1607 """Get the value of a property on an existing node of this class.
1609 'nodeid' must be the id of an existing node of this class or an
1610 IndexError is raised. 'propname' must be the name of a property
1611 of this class or a KeyError is raised.
1613 'cache' exists for backwards compatibility, and is not used.
1614 """
1615 if propname == 'id':
1616 return nodeid
1618 # get the node's dict
1619 d = self.db.getnode(self.classname, nodeid)
1620 # handle common case -- that property is in dict -- first
1621 # if None and one of creator/creation actor/activity return None
1622 if propname in d:
1623 r = d [propname]
1624 # return copy of our list
1625 if isinstance (r, list):
1626 return r[:]
1627 if r is not None:
1628 return r
1629 elif propname in ('creation', 'activity', 'creator', 'actor'):
1630 return r
1632 # propname not in d:
1633 if propname == 'creation' or propname == 'activity':
1634 return date.Date()
1635 if propname == 'creator' or propname == 'actor':
1636 return self.db.getuid()
1638 # get the property (raises KeyError if invalid)
1639 prop = self.properties[propname]
1641 # lazy evaluation of Multilink
1642 if propname not in d and isinstance(prop, Multilink):
1643 sql = 'select linkid from %s_%s where nodeid=%s'%(self.classname,
1644 propname, self.db.arg)
1645 self.db.sql(sql, (nodeid,))
1646 # extract the first column from the result
1647 # XXX numeric ids
1648 items = [int(x[0]) for x in self.db.cursor.fetchall()]
1649 items.sort ()
1650 d[propname] = [str(x) for x in items]
1652 # handle there being no value in the table for the property
1653 if propname not in d or d[propname] is None:
1654 if default is _marker:
1655 if isinstance(prop, Multilink):
1656 return []
1657 else:
1658 return None
1659 else:
1660 return default
1662 # don't pass our list to other code
1663 if isinstance(prop, Multilink):
1664 return d[propname][:]
1666 return d[propname]
1668 def set(self, nodeid, **propvalues):
1669 """Modify a property on an existing node of this class.
1671 'nodeid' must be the id of an existing node of this class or an
1672 IndexError is raised.
1674 Each key in 'propvalues' must be the name of a property of this
1675 class or a KeyError is raised.
1677 All values in 'propvalues' must be acceptable types for their
1678 corresponding properties or a TypeError is raised.
1680 If the value of the key property is set, it must not collide with
1681 other key strings or a ValueError is raised.
1683 If the value of a Link or Multilink property contains an invalid
1684 node id, a ValueError is raised.
1685 """
1686 self.fireAuditors('set', nodeid, propvalues)
1687 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1688 propvalues = self.set_inner(nodeid, **propvalues)
1689 self.fireReactors('set', nodeid, oldvalues)
1690 return propvalues
1692 def set_inner(self, nodeid, **propvalues):
1693 """ Called by set, in-between the audit and react calls.
1694 """
1695 if not propvalues:
1696 return propvalues
1698 if ('creator' in propvalues or 'actor' in propvalues or
1699 'creation' in propvalues or 'activity' in propvalues):
1700 raise KeyError('"creator", "actor", "creation" and '
1701 '"activity" are reserved')
1703 if 'id' in propvalues:
1704 raise KeyError('"id" is reserved')
1706 if self.db.journaltag is None:
1707 raise DatabaseError(_('Database open read-only'))
1709 node = self.db.getnode(self.classname, nodeid)
1710 if self.is_retired(nodeid):
1711 raise IndexError('Requested item is retired')
1712 num_re = re.compile('^\d+$')
1714 # make a copy of the values dictionary - we'll modify the contents
1715 propvalues = propvalues.copy()
1717 # if the journal value is to be different, store it in here
1718 journalvalues = {}
1720 # remember the add/remove stuff for multilinks, making it easier
1721 # for the Database layer to do its stuff
1722 multilink_changes = {}
1724 for propname, value in list(propvalues.items()):
1725 # check to make sure we're not duplicating an existing key
1726 if propname == self.key and node[propname] != value:
1727 try:
1728 self.lookup(value)
1729 except KeyError:
1730 pass
1731 else:
1732 raise ValueError('node with key "%s" exists'%value)
1734 # this will raise the KeyError if the property isn't valid
1735 # ... we don't use getprops() here because we only care about
1736 # the writeable properties.
1737 try:
1738 prop = self.properties[propname]
1739 except KeyError:
1740 raise KeyError('"%s" has no property named "%s"'%(
1741 self.classname, propname))
1743 # if the value's the same as the existing value, no sense in
1744 # doing anything
1745 current = node.get(propname, None)
1746 if value == current:
1747 del propvalues[propname]
1748 continue
1749 journalvalues[propname] = current
1751 # do stuff based on the prop type
1752 if isinstance(prop, Link):
1753 link_class = prop.classname
1754 # if it isn't a number, it's a key
1755 if value is not None and not isinstance(value, type('')):
1756 raise ValueError('property "%s" link value be a string'%(
1757 propname))
1758 if isinstance(value, type('')) and not num_re.match(value):
1759 try:
1760 value = self.db.classes[link_class].lookup(value)
1761 except (TypeError, KeyError):
1762 raise IndexError('new property "%s": %s not a %s'%(
1763 propname, value, prop.classname))
1765 if (value is not None and
1766 not self.db.getclass(link_class).hasnode(value)):
1767 raise IndexError('%s has no node %s'%(link_class,
1768 value))
1770 if self.do_journal and prop.do_journal:
1771 # register the unlink with the old linked node
1772 if node[propname] is not None:
1773 self.db.addjournal(link_class, node[propname],
1774 ''"unlink", (self.classname, nodeid, propname))
1776 # register the link with the newly linked node
1777 if value is not None:
1778 self.db.addjournal(link_class, value, ''"link",
1779 (self.classname, nodeid, propname))
1781 elif isinstance(prop, Multilink):
1782 if value is None:
1783 value = []
1784 if not hasattr(value, '__iter__'):
1785 raise TypeError('new property "%s" not an iterable of'
1786 ' ids'%propname)
1787 link_class = self.properties[propname].classname
1788 l = []
1789 for entry in value:
1790 # if it isn't a number, it's a key
1791 if type(entry) != type(''):
1792 raise ValueError('new property "%s" link value '
1793 'must be a string'%propname)
1794 if not num_re.match(entry):
1795 try:
1796 entry = self.db.classes[link_class].lookup(entry)
1797 except (TypeError, KeyError):
1798 raise IndexError('new property "%s": %s not a %s'%(
1799 propname, entry,
1800 self.properties[propname].classname))
1801 l.append(entry)
1802 value = l
1803 propvalues[propname] = value
1805 # figure the journal entry for this property
1806 add = []
1807 remove = []
1809 # handle removals
1810 if propname in node:
1811 l = node[propname]
1812 else:
1813 l = []
1814 for id in l[:]:
1815 if id in value:
1816 continue
1817 # register the unlink with the old linked node
1818 if self.do_journal and self.properties[propname].do_journal:
1819 self.db.addjournal(link_class, id, 'unlink',
1820 (self.classname, nodeid, propname))
1821 l.remove(id)
1822 remove.append(id)
1824 # handle additions
1825 for id in value:
1826 if id in l:
1827 continue
1828 # We can safely check this condition after
1829 # checking that this is an addition to the
1830 # multilink since the condition was checked for
1831 # existing entries at the point they were added to
1832 # the multilink. Since the hasnode call will
1833 # result in a SQL query, it is more efficient to
1834 # avoid the check if possible.
1835 if not self.db.getclass(link_class).hasnode(id):
1836 raise IndexError('%s has no node %s'%(link_class,
1837 id))
1838 # register the link with the newly linked node
1839 if self.do_journal and self.properties[propname].do_journal:
1840 self.db.addjournal(link_class, id, 'link',
1841 (self.classname, nodeid, propname))
1842 l.append(id)
1843 add.append(id)
1845 # figure the journal entry
1846 l = []
1847 if add:
1848 l.append(('+', add))
1849 if remove:
1850 l.append(('-', remove))
1851 multilink_changes[propname] = (add, remove)
1852 if l:
1853 journalvalues[propname] = tuple(l)
1855 elif isinstance(prop, String):
1856 if value is not None and type(value) != type('') and type(value) != type(u''):
1857 raise TypeError('new property "%s" not a string'%propname)
1858 if prop.indexme:
1859 if value is None: value = ''
1860 self.db.indexer.add_text((self.classname, nodeid, propname),
1861 value)
1863 elif isinstance(prop, Password):
1864 if not isinstance(value, password.Password):
1865 raise TypeError('new property "%s" not a Password'%propname)
1866 propvalues[propname] = value
1867 journalvalues[propname] = \
1868 current and password.JournalPassword(current)
1870 elif value is not None and isinstance(prop, Date):
1871 if not isinstance(value, date.Date):
1872 raise TypeError('new property "%s" not a Date'% propname)
1873 propvalues[propname] = value
1875 elif value is not None and isinstance(prop, Interval):
1876 if not isinstance(value, date.Interval):
1877 raise TypeError('new property "%s" not an '
1878 'Interval'%propname)
1879 propvalues[propname] = value
1881 elif value is not None and isinstance(prop, Number):
1882 try:
1883 float(value)
1884 except ValueError:
1885 raise TypeError('new property "%s" not numeric'%propname)
1887 elif value is not None and isinstance(prop, Boolean):
1888 try:
1889 int(value)
1890 except ValueError:
1891 raise TypeError('new property "%s" not boolean'%propname)
1893 # nothing to do?
1894 if not propvalues:
1895 return propvalues
1897 # update the activity time
1898 propvalues['activity'] = date.Date()
1899 propvalues['actor'] = self.db.getuid()
1901 # do the set
1902 self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
1904 # remove the activity props now they're handled
1905 del propvalues['activity']
1906 del propvalues['actor']
1908 # journal the set
1909 if self.do_journal:
1910 self.db.addjournal(self.classname, nodeid, ''"set", journalvalues)
1912 return propvalues
1914 def retire(self, nodeid):
1915 """Retire a node.
1917 The properties on the node remain available from the get() method,
1918 and the node's id is never reused.
1920 Retired nodes are not returned by the find(), list(), or lookup()
1921 methods, and other nodes may reuse the values of their key properties.
1922 """
1923 if self.db.journaltag is None:
1924 raise DatabaseError(_('Database open read-only'))
1926 self.fireAuditors('retire', nodeid, None)
1928 # use the arg for __retired__ to cope with any odd database type
1929 # conversion (hello, sqlite)
1930 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1931 self.db.arg, self.db.arg)
1932 self.db.sql(sql, (nodeid, nodeid))
1933 if self.do_journal:
1934 self.db.addjournal(self.classname, nodeid, ''"retired", None)
1936 self.fireReactors('retire', nodeid, None)
1938 def restore(self, nodeid):
1939 """Restore a retired node.
1941 Make node available for all operations like it was before retirement.
1942 """
1943 if self.db.journaltag is None:
1944 raise DatabaseError(_('Database open read-only'))
1946 node = self.db.getnode(self.classname, nodeid)
1947 # check if key property was overrided
1948 key = self.getkey()
1949 try:
1950 id = self.lookup(node[key])
1951 except KeyError:
1952 pass
1953 else:
1954 raise KeyError("Key property (%s) of retired node clashes "
1955 "with existing one (%s)" % (key, node[key]))
1957 self.fireAuditors('restore', nodeid, None)
1958 # use the arg for __retired__ to cope with any odd database type
1959 # conversion (hello, sqlite)
1960 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1961 self.db.arg, self.db.arg)
1962 self.db.sql(sql, (0, nodeid))
1963 if self.do_journal:
1964 self.db.addjournal(self.classname, nodeid, ''"restored", None)
1966 self.fireReactors('restore', nodeid, None)
1968 def is_retired(self, nodeid):
1969 """Return true if the node is rerired
1970 """
1971 sql = 'select __retired__ from _%s where id=%s'%(self.classname,
1972 self.db.arg)
1973 self.db.sql(sql, (nodeid,))
1974 return int(self.db.sql_fetchone()[0]) > 0
1976 def destroy(self, nodeid):
1977 """Destroy a node.
1979 WARNING: this method should never be used except in extremely rare
1980 situations where there could never be links to the node being
1981 deleted
1983 WARNING: use retire() instead
1985 WARNING: the properties of this node will not be available ever again
1987 WARNING: really, use retire() instead
1989 Well, I think that's enough warnings. This method exists mostly to
1990 support the session storage of the cgi interface.
1992 The node is completely removed from the hyperdb, including all journal
1993 entries. It will no longer be available, and will generally break code
1994 if there are any references to the node.
1995 """
1996 if self.db.journaltag is None:
1997 raise DatabaseError(_('Database open read-only'))
1998 self.db.destroynode(self.classname, nodeid)
2000 # Locating nodes:
2001 def hasnode(self, nodeid):
2002 """Determine if the given nodeid actually exists
2003 """
2004 return self.db.hasnode(self.classname, nodeid)
2006 def setkey(self, propname):
2007 """Select a String property of this class to be the key property.
2009 'propname' must be the name of a String property of this class or
2010 None, or a TypeError is raised. The values of the key property on
2011 all existing nodes must be unique or a ValueError is raised.
2012 """
2013 prop = self.getprops()[propname]
2014 if not isinstance(prop, String):
2015 raise TypeError('key properties must be String')
2016 self.key = propname
2018 def getkey(self):
2019 """Return the name of the key property for this class or None."""
2020 return self.key
2022 def lookup(self, keyvalue):
2023 """Locate a particular node by its key property and return its id.
2025 If this class has no key property, a TypeError is raised. If the
2026 'keyvalue' matches one of the values for the key property among
2027 the nodes in this class, the matching node's id is returned;
2028 otherwise a KeyError is raised.
2029 """
2030 if not self.key:
2031 raise TypeError('No key property set for class %s'%self.classname)
2033 # use the arg to handle any odd database type conversion (hello,
2034 # sqlite)
2035 sql = "select id from _%s where _%s=%s and __retired__=%s"%(
2036 self.classname, self.key, self.db.arg, self.db.arg)
2037 self.db.sql(sql, (str(keyvalue), 0))
2039 # see if there was a result that's not retired
2040 row = self.db.sql_fetchone()
2041 if not row:
2042 raise KeyError('No key (%s) value "%s" for "%s"'%(self.key,
2043 keyvalue, self.classname))
2045 # return the id
2046 # XXX numeric ids
2047 return str(row[0])
2049 def find(self, **propspec):
2050 """Get the ids of nodes in this class which link to the given nodes.
2052 'propspec' consists of keyword args propname=nodeid or
2053 propname={nodeid:1, }
2054 'propname' must be the name of a property in this class, or a
2055 KeyError is raised. That property must be a Link or
2056 Multilink property, or a TypeError is raised.
2058 Any node in this class whose 'propname' property links to any of
2059 the nodeids will be returned. Examples::
2061 db.issue.find(messages='1')
2062 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
2063 """
2064 # shortcut
2065 if not propspec:
2066 return []
2068 # validate the args
2069 props = self.getprops()
2070 for propname, nodeids in propspec.iteritems():
2071 # check the prop is OK
2072 prop = props[propname]
2073 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
2074 raise TypeError("'%s' not a Link/Multilink property"%propname)
2076 # first, links
2077 a = self.db.arg
2078 allvalues = ()
2079 sql = []
2080 where = []
2081 for prop, values in propspec.iteritems():
2082 if not isinstance(props[prop], hyperdb.Link):
2083 continue
2084 if type(values) is type({}) and len(values) == 1:
2085 values = list(values)[0]
2086 if type(values) is type(''):
2087 allvalues += (values,)
2088 where.append('_%s = %s'%(prop, a))
2089 elif values is None:
2090 where.append('_%s is NULL'%prop)
2091 else:
2092 values = list(values)
2093 s = ''
2094 if None in values:
2095 values.remove(None)
2096 s = '_%s is NULL or '%prop
2097 allvalues += tuple(values)
2098 s += '_%s in (%s)'%(prop, ','.join([a]*len(values)))
2099 where.append('(' + s +')')
2100 if where:
2101 allvalues = (0, ) + allvalues
2102 sql.append("""select id from _%s where __retired__=%s
2103 and %s"""%(self.classname, a, ' and '.join(where)))
2105 # now multilinks
2106 for prop, values in propspec.iteritems():
2107 if not isinstance(props[prop], hyperdb.Multilink):
2108 continue
2109 if not values:
2110 continue
2111 allvalues += (0, )
2112 if type(values) is type(''):
2113 allvalues += (values,)
2114 s = a
2115 else:
2116 allvalues += tuple(values)
2117 s = ','.join([a]*len(values))
2118 tn = '%s_%s'%(self.classname, prop)
2119 sql.append("""select id from _%s, %s where __retired__=%s
2120 and id = %s.nodeid and %s.linkid in (%s)"""%(self.classname,
2121 tn, a, tn, tn, s))
2123 if not sql:
2124 return []
2125 sql = ' union '.join(sql)
2126 self.db.sql(sql, allvalues)
2127 # XXX numeric ids
2128 l = [str(x[0]) for x in self.db.sql_fetchall()]
2129 return l
2131 def stringFind(self, **requirements):
2132 """Locate a particular node by matching a set of its String
2133 properties in a caseless search.
2135 If the property is not a String property, a TypeError is raised.
2137 The return is a list of the id of all nodes that match.
2138 """
2139 where = []
2140 args = []
2141 for propname in requirements:
2142 prop = self.properties[propname]
2143 if not isinstance(prop, String):
2144 raise TypeError("'%s' not a String property"%propname)
2145 where.append(propname)
2146 args.append(requirements[propname].lower())
2148 # generate the where clause
2149 s = ' and '.join(['lower(_%s)=%s'%(col, self.db.arg) for col in where])
2150 sql = 'select id from _%s where %s and __retired__=%s'%(
2151 self.classname, s, self.db.arg)
2152 args.append(0)
2153 self.db.sql(sql, tuple(args))
2154 # XXX numeric ids
2155 l = [str(x[0]) for x in self.db.sql_fetchall()]
2156 return l
2158 def list(self):
2159 """ Return a list of the ids of the active nodes in this class.
2160 """
2161 return self.getnodeids(retired=0)
2163 def getnodeids(self, retired=None):
2164 """ Retrieve all the ids of the nodes for a particular Class.
2166 Set retired=None to get all nodes. Otherwise it'll get all the
2167 retired or non-retired nodes, depending on the flag.
2168 """
2169 # flip the sense of the 'retired' flag if we don't want all of them
2170 if retired is not None:
2171 args = (0, )
2172 if retired:
2173 compare = '>'
2174 else:
2175 compare = '='
2176 sql = 'select id from _%s where __retired__%s%s'%(self.classname,
2177 compare, self.db.arg)
2178 else:
2179 args = ()
2180 sql = 'select id from _%s'%self.classname
2181 self.db.sql(sql, args)
2182 # XXX numeric ids
2183 ids = [str(x[0]) for x in self.db.cursor.fetchall()]
2184 return ids
2186 def _subselect(self, classname, multilink_table):
2187 """Create a subselect. This is factored out because some
2188 databases (hmm only one, so far) doesn't support subselects
2189 look for "I can't believe it's not a toy RDBMS" in the mysql
2190 backend.
2191 """
2192 return '_%s.id not in (select nodeid from %s)'%(classname,
2193 multilink_table)
2195 # Some DBs order NULL values last. Set this variable in the backend
2196 # for prepending an order by clause for each attribute that causes
2197 # correct sort order for NULLs. Examples:
2198 # order_by_null_values = '(%s is not NULL)'
2199 # order_by_null_values = 'notnull(%s)'
2200 # The format parameter is replaced with the attribute.
2201 order_by_null_values = None
2203 def supports_subselects(self):
2204 '''Assuming DBs can do subselects, overwrite if they cannot.
2205 '''
2206 return True
2208 def _filter_multilink_expression_fallback(
2209 self, classname, multilink_table, expr):
2210 '''This is a fallback for database that do not support
2211 subselects.'''
2213 is_valid = expr.evaluate
2215 last_id, kws = None, []
2217 ids = IdListOptimizer()
2218 append = ids.append
2220 # This join and the evaluation in program space
2221 # can be expensive for larger databases!
2222 # TODO: Find a faster way to collect the data needed
2223 # to evalute the expression.
2224 # Moving the expression evaluation into the database
2225 # would be nice but this tricky: Think about the cases
2226 # where the multilink table does not have join values
2227 # needed in evaluation.
2229 stmnt = "SELECT c.id, m.linkid FROM _%s c " \
2230 "LEFT OUTER JOIN %s m " \
2231 "ON c.id = m.nodeid ORDER BY c.id" % (
2232 classname, multilink_table)
2233 self.db.sql(stmnt)
2235 # collect all multilink items for a class item
2236 for nid, kw in self.db.sql_fetchiter():
2237 if nid != last_id:
2238 if last_id is None:
2239 last_id = nid
2240 else:
2241 # we have all multilink items -> evaluate!
2242 if is_valid(kws): append(last_id)
2243 last_id, kws = nid, []
2244 if kw is not None:
2245 kws.append(kw)
2247 if last_id is not None and is_valid(kws):
2248 append(last_id)
2250 # we have ids of the classname table
2251 return ids.where("_%s.id" % classname, self.db.arg)
2253 def _filter_multilink_expression(self, classname, multilink_table, v):
2254 """ Filters out elements of the classname table that do not
2255 match the given expression.
2256 Returns tuple of 'WHERE' introns for the overall filter.
2257 """
2258 try:
2259 opcodes = [int(x) for x in v]
2260 if min(opcodes) >= -1: raise ValueError()
2262 expr = compile_expression(opcodes)
2264 if not self.supports_subselects():
2265 # We heavily rely on subselects. If there is
2266 # no decent support fall back to slower variant.
2267 return self._filter_multilink_expression_fallback(
2268 classname, multilink_table, expr)
2270 atom = \
2271 "%s IN(SELECT linkid FROM %s WHERE nodeid=a.id)" % (
2272 self.db.arg,
2273 multilink_table)
2275 intron = \
2276 "_%(classname)s.id in (SELECT id " \
2277 "FROM _%(classname)s AS a WHERE %(condition)s) " % {
2278 'classname' : classname,
2279 'condition' : expr.generate(lambda n: atom) }
2281 values = []
2282 def collect_values(n): values.append(n.x)
2283 expr.visit(collect_values)
2285 return intron, values
2286 except:
2287 # original behavior
2288 where = "%s.linkid in (%s)" % (
2289 multilink_table, ','.join([self.db.arg] * len(v)))
2290 return where, v, True # True to indicate original
2292 def _filter_sql (self, search_matches, filterspec, srt=[], grp=[], retr=0):
2293 """ Compute the proptree and the SQL/ARGS for a filter.
2294 For argument description see filter below.
2295 We return a 3-tuple, the proptree, the sql and the sql-args
2296 or None if no SQL is necessary.
2297 The flag retr serves to retrieve *all* non-Multilink properties
2298 (for filling the cache during a filter_iter)
2299 """
2300 # we can't match anything if search_matches is empty
2301 if not search_matches and search_matches is not None:
2302 return None
2304 icn = self.classname
2306 # vars to hold the components of the SQL statement
2307 frum = [] # FROM clauses
2308 loj = [] # LEFT OUTER JOIN clauses
2309 where = [] # WHERE clauses
2310 args = [] # *any* positional arguments
2311 a = self.db.arg
2313 # figure the WHERE clause from the filterspec
2314 mlfilt = 0 # are we joining with Multilink tables?
2315 sortattr = self._sortattr (group = grp, sort = srt)
2316 proptree = self._proptree(filterspec, sortattr, retr)
2317 mlseen = 0
2318 for pt in reversed(proptree.sortattr):
2319 p = pt
2320 while p.parent:
2321 if isinstance (p.propclass, Multilink):
2322 mlseen = True
2323 if mlseen:
2324 p.sort_ids_needed = True
2325 p.tree_sort_done = False
2326 p = p.parent
2327 if not mlseen:
2328 pt.attr_sort_done = pt.tree_sort_done = True
2329 proptree.compute_sort_done()
2331 cols = ['_%s.id'%icn]
2332 mlsort = []
2333 rhsnum = 0
2334 for p in proptree:
2335 rc = ac = oc = None
2336 cn = p.classname
2337 ln = p.uniqname
2338 pln = p.parent.uniqname
2339 pcn = p.parent.classname
2340 k = p.name
2341 v = p.val
2342 propclass = p.propclass
2343 if p.parent == proptree and p.name == 'id' \
2344 and 'retrieve' in p.need_for:
2345 p.sql_idx = 0
2346 if 'sort' in p.need_for or 'retrieve' in p.need_for:
2347 rc = oc = ac = '_%s._%s'%(pln, k)
2348 if isinstance(propclass, Multilink):
2349 if 'search' in p.need_for:
2350 mlfilt = 1
2351 tn = '%s_%s'%(pcn, k)
2352 if v in ('-1', ['-1'], []):
2353 # only match rows that have count(linkid)=0 in the
2354 # corresponding multilink table)
2355 where.append(self._subselect(pcn, tn))
2356 else:
2357 frum.append(tn)
2358 gen_join = True
2360 if p.has_values and isinstance(v, type([])):
2361 result = self._filter_multilink_expression(pln, tn, v)
2362 # XXX: We dont need an id join if we used the filter
2363 gen_join = len(result) == 3
2365 if gen_join:
2366 where.append('_%s.id=%s.nodeid'%(pln,tn))
2368 if p.children:
2369 frum.append('_%s as _%s' % (cn, ln))
2370 where.append('%s.linkid=_%s.id'%(tn, ln))
2372 if p.has_values:
2373 if isinstance(v, type([])):
2374 where.append(result[0])
2375 args += result[1]
2376 else:
2377 where.append('%s.linkid=%s'%(tn, a))
2378 args.append(v)
2379 if 'sort' in p.need_for:
2380 assert not p.attr_sort_done and not p.sort_ids_needed
2381 elif k == 'id':
2382 if 'search' in p.need_for:
2383 if isinstance(v, type([])):
2384 # If there are no permitted values, then the
2385 # where clause will always be false, and we
2386 # can optimize the query away.
2387 if not v:
2388 return []
2389 s = ','.join([a for x in v])
2390 where.append('_%s.%s in (%s)'%(pln, k, s))
2391 args = args + v
2392 else:
2393 where.append('_%s.%s=%s'%(pln, k, a))
2394 args.append(v)
2395 if 'sort' in p.need_for or 'retrieve' in p.need_for:
2396 rc = oc = ac = '_%s.id'%pln
2397 elif isinstance(propclass, String):
2398 if 'search' in p.need_for:
2399 if not isinstance(v, type([])):
2400 v = [v]
2402 # Quote the bits in the string that need it and then embed
2403 # in a "substring" search. Note - need to quote the '%' so
2404 # they make it through the python layer happily
2405 v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
2407 # now add to the where clause
2408 where.append('('
2409 +' and '.join(["_%s._%s LIKE '%s'"%(pln, k, s) for s in v])
2410 +')')
2411 # note: args are embedded in the query string now
2412 if 'sort' in p.need_for:
2413 oc = ac = 'lower(_%s._%s)'%(pln, k)
2414 elif isinstance(propclass, Link):
2415 if 'search' in p.need_for:
2416 if p.children:
2417 if 'sort' not in p.need_for:
2418 frum.append('_%s as _%s' % (cn, ln))
2419 where.append('_%s._%s=_%s.id'%(pln, k, ln))
2420 if p.has_values:
2421 if isinstance(v, type([])):
2422 d = {}
2423 for entry in v:
2424 if entry == '-1':
2425 entry = None
2426 d[entry] = entry
2427 l = []
2428 if None in d or not d:
2429 if None in d: del d[None]
2430 l.append('_%s._%s is NULL'%(pln, k))
2431 if d:
2432 v = list(d)
2433 s = ','.join([a for x in v])
2434 l.append('(_%s._%s in (%s))'%(pln, k, s))
2435 args = args + v
2436 if l:
2437 where.append('(' + ' or '.join(l) +')')
2438 else:
2439 if v in ('-1', None):
2440 v = None
2441 where.append('_%s._%s is NULL'%(pln, k))
2442 else:
2443 where.append('_%s._%s=%s'%(pln, k, a))
2444 args.append(v)
2445 if 'sort' in p.need_for:
2446 lp = p.cls.labelprop()
2447 oc = ac = '_%s._%s'%(pln, k)
2448 if lp != 'id':
2449 if p.tree_sort_done:
2450 loj.append(
2451 'LEFT OUTER JOIN _%s as _%s on _%s._%s=_%s.id'%(
2452 cn, ln, pln, k, ln))
2453 oc = '_%s._%s'%(ln, lp)
2454 if 'retrieve' in p.need_for:
2455 rc = '_%s._%s'%(pln, k)
2456 elif isinstance(propclass, Date) and 'search' in p.need_for:
2457 dc = self.db.to_sql_value(hyperdb.Date)
2458 if isinstance(v, type([])):
2459 s = ','.join([a for x in v])
2460 where.append('_%s._%s in (%s)'%(pln, k, s))
2461 args = args + [dc(date.Date(x)) for x in v]
2462 else:
2463 try:
2464 # Try to filter on range of dates
2465 date_rng = propclass.range_from_raw(v, self.db)
2466 if date_rng.from_value:
2467 where.append('_%s._%s >= %s'%(pln, k, a))
2468 args.append(dc(date_rng.from_value))
2469 if date_rng.to_value:
2470 where.append('_%s._%s <= %s'%(pln, k, a))
2471 args.append(dc(date_rng.to_value))
2472 except ValueError:
2473 # If range creation fails - ignore that search parameter
2474 pass
2475 elif isinstance(propclass, Interval):
2476 # filter/sort using the __<prop>_int__ column
2477 if 'search' in p.need_for:
2478 if isinstance(v, type([])):
2479 s = ','.join([a for x in v])
2480 where.append('_%s.__%s_int__ in (%s)'%(pln, k, s))
2481 args = args + [date.Interval(x).as_seconds() for x in v]
2482 else:
2483 try:
2484 # Try to filter on range of intervals
2485 date_rng = Range(v, date.Interval)
2486 if date_rng.from_value:
2487 where.append('_%s.__%s_int__ >= %s'%(pln, k, a))
2488 args.append(date_rng.from_value.as_seconds())
2489 if date_rng.to_value:
2490 where.append('_%s.__%s_int__ <= %s'%(pln, k, a))
2491 args.append(date_rng.to_value.as_seconds())
2492 except ValueError:
2493 # If range creation fails - ignore search parameter
2494 pass
2495 if 'sort' in p.need_for:
2496 oc = ac = '_%s.__%s_int__'%(pln,k)
2497 if 'retrieve' in p.need_for:
2498 rc = '_%s._%s'%(pln,k)
2499 elif isinstance(propclass, Boolean) and 'search' in p.need_for:
2500 if type(v) == type(""):
2501 v = v.split(',')
2502 if type(v) != type([]):
2503 v = [v]
2504 bv = []
2505 for val in v:
2506 if type(val) is type(''):
2507 bv.append(propclass.from_raw (val))
2508 else:
2509 bv.append(bool(val))
2510 if len(bv) == 1:
2511 where.append('_%s._%s=%s'%(pln, k, a))
2512 args = args + bv
2513 else:
2514 s = ','.join([a for x in v])
2515 where.append('_%s._%s in (%s)'%(pln, k, s))
2516 args = args + bv
2517 elif 'search' in p.need_for:
2518 if isinstance(v, type([])):
2519 s = ','.join([a for x in v])
2520 where.append('_%s._%s in (%s)'%(pln, k, s))
2521 args = args + v
2522 else:
2523 where.append('_%s._%s=%s'%(pln, k, a))
2524 args.append(v)
2525 if oc:
2526 if p.sort_ids_needed:
2527 if rc == ac:
2528 p.sql_idx = len(cols)
2529 p.auxcol = len(cols)
2530 cols.append(ac)
2531 if p.tree_sort_done and p.sort_direction:
2532 # Don't select top-level id or multilink twice
2533 if (not p.sort_ids_needed or ac != oc) and (p.name != 'id'
2534 or p.parent != proptree):
2535 if rc == oc:
2536 p.sql_idx = len(cols)
2537 cols.append(oc)
2538 desc = ['', ' desc'][p.sort_direction == '-']
2539 # Some SQL dbs sort NULL values last -- we want them first.
2540 if (self.order_by_null_values and p.name != 'id'):
2541 nv = self.order_by_null_values % oc
2542 cols.append(nv)
2543 p.orderby.append(nv + desc)
2544 p.orderby.append(oc + desc)
2545 if 'retrieve' in p.need_for and p.sql_idx is None:
2546 assert(rc)
2547 p.sql_idx = len(cols)
2548 cols.append (rc)
2550 props = self.getprops()
2552 # don't match retired nodes
2553 where.append('_%s.__retired__=0'%icn)
2555 # add results of full text search
2556 if search_matches is not None:
2557 s = ','.join([a for x in search_matches])
2558 where.append('_%s.id in (%s)'%(icn, s))
2559 args = args + [x for x in search_matches]
2561 # construct the SQL
2562 frum.append('_'+icn)
2563 frum = ','.join(frum)
2564 if where:
2565 where = ' where ' + (' and '.join(where))
2566 else:
2567 where = ''
2568 if mlfilt:
2569 # we're joining tables on the id, so we will get dupes if we
2570 # don't distinct()
2571 cols[0] = 'distinct(_%s.id)'%icn
2573 order = []
2574 # keep correct sequence of order attributes.
2575 for sa in proptree.sortattr:
2576 if not sa.attr_sort_done:
2577 continue
2578 order.extend(sa.orderby)
2579 if order:
2580 order = ' order by %s'%(','.join(order))
2581 else:
2582 order = ''
2584 cols = ','.join(cols)
2585 loj = ' '.join(loj)
2586 sql = 'select %s from %s %s %s%s'%(cols, frum, loj, where, order)
2587 args = tuple(args)
2588 __traceback_info__ = (sql, args)
2589 return proptree, sql, args
2591 def filter(self, search_matches, filterspec, sort=[], group=[]):
2592 """Return a list of the ids of the active nodes in this class that
2593 match the 'filter' spec, sorted by the group spec and then the
2594 sort spec
2596 "filterspec" is {propname: value(s)}
2598 "sort" and "group" are [(dir, prop), ...] where dir is '+', '-'
2599 or None and prop is a prop name or None. Note that for
2600 backward-compatibility reasons a single (dir, prop) tuple is
2601 also allowed.
2603 "search_matches" is a container type or None
2605 The filter must match all properties specificed. If the property
2606 value to match is a list:
2608 1. String properties must match all elements in the list, and
2609 2. Other properties must match any of the elements in the list.
2610 """
2611 if __debug__:
2612 start_t = time.time()
2614 sq = self._filter_sql (search_matches, filterspec, sort, group)
2615 # nothing to match?
2616 if sq is None:
2617 return []
2618 proptree, sql, args = sq
2620 self.db.sql(sql, args)
2621 l = self.db.sql_fetchall()
2623 # Compute values needed for sorting in proptree.sort
2624 for p in proptree:
2625 if hasattr(p, 'auxcol'):
2626 p.sort_ids = p.sort_result = [row[p.auxcol] for row in l]
2627 # return the IDs (the first column)
2628 # XXX numeric ids
2629 l = [str(row[0]) for row in l]
2630 l = proptree.sort (l)
2632 if __debug__:
2633 self.db.stats['filtering'] += (time.time() - start_t)
2634 return l
2636 def filter_iter(self, search_matches, filterspec, sort=[], group=[]):
2637 """Iterator similar to filter above with same args.
2638 Limitation: We don't sort on multilinks.
2639 This uses an optimisation: We put all nodes that are in the
2640 current row into the node cache. Then we return the node id.
2641 That way a fetch of a node won't create another sql-fetch (with
2642 a join) from the database because the nodes are already in the
2643 cache. We're using our own temporary cursor.
2644 """
2645 sq = self._filter_sql(search_matches, filterspec, sort, group, retr=1)
2646 # nothing to match?
2647 if sq is None:
2648 return
2649 proptree, sql, args = sq
2650 cursor = self.db.conn.cursor()
2651 self.db.sql(sql, args, cursor)
2652 classes = {}
2653 for p in proptree:
2654 if 'retrieve' in p.need_for:
2655 cn = p.parent.classname
2656 ptid = p.parent.id # not the nodeid!
2657 key = (cn, ptid)
2658 if key not in classes:
2659 classes[key] = {}
2660 name = p.name
2661 assert (name)
2662 classes[key][name] = p
2663 p.to_hyperdb = self.db.to_hyperdb_value(p.propclass.__class__)
2664 while True:
2665 row = cursor.fetchone()
2666 if not row: break
2667 # populate cache with current items
2668 for (classname, ptid), pt in classes.iteritems():
2669 nodeid = str(row[pt['id'].sql_idx])
2670 key = (classname, nodeid)
2671 if key in self.db.cache:
2672 self.db._cache_refresh(key)
2673 continue
2674 node = {}
2675 for propname, p in pt.iteritems():
2676 value = row[p.sql_idx]
2677 if value is not None:
2678 value = p.to_hyperdb(value)
2679 node[propname] = value
2680 self.db._cache_save(key, node)
2681 yield str(row[0])
2683 def filter_sql(self, sql):
2684 """Return a list of the ids of the items in this class that match
2685 the SQL provided. The SQL is a complete "select" statement.
2687 The SQL select must include the item id as the first column.
2689 This function DOES NOT filter out retired items, add on a where
2690 clause "__retired__=0" if you don't want retired nodes.
2691 """
2692 if __debug__:
2693 start_t = time.time()
2695 self.db.sql(sql)
2696 l = self.db.sql_fetchall()
2698 if __debug__:
2699 self.db.stats['filtering'] += (time.time() - start_t)
2700 return l
2702 def count(self):
2703 """Get the number of nodes in this class.
2705 If the returned integer is 'numnodes', the ids of all the nodes
2706 in this class run from 1 to numnodes, and numnodes+1 will be the
2707 id of the next node to be created in this class.
2708 """
2709 return self.db.countnodes(self.classname)
2711 # Manipulating properties:
2712 def getprops(self, protected=1):
2713 """Return a dictionary mapping property names to property objects.
2714 If the "protected" flag is true, we include protected properties -
2715 those which may not be modified.
2716 """
2717 d = self.properties.copy()
2718 if protected:
2719 d['id'] = String()
2720 d['creation'] = hyperdb.Date()
2721 d['activity'] = hyperdb.Date()
2722 d['creator'] = hyperdb.Link('user')
2723 d['actor'] = hyperdb.Link('user')
2724 return d
2726 def addprop(self, **properties):
2727 """Add properties to this class.
2729 The keyword arguments in 'properties' must map names to property
2730 objects, or a TypeError is raised. None of the keys in 'properties'
2731 may collide with the names of existing properties, or a ValueError
2732 is raised before any properties have been added.
2733 """
2734 for key in properties:
2735 if key in self.properties:
2736 raise ValueError(key)
2737 self.properties.update(properties)
2739 def index(self, nodeid):
2740 """Add (or refresh) the node to search indexes
2741 """
2742 # find all the String properties that have indexme
2743 for prop, propclass in self.getprops().iteritems():
2744 if isinstance(propclass, String) and propclass.indexme:
2745 self.db.indexer.add_text((self.classname, nodeid, prop),
2746 str(self.get(nodeid, prop)))
2748 #
2749 # import / export support
2750 #
2751 def export_list(self, propnames, nodeid):
2752 """ Export a node - generate a list of CSV-able data in the order
2753 specified by propnames for the given node.
2754 """
2755 properties = self.getprops()
2756 l = []
2757 for prop in propnames:
2758 proptype = properties[prop]
2759 value = self.get(nodeid, prop)
2760 # "marshal" data where needed
2761 if value is None:
2762 pass
2763 elif isinstance(proptype, hyperdb.Date):
2764 value = value.get_tuple()
2765 elif isinstance(proptype, hyperdb.Interval):
2766 value = value.get_tuple()
2767 elif isinstance(proptype, hyperdb.Password):
2768 value = str(value)
2769 l.append(repr(value))
2770 l.append(repr(self.is_retired(nodeid)))
2771 return l
2773 def import_list(self, propnames, proplist):
2774 """ Import a node - all information including "id" is present and
2775 should not be sanity checked. Triggers are not triggered. The
2776 journal should be initialised using the "creator" and "created"
2777 information.
2779 Return the nodeid of the node imported.
2780 """
2781 if self.db.journaltag is None:
2782 raise DatabaseError(_('Database open read-only'))
2783 properties = self.getprops()
2785 # make the new node's property map
2786 d = {}
2787 retire = 0
2788 if not "id" in propnames:
2789 newid = self.db.newid(self.classname)
2790 else:
2791 newid = eval(proplist[propnames.index("id")])
2792 for i in range(len(propnames)):
2793 # Use eval to reverse the repr() used to output the CSV
2794 value = eval(proplist[i])
2796 # Figure the property for this column
2797 propname = propnames[i]
2799 # "unmarshal" where necessary
2800 if propname == 'id':
2801 continue
2802 elif propname == 'is retired':
2803 # is the item retired?
2804 if int(value):
2805 retire = 1
2806 continue
2807 elif value is None:
2808 d[propname] = None
2809 continue
2811 prop = properties[propname]
2812 if value is None:
2813 # don't set Nones
2814 continue
2815 elif isinstance(prop, hyperdb.Date):
2816 value = date.Date(value)
2817 elif isinstance(prop, hyperdb.Interval):
2818 value = date.Interval(value)
2819 elif isinstance(prop, hyperdb.Password):
2820 value = password.Password(encrypted=value)
2821 elif isinstance(prop, String):
2822 if isinstance(value, unicode):
2823 value = value.encode('utf8')
2824 if not isinstance(value, str):
2825 raise TypeError('new property "%(propname)s" not a '
2826 'string: %(value)r'%locals())
2827 if prop.indexme:
2828 self.db.indexer.add_text((self.classname, newid, propname),
2829 value)
2830 d[propname] = value
2832 # get a new id if necessary
2833 if newid is None:
2834 newid = self.db.newid(self.classname)
2836 # insert new node or update existing?
2837 if not self.hasnode(newid):
2838 self.db.addnode(self.classname, newid, d) # insert
2839 else:
2840 self.db.setnode(self.classname, newid, d) # update
2842 # retire?
2843 if retire:
2844 # use the arg for __retired__ to cope with any odd database type
2845 # conversion (hello, sqlite)
2846 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
2847 self.db.arg, self.db.arg)
2848 self.db.sql(sql, (newid, newid))
2849 return newid
2851 def export_journals(self):
2852 """Export a class's journal - generate a list of lists of
2853 CSV-able data:
2855 nodeid, date, user, action, params
2857 No heading here - the columns are fixed.
2858 """
2859 properties = self.getprops()
2860 r = []
2861 for nodeid in self.getnodeids():
2862 for nodeid, date, user, action, params in self.history(nodeid):
2863 date = date.get_tuple()
2864 if action == 'set':
2865 export_data = {}
2866 for propname, value in params.iteritems():
2867 if propname not in properties:
2868 # property no longer in the schema
2869 continue
2871 prop = properties[propname]
2872 # make sure the params are eval()'able
2873 if value is None:
2874 pass
2875 elif isinstance(prop, Date):
2876 value = value.get_tuple()
2877 elif isinstance(prop, Interval):
2878 value = value.get_tuple()
2879 elif isinstance(prop, Password):
2880 value = str(value)
2881 export_data[propname] = value
2882 params = export_data
2883 elif action == 'create' and params:
2884 # old tracker with data stored in the create!
2885 params = {}
2886 l = [nodeid, date, user, action, params]
2887 r.append(list(map(repr, l)))
2888 return r
2890 class FileClass(hyperdb.FileClass, Class):
2891 """This class defines a large chunk of data. To support this, it has a
2892 mandatory String property "content" which is typically saved off
2893 externally to the hyperdb.
2895 The default MIME type of this data is defined by the
2896 "default_mime_type" class attribute, which may be overridden by each
2897 node if the class defines a "type" String property.
2898 """
2899 def __init__(self, db, classname, **properties):
2900 """The newly-created class automatically includes the "content"
2901 and "type" properties.
2902 """
2903 if 'content' not in properties:
2904 properties['content'] = hyperdb.String(indexme='yes')
2905 if 'type' not in properties:
2906 properties['type'] = hyperdb.String()
2907 Class.__init__(self, db, classname, **properties)
2909 def create(self, **propvalues):
2910 """ snaffle the file propvalue and store in a file
2911 """
2912 # we need to fire the auditors now, or the content property won't
2913 # be in propvalues for the auditors to play with
2914 self.fireAuditors('create', None, propvalues)
2916 # now remove the content property so it's not stored in the db
2917 content = propvalues['content']
2918 del propvalues['content']
2920 # do the database create
2921 newid = self.create_inner(**propvalues)
2923 # figure the mime type
2924 mime_type = propvalues.get('type', self.default_mime_type)
2926 # and index!
2927 if self.properties['content'].indexme:
2928 self.db.indexer.add_text((self.classname, newid, 'content'),
2929 content, mime_type)
2931 # store off the content as a file
2932 self.db.storefile(self.classname, newid, None, content)
2934 # fire reactors
2935 self.fireReactors('create', newid, None)
2937 return newid
2939 def get(self, nodeid, propname, default=_marker, cache=1):
2940 """ Trap the content propname and get it from the file
2942 'cache' exists for backwards compatibility, and is not used.
2943 """
2944 poss_msg = 'Possibly a access right configuration problem.'
2945 if propname == 'content':
2946 try:
2947 return self.db.getfile(self.classname, nodeid, None)
2948 except IOError, strerror:
2949 # BUG: by catching this we donot see an error in the log.
2950 return 'ERROR reading file: %s%s\n%s\n%s'%(
2951 self.classname, nodeid, poss_msg, strerror)
2952 if default is not _marker:
2953 return Class.get(self, nodeid, propname, default)
2954 else:
2955 return Class.get(self, nodeid, propname)
2957 def set(self, itemid, **propvalues):
2958 """ Snarf the "content" propvalue and update it in a file
2959 """
2960 self.fireAuditors('set', itemid, propvalues)
2961 oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid))
2963 # now remove the content property so it's not stored in the db
2964 content = None
2965 if 'content' in propvalues:
2966 content = propvalues['content']
2967 del propvalues['content']
2969 # do the database create
2970 propvalues = self.set_inner(itemid, **propvalues)
2972 # do content?
2973 if content:
2974 # store and possibly index
2975 self.db.storefile(self.classname, itemid, None, content)
2976 if self.properties['content'].indexme:
2977 mime_type = self.get(itemid, 'type', self.default_mime_type)
2978 self.db.indexer.add_text((self.classname, itemid, 'content'),
2979 content, mime_type)
2980 propvalues['content'] = content
2982 # fire reactors
2983 self.fireReactors('set', itemid, oldvalues)
2984 return propvalues
2986 def index(self, nodeid):
2987 """ Add (or refresh) the node to search indexes.
2989 Use the content-type property for the content property.
2990 """
2991 # find all the String properties that have indexme
2992 for prop, propclass in self.getprops().iteritems():
2993 if prop == 'content' and propclass.indexme:
2994 mime_type = self.get(nodeid, 'type', self.default_mime_type)
2995 self.db.indexer.add_text((self.classname, nodeid, 'content'),
2996 str(self.get(nodeid, 'content')), mime_type)
2997 elif isinstance(propclass, hyperdb.String) and propclass.indexme:
2998 # index them under (classname, nodeid, property)
2999 try:
3000 value = str(self.get(nodeid, prop))
3001 except IndexError:
3002 # node has been destroyed
3003 continue
3004 self.db.indexer.add_text((self.classname, nodeid, prop), value)
3006 # XXX deviation from spec - was called ItemClass
3007 class IssueClass(Class, roundupdb.IssueClass):
3008 # Overridden methods:
3009 def __init__(self, db, classname, **properties):
3010 """The newly-created class automatically includes the "messages",
3011 "files", "nosy", and "superseder" properties. If the 'properties'
3012 dictionary attempts to specify any of these properties or a
3013 "creation", "creator", "activity" or "actor" property, a ValueError
3014 is raised.
3015 """
3016 if 'title' not in properties:
3017 properties['title'] = hyperdb.String(indexme='yes')
3018 if 'messages' not in properties:
3019 properties['messages'] = hyperdb.Multilink("msg")
3020 if 'files' not in properties:
3021 properties['files'] = hyperdb.Multilink("file")
3022 if 'nosy' not in properties:
3023 # note: journalling is turned off as it really just wastes
3024 # space. this behaviour may be overridden in an instance
3025 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
3026 if 'superseder' not in properties:
3027 properties['superseder'] = hyperdb.Multilink(classname)
3028 Class.__init__(self, db, classname, **properties)
3030 # vim: set et sts=4 sw=4 :