1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 """ Relational database (SQL) backend common code.
20 Basics:
22 - map roundup classes to relational tables
23 - automatically detect schema changes and modify the table schemas
24 appropriately (we store the "database version" of the schema in the
25 database itself as the only row of the "schema" table)
26 - multilinks (which represent a many-to-many relationship) are handled through
27 intermediate tables
28 - journals are stored adjunct to the per-class tables
29 - table names and columns have "_" prepended so the names can't clash with
30 restricted names (like "order")
31 - retirement is determined by the __retired__ column being > 0
33 Database-specific changes may generally be pushed out to the overridable
34 sql_* methods, since everything else should be fairly generic. There's
35 probably a bit of work to be done if a database is used that actually
36 honors column typing, since the initial databases don't (sqlite stores
37 everything as a string.)
39 The schema of the hyperdb being mapped to the database is stored in the
40 database itself as a repr()'ed dictionary of information about each Class
41 that maps to a table. If that information differs from the hyperdb schema,
42 then we update it. We also store in the schema dict a version which
43 allows us to upgrade the database schema when necessary. See upgrade_db().
45 To force a unqiueness constraint on the key properties we put the item
46 id into the __retired__ column duing retirement (so it's 0 for "active"
47 items) and place a unqiueness constraint on key + __retired__. This is
48 particularly important for the users class where multiple users may
49 try to have the same username, with potentially many retired users with
50 the same name.
51 """
52 __docformat__ = 'restructuredtext'
54 # standard python modules
55 import sys, os, time, re, errno, weakref, copy, logging, datetime
57 # roundup modules
58 from roundup import hyperdb, date, password, roundupdb, security, support
59 from roundup.hyperdb import String, Password, Date, Interval, Link, \
60 Multilink, DatabaseError, Boolean, Number, Node
61 from roundup.backends import locking
62 from roundup.support import reversed
63 from roundup.i18n import _
66 # support
67 from roundup.backends.blobfiles import FileStorage
68 try:
69 from roundup.backends.indexer_xapian import Indexer
70 except ImportError:
71 from roundup.backends.indexer_rdbms import Indexer
72 from roundup.backends.sessions_rdbms import Sessions, OneTimeKeys
73 from roundup.date import Range
75 from roundup.backends.back_anydbm import compile_expression
78 # dummy value meaning "argument not passed"
79 _marker = []
81 def _num_cvt(num):
82 num = str(num)
83 try:
84 return int(num)
85 except:
86 return float(num)
88 def _bool_cvt(value):
89 if value in ('TRUE', 'FALSE'):
90 return {'TRUE': 1, 'FALSE': 0}[value]
91 # assume it's a number returned from the db API
92 return int(value)
94 def date_to_hyperdb_value(d):
95 """ convert date d to a roundup date """
96 if isinstance (d, datetime.datetime):
97 return date.Date(d)
98 return date.Date (str(d).replace(' ', '.'))
101 def connection_dict(config, dbnamestr=None):
102 """ Used by Postgresql and MySQL to detemine the keyword args for
103 opening the database connection."""
104 d = { }
105 if dbnamestr:
106 d[dbnamestr] = config.RDBMS_NAME
107 for name in ('host', 'port', 'password', 'user', 'read_default_group',
108 'read_default_file'):
109 cvar = 'RDBMS_'+name.upper()
110 if config[cvar] is not None:
111 d[name] = config[cvar]
112 return d
115 class IdListOptimizer:
116 """ To prevent flooding the SQL parser of the underlaying
117 db engine with "x IN (1, 2, 3, ..., <large number>)" collapses
118 these cases to "x BETWEEN 1 AND <large number>".
119 """
121 def __init__(self):
122 self.ranges = []
123 self.singles = []
125 def append(self, nid):
126 """ Invariant: nid are ordered ascending """
127 if self.ranges:
128 last = self.ranges[-1]
129 if last[1] == nid-1:
130 last[1] = nid
131 return
132 if self.singles:
133 last = self.singles[-1]
134 if last == nid-1:
135 self.singles.pop()
136 self.ranges.append([last, nid])
137 return
138 self.singles.append(nid)
140 def where(self, field, placeholder):
141 ranges = self.ranges
142 singles = self.singles
144 if not singles and not ranges: return "(1=0)", []
146 if ranges:
147 between = '%s BETWEEN %s AND %s' % (
148 field, placeholder, placeholder)
149 stmnt = [between] * len(ranges)
150 else:
151 stmnt = []
152 if singles:
153 stmnt.append('%s in (%s)' % (
154 field, ','.join([placeholder]*len(singles))))
156 return '(%s)' % ' OR '.join(stmnt), sum(ranges, []) + singles
158 def __str__(self):
159 return "ranges: %r / singles: %r" % (self.ranges, self.singles)
162 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
163 """ Wrapper around an SQL database that presents a hyperdb interface.
165 - some functionality is specific to the actual SQL database, hence
166 the sql_* methods that are NotImplemented
167 - we keep a cache of the latest N row fetches (where N is configurable).
168 """
169 def __init__(self, config, journaltag=None):
170 """ Open the database and load the schema from it.
171 """
172 FileStorage.__init__(self, config.UMASK)
173 self.config, self.journaltag = config, journaltag
174 self.dir = config.DATABASE
175 self.classes = {}
176 self.indexer = Indexer(self)
177 self.security = security.Security(self)
179 # additional transaction support for external files and the like
180 self.transactions = []
182 # keep a cache of the N most recently retrieved rows of any kind
183 # (classname, nodeid) = row
184 self.cache_size = config.RDBMS_CACHE_SIZE
185 self.clearCache()
186 self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0,
187 'filtering': 0}
189 # database lock
190 self.lockfile = None
192 # open a connection to the database, creating the "conn" attribute
193 self.open_connection()
195 def clearCache(self):
196 self.cache = {}
197 self.cache_lru = []
199 def getSessionManager(self):
200 return Sessions(self)
202 def getOTKManager(self):
203 return OneTimeKeys(self)
205 def open_connection(self):
206 """ Open a connection to the database, creating it if necessary.
208 Must call self.load_dbschema()
209 """
210 raise NotImplemented
212 def sql(self, sql, args=None, cursor=None):
213 """ Execute the sql with the optional args.
214 """
215 self.log_debug('SQL %r %r'%(sql, args))
216 if not cursor:
217 cursor = self.cursor
218 if args:
219 cursor.execute(sql, args)
220 else:
221 cursor.execute(sql)
223 def sql_fetchone(self):
224 """ Fetch a single row. If there's nothing to fetch, return None.
225 """
226 return self.cursor.fetchone()
228 def sql_fetchall(self):
229 """ Fetch all rows. If there's nothing to fetch, return [].
230 """
231 return self.cursor.fetchall()
233 def sql_fetchiter(self):
234 """ Fetch all row as a generator
235 """
236 while True:
237 row = self.cursor.fetchone()
238 if not row: break
239 yield row
241 def sql_stringquote(self, value):
242 """ Quote the string so it's safe to put in the 'sql quotes'
243 """
244 return re.sub("'", "''", str(value))
246 def init_dbschema(self):
247 self.database_schema = {
248 'version': self.current_db_version,
249 'tables': {}
250 }
252 def load_dbschema(self):
253 """ Load the schema definition that the database currently implements
254 """
255 self.cursor.execute('select schema from schema')
256 schema = self.cursor.fetchone()
257 if schema:
258 self.database_schema = eval(schema[0])
259 else:
260 self.database_schema = {}
262 def save_dbschema(self):
263 """ Save the schema definition that the database currently implements
264 """
265 s = repr(self.database_schema)
266 self.sql('delete from schema')
267 self.sql('insert into schema values (%s)'%self.arg, (s,))
269 def post_init(self):
270 """ Called once the schema initialisation has finished.
272 We should now confirm that the schema defined by our "classes"
273 attribute actually matches the schema in the database.
274 """
275 save = 0
277 # handle changes in the schema
278 tables = self.database_schema['tables']
279 for classname, spec in self.classes.iteritems():
280 if classname in tables:
281 dbspec = tables[classname]
282 if self.update_class(spec, dbspec):
283 tables[classname] = spec.schema()
284 save = 1
285 else:
286 self.create_class(spec)
287 tables[classname] = spec.schema()
288 save = 1
290 for classname, spec in list(tables.items()):
291 if classname not in self.classes:
292 self.drop_class(classname, tables[classname])
293 del tables[classname]
294 save = 1
296 # now upgrade the database for column type changes, new internal
297 # tables, etc.
298 save = save | self.upgrade_db()
300 # update the database version of the schema
301 if save:
302 self.save_dbschema()
304 # reindex the db if necessary
305 if self.indexer.should_reindex():
306 self.reindex()
308 # commit
309 self.sql_commit()
311 # update this number when we need to make changes to the SQL structure
312 # of the backen database
313 current_db_version = 5
314 db_version_updated = False
315 def upgrade_db(self):
316 """ Update the SQL database to reflect changes in the backend code.
318 Return boolean whether we need to save the schema.
319 """
320 version = self.database_schema.get('version', 1)
321 if version > self.current_db_version:
322 raise DatabaseError('attempting to run rev %d DATABASE with rev '
323 '%d CODE!'%(version, self.current_db_version))
324 if version == self.current_db_version:
325 # nothing to do
326 return 0
328 if version < 2:
329 self.log_info('upgrade to version 2')
330 # change the schema structure
331 self.database_schema = {'tables': self.database_schema}
333 # version 1 didn't have the actor column (note that in
334 # MySQL this will also transition the tables to typed columns)
335 self.add_new_columns_v2()
337 # version 1 doesn't have the OTK, session and indexing in the
338 # database
339 self.create_version_2_tables()
341 if version < 3:
342 self.log_info('upgrade to version 3')
343 self.fix_version_2_tables()
345 if version < 4:
346 self.fix_version_3_tables()
348 if version < 5:
349 self.fix_version_4_tables()
351 self.database_schema['version'] = self.current_db_version
352 self.db_version_updated = True
353 return 1
355 def fix_version_3_tables(self):
356 # drop the shorter VARCHAR OTK column and add a new TEXT one
357 for name in ('otk', 'session'):
358 self.sql('DELETE FROM %ss'%name)
359 self.sql('ALTER TABLE %ss DROP %s_value'%(name, name))
360 self.sql('ALTER TABLE %ss ADD %s_value TEXT'%(name, name))
362 def fix_version_2_tables(self):
363 # Default (used by sqlite): NOOP
364 pass
366 def fix_version_4_tables(self):
367 # note this is an explicit call now
368 c = self.cursor
369 for cn, klass in self.classes.iteritems():
370 c.execute('select id from _%s where __retired__<>0'%(cn,))
371 for (id,) in c.fetchall():
372 c.execute('update _%s set __retired__=%s where id=%s'%(cn,
373 self.arg, self.arg), (id, id))
375 if klass.key:
376 self.add_class_key_required_unique_constraint(cn, klass.key)
378 def _convert_journal_tables(self):
379 """Get current journal table contents, drop the table and re-create"""
380 c = self.cursor
381 cols = ','.join('nodeid date tag action params'.split())
382 for klass in self.classes.itervalues():
383 # slurp and drop
384 sql = 'select %s from %s__journal order by date'%(cols,
385 klass.classname)
386 c.execute(sql)
387 contents = c.fetchall()
388 self.drop_journal_table_indexes(klass.classname)
389 c.execute('drop table %s__journal'%klass.classname)
391 # re-create and re-populate
392 self.create_journal_table(klass)
393 a = self.arg
394 sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(
395 klass.classname, cols, a, a, a, a, a)
396 for row in contents:
397 # no data conversion needed
398 self.cursor.execute(sql, row)
400 def _convert_string_properties(self):
401 """Get current Class tables that contain String properties, and
402 convert the VARCHAR columns to TEXT"""
403 c = self.cursor
404 for klass in self.classes.itervalues():
405 # slurp and drop
406 cols, mls = self.determine_columns(list(klass.properties.iteritems()))
407 scols = ','.join([i[0] for i in cols])
408 sql = 'select id,%s from _%s'%(scols, klass.classname)
409 c.execute(sql)
410 contents = c.fetchall()
411 self.drop_class_table_indexes(klass.classname, klass.getkey())
412 c.execute('drop table _%s'%klass.classname)
414 # re-create and re-populate
415 self.create_class_table(klass, create_sequence=0)
416 a = ','.join([self.arg for i in range(len(cols)+1)])
417 sql = 'insert into _%s (id,%s) values (%s)'%(klass.classname,
418 scols, a)
419 for row in contents:
420 l = []
421 for entry in row:
422 # mysql will already be a string - psql needs "help"
423 if entry is not None and not isinstance(entry, type('')):
424 entry = str(entry)
425 l.append(entry)
426 self.cursor.execute(sql, l)
428 def refresh_database(self):
429 self.post_init()
432 def reindex(self, classname=None, show_progress=False):
433 if classname:
434 classes = [self.getclass(classname)]
435 else:
436 classes = list(self.classes.itervalues())
437 for klass in classes:
438 if show_progress:
439 for nodeid in support.Progress('Reindex %s'%klass.classname,
440 klass.list()):
441 klass.index(nodeid)
442 else:
443 for nodeid in klass.list():
444 klass.index(nodeid)
445 self.indexer.save_index()
447 hyperdb_to_sql_datatypes = {
448 hyperdb.String : 'TEXT',
449 hyperdb.Date : 'TIMESTAMP',
450 hyperdb.Link : 'INTEGER',
451 hyperdb.Interval : 'VARCHAR(255)',
452 hyperdb.Password : 'VARCHAR(255)',
453 hyperdb.Boolean : 'BOOLEAN',
454 hyperdb.Number : 'REAL',
455 }
457 def hyperdb_to_sql_datatype(self, propclass):
459 datatype = self.hyperdb_to_sql_datatypes.get(propclass)
460 if datatype:
461 return datatype
463 for k, v in self.hyperdb_to_sql_datatypes.iteritems():
464 if issubclass(propclass, k):
465 return v
467 raise ValueError('%r is not a hyperdb property class' % propclass)
469 def determine_columns(self, properties):
470 """ Figure the column names and multilink properties from the spec
472 "properties" is a list of (name, prop) where prop may be an
473 instance of a hyperdb "type" _or_ a string repr of that type.
474 """
475 cols = [
476 ('_actor', self.hyperdb_to_sql_datatype(hyperdb.Link)),
477 ('_activity', self.hyperdb_to_sql_datatype(hyperdb.Date)),
478 ('_creator', self.hyperdb_to_sql_datatype(hyperdb.Link)),
479 ('_creation', self.hyperdb_to_sql_datatype(hyperdb.Date)),
480 ]
481 mls = []
482 # add the multilinks separately
483 for col, prop in properties:
484 if isinstance(prop, Multilink):
485 mls.append(col)
486 continue
488 if isinstance(prop, type('')):
489 raise ValueError("string property spec!")
490 #and prop.find('Multilink') != -1:
491 #mls.append(col)
493 datatype = self.hyperdb_to_sql_datatype(prop.__class__)
494 cols.append(('_'+col, datatype))
496 # Intervals stored as two columns
497 if isinstance(prop, Interval):
498 cols.append(('__'+col+'_int__', 'BIGINT'))
500 cols.sort()
501 return cols, mls
503 def update_class(self, spec, old_spec, force=0):
504 """ Determine the differences between the current spec and the
505 database version of the spec, and update where necessary.
507 If 'force' is true, update the database anyway.
508 """
509 new_spec = spec.schema()
510 new_spec[1].sort()
511 old_spec[1].sort()
512 if not force and new_spec == old_spec:
513 # no changes
514 return 0
516 logger = logging.getLogger('roundup.hyperdb')
517 logger.info('update_class %s'%spec.classname)
519 logger.debug('old_spec %r'%(old_spec,))
520 logger.debug('new_spec %r'%(new_spec,))
522 # detect key prop change for potential index change
523 keyprop_changes = {}
524 if new_spec[0] != old_spec[0]:
525 if old_spec[0]:
526 keyprop_changes['remove'] = old_spec[0]
527 if new_spec[0]:
528 keyprop_changes['add'] = new_spec[0]
530 # detect multilinks that have been removed, and drop their table
531 old_has = {}
532 for name, prop in old_spec[1]:
533 old_has[name] = 1
534 if name in spec.properties:
535 continue
537 if prop.find('Multilink to') != -1:
538 # first drop indexes.
539 self.drop_multilink_table_indexes(spec.classname, name)
541 # now the multilink table itself
542 sql = 'drop table %s_%s'%(spec.classname, name)
543 else:
544 # if this is the key prop, drop the index first
545 if old_spec[0] == prop:
546 self.drop_class_table_key_index(spec.classname, name)
547 del keyprop_changes['remove']
549 # drop the column
550 sql = 'alter table _%s drop column _%s'%(spec.classname, name)
552 self.sql(sql)
554 # if we didn't remove the key prop just then, but the key prop has
555 # changed, we still need to remove the old index
556 if 'remove' in keyprop_changes:
557 self.drop_class_table_key_index(spec.classname,
558 keyprop_changes['remove'])
560 # add new columns
561 for propname, prop in new_spec[1]:
562 if propname in old_has:
563 continue
564 prop = spec.properties[propname]
565 if isinstance(prop, Multilink):
566 self.create_multilink_table(spec, propname)
567 else:
568 # add the column
569 coltype = self.hyperdb_to_sql_datatype(prop.__class__)
570 sql = 'alter table _%s add column _%s %s'%(
571 spec.classname, propname, coltype)
572 self.sql(sql)
574 # extra Interval column
575 if isinstance(prop, Interval):
576 sql = 'alter table _%s add column __%s_int__ BIGINT'%(
577 spec.classname, propname)
578 self.sql(sql)
580 # if the new column is a key prop, we need an index!
581 if new_spec[0] == propname:
582 self.create_class_table_key_index(spec.classname, propname)
583 del keyprop_changes['add']
585 # if we didn't add the key prop just then, but the key prop has
586 # changed, we still need to add the new index
587 if 'add' in keyprop_changes:
588 self.create_class_table_key_index(spec.classname,
589 keyprop_changes['add'])
591 return 1
593 def determine_all_columns(self, spec):
594 """Figure out the columns from the spec and also add internal columns
596 """
597 cols, mls = self.determine_columns(list(spec.properties.iteritems()))
599 # add on our special columns
600 cols.append(('id', 'INTEGER PRIMARY KEY'))
601 cols.append(('__retired__', 'INTEGER DEFAULT 0'))
602 return cols, mls
604 def create_class_table(self, spec):
605 """Create the class table for the given Class "spec". Creates the
606 indexes too."""
607 cols, mls = self.determine_all_columns(spec)
609 # create the base table
610 scols = ','.join(['%s %s'%x for x in cols])
611 sql = 'create table _%s (%s)'%(spec.classname, scols)
612 self.sql(sql)
614 self.create_class_table_indexes(spec)
616 return cols, mls
618 def create_class_table_indexes(self, spec):
619 """ create the class table for the given spec
620 """
621 # create __retired__ index
622 index_sql2 = 'create index _%s_retired_idx on _%s(__retired__)'%(
623 spec.classname, spec.classname)
624 self.sql(index_sql2)
626 # create index for key property
627 if spec.key:
628 index_sql3 = 'create index _%s_%s_idx on _%s(_%s)'%(
629 spec.classname, spec.key,
630 spec.classname, spec.key)
631 self.sql(index_sql3)
633 # and the unique index for key / retired(id)
634 self.add_class_key_required_unique_constraint(spec.classname,
635 spec.key)
637 # TODO: create indexes on (selected?) Link property columns, as
638 # they're more likely to be used for lookup
640 def add_class_key_required_unique_constraint(self, cn, key):
641 sql = '''create unique index _%s_key_retired_idx
642 on _%s(__retired__, _%s)'''%(cn, cn, key)
643 self.sql(sql)
645 def drop_class_table_indexes(self, cn, key):
646 # drop the old table indexes first
647 l = ['_%s_id_idx'%cn, '_%s_retired_idx'%cn]
648 if key:
649 l.append('_%s_%s_idx'%(cn, key))
651 table_name = '_%s'%cn
652 for index_name in l:
653 if not self.sql_index_exists(table_name, index_name):
654 continue
655 index_sql = 'drop index '+index_name
656 self.sql(index_sql)
658 def create_class_table_key_index(self, cn, key):
659 """ create the class table for the given spec
660 """
661 sql = 'create index _%s_%s_idx on _%s(_%s)'%(cn, key, cn, key)
662 self.sql(sql)
664 def drop_class_table_key_index(self, cn, key):
665 table_name = '_%s'%cn
666 index_name = '_%s_%s_idx'%(cn, key)
667 if self.sql_index_exists(table_name, index_name):
668 sql = 'drop index '+index_name
669 self.sql(sql)
671 # and now the retired unique index too
672 index_name = '_%s_key_retired_idx'%cn
673 if self.sql_index_exists(table_name, index_name):
674 sql = 'drop index '+index_name
675 self.sql(sql)
677 def create_journal_table(self, spec):
678 """ create the journal table for a class given the spec and
679 already-determined cols
680 """
681 # journal table
682 cols = ','.join(['%s varchar'%x
683 for x in 'nodeid date tag action params'.split()])
684 sql = """create table %s__journal (
685 nodeid integer, date %s, tag varchar(255),
686 action varchar(255), params text)""" % (spec.classname,
687 self.hyperdb_to_sql_datatype(hyperdb.Date))
688 self.sql(sql)
689 self.create_journal_table_indexes(spec)
691 def create_journal_table_indexes(self, spec):
692 # index on nodeid
693 sql = 'create index %s_journ_idx on %s__journal(nodeid)'%(
694 spec.classname, spec.classname)
695 self.sql(sql)
697 def drop_journal_table_indexes(self, classname):
698 index_name = '%s_journ_idx'%classname
699 if not self.sql_index_exists('%s__journal'%classname, index_name):
700 return
701 index_sql = 'drop index '+index_name
702 self.sql(index_sql)
704 def create_multilink_table(self, spec, ml):
705 """ Create a multilink table for the "ml" property of the class
706 given by the spec
707 """
708 # create the table
709 sql = 'create table %s_%s (linkid INTEGER, nodeid INTEGER)'%(
710 spec.classname, ml)
711 self.sql(sql)
712 self.create_multilink_table_indexes(spec, ml)
714 def create_multilink_table_indexes(self, spec, ml):
715 # create index on linkid
716 index_sql = 'create index %s_%s_l_idx on %s_%s(linkid)'%(
717 spec.classname, ml, spec.classname, ml)
718 self.sql(index_sql)
720 # create index on nodeid
721 index_sql = 'create index %s_%s_n_idx on %s_%s(nodeid)'%(
722 spec.classname, ml, spec.classname, ml)
723 self.sql(index_sql)
725 def drop_multilink_table_indexes(self, classname, ml):
726 l = [
727 '%s_%s_l_idx'%(classname, ml),
728 '%s_%s_n_idx'%(classname, ml)
729 ]
730 table_name = '%s_%s'%(classname, ml)
731 for index_name in l:
732 if not self.sql_index_exists(table_name, index_name):
733 continue
734 index_sql = 'drop index %s'%index_name
735 self.sql(index_sql)
737 def create_class(self, spec):
738 """ Create a database table according to the given spec.
739 """
740 cols, mls = self.create_class_table(spec)
741 self.create_journal_table(spec)
743 # now create the multilink tables
744 for ml in mls:
745 self.create_multilink_table(spec, ml)
747 def drop_class(self, cn, spec):
748 """ Drop the given table from the database.
750 Drop the journal and multilink tables too.
751 """
752 properties = spec[1]
753 # figure the multilinks
754 mls = []
755 for propname, prop in properties:
756 if isinstance(prop, Multilink):
757 mls.append(propname)
759 # drop class table and indexes
760 self.drop_class_table_indexes(cn, spec[0])
762 self.drop_class_table(cn)
764 # drop journal table and indexes
765 self.drop_journal_table_indexes(cn)
766 sql = 'drop table %s__journal'%cn
767 self.sql(sql)
769 for ml in mls:
770 # drop multilink table and indexes
771 self.drop_multilink_table_indexes(cn, ml)
772 sql = 'drop table %s_%s'%(spec.classname, ml)
773 self.sql(sql)
775 def drop_class_table(self, cn):
776 sql = 'drop table _%s'%cn
777 self.sql(sql)
779 #
780 # Classes
781 #
782 def __getattr__(self, classname):
783 """ A convenient way of calling self.getclass(classname).
784 """
785 if classname in self.classes:
786 return self.classes[classname]
787 raise AttributeError(classname)
789 def addclass(self, cl):
790 """ Add a Class to the hyperdatabase.
791 """
792 cn = cl.classname
793 if cn in self.classes:
794 raise ValueError(cn)
795 self.classes[cn] = cl
797 # add default Edit and View permissions
798 self.security.addPermission(name="Create", klass=cn,
799 description="User is allowed to create "+cn)
800 self.security.addPermission(name="Edit", klass=cn,
801 description="User is allowed to edit "+cn)
802 self.security.addPermission(name="View", klass=cn,
803 description="User is allowed to access "+cn)
805 def getclasses(self):
806 """ Return a list of the names of all existing classes.
807 """
808 return sorted(self.classes)
810 def getclass(self, classname):
811 """Get the Class object representing a particular class.
813 If 'classname' is not a valid class name, a KeyError is raised.
814 """
815 try:
816 return self.classes[classname]
817 except KeyError:
818 raise KeyError('There is no class called "%s"'%classname)
820 def clear(self):
821 """Delete all database contents.
823 Note: I don't commit here, which is different behaviour to the
824 "nuke from orbit" behaviour in the dbs.
825 """
826 logging.getLogger('roundup.hyperdb').info('clear')
827 for cn in self.classes:
828 sql = 'delete from _%s'%cn
829 self.sql(sql)
831 #
832 # Nodes
833 #
835 hyperdb_to_sql_value = {
836 hyperdb.String : str,
837 # fractional seconds by default
838 hyperdb.Date : lambda x: x.formal(sep=' ', sec='%06.3f'),
839 hyperdb.Link : int,
840 hyperdb.Interval : str,
841 hyperdb.Password : str,
842 hyperdb.Boolean : lambda x: x and 'TRUE' or 'FALSE',
843 hyperdb.Number : lambda x: x,
844 hyperdb.Multilink : lambda x: x, # used in journal marshalling
845 }
847 def to_sql_value(self, propklass):
849 fn = self.hyperdb_to_sql_value.get(propklass)
850 if fn:
851 return fn
853 for k, v in self.hyperdb_to_sql_value.iteritems():
854 if issubclass(propklass, k):
855 return v
857 raise ValueError('%r is not a hyperdb property class' % propklass)
859 def _cache_del(self, key):
860 del self.cache[key]
861 self.cache_lru.remove(key)
863 def _cache_refresh(self, key):
864 self.cache_lru.remove(key)
865 self.cache_lru.insert(0, key)
867 def _cache_save(self, key, node):
868 self.cache[key] = node
869 # update the LRU
870 self.cache_lru.insert(0, key)
871 if len(self.cache_lru) > self.cache_size:
872 del self.cache[self.cache_lru.pop()]
874 def addnode(self, classname, nodeid, node):
875 """ Add the specified node to its class's db.
876 """
877 self.log_debug('addnode %s%s %r'%(classname,
878 nodeid, node))
880 # determine the column definitions and multilink tables
881 cl = self.classes[classname]
882 cols, mls = self.determine_columns(list(cl.properties.iteritems()))
884 # we'll be supplied these props if we're doing an import
885 values = node.copy()
886 if 'creator' not in values:
887 # add in the "calculated" properties (dupe so we don't affect
888 # calling code's node assumptions)
889 values['creation'] = values['activity'] = date.Date()
890 values['actor'] = values['creator'] = self.getuid()
892 cl = self.classes[classname]
893 props = cl.getprops(protected=1)
894 del props['id']
896 # default the non-multilink columns
897 for col, prop in props.iteritems():
898 if col not in values:
899 if isinstance(prop, Multilink):
900 values[col] = []
901 else:
902 values[col] = None
904 # clear this node out of the cache if it's in there
905 key = (classname, nodeid)
906 if key in self.cache:
907 self._cache_del(key)
909 # figure the values to insert
910 vals = []
911 for col,dt in cols:
912 # this is somewhat dodgy....
913 if col.endswith('_int__'):
914 # XXX eugh, this test suxxors
915 value = values[col[2:-6]]
916 # this is an Interval special "int" column
917 if value is not None:
918 vals.append(value.as_seconds())
919 else:
920 vals.append(value)
921 continue
923 prop = props[col[1:]]
924 value = values[col[1:]]
925 if value is not None:
926 value = self.to_sql_value(prop.__class__)(value)
927 vals.append(value)
928 vals.append(nodeid)
929 vals = tuple(vals)
931 # make sure the ordering is correct for column name -> column value
932 s = ','.join([self.arg for x in cols]) + ',%s'%self.arg
933 cols = ','.join([col for col,dt in cols]) + ',id'
935 # perform the inserts
936 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
937 self.sql(sql, vals)
939 # insert the multilink rows
940 for col in mls:
941 t = '%s_%s'%(classname, col)
942 for entry in node[col]:
943 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
944 self.arg, self.arg)
945 self.sql(sql, (entry, nodeid))
947 def setnode(self, classname, nodeid, values, multilink_changes={}):
948 """ Change the specified node.
949 """
950 self.log_debug('setnode %s%s %r'
951 % (classname, nodeid, values))
953 # clear this node out of the cache if it's in there
954 key = (classname, nodeid)
955 if key in self.cache:
956 self._cache_del(key)
958 cl = self.classes[classname]
959 props = cl.getprops()
961 cols = []
962 mls = []
963 # add the multilinks separately
964 for col in values:
965 prop = props[col]
966 if isinstance(prop, Multilink):
967 mls.append(col)
968 elif isinstance(prop, Interval):
969 # Intervals store the seconds value too
970 cols.append(col)
971 # extra leading '_' added by code below
972 cols.append('_' +col + '_int__')
973 else:
974 cols.append(col)
975 cols.sort()
977 # figure the values to insert
978 vals = []
979 for col in cols:
980 if col.endswith('_int__'):
981 # XXX eugh, this test suxxors
982 # Intervals store the seconds value too
983 col = col[1:-6]
984 prop = props[col]
985 value = values[col]
986 if value is None:
987 vals.append(None)
988 else:
989 vals.append(value.as_seconds())
990 else:
991 prop = props[col]
992 value = values[col]
993 if value is None:
994 e = None
995 else:
996 e = self.to_sql_value(prop.__class__)(value)
997 vals.append(e)
999 vals.append(int(nodeid))
1000 vals = tuple(vals)
1002 # if there's any updates to regular columns, do them
1003 if cols:
1004 # make sure the ordering is correct for column name -> column value
1005 s = ','.join(['_%s=%s'%(x, self.arg) for x in cols])
1006 cols = ','.join(cols)
1008 # perform the update
1009 sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
1010 self.sql(sql, vals)
1012 # we're probably coming from an import, not a change
1013 if not multilink_changes:
1014 for name in mls:
1015 prop = props[name]
1016 value = values[name]
1018 t = '%s_%s'%(classname, name)
1020 # clear out previous values for this node
1021 # XXX numeric ids
1022 self.sql('delete from %s where nodeid=%s'%(t, self.arg),
1023 (nodeid,))
1025 # insert the values for this node
1026 for entry in values[name]:
1027 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
1028 self.arg, self.arg)
1029 # XXX numeric ids
1030 self.sql(sql, (entry, nodeid))
1032 # we have multilink changes to apply
1033 for col, (add, remove) in multilink_changes.iteritems():
1034 tn = '%s_%s'%(classname, col)
1035 if add:
1036 sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
1037 self.arg, self.arg)
1038 for addid in add:
1039 # XXX numeric ids
1040 self.sql(sql, (int(nodeid), int(addid)))
1041 if remove:
1042 s = ','.join([self.arg]*len(remove))
1043 sql = 'delete from %s where nodeid=%s and linkid in (%s)'%(tn,
1044 self.arg, s)
1045 # XXX numeric ids
1046 self.sql(sql, [int(nodeid)] + remove)
1048 sql_to_hyperdb_value = {
1049 hyperdb.String : str,
1050 hyperdb.Date : date_to_hyperdb_value,
1051 # hyperdb.Link : int, # XXX numeric ids
1052 hyperdb.Link : str,
1053 hyperdb.Interval : date.Interval,
1054 hyperdb.Password : lambda x: password.Password(encrypted=x),
1055 hyperdb.Boolean : _bool_cvt,
1056 hyperdb.Number : _num_cvt,
1057 hyperdb.Multilink : lambda x: x, # used in journal marshalling
1058 }
1060 def to_hyperdb_value(self, propklass):
1062 fn = self.sql_to_hyperdb_value.get(propklass)
1063 if fn:
1064 return fn
1066 for k, v in self.sql_to_hyperdb_value.iteritems():
1067 if issubclass(propklass, k):
1068 return v
1070 raise ValueError('%r is not a hyperdb property class' % propklass)
1072 def getnode(self, classname, nodeid):
1073 """ Get a node from the database.
1074 """
1075 # see if we have this node cached
1076 key = (classname, nodeid)
1077 if key in self.cache:
1078 # push us back to the top of the LRU
1079 self._cache_refresh(key)
1080 if __debug__:
1081 self.stats['cache_hits'] += 1
1082 # return the cached information
1083 return self.cache[key]
1085 if __debug__:
1086 self.stats['cache_misses'] += 1
1087 start_t = time.time()
1089 # figure the columns we're fetching
1090 cl = self.classes[classname]
1091 cols, mls = self.determine_columns(list(cl.properties.iteritems()))
1092 scols = ','.join([col for col,dt in cols])
1094 # perform the basic property fetch
1095 sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
1096 self.sql(sql, (nodeid,))
1098 values = self.sql_fetchone()
1099 if values is None:
1100 raise IndexError('no such %s node %s'%(classname, nodeid))
1102 # make up the node
1103 node = {}
1104 props = cl.getprops(protected=1)
1105 for col in range(len(cols)):
1106 name = cols[col][0][1:]
1107 if name.endswith('_int__'):
1108 # XXX eugh, this test suxxors
1109 # ignore the special Interval-as-seconds column
1110 continue
1111 value = values[col]
1112 if value is not None:
1113 value = self.to_hyperdb_value(props[name].__class__)(value)
1114 node[name] = value
1116 # save off in the cache
1117 key = (classname, nodeid)
1118 self._cache_save(key, node)
1120 if __debug__:
1121 self.stats['get_items'] += (time.time() - start_t)
1123 return node
1125 def destroynode(self, classname, nodeid):
1126 """Remove a node from the database. Called exclusively by the
1127 destroy() method on Class.
1128 """
1129 logging.getLogger('roundup.hyperdb').info('destroynode %s%s'%(
1130 classname, nodeid))
1132 # make sure the node exists
1133 if not self.hasnode(classname, nodeid):
1134 raise IndexError('%s has no node %s'%(classname, nodeid))
1136 # see if we have this node cached
1137 if (classname, nodeid) in self.cache:
1138 del self.cache[(classname, nodeid)]
1140 # see if there's any obvious commit actions that we should get rid of
1141 for entry in self.transactions[:]:
1142 if entry[1][:2] == (classname, nodeid):
1143 self.transactions.remove(entry)
1145 # now do the SQL
1146 sql = 'delete from _%s where id=%s'%(classname, self.arg)
1147 self.sql(sql, (nodeid,))
1149 # remove from multilnks
1150 cl = self.getclass(classname)
1151 x, mls = self.determine_columns(list(cl.properties.iteritems()))
1152 for col in mls:
1153 # get the link ids
1154 sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
1155 self.sql(sql, (nodeid,))
1157 # remove journal entries
1158 sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
1159 self.sql(sql, (nodeid,))
1161 # cleanup any blob filestorage when we commit
1162 self.transactions.append((FileStorage.destroy, (self, classname, nodeid)))
1164 def hasnode(self, classname, nodeid):
1165 """ Determine if the database has a given node.
1166 """
1167 # If this node is in the cache, then we do not need to go to
1168 # the database. (We don't consider this an LRU hit, though.)
1169 if (classname, nodeid) in self.cache:
1170 # Return 1, not True, to match the type of the result of
1171 # the SQL operation below.
1172 return 1
1173 sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
1174 self.sql(sql, (nodeid,))
1175 return int(self.cursor.fetchone()[0])
1177 def countnodes(self, classname):
1178 """ Count the number of nodes that exist for a particular Class.
1179 """
1180 sql = 'select count(*) from _%s'%classname
1181 self.sql(sql)
1182 return self.cursor.fetchone()[0]
1184 def addjournal(self, classname, nodeid, action, params, creator=None,
1185 creation=None):
1186 """ Journal the Action
1187 'action' may be:
1189 'create' or 'set' -- 'params' is a dictionary of property values
1190 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
1191 'retire' -- 'params' is None
1192 """
1193 # handle supply of the special journalling parameters (usually
1194 # supplied on importing an existing database)
1195 if creator:
1196 journaltag = creator
1197 else:
1198 journaltag = self.getuid()
1199 if creation:
1200 journaldate = creation
1201 else:
1202 journaldate = date.Date()
1204 # create the journal entry
1205 cols = 'nodeid,date,tag,action,params'
1207 self.log_debug('addjournal %s%s %r %s %s %r'%(classname,
1208 nodeid, journaldate, journaltag, action, params))
1210 # make the journalled data marshallable
1211 if isinstance(params, type({})):
1212 self._journal_marshal(params, classname)
1214 params = repr(params)
1216 dc = self.to_sql_value(hyperdb.Date)
1217 journaldate = dc(journaldate)
1219 self.save_journal(classname, cols, nodeid, journaldate,
1220 journaltag, action, params)
1222 def setjournal(self, classname, nodeid, journal):
1223 """Set the journal to the "journal" list."""
1224 # clear out any existing entries
1225 self.sql('delete from %s__journal where nodeid=%s'%(classname,
1226 self.arg), (nodeid,))
1228 # create the journal entry
1229 cols = 'nodeid,date,tag,action,params'
1231 dc = self.to_sql_value(hyperdb.Date)
1232 for nodeid, journaldate, journaltag, action, params in journal:
1233 self.log_debug('addjournal %s%s %r %s %s %r'%(
1234 classname, nodeid, journaldate, journaltag, action,
1235 params))
1237 # make the journalled data marshallable
1238 if isinstance(params, type({})):
1239 self._journal_marshal(params, classname)
1240 params = repr(params)
1242 self.save_journal(classname, cols, nodeid, dc(journaldate),
1243 journaltag, action, params)
1245 def _journal_marshal(self, params, classname):
1246 """Convert the journal params values into safely repr'able and
1247 eval'able values."""
1248 properties = self.getclass(classname).getprops()
1249 for param, value in params.iteritems():
1250 if not value:
1251 continue
1252 property = properties[param]
1253 cvt = self.to_sql_value(property.__class__)
1254 if isinstance(property, Password):
1255 params[param] = cvt(value)
1256 elif isinstance(property, Date):
1257 params[param] = cvt(value)
1258 elif isinstance(property, Interval):
1259 params[param] = cvt(value)
1260 elif isinstance(property, Boolean):
1261 params[param] = cvt(value)
1263 def getjournal(self, classname, nodeid):
1264 """ get the journal for id
1265 """
1266 # make sure the node exists
1267 if not self.hasnode(classname, nodeid):
1268 raise IndexError('%s has no node %s'%(classname, nodeid))
1270 cols = ','.join('nodeid date tag action params'.split())
1271 journal = self.load_journal(classname, cols, nodeid)
1273 # now unmarshal the data
1274 dc = self.to_hyperdb_value(hyperdb.Date)
1275 res = []
1276 properties = self.getclass(classname).getprops()
1277 for nodeid, date_stamp, user, action, params in journal:
1278 params = eval(params)
1279 if isinstance(params, type({})):
1280 for param, value in params.iteritems():
1281 if not value:
1282 continue
1283 property = properties.get(param, None)
1284 if property is None:
1285 # deleted property
1286 continue
1287 cvt = self.to_hyperdb_value(property.__class__)
1288 if isinstance(property, Password):
1289 params[param] = cvt(value)
1290 elif isinstance(property, Date):
1291 params[param] = cvt(value)
1292 elif isinstance(property, Interval):
1293 params[param] = cvt(value)
1294 elif isinstance(property, Boolean):
1295 params[param] = cvt(value)
1296 # XXX numeric ids
1297 res.append((str(nodeid), dc(date_stamp), user, action, params))
1298 return res
1300 def save_journal(self, classname, cols, nodeid, journaldate,
1301 journaltag, action, params):
1302 """ Save the journal entry to the database
1303 """
1304 entry = (nodeid, journaldate, journaltag, action, params)
1306 # do the insert
1307 a = self.arg
1308 sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(
1309 classname, cols, a, a, a, a, a)
1310 self.sql(sql, entry)
1312 def load_journal(self, classname, cols, nodeid):
1313 """ Load the journal from the database
1314 """
1315 # now get the journal entries
1316 sql = 'select %s from %s__journal where nodeid=%s order by date'%(
1317 cols, classname, self.arg)
1318 self.sql(sql, (nodeid,))
1319 return self.cursor.fetchall()
1321 def pack(self, pack_before):
1322 """ Delete all journal entries except "create" before 'pack_before'.
1323 """
1324 date_stamp = self.to_sql_value(Date)(pack_before)
1326 # do the delete
1327 for classname in self.classes:
1328 sql = "delete from %s__journal where date<%s and "\
1329 "action<>'create'"%(classname, self.arg)
1330 self.sql(sql, (date_stamp,))
1332 def sql_commit(self, fail_ok=False):
1333 """ Actually commit to the database.
1334 """
1335 logging.getLogger('roundup.hyperdb').info('commit')
1337 self.conn.commit()
1339 # open a new cursor for subsequent work
1340 self.cursor = self.conn.cursor()
1342 def commit(self, fail_ok=False):
1343 """ Commit the current transactions.
1345 Save all data changed since the database was opened or since the
1346 last commit() or rollback().
1348 fail_ok indicates that the commit is allowed to fail. This is used
1349 in the web interface when committing cleaning of the session
1350 database. We don't care if there's a concurrency issue there.
1352 The only backend this seems to affect is postgres.
1353 """
1354 # commit the database
1355 self.sql_commit(fail_ok)
1357 # now, do all the other transaction stuff
1358 for method, args in self.transactions:
1359 method(*args)
1361 # save the indexer
1362 self.indexer.save_index()
1364 # clear out the transactions
1365 self.transactions = []
1367 # clear the cache: Don't carry over cached values from one
1368 # transaction to the next (there may be other changes from other
1369 # transactions)
1370 self.clearCache()
1372 def sql_rollback(self):
1373 self.conn.rollback()
1375 def rollback(self):
1376 """ Reverse all actions from the current transaction.
1378 Undo all the changes made since the database was opened or the last
1379 commit() or rollback() was performed.
1380 """
1381 logging.getLogger('roundup.hyperdb').info('rollback')
1383 self.sql_rollback()
1385 # roll back "other" transaction stuff
1386 for method, args in self.transactions:
1387 # delete temporary files
1388 if method == self.doStoreFile:
1389 self.rollbackStoreFile(*args)
1390 self.transactions = []
1392 # clear the cache
1393 self.clearCache()
1395 def sql_close(self):
1396 logging.getLogger('roundup.hyperdb').info('close')
1397 self.conn.close()
1399 def close(self):
1400 """ Close off the connection.
1401 """
1402 self.indexer.close()
1403 self.sql_close()
1405 #
1406 # The base Class class
1407 #
1408 class Class(hyperdb.Class):
1409 """ The handle to a particular class of nodes in a hyperdatabase.
1411 All methods except __repr__ and getnode must be implemented by a
1412 concrete backend Class.
1413 """
1415 def schema(self):
1416 """ A dumpable version of the schema that we can store in the
1417 database
1418 """
1419 return (self.key, [(x, repr(y)) for x,y in self.properties.iteritems()])
1421 def enableJournalling(self):
1422 """Turn journalling on for this class
1423 """
1424 self.do_journal = 1
1426 def disableJournalling(self):
1427 """Turn journalling off for this class
1428 """
1429 self.do_journal = 0
1431 # Editing nodes:
1432 def create(self, **propvalues):
1433 """ Create a new node of this class and return its id.
1435 The keyword arguments in 'propvalues' map property names to values.
1437 The values of arguments must be acceptable for the types of their
1438 corresponding properties or a TypeError is raised.
1440 If this class has a key property, it must be present and its value
1441 must not collide with other key strings or a ValueError is raised.
1443 Any other properties on this class that are missing from the
1444 'propvalues' dictionary are set to None.
1446 If an id in a link or multilink property does not refer to a valid
1447 node, an IndexError is raised.
1448 """
1449 self.fireAuditors('create', None, propvalues)
1450 newid = self.create_inner(**propvalues)
1451 self.fireReactors('create', newid, None)
1452 return newid
1454 def create_inner(self, **propvalues):
1455 """ Called by create, in-between the audit and react calls.
1456 """
1457 if 'id' in propvalues:
1458 raise KeyError('"id" is reserved')
1460 if self.db.journaltag is None:
1461 raise DatabaseError(_('Database open read-only'))
1463 if ('creator' in propvalues or 'actor' in propvalues or
1464 'creation' in propvalues or 'activity' in propvalues):
1465 raise KeyError('"creator", "actor", "creation" and '
1466 '"activity" are reserved')
1468 # new node's id
1469 newid = self.db.newid(self.classname)
1471 # validate propvalues
1472 num_re = re.compile('^\d+$')
1473 for key, value in propvalues.iteritems():
1474 if key == self.key:
1475 try:
1476 self.lookup(value)
1477 except KeyError:
1478 pass
1479 else:
1480 raise ValueError('node with key "%s" exists'%value)
1482 # try to handle this property
1483 try:
1484 prop = self.properties[key]
1485 except KeyError:
1486 raise KeyError('"%s" has no property "%s"'%(self.classname,
1487 key))
1489 if value is not None and isinstance(prop, Link):
1490 if type(value) != type(''):
1491 raise ValueError('link value must be String')
1492 link_class = self.properties[key].classname
1493 # if it isn't a number, it's a key
1494 if not num_re.match(value):
1495 try:
1496 value = self.db.classes[link_class].lookup(value)
1497 except (TypeError, KeyError):
1498 raise IndexError('new property "%s": %s not a %s'%(
1499 key, value, link_class))
1500 elif not self.db.getclass(link_class).hasnode(value):
1501 raise IndexError('%s has no node %s'%(link_class,
1502 value))
1504 # save off the value
1505 propvalues[key] = value
1507 # register the link with the newly linked node
1508 if self.do_journal and self.properties[key].do_journal:
1509 self.db.addjournal(link_class, value, 'link',
1510 (self.classname, newid, key))
1512 elif isinstance(prop, Multilink):
1513 if value is None:
1514 value = []
1515 if not hasattr(value, '__iter__'):
1516 raise TypeError('new property "%s" not an iterable of ids'%key)
1517 # clean up and validate the list of links
1518 link_class = self.properties[key].classname
1519 l = []
1520 for entry in value:
1521 if type(entry) != type(''):
1522 raise ValueError('"%s" multilink value (%r) '
1523 'must contain Strings'%(key, value))
1524 # if it isn't a number, it's a key
1525 if not num_re.match(entry):
1526 try:
1527 entry = self.db.classes[link_class].lookup(entry)
1528 except (TypeError, KeyError):
1529 raise IndexError('new property "%s": %s not a %s'%(
1530 key, entry, self.properties[key].classname))
1531 l.append(entry)
1532 value = l
1533 propvalues[key] = value
1535 # handle additions
1536 for nodeid in value:
1537 if not self.db.getclass(link_class).hasnode(nodeid):
1538 raise IndexError('%s has no node %s'%(link_class,
1539 nodeid))
1540 # register the link with the newly linked node
1541 if self.do_journal and self.properties[key].do_journal:
1542 self.db.addjournal(link_class, nodeid, 'link',
1543 (self.classname, newid, key))
1545 elif isinstance(prop, String):
1546 if type(value) != type('') and type(value) != type(u''):
1547 raise TypeError('new property "%s" not a string'%key)
1548 if prop.indexme:
1549 self.db.indexer.add_text((self.classname, newid, key),
1550 value)
1552 elif isinstance(prop, Password):
1553 if not isinstance(value, password.Password):
1554 raise TypeError('new property "%s" not a Password'%key)
1556 elif isinstance(prop, Date):
1557 if value is not None and not isinstance(value, date.Date):
1558 raise TypeError('new property "%s" not a Date'%key)
1560 elif isinstance(prop, Interval):
1561 if value is not None and not isinstance(value, date.Interval):
1562 raise TypeError('new property "%s" not an Interval'%key)
1564 elif value is not None and isinstance(prop, Number):
1565 try:
1566 float(value)
1567 except ValueError:
1568 raise TypeError('new property "%s" not numeric'%key)
1570 elif value is not None and isinstance(prop, Boolean):
1571 try:
1572 int(value)
1573 except ValueError:
1574 raise TypeError('new property "%s" not boolean'%key)
1576 # make sure there's data where there needs to be
1577 for key, prop in self.properties.iteritems():
1578 if key in propvalues:
1579 continue
1580 if key == self.key:
1581 raise ValueError('key property "%s" is required'%key)
1582 if isinstance(prop, Multilink):
1583 propvalues[key] = []
1584 else:
1585 propvalues[key] = None
1587 # done
1588 self.db.addnode(self.classname, newid, propvalues)
1589 if self.do_journal:
1590 self.db.addjournal(self.classname, newid, ''"create", {})
1592 # XXX numeric ids
1593 return str(newid)
1595 def get(self, nodeid, propname, default=_marker, cache=1):
1596 """Get the value of a property on an existing node of this class.
1598 'nodeid' must be the id of an existing node of this class or an
1599 IndexError is raised. 'propname' must be the name of a property
1600 of this class or a KeyError is raised.
1602 'cache' exists for backwards compatibility, and is not used.
1603 """
1604 if propname == 'id':
1605 return nodeid
1607 # get the node's dict
1608 d = self.db.getnode(self.classname, nodeid)
1610 if propname == 'creation':
1611 if 'creation' in d:
1612 return d['creation']
1613 else:
1614 return date.Date()
1615 if propname == 'activity':
1616 if 'activity' in d:
1617 return d['activity']
1618 else:
1619 return date.Date()
1620 if propname == 'creator':
1621 if 'creator' in d:
1622 return d['creator']
1623 else:
1624 return self.db.getuid()
1625 if propname == 'actor':
1626 if 'actor' in d:
1627 return d['actor']
1628 else:
1629 return self.db.getuid()
1631 # get the property (raises KeyError if invalid)
1632 prop = self.properties[propname]
1634 # lazy evaluation of Multilink
1635 if propname not in d and isinstance(prop, Multilink):
1636 sql = 'select linkid from %s_%s where nodeid=%s'%(self.classname,
1637 propname, self.db.arg)
1638 self.db.sql(sql, (nodeid,))
1639 # extract the first column from the result
1640 # XXX numeric ids
1641 items = [int(x[0]) for x in self.db.cursor.fetchall()]
1642 items.sort ()
1643 d[propname] = [str(x) for x in items]
1645 # handle there being no value in the table for the property
1646 if propname not in d or d[propname] is None:
1647 if default is _marker:
1648 if isinstance(prop, Multilink):
1649 return []
1650 else:
1651 return None
1652 else:
1653 return default
1655 # don't pass our list to other code
1656 if isinstance(prop, Multilink):
1657 return d[propname][:]
1659 return d[propname]
1661 def set(self, nodeid, **propvalues):
1662 """Modify a property on an existing node of this class.
1664 'nodeid' must be the id of an existing node of this class or an
1665 IndexError is raised.
1667 Each key in 'propvalues' must be the name of a property of this
1668 class or a KeyError is raised.
1670 All values in 'propvalues' must be acceptable types for their
1671 corresponding properties or a TypeError is raised.
1673 If the value of the key property is set, it must not collide with
1674 other key strings or a ValueError is raised.
1676 If the value of a Link or Multilink property contains an invalid
1677 node id, a ValueError is raised.
1678 """
1679 self.fireAuditors('set', nodeid, propvalues)
1680 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1681 propvalues = self.set_inner(nodeid, **propvalues)
1682 self.fireReactors('set', nodeid, oldvalues)
1683 return propvalues
1685 def set_inner(self, nodeid, **propvalues):
1686 """ Called by set, in-between the audit and react calls.
1687 """
1688 if not propvalues:
1689 return propvalues
1691 if ('creator' in propvalues or 'actor' in propvalues or
1692 'creation' in propvalues or 'activity' in propvalues):
1693 raise KeyError('"creator", "actor", "creation" and '
1694 '"activity" are reserved')
1696 if 'id' in propvalues:
1697 raise KeyError('"id" is reserved')
1699 if self.db.journaltag is None:
1700 raise DatabaseError(_('Database open read-only'))
1702 node = self.db.getnode(self.classname, nodeid)
1703 if self.is_retired(nodeid):
1704 raise IndexError('Requested item is retired')
1705 num_re = re.compile('^\d+$')
1707 # make a copy of the values dictionary - we'll modify the contents
1708 propvalues = propvalues.copy()
1710 # if the journal value is to be different, store it in here
1711 journalvalues = {}
1713 # remember the add/remove stuff for multilinks, making it easier
1714 # for the Database layer to do its stuff
1715 multilink_changes = {}
1717 for propname, value in list(propvalues.items()):
1718 # check to make sure we're not duplicating an existing key
1719 if propname == self.key and node[propname] != value:
1720 try:
1721 self.lookup(value)
1722 except KeyError:
1723 pass
1724 else:
1725 raise ValueError('node with key "%s" exists'%value)
1727 # this will raise the KeyError if the property isn't valid
1728 # ... we don't use getprops() here because we only care about
1729 # the writeable properties.
1730 try:
1731 prop = self.properties[propname]
1732 except KeyError:
1733 raise KeyError('"%s" has no property named "%s"'%(
1734 self.classname, propname))
1736 # if the value's the same as the existing value, no sense in
1737 # doing anything
1738 current = node.get(propname, None)
1739 if value == current:
1740 del propvalues[propname]
1741 continue
1742 journalvalues[propname] = current
1744 # do stuff based on the prop type
1745 if isinstance(prop, Link):
1746 link_class = prop.classname
1747 # if it isn't a number, it's a key
1748 if value is not None and not isinstance(value, type('')):
1749 raise ValueError('property "%s" link value be a string'%(
1750 propname))
1751 if isinstance(value, type('')) and not num_re.match(value):
1752 try:
1753 value = self.db.classes[link_class].lookup(value)
1754 except (TypeError, KeyError):
1755 raise IndexError('new property "%s": %s not a %s'%(
1756 propname, value, prop.classname))
1758 if (value is not None and
1759 not self.db.getclass(link_class).hasnode(value)):
1760 raise IndexError('%s has no node %s'%(link_class,
1761 value))
1763 if self.do_journal and prop.do_journal:
1764 # register the unlink with the old linked node
1765 if node[propname] is not None:
1766 self.db.addjournal(link_class, node[propname],
1767 ''"unlink", (self.classname, nodeid, propname))
1769 # register the link with the newly linked node
1770 if value is not None:
1771 self.db.addjournal(link_class, value, ''"link",
1772 (self.classname, nodeid, propname))
1774 elif isinstance(prop, Multilink):
1775 if value is None:
1776 value = []
1777 if not hasattr(value, '__iter__'):
1778 raise TypeError('new property "%s" not an iterable of'
1779 ' ids'%propname)
1780 link_class = self.properties[propname].classname
1781 l = []
1782 for entry in value:
1783 # if it isn't a number, it's a key
1784 if type(entry) != type(''):
1785 raise ValueError('new property "%s" link value '
1786 'must be a string'%propname)
1787 if not num_re.match(entry):
1788 try:
1789 entry = self.db.classes[link_class].lookup(entry)
1790 except (TypeError, KeyError):
1791 raise IndexError('new property "%s": %s not a %s'%(
1792 propname, entry,
1793 self.properties[propname].classname))
1794 l.append(entry)
1795 value = l
1796 propvalues[propname] = value
1798 # figure the journal entry for this property
1799 add = []
1800 remove = []
1802 # handle removals
1803 if propname in node:
1804 l = node[propname]
1805 else:
1806 l = []
1807 for id in l[:]:
1808 if id in value:
1809 continue
1810 # register the unlink with the old linked node
1811 if self.do_journal and self.properties[propname].do_journal:
1812 self.db.addjournal(link_class, id, 'unlink',
1813 (self.classname, nodeid, propname))
1814 l.remove(id)
1815 remove.append(id)
1817 # handle additions
1818 for id in value:
1819 if id in l:
1820 continue
1821 # We can safely check this condition after
1822 # checking that this is an addition to the
1823 # multilink since the condition was checked for
1824 # existing entries at the point they were added to
1825 # the multilink. Since the hasnode call will
1826 # result in a SQL query, it is more efficient to
1827 # avoid the check if possible.
1828 if not self.db.getclass(link_class).hasnode(id):
1829 raise IndexError('%s has no node %s'%(link_class,
1830 id))
1831 # register the link with the newly linked node
1832 if self.do_journal and self.properties[propname].do_journal:
1833 self.db.addjournal(link_class, id, 'link',
1834 (self.classname, nodeid, propname))
1835 l.append(id)
1836 add.append(id)
1838 # figure the journal entry
1839 l = []
1840 if add:
1841 l.append(('+', add))
1842 if remove:
1843 l.append(('-', remove))
1844 multilink_changes[propname] = (add, remove)
1845 if l:
1846 journalvalues[propname] = tuple(l)
1848 elif isinstance(prop, String):
1849 if value is not None and type(value) != type('') and type(value) != type(u''):
1850 raise TypeError('new property "%s" not a string'%propname)
1851 if prop.indexme:
1852 if value is None: value = ''
1853 self.db.indexer.add_text((self.classname, nodeid, propname),
1854 value)
1856 elif isinstance(prop, Password):
1857 if not isinstance(value, password.Password):
1858 raise TypeError('new property "%s" not a Password'%propname)
1859 propvalues[propname] = value
1861 elif value is not None and isinstance(prop, Date):
1862 if not isinstance(value, date.Date):
1863 raise TypeError('new property "%s" not a Date'% propname)
1864 propvalues[propname] = value
1866 elif value is not None and isinstance(prop, Interval):
1867 if not isinstance(value, date.Interval):
1868 raise TypeError('new property "%s" not an '
1869 'Interval'%propname)
1870 propvalues[propname] = value
1872 elif value is not None and isinstance(prop, Number):
1873 try:
1874 float(value)
1875 except ValueError:
1876 raise TypeError('new property "%s" not numeric'%propname)
1878 elif value is not None and isinstance(prop, Boolean):
1879 try:
1880 int(value)
1881 except ValueError:
1882 raise TypeError('new property "%s" not boolean'%propname)
1884 # nothing to do?
1885 if not propvalues:
1886 return propvalues
1888 # update the activity time
1889 propvalues['activity'] = date.Date()
1890 propvalues['actor'] = self.db.getuid()
1892 # do the set
1893 self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
1895 # remove the activity props now they're handled
1896 del propvalues['activity']
1897 del propvalues['actor']
1899 # journal the set
1900 if self.do_journal:
1901 self.db.addjournal(self.classname, nodeid, ''"set", journalvalues)
1903 return propvalues
1905 def retire(self, nodeid):
1906 """Retire a node.
1908 The properties on the node remain available from the get() method,
1909 and the node's id is never reused.
1911 Retired nodes are not returned by the find(), list(), or lookup()
1912 methods, and other nodes may reuse the values of their key properties.
1913 """
1914 if self.db.journaltag is None:
1915 raise DatabaseError(_('Database open read-only'))
1917 self.fireAuditors('retire', nodeid, None)
1919 # use the arg for __retired__ to cope with any odd database type
1920 # conversion (hello, sqlite)
1921 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1922 self.db.arg, self.db.arg)
1923 self.db.sql(sql, (nodeid, nodeid))
1924 if self.do_journal:
1925 self.db.addjournal(self.classname, nodeid, ''"retired", None)
1927 self.fireReactors('retire', nodeid, None)
1929 def restore(self, nodeid):
1930 """Restore a retired node.
1932 Make node available for all operations like it was before retirement.
1933 """
1934 if self.db.journaltag is None:
1935 raise DatabaseError(_('Database open read-only'))
1937 node = self.db.getnode(self.classname, nodeid)
1938 # check if key property was overrided
1939 key = self.getkey()
1940 try:
1941 id = self.lookup(node[key])
1942 except KeyError:
1943 pass
1944 else:
1945 raise KeyError("Key property (%s) of retired node clashes "
1946 "with existing one (%s)" % (key, node[key]))
1948 self.fireAuditors('restore', nodeid, None)
1949 # use the arg for __retired__ to cope with any odd database type
1950 # conversion (hello, sqlite)
1951 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1952 self.db.arg, self.db.arg)
1953 self.db.sql(sql, (0, nodeid))
1954 if self.do_journal:
1955 self.db.addjournal(self.classname, nodeid, ''"restored", None)
1957 self.fireReactors('restore', nodeid, None)
1959 def is_retired(self, nodeid):
1960 """Return true if the node is rerired
1961 """
1962 sql = 'select __retired__ from _%s where id=%s'%(self.classname,
1963 self.db.arg)
1964 self.db.sql(sql, (nodeid,))
1965 return int(self.db.sql_fetchone()[0]) > 0
1967 def destroy(self, nodeid):
1968 """Destroy a node.
1970 WARNING: this method should never be used except in extremely rare
1971 situations where there could never be links to the node being
1972 deleted
1974 WARNING: use retire() instead
1976 WARNING: the properties of this node will not be available ever again
1978 WARNING: really, use retire() instead
1980 Well, I think that's enough warnings. This method exists mostly to
1981 support the session storage of the cgi interface.
1983 The node is completely removed from the hyperdb, including all journal
1984 entries. It will no longer be available, and will generally break code
1985 if there are any references to the node.
1986 """
1987 if self.db.journaltag is None:
1988 raise DatabaseError(_('Database open read-only'))
1989 self.db.destroynode(self.classname, nodeid)
1991 def history(self, nodeid):
1992 """Retrieve the journal of edits on a particular node.
1994 'nodeid' must be the id of an existing node of this class or an
1995 IndexError is raised.
1997 The returned list contains tuples of the form
1999 (nodeid, date, tag, action, params)
2001 'date' is a Timestamp object specifying the time of the change and
2002 'tag' is the journaltag specified when the database was opened.
2003 """
2004 if not self.do_journal:
2005 raise ValueError('Journalling is disabled for this class')
2006 return self.db.getjournal(self.classname, nodeid)
2008 # Locating nodes:
2009 def hasnode(self, nodeid):
2010 """Determine if the given nodeid actually exists
2011 """
2012 return self.db.hasnode(self.classname, nodeid)
2014 def setkey(self, propname):
2015 """Select a String property of this class to be the key property.
2017 'propname' must be the name of a String property of this class or
2018 None, or a TypeError is raised. The values of the key property on
2019 all existing nodes must be unique or a ValueError is raised.
2020 """
2021 prop = self.getprops()[propname]
2022 if not isinstance(prop, String):
2023 raise TypeError('key properties must be String')
2024 self.key = propname
2026 def getkey(self):
2027 """Return the name of the key property for this class or None."""
2028 return self.key
2030 def lookup(self, keyvalue):
2031 """Locate a particular node by its key property and return its id.
2033 If this class has no key property, a TypeError is raised. If the
2034 'keyvalue' matches one of the values for the key property among
2035 the nodes in this class, the matching node's id is returned;
2036 otherwise a KeyError is raised.
2037 """
2038 if not self.key:
2039 raise TypeError('No key property set for class %s'%self.classname)
2041 # use the arg to handle any odd database type conversion (hello,
2042 # sqlite)
2043 sql = "select id from _%s where _%s=%s and __retired__=%s"%(
2044 self.classname, self.key, self.db.arg, self.db.arg)
2045 self.db.sql(sql, (str(keyvalue), 0))
2047 # see if there was a result that's not retired
2048 row = self.db.sql_fetchone()
2049 if not row:
2050 raise KeyError('No key (%s) value "%s" for "%s"'%(self.key,
2051 keyvalue, self.classname))
2053 # return the id
2054 # XXX numeric ids
2055 return str(row[0])
2057 def find(self, **propspec):
2058 """Get the ids of nodes in this class which link to the given nodes.
2060 'propspec' consists of keyword args propname=nodeid or
2061 propname={nodeid:1, }
2062 'propname' must be the name of a property in this class, or a
2063 KeyError is raised. That property must be a Link or
2064 Multilink property, or a TypeError is raised.
2066 Any node in this class whose 'propname' property links to any of
2067 the nodeids will be returned. Examples::
2069 db.issue.find(messages='1')
2070 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
2071 """
2072 # shortcut
2073 if not propspec:
2074 return []
2076 # validate the args
2077 props = self.getprops()
2078 for propname, nodeids in propspec.iteritems():
2079 # check the prop is OK
2080 prop = props[propname]
2081 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
2082 raise TypeError("'%s' not a Link/Multilink property"%propname)
2084 # first, links
2085 a = self.db.arg
2086 allvalues = ()
2087 sql = []
2088 where = []
2089 for prop, values in propspec.iteritems():
2090 if not isinstance(props[prop], hyperdb.Link):
2091 continue
2092 if type(values) is type({}) and len(values) == 1:
2093 values = list(values)[0]
2094 if type(values) is type(''):
2095 allvalues += (values,)
2096 where.append('_%s = %s'%(prop, a))
2097 elif values is None:
2098 where.append('_%s is NULL'%prop)
2099 else:
2100 values = list(values)
2101 s = ''
2102 if None in values:
2103 values.remove(None)
2104 s = '_%s is NULL or '%prop
2105 allvalues += tuple(values)
2106 s += '_%s in (%s)'%(prop, ','.join([a]*len(values)))
2107 where.append('(' + s +')')
2108 if where:
2109 allvalues = (0, ) + allvalues
2110 sql.append("""select id from _%s where __retired__=%s
2111 and %s"""%(self.classname, a, ' and '.join(where)))
2113 # now multilinks
2114 for prop, values in propspec.iteritems():
2115 if not isinstance(props[prop], hyperdb.Multilink):
2116 continue
2117 if not values:
2118 continue
2119 allvalues += (0, )
2120 if type(values) is type(''):
2121 allvalues += (values,)
2122 s = a
2123 else:
2124 allvalues += tuple(values)
2125 s = ','.join([a]*len(values))
2126 tn = '%s_%s'%(self.classname, prop)
2127 sql.append("""select id from _%s, %s where __retired__=%s
2128 and id = %s.nodeid and %s.linkid in (%s)"""%(self.classname,
2129 tn, a, tn, tn, s))
2131 if not sql:
2132 return []
2133 sql = ' union '.join(sql)
2134 self.db.sql(sql, allvalues)
2135 # XXX numeric ids
2136 l = [str(x[0]) for x in self.db.sql_fetchall()]
2137 return l
2139 def stringFind(self, **requirements):
2140 """Locate a particular node by matching a set of its String
2141 properties in a caseless search.
2143 If the property is not a String property, a TypeError is raised.
2145 The return is a list of the id of all nodes that match.
2146 """
2147 where = []
2148 args = []
2149 for propname in requirements:
2150 prop = self.properties[propname]
2151 if not isinstance(prop, String):
2152 raise TypeError("'%s' not a String property"%propname)
2153 where.append(propname)
2154 args.append(requirements[propname].lower())
2156 # generate the where clause
2157 s = ' and '.join(['lower(_%s)=%s'%(col, self.db.arg) for col in where])
2158 sql = 'select id from _%s where %s and __retired__=%s'%(
2159 self.classname, s, self.db.arg)
2160 args.append(0)
2161 self.db.sql(sql, tuple(args))
2162 # XXX numeric ids
2163 l = [str(x[0]) for x in self.db.sql_fetchall()]
2164 return l
2166 def list(self):
2167 """ Return a list of the ids of the active nodes in this class.
2168 """
2169 return self.getnodeids(retired=0)
2171 def getnodeids(self, retired=None):
2172 """ Retrieve all the ids of the nodes for a particular Class.
2174 Set retired=None to get all nodes. Otherwise it'll get all the
2175 retired or non-retired nodes, depending on the flag.
2176 """
2177 # flip the sense of the 'retired' flag if we don't want all of them
2178 if retired is not None:
2179 args = (0, )
2180 if retired:
2181 compare = '>'
2182 else:
2183 compare = '='
2184 sql = 'select id from _%s where __retired__%s%s'%(self.classname,
2185 compare, self.db.arg)
2186 else:
2187 args = ()
2188 sql = 'select id from _%s'%self.classname
2189 self.db.sql(sql, args)
2190 # XXX numeric ids
2191 ids = [str(x[0]) for x in self.db.cursor.fetchall()]
2192 return ids
2194 def _subselect(self, classname, multilink_table):
2195 """Create a subselect. This is factored out because some
2196 databases (hmm only one, so far) doesn't support subselects
2197 look for "I can't believe it's not a toy RDBMS" in the mysql
2198 backend.
2199 """
2200 return '_%s.id not in (select nodeid from %s)'%(classname,
2201 multilink_table)
2203 # Some DBs order NULL values last. Set this variable in the backend
2204 # for prepending an order by clause for each attribute that causes
2205 # correct sort order for NULLs. Examples:
2206 # order_by_null_values = '(%s is not NULL)'
2207 # order_by_null_values = 'notnull(%s)'
2208 # The format parameter is replaced with the attribute.
2209 order_by_null_values = None
2211 def supports_subselects(self):
2212 '''Assuming DBs can do subselects, overwrite if they cannot.
2213 '''
2214 return True
2216 def _filter_multilink_expression_fallback(
2217 self, classname, multilink_table, expr):
2218 '''This is a fallback for database that do not support
2219 subselects.'''
2221 is_valid = expr.evaluate
2223 last_id, kws = None, []
2225 ids = IdListOptimizer()
2226 append = ids.append
2228 # This join and the evaluation in program space
2229 # can be expensive for larger databases!
2230 # TODO: Find a faster way to collect the data needed
2231 # to evalute the expression.
2232 # Moving the expression evaluation into the database
2233 # would be nice but this tricky: Think about the cases
2234 # where the multilink table does not have join values
2235 # needed in evaluation.
2237 stmnt = "SELECT c.id, m.linkid FROM _%s c " \
2238 "LEFT OUTER JOIN %s m " \
2239 "ON c.id = m.nodeid ORDER BY c.id" % (
2240 classname, multilink_table)
2241 self.db.sql(stmnt)
2243 # collect all multilink items for a class item
2244 for nid, kw in self.db.sql_fetchiter():
2245 if nid != last_id:
2246 if last_id is None:
2247 last_id = nid
2248 else:
2249 # we have all multilink items -> evaluate!
2250 if is_valid(kws): append(last_id)
2251 last_id, kws = nid, []
2252 if kw is not None:
2253 kws.append(kw)
2255 if last_id is not None and is_valid(kws):
2256 append(last_id)
2258 # we have ids of the classname table
2259 return ids.where("_%s.id" % classname, self.db.arg)
2261 def _filter_multilink_expression(self, classname, multilink_table, v):
2262 """ Filters out elements of the classname table that do not
2263 match the given expression.
2264 Returns tuple of 'WHERE' introns for the overall filter.
2265 """
2266 try:
2267 opcodes = [int(x) for x in v]
2268 if min(opcodes) >= -1: raise ValueError()
2270 expr = compile_expression(opcodes)
2272 if not self.supports_subselects():
2273 # We heavily rely on subselects. If there is
2274 # no decent support fall back to slower variant.
2275 return self._filter_multilink_expression_fallback(
2276 classname, multilink_table, expr)
2278 atom = \
2279 "%s IN(SELECT linkid FROM %s WHERE nodeid=a.id)" % (
2280 self.db.arg,
2281 multilink_table)
2283 intron = \
2284 "_%(classname)s.id in (SELECT id " \
2285 "FROM _%(classname)s AS a WHERE %(condition)s) " % {
2286 'classname' : classname,
2287 'condition' : expr.generate(lambda n: atom) }
2289 values = []
2290 def collect_values(n): values.append(n.x)
2291 expr.visit(collect_values)
2293 return intron, values
2294 except:
2295 # original behavior
2296 where = "%s.linkid in (%s)" % (
2297 multilink_table, ','.join([self.db.arg] * len(v)))
2298 return where, v, True # True to indicate original
2300 def _filter_sql (self, search_matches, filterspec, srt=[], grp=[], retr=0):
2301 """ Compute the proptree and the SQL/ARGS for a filter.
2302 For argument description see filter below.
2303 We return a 3-tuple, the proptree, the sql and the sql-args
2304 or None if no SQL is necessary.
2305 The flag retr serves to retrieve *all* non-Multilink properties
2306 (for filling the cache during a filter_iter)
2307 """
2308 # we can't match anything if search_matches is empty
2309 if not search_matches and search_matches is not None:
2310 return None
2312 icn = self.classname
2314 # vars to hold the components of the SQL statement
2315 frum = [] # FROM clauses
2316 loj = [] # LEFT OUTER JOIN clauses
2317 where = [] # WHERE clauses
2318 args = [] # *any* positional arguments
2319 a = self.db.arg
2321 # figure the WHERE clause from the filterspec
2322 mlfilt = 0 # are we joining with Multilink tables?
2323 sortattr = self._sortattr (group = grp, sort = srt)
2324 proptree = self._proptree(filterspec, sortattr, retr)
2325 mlseen = 0
2326 for pt in reversed(proptree.sortattr):
2327 p = pt
2328 while p.parent:
2329 if isinstance (p.propclass, Multilink):
2330 mlseen = True
2331 if mlseen:
2332 p.sort_ids_needed = True
2333 p.tree_sort_done = False
2334 p = p.parent
2335 if not mlseen:
2336 pt.attr_sort_done = pt.tree_sort_done = True
2337 proptree.compute_sort_done()
2339 cols = ['_%s.id'%icn]
2340 mlsort = []
2341 rhsnum = 0
2342 for p in proptree:
2343 rc = ac = oc = None
2344 cn = p.classname
2345 ln = p.uniqname
2346 pln = p.parent.uniqname
2347 pcn = p.parent.classname
2348 k = p.name
2349 v = p.val
2350 propclass = p.propclass
2351 if p.parent == proptree and p.name == 'id' \
2352 and 'retrieve' in p.need_for:
2353 p.sql_idx = 0
2354 if 'sort' in p.need_for or 'retrieve' in p.need_for:
2355 rc = oc = ac = '_%s._%s'%(pln, k)
2356 if isinstance(propclass, Multilink):
2357 if 'search' in p.need_for:
2358 mlfilt = 1
2359 tn = '%s_%s'%(pcn, k)
2360 if v in ('-1', ['-1'], []):
2361 # only match rows that have count(linkid)=0 in the
2362 # corresponding multilink table)
2363 where.append(self._subselect(pcn, tn))
2364 else:
2365 frum.append(tn)
2366 gen_join = True
2368 if p.has_values and isinstance(v, type([])):
2369 result = self._filter_multilink_expression(pln, tn, v)
2370 # XXX: We dont need an id join if we used the filter
2371 gen_join = len(result) == 3
2373 if gen_join:
2374 where.append('_%s.id=%s.nodeid'%(pln,tn))
2376 if p.children:
2377 frum.append('_%s as _%s' % (cn, ln))
2378 where.append('%s.linkid=_%s.id'%(tn, ln))
2380 if p.has_values:
2381 if isinstance(v, type([])):
2382 where.append(result[0])
2383 args += result[1]
2384 else:
2385 where.append('%s.linkid=%s'%(tn, a))
2386 args.append(v)
2387 if 'sort' in p.need_for:
2388 assert not p.attr_sort_done and not p.sort_ids_needed
2389 elif k == 'id':
2390 if 'search' in p.need_for:
2391 if isinstance(v, type([])):
2392 # If there are no permitted values, then the
2393 # where clause will always be false, and we
2394 # can optimize the query away.
2395 if not v:
2396 return []
2397 s = ','.join([a for x in v])
2398 where.append('_%s.%s in (%s)'%(pln, k, s))
2399 args = args + v
2400 else:
2401 where.append('_%s.%s=%s'%(pln, k, a))
2402 args.append(v)
2403 if 'sort' in p.need_for or 'retrieve' in p.need_for:
2404 rc = oc = ac = '_%s.id'%pln
2405 elif isinstance(propclass, String):
2406 if 'search' in p.need_for:
2407 if not isinstance(v, type([])):
2408 v = [v]
2410 # Quote the bits in the string that need it and then embed
2411 # in a "substring" search. Note - need to quote the '%' so
2412 # they make it through the python layer happily
2413 v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
2415 # now add to the where clause
2416 where.append('('
2417 +' and '.join(["_%s._%s LIKE '%s'"%(pln, k, s) for s in v])
2418 +')')
2419 # note: args are embedded in the query string now
2420 if 'sort' in p.need_for:
2421 oc = ac = 'lower(_%s._%s)'%(pln, k)
2422 elif isinstance(propclass, Link):
2423 if 'search' in p.need_for:
2424 if p.children:
2425 if 'sort' not in p.need_for:
2426 frum.append('_%s as _%s' % (cn, ln))
2427 where.append('_%s._%s=_%s.id'%(pln, k, ln))
2428 if p.has_values:
2429 if isinstance(v, type([])):
2430 d = {}
2431 for entry in v:
2432 if entry == '-1':
2433 entry = None
2434 d[entry] = entry
2435 l = []
2436 if None in d or not d:
2437 if None in d: del d[None]
2438 l.append('_%s._%s is NULL'%(pln, k))
2439 if d:
2440 v = list(d)
2441 s = ','.join([a for x in v])
2442 l.append('(_%s._%s in (%s))'%(pln, k, s))
2443 args = args + v
2444 if l:
2445 where.append('(' + ' or '.join(l) +')')
2446 else:
2447 if v in ('-1', None):
2448 v = None
2449 where.append('_%s._%s is NULL'%(pln, k))
2450 else:
2451 where.append('_%s._%s=%s'%(pln, k, a))
2452 args.append(v)
2453 if 'sort' in p.need_for:
2454 lp = p.cls.labelprop()
2455 oc = ac = '_%s._%s'%(pln, k)
2456 if lp != 'id':
2457 if p.tree_sort_done:
2458 loj.append(
2459 'LEFT OUTER JOIN _%s as _%s on _%s._%s=_%s.id'%(
2460 cn, ln, pln, k, ln))
2461 oc = '_%s._%s'%(ln, lp)
2462 if 'retrieve' in p.need_for:
2463 rc = '_%s._%s'%(pln, k)
2464 elif isinstance(propclass, Date) and 'search' in p.need_for:
2465 dc = self.db.to_sql_value(hyperdb.Date)
2466 if isinstance(v, type([])):
2467 s = ','.join([a for x in v])
2468 where.append('_%s._%s in (%s)'%(pln, k, s))
2469 args = args + [dc(date.Date(x)) for x in v]
2470 else:
2471 try:
2472 # Try to filter on range of dates
2473 date_rng = propclass.range_from_raw(v, self.db)
2474 if date_rng.from_value:
2475 where.append('_%s._%s >= %s'%(pln, k, a))
2476 args.append(dc(date_rng.from_value))
2477 if date_rng.to_value:
2478 where.append('_%s._%s <= %s'%(pln, k, a))
2479 args.append(dc(date_rng.to_value))
2480 except ValueError:
2481 # If range creation fails - ignore that search parameter
2482 pass
2483 elif isinstance(propclass, Interval):
2484 # filter/sort using the __<prop>_int__ column
2485 if 'search' in p.need_for:
2486 if isinstance(v, type([])):
2487 s = ','.join([a for x in v])
2488 where.append('_%s.__%s_int__ in (%s)'%(pln, k, s))
2489 args = args + [date.Interval(x).as_seconds() for x in v]
2490 else:
2491 try:
2492 # Try to filter on range of intervals
2493 date_rng = Range(v, date.Interval)
2494 if date_rng.from_value:
2495 where.append('_%s.__%s_int__ >= %s'%(pln, k, a))
2496 args.append(date_rng.from_value.as_seconds())
2497 if date_rng.to_value:
2498 where.append('_%s.__%s_int__ <= %s'%(pln, k, a))
2499 args.append(date_rng.to_value.as_seconds())
2500 except ValueError:
2501 # If range creation fails - ignore search parameter
2502 pass
2503 if 'sort' in p.need_for:
2504 oc = ac = '_%s.__%s_int__'%(pln,k)
2505 if 'retrieve' in p.need_for:
2506 rc = '_%s._%s'%(pln,k)
2507 elif isinstance(propclass, Boolean) and 'search' in p.need_for:
2508 if type(v) == type(""):
2509 v = v.split(',')
2510 if type(v) != type([]):
2511 v = [v]
2512 bv = []
2513 for val in v:
2514 if type(val) is type(''):
2515 bv.append(propclass.from_raw (val))
2516 else:
2517 bv.append(bool(val))
2518 if len(bv) == 1:
2519 where.append('_%s._%s=%s'%(pln, k, a))
2520 args = args + bv
2521 else:
2522 s = ','.join([a for x in v])
2523 where.append('_%s._%s in (%s)'%(pln, k, s))
2524 args = args + bv
2525 elif 'search' in p.need_for:
2526 if isinstance(v, type([])):
2527 s = ','.join([a for x in v])
2528 where.append('_%s._%s in (%s)'%(pln, k, s))
2529 args = args + v
2530 else:
2531 where.append('_%s._%s=%s'%(pln, k, a))
2532 args.append(v)
2533 if oc:
2534 if p.sort_ids_needed:
2535 if rc == ac:
2536 p.sql_idx = len(cols)
2537 p.auxcol = len(cols)
2538 cols.append(ac)
2539 if p.tree_sort_done and p.sort_direction:
2540 # Don't select top-level id or multilink twice
2541 if (not p.sort_ids_needed or ac != oc) and (p.name != 'id'
2542 or p.parent != proptree):
2543 if rc == oc:
2544 p.sql_idx = len(cols)
2545 cols.append(oc)
2546 desc = ['', ' desc'][p.sort_direction == '-']
2547 # Some SQL dbs sort NULL values last -- we want them first.
2548 if (self.order_by_null_values and p.name != 'id'):
2549 nv = self.order_by_null_values % oc
2550 cols.append(nv)
2551 p.orderby.append(nv + desc)
2552 p.orderby.append(oc + desc)
2553 if 'retrieve' in p.need_for and p.sql_idx is None:
2554 assert(rc)
2555 p.sql_idx = len(cols)
2556 cols.append (rc)
2558 props = self.getprops()
2560 # don't match retired nodes
2561 where.append('_%s.__retired__=0'%icn)
2563 # add results of full text search
2564 if search_matches is not None:
2565 s = ','.join([a for x in search_matches])
2566 where.append('_%s.id in (%s)'%(icn, s))
2567 args = args + [x for x in search_matches]
2569 # construct the SQL
2570 frum.append('_'+icn)
2571 frum = ','.join(frum)
2572 if where:
2573 where = ' where ' + (' and '.join(where))
2574 else:
2575 where = ''
2576 if mlfilt:
2577 # we're joining tables on the id, so we will get dupes if we
2578 # don't distinct()
2579 cols[0] = 'distinct(_%s.id)'%icn
2581 order = []
2582 # keep correct sequence of order attributes.
2583 for sa in proptree.sortattr:
2584 if not sa.attr_sort_done:
2585 continue
2586 order.extend(sa.orderby)
2587 if order:
2588 order = ' order by %s'%(','.join(order))
2589 else:
2590 order = ''
2592 cols = ','.join(cols)
2593 loj = ' '.join(loj)
2594 sql = 'select %s from %s %s %s%s'%(cols, frum, loj, where, order)
2595 args = tuple(args)
2596 __traceback_info__ = (sql, args)
2597 return proptree, sql, args
2599 def filter(self, search_matches, filterspec, sort=[], group=[]):
2600 """Return a list of the ids of the active nodes in this class that
2601 match the 'filter' spec, sorted by the group spec and then the
2602 sort spec
2604 "filterspec" is {propname: value(s)}
2606 "sort" and "group" are [(dir, prop), ...] where dir is '+', '-'
2607 or None and prop is a prop name or None. Note that for
2608 backward-compatibility reasons a single (dir, prop) tuple is
2609 also allowed.
2611 "search_matches" is a container type or None
2613 The filter must match all properties specificed. If the property
2614 value to match is a list:
2616 1. String properties must match all elements in the list, and
2617 2. Other properties must match any of the elements in the list.
2618 """
2619 if __debug__:
2620 start_t = time.time()
2622 sq = self._filter_sql (search_matches, filterspec, sort, group)
2623 # nothing to match?
2624 if sq is None:
2625 return []
2626 proptree, sql, args = sq
2628 self.db.sql(sql, args)
2629 l = self.db.sql_fetchall()
2631 # Compute values needed for sorting in proptree.sort
2632 for p in proptree:
2633 if hasattr(p, 'auxcol'):
2634 p.sort_ids = p.sort_result = [row[p.auxcol] for row in l]
2635 # return the IDs (the first column)
2636 # XXX numeric ids
2637 l = [str(row[0]) for row in l]
2638 l = proptree.sort (l)
2640 if __debug__:
2641 self.db.stats['filtering'] += (time.time() - start_t)
2642 return l
2644 def filter_iter(self, search_matches, filterspec, sort=[], group=[]):
2645 """Iterator similar to filter above with same args.
2646 Limitation: We don't sort on multilinks.
2647 This uses an optimisation: We put all nodes that are in the
2648 current row into the node cache. Then we return the node id.
2649 That way a fetch of a node won't create another sql-fetch (with
2650 a join) from the database because the nodes are already in the
2651 cache. We're using our own temporary cursor.
2652 """
2653 sq = self._filter_sql(search_matches, filterspec, sort, group, retr=1)
2654 # nothing to match?
2655 if sq is None:
2656 return
2657 proptree, sql, args = sq
2658 cursor = self.db.conn.cursor()
2659 self.db.sql(sql, args, cursor)
2660 classes = {}
2661 for p in proptree:
2662 if 'retrieve' in p.need_for:
2663 cn = p.parent.classname
2664 ptid = p.parent.id # not the nodeid!
2665 key = (cn, ptid)
2666 if key not in classes:
2667 classes[key] = {}
2668 name = p.name
2669 assert (name)
2670 classes[key][name] = p
2671 p.to_hyperdb = self.db.to_hyperdb_value(p.propclass.__class__)
2672 while True:
2673 row = cursor.fetchone()
2674 if not row: break
2675 # populate cache with current items
2676 for (classname, ptid), pt in classes.iteritems():
2677 nodeid = str(row[pt['id'].sql_idx])
2678 key = (classname, nodeid)
2679 if key in self.db.cache:
2680 self.db._cache_refresh(key)
2681 continue
2682 node = {}
2683 for propname, p in pt.iteritems():
2684 value = row[p.sql_idx]
2685 if value is not None:
2686 value = p.to_hyperdb(value)
2687 node[propname] = value
2688 self.db._cache_save(key, node)
2689 yield str(row[0])
2691 def filter_sql(self, sql):
2692 """Return a list of the ids of the items in this class that match
2693 the SQL provided. The SQL is a complete "select" statement.
2695 The SQL select must include the item id as the first column.
2697 This function DOES NOT filter out retired items, add on a where
2698 clause "__retired__=0" if you don't want retired nodes.
2699 """
2700 if __debug__:
2701 start_t = time.time()
2703 self.db.sql(sql)
2704 l = self.db.sql_fetchall()
2706 if __debug__:
2707 self.db.stats['filtering'] += (time.time() - start_t)
2708 return l
2710 def count(self):
2711 """Get the number of nodes in this class.
2713 If the returned integer is 'numnodes', the ids of all the nodes
2714 in this class run from 1 to numnodes, and numnodes+1 will be the
2715 id of the next node to be created in this class.
2716 """
2717 return self.db.countnodes(self.classname)
2719 # Manipulating properties:
2720 def getprops(self, protected=1):
2721 """Return a dictionary mapping property names to property objects.
2722 If the "protected" flag is true, we include protected properties -
2723 those which may not be modified.
2724 """
2725 d = self.properties.copy()
2726 if protected:
2727 d['id'] = String()
2728 d['creation'] = hyperdb.Date()
2729 d['activity'] = hyperdb.Date()
2730 d['creator'] = hyperdb.Link('user')
2731 d['actor'] = hyperdb.Link('user')
2732 return d
2734 def addprop(self, **properties):
2735 """Add properties to this class.
2737 The keyword arguments in 'properties' must map names to property
2738 objects, or a TypeError is raised. None of the keys in 'properties'
2739 may collide with the names of existing properties, or a ValueError
2740 is raised before any properties have been added.
2741 """
2742 for key in properties:
2743 if key in self.properties:
2744 raise ValueError(key)
2745 self.properties.update(properties)
2747 def index(self, nodeid):
2748 """Add (or refresh) the node to search indexes
2749 """
2750 # find all the String properties that have indexme
2751 for prop, propclass in self.getprops().iteritems():
2752 if isinstance(propclass, String) and propclass.indexme:
2753 self.db.indexer.add_text((self.classname, nodeid, prop),
2754 str(self.get(nodeid, prop)))
2756 #
2757 # import / export support
2758 #
2759 def export_list(self, propnames, nodeid):
2760 """ Export a node - generate a list of CSV-able data in the order
2761 specified by propnames for the given node.
2762 """
2763 properties = self.getprops()
2764 l = []
2765 for prop in propnames:
2766 proptype = properties[prop]
2767 value = self.get(nodeid, prop)
2768 # "marshal" data where needed
2769 if value is None:
2770 pass
2771 elif isinstance(proptype, hyperdb.Date):
2772 value = value.get_tuple()
2773 elif isinstance(proptype, hyperdb.Interval):
2774 value = value.get_tuple()
2775 elif isinstance(proptype, hyperdb.Password):
2776 value = str(value)
2777 l.append(repr(value))
2778 l.append(repr(self.is_retired(nodeid)))
2779 return l
2781 def import_list(self, propnames, proplist):
2782 """ Import a node - all information including "id" is present and
2783 should not be sanity checked. Triggers are not triggered. The
2784 journal should be initialised using the "creator" and "created"
2785 information.
2787 Return the nodeid of the node imported.
2788 """
2789 if self.db.journaltag is None:
2790 raise DatabaseError(_('Database open read-only'))
2791 properties = self.getprops()
2793 # make the new node's property map
2794 d = {}
2795 retire = 0
2796 if not "id" in propnames:
2797 newid = self.db.newid(self.classname)
2798 else:
2799 newid = eval(proplist[propnames.index("id")])
2800 for i in range(len(propnames)):
2801 # Use eval to reverse the repr() used to output the CSV
2802 value = eval(proplist[i])
2804 # Figure the property for this column
2805 propname = propnames[i]
2807 # "unmarshal" where necessary
2808 if propname == 'id':
2809 continue
2810 elif propname == 'is retired':
2811 # is the item retired?
2812 if int(value):
2813 retire = 1
2814 continue
2815 elif value is None:
2816 d[propname] = None
2817 continue
2819 prop = properties[propname]
2820 if value is None:
2821 # don't set Nones
2822 continue
2823 elif isinstance(prop, hyperdb.Date):
2824 value = date.Date(value)
2825 elif isinstance(prop, hyperdb.Interval):
2826 value = date.Interval(value)
2827 elif isinstance(prop, hyperdb.Password):
2828 pwd = password.Password()
2829 pwd.unpack(value)
2830 value = pwd
2831 elif isinstance(prop, String):
2832 if isinstance(value, unicode):
2833 value = value.encode('utf8')
2834 if not isinstance(value, str):
2835 raise TypeError('new property "%(propname)s" not a '
2836 'string: %(value)r'%locals())
2837 if prop.indexme:
2838 self.db.indexer.add_text((self.classname, newid, propname),
2839 value)
2840 d[propname] = value
2842 # get a new id if necessary
2843 if newid is None:
2844 newid = self.db.newid(self.classname)
2846 # insert new node or update existing?
2847 if not self.hasnode(newid):
2848 self.db.addnode(self.classname, newid, d) # insert
2849 else:
2850 self.db.setnode(self.classname, newid, d) # update
2852 # retire?
2853 if retire:
2854 # use the arg for __retired__ to cope with any odd database type
2855 # conversion (hello, sqlite)
2856 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
2857 self.db.arg, self.db.arg)
2858 self.db.sql(sql, (newid, newid))
2859 return newid
2861 def export_journals(self):
2862 """Export a class's journal - generate a list of lists of
2863 CSV-able data:
2865 nodeid, date, user, action, params
2867 No heading here - the columns are fixed.
2868 """
2869 properties = self.getprops()
2870 r = []
2871 for nodeid in self.getnodeids():
2872 for nodeid, date, user, action, params in self.history(nodeid):
2873 date = date.get_tuple()
2874 if action == 'set':
2875 export_data = {}
2876 for propname, value in params.iteritems():
2877 if propname not in properties:
2878 # property no longer in the schema
2879 continue
2881 prop = properties[propname]
2882 # make sure the params are eval()'able
2883 if value is None:
2884 pass
2885 elif isinstance(prop, Date):
2886 value = value.get_tuple()
2887 elif isinstance(prop, Interval):
2888 value = value.get_tuple()
2889 elif isinstance(prop, Password):
2890 value = str(value)
2891 export_data[propname] = value
2892 params = export_data
2893 elif action == 'create' and params:
2894 # old tracker with data stored in the create!
2895 params = {}
2896 l = [nodeid, date, user, action, params]
2897 r.append(list(map(repr, l)))
2898 return r
2900 class FileClass(hyperdb.FileClass, Class):
2901 """This class defines a large chunk of data. To support this, it has a
2902 mandatory String property "content" which is typically saved off
2903 externally to the hyperdb.
2905 The default MIME type of this data is defined by the
2906 "default_mime_type" class attribute, which may be overridden by each
2907 node if the class defines a "type" String property.
2908 """
2909 def __init__(self, db, classname, **properties):
2910 """The newly-created class automatically includes the "content"
2911 and "type" properties.
2912 """
2913 if 'content' not in properties:
2914 properties['content'] = hyperdb.String(indexme='yes')
2915 if 'type' not in properties:
2916 properties['type'] = hyperdb.String()
2917 Class.__init__(self, db, classname, **properties)
2919 def create(self, **propvalues):
2920 """ snaffle the file propvalue and store in a file
2921 """
2922 # we need to fire the auditors now, or the content property won't
2923 # be in propvalues for the auditors to play with
2924 self.fireAuditors('create', None, propvalues)
2926 # now remove the content property so it's not stored in the db
2927 content = propvalues['content']
2928 del propvalues['content']
2930 # do the database create
2931 newid = self.create_inner(**propvalues)
2933 # figure the mime type
2934 mime_type = propvalues.get('type', self.default_mime_type)
2936 # and index!
2937 if self.properties['content'].indexme:
2938 self.db.indexer.add_text((self.classname, newid, 'content'),
2939 content, mime_type)
2941 # store off the content as a file
2942 self.db.storefile(self.classname, newid, None, content)
2944 # fire reactors
2945 self.fireReactors('create', newid, None)
2947 return newid
2949 def get(self, nodeid, propname, default=_marker, cache=1):
2950 """ Trap the content propname and get it from the file
2952 'cache' exists for backwards compatibility, and is not used.
2953 """
2954 poss_msg = 'Possibly a access right configuration problem.'
2955 if propname == 'content':
2956 try:
2957 return self.db.getfile(self.classname, nodeid, None)
2958 except IOError, strerror:
2959 # BUG: by catching this we donot see an error in the log.
2960 return 'ERROR reading file: %s%s\n%s\n%s'%(
2961 self.classname, nodeid, poss_msg, strerror)
2962 if default is not _marker:
2963 return Class.get(self, nodeid, propname, default)
2964 else:
2965 return Class.get(self, nodeid, propname)
2967 def set(self, itemid, **propvalues):
2968 """ Snarf the "content" propvalue and update it in a file
2969 """
2970 self.fireAuditors('set', itemid, propvalues)
2971 oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid))
2973 # now remove the content property so it's not stored in the db
2974 content = None
2975 if 'content' in propvalues:
2976 content = propvalues['content']
2977 del propvalues['content']
2979 # do the database create
2980 propvalues = self.set_inner(itemid, **propvalues)
2982 # do content?
2983 if content:
2984 # store and possibly index
2985 self.db.storefile(self.classname, itemid, None, content)
2986 if self.properties['content'].indexme:
2987 mime_type = self.get(itemid, 'type', self.default_mime_type)
2988 self.db.indexer.add_text((self.classname, itemid, 'content'),
2989 content, mime_type)
2990 propvalues['content'] = content
2992 # fire reactors
2993 self.fireReactors('set', itemid, oldvalues)
2994 return propvalues
2996 def index(self, nodeid):
2997 """ Add (or refresh) the node to search indexes.
2999 Use the content-type property for the content property.
3000 """
3001 # find all the String properties that have indexme
3002 for prop, propclass in self.getprops().iteritems():
3003 if prop == 'content' and propclass.indexme:
3004 mime_type = self.get(nodeid, 'type', self.default_mime_type)
3005 self.db.indexer.add_text((self.classname, nodeid, 'content'),
3006 str(self.get(nodeid, 'content')), mime_type)
3007 elif isinstance(propclass, hyperdb.String) and propclass.indexme:
3008 # index them under (classname, nodeid, property)
3009 try:
3010 value = str(self.get(nodeid, prop))
3011 except IndexError:
3012 # node has been destroyed
3013 continue
3014 self.db.indexer.add_text((self.classname, nodeid, prop), value)
3016 # XXX deviation from spec - was called ItemClass
3017 class IssueClass(Class, roundupdb.IssueClass):
3018 # Overridden methods:
3019 def __init__(self, db, classname, **properties):
3020 """The newly-created class automatically includes the "messages",
3021 "files", "nosy", and "superseder" properties. If the 'properties'
3022 dictionary attempts to specify any of these properties or a
3023 "creation", "creator", "activity" or "actor" property, a ValueError
3024 is raised.
3025 """
3026 if 'title' not in properties:
3027 properties['title'] = hyperdb.String(indexme='yes')
3028 if 'messages' not in properties:
3029 properties['messages'] = hyperdb.Multilink("msg")
3030 if 'files' not in properties:
3031 properties['files'] = hyperdb.Multilink("file")
3032 if 'nosy' not in properties:
3033 # note: journalling is turned off as it really just wastes
3034 # space. this behaviour may be overridden in an instance
3035 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
3036 if 'superseder' not in properties:
3037 properties['superseder'] = hyperdb.Multilink(classname)
3038 Class.__init__(self, db, classname, **properties)
3040 # vim: set et sts=4 sw=4 :