1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 """ Relational database (SQL) backend common code.
20 Basics:
22 - map roundup classes to relational tables
23 - automatically detect schema changes and modify the table schemas
24 appropriately (we store the "database version" of the schema in the
25 database itself as the only row of the "schema" table)
26 - multilinks (which represent a many-to-many relationship) are handled through
27 intermediate tables
28 - journals are stored adjunct to the per-class tables
29 - table names and columns have "_" prepended so the names can't clash with
30 restricted names (like "order")
31 - retirement is determined by the __retired__ column being > 0
33 Database-specific changes may generally be pushed out to the overridable
34 sql_* methods, since everything else should be fairly generic. There's
35 probably a bit of work to be done if a database is used that actually
36 honors column typing, since the initial databases don't (sqlite stores
37 everything as a string.)
39 The schema of the hyperdb being mapped to the database is stored in the
40 database itself as a repr()'ed dictionary of information about each Class
41 that maps to a table. If that information differs from the hyperdb schema,
42 then we update it. We also store in the schema dict a version which
43 allows us to upgrade the database schema when necessary. See upgrade_db().
45 To force a unqiueness constraint on the key properties we put the item
46 id into the __retired__ column duing retirement (so it's 0 for "active"
47 items) and place a unqiueness constraint on key + __retired__. This is
48 particularly important for the users class where multiple users may
49 try to have the same username, with potentially many retired users with
50 the same name.
51 """
52 __docformat__ = 'restructuredtext'
54 # standard python modules
55 import sys, os, time, re, errno, weakref, copy, logging, datetime
57 # roundup modules
58 from roundup import hyperdb, date, password, roundupdb, security, support
59 from roundup.hyperdb import String, Password, Date, Interval, Link, \
60 Multilink, DatabaseError, Boolean, Number, Node
61 from roundup.backends import locking
62 from roundup.support import reversed
63 from roundup.i18n import _
66 # support
67 from roundup.backends.blobfiles import FileStorage
68 try:
69 from roundup.backends.indexer_xapian import Indexer
70 except ImportError:
71 from roundup.backends.indexer_rdbms import Indexer
72 from roundup.backends.sessions_rdbms import Sessions, OneTimeKeys
73 from roundup.date import Range
75 from roundup.backends.back_anydbm import compile_expression
78 # dummy value meaning "argument not passed"
79 _marker = []
81 def _num_cvt(num):
82 num = str(num)
83 try:
84 return int(num)
85 except:
86 return float(num)
88 def _bool_cvt(value):
89 if value in ('TRUE', 'FALSE'):
90 return {'TRUE': 1, 'FALSE': 0}[value]
91 # assume it's a number returned from the db API
92 return int(value)
94 def date_to_hyperdb_value(d):
95 """ convert date d to a roundup date """
96 if isinstance (d, datetime.datetime):
97 return date.Date(d)
98 return date.Date (str(d).replace(' ', '.'))
101 def connection_dict(config, dbnamestr=None):
102 """ Used by Postgresql and MySQL to detemine the keyword args for
103 opening the database connection."""
104 d = { }
105 if dbnamestr:
106 d[dbnamestr] = config.RDBMS_NAME
107 for name in ('host', 'port', 'password', 'user', 'read_default_group',
108 'read_default_file'):
109 cvar = 'RDBMS_'+name.upper()
110 if config[cvar] is not None:
111 d[name] = config[cvar]
112 return d
115 class IdListOptimizer:
116 """ To prevent flooding the SQL parser of the underlaying
117 db engine with "x IN (1, 2, 3, ..., <large number>)" collapses
118 these cases to "x BETWEEN 1 AND <large number>".
119 """
121 def __init__(self):
122 self.ranges = []
123 self.singles = []
125 def append(self, nid):
126 """ Invariant: nid are ordered ascending """
127 if self.ranges:
128 last = self.ranges[-1]
129 if last[1] == nid-1:
130 last[1] = nid
131 return
132 if self.singles:
133 last = self.singles[-1]
134 if last == nid-1:
135 self.singles.pop()
136 self.ranges.append([last, nid])
137 return
138 self.singles.append(nid)
140 def where(self, field, placeholder):
141 ranges = self.ranges
142 singles = self.singles
144 if not singles and not ranges: return "(1=0)", []
146 if ranges:
147 between = '%s BETWEEN %s AND %s' % (
148 field, placeholder, placeholder)
149 stmnt = [between] * len(ranges)
150 else:
151 stmnt = []
152 if singles:
153 stmnt.append('%s in (%s)' % (
154 field, ','.join([placeholder]*len(singles))))
156 return '(%s)' % ' OR '.join(stmnt), sum(ranges, []) + singles
158 def __str__(self):
159 return "ranges: %r / singles: %r" % (self.ranges, self.singles)
162 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
163 """ Wrapper around an SQL database that presents a hyperdb interface.
165 - some functionality is specific to the actual SQL database, hence
166 the sql_* methods that are NotImplemented
167 - we keep a cache of the latest N row fetches (where N is configurable).
168 """
169 def __init__(self, config, journaltag=None):
170 """ Open the database and load the schema from it.
171 """
172 FileStorage.__init__(self, config.UMASK)
173 self.config, self.journaltag = config, journaltag
174 self.dir = config.DATABASE
175 self.classes = {}
176 self.indexer = Indexer(self)
177 self.security = security.Security(self)
179 # additional transaction support for external files and the like
180 self.transactions = []
182 # keep a cache of the N most recently retrieved rows of any kind
183 # (classname, nodeid) = row
184 self.cache_size = config.RDBMS_CACHE_SIZE
185 self.clearCache()
186 self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0,
187 'filtering': 0}
189 # database lock
190 self.lockfile = None
192 # open a connection to the database, creating the "conn" attribute
193 self.open_connection()
195 def clearCache(self):
196 self.cache = {}
197 self.cache_lru = []
199 def getSessionManager(self):
200 return Sessions(self)
202 def getOTKManager(self):
203 return OneTimeKeys(self)
205 def open_connection(self):
206 """ Open a connection to the database, creating it if necessary.
208 Must call self.load_dbschema()
209 """
210 raise NotImplemented
212 def sql(self, sql, args=None, cursor=None):
213 """ Execute the sql with the optional args.
214 """
215 self.log_debug('SQL %r %r'%(sql, args))
216 if not cursor:
217 cursor = self.cursor
218 if args:
219 cursor.execute(sql, args)
220 else:
221 cursor.execute(sql)
223 def sql_fetchone(self):
224 """ Fetch a single row. If there's nothing to fetch, return None.
225 """
226 return self.cursor.fetchone()
228 def sql_fetchall(self):
229 """ Fetch all rows. If there's nothing to fetch, return [].
230 """
231 return self.cursor.fetchall()
233 def sql_fetchiter(self):
234 """ Fetch all row as a generator
235 """
236 while True:
237 row = self.cursor.fetchone()
238 if not row: break
239 yield row
241 def sql_stringquote(self, value):
242 """ Quote the string so it's safe to put in the 'sql quotes'
243 """
244 return re.sub("'", "''", str(value))
246 def init_dbschema(self):
247 self.database_schema = {
248 'version': self.current_db_version,
249 'tables': {}
250 }
252 def load_dbschema(self):
253 """ Load the schema definition that the database currently implements
254 """
255 self.cursor.execute('select schema from schema')
256 schema = self.cursor.fetchone()
257 if schema:
258 self.database_schema = eval(schema[0])
259 else:
260 self.database_schema = {}
262 def save_dbschema(self):
263 """ Save the schema definition that the database currently implements
264 """
265 s = repr(self.database_schema)
266 self.sql('delete from schema')
267 self.sql('insert into schema values (%s)'%self.arg, (s,))
269 def post_init(self):
270 """ Called once the schema initialisation has finished.
272 We should now confirm that the schema defined by our "classes"
273 attribute actually matches the schema in the database.
274 """
275 save = 0
277 # handle changes in the schema
278 tables = self.database_schema['tables']
279 for classname, spec in self.classes.iteritems():
280 if classname in tables:
281 dbspec = tables[classname]
282 if self.update_class(spec, dbspec):
283 tables[classname] = spec.schema()
284 save = 1
285 else:
286 self.create_class(spec)
287 tables[classname] = spec.schema()
288 save = 1
290 for classname, spec in list(tables.items()):
291 if classname not in self.classes:
292 self.drop_class(classname, tables[classname])
293 del tables[classname]
294 save = 1
296 # now upgrade the database for column type changes, new internal
297 # tables, etc.
298 save = save | self.upgrade_db()
300 # update the database version of the schema
301 if save:
302 self.save_dbschema()
304 # reindex the db if necessary
305 if self.indexer.should_reindex():
306 self.reindex()
308 # commit
309 self.sql_commit()
311 # update this number when we need to make changes to the SQL structure
312 # of the backen database
313 current_db_version = 5
314 db_version_updated = False
315 def upgrade_db(self):
316 """ Update the SQL database to reflect changes in the backend code.
318 Return boolean whether we need to save the schema.
319 """
320 version = self.database_schema.get('version', 1)
321 if version > self.current_db_version:
322 raise DatabaseError('attempting to run rev %d DATABASE with rev '
323 '%d CODE!'%(version, self.current_db_version))
324 if version == self.current_db_version:
325 # nothing to do
326 return 0
328 if version < 2:
329 self.log_info('upgrade to version 2')
330 # change the schema structure
331 self.database_schema = {'tables': self.database_schema}
333 # version 1 didn't have the actor column (note that in
334 # MySQL this will also transition the tables to typed columns)
335 self.add_new_columns_v2()
337 # version 1 doesn't have the OTK, session and indexing in the
338 # database
339 self.create_version_2_tables()
341 if version < 3:
342 self.log_info('upgrade to version 3')
343 self.fix_version_2_tables()
345 if version < 4:
346 self.fix_version_3_tables()
348 if version < 5:
349 self.fix_version_4_tables()
351 self.database_schema['version'] = self.current_db_version
352 self.db_version_updated = True
353 return 1
355 def fix_version_3_tables(self):
356 # drop the shorter VARCHAR OTK column and add a new TEXT one
357 for name in ('otk', 'session'):
358 self.sql('DELETE FROM %ss'%name)
359 self.sql('ALTER TABLE %ss DROP %s_value'%(name, name))
360 self.sql('ALTER TABLE %ss ADD %s_value TEXT'%(name, name))
362 def fix_version_2_tables(self):
363 # Default (used by sqlite): NOOP
364 pass
366 def fix_version_4_tables(self):
367 # note this is an explicit call now
368 c = self.cursor
369 for cn, klass in self.classes.iteritems():
370 c.execute('select id from _%s where __retired__<>0'%(cn,))
371 for (id,) in c.fetchall():
372 c.execute('update _%s set __retired__=%s where id=%s'%(cn,
373 self.arg, self.arg), (id, id))
375 if klass.key:
376 self.add_class_key_required_unique_constraint(cn, klass.key)
378 def _convert_journal_tables(self):
379 """Get current journal table contents, drop the table and re-create"""
380 c = self.cursor
381 cols = ','.join('nodeid date tag action params'.split())
382 for klass in self.classes.itervalues():
383 # slurp and drop
384 sql = 'select %s from %s__journal order by date'%(cols,
385 klass.classname)
386 c.execute(sql)
387 contents = c.fetchall()
388 self.drop_journal_table_indexes(klass.classname)
389 c.execute('drop table %s__journal'%klass.classname)
391 # re-create and re-populate
392 self.create_journal_table(klass)
393 a = self.arg
394 sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(
395 klass.classname, cols, a, a, a, a, a)
396 for row in contents:
397 # no data conversion needed
398 self.cursor.execute(sql, row)
400 def _convert_string_properties(self):
401 """Get current Class tables that contain String properties, and
402 convert the VARCHAR columns to TEXT"""
403 c = self.cursor
404 for klass in self.classes.itervalues():
405 # slurp and drop
406 cols, mls = self.determine_columns(list(klass.properties.iteritems()))
407 scols = ','.join([i[0] for i in cols])
408 sql = 'select id,%s from _%s'%(scols, klass.classname)
409 c.execute(sql)
410 contents = c.fetchall()
411 self.drop_class_table_indexes(klass.classname, klass.getkey())
412 c.execute('drop table _%s'%klass.classname)
414 # re-create and re-populate
415 self.create_class_table(klass, create_sequence=0)
416 a = ','.join([self.arg for i in range(len(cols)+1)])
417 sql = 'insert into _%s (id,%s) values (%s)'%(klass.classname,
418 scols, a)
419 for row in contents:
420 l = []
421 for entry in row:
422 # mysql will already be a string - psql needs "help"
423 if entry is not None and not isinstance(entry, type('')):
424 entry = str(entry)
425 l.append(entry)
426 self.cursor.execute(sql, l)
428 def refresh_database(self):
429 self.post_init()
432 def reindex(self, classname=None, show_progress=False):
433 if classname:
434 classes = [self.getclass(classname)]
435 else:
436 classes = list(self.classes.itervalues())
437 for klass in classes:
438 if show_progress:
439 for nodeid in support.Progress('Reindex %s'%klass.classname,
440 klass.list()):
441 klass.index(nodeid)
442 else:
443 for nodeid in klass.list():
444 klass.index(nodeid)
445 self.indexer.save_index()
447 hyperdb_to_sql_datatypes = {
448 hyperdb.String : 'TEXT',
449 hyperdb.Date : 'TIMESTAMP',
450 hyperdb.Link : 'INTEGER',
451 hyperdb.Interval : 'VARCHAR(255)',
452 hyperdb.Password : 'VARCHAR(255)',
453 hyperdb.Boolean : 'BOOLEAN',
454 hyperdb.Number : 'REAL',
455 }
457 def hyperdb_to_sql_datatype(self, propclass):
459 datatype = self.hyperdb_to_sql_datatypes.get(propclass)
460 if datatype:
461 return datatype
463 for k, v in self.hyperdb_to_sql_datatypes.iteritems():
464 if issubclass(propclass, k):
465 return v
467 raise ValueError('%r is not a hyperdb property class' % propclass)
469 def determine_columns(self, properties):
470 """ Figure the column names and multilink properties from the spec
472 "properties" is a list of (name, prop) where prop may be an
473 instance of a hyperdb "type" _or_ a string repr of that type.
474 """
475 cols = [
476 ('_actor', self.hyperdb_to_sql_datatype(hyperdb.Link)),
477 ('_activity', self.hyperdb_to_sql_datatype(hyperdb.Date)),
478 ('_creator', self.hyperdb_to_sql_datatype(hyperdb.Link)),
479 ('_creation', self.hyperdb_to_sql_datatype(hyperdb.Date)),
480 ]
481 mls = []
482 # add the multilinks separately
483 for col, prop in properties:
484 if isinstance(prop, Multilink):
485 mls.append(col)
486 continue
488 if isinstance(prop, type('')):
489 raise ValueError("string property spec!")
490 #and prop.find('Multilink') != -1:
491 #mls.append(col)
493 datatype = self.hyperdb_to_sql_datatype(prop.__class__)
494 cols.append(('_'+col, datatype))
496 # Intervals stored as two columns
497 if isinstance(prop, Interval):
498 cols.append(('__'+col+'_int__', 'BIGINT'))
500 cols.sort()
501 return cols, mls
503 def update_class(self, spec, old_spec, force=0):
504 """ Determine the differences between the current spec and the
505 database version of the spec, and update where necessary.
507 If 'force' is true, update the database anyway.
508 """
509 new_spec = spec.schema()
510 new_spec[1].sort()
511 old_spec[1].sort()
512 if not force and new_spec == old_spec:
513 # no changes
514 return 0
516 if not self.config.RDBMS_ALLOW_ALTER:
517 raise DatabaseError(_('ALTER operation disallowed: %r -> %r.'%(old_spec, new_spec)))
519 logger = logging.getLogger('roundup.hyperdb')
520 logger.info('update_class %s'%spec.classname)
522 logger.debug('old_spec %r'%(old_spec,))
523 logger.debug('new_spec %r'%(new_spec,))
525 # detect key prop change for potential index change
526 keyprop_changes = {}
527 if new_spec[0] != old_spec[0]:
528 if old_spec[0]:
529 keyprop_changes['remove'] = old_spec[0]
530 if new_spec[0]:
531 keyprop_changes['add'] = new_spec[0]
533 # detect multilinks that have been removed, and drop their table
534 old_has = {}
535 for name, prop in old_spec[1]:
536 old_has[name] = 1
537 if name in spec.properties:
538 continue
540 if prop.find('Multilink to') != -1:
541 # first drop indexes.
542 self.drop_multilink_table_indexes(spec.classname, name)
544 # now the multilink table itself
545 sql = 'drop table %s_%s'%(spec.classname, name)
546 else:
547 # if this is the key prop, drop the index first
548 if old_spec[0] == prop:
549 self.drop_class_table_key_index(spec.classname, name)
550 del keyprop_changes['remove']
552 # drop the column
553 sql = 'alter table _%s drop column _%s'%(spec.classname, name)
555 self.sql(sql)
557 # if we didn't remove the key prop just then, but the key prop has
558 # changed, we still need to remove the old index
559 if 'remove' in keyprop_changes:
560 self.drop_class_table_key_index(spec.classname,
561 keyprop_changes['remove'])
563 # add new columns
564 for propname, prop in new_spec[1]:
565 if propname in old_has:
566 continue
567 prop = spec.properties[propname]
568 if isinstance(prop, Multilink):
569 self.create_multilink_table(spec, propname)
570 else:
571 # add the column
572 coltype = self.hyperdb_to_sql_datatype(prop.__class__)
573 sql = 'alter table _%s add column _%s %s'%(
574 spec.classname, propname, coltype)
575 self.sql(sql)
577 # extra Interval column
578 if isinstance(prop, Interval):
579 sql = 'alter table _%s add column __%s_int__ BIGINT'%(
580 spec.classname, propname)
581 self.sql(sql)
583 # if the new column is a key prop, we need an index!
584 if new_spec[0] == propname:
585 self.create_class_table_key_index(spec.classname, propname)
586 del keyprop_changes['add']
588 # if we didn't add the key prop just then, but the key prop has
589 # changed, we still need to add the new index
590 if 'add' in keyprop_changes:
591 self.create_class_table_key_index(spec.classname,
592 keyprop_changes['add'])
594 return 1
596 def determine_all_columns(self, spec):
597 """Figure out the columns from the spec and also add internal columns
599 """
600 cols, mls = self.determine_columns(list(spec.properties.iteritems()))
602 # add on our special columns
603 cols.append(('id', 'INTEGER PRIMARY KEY'))
604 cols.append(('__retired__', 'INTEGER DEFAULT 0'))
605 return cols, mls
607 def create_class_table(self, spec):
608 """Create the class table for the given Class "spec". Creates the
609 indexes too."""
610 cols, mls = self.determine_all_columns(spec)
612 # create the base table
613 scols = ','.join(['%s %s'%x for x in cols])
614 sql = 'create table _%s (%s)'%(spec.classname, scols)
615 self.sql(sql)
617 self.create_class_table_indexes(spec)
619 return cols, mls
621 def create_class_table_indexes(self, spec):
622 """ create the class table for the given spec
623 """
624 # create __retired__ index
625 index_sql2 = 'create index _%s_retired_idx on _%s(__retired__)'%(
626 spec.classname, spec.classname)
627 self.sql(index_sql2)
629 # create index for key property
630 if spec.key:
631 index_sql3 = 'create index _%s_%s_idx on _%s(_%s)'%(
632 spec.classname, spec.key,
633 spec.classname, spec.key)
634 self.sql(index_sql3)
636 # and the unique index for key / retired(id)
637 self.add_class_key_required_unique_constraint(spec.classname,
638 spec.key)
640 # TODO: create indexes on (selected?) Link property columns, as
641 # they're more likely to be used for lookup
643 def add_class_key_required_unique_constraint(self, cn, key):
644 sql = '''create unique index _%s_key_retired_idx
645 on _%s(__retired__, _%s)'''%(cn, cn, key)
646 self.sql(sql)
648 def drop_class_table_indexes(self, cn, key):
649 # drop the old table indexes first
650 l = ['_%s_id_idx'%cn, '_%s_retired_idx'%cn]
651 if key:
652 l.append('_%s_%s_idx'%(cn, key))
654 table_name = '_%s'%cn
655 for index_name in l:
656 if not self.sql_index_exists(table_name, index_name):
657 continue
658 index_sql = 'drop index '+index_name
659 self.sql(index_sql)
661 def create_class_table_key_index(self, cn, key):
662 """ create the class table for the given spec
663 """
664 sql = 'create index _%s_%s_idx on _%s(_%s)'%(cn, key, cn, key)
665 self.sql(sql)
667 def drop_class_table_key_index(self, cn, key):
668 table_name = '_%s'%cn
669 index_name = '_%s_%s_idx'%(cn, key)
670 if self.sql_index_exists(table_name, index_name):
671 sql = 'drop index '+index_name
672 self.sql(sql)
674 # and now the retired unique index too
675 index_name = '_%s_key_retired_idx'%cn
676 if self.sql_index_exists(table_name, index_name):
677 sql = 'drop index '+index_name
678 self.sql(sql)
680 def create_journal_table(self, spec):
681 """ create the journal table for a class given the spec and
682 already-determined cols
683 """
684 # journal table
685 cols = ','.join(['%s varchar'%x
686 for x in 'nodeid date tag action params'.split()])
687 sql = """create table %s__journal (
688 nodeid integer, date %s, tag varchar(255),
689 action varchar(255), params text)""" % (spec.classname,
690 self.hyperdb_to_sql_datatype(hyperdb.Date))
691 self.sql(sql)
692 self.create_journal_table_indexes(spec)
694 def create_journal_table_indexes(self, spec):
695 # index on nodeid
696 sql = 'create index %s_journ_idx on %s__journal(nodeid)'%(
697 spec.classname, spec.classname)
698 self.sql(sql)
700 def drop_journal_table_indexes(self, classname):
701 index_name = '%s_journ_idx'%classname
702 if not self.sql_index_exists('%s__journal'%classname, index_name):
703 return
704 index_sql = 'drop index '+index_name
705 self.sql(index_sql)
707 def create_multilink_table(self, spec, ml):
708 """ Create a multilink table for the "ml" property of the class
709 given by the spec
710 """
711 # create the table
712 sql = 'create table %s_%s (linkid INTEGER, nodeid INTEGER)'%(
713 spec.classname, ml)
714 self.sql(sql)
715 self.create_multilink_table_indexes(spec, ml)
717 def create_multilink_table_indexes(self, spec, ml):
718 # create index on linkid
719 index_sql = 'create index %s_%s_l_idx on %s_%s(linkid)'%(
720 spec.classname, ml, spec.classname, ml)
721 self.sql(index_sql)
723 # create index on nodeid
724 index_sql = 'create index %s_%s_n_idx on %s_%s(nodeid)'%(
725 spec.classname, ml, spec.classname, ml)
726 self.sql(index_sql)
728 def drop_multilink_table_indexes(self, classname, ml):
729 l = [
730 '%s_%s_l_idx'%(classname, ml),
731 '%s_%s_n_idx'%(classname, ml)
732 ]
733 table_name = '%s_%s'%(classname, ml)
734 for index_name in l:
735 if not self.sql_index_exists(table_name, index_name):
736 continue
737 index_sql = 'drop index %s'%index_name
738 self.sql(index_sql)
740 def create_class(self, spec):
741 """ Create a database table according to the given spec.
742 """
744 if not self.config.RDBMS_ALLOW_CREATE:
745 raise DatabaseError(_('CREATE operation disallowed: "%s".'%spec.classname))
747 cols, mls = self.create_class_table(spec)
748 self.create_journal_table(spec)
750 # now create the multilink tables
751 for ml in mls:
752 self.create_multilink_table(spec, ml)
754 def drop_class(self, cn, spec):
755 """ Drop the given table from the database.
757 Drop the journal and multilink tables too.
758 """
760 if not self.config.RDBMS_ALLOW_DROP:
761 raise DatabaseError(_('DROP operation disallowed: "%s".'%cn))
763 properties = spec[1]
764 # figure the multilinks
765 mls = []
766 for propname, prop in properties:
767 if isinstance(prop, Multilink):
768 mls.append(propname)
770 # drop class table and indexes
771 self.drop_class_table_indexes(cn, spec[0])
773 self.drop_class_table(cn)
775 # drop journal table and indexes
776 self.drop_journal_table_indexes(cn)
777 sql = 'drop table %s__journal'%cn
778 self.sql(sql)
780 for ml in mls:
781 # drop multilink table and indexes
782 self.drop_multilink_table_indexes(cn, ml)
783 sql = 'drop table %s_%s'%(spec.classname, ml)
784 self.sql(sql)
786 def drop_class_table(self, cn):
787 sql = 'drop table _%s'%cn
788 self.sql(sql)
790 #
791 # Classes
792 #
793 def __getattr__(self, classname):
794 """ A convenient way of calling self.getclass(classname).
795 """
796 if classname in self.classes:
797 return self.classes[classname]
798 raise AttributeError(classname)
800 def addclass(self, cl):
801 """ Add a Class to the hyperdatabase.
802 """
803 cn = cl.classname
804 if cn in self.classes:
805 raise ValueError(cn)
806 self.classes[cn] = cl
808 # add default Edit and View permissions
809 self.security.addPermission(name="Create", klass=cn,
810 description="User is allowed to create "+cn)
811 self.security.addPermission(name="Edit", klass=cn,
812 description="User is allowed to edit "+cn)
813 self.security.addPermission(name="View", klass=cn,
814 description="User is allowed to access "+cn)
815 self.security.addPermission(name="Retire", klass=cn,
816 description="User is allowed to retire "+cn)
818 def getclasses(self):
819 """ Return a list of the names of all existing classes.
820 """
821 return sorted(self.classes)
823 def getclass(self, classname):
824 """Get the Class object representing a particular class.
826 If 'classname' is not a valid class name, a KeyError is raised.
827 """
828 try:
829 return self.classes[classname]
830 except KeyError:
831 raise KeyError('There is no class called "%s"'%classname)
833 def clear(self):
834 """Delete all database contents.
836 Note: I don't commit here, which is different behaviour to the
837 "nuke from orbit" behaviour in the dbs.
838 """
839 logging.getLogger('roundup.hyperdb').info('clear')
840 for cn in self.classes:
841 sql = 'delete from _%s'%cn
842 self.sql(sql)
844 #
845 # Nodes
846 #
848 hyperdb_to_sql_value = {
849 hyperdb.String : str,
850 # fractional seconds by default
851 hyperdb.Date : lambda x: x.formal(sep=' ', sec='%06.3f'),
852 hyperdb.Link : int,
853 hyperdb.Interval : str,
854 hyperdb.Password : str,
855 hyperdb.Boolean : lambda x: x and 'TRUE' or 'FALSE',
856 hyperdb.Number : lambda x: x,
857 hyperdb.Multilink : lambda x: x, # used in journal marshalling
858 }
860 def to_sql_value(self, propklass):
862 fn = self.hyperdb_to_sql_value.get(propklass)
863 if fn:
864 return fn
866 for k, v in self.hyperdb_to_sql_value.iteritems():
867 if issubclass(propklass, k):
868 return v
870 raise ValueError('%r is not a hyperdb property class' % propklass)
872 def _cache_del(self, key):
873 del self.cache[key]
874 self.cache_lru.remove(key)
876 def _cache_refresh(self, key):
877 self.cache_lru.remove(key)
878 self.cache_lru.insert(0, key)
880 def _cache_save(self, key, node):
881 self.cache[key] = node
882 # update the LRU
883 self.cache_lru.insert(0, key)
884 if len(self.cache_lru) > self.cache_size:
885 del self.cache[self.cache_lru.pop()]
887 def addnode(self, classname, nodeid, node):
888 """ Add the specified node to its class's db.
889 """
890 self.log_debug('addnode %s%s %r'%(classname,
891 nodeid, node))
893 # determine the column definitions and multilink tables
894 cl = self.classes[classname]
895 cols, mls = self.determine_columns(list(cl.properties.iteritems()))
897 # we'll be supplied these props if we're doing an import
898 values = node.copy()
899 if 'creator' not in values:
900 # add in the "calculated" properties (dupe so we don't affect
901 # calling code's node assumptions)
902 values['creation'] = values['activity'] = date.Date()
903 values['actor'] = values['creator'] = self.getuid()
905 cl = self.classes[classname]
906 props = cl.getprops(protected=1)
907 del props['id']
909 # default the non-multilink columns
910 for col, prop in props.iteritems():
911 if col not in values:
912 if isinstance(prop, Multilink):
913 values[col] = []
914 else:
915 values[col] = None
917 # clear this node out of the cache if it's in there
918 key = (classname, nodeid)
919 if key in self.cache:
920 self._cache_del(key)
922 # figure the values to insert
923 vals = []
924 for col,dt in cols:
925 # this is somewhat dodgy....
926 if col.endswith('_int__'):
927 # XXX eugh, this test suxxors
928 value = values[col[2:-6]]
929 # this is an Interval special "int" column
930 if value is not None:
931 vals.append(value.as_seconds())
932 else:
933 vals.append(value)
934 continue
936 prop = props[col[1:]]
937 value = values[col[1:]]
938 if value is not None:
939 value = self.to_sql_value(prop.__class__)(value)
940 vals.append(value)
941 vals.append(nodeid)
942 vals = tuple(vals)
944 # make sure the ordering is correct for column name -> column value
945 s = ','.join([self.arg for x in cols]) + ',%s'%self.arg
946 cols = ','.join([col for col,dt in cols]) + ',id'
948 # perform the inserts
949 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
950 self.sql(sql, vals)
952 # insert the multilink rows
953 for col in mls:
954 t = '%s_%s'%(classname, col)
955 for entry in node[col]:
956 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
957 self.arg, self.arg)
958 self.sql(sql, (entry, nodeid))
960 def setnode(self, classname, nodeid, values, multilink_changes={}):
961 """ Change the specified node.
962 """
963 self.log_debug('setnode %s%s %r'
964 % (classname, nodeid, values))
966 # clear this node out of the cache if it's in there
967 key = (classname, nodeid)
968 if key in self.cache:
969 self._cache_del(key)
971 cl = self.classes[classname]
972 props = cl.getprops()
974 cols = []
975 mls = []
976 # add the multilinks separately
977 for col in values:
978 prop = props[col]
979 if isinstance(prop, Multilink):
980 mls.append(col)
981 elif isinstance(prop, Interval):
982 # Intervals store the seconds value too
983 cols.append(col)
984 # extra leading '_' added by code below
985 cols.append('_' +col + '_int__')
986 else:
987 cols.append(col)
988 cols.sort()
990 # figure the values to insert
991 vals = []
992 for col in cols:
993 if col.endswith('_int__'):
994 # XXX eugh, this test suxxors
995 # Intervals store the seconds value too
996 col = col[1:-6]
997 prop = props[col]
998 value = values[col]
999 if value is None:
1000 vals.append(None)
1001 else:
1002 vals.append(value.as_seconds())
1003 else:
1004 prop = props[col]
1005 value = values[col]
1006 if value is None:
1007 e = None
1008 else:
1009 e = self.to_sql_value(prop.__class__)(value)
1010 vals.append(e)
1012 vals.append(int(nodeid))
1013 vals = tuple(vals)
1015 # if there's any updates to regular columns, do them
1016 if cols:
1017 # make sure the ordering is correct for column name -> column value
1018 s = ','.join(['_%s=%s'%(x, self.arg) for x in cols])
1019 cols = ','.join(cols)
1021 # perform the update
1022 sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
1023 self.sql(sql, vals)
1025 # we're probably coming from an import, not a change
1026 if not multilink_changes:
1027 for name in mls:
1028 prop = props[name]
1029 value = values[name]
1031 t = '%s_%s'%(classname, name)
1033 # clear out previous values for this node
1034 # XXX numeric ids
1035 self.sql('delete from %s where nodeid=%s'%(t, self.arg),
1036 (nodeid,))
1038 # insert the values for this node
1039 for entry in values[name]:
1040 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
1041 self.arg, self.arg)
1042 # XXX numeric ids
1043 self.sql(sql, (entry, nodeid))
1045 # we have multilink changes to apply
1046 for col, (add, remove) in multilink_changes.iteritems():
1047 tn = '%s_%s'%(classname, col)
1048 if add:
1049 sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
1050 self.arg, self.arg)
1051 for addid in add:
1052 # XXX numeric ids
1053 self.sql(sql, (int(nodeid), int(addid)))
1054 if remove:
1055 s = ','.join([self.arg]*len(remove))
1056 sql = 'delete from %s where nodeid=%s and linkid in (%s)'%(tn,
1057 self.arg, s)
1058 # XXX numeric ids
1059 self.sql(sql, [int(nodeid)] + remove)
1061 sql_to_hyperdb_value = {
1062 hyperdb.String : str,
1063 hyperdb.Date : date_to_hyperdb_value,
1064 # hyperdb.Link : int, # XXX numeric ids
1065 hyperdb.Link : str,
1066 hyperdb.Interval : date.Interval,
1067 hyperdb.Password : lambda x: password.Password(encrypted=x),
1068 hyperdb.Boolean : _bool_cvt,
1069 hyperdb.Number : _num_cvt,
1070 hyperdb.Multilink : lambda x: x, # used in journal marshalling
1071 }
1073 def to_hyperdb_value(self, propklass):
1075 fn = self.sql_to_hyperdb_value.get(propklass)
1076 if fn:
1077 return fn
1079 for k, v in self.sql_to_hyperdb_value.iteritems():
1080 if issubclass(propklass, k):
1081 return v
1083 raise ValueError('%r is not a hyperdb property class' % propklass)
1085 def _materialize_multilink(self, classname, nodeid, node, propname):
1086 """ evaluation of single Multilink (lazy eval may have skipped this)
1087 """
1088 if propname not in node:
1089 sql = 'select linkid from %s_%s where nodeid=%s'%(classname,
1090 propname, self.arg)
1091 self.sql(sql, (nodeid,))
1092 # extract the first column from the result
1093 # XXX numeric ids
1094 items = [int(x[0]) for x in self.cursor.fetchall()]
1095 items.sort ()
1096 node[propname] = [str(x) for x in items]
1098 def _materialize_multilinks(self, classname, nodeid, node, props=None):
1099 """ get all Multilinks of a node (lazy eval may have skipped this)
1100 """
1101 cl = self.classes[classname]
1102 props = props or [pn for (pn, p) in cl.properties.iteritems()
1103 if isinstance(p, Multilink)]
1104 for propname in props:
1105 if propname not in node:
1106 self._materialize_multilink(classname, nodeid, node, propname)
1108 def getnode(self, classname, nodeid, fetch_multilinks=True):
1109 """ Get a node from the database.
1110 For optimisation optionally we don't fetch multilinks
1111 (lazy Multilinks).
1112 But for internal database operations we need them.
1113 """
1114 # see if we have this node cached
1115 key = (classname, nodeid)
1116 if key in self.cache:
1117 # push us back to the top of the LRU
1118 self._cache_refresh(key)
1119 if __debug__:
1120 self.stats['cache_hits'] += 1
1121 # return the cached information
1122 if fetch_multilinks:
1123 self._materialize_multilinks(classname, nodeid, self.cache[key])
1124 return self.cache[key]
1126 if __debug__:
1127 self.stats['cache_misses'] += 1
1128 start_t = time.time()
1130 # figure the columns we're fetching
1131 cl = self.classes[classname]
1132 cols, mls = self.determine_columns(list(cl.properties.iteritems()))
1133 scols = ','.join([col for col,dt in cols])
1135 # perform the basic property fetch
1136 sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
1137 self.sql(sql, (nodeid,))
1139 values = self.sql_fetchone()
1140 if values is None:
1141 raise IndexError('no such %s node %s'%(classname, nodeid))
1143 # make up the node
1144 node = {}
1145 props = cl.getprops(protected=1)
1146 for col in range(len(cols)):
1147 name = cols[col][0][1:]
1148 if name.endswith('_int__'):
1149 # XXX eugh, this test suxxors
1150 # ignore the special Interval-as-seconds column
1151 continue
1152 value = values[col]
1153 if value is not None:
1154 value = self.to_hyperdb_value(props[name].__class__)(value)
1155 node[name] = value
1157 if fetch_multilinks and mls:
1158 self._materialize_multilinks(classname, nodeid, node, mls)
1160 # save off in the cache
1161 key = (classname, nodeid)
1162 self._cache_save(key, node)
1164 if __debug__:
1165 self.stats['get_items'] += (time.time() - start_t)
1167 return node
1169 def destroynode(self, classname, nodeid):
1170 """Remove a node from the database. Called exclusively by the
1171 destroy() method on Class.
1172 """
1173 logging.getLogger('roundup.hyperdb').info('destroynode %s%s'%(
1174 classname, nodeid))
1176 # make sure the node exists
1177 if not self.hasnode(classname, nodeid):
1178 raise IndexError('%s has no node %s'%(classname, nodeid))
1180 # see if we have this node cached
1181 if (classname, nodeid) in self.cache:
1182 del self.cache[(classname, nodeid)]
1184 # see if there's any obvious commit actions that we should get rid of
1185 for entry in self.transactions[:]:
1186 if entry[1][:2] == (classname, nodeid):
1187 self.transactions.remove(entry)
1189 # now do the SQL
1190 sql = 'delete from _%s where id=%s'%(classname, self.arg)
1191 self.sql(sql, (nodeid,))
1193 # remove from multilnks
1194 cl = self.getclass(classname)
1195 x, mls = self.determine_columns(list(cl.properties.iteritems()))
1196 for col in mls:
1197 # get the link ids
1198 sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
1199 self.sql(sql, (nodeid,))
1201 # remove journal entries
1202 sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
1203 self.sql(sql, (nodeid,))
1205 # cleanup any blob filestorage when we commit
1206 self.transactions.append((FileStorage.destroy, (self, classname, nodeid)))
1208 def hasnode(self, classname, nodeid):
1209 """ Determine if the database has a given node.
1210 """
1211 # If this node is in the cache, then we do not need to go to
1212 # the database. (We don't consider this an LRU hit, though.)
1213 if (classname, nodeid) in self.cache:
1214 # Return 1, not True, to match the type of the result of
1215 # the SQL operation below.
1216 return 1
1217 sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
1218 self.sql(sql, (nodeid,))
1219 return int(self.cursor.fetchone()[0])
1221 def countnodes(self, classname):
1222 """ Count the number of nodes that exist for a particular Class.
1223 """
1224 sql = 'select count(*) from _%s'%classname
1225 self.sql(sql)
1226 return self.cursor.fetchone()[0]
1228 def addjournal(self, classname, nodeid, action, params, creator=None,
1229 creation=None):
1230 """ Journal the Action
1231 'action' may be:
1233 'create' or 'set' -- 'params' is a dictionary of property values
1234 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
1235 'retire' -- 'params' is None
1236 """
1237 # handle supply of the special journalling parameters (usually
1238 # supplied on importing an existing database)
1239 if creator:
1240 journaltag = creator
1241 else:
1242 journaltag = self.getuid()
1243 if creation:
1244 journaldate = creation
1245 else:
1246 journaldate = date.Date()
1248 # create the journal entry
1249 cols = 'nodeid,date,tag,action,params'
1251 self.log_debug('addjournal %s%s %r %s %s %r'%(classname,
1252 nodeid, journaldate, journaltag, action, params))
1254 # make the journalled data marshallable
1255 if isinstance(params, type({})):
1256 self._journal_marshal(params, classname)
1258 params = repr(params)
1260 dc = self.to_sql_value(hyperdb.Date)
1261 journaldate = dc(journaldate)
1263 self.save_journal(classname, cols, nodeid, journaldate,
1264 journaltag, action, params)
1266 def setjournal(self, classname, nodeid, journal):
1267 """Set the journal to the "journal" list."""
1268 # clear out any existing entries
1269 self.sql('delete from %s__journal where nodeid=%s'%(classname,
1270 self.arg), (nodeid,))
1272 # create the journal entry
1273 cols = 'nodeid,date,tag,action,params'
1275 dc = self.to_sql_value(hyperdb.Date)
1276 for nodeid, journaldate, journaltag, action, params in journal:
1277 self.log_debug('addjournal %s%s %r %s %s %r'%(
1278 classname, nodeid, journaldate, journaltag, action,
1279 params))
1281 # make the journalled data marshallable
1282 if isinstance(params, type({})):
1283 self._journal_marshal(params, classname)
1284 params = repr(params)
1286 self.save_journal(classname, cols, nodeid, dc(journaldate),
1287 journaltag, action, params)
1289 def _journal_marshal(self, params, classname):
1290 """Convert the journal params values into safely repr'able and
1291 eval'able values."""
1292 properties = self.getclass(classname).getprops()
1293 for param, value in params.iteritems():
1294 if not value:
1295 continue
1296 property = properties[param]
1297 cvt = self.to_sql_value(property.__class__)
1298 if isinstance(property, Password):
1299 params[param] = cvt(value)
1300 elif isinstance(property, Date):
1301 params[param] = cvt(value)
1302 elif isinstance(property, Interval):
1303 params[param] = cvt(value)
1304 elif isinstance(property, Boolean):
1305 params[param] = cvt(value)
1307 def getjournal(self, classname, nodeid):
1308 """ get the journal for id
1309 """
1310 # make sure the node exists
1311 if not self.hasnode(classname, nodeid):
1312 raise IndexError('%s has no node %s'%(classname, nodeid))
1314 cols = ','.join('nodeid date tag action params'.split())
1315 journal = self.load_journal(classname, cols, nodeid)
1317 # now unmarshal the data
1318 dc = self.to_hyperdb_value(hyperdb.Date)
1319 res = []
1320 properties = self.getclass(classname).getprops()
1321 for nodeid, date_stamp, user, action, params in journal:
1322 params = eval(params)
1323 if isinstance(params, type({})):
1324 for param, value in params.iteritems():
1325 if not value:
1326 continue
1327 property = properties.get(param, None)
1328 if property is None:
1329 # deleted property
1330 continue
1331 cvt = self.to_hyperdb_value(property.__class__)
1332 if isinstance(property, Password):
1333 params[param] = password.JournalPassword(value)
1334 elif isinstance(property, Date):
1335 params[param] = cvt(value)
1336 elif isinstance(property, Interval):
1337 params[param] = cvt(value)
1338 elif isinstance(property, Boolean):
1339 params[param] = cvt(value)
1340 # XXX numeric ids
1341 res.append((str(nodeid), dc(date_stamp), user, action, params))
1342 return res
1344 def save_journal(self, classname, cols, nodeid, journaldate,
1345 journaltag, action, params):
1346 """ Save the journal entry to the database
1347 """
1348 entry = (nodeid, journaldate, journaltag, action, params)
1350 # do the insert
1351 a = self.arg
1352 sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(
1353 classname, cols, a, a, a, a, a)
1354 self.sql(sql, entry)
1356 def load_journal(self, classname, cols, nodeid):
1357 """ Load the journal from the database
1358 """
1359 # now get the journal entries
1360 sql = 'select %s from %s__journal where nodeid=%s order by date'%(
1361 cols, classname, self.arg)
1362 self.sql(sql, (nodeid,))
1363 return self.cursor.fetchall()
1365 def pack(self, pack_before):
1366 """ Delete all journal entries except "create" before 'pack_before'.
1367 """
1368 date_stamp = self.to_sql_value(Date)(pack_before)
1370 # do the delete
1371 for classname in self.classes:
1372 sql = "delete from %s__journal where date<%s and "\
1373 "action<>'create'"%(classname, self.arg)
1374 self.sql(sql, (date_stamp,))
1376 def sql_commit(self, fail_ok=False):
1377 """ Actually commit to the database.
1378 """
1379 logging.getLogger('roundup.hyperdb').info('commit')
1381 self.conn.commit()
1383 # open a new cursor for subsequent work
1384 self.cursor = self.conn.cursor()
1386 def commit(self, fail_ok=False):
1387 """ Commit the current transactions.
1389 Save all data changed since the database was opened or since the
1390 last commit() or rollback().
1392 fail_ok indicates that the commit is allowed to fail. This is used
1393 in the web interface when committing cleaning of the session
1394 database. We don't care if there's a concurrency issue there.
1396 The only backend this seems to affect is postgres.
1397 """
1398 # commit the database
1399 self.sql_commit(fail_ok)
1401 # now, do all the other transaction stuff
1402 for method, args in self.transactions:
1403 method(*args)
1405 # save the indexer
1406 self.indexer.save_index()
1408 # clear out the transactions
1409 self.transactions = []
1411 # clear the cache: Don't carry over cached values from one
1412 # transaction to the next (there may be other changes from other
1413 # transactions)
1414 self.clearCache()
1416 def sql_rollback(self):
1417 self.conn.rollback()
1419 def rollback(self):
1420 """ Reverse all actions from the current transaction.
1422 Undo all the changes made since the database was opened or the last
1423 commit() or rollback() was performed.
1424 """
1425 logging.getLogger('roundup.hyperdb').info('rollback')
1427 self.sql_rollback()
1429 # roll back "other" transaction stuff
1430 for method, args in self.transactions:
1431 # delete temporary files
1432 if method == self.doStoreFile:
1433 self.rollbackStoreFile(*args)
1434 self.transactions = []
1436 # clear the cache
1437 self.clearCache()
1439 def sql_close(self):
1440 logging.getLogger('roundup.hyperdb').info('close')
1441 self.conn.close()
1443 def close(self):
1444 """ Close off the connection.
1445 """
1446 self.indexer.close()
1447 self.sql_close()
1449 #
1450 # The base Class class
1451 #
1452 class Class(hyperdb.Class):
1453 """ The handle to a particular class of nodes in a hyperdatabase.
1455 All methods except __repr__ and getnode must be implemented by a
1456 concrete backend Class.
1457 """
1459 def schema(self):
1460 """ A dumpable version of the schema that we can store in the
1461 database
1462 """
1463 return (self.key, [(x, repr(y)) for x,y in self.properties.iteritems()])
1465 def enableJournalling(self):
1466 """Turn journalling on for this class
1467 """
1468 self.do_journal = 1
1470 def disableJournalling(self):
1471 """Turn journalling off for this class
1472 """
1473 self.do_journal = 0
1475 # Editing nodes:
1476 def create(self, **propvalues):
1477 """ Create a new node of this class and return its id.
1479 The keyword arguments in 'propvalues' map property names to values.
1481 The values of arguments must be acceptable for the types of their
1482 corresponding properties or a TypeError is raised.
1484 If this class has a key property, it must be present and its value
1485 must not collide with other key strings or a ValueError is raised.
1487 Any other properties on this class that are missing from the
1488 'propvalues' dictionary are set to None.
1490 If an id in a link or multilink property does not refer to a valid
1491 node, an IndexError is raised.
1492 """
1493 self.fireAuditors('create', None, propvalues)
1494 newid = self.create_inner(**propvalues)
1495 self.fireReactors('create', newid, None)
1496 return newid
1498 def create_inner(self, **propvalues):
1499 """ Called by create, in-between the audit and react calls.
1500 """
1501 if 'id' in propvalues:
1502 raise KeyError('"id" is reserved')
1504 if self.db.journaltag is None:
1505 raise DatabaseError(_('Database open read-only'))
1507 if ('creator' in propvalues or 'actor' in propvalues or
1508 'creation' in propvalues or 'activity' in propvalues):
1509 raise KeyError('"creator", "actor", "creation" and '
1510 '"activity" are reserved')
1512 # new node's id
1513 newid = self.db.newid(self.classname)
1515 # validate propvalues
1516 num_re = re.compile('^\d+$')
1517 for key, value in propvalues.iteritems():
1518 if key == self.key:
1519 try:
1520 self.lookup(value)
1521 except KeyError:
1522 pass
1523 else:
1524 raise ValueError('node with key "%s" exists'%value)
1526 # try to handle this property
1527 try:
1528 prop = self.properties[key]
1529 except KeyError:
1530 raise KeyError('"%s" has no property "%s"'%(self.classname,
1531 key))
1533 if value is not None and isinstance(prop, Link):
1534 if type(value) != type(''):
1535 raise ValueError('link value must be String')
1536 link_class = self.properties[key].classname
1537 # if it isn't a number, it's a key
1538 if not num_re.match(value):
1539 try:
1540 value = self.db.classes[link_class].lookup(value)
1541 except (TypeError, KeyError):
1542 raise IndexError('new property "%s": %s not a %s'%(
1543 key, value, link_class))
1544 elif not self.db.getclass(link_class).hasnode(value):
1545 raise IndexError('%s has no node %s'%(link_class,
1546 value))
1548 # save off the value
1549 propvalues[key] = value
1551 # register the link with the newly linked node
1552 if self.do_journal and self.properties[key].do_journal:
1553 self.db.addjournal(link_class, value, 'link',
1554 (self.classname, newid, key))
1556 elif isinstance(prop, Multilink):
1557 if value is None:
1558 value = []
1559 if not hasattr(value, '__iter__'):
1560 raise TypeError('new property "%s" not an iterable of ids'%key)
1561 # clean up and validate the list of links
1562 link_class = self.properties[key].classname
1563 l = []
1564 for entry in value:
1565 if type(entry) != type(''):
1566 raise ValueError('"%s" multilink value (%r) '
1567 'must contain Strings'%(key, value))
1568 # if it isn't a number, it's a key
1569 if not num_re.match(entry):
1570 try:
1571 entry = self.db.classes[link_class].lookup(entry)
1572 except (TypeError, KeyError):
1573 raise IndexError('new property "%s": %s not a %s'%(
1574 key, entry, self.properties[key].classname))
1575 l.append(entry)
1576 value = l
1577 propvalues[key] = value
1579 # handle additions
1580 for nodeid in value:
1581 if not self.db.getclass(link_class).hasnode(nodeid):
1582 raise IndexError('%s has no node %s'%(link_class,
1583 nodeid))
1584 # register the link with the newly linked node
1585 if self.do_journal and self.properties[key].do_journal:
1586 self.db.addjournal(link_class, nodeid, 'link',
1587 (self.classname, newid, key))
1589 elif isinstance(prop, String):
1590 if type(value) != type('') and type(value) != type(u''):
1591 raise TypeError('new property "%s" not a string'%key)
1592 if prop.indexme:
1593 self.db.indexer.add_text((self.classname, newid, key),
1594 value)
1596 elif isinstance(prop, Password):
1597 if not isinstance(value, password.Password):
1598 raise TypeError('new property "%s" not a Password'%key)
1600 elif isinstance(prop, Date):
1601 if value is not None and not isinstance(value, date.Date):
1602 raise TypeError('new property "%s" not a Date'%key)
1604 elif isinstance(prop, Interval):
1605 if value is not None and not isinstance(value, date.Interval):
1606 raise TypeError('new property "%s" not an Interval'%key)
1608 elif value is not None and isinstance(prop, Number):
1609 try:
1610 float(value)
1611 except ValueError:
1612 raise TypeError('new property "%s" not numeric'%key)
1614 elif value is not None and isinstance(prop, Boolean):
1615 try:
1616 int(value)
1617 except ValueError:
1618 raise TypeError('new property "%s" not boolean'%key)
1620 # make sure there's data where there needs to be
1621 for key, prop in self.properties.iteritems():
1622 if key in propvalues:
1623 continue
1624 if key == self.key:
1625 raise ValueError('key property "%s" is required'%key)
1626 if isinstance(prop, Multilink):
1627 propvalues[key] = []
1628 else:
1629 propvalues[key] = None
1631 # done
1632 self.db.addnode(self.classname, newid, propvalues)
1633 if self.do_journal:
1634 self.db.addjournal(self.classname, newid, ''"create", {})
1636 # XXX numeric ids
1637 return str(newid)
1639 def get(self, nodeid, propname, default=_marker, cache=1):
1640 """Get the value of a property on an existing node of this class.
1642 'nodeid' must be the id of an existing node of this class or an
1643 IndexError is raised. 'propname' must be the name of a property
1644 of this class or a KeyError is raised.
1646 'cache' exists for backwards compatibility, and is not used.
1647 """
1648 if propname == 'id':
1649 return nodeid
1651 # get the node's dict
1652 d = self.db.getnode(self.classname, nodeid, fetch_multilinks=False)
1653 # handle common case -- that property is in dict -- first
1654 # if None and one of creator/creation actor/activity return None
1655 if propname in d:
1656 r = d [propname]
1657 # return copy of our list
1658 if isinstance (r, list):
1659 return r[:]
1660 if r is not None:
1661 return r
1662 elif propname in ('creation', 'activity', 'creator', 'actor'):
1663 return r
1665 # propname not in d:
1666 if propname == 'creation' or propname == 'activity':
1667 return date.Date()
1668 if propname == 'creator' or propname == 'actor':
1669 return self.db.getuid()
1671 # get the property (raises KeyError if invalid)
1672 prop = self.properties[propname]
1674 # lazy evaluation of Multilink
1675 if propname not in d and isinstance(prop, Multilink):
1676 self.db._materialize_multilink(self.classname, nodeid, d, propname)
1678 # handle there being no value in the table for the property
1679 if propname not in d or d[propname] is None:
1680 if default is _marker:
1681 if isinstance(prop, Multilink):
1682 return []
1683 else:
1684 return None
1685 else:
1686 return default
1688 # don't pass our list to other code
1689 if isinstance(prop, Multilink):
1690 return d[propname][:]
1692 return d[propname]
1694 def set(self, nodeid, **propvalues):
1695 """Modify a property on an existing node of this class.
1697 'nodeid' must be the id of an existing node of this class or an
1698 IndexError is raised.
1700 Each key in 'propvalues' must be the name of a property of this
1701 class or a KeyError is raised.
1703 All values in 'propvalues' must be acceptable types for their
1704 corresponding properties or a TypeError is raised.
1706 If the value of the key property is set, it must not collide with
1707 other key strings or a ValueError is raised.
1709 If the value of a Link or Multilink property contains an invalid
1710 node id, a ValueError is raised.
1711 """
1712 self.fireAuditors('set', nodeid, propvalues)
1713 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1714 propvalues = self.set_inner(nodeid, **propvalues)
1715 self.fireReactors('set', nodeid, oldvalues)
1716 return propvalues
1718 def set_inner(self, nodeid, **propvalues):
1719 """ Called by set, in-between the audit and react calls.
1720 """
1721 if not propvalues:
1722 return propvalues
1724 if ('creator' in propvalues or 'actor' in propvalues or
1725 'creation' in propvalues or 'activity' in propvalues):
1726 raise KeyError('"creator", "actor", "creation" and '
1727 '"activity" are reserved')
1729 if 'id' in propvalues:
1730 raise KeyError('"id" is reserved')
1732 if self.db.journaltag is None:
1733 raise DatabaseError(_('Database open read-only'))
1735 node = self.db.getnode(self.classname, nodeid)
1736 if self.is_retired(nodeid):
1737 raise IndexError('Requested item is retired')
1738 num_re = re.compile('^\d+$')
1740 # make a copy of the values dictionary - we'll modify the contents
1741 propvalues = propvalues.copy()
1743 # if the journal value is to be different, store it in here
1744 journalvalues = {}
1746 # remember the add/remove stuff for multilinks, making it easier
1747 # for the Database layer to do its stuff
1748 multilink_changes = {}
1750 for propname, value in list(propvalues.items()):
1751 # check to make sure we're not duplicating an existing key
1752 if propname == self.key and node[propname] != value:
1753 try:
1754 self.lookup(value)
1755 except KeyError:
1756 pass
1757 else:
1758 raise ValueError('node with key "%s" exists'%value)
1760 # this will raise the KeyError if the property isn't valid
1761 # ... we don't use getprops() here because we only care about
1762 # the writeable properties.
1763 try:
1764 prop = self.properties[propname]
1765 except KeyError:
1766 raise KeyError('"%s" has no property named "%s"'%(
1767 self.classname, propname))
1769 # if the value's the same as the existing value, no sense in
1770 # doing anything
1771 current = node.get(propname, None)
1772 if value == current:
1773 del propvalues[propname]
1774 continue
1775 journalvalues[propname] = current
1777 # do stuff based on the prop type
1778 if isinstance(prop, Link):
1779 link_class = prop.classname
1780 # if it isn't a number, it's a key
1781 if value is not None and not isinstance(value, type('')):
1782 raise ValueError('property "%s" link value be a string'%(
1783 propname))
1784 if isinstance(value, type('')) and not num_re.match(value):
1785 try:
1786 value = self.db.classes[link_class].lookup(value)
1787 except (TypeError, KeyError):
1788 raise IndexError('new property "%s": %s not a %s'%(
1789 propname, value, prop.classname))
1791 if (value is not None and
1792 not self.db.getclass(link_class).hasnode(value)):
1793 raise IndexError('%s has no node %s'%(link_class,
1794 value))
1796 if self.do_journal and prop.do_journal:
1797 # register the unlink with the old linked node
1798 if node[propname] is not None:
1799 self.db.addjournal(link_class, node[propname],
1800 ''"unlink", (self.classname, nodeid, propname))
1802 # register the link with the newly linked node
1803 if value is not None:
1804 self.db.addjournal(link_class, value, ''"link",
1805 (self.classname, nodeid, propname))
1807 elif isinstance(prop, Multilink):
1808 if value is None:
1809 value = []
1810 if not hasattr(value, '__iter__'):
1811 raise TypeError('new property "%s" not an iterable of'
1812 ' ids'%propname)
1813 link_class = self.properties[propname].classname
1814 l = []
1815 for entry in value:
1816 # if it isn't a number, it's a key
1817 if type(entry) != type(''):
1818 raise ValueError('new property "%s" link value '
1819 'must be a string'%propname)
1820 if not num_re.match(entry):
1821 try:
1822 entry = self.db.classes[link_class].lookup(entry)
1823 except (TypeError, KeyError):
1824 raise IndexError('new property "%s": %s not a %s'%(
1825 propname, entry,
1826 self.properties[propname].classname))
1827 l.append(entry)
1828 value = l
1829 propvalues[propname] = value
1831 # figure the journal entry for this property
1832 add = []
1833 remove = []
1835 # handle removals
1836 if propname in node:
1837 l = node[propname]
1838 else:
1839 l = []
1840 for id in l[:]:
1841 if id in value:
1842 continue
1843 # register the unlink with the old linked node
1844 if self.do_journal and self.properties[propname].do_journal:
1845 self.db.addjournal(link_class, id, 'unlink',
1846 (self.classname, nodeid, propname))
1847 l.remove(id)
1848 remove.append(id)
1850 # handle additions
1851 for id in value:
1852 if id in l:
1853 continue
1854 # We can safely check this condition after
1855 # checking that this is an addition to the
1856 # multilink since the condition was checked for
1857 # existing entries at the point they were added to
1858 # the multilink. Since the hasnode call will
1859 # result in a SQL query, it is more efficient to
1860 # avoid the check if possible.
1861 if not self.db.getclass(link_class).hasnode(id):
1862 raise IndexError('%s has no node %s'%(link_class,
1863 id))
1864 # register the link with the newly linked node
1865 if self.do_journal and self.properties[propname].do_journal:
1866 self.db.addjournal(link_class, id, 'link',
1867 (self.classname, nodeid, propname))
1868 l.append(id)
1869 add.append(id)
1871 # figure the journal entry
1872 l = []
1873 if add:
1874 l.append(('+', add))
1875 if remove:
1876 l.append(('-', remove))
1877 multilink_changes[propname] = (add, remove)
1878 if l:
1879 journalvalues[propname] = tuple(l)
1881 elif isinstance(prop, String):
1882 if value is not None and type(value) != type('') and type(value) != type(u''):
1883 raise TypeError('new property "%s" not a string'%propname)
1884 if prop.indexme:
1885 if value is None: value = ''
1886 self.db.indexer.add_text((self.classname, nodeid, propname),
1887 value)
1889 elif isinstance(prop, Password):
1890 if not isinstance(value, password.Password):
1891 raise TypeError('new property "%s" not a Password'%propname)
1892 propvalues[propname] = value
1893 journalvalues[propname] = \
1894 current and password.JournalPassword(current)
1896 elif value is not None and isinstance(prop, Date):
1897 if not isinstance(value, date.Date):
1898 raise TypeError('new property "%s" not a Date'% propname)
1899 propvalues[propname] = value
1901 elif value is not None and isinstance(prop, Interval):
1902 if not isinstance(value, date.Interval):
1903 raise TypeError('new property "%s" not an '
1904 'Interval'%propname)
1905 propvalues[propname] = value
1907 elif value is not None and isinstance(prop, Number):
1908 try:
1909 float(value)
1910 except ValueError:
1911 raise TypeError('new property "%s" not numeric'%propname)
1913 elif value is not None and isinstance(prop, Boolean):
1914 try:
1915 int(value)
1916 except ValueError:
1917 raise TypeError('new property "%s" not boolean'%propname)
1919 # nothing to do?
1920 if not propvalues:
1921 return propvalues
1923 # update the activity time
1924 propvalues['activity'] = date.Date()
1925 propvalues['actor'] = self.db.getuid()
1927 # do the set
1928 self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
1930 # remove the activity props now they're handled
1931 del propvalues['activity']
1932 del propvalues['actor']
1934 # journal the set
1935 if self.do_journal:
1936 self.db.addjournal(self.classname, nodeid, ''"set", journalvalues)
1938 return propvalues
1940 def retire(self, nodeid):
1941 """Retire a node.
1943 The properties on the node remain available from the get() method,
1944 and the node's id is never reused.
1946 Retired nodes are not returned by the find(), list(), or lookup()
1947 methods, and other nodes may reuse the values of their key properties.
1948 """
1949 if self.db.journaltag is None:
1950 raise DatabaseError(_('Database open read-only'))
1952 self.fireAuditors('retire', nodeid, None)
1954 # use the arg for __retired__ to cope with any odd database type
1955 # conversion (hello, sqlite)
1956 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1957 self.db.arg, self.db.arg)
1958 self.db.sql(sql, (nodeid, nodeid))
1959 if self.do_journal:
1960 self.db.addjournal(self.classname, nodeid, ''"retired", None)
1962 self.fireReactors('retire', nodeid, None)
1964 def restore(self, nodeid):
1965 """Restore a retired node.
1967 Make node available for all operations like it was before retirement.
1968 """
1969 if self.db.journaltag is None:
1970 raise DatabaseError(_('Database open read-only'))
1972 node = self.db.getnode(self.classname, nodeid)
1973 # check if key property was overrided
1974 key = self.getkey()
1975 try:
1976 id = self.lookup(node[key])
1977 except KeyError:
1978 pass
1979 else:
1980 raise KeyError("Key property (%s) of retired node clashes "
1981 "with existing one (%s)" % (key, node[key]))
1983 self.fireAuditors('restore', nodeid, None)
1984 # use the arg for __retired__ to cope with any odd database type
1985 # conversion (hello, sqlite)
1986 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1987 self.db.arg, self.db.arg)
1988 self.db.sql(sql, (0, nodeid))
1989 if self.do_journal:
1990 self.db.addjournal(self.classname, nodeid, ''"restored", None)
1992 self.fireReactors('restore', nodeid, None)
1994 def is_retired(self, nodeid):
1995 """Return true if the node is rerired
1996 """
1997 sql = 'select __retired__ from _%s where id=%s'%(self.classname,
1998 self.db.arg)
1999 self.db.sql(sql, (nodeid,))
2000 return int(self.db.sql_fetchone()[0]) > 0
2002 def destroy(self, nodeid):
2003 """Destroy a node.
2005 WARNING: this method should never be used except in extremely rare
2006 situations where there could never be links to the node being
2007 deleted
2009 WARNING: use retire() instead
2011 WARNING: the properties of this node will not be available ever again
2013 WARNING: really, use retire() instead
2015 Well, I think that's enough warnings. This method exists mostly to
2016 support the session storage of the cgi interface.
2018 The node is completely removed from the hyperdb, including all journal
2019 entries. It will no longer be available, and will generally break code
2020 if there are any references to the node.
2021 """
2022 if self.db.journaltag is None:
2023 raise DatabaseError(_('Database open read-only'))
2024 self.db.destroynode(self.classname, nodeid)
2026 # Locating nodes:
2027 def hasnode(self, nodeid):
2028 """Determine if the given nodeid actually exists
2029 """
2030 return self.db.hasnode(self.classname, nodeid)
2032 def setkey(self, propname):
2033 """Select a String property of this class to be the key property.
2035 'propname' must be the name of a String property of this class or
2036 None, or a TypeError is raised. The values of the key property on
2037 all existing nodes must be unique or a ValueError is raised.
2038 """
2039 prop = self.getprops()[propname]
2040 if not isinstance(prop, String):
2041 raise TypeError('key properties must be String')
2042 self.key = propname
2044 def getkey(self):
2045 """Return the name of the key property for this class or None."""
2046 return self.key
2048 def lookup(self, keyvalue):
2049 """Locate a particular node by its key property and return its id.
2051 If this class has no key property, a TypeError is raised. If the
2052 'keyvalue' matches one of the values for the key property among
2053 the nodes in this class, the matching node's id is returned;
2054 otherwise a KeyError is raised.
2055 """
2056 if not self.key:
2057 raise TypeError('No key property set for class %s'%self.classname)
2059 # use the arg to handle any odd database type conversion (hello,
2060 # sqlite)
2061 sql = "select id from _%s where _%s=%s and __retired__=%s"%(
2062 self.classname, self.key, self.db.arg, self.db.arg)
2063 self.db.sql(sql, (str(keyvalue), 0))
2065 # see if there was a result that's not retired
2066 row = self.db.sql_fetchone()
2067 if not row:
2068 raise KeyError('No key (%s) value "%s" for "%s"'%(self.key,
2069 keyvalue, self.classname))
2071 # return the id
2072 # XXX numeric ids
2073 return str(row[0])
2075 def find(self, **propspec):
2076 """Get the ids of nodes in this class which link to the given nodes.
2078 'propspec' consists of keyword args propname=nodeid or
2079 propname={nodeid:1, }
2080 'propname' must be the name of a property in this class, or a
2081 KeyError is raised. That property must be a Link or
2082 Multilink property, or a TypeError is raised.
2084 Any node in this class whose 'propname' property links to any of
2085 the nodeids will be returned. Examples::
2087 db.issue.find(messages='1')
2088 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
2089 """
2090 # shortcut
2091 if not propspec:
2092 return []
2094 # validate the args
2095 props = self.getprops()
2096 for propname, nodeids in propspec.iteritems():
2097 # check the prop is OK
2098 prop = props[propname]
2099 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
2100 raise TypeError("'%s' not a Link/Multilink property"%propname)
2102 # first, links
2103 a = self.db.arg
2104 allvalues = ()
2105 sql = []
2106 where = []
2107 for prop, values in propspec.iteritems():
2108 if not isinstance(props[prop], hyperdb.Link):
2109 continue
2110 if type(values) is type({}) and len(values) == 1:
2111 values = list(values)[0]
2112 if type(values) is type(''):
2113 allvalues += (values,)
2114 where.append('_%s = %s'%(prop, a))
2115 elif values is None:
2116 where.append('_%s is NULL'%prop)
2117 else:
2118 values = list(values)
2119 s = ''
2120 if None in values:
2121 values.remove(None)
2122 s = '_%s is NULL or '%prop
2123 allvalues += tuple(values)
2124 s += '_%s in (%s)'%(prop, ','.join([a]*len(values)))
2125 where.append('(' + s +')')
2126 if where:
2127 allvalues = (0, ) + allvalues
2128 sql.append("""select id from _%s where __retired__=%s
2129 and %s"""%(self.classname, a, ' and '.join(where)))
2131 # now multilinks
2132 for prop, values in propspec.iteritems():
2133 if not isinstance(props[prop], hyperdb.Multilink):
2134 continue
2135 if not values:
2136 continue
2137 allvalues += (0, )
2138 if type(values) is type(''):
2139 allvalues += (values,)
2140 s = a
2141 else:
2142 allvalues += tuple(values)
2143 s = ','.join([a]*len(values))
2144 tn = '%s_%s'%(self.classname, prop)
2145 sql.append("""select id from _%s, %s where __retired__=%s
2146 and id = %s.nodeid and %s.linkid in (%s)"""%(self.classname,
2147 tn, a, tn, tn, s))
2149 if not sql:
2150 return []
2151 sql = ' union '.join(sql)
2152 self.db.sql(sql, allvalues)
2153 # XXX numeric ids
2154 l = [str(x[0]) for x in self.db.sql_fetchall()]
2155 return l
2157 def stringFind(self, **requirements):
2158 """Locate a particular node by matching a set of its String
2159 properties in a caseless search.
2161 If the property is not a String property, a TypeError is raised.
2163 The return is a list of the id of all nodes that match.
2164 """
2165 where = []
2166 args = []
2167 for propname in requirements:
2168 prop = self.properties[propname]
2169 if not isinstance(prop, String):
2170 raise TypeError("'%s' not a String property"%propname)
2171 where.append(propname)
2172 args.append(requirements[propname].lower())
2174 # generate the where clause
2175 s = ' and '.join(['lower(_%s)=%s'%(col, self.db.arg) for col in where])
2176 sql = 'select id from _%s where %s and __retired__=%s'%(
2177 self.classname, s, self.db.arg)
2178 args.append(0)
2179 self.db.sql(sql, tuple(args))
2180 # XXX numeric ids
2181 l = [str(x[0]) for x in self.db.sql_fetchall()]
2182 return l
2184 def list(self):
2185 """ Return a list of the ids of the active nodes in this class.
2186 """
2187 return self.getnodeids(retired=0)
2189 def getnodeids(self, retired=None):
2190 """ Retrieve all the ids of the nodes for a particular Class.
2192 Set retired=None to get all nodes. Otherwise it'll get all the
2193 retired or non-retired nodes, depending on the flag.
2194 """
2195 # flip the sense of the 'retired' flag if we don't want all of them
2196 if retired is not None:
2197 args = (0, )
2198 if retired:
2199 compare = '>'
2200 else:
2201 compare = '='
2202 sql = 'select id from _%s where __retired__%s%s'%(self.classname,
2203 compare, self.db.arg)
2204 else:
2205 args = ()
2206 sql = 'select id from _%s'%self.classname
2207 self.db.sql(sql, args)
2208 # XXX numeric ids
2209 ids = [str(x[0]) for x in self.db.cursor.fetchall()]
2210 return ids
2212 def _subselect(self, classname, multilink_table):
2213 """Create a subselect. This is factored out because some
2214 databases (hmm only one, so far) doesn't support subselects
2215 look for "I can't believe it's not a toy RDBMS" in the mysql
2216 backend.
2217 """
2218 return '_%s.id not in (select nodeid from %s)'%(classname,
2219 multilink_table)
2221 # Some DBs order NULL values last. Set this variable in the backend
2222 # for prepending an order by clause for each attribute that causes
2223 # correct sort order for NULLs. Examples:
2224 # order_by_null_values = '(%s is not NULL)'
2225 # order_by_null_values = 'notnull(%s)'
2226 # The format parameter is replaced with the attribute.
2227 order_by_null_values = None
2229 def supports_subselects(self):
2230 '''Assuming DBs can do subselects, overwrite if they cannot.
2231 '''
2232 return True
2234 def _filter_multilink_expression_fallback(
2235 self, classname, multilink_table, expr):
2236 '''This is a fallback for database that do not support
2237 subselects.'''
2239 is_valid = expr.evaluate
2241 last_id, kws = None, []
2243 ids = IdListOptimizer()
2244 append = ids.append
2246 # This join and the evaluation in program space
2247 # can be expensive for larger databases!
2248 # TODO: Find a faster way to collect the data needed
2249 # to evalute the expression.
2250 # Moving the expression evaluation into the database
2251 # would be nice but this tricky: Think about the cases
2252 # where the multilink table does not have join values
2253 # needed in evaluation.
2255 stmnt = "SELECT c.id, m.linkid FROM _%s c " \
2256 "LEFT OUTER JOIN %s m " \
2257 "ON c.id = m.nodeid ORDER BY c.id" % (
2258 classname, multilink_table)
2259 self.db.sql(stmnt)
2261 # collect all multilink items for a class item
2262 for nid, kw in self.db.sql_fetchiter():
2263 if nid != last_id:
2264 if last_id is None:
2265 last_id = nid
2266 else:
2267 # we have all multilink items -> evaluate!
2268 if is_valid(kws): append(last_id)
2269 last_id, kws = nid, []
2270 if kw is not None:
2271 kws.append(kw)
2273 if last_id is not None and is_valid(kws):
2274 append(last_id)
2276 # we have ids of the classname table
2277 return ids.where("_%s.id" % classname, self.db.arg)
2279 def _filter_multilink_expression(self, classname, multilink_table, v):
2280 """ Filters out elements of the classname table that do not
2281 match the given expression.
2282 Returns tuple of 'WHERE' introns for the overall filter.
2283 """
2284 try:
2285 opcodes = [int(x) for x in v]
2286 if min(opcodes) >= -1: raise ValueError()
2288 expr = compile_expression(opcodes)
2290 if not self.supports_subselects():
2291 # We heavily rely on subselects. If there is
2292 # no decent support fall back to slower variant.
2293 return self._filter_multilink_expression_fallback(
2294 classname, multilink_table, expr)
2296 atom = \
2297 "%s IN(SELECT linkid FROM %s WHERE nodeid=a.id)" % (
2298 self.db.arg,
2299 multilink_table)
2301 intron = \
2302 "_%(classname)s.id in (SELECT id " \
2303 "FROM _%(classname)s AS a WHERE %(condition)s) " % {
2304 'classname' : classname,
2305 'condition' : expr.generate(lambda n: atom) }
2307 values = []
2308 def collect_values(n): values.append(n.x)
2309 expr.visit(collect_values)
2311 return intron, values
2312 except:
2313 # original behavior
2314 where = "%s.linkid in (%s)" % (
2315 multilink_table, ','.join([self.db.arg] * len(v)))
2316 return where, v, True # True to indicate original
2318 def _filter_sql (self, search_matches, filterspec, srt=[], grp=[], retr=0):
2319 """ Compute the proptree and the SQL/ARGS for a filter.
2320 For argument description see filter below.
2321 We return a 3-tuple, the proptree, the sql and the sql-args
2322 or None if no SQL is necessary.
2323 The flag retr serves to retrieve *all* non-Multilink properties
2324 (for filling the cache during a filter_iter)
2325 """
2326 # we can't match anything if search_matches is empty
2327 if not search_matches and search_matches is not None:
2328 return None
2330 icn = self.classname
2332 # vars to hold the components of the SQL statement
2333 frum = [] # FROM clauses
2334 loj = [] # LEFT OUTER JOIN clauses
2335 where = [] # WHERE clauses
2336 args = [] # *any* positional arguments
2337 a = self.db.arg
2339 # figure the WHERE clause from the filterspec
2340 mlfilt = 0 # are we joining with Multilink tables?
2341 sortattr = self._sortattr (group = grp, sort = srt)
2342 proptree = self._proptree(filterspec, sortattr, retr)
2343 mlseen = 0
2344 for pt in reversed(proptree.sortattr):
2345 p = pt
2346 while p.parent:
2347 if isinstance (p.propclass, Multilink):
2348 mlseen = True
2349 if mlseen:
2350 p.sort_ids_needed = True
2351 p.tree_sort_done = False
2352 p = p.parent
2353 if not mlseen:
2354 pt.attr_sort_done = pt.tree_sort_done = True
2355 proptree.compute_sort_done()
2357 cols = ['_%s.id'%icn]
2358 mlsort = []
2359 rhsnum = 0
2360 for p in proptree:
2361 rc = ac = oc = None
2362 cn = p.classname
2363 ln = p.uniqname
2364 pln = p.parent.uniqname
2365 pcn = p.parent.classname
2366 k = p.name
2367 v = p.val
2368 propclass = p.propclass
2369 if p.parent == proptree and p.name == 'id' \
2370 and 'retrieve' in p.need_for:
2371 p.sql_idx = 0
2372 if 'sort' in p.need_for or 'retrieve' in p.need_for:
2373 rc = oc = ac = '_%s._%s'%(pln, k)
2374 if isinstance(propclass, Multilink):
2375 if 'search' in p.need_for:
2376 mlfilt = 1
2377 tn = '%s_%s'%(pcn, k)
2378 if v in ('-1', ['-1'], []):
2379 # only match rows that have count(linkid)=0 in the
2380 # corresponding multilink table)
2381 where.append(self._subselect(pcn, tn))
2382 else:
2383 frum.append(tn)
2384 gen_join = True
2386 if p.has_values and isinstance(v, type([])):
2387 result = self._filter_multilink_expression(pln, tn, v)
2388 # XXX: We dont need an id join if we used the filter
2389 gen_join = len(result) == 3
2391 if gen_join:
2392 where.append('_%s.id=%s.nodeid'%(pln,tn))
2394 if p.children:
2395 frum.append('_%s as _%s' % (cn, ln))
2396 where.append('%s.linkid=_%s.id'%(tn, ln))
2398 if p.has_values:
2399 if isinstance(v, type([])):
2400 where.append(result[0])
2401 args += result[1]
2402 else:
2403 where.append('%s.linkid=%s'%(tn, a))
2404 args.append(v)
2405 if 'sort' in p.need_for:
2406 assert not p.attr_sort_done and not p.sort_ids_needed
2407 elif k == 'id':
2408 if 'search' in p.need_for:
2409 if isinstance(v, type([])):
2410 # If there are no permitted values, then the
2411 # where clause will always be false, and we
2412 # can optimize the query away.
2413 if not v:
2414 return []
2415 s = ','.join([a for x in v])
2416 where.append('_%s.%s in (%s)'%(pln, k, s))
2417 args = args + v
2418 else:
2419 where.append('_%s.%s=%s'%(pln, k, a))
2420 args.append(v)
2421 if 'sort' in p.need_for or 'retrieve' in p.need_for:
2422 rc = oc = ac = '_%s.id'%pln
2423 elif isinstance(propclass, String):
2424 if 'search' in p.need_for:
2425 if not isinstance(v, type([])):
2426 v = [v]
2428 # Quote the bits in the string that need it and then embed
2429 # in a "substring" search. Note - need to quote the '%' so
2430 # they make it through the python layer happily
2431 v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
2433 # now add to the where clause
2434 where.append('('
2435 +' and '.join(["_%s._%s LIKE '%s'"%(pln, k, s) for s in v])
2436 +')')
2437 # note: args are embedded in the query string now
2438 if 'sort' in p.need_for:
2439 oc = ac = 'lower(_%s._%s)'%(pln, k)
2440 elif isinstance(propclass, Link):
2441 if 'search' in p.need_for:
2442 if p.children:
2443 if 'sort' not in p.need_for:
2444 frum.append('_%s as _%s' % (cn, ln))
2445 where.append('_%s._%s=_%s.id'%(pln, k, ln))
2446 if p.has_values:
2447 if isinstance(v, type([])):
2448 d = {}
2449 for entry in v:
2450 if entry == '-1':
2451 entry = None
2452 d[entry] = entry
2453 l = []
2454 if None in d or not d:
2455 if None in d: del d[None]
2456 l.append('_%s._%s is NULL'%(pln, k))
2457 if d:
2458 v = list(d)
2459 s = ','.join([a for x in v])
2460 l.append('(_%s._%s in (%s))'%(pln, k, s))
2461 args = args + v
2462 if l:
2463 where.append('(' + ' or '.join(l) +')')
2464 else:
2465 if v in ('-1', None):
2466 v = None
2467 where.append('_%s._%s is NULL'%(pln, k))
2468 else:
2469 where.append('_%s._%s=%s'%(pln, k, a))
2470 args.append(v)
2471 if 'sort' in p.need_for:
2472 lp = p.cls.labelprop()
2473 oc = ac = '_%s._%s'%(pln, k)
2474 if lp != 'id':
2475 if p.tree_sort_done:
2476 loj.append(
2477 'LEFT OUTER JOIN _%s as _%s on _%s._%s=_%s.id'%(
2478 cn, ln, pln, k, ln))
2479 oc = '_%s._%s'%(ln, lp)
2480 if 'retrieve' in p.need_for:
2481 rc = '_%s._%s'%(pln, k)
2482 elif isinstance(propclass, Date) and 'search' in p.need_for:
2483 dc = self.db.to_sql_value(hyperdb.Date)
2484 if isinstance(v, type([])):
2485 s = ','.join([a for x in v])
2486 where.append('_%s._%s in (%s)'%(pln, k, s))
2487 args = args + [dc(date.Date(x)) for x in v]
2488 else:
2489 try:
2490 # Try to filter on range of dates
2491 date_rng = propclass.range_from_raw(v, self.db)
2492 if date_rng.from_value:
2493 where.append('_%s._%s >= %s'%(pln, k, a))
2494 args.append(dc(date_rng.from_value))
2495 if date_rng.to_value:
2496 where.append('_%s._%s <= %s'%(pln, k, a))
2497 args.append(dc(date_rng.to_value))
2498 except ValueError:
2499 # If range creation fails - ignore that search parameter
2500 pass
2501 elif isinstance(propclass, Interval):
2502 # filter/sort using the __<prop>_int__ column
2503 if 'search' in p.need_for:
2504 if isinstance(v, type([])):
2505 s = ','.join([a for x in v])
2506 where.append('_%s.__%s_int__ in (%s)'%(pln, k, s))
2507 args = args + [date.Interval(x).as_seconds() for x in v]
2508 else:
2509 try:
2510 # Try to filter on range of intervals
2511 date_rng = Range(v, date.Interval)
2512 if date_rng.from_value:
2513 where.append('_%s.__%s_int__ >= %s'%(pln, k, a))
2514 args.append(date_rng.from_value.as_seconds())
2515 if date_rng.to_value:
2516 where.append('_%s.__%s_int__ <= %s'%(pln, k, a))
2517 args.append(date_rng.to_value.as_seconds())
2518 except ValueError:
2519 # If range creation fails - ignore search parameter
2520 pass
2521 if 'sort' in p.need_for:
2522 oc = ac = '_%s.__%s_int__'%(pln,k)
2523 if 'retrieve' in p.need_for:
2524 rc = '_%s._%s'%(pln,k)
2525 elif isinstance(propclass, Boolean) and 'search' in p.need_for:
2526 if type(v) == type(""):
2527 v = v.split(',')
2528 if type(v) != type([]):
2529 v = [v]
2530 bv = []
2531 for val in v:
2532 if type(val) is type(''):
2533 bv.append(propclass.from_raw (val))
2534 else:
2535 bv.append(bool(val))
2536 if len(bv) == 1:
2537 where.append('_%s._%s=%s'%(pln, k, a))
2538 args = args + bv
2539 else:
2540 s = ','.join([a for x in v])
2541 where.append('_%s._%s in (%s)'%(pln, k, s))
2542 args = args + bv
2543 elif 'search' in p.need_for:
2544 if isinstance(v, type([])):
2545 s = ','.join([a for x in v])
2546 where.append('_%s._%s in (%s)'%(pln, k, s))
2547 args = args + v
2548 else:
2549 where.append('_%s._%s=%s'%(pln, k, a))
2550 args.append(v)
2551 if oc:
2552 if p.sort_ids_needed:
2553 if rc == ac:
2554 p.sql_idx = len(cols)
2555 p.auxcol = len(cols)
2556 cols.append(ac)
2557 if p.tree_sort_done and p.sort_direction:
2558 # Don't select top-level id or multilink twice
2559 if (not p.sort_ids_needed or ac != oc) and (p.name != 'id'
2560 or p.parent != proptree):
2561 if rc == oc:
2562 p.sql_idx = len(cols)
2563 cols.append(oc)
2564 desc = ['', ' desc'][p.sort_direction == '-']
2565 # Some SQL dbs sort NULL values last -- we want them first.
2566 if (self.order_by_null_values and p.name != 'id'):
2567 nv = self.order_by_null_values % oc
2568 cols.append(nv)
2569 p.orderby.append(nv + desc)
2570 p.orderby.append(oc + desc)
2571 if 'retrieve' in p.need_for and p.sql_idx is None:
2572 assert(rc)
2573 p.sql_idx = len(cols)
2574 cols.append (rc)
2576 props = self.getprops()
2578 # don't match retired nodes
2579 where.append('_%s.__retired__=0'%icn)
2581 # add results of full text search
2582 if search_matches is not None:
2583 s = ','.join([a for x in search_matches])
2584 where.append('_%s.id in (%s)'%(icn, s))
2585 args = args + [x for x in search_matches]
2587 # construct the SQL
2588 frum.append('_'+icn)
2589 frum = ','.join(frum)
2590 if where:
2591 where = ' where ' + (' and '.join(where))
2592 else:
2593 where = ''
2594 if mlfilt:
2595 # we're joining tables on the id, so we will get dupes if we
2596 # don't distinct()
2597 cols[0] = 'distinct(_%s.id)'%icn
2599 order = []
2600 # keep correct sequence of order attributes.
2601 for sa in proptree.sortattr:
2602 if not sa.attr_sort_done:
2603 continue
2604 order.extend(sa.orderby)
2605 if order:
2606 order = ' order by %s'%(','.join(order))
2607 else:
2608 order = ''
2610 cols = ','.join(cols)
2611 loj = ' '.join(loj)
2612 sql = 'select %s from %s %s %s%s'%(cols, frum, loj, where, order)
2613 args = tuple(args)
2614 __traceback_info__ = (sql, args)
2615 return proptree, sql, args
2617 def filter(self, search_matches, filterspec, sort=[], group=[]):
2618 """Return a list of the ids of the active nodes in this class that
2619 match the 'filter' spec, sorted by the group spec and then the
2620 sort spec
2622 "filterspec" is {propname: value(s)}
2624 "sort" and "group" are [(dir, prop), ...] where dir is '+', '-'
2625 or None and prop is a prop name or None. Note that for
2626 backward-compatibility reasons a single (dir, prop) tuple is
2627 also allowed.
2629 "search_matches" is a container type or None
2631 The filter must match all properties specificed. If the property
2632 value to match is a list:
2634 1. String properties must match all elements in the list, and
2635 2. Other properties must match any of the elements in the list.
2636 """
2637 if __debug__:
2638 start_t = time.time()
2640 sq = self._filter_sql (search_matches, filterspec, sort, group)
2641 # nothing to match?
2642 if sq is None:
2643 return []
2644 proptree, sql, args = sq
2646 self.db.sql(sql, args)
2647 l = self.db.sql_fetchall()
2649 # Compute values needed for sorting in proptree.sort
2650 for p in proptree:
2651 if hasattr(p, 'auxcol'):
2652 p.sort_ids = p.sort_result = [row[p.auxcol] for row in l]
2653 # return the IDs (the first column)
2654 # XXX numeric ids
2655 l = [str(row[0]) for row in l]
2656 l = proptree.sort (l)
2658 if __debug__:
2659 self.db.stats['filtering'] += (time.time() - start_t)
2660 return l
2662 def filter_iter(self, search_matches, filterspec, sort=[], group=[]):
2663 """Iterator similar to filter above with same args.
2664 Limitation: We don't sort on multilinks.
2665 This uses an optimisation: We put all nodes that are in the
2666 current row into the node cache. Then we return the node id.
2667 That way a fetch of a node won't create another sql-fetch (with
2668 a join) from the database because the nodes are already in the
2669 cache. We're using our own temporary cursor.
2670 """
2671 sq = self._filter_sql(search_matches, filterspec, sort, group, retr=1)
2672 # nothing to match?
2673 if sq is None:
2674 return
2675 proptree, sql, args = sq
2676 cursor = self.db.conn.cursor()
2677 self.db.sql(sql, args, cursor)
2678 classes = {}
2679 for p in proptree:
2680 if 'retrieve' in p.need_for:
2681 cn = p.parent.classname
2682 ptid = p.parent.id # not the nodeid!
2683 key = (cn, ptid)
2684 if key not in classes:
2685 classes[key] = {}
2686 name = p.name
2687 assert (name)
2688 classes[key][name] = p
2689 p.to_hyperdb = self.db.to_hyperdb_value(p.propclass.__class__)
2690 while True:
2691 row = cursor.fetchone()
2692 if not row: break
2693 # populate cache with current items
2694 for (classname, ptid), pt in classes.iteritems():
2695 nodeid = str(row[pt['id'].sql_idx])
2696 key = (classname, nodeid)
2697 if key in self.db.cache:
2698 self.db._cache_refresh(key)
2699 continue
2700 node = {}
2701 for propname, p in pt.iteritems():
2702 value = row[p.sql_idx]
2703 if value is not None:
2704 value = p.to_hyperdb(value)
2705 node[propname] = value
2706 self.db._cache_save(key, node)
2707 yield str(row[0])
2709 def filter_sql(self, sql):
2710 """Return a list of the ids of the items in this class that match
2711 the SQL provided. The SQL is a complete "select" statement.
2713 The SQL select must include the item id as the first column.
2715 This function DOES NOT filter out retired items, add on a where
2716 clause "__retired__=0" if you don't want retired nodes.
2717 """
2718 if __debug__:
2719 start_t = time.time()
2721 self.db.sql(sql)
2722 l = self.db.sql_fetchall()
2724 if __debug__:
2725 self.db.stats['filtering'] += (time.time() - start_t)
2726 return l
2728 def count(self):
2729 """Get the number of nodes in this class.
2731 If the returned integer is 'numnodes', the ids of all the nodes
2732 in this class run from 1 to numnodes, and numnodes+1 will be the
2733 id of the next node to be created in this class.
2734 """
2735 return self.db.countnodes(self.classname)
2737 # Manipulating properties:
2738 def getprops(self, protected=1):
2739 """Return a dictionary mapping property names to property objects.
2740 If the "protected" flag is true, we include protected properties -
2741 those which may not be modified.
2742 """
2743 d = self.properties.copy()
2744 if protected:
2745 d['id'] = String()
2746 d['creation'] = hyperdb.Date()
2747 d['activity'] = hyperdb.Date()
2748 d['creator'] = hyperdb.Link('user')
2749 d['actor'] = hyperdb.Link('user')
2750 return d
2752 def addprop(self, **properties):
2753 """Add properties to this class.
2755 The keyword arguments in 'properties' must map names to property
2756 objects, or a TypeError is raised. None of the keys in 'properties'
2757 may collide with the names of existing properties, or a ValueError
2758 is raised before any properties have been added.
2759 """
2760 for key in properties:
2761 if key in self.properties:
2762 raise ValueError(key)
2763 self.properties.update(properties)
2765 def index(self, nodeid):
2766 """Add (or refresh) the node to search indexes
2767 """
2768 # find all the String properties that have indexme
2769 for prop, propclass in self.getprops().iteritems():
2770 if isinstance(propclass, String) and propclass.indexme:
2771 self.db.indexer.add_text((self.classname, nodeid, prop),
2772 str(self.get(nodeid, prop)))
2774 #
2775 # import / export support
2776 #
2777 def export_list(self, propnames, nodeid):
2778 """ Export a node - generate a list of CSV-able data in the order
2779 specified by propnames for the given node.
2780 """
2781 properties = self.getprops()
2782 l = []
2783 for prop in propnames:
2784 proptype = properties[prop]
2785 value = self.get(nodeid, prop)
2786 # "marshal" data where needed
2787 if value is None:
2788 pass
2789 elif isinstance(proptype, hyperdb.Date):
2790 value = value.get_tuple()
2791 elif isinstance(proptype, hyperdb.Interval):
2792 value = value.get_tuple()
2793 elif isinstance(proptype, hyperdb.Password):
2794 value = str(value)
2795 l.append(repr(value))
2796 l.append(repr(self.is_retired(nodeid)))
2797 return l
2799 def import_list(self, propnames, proplist):
2800 """ Import a node - all information including "id" is present and
2801 should not be sanity checked. Triggers are not triggered. The
2802 journal should be initialised using the "creator" and "created"
2803 information.
2805 Return the nodeid of the node imported.
2806 """
2807 if self.db.journaltag is None:
2808 raise DatabaseError(_('Database open read-only'))
2809 properties = self.getprops()
2811 # make the new node's property map
2812 d = {}
2813 retire = 0
2814 if not "id" in propnames:
2815 newid = self.db.newid(self.classname)
2816 else:
2817 newid = eval(proplist[propnames.index("id")])
2818 for i in range(len(propnames)):
2819 # Use eval to reverse the repr() used to output the CSV
2820 value = eval(proplist[i])
2822 # Figure the property for this column
2823 propname = propnames[i]
2825 # "unmarshal" where necessary
2826 if propname == 'id':
2827 continue
2828 elif propname == 'is retired':
2829 # is the item retired?
2830 if int(value):
2831 retire = 1
2832 continue
2833 elif value is None:
2834 d[propname] = None
2835 continue
2837 prop = properties[propname]
2838 if value is None:
2839 # don't set Nones
2840 continue
2841 elif isinstance(prop, hyperdb.Date):
2842 value = date.Date(value)
2843 elif isinstance(prop, hyperdb.Interval):
2844 value = date.Interval(value)
2845 elif isinstance(prop, hyperdb.Password):
2846 value = password.Password(encrypted=value)
2847 elif isinstance(prop, String):
2848 if isinstance(value, unicode):
2849 value = value.encode('utf8')
2850 if not isinstance(value, str):
2851 raise TypeError('new property "%(propname)s" not a '
2852 'string: %(value)r'%locals())
2853 if prop.indexme:
2854 self.db.indexer.add_text((self.classname, newid, propname),
2855 value)
2856 d[propname] = value
2858 # get a new id if necessary
2859 if newid is None:
2860 newid = self.db.newid(self.classname)
2862 # insert new node or update existing?
2863 if not self.hasnode(newid):
2864 self.db.addnode(self.classname, newid, d) # insert
2865 else:
2866 self.db.setnode(self.classname, newid, d) # update
2868 # retire?
2869 if retire:
2870 # use the arg for __retired__ to cope with any odd database type
2871 # conversion (hello, sqlite)
2872 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
2873 self.db.arg, self.db.arg)
2874 self.db.sql(sql, (newid, newid))
2875 return newid
2877 def export_journals(self):
2878 """Export a class's journal - generate a list of lists of
2879 CSV-able data:
2881 nodeid, date, user, action, params
2883 No heading here - the columns are fixed.
2884 """
2885 properties = self.getprops()
2886 r = []
2887 for nodeid in self.getnodeids():
2888 for nodeid, date, user, action, params in self.history(nodeid):
2889 date = date.get_tuple()
2890 if action == 'set':
2891 export_data = {}
2892 for propname, value in params.iteritems():
2893 if propname not in properties:
2894 # property no longer in the schema
2895 continue
2897 prop = properties[propname]
2898 # make sure the params are eval()'able
2899 if value is None:
2900 pass
2901 elif isinstance(prop, Date):
2902 value = value.get_tuple()
2903 elif isinstance(prop, Interval):
2904 value = value.get_tuple()
2905 elif isinstance(prop, Password):
2906 value = str(value)
2907 export_data[propname] = value
2908 params = export_data
2909 elif action == 'create' and params:
2910 # old tracker with data stored in the create!
2911 params = {}
2912 l = [nodeid, date, user, action, params]
2913 r.append(list(map(repr, l)))
2914 return r
2916 class FileClass(hyperdb.FileClass, Class):
2917 """This class defines a large chunk of data. To support this, it has a
2918 mandatory String property "content" which is typically saved off
2919 externally to the hyperdb.
2921 The default MIME type of this data is defined by the
2922 "default_mime_type" class attribute, which may be overridden by each
2923 node if the class defines a "type" String property.
2924 """
2925 def __init__(self, db, classname, **properties):
2926 """The newly-created class automatically includes the "content"
2927 and "type" properties.
2928 """
2929 if 'content' not in properties:
2930 properties['content'] = hyperdb.String(indexme='yes')
2931 if 'type' not in properties:
2932 properties['type'] = hyperdb.String()
2933 Class.__init__(self, db, classname, **properties)
2935 def create(self, **propvalues):
2936 """ snaffle the file propvalue and store in a file
2937 """
2938 # we need to fire the auditors now, or the content property won't
2939 # be in propvalues for the auditors to play with
2940 self.fireAuditors('create', None, propvalues)
2942 # now remove the content property so it's not stored in the db
2943 content = propvalues['content']
2944 del propvalues['content']
2946 # do the database create
2947 newid = self.create_inner(**propvalues)
2949 # figure the mime type
2950 mime_type = propvalues.get('type', self.default_mime_type)
2952 # and index!
2953 if self.properties['content'].indexme:
2954 self.db.indexer.add_text((self.classname, newid, 'content'),
2955 content, mime_type)
2957 # store off the content as a file
2958 self.db.storefile(self.classname, newid, None, content)
2960 # fire reactors
2961 self.fireReactors('create', newid, None)
2963 return newid
2965 def get(self, nodeid, propname, default=_marker, cache=1):
2966 """ Trap the content propname and get it from the file
2968 'cache' exists for backwards compatibility, and is not used.
2969 """
2970 poss_msg = 'Possibly a access right configuration problem.'
2971 if propname == 'content':
2972 try:
2973 return self.db.getfile(self.classname, nodeid, None)
2974 except IOError, strerror:
2975 # BUG: by catching this we donot see an error in the log.
2976 return 'ERROR reading file: %s%s\n%s\n%s'%(
2977 self.classname, nodeid, poss_msg, strerror)
2978 if default is not _marker:
2979 return Class.get(self, nodeid, propname, default)
2980 else:
2981 return Class.get(self, nodeid, propname)
2983 def set(self, itemid, **propvalues):
2984 """ Snarf the "content" propvalue and update it in a file
2985 """
2986 self.fireAuditors('set', itemid, propvalues)
2987 oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid))
2989 # now remove the content property so it's not stored in the db
2990 content = None
2991 if 'content' in propvalues:
2992 content = propvalues['content']
2993 del propvalues['content']
2995 # do the database create
2996 propvalues = self.set_inner(itemid, **propvalues)
2998 # do content?
2999 if content:
3000 # store and possibly index
3001 self.db.storefile(self.classname, itemid, None, content)
3002 if self.properties['content'].indexme:
3003 mime_type = self.get(itemid, 'type', self.default_mime_type)
3004 self.db.indexer.add_text((self.classname, itemid, 'content'),
3005 content, mime_type)
3006 propvalues['content'] = content
3008 # fire reactors
3009 self.fireReactors('set', itemid, oldvalues)
3010 return propvalues
3012 def index(self, nodeid):
3013 """ Add (or refresh) the node to search indexes.
3015 Use the content-type property for the content property.
3016 """
3017 # find all the String properties that have indexme
3018 for prop, propclass in self.getprops().iteritems():
3019 if prop == 'content' and propclass.indexme:
3020 mime_type = self.get(nodeid, 'type', self.default_mime_type)
3021 self.db.indexer.add_text((self.classname, nodeid, 'content'),
3022 str(self.get(nodeid, 'content')), mime_type)
3023 elif isinstance(propclass, hyperdb.String) and propclass.indexme:
3024 # index them under (classname, nodeid, property)
3025 try:
3026 value = str(self.get(nodeid, prop))
3027 except IndexError:
3028 # node has been destroyed
3029 continue
3030 self.db.indexer.add_text((self.classname, nodeid, prop), value)
3032 # XXX deviation from spec - was called ItemClass
3033 class IssueClass(Class, roundupdb.IssueClass):
3034 # Overridden methods:
3035 def __init__(self, db, classname, **properties):
3036 """The newly-created class automatically includes the "messages",
3037 "files", "nosy", and "superseder" properties. If the 'properties'
3038 dictionary attempts to specify any of these properties or a
3039 "creation", "creator", "activity" or "actor" property, a ValueError
3040 is raised.
3041 """
3042 if 'title' not in properties:
3043 properties['title'] = hyperdb.String(indexme='yes')
3044 if 'messages' not in properties:
3045 properties['messages'] = hyperdb.Multilink("msg")
3046 if 'files' not in properties:
3047 properties['files'] = hyperdb.Multilink("file")
3048 if 'nosy' not in properties:
3049 # note: journalling is turned off as it really just wastes
3050 # space. this behaviour may be overridden in an instance
3051 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
3052 if 'superseder' not in properties:
3053 properties['superseder'] = hyperdb.Multilink(classname)
3054 Class.__init__(self, db, classname, **properties)
3056 # vim: set et sts=4 sw=4 :