09425b2ac35ce557ac24c11d5f93829cb8fbd827
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 """ Relational database (SQL) backend common code.
20 Basics:
22 - map roundup classes to relational tables
23 - automatically detect schema changes and modify the table schemas
24 appropriately (we store the "database version" of the schema in the
25 database itself as the only row of the "schema" table)
26 - multilinks (which represent a many-to-many relationship) are handled through
27 intermediate tables
28 - journals are stored adjunct to the per-class tables
29 - table names and columns have "_" prepended so the names can't clash with
30 restricted names (like "order")
31 - retirement is determined by the __retired__ column being > 0
33 Database-specific changes may generally be pushed out to the overridable
34 sql_* methods, since everything else should be fairly generic. There's
35 probably a bit of work to be done if a database is used that actually
36 honors column typing, since the initial databases don't (sqlite stores
37 everything as a string.)
39 The schema of the hyperdb being mapped to the database is stored in the
40 database itself as a repr()'ed dictionary of information about each Class
41 that maps to a table. If that information differs from the hyperdb schema,
42 then we update it. We also store in the schema dict a version which
43 allows us to upgrade the database schema when necessary. See upgrade_db().
45 To force a unqiueness constraint on the key properties we put the item
46 id into the __retired__ column duing retirement (so it's 0 for "active"
47 items) and place a unqiueness constraint on key + __retired__. This is
48 particularly important for the users class where multiple users may
49 try to have the same username, with potentially many retired users with
50 the same name.
51 """
52 __docformat__ = 'restructuredtext'
54 # standard python modules
55 import sys, os, time, re, errno, weakref, copy, logging, datetime
57 # roundup modules
58 from roundup import hyperdb, date, password, roundupdb, security, support
59 from roundup.hyperdb import String, Password, Date, Interval, Link, \
60 Multilink, DatabaseError, Boolean, Number, Node
61 from roundup.backends import locking
62 from roundup.support import reversed
63 from roundup.i18n import _
66 # support
67 from roundup.backends.blobfiles import FileStorage
68 try:
69 from roundup.backends.indexer_xapian import Indexer
70 except ImportError:
71 from roundup.backends.indexer_rdbms import Indexer
72 from roundup.backends.sessions_rdbms import Sessions, OneTimeKeys
73 from roundup.date import Range
75 from roundup.backends.back_anydbm import compile_expression
78 # dummy value meaning "argument not passed"
79 _marker = []
81 def _num_cvt(num):
82 num = str(num)
83 try:
84 return int(num)
85 except:
86 return float(num)
88 def _bool_cvt(value):
89 if value in ('TRUE', 'FALSE'):
90 return {'TRUE': 1, 'FALSE': 0}[value]
91 # assume it's a number returned from the db API
92 return int(value)
94 def date_to_hyperdb_value(d):
95 """ convert date d to a roundup date """
96 if isinstance (d, datetime.datetime):
97 return date.Date(d)
98 return date.Date (str(d).replace(' ', '.'))
101 def connection_dict(config, dbnamestr=None):
102 """ Used by Postgresql and MySQL to detemine the keyword args for
103 opening the database connection."""
104 d = { }
105 if dbnamestr:
106 d[dbnamestr] = config.RDBMS_NAME
107 for name in ('host', 'port', 'password', 'user', 'read_default_group',
108 'read_default_file'):
109 cvar = 'RDBMS_'+name.upper()
110 if config[cvar] is not None:
111 d[name] = config[cvar]
112 return d
115 class IdListOptimizer:
116 """ To prevent flooding the SQL parser of the underlaying
117 db engine with "x IN (1, 2, 3, ..., <large number>)" collapses
118 these cases to "x BETWEEN 1 AND <large number>".
119 """
121 def __init__(self):
122 self.ranges = []
123 self.singles = []
125 def append(self, nid):
126 """ Invariant: nid are ordered ascending """
127 if self.ranges:
128 last = self.ranges[-1]
129 if last[1] == nid-1:
130 last[1] = nid
131 return
132 if self.singles:
133 last = self.singles[-1]
134 if last == nid-1:
135 self.singles.pop()
136 self.ranges.append([last, nid])
137 return
138 self.singles.append(nid)
140 def where(self, field, placeholder):
141 ranges = self.ranges
142 singles = self.singles
144 if not singles and not ranges: return "(1=0)", []
146 if ranges:
147 between = '%s BETWEEN %s AND %s' % (
148 field, placeholder, placeholder)
149 stmnt = [between] * len(ranges)
150 else:
151 stmnt = []
152 if singles:
153 stmnt.append('%s in (%s)' % (
154 field, ','.join([placeholder]*len(singles))))
156 return '(%s)' % ' OR '.join(stmnt), sum(ranges, []) + singles
158 def __str__(self):
159 return "ranges: %r / singles: %r" % (self.ranges, self.singles)
162 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
163 """ Wrapper around an SQL database that presents a hyperdb interface.
165 - some functionality is specific to the actual SQL database, hence
166 the sql_* methods that are NotImplemented
167 - we keep a cache of the latest N row fetches (where N is configurable).
168 """
169 def __init__(self, config, journaltag=None):
170 """ Open the database and load the schema from it.
171 """
172 FileStorage.__init__(self, config.UMASK)
173 self.config, self.journaltag = config, journaltag
174 self.dir = config.DATABASE
175 self.classes = {}
176 self.indexer = Indexer(self)
177 self.security = security.Security(self)
179 # additional transaction support for external files and the like
180 self.transactions = []
182 # keep a cache of the N most recently retrieved rows of any kind
183 # (classname, nodeid) = row
184 self.cache_size = config.RDBMS_CACHE_SIZE
185 self.clearCache()
186 self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0,
187 'filtering': 0}
189 # database lock
190 self.lockfile = None
192 # open a connection to the database, creating the "conn" attribute
193 self.open_connection()
195 def clearCache(self):
196 self.cache = {}
197 self.cache_lru = []
199 def getSessionManager(self):
200 return Sessions(self)
202 def getOTKManager(self):
203 return OneTimeKeys(self)
205 def open_connection(self):
206 """ Open a connection to the database, creating it if necessary.
208 Must call self.load_dbschema()
209 """
210 raise NotImplemented
212 def sql(self, sql, args=None, cursor=None):
213 """ Execute the sql with the optional args.
214 """
215 self.log_debug('SQL %r %r'%(sql, args))
216 if not cursor:
217 cursor = self.cursor
218 if args:
219 cursor.execute(sql, args)
220 else:
221 cursor.execute(sql)
223 def sql_fetchone(self):
224 """ Fetch a single row. If there's nothing to fetch, return None.
225 """
226 return self.cursor.fetchone()
228 def sql_fetchall(self):
229 """ Fetch all rows. If there's nothing to fetch, return [].
230 """
231 return self.cursor.fetchall()
233 def sql_fetchiter(self):
234 """ Fetch all row as a generator
235 """
236 while True:
237 row = self.cursor.fetchone()
238 if not row: break
239 yield row
241 def sql_stringquote(self, value):
242 """ Quote the string so it's safe to put in the 'sql quotes'
243 """
244 return re.sub("'", "''", str(value))
246 def init_dbschema(self):
247 self.database_schema = {
248 'version': self.current_db_version,
249 'tables': {}
250 }
252 def load_dbschema(self):
253 """ Load the schema definition that the database currently implements
254 """
255 self.cursor.execute('select schema from schema')
256 schema = self.cursor.fetchone()
257 if schema:
258 self.database_schema = eval(schema[0])
259 else:
260 self.database_schema = {}
262 def save_dbschema(self):
263 """ Save the schema definition that the database currently implements
264 """
265 s = repr(self.database_schema)
266 self.sql('delete from schema')
267 self.sql('insert into schema values (%s)'%self.arg, (s,))
269 def post_init(self):
270 """ Called once the schema initialisation has finished.
272 We should now confirm that the schema defined by our "classes"
273 attribute actually matches the schema in the database.
274 """
275 save = 0
277 # handle changes in the schema
278 tables = self.database_schema['tables']
279 for classname, spec in self.classes.iteritems():
280 if classname in tables:
281 dbspec = tables[classname]
282 if self.update_class(spec, dbspec):
283 tables[classname] = spec.schema()
284 save = 1
285 else:
286 self.create_class(spec)
287 tables[classname] = spec.schema()
288 save = 1
290 for classname, spec in list(tables.items()):
291 if classname not in self.classes:
292 self.drop_class(classname, tables[classname])
293 del tables[classname]
294 save = 1
296 # now upgrade the database for column type changes, new internal
297 # tables, etc.
298 save = save | self.upgrade_db()
300 # update the database version of the schema
301 if save:
302 self.save_dbschema()
304 # reindex the db if necessary
305 if self.indexer.should_reindex():
306 self.reindex()
308 # commit
309 self.sql_commit()
311 # update this number when we need to make changes to the SQL structure
312 # of the backen database
313 current_db_version = 5
314 db_version_updated = False
315 def upgrade_db(self):
316 """ Update the SQL database to reflect changes in the backend code.
318 Return boolean whether we need to save the schema.
319 """
320 version = self.database_schema.get('version', 1)
321 if version > self.current_db_version:
322 raise DatabaseError('attempting to run rev %d DATABASE with rev '
323 '%d CODE!'%(version, self.current_db_version))
324 if version == self.current_db_version:
325 # nothing to do
326 return 0
328 if version < 2:
329 self.log_info('upgrade to version 2')
330 # change the schema structure
331 self.database_schema = {'tables': self.database_schema}
333 # version 1 didn't have the actor column (note that in
334 # MySQL this will also transition the tables to typed columns)
335 self.add_new_columns_v2()
337 # version 1 doesn't have the OTK, session and indexing in the
338 # database
339 self.create_version_2_tables()
341 if version < 3:
342 self.log_info('upgrade to version 3')
343 self.fix_version_2_tables()
345 if version < 4:
346 self.fix_version_3_tables()
348 if version < 5:
349 self.fix_version_4_tables()
351 self.database_schema['version'] = self.current_db_version
352 self.db_version_updated = True
353 return 1
355 def fix_version_3_tables(self):
356 # drop the shorter VARCHAR OTK column and add a new TEXT one
357 for name in ('otk', 'session'):
358 self.sql('DELETE FROM %ss'%name)
359 self.sql('ALTER TABLE %ss DROP %s_value'%(name, name))
360 self.sql('ALTER TABLE %ss ADD %s_value TEXT'%(name, name))
362 def fix_version_2_tables(self):
363 # Default (used by sqlite): NOOP
364 pass
366 def fix_version_4_tables(self):
367 # note this is an explicit call now
368 c = self.cursor
369 for cn, klass in self.classes.iteritems():
370 c.execute('select id from _%s where __retired__<>0'%(cn,))
371 for (id,) in c.fetchall():
372 c.execute('update _%s set __retired__=%s where id=%s'%(cn,
373 self.arg, self.arg), (id, id))
375 if klass.key:
376 self.add_class_key_required_unique_constraint(cn, klass.key)
378 def _convert_journal_tables(self):
379 """Get current journal table contents, drop the table and re-create"""
380 c = self.cursor
381 cols = ','.join('nodeid date tag action params'.split())
382 for klass in self.classes.itervalues():
383 # slurp and drop
384 sql = 'select %s from %s__journal order by date'%(cols,
385 klass.classname)
386 c.execute(sql)
387 contents = c.fetchall()
388 self.drop_journal_table_indexes(klass.classname)
389 c.execute('drop table %s__journal'%klass.classname)
391 # re-create and re-populate
392 self.create_journal_table(klass)
393 a = self.arg
394 sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(
395 klass.classname, cols, a, a, a, a, a)
396 for row in contents:
397 # no data conversion needed
398 self.cursor.execute(sql, row)
400 def _convert_string_properties(self):
401 """Get current Class tables that contain String properties, and
402 convert the VARCHAR columns to TEXT"""
403 c = self.cursor
404 for klass in self.classes.itervalues():
405 # slurp and drop
406 cols, mls = self.determine_columns(list(klass.properties.iteritems()))
407 scols = ','.join([i[0] for i in cols])
408 sql = 'select id,%s from _%s'%(scols, klass.classname)
409 c.execute(sql)
410 contents = c.fetchall()
411 self.drop_class_table_indexes(klass.classname, klass.getkey())
412 c.execute('drop table _%s'%klass.classname)
414 # re-create and re-populate
415 self.create_class_table(klass, create_sequence=0)
416 a = ','.join([self.arg for i in range(len(cols)+1)])
417 sql = 'insert into _%s (id,%s) values (%s)'%(klass.classname,
418 scols, a)
419 for row in contents:
420 l = []
421 for entry in row:
422 # mysql will already be a string - psql needs "help"
423 if entry is not None and not isinstance(entry, type('')):
424 entry = str(entry)
425 l.append(entry)
426 self.cursor.execute(sql, l)
428 def refresh_database(self):
429 self.post_init()
432 def reindex(self, classname=None, show_progress=False):
433 if classname:
434 classes = [self.getclass(classname)]
435 else:
436 classes = list(self.classes.itervalues())
437 for klass in classes:
438 if show_progress:
439 for nodeid in support.Progress('Reindex %s'%klass.classname,
440 klass.list()):
441 klass.index(nodeid)
442 else:
443 for nodeid in klass.list():
444 klass.index(nodeid)
445 self.indexer.save_index()
447 hyperdb_to_sql_datatypes = {
448 hyperdb.String : 'TEXT',
449 hyperdb.Date : 'TIMESTAMP',
450 hyperdb.Link : 'INTEGER',
451 hyperdb.Interval : 'VARCHAR(255)',
452 hyperdb.Password : 'VARCHAR(255)',
453 hyperdb.Boolean : 'BOOLEAN',
454 hyperdb.Number : 'REAL',
455 }
457 def hyperdb_to_sql_datatype(self, propclass):
459 datatype = self.hyperdb_to_sql_datatypes.get(propclass)
460 if datatype:
461 return datatype
463 for k, v in self.hyperdb_to_sql_datatypes.iteritems():
464 if issubclass(propclass, k):
465 return v
467 raise ValueError('%r is not a hyperdb property class' % propclass)
469 def determine_columns(self, properties):
470 """ Figure the column names and multilink properties from the spec
472 "properties" is a list of (name, prop) where prop may be an
473 instance of a hyperdb "type" _or_ a string repr of that type.
474 """
475 cols = [
476 ('_actor', self.hyperdb_to_sql_datatype(hyperdb.Link)),
477 ('_activity', self.hyperdb_to_sql_datatype(hyperdb.Date)),
478 ('_creator', self.hyperdb_to_sql_datatype(hyperdb.Link)),
479 ('_creation', self.hyperdb_to_sql_datatype(hyperdb.Date)),
480 ]
481 mls = []
482 # add the multilinks separately
483 for col, prop in properties:
484 if isinstance(prop, Multilink):
485 mls.append(col)
486 continue
488 if isinstance(prop, type('')):
489 raise ValueError("string property spec!")
490 #and prop.find('Multilink') != -1:
491 #mls.append(col)
493 datatype = self.hyperdb_to_sql_datatype(prop.__class__)
494 cols.append(('_'+col, datatype))
496 # Intervals stored as two columns
497 if isinstance(prop, Interval):
498 cols.append(('__'+col+'_int__', 'BIGINT'))
500 cols.sort()
501 return cols, mls
503 def update_class(self, spec, old_spec, force=0):
504 """ Determine the differences between the current spec and the
505 database version of the spec, and update where necessary.
507 If 'force' is true, update the database anyway.
508 """
509 new_spec = spec.schema()
510 new_spec[1].sort()
511 old_spec[1].sort()
512 if not force and new_spec == old_spec:
513 # no changes
514 return 0
516 if not self.config.RDBMS_ALLOW_ALTER:
517 raise DatabaseError(_('ALTER operation disallowed: %r -> %r.'%(old_spec, new_spec)))
519 logger = logging.getLogger('roundup.hyperdb')
520 logger.info('update_class %s'%spec.classname)
522 logger.debug('old_spec %r'%(old_spec,))
523 logger.debug('new_spec %r'%(new_spec,))
525 # detect key prop change for potential index change
526 keyprop_changes = {}
527 if new_spec[0] != old_spec[0]:
528 if old_spec[0]:
529 keyprop_changes['remove'] = old_spec[0]
530 if new_spec[0]:
531 keyprop_changes['add'] = new_spec[0]
533 # detect multilinks that have been removed, and drop their table
534 old_has = {}
535 for name, prop in old_spec[1]:
536 old_has[name] = 1
537 if name in spec.properties:
538 continue
540 if prop.find('Multilink to') != -1:
541 # first drop indexes.
542 self.drop_multilink_table_indexes(spec.classname, name)
544 # now the multilink table itself
545 sql = 'drop table %s_%s'%(spec.classname, name)
546 else:
547 # if this is the key prop, drop the index first
548 if old_spec[0] == prop:
549 self.drop_class_table_key_index(spec.classname, name)
550 del keyprop_changes['remove']
552 # drop the column
553 sql = 'alter table _%s drop column _%s'%(spec.classname, name)
555 self.sql(sql)
557 # if we didn't remove the key prop just then, but the key prop has
558 # changed, we still need to remove the old index
559 if 'remove' in keyprop_changes:
560 self.drop_class_table_key_index(spec.classname,
561 keyprop_changes['remove'])
563 # add new columns
564 for propname, prop in new_spec[1]:
565 if propname in old_has:
566 continue
567 prop = spec.properties[propname]
568 if isinstance(prop, Multilink):
569 self.create_multilink_table(spec, propname)
570 else:
571 # add the column
572 coltype = self.hyperdb_to_sql_datatype(prop.__class__)
573 sql = 'alter table _%s add column _%s %s'%(
574 spec.classname, propname, coltype)
575 self.sql(sql)
577 # extra Interval column
578 if isinstance(prop, Interval):
579 sql = 'alter table _%s add column __%s_int__ BIGINT'%(
580 spec.classname, propname)
581 self.sql(sql)
583 # if the new column is a key prop, we need an index!
584 if new_spec[0] == propname:
585 self.create_class_table_key_index(spec.classname, propname)
586 del keyprop_changes['add']
588 # if we didn't add the key prop just then, but the key prop has
589 # changed, we still need to add the new index
590 if 'add' in keyprop_changes:
591 self.create_class_table_key_index(spec.classname,
592 keyprop_changes['add'])
594 return 1
596 def determine_all_columns(self, spec):
597 """Figure out the columns from the spec and also add internal columns
599 """
600 cols, mls = self.determine_columns(list(spec.properties.iteritems()))
602 # add on our special columns
603 cols.append(('id', 'INTEGER PRIMARY KEY'))
604 cols.append(('__retired__', 'INTEGER DEFAULT 0'))
605 return cols, mls
607 def create_class_table(self, spec):
608 """Create the class table for the given Class "spec". Creates the
609 indexes too."""
610 cols, mls = self.determine_all_columns(spec)
612 # create the base table
613 scols = ','.join(['%s %s'%x for x in cols])
614 sql = 'create table _%s (%s)'%(spec.classname, scols)
615 self.sql(sql)
617 self.create_class_table_indexes(spec)
619 return cols, mls
621 def create_class_table_indexes(self, spec):
622 """ create the class table for the given spec
623 """
624 # create __retired__ index
625 index_sql2 = 'create index _%s_retired_idx on _%s(__retired__)'%(
626 spec.classname, spec.classname)
627 self.sql(index_sql2)
629 # create index for key property
630 if spec.key:
631 index_sql3 = 'create index _%s_%s_idx on _%s(_%s)'%(
632 spec.classname, spec.key,
633 spec.classname, spec.key)
634 self.sql(index_sql3)
636 # and the unique index for key / retired(id)
637 self.add_class_key_required_unique_constraint(spec.classname,
638 spec.key)
640 # TODO: create indexes on (selected?) Link property columns, as
641 # they're more likely to be used for lookup
643 def add_class_key_required_unique_constraint(self, cn, key):
644 sql = '''create unique index _%s_key_retired_idx
645 on _%s(__retired__, _%s)'''%(cn, cn, key)
646 self.sql(sql)
648 def drop_class_table_indexes(self, cn, key):
649 # drop the old table indexes first
650 l = ['_%s_id_idx'%cn, '_%s_retired_idx'%cn]
651 if key:
652 l.append('_%s_%s_idx'%(cn, key))
654 table_name = '_%s'%cn
655 for index_name in l:
656 if not self.sql_index_exists(table_name, index_name):
657 continue
658 index_sql = 'drop index '+index_name
659 self.sql(index_sql)
661 def create_class_table_key_index(self, cn, key):
662 """ create the class table for the given spec
663 """
664 sql = 'create index _%s_%s_idx on _%s(_%s)'%(cn, key, cn, key)
665 self.sql(sql)
667 def drop_class_table_key_index(self, cn, key):
668 table_name = '_%s'%cn
669 index_name = '_%s_%s_idx'%(cn, key)
670 if self.sql_index_exists(table_name, index_name):
671 sql = 'drop index '+index_name
672 self.sql(sql)
674 # and now the retired unique index too
675 index_name = '_%s_key_retired_idx'%cn
676 if self.sql_index_exists(table_name, index_name):
677 sql = 'drop index '+index_name
678 self.sql(sql)
680 def create_journal_table(self, spec):
681 """ create the journal table for a class given the spec and
682 already-determined cols
683 """
684 # journal table
685 cols = ','.join(['%s varchar'%x
686 for x in 'nodeid date tag action params'.split()])
687 sql = """create table %s__journal (
688 nodeid integer, date %s, tag varchar(255),
689 action varchar(255), params text)""" % (spec.classname,
690 self.hyperdb_to_sql_datatype(hyperdb.Date))
691 self.sql(sql)
692 self.create_journal_table_indexes(spec)
694 def create_journal_table_indexes(self, spec):
695 # index on nodeid
696 sql = 'create index %s_journ_idx on %s__journal(nodeid)'%(
697 spec.classname, spec.classname)
698 self.sql(sql)
700 def drop_journal_table_indexes(self, classname):
701 index_name = '%s_journ_idx'%classname
702 if not self.sql_index_exists('%s__journal'%classname, index_name):
703 return
704 index_sql = 'drop index '+index_name
705 self.sql(index_sql)
707 def create_multilink_table(self, spec, ml):
708 """ Create a multilink table for the "ml" property of the class
709 given by the spec
710 """
711 # create the table
712 sql = 'create table %s_%s (linkid INTEGER, nodeid INTEGER)'%(
713 spec.classname, ml)
714 self.sql(sql)
715 self.create_multilink_table_indexes(spec, ml)
717 def create_multilink_table_indexes(self, spec, ml):
718 # create index on linkid
719 index_sql = 'create index %s_%s_l_idx on %s_%s(linkid)'%(
720 spec.classname, ml, spec.classname, ml)
721 self.sql(index_sql)
723 # create index on nodeid
724 index_sql = 'create index %s_%s_n_idx on %s_%s(nodeid)'%(
725 spec.classname, ml, spec.classname, ml)
726 self.sql(index_sql)
728 def drop_multilink_table_indexes(self, classname, ml):
729 l = [
730 '%s_%s_l_idx'%(classname, ml),
731 '%s_%s_n_idx'%(classname, ml)
732 ]
733 table_name = '%s_%s'%(classname, ml)
734 for index_name in l:
735 if not self.sql_index_exists(table_name, index_name):
736 continue
737 index_sql = 'drop index %s'%index_name
738 self.sql(index_sql)
740 def create_class(self, spec):
741 """ Create a database table according to the given spec.
742 """
744 if not self.config.RDBMS_ALLOW_CREATE:
745 raise DatabaseError(_('CREATE operation disallowed: "%s".'%spec.classname))
747 cols, mls = self.create_class_table(spec)
748 self.create_journal_table(spec)
750 # now create the multilink tables
751 for ml in mls:
752 self.create_multilink_table(spec, ml)
754 def drop_class(self, cn, spec):
755 """ Drop the given table from the database.
757 Drop the journal and multilink tables too.
758 """
760 if not self.config.RDBMS_ALLOW_DROP:
761 raise DatabaseError(_('DROP operation disallowed: "%s".'%cn))
763 properties = spec[1]
764 # figure the multilinks
765 mls = []
766 for propname, prop in properties:
767 if isinstance(prop, Multilink):
768 mls.append(propname)
770 # drop class table and indexes
771 self.drop_class_table_indexes(cn, spec[0])
773 self.drop_class_table(cn)
775 # drop journal table and indexes
776 self.drop_journal_table_indexes(cn)
777 sql = 'drop table %s__journal'%cn
778 self.sql(sql)
780 for ml in mls:
781 # drop multilink table and indexes
782 self.drop_multilink_table_indexes(cn, ml)
783 sql = 'drop table %s_%s'%(spec.classname, ml)
784 self.sql(sql)
786 def drop_class_table(self, cn):
787 sql = 'drop table _%s'%cn
788 self.sql(sql)
790 #
791 # Classes
792 #
793 def __getattr__(self, classname):
794 """ A convenient way of calling self.getclass(classname).
795 """
796 if classname in self.classes:
797 return self.classes[classname]
798 raise AttributeError(classname)
800 def addclass(self, cl):
801 """ Add a Class to the hyperdatabase.
802 """
803 cn = cl.classname
804 if cn in self.classes:
805 raise ValueError(cn)
806 self.classes[cn] = cl
808 # add default Edit and View permissions
809 self.security.addPermission(name="Create", klass=cn,
810 description="User is allowed to create "+cn)
811 self.security.addPermission(name="Edit", klass=cn,
812 description="User is allowed to edit "+cn)
813 self.security.addPermission(name="View", klass=cn,
814 description="User is allowed to access "+cn)
816 def getclasses(self):
817 """ Return a list of the names of all existing classes.
818 """
819 return sorted(self.classes)
821 def getclass(self, classname):
822 """Get the Class object representing a particular class.
824 If 'classname' is not a valid class name, a KeyError is raised.
825 """
826 try:
827 return self.classes[classname]
828 except KeyError:
829 raise KeyError('There is no class called "%s"'%classname)
831 def clear(self):
832 """Delete all database contents.
834 Note: I don't commit here, which is different behaviour to the
835 "nuke from orbit" behaviour in the dbs.
836 """
837 logging.getLogger('roundup.hyperdb').info('clear')
838 for cn in self.classes:
839 sql = 'delete from _%s'%cn
840 self.sql(sql)
842 #
843 # Nodes
844 #
846 hyperdb_to_sql_value = {
847 hyperdb.String : str,
848 # fractional seconds by default
849 hyperdb.Date : lambda x: x.formal(sep=' ', sec='%06.3f'),
850 hyperdb.Link : int,
851 hyperdb.Interval : str,
852 hyperdb.Password : str,
853 hyperdb.Boolean : lambda x: x and 'TRUE' or 'FALSE',
854 hyperdb.Number : lambda x: x,
855 hyperdb.Multilink : lambda x: x, # used in journal marshalling
856 }
858 def to_sql_value(self, propklass):
860 fn = self.hyperdb_to_sql_value.get(propklass)
861 if fn:
862 return fn
864 for k, v in self.hyperdb_to_sql_value.iteritems():
865 if issubclass(propklass, k):
866 return v
868 raise ValueError('%r is not a hyperdb property class' % propklass)
870 def _cache_del(self, key):
871 del self.cache[key]
872 self.cache_lru.remove(key)
874 def _cache_refresh(self, key):
875 self.cache_lru.remove(key)
876 self.cache_lru.insert(0, key)
878 def _cache_save(self, key, node):
879 self.cache[key] = node
880 # update the LRU
881 self.cache_lru.insert(0, key)
882 if len(self.cache_lru) > self.cache_size:
883 del self.cache[self.cache_lru.pop()]
885 def addnode(self, classname, nodeid, node):
886 """ Add the specified node to its class's db.
887 """
888 self.log_debug('addnode %s%s %r'%(classname,
889 nodeid, node))
891 # determine the column definitions and multilink tables
892 cl = self.classes[classname]
893 cols, mls = self.determine_columns(list(cl.properties.iteritems()))
895 # we'll be supplied these props if we're doing an import
896 values = node.copy()
897 if 'creator' not in values:
898 # add in the "calculated" properties (dupe so we don't affect
899 # calling code's node assumptions)
900 values['creation'] = values['activity'] = date.Date()
901 values['actor'] = values['creator'] = self.getuid()
903 cl = self.classes[classname]
904 props = cl.getprops(protected=1)
905 del props['id']
907 # default the non-multilink columns
908 for col, prop in props.iteritems():
909 if col not in values:
910 if isinstance(prop, Multilink):
911 values[col] = []
912 else:
913 values[col] = None
915 # clear this node out of the cache if it's in there
916 key = (classname, nodeid)
917 if key in self.cache:
918 self._cache_del(key)
920 # figure the values to insert
921 vals = []
922 for col,dt in cols:
923 # this is somewhat dodgy....
924 if col.endswith('_int__'):
925 # XXX eugh, this test suxxors
926 value = values[col[2:-6]]
927 # this is an Interval special "int" column
928 if value is not None:
929 vals.append(value.as_seconds())
930 else:
931 vals.append(value)
932 continue
934 prop = props[col[1:]]
935 value = values[col[1:]]
936 if value is not None:
937 value = self.to_sql_value(prop.__class__)(value)
938 vals.append(value)
939 vals.append(nodeid)
940 vals = tuple(vals)
942 # make sure the ordering is correct for column name -> column value
943 s = ','.join([self.arg for x in cols]) + ',%s'%self.arg
944 cols = ','.join([col for col,dt in cols]) + ',id'
946 # perform the inserts
947 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
948 self.sql(sql, vals)
950 # insert the multilink rows
951 for col in mls:
952 t = '%s_%s'%(classname, col)
953 for entry in node[col]:
954 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
955 self.arg, self.arg)
956 self.sql(sql, (entry, nodeid))
958 def setnode(self, classname, nodeid, values, multilink_changes={}):
959 """ Change the specified node.
960 """
961 self.log_debug('setnode %s%s %r'
962 % (classname, nodeid, values))
964 # clear this node out of the cache if it's in there
965 key = (classname, nodeid)
966 if key in self.cache:
967 self._cache_del(key)
969 cl = self.classes[classname]
970 props = cl.getprops()
972 cols = []
973 mls = []
974 # add the multilinks separately
975 for col in values:
976 prop = props[col]
977 if isinstance(prop, Multilink):
978 mls.append(col)
979 elif isinstance(prop, Interval):
980 # Intervals store the seconds value too
981 cols.append(col)
982 # extra leading '_' added by code below
983 cols.append('_' +col + '_int__')
984 else:
985 cols.append(col)
986 cols.sort()
988 # figure the values to insert
989 vals = []
990 for col in cols:
991 if col.endswith('_int__'):
992 # XXX eugh, this test suxxors
993 # Intervals store the seconds value too
994 col = col[1:-6]
995 prop = props[col]
996 value = values[col]
997 if value is None:
998 vals.append(None)
999 else:
1000 vals.append(value.as_seconds())
1001 else:
1002 prop = props[col]
1003 value = values[col]
1004 if value is None:
1005 e = None
1006 else:
1007 e = self.to_sql_value(prop.__class__)(value)
1008 vals.append(e)
1010 vals.append(int(nodeid))
1011 vals = tuple(vals)
1013 # if there's any updates to regular columns, do them
1014 if cols:
1015 # make sure the ordering is correct for column name -> column value
1016 s = ','.join(['_%s=%s'%(x, self.arg) for x in cols])
1017 cols = ','.join(cols)
1019 # perform the update
1020 sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
1021 self.sql(sql, vals)
1023 # we're probably coming from an import, not a change
1024 if not multilink_changes:
1025 for name in mls:
1026 prop = props[name]
1027 value = values[name]
1029 t = '%s_%s'%(classname, name)
1031 # clear out previous values for this node
1032 # XXX numeric ids
1033 self.sql('delete from %s where nodeid=%s'%(t, self.arg),
1034 (nodeid,))
1036 # insert the values for this node
1037 for entry in values[name]:
1038 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
1039 self.arg, self.arg)
1040 # XXX numeric ids
1041 self.sql(sql, (entry, nodeid))
1043 # we have multilink changes to apply
1044 for col, (add, remove) in multilink_changes.iteritems():
1045 tn = '%s_%s'%(classname, col)
1046 if add:
1047 sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
1048 self.arg, self.arg)
1049 for addid in add:
1050 # XXX numeric ids
1051 self.sql(sql, (int(nodeid), int(addid)))
1052 if remove:
1053 s = ','.join([self.arg]*len(remove))
1054 sql = 'delete from %s where nodeid=%s and linkid in (%s)'%(tn,
1055 self.arg, s)
1056 # XXX numeric ids
1057 self.sql(sql, [int(nodeid)] + remove)
1059 sql_to_hyperdb_value = {
1060 hyperdb.String : str,
1061 hyperdb.Date : date_to_hyperdb_value,
1062 # hyperdb.Link : int, # XXX numeric ids
1063 hyperdb.Link : str,
1064 hyperdb.Interval : date.Interval,
1065 hyperdb.Password : lambda x: password.Password(encrypted=x),
1066 hyperdb.Boolean : _bool_cvt,
1067 hyperdb.Number : _num_cvt,
1068 hyperdb.Multilink : lambda x: x, # used in journal marshalling
1069 }
1071 def to_hyperdb_value(self, propklass):
1073 fn = self.sql_to_hyperdb_value.get(propklass)
1074 if fn:
1075 return fn
1077 for k, v in self.sql_to_hyperdb_value.iteritems():
1078 if issubclass(propklass, k):
1079 return v
1081 raise ValueError('%r is not a hyperdb property class' % propklass)
1083 def getnode(self, classname, nodeid):
1084 """ Get a node from the database.
1085 """
1086 # see if we have this node cached
1087 key = (classname, nodeid)
1088 if key in self.cache:
1089 # push us back to the top of the LRU
1090 self._cache_refresh(key)
1091 if __debug__:
1092 self.stats['cache_hits'] += 1
1093 # return the cached information
1094 return self.cache[key]
1096 if __debug__:
1097 self.stats['cache_misses'] += 1
1098 start_t = time.time()
1100 # figure the columns we're fetching
1101 cl = self.classes[classname]
1102 cols, mls = self.determine_columns(list(cl.properties.iteritems()))
1103 scols = ','.join([col for col,dt in cols])
1105 # perform the basic property fetch
1106 sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
1107 self.sql(sql, (nodeid,))
1109 values = self.sql_fetchone()
1110 if values is None:
1111 raise IndexError('no such %s node %s'%(classname, nodeid))
1113 # make up the node
1114 node = {}
1115 props = cl.getprops(protected=1)
1116 for col in range(len(cols)):
1117 name = cols[col][0][1:]
1118 if name.endswith('_int__'):
1119 # XXX eugh, this test suxxors
1120 # ignore the special Interval-as-seconds column
1121 continue
1122 value = values[col]
1123 if value is not None:
1124 value = self.to_hyperdb_value(props[name].__class__)(value)
1125 node[name] = value
1127 # save off in the cache
1128 key = (classname, nodeid)
1129 self._cache_save(key, node)
1131 if __debug__:
1132 self.stats['get_items'] += (time.time() - start_t)
1134 return node
1136 def destroynode(self, classname, nodeid):
1137 """Remove a node from the database. Called exclusively by the
1138 destroy() method on Class.
1139 """
1140 logging.getLogger('roundup.hyperdb').info('destroynode %s%s'%(
1141 classname, nodeid))
1143 # make sure the node exists
1144 if not self.hasnode(classname, nodeid):
1145 raise IndexError('%s has no node %s'%(classname, nodeid))
1147 # see if we have this node cached
1148 if (classname, nodeid) in self.cache:
1149 del self.cache[(classname, nodeid)]
1151 # see if there's any obvious commit actions that we should get rid of
1152 for entry in self.transactions[:]:
1153 if entry[1][:2] == (classname, nodeid):
1154 self.transactions.remove(entry)
1156 # now do the SQL
1157 sql = 'delete from _%s where id=%s'%(classname, self.arg)
1158 self.sql(sql, (nodeid,))
1160 # remove from multilnks
1161 cl = self.getclass(classname)
1162 x, mls = self.determine_columns(list(cl.properties.iteritems()))
1163 for col in mls:
1164 # get the link ids
1165 sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
1166 self.sql(sql, (nodeid,))
1168 # remove journal entries
1169 sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
1170 self.sql(sql, (nodeid,))
1172 # cleanup any blob filestorage when we commit
1173 self.transactions.append((FileStorage.destroy, (self, classname, nodeid)))
1175 def hasnode(self, classname, nodeid):
1176 """ Determine if the database has a given node.
1177 """
1178 # If this node is in the cache, then we do not need to go to
1179 # the database. (We don't consider this an LRU hit, though.)
1180 if (classname, nodeid) in self.cache:
1181 # Return 1, not True, to match the type of the result of
1182 # the SQL operation below.
1183 return 1
1184 sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
1185 self.sql(sql, (nodeid,))
1186 return int(self.cursor.fetchone()[0])
1188 def countnodes(self, classname):
1189 """ Count the number of nodes that exist for a particular Class.
1190 """
1191 sql = 'select count(*) from _%s'%classname
1192 self.sql(sql)
1193 return self.cursor.fetchone()[0]
1195 def addjournal(self, classname, nodeid, action, params, creator=None,
1196 creation=None):
1197 """ Journal the Action
1198 'action' may be:
1200 'create' or 'set' -- 'params' is a dictionary of property values
1201 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
1202 'retire' -- 'params' is None
1203 """
1204 # handle supply of the special journalling parameters (usually
1205 # supplied on importing an existing database)
1206 if creator:
1207 journaltag = creator
1208 else:
1209 journaltag = self.getuid()
1210 if creation:
1211 journaldate = creation
1212 else:
1213 journaldate = date.Date()
1215 # create the journal entry
1216 cols = 'nodeid,date,tag,action,params'
1218 self.log_debug('addjournal %s%s %r %s %s %r'%(classname,
1219 nodeid, journaldate, journaltag, action, params))
1221 # make the journalled data marshallable
1222 if isinstance(params, type({})):
1223 self._journal_marshal(params, classname)
1225 params = repr(params)
1227 dc = self.to_sql_value(hyperdb.Date)
1228 journaldate = dc(journaldate)
1230 self.save_journal(classname, cols, nodeid, journaldate,
1231 journaltag, action, params)
1233 def setjournal(self, classname, nodeid, journal):
1234 """Set the journal to the "journal" list."""
1235 # clear out any existing entries
1236 self.sql('delete from %s__journal where nodeid=%s'%(classname,
1237 self.arg), (nodeid,))
1239 # create the journal entry
1240 cols = 'nodeid,date,tag,action,params'
1242 dc = self.to_sql_value(hyperdb.Date)
1243 for nodeid, journaldate, journaltag, action, params in journal:
1244 self.log_debug('addjournal %s%s %r %s %s %r'%(
1245 classname, nodeid, journaldate, journaltag, action,
1246 params))
1248 # make the journalled data marshallable
1249 if isinstance(params, type({})):
1250 self._journal_marshal(params, classname)
1251 params = repr(params)
1253 self.save_journal(classname, cols, nodeid, dc(journaldate),
1254 journaltag, action, params)
1256 def _journal_marshal(self, params, classname):
1257 """Convert the journal params values into safely repr'able and
1258 eval'able values."""
1259 properties = self.getclass(classname).getprops()
1260 for param, value in params.iteritems():
1261 if not value:
1262 continue
1263 property = properties[param]
1264 cvt = self.to_sql_value(property.__class__)
1265 if isinstance(property, Password):
1266 params[param] = cvt(value)
1267 elif isinstance(property, Date):
1268 params[param] = cvt(value)
1269 elif isinstance(property, Interval):
1270 params[param] = cvt(value)
1271 elif isinstance(property, Boolean):
1272 params[param] = cvt(value)
1274 def getjournal(self, classname, nodeid):
1275 """ get the journal for id
1276 """
1277 # make sure the node exists
1278 if not self.hasnode(classname, nodeid):
1279 raise IndexError('%s has no node %s'%(classname, nodeid))
1281 cols = ','.join('nodeid date tag action params'.split())
1282 journal = self.load_journal(classname, cols, nodeid)
1284 # now unmarshal the data
1285 dc = self.to_hyperdb_value(hyperdb.Date)
1286 res = []
1287 properties = self.getclass(classname).getprops()
1288 for nodeid, date_stamp, user, action, params in journal:
1289 params = eval(params)
1290 if isinstance(params, type({})):
1291 for param, value in params.iteritems():
1292 if not value:
1293 continue
1294 property = properties.get(param, None)
1295 if property is None:
1296 # deleted property
1297 continue
1298 cvt = self.to_hyperdb_value(property.__class__)
1299 if isinstance(property, Password):
1300 params[param] = cvt(value)
1301 elif isinstance(property, Date):
1302 params[param] = cvt(value)
1303 elif isinstance(property, Interval):
1304 params[param] = cvt(value)
1305 elif isinstance(property, Boolean):
1306 params[param] = cvt(value)
1307 # XXX numeric ids
1308 res.append((str(nodeid), dc(date_stamp), user, action, params))
1309 return res
1311 def save_journal(self, classname, cols, nodeid, journaldate,
1312 journaltag, action, params):
1313 """ Save the journal entry to the database
1314 """
1315 entry = (nodeid, journaldate, journaltag, action, params)
1317 # do the insert
1318 a = self.arg
1319 sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(
1320 classname, cols, a, a, a, a, a)
1321 self.sql(sql, entry)
1323 def load_journal(self, classname, cols, nodeid):
1324 """ Load the journal from the database
1325 """
1326 # now get the journal entries
1327 sql = 'select %s from %s__journal where nodeid=%s order by date'%(
1328 cols, classname, self.arg)
1329 self.sql(sql, (nodeid,))
1330 return self.cursor.fetchall()
1332 def pack(self, pack_before):
1333 """ Delete all journal entries except "create" before 'pack_before'.
1334 """
1335 date_stamp = self.to_sql_value(Date)(pack_before)
1337 # do the delete
1338 for classname in self.classes:
1339 sql = "delete from %s__journal where date<%s and "\
1340 "action<>'create'"%(classname, self.arg)
1341 self.sql(sql, (date_stamp,))
1343 def sql_commit(self, fail_ok=False):
1344 """ Actually commit to the database.
1345 """
1346 logging.getLogger('roundup.hyperdb').info('commit')
1348 self.conn.commit()
1350 # open a new cursor for subsequent work
1351 self.cursor = self.conn.cursor()
1353 def commit(self, fail_ok=False):
1354 """ Commit the current transactions.
1356 Save all data changed since the database was opened or since the
1357 last commit() or rollback().
1359 fail_ok indicates that the commit is allowed to fail. This is used
1360 in the web interface when committing cleaning of the session
1361 database. We don't care if there's a concurrency issue there.
1363 The only backend this seems to affect is postgres.
1364 """
1365 # commit the database
1366 self.sql_commit(fail_ok)
1368 # now, do all the other transaction stuff
1369 for method, args in self.transactions:
1370 method(*args)
1372 # save the indexer
1373 self.indexer.save_index()
1375 # clear out the transactions
1376 self.transactions = []
1378 # clear the cache: Don't carry over cached values from one
1379 # transaction to the next (there may be other changes from other
1380 # transactions)
1381 self.clearCache()
1383 def sql_rollback(self):
1384 self.conn.rollback()
1386 def rollback(self):
1387 """ Reverse all actions from the current transaction.
1389 Undo all the changes made since the database was opened or the last
1390 commit() or rollback() was performed.
1391 """
1392 logging.getLogger('roundup.hyperdb').info('rollback')
1394 self.sql_rollback()
1396 # roll back "other" transaction stuff
1397 for method, args in self.transactions:
1398 # delete temporary files
1399 if method == self.doStoreFile:
1400 self.rollbackStoreFile(*args)
1401 self.transactions = []
1403 # clear the cache
1404 self.clearCache()
1406 def sql_close(self):
1407 logging.getLogger('roundup.hyperdb').info('close')
1408 self.conn.close()
1410 def close(self):
1411 """ Close off the connection.
1412 """
1413 self.indexer.close()
1414 self.sql_close()
1416 #
1417 # The base Class class
1418 #
1419 class Class(hyperdb.Class):
1420 """ The handle to a particular class of nodes in a hyperdatabase.
1422 All methods except __repr__ and getnode must be implemented by a
1423 concrete backend Class.
1424 """
1426 def schema(self):
1427 """ A dumpable version of the schema that we can store in the
1428 database
1429 """
1430 return (self.key, [(x, repr(y)) for x,y in self.properties.iteritems()])
1432 def enableJournalling(self):
1433 """Turn journalling on for this class
1434 """
1435 self.do_journal = 1
1437 def disableJournalling(self):
1438 """Turn journalling off for this class
1439 """
1440 self.do_journal = 0
1442 # Editing nodes:
1443 def create(self, **propvalues):
1444 """ Create a new node of this class and return its id.
1446 The keyword arguments in 'propvalues' map property names to values.
1448 The values of arguments must be acceptable for the types of their
1449 corresponding properties or a TypeError is raised.
1451 If this class has a key property, it must be present and its value
1452 must not collide with other key strings or a ValueError is raised.
1454 Any other properties on this class that are missing from the
1455 'propvalues' dictionary are set to None.
1457 If an id in a link or multilink property does not refer to a valid
1458 node, an IndexError is raised.
1459 """
1460 self.fireAuditors('create', None, propvalues)
1461 newid = self.create_inner(**propvalues)
1462 self.fireReactors('create', newid, None)
1463 return newid
1465 def create_inner(self, **propvalues):
1466 """ Called by create, in-between the audit and react calls.
1467 """
1468 if 'id' in propvalues:
1469 raise KeyError('"id" is reserved')
1471 if self.db.journaltag is None:
1472 raise DatabaseError(_('Database open read-only'))
1474 if ('creator' in propvalues or 'actor' in propvalues or
1475 'creation' in propvalues or 'activity' in propvalues):
1476 raise KeyError('"creator", "actor", "creation" and '
1477 '"activity" are reserved')
1479 # new node's id
1480 newid = self.db.newid(self.classname)
1482 # validate propvalues
1483 num_re = re.compile('^\d+$')
1484 for key, value in propvalues.iteritems():
1485 if key == self.key:
1486 try:
1487 self.lookup(value)
1488 except KeyError:
1489 pass
1490 else:
1491 raise ValueError('node with key "%s" exists'%value)
1493 # try to handle this property
1494 try:
1495 prop = self.properties[key]
1496 except KeyError:
1497 raise KeyError('"%s" has no property "%s"'%(self.classname,
1498 key))
1500 if value is not None and isinstance(prop, Link):
1501 if type(value) != type(''):
1502 raise ValueError('link value must be String')
1503 link_class = self.properties[key].classname
1504 # if it isn't a number, it's a key
1505 if not num_re.match(value):
1506 try:
1507 value = self.db.classes[link_class].lookup(value)
1508 except (TypeError, KeyError):
1509 raise IndexError('new property "%s": %s not a %s'%(
1510 key, value, link_class))
1511 elif not self.db.getclass(link_class).hasnode(value):
1512 raise IndexError('%s has no node %s'%(link_class,
1513 value))
1515 # save off the value
1516 propvalues[key] = value
1518 # register the link with the newly linked node
1519 if self.do_journal and self.properties[key].do_journal:
1520 self.db.addjournal(link_class, value, 'link',
1521 (self.classname, newid, key))
1523 elif isinstance(prop, Multilink):
1524 if value is None:
1525 value = []
1526 if not hasattr(value, '__iter__'):
1527 raise TypeError('new property "%s" not an iterable of ids'%key)
1528 # clean up and validate the list of links
1529 link_class = self.properties[key].classname
1530 l = []
1531 for entry in value:
1532 if type(entry) != type(''):
1533 raise ValueError('"%s" multilink value (%r) '
1534 'must contain Strings'%(key, value))
1535 # if it isn't a number, it's a key
1536 if not num_re.match(entry):
1537 try:
1538 entry = self.db.classes[link_class].lookup(entry)
1539 except (TypeError, KeyError):
1540 raise IndexError('new property "%s": %s not a %s'%(
1541 key, entry, self.properties[key].classname))
1542 l.append(entry)
1543 value = l
1544 propvalues[key] = value
1546 # handle additions
1547 for nodeid in value:
1548 if not self.db.getclass(link_class).hasnode(nodeid):
1549 raise IndexError('%s has no node %s'%(link_class,
1550 nodeid))
1551 # register the link with the newly linked node
1552 if self.do_journal and self.properties[key].do_journal:
1553 self.db.addjournal(link_class, nodeid, 'link',
1554 (self.classname, newid, key))
1556 elif isinstance(prop, String):
1557 if type(value) != type('') and type(value) != type(u''):
1558 raise TypeError('new property "%s" not a string'%key)
1559 if prop.indexme:
1560 self.db.indexer.add_text((self.classname, newid, key),
1561 value)
1563 elif isinstance(prop, Password):
1564 if not isinstance(value, password.Password):
1565 raise TypeError('new property "%s" not a Password'%key)
1567 elif isinstance(prop, Date):
1568 if value is not None and not isinstance(value, date.Date):
1569 raise TypeError('new property "%s" not a Date'%key)
1571 elif isinstance(prop, Interval):
1572 if value is not None and not isinstance(value, date.Interval):
1573 raise TypeError('new property "%s" not an Interval'%key)
1575 elif value is not None and isinstance(prop, Number):
1576 try:
1577 float(value)
1578 except ValueError:
1579 raise TypeError('new property "%s" not numeric'%key)
1581 elif value is not None and isinstance(prop, Boolean):
1582 try:
1583 int(value)
1584 except ValueError:
1585 raise TypeError('new property "%s" not boolean'%key)
1587 # make sure there's data where there needs to be
1588 for key, prop in self.properties.iteritems():
1589 if key in propvalues:
1590 continue
1591 if key == self.key:
1592 raise ValueError('key property "%s" is required'%key)
1593 if isinstance(prop, Multilink):
1594 propvalues[key] = []
1595 else:
1596 propvalues[key] = None
1598 # done
1599 self.db.addnode(self.classname, newid, propvalues)
1600 if self.do_journal:
1601 self.db.addjournal(self.classname, newid, ''"create", {})
1603 # XXX numeric ids
1604 return str(newid)
1606 def get(self, nodeid, propname, default=_marker, cache=1):
1607 """Get the value of a property on an existing node of this class.
1609 'nodeid' must be the id of an existing node of this class or an
1610 IndexError is raised. 'propname' must be the name of a property
1611 of this class or a KeyError is raised.
1613 'cache' exists for backwards compatibility, and is not used.
1614 """
1615 if propname == 'id':
1616 return nodeid
1618 # get the node's dict
1619 d = self.db.getnode(self.classname, nodeid)
1620 # handle common case -- that property is in dict -- first
1621 # if None and one of creator/creation actor/activity return None
1622 if propname in d:
1623 r = d [propname]
1624 # return copy of our list
1625 if isinstance (r, list):
1626 return r[:]
1627 if r is not None:
1628 return r
1629 elif propname in ('creation', 'activity', 'creator', 'actor'):
1630 return r
1632 # propname not in d:
1633 if propname == 'creation' or propname == 'activity':
1634 return date.Date()
1635 if propname == 'creator' or propname == 'actor':
1636 return self.db.getuid()
1638 # get the property (raises KeyError if invalid)
1639 prop = self.properties[propname]
1641 # lazy evaluation of Multilink
1642 if propname not in d and isinstance(prop, Multilink):
1643 sql = 'select linkid from %s_%s where nodeid=%s'%(self.classname,
1644 propname, self.db.arg)
1645 self.db.sql(sql, (nodeid,))
1646 # extract the first column from the result
1647 # XXX numeric ids
1648 items = [int(x[0]) for x in self.db.cursor.fetchall()]
1649 items.sort ()
1650 d[propname] = [str(x) for x in items]
1652 # handle there being no value in the table for the property
1653 if propname not in d or d[propname] is None:
1654 if default is _marker:
1655 if isinstance(prop, Multilink):
1656 return []
1657 else:
1658 return None
1659 else:
1660 return default
1662 # don't pass our list to other code
1663 if isinstance(prop, Multilink):
1664 return d[propname][:]
1666 return d[propname]
1668 def set(self, nodeid, **propvalues):
1669 """Modify a property on an existing node of this class.
1671 'nodeid' must be the id of an existing node of this class or an
1672 IndexError is raised.
1674 Each key in 'propvalues' must be the name of a property of this
1675 class or a KeyError is raised.
1677 All values in 'propvalues' must be acceptable types for their
1678 corresponding properties or a TypeError is raised.
1680 If the value of the key property is set, it must not collide with
1681 other key strings or a ValueError is raised.
1683 If the value of a Link or Multilink property contains an invalid
1684 node id, a ValueError is raised.
1685 """
1686 self.fireAuditors('set', nodeid, propvalues)
1687 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1688 propvalues = self.set_inner(nodeid, **propvalues)
1689 self.fireReactors('set', nodeid, oldvalues)
1690 return propvalues
1692 def set_inner(self, nodeid, **propvalues):
1693 """ Called by set, in-between the audit and react calls.
1694 """
1695 if not propvalues:
1696 return propvalues
1698 if ('creator' in propvalues or 'actor' in propvalues or
1699 'creation' in propvalues or 'activity' in propvalues):
1700 raise KeyError('"creator", "actor", "creation" and '
1701 '"activity" are reserved')
1703 if 'id' in propvalues:
1704 raise KeyError('"id" is reserved')
1706 if self.db.journaltag is None:
1707 raise DatabaseError(_('Database open read-only'))
1709 node = self.db.getnode(self.classname, nodeid)
1710 if self.is_retired(nodeid):
1711 raise IndexError('Requested item is retired')
1712 num_re = re.compile('^\d+$')
1714 # make a copy of the values dictionary - we'll modify the contents
1715 propvalues = propvalues.copy()
1717 # if the journal value is to be different, store it in here
1718 journalvalues = {}
1720 # remember the add/remove stuff for multilinks, making it easier
1721 # for the Database layer to do its stuff
1722 multilink_changes = {}
1724 for propname, value in list(propvalues.items()):
1725 # check to make sure we're not duplicating an existing key
1726 if propname == self.key and node[propname] != value:
1727 try:
1728 self.lookup(value)
1729 except KeyError:
1730 pass
1731 else:
1732 raise ValueError('node with key "%s" exists'%value)
1734 # this will raise the KeyError if the property isn't valid
1735 # ... we don't use getprops() here because we only care about
1736 # the writeable properties.
1737 try:
1738 prop = self.properties[propname]
1739 except KeyError:
1740 raise KeyError('"%s" has no property named "%s"'%(
1741 self.classname, propname))
1743 # if the value's the same as the existing value, no sense in
1744 # doing anything
1745 current = node.get(propname, None)
1746 if value == current:
1747 del propvalues[propname]
1748 continue
1749 journalvalues[propname] = current
1751 # do stuff based on the prop type
1752 if isinstance(prop, Link):
1753 link_class = prop.classname
1754 # if it isn't a number, it's a key
1755 if value is not None and not isinstance(value, type('')):
1756 raise ValueError('property "%s" link value be a string'%(
1757 propname))
1758 if isinstance(value, type('')) and not num_re.match(value):
1759 try:
1760 value = self.db.classes[link_class].lookup(value)
1761 except (TypeError, KeyError):
1762 raise IndexError('new property "%s": %s not a %s'%(
1763 propname, value, prop.classname))
1765 if (value is not None and
1766 not self.db.getclass(link_class).hasnode(value)):
1767 raise IndexError('%s has no node %s'%(link_class,
1768 value))
1770 if self.do_journal and prop.do_journal:
1771 # register the unlink with the old linked node
1772 if node[propname] is not None:
1773 self.db.addjournal(link_class, node[propname],
1774 ''"unlink", (self.classname, nodeid, propname))
1776 # register the link with the newly linked node
1777 if value is not None:
1778 self.db.addjournal(link_class, value, ''"link",
1779 (self.classname, nodeid, propname))
1781 elif isinstance(prop, Multilink):
1782 if value is None:
1783 value = []
1784 if not hasattr(value, '__iter__'):
1785 raise TypeError('new property "%s" not an iterable of'
1786 ' ids'%propname)
1787 link_class = self.properties[propname].classname
1788 l = []
1789 for entry in value:
1790 # if it isn't a number, it's a key
1791 if type(entry) != type(''):
1792 raise ValueError('new property "%s" link value '
1793 'must be a string'%propname)
1794 if not num_re.match(entry):
1795 try:
1796 entry = self.db.classes[link_class].lookup(entry)
1797 except (TypeError, KeyError):
1798 raise IndexError('new property "%s": %s not a %s'%(
1799 propname, entry,
1800 self.properties[propname].classname))
1801 l.append(entry)
1802 value = l
1803 propvalues[propname] = value
1805 # figure the journal entry for this property
1806 add = []
1807 remove = []
1809 # handle removals
1810 if propname in node:
1811 l = node[propname]
1812 else:
1813 l = []
1814 for id in l[:]:
1815 if id in value:
1816 continue
1817 # register the unlink with the old linked node
1818 if self.do_journal and self.properties[propname].do_journal:
1819 self.db.addjournal(link_class, id, 'unlink',
1820 (self.classname, nodeid, propname))
1821 l.remove(id)
1822 remove.append(id)
1824 # handle additions
1825 for id in value:
1826 if id in l:
1827 continue
1828 # We can safely check this condition after
1829 # checking that this is an addition to the
1830 # multilink since the condition was checked for
1831 # existing entries at the point they were added to
1832 # the multilink. Since the hasnode call will
1833 # result in a SQL query, it is more efficient to
1834 # avoid the check if possible.
1835 if not self.db.getclass(link_class).hasnode(id):
1836 raise IndexError('%s has no node %s'%(link_class,
1837 id))
1838 # register the link with the newly linked node
1839 if self.do_journal and self.properties[propname].do_journal:
1840 self.db.addjournal(link_class, id, 'link',
1841 (self.classname, nodeid, propname))
1842 l.append(id)
1843 add.append(id)
1845 # figure the journal entry
1846 l = []
1847 if add:
1848 l.append(('+', add))
1849 if remove:
1850 l.append(('-', remove))
1851 multilink_changes[propname] = (add, remove)
1852 if l:
1853 journalvalues[propname] = tuple(l)
1855 elif isinstance(prop, String):
1856 if value is not None and type(value) != type('') and type(value) != type(u''):
1857 raise TypeError('new property "%s" not a string'%propname)
1858 if prop.indexme:
1859 if value is None: value = ''
1860 self.db.indexer.add_text((self.classname, nodeid, propname),
1861 value)
1863 elif isinstance(prop, Password):
1864 if not isinstance(value, password.Password):
1865 raise TypeError('new property "%s" not a Password'%propname)
1866 propvalues[propname] = value
1868 elif value is not None and isinstance(prop, Date):
1869 if not isinstance(value, date.Date):
1870 raise TypeError('new property "%s" not a Date'% propname)
1871 propvalues[propname] = value
1873 elif value is not None and isinstance(prop, Interval):
1874 if not isinstance(value, date.Interval):
1875 raise TypeError('new property "%s" not an '
1876 'Interval'%propname)
1877 propvalues[propname] = value
1879 elif value is not None and isinstance(prop, Number):
1880 try:
1881 float(value)
1882 except ValueError:
1883 raise TypeError('new property "%s" not numeric'%propname)
1885 elif value is not None and isinstance(prop, Boolean):
1886 try:
1887 int(value)
1888 except ValueError:
1889 raise TypeError('new property "%s" not boolean'%propname)
1891 # nothing to do?
1892 if not propvalues:
1893 return propvalues
1895 # update the activity time
1896 propvalues['activity'] = date.Date()
1897 propvalues['actor'] = self.db.getuid()
1899 # do the set
1900 self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
1902 # remove the activity props now they're handled
1903 del propvalues['activity']
1904 del propvalues['actor']
1906 # journal the set
1907 if self.do_journal:
1908 self.db.addjournal(self.classname, nodeid, ''"set", journalvalues)
1910 return propvalues
1912 def retire(self, nodeid):
1913 """Retire a node.
1915 The properties on the node remain available from the get() method,
1916 and the node's id is never reused.
1918 Retired nodes are not returned by the find(), list(), or lookup()
1919 methods, and other nodes may reuse the values of their key properties.
1920 """
1921 if self.db.journaltag is None:
1922 raise DatabaseError(_('Database open read-only'))
1924 self.fireAuditors('retire', nodeid, None)
1926 # use the arg for __retired__ to cope with any odd database type
1927 # conversion (hello, sqlite)
1928 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1929 self.db.arg, self.db.arg)
1930 self.db.sql(sql, (nodeid, nodeid))
1931 if self.do_journal:
1932 self.db.addjournal(self.classname, nodeid, ''"retired", None)
1934 self.fireReactors('retire', nodeid, None)
1936 def restore(self, nodeid):
1937 """Restore a retired node.
1939 Make node available for all operations like it was before retirement.
1940 """
1941 if self.db.journaltag is None:
1942 raise DatabaseError(_('Database open read-only'))
1944 node = self.db.getnode(self.classname, nodeid)
1945 # check if key property was overrided
1946 key = self.getkey()
1947 try:
1948 id = self.lookup(node[key])
1949 except KeyError:
1950 pass
1951 else:
1952 raise KeyError("Key property (%s) of retired node clashes "
1953 "with existing one (%s)" % (key, node[key]))
1955 self.fireAuditors('restore', nodeid, None)
1956 # use the arg for __retired__ to cope with any odd database type
1957 # conversion (hello, sqlite)
1958 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1959 self.db.arg, self.db.arg)
1960 self.db.sql(sql, (0, nodeid))
1961 if self.do_journal:
1962 self.db.addjournal(self.classname, nodeid, ''"restored", None)
1964 self.fireReactors('restore', nodeid, None)
1966 def is_retired(self, nodeid):
1967 """Return true if the node is rerired
1968 """
1969 sql = 'select __retired__ from _%s where id=%s'%(self.classname,
1970 self.db.arg)
1971 self.db.sql(sql, (nodeid,))
1972 return int(self.db.sql_fetchone()[0]) > 0
1974 def destroy(self, nodeid):
1975 """Destroy a node.
1977 WARNING: this method should never be used except in extremely rare
1978 situations where there could never be links to the node being
1979 deleted
1981 WARNING: use retire() instead
1983 WARNING: the properties of this node will not be available ever again
1985 WARNING: really, use retire() instead
1987 Well, I think that's enough warnings. This method exists mostly to
1988 support the session storage of the cgi interface.
1990 The node is completely removed from the hyperdb, including all journal
1991 entries. It will no longer be available, and will generally break code
1992 if there are any references to the node.
1993 """
1994 if self.db.journaltag is None:
1995 raise DatabaseError(_('Database open read-only'))
1996 self.db.destroynode(self.classname, nodeid)
1998 def history(self, nodeid):
1999 """Retrieve the journal of edits on a particular node.
2001 'nodeid' must be the id of an existing node of this class or an
2002 IndexError is raised.
2004 The returned list contains tuples of the form
2006 (nodeid, date, tag, action, params)
2008 'date' is a Timestamp object specifying the time of the change and
2009 'tag' is the journaltag specified when the database was opened.
2010 """
2011 if not self.do_journal:
2012 raise ValueError('Journalling is disabled for this class')
2013 return self.db.getjournal(self.classname, nodeid)
2015 # Locating nodes:
2016 def hasnode(self, nodeid):
2017 """Determine if the given nodeid actually exists
2018 """
2019 return self.db.hasnode(self.classname, nodeid)
2021 def setkey(self, propname):
2022 """Select a String property of this class to be the key property.
2024 'propname' must be the name of a String property of this class or
2025 None, or a TypeError is raised. The values of the key property on
2026 all existing nodes must be unique or a ValueError is raised.
2027 """
2028 prop = self.getprops()[propname]
2029 if not isinstance(prop, String):
2030 raise TypeError('key properties must be String')
2031 self.key = propname
2033 def getkey(self):
2034 """Return the name of the key property for this class or None."""
2035 return self.key
2037 def lookup(self, keyvalue):
2038 """Locate a particular node by its key property and return its id.
2040 If this class has no key property, a TypeError is raised. If the
2041 'keyvalue' matches one of the values for the key property among
2042 the nodes in this class, the matching node's id is returned;
2043 otherwise a KeyError is raised.
2044 """
2045 if not self.key:
2046 raise TypeError('No key property set for class %s'%self.classname)
2048 # use the arg to handle any odd database type conversion (hello,
2049 # sqlite)
2050 sql = "select id from _%s where _%s=%s and __retired__=%s"%(
2051 self.classname, self.key, self.db.arg, self.db.arg)
2052 self.db.sql(sql, (str(keyvalue), 0))
2054 # see if there was a result that's not retired
2055 row = self.db.sql_fetchone()
2056 if not row:
2057 raise KeyError('No key (%s) value "%s" for "%s"'%(self.key,
2058 keyvalue, self.classname))
2060 # return the id
2061 # XXX numeric ids
2062 return str(row[0])
2064 def find(self, **propspec):
2065 """Get the ids of nodes in this class which link to the given nodes.
2067 'propspec' consists of keyword args propname=nodeid or
2068 propname={nodeid:1, }
2069 'propname' must be the name of a property in this class, or a
2070 KeyError is raised. That property must be a Link or
2071 Multilink property, or a TypeError is raised.
2073 Any node in this class whose 'propname' property links to any of
2074 the nodeids will be returned. Examples::
2076 db.issue.find(messages='1')
2077 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
2078 """
2079 # shortcut
2080 if not propspec:
2081 return []
2083 # validate the args
2084 props = self.getprops()
2085 for propname, nodeids in propspec.iteritems():
2086 # check the prop is OK
2087 prop = props[propname]
2088 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
2089 raise TypeError("'%s' not a Link/Multilink property"%propname)
2091 # first, links
2092 a = self.db.arg
2093 allvalues = ()
2094 sql = []
2095 where = []
2096 for prop, values in propspec.iteritems():
2097 if not isinstance(props[prop], hyperdb.Link):
2098 continue
2099 if type(values) is type({}) and len(values) == 1:
2100 values = list(values)[0]
2101 if type(values) is type(''):
2102 allvalues += (values,)
2103 where.append('_%s = %s'%(prop, a))
2104 elif values is None:
2105 where.append('_%s is NULL'%prop)
2106 else:
2107 values = list(values)
2108 s = ''
2109 if None in values:
2110 values.remove(None)
2111 s = '_%s is NULL or '%prop
2112 allvalues += tuple(values)
2113 s += '_%s in (%s)'%(prop, ','.join([a]*len(values)))
2114 where.append('(' + s +')')
2115 if where:
2116 allvalues = (0, ) + allvalues
2117 sql.append("""select id from _%s where __retired__=%s
2118 and %s"""%(self.classname, a, ' and '.join(where)))
2120 # now multilinks
2121 for prop, values in propspec.iteritems():
2122 if not isinstance(props[prop], hyperdb.Multilink):
2123 continue
2124 if not values:
2125 continue
2126 allvalues += (0, )
2127 if type(values) is type(''):
2128 allvalues += (values,)
2129 s = a
2130 else:
2131 allvalues += tuple(values)
2132 s = ','.join([a]*len(values))
2133 tn = '%s_%s'%(self.classname, prop)
2134 sql.append("""select id from _%s, %s where __retired__=%s
2135 and id = %s.nodeid and %s.linkid in (%s)"""%(self.classname,
2136 tn, a, tn, tn, s))
2138 if not sql:
2139 return []
2140 sql = ' union '.join(sql)
2141 self.db.sql(sql, allvalues)
2142 # XXX numeric ids
2143 l = [str(x[0]) for x in self.db.sql_fetchall()]
2144 return l
2146 def stringFind(self, **requirements):
2147 """Locate a particular node by matching a set of its String
2148 properties in a caseless search.
2150 If the property is not a String property, a TypeError is raised.
2152 The return is a list of the id of all nodes that match.
2153 """
2154 where = []
2155 args = []
2156 for propname in requirements:
2157 prop = self.properties[propname]
2158 if not isinstance(prop, String):
2159 raise TypeError("'%s' not a String property"%propname)
2160 where.append(propname)
2161 args.append(requirements[propname].lower())
2163 # generate the where clause
2164 s = ' and '.join(['lower(_%s)=%s'%(col, self.db.arg) for col in where])
2165 sql = 'select id from _%s where %s and __retired__=%s'%(
2166 self.classname, s, self.db.arg)
2167 args.append(0)
2168 self.db.sql(sql, tuple(args))
2169 # XXX numeric ids
2170 l = [str(x[0]) for x in self.db.sql_fetchall()]
2171 return l
2173 def list(self):
2174 """ Return a list of the ids of the active nodes in this class.
2175 """
2176 return self.getnodeids(retired=0)
2178 def getnodeids(self, retired=None):
2179 """ Retrieve all the ids of the nodes for a particular Class.
2181 Set retired=None to get all nodes. Otherwise it'll get all the
2182 retired or non-retired nodes, depending on the flag.
2183 """
2184 # flip the sense of the 'retired' flag if we don't want all of them
2185 if retired is not None:
2186 args = (0, )
2187 if retired:
2188 compare = '>'
2189 else:
2190 compare = '='
2191 sql = 'select id from _%s where __retired__%s%s'%(self.classname,
2192 compare, self.db.arg)
2193 else:
2194 args = ()
2195 sql = 'select id from _%s'%self.classname
2196 self.db.sql(sql, args)
2197 # XXX numeric ids
2198 ids = [str(x[0]) for x in self.db.cursor.fetchall()]
2199 return ids
2201 def _subselect(self, classname, multilink_table):
2202 """Create a subselect. This is factored out because some
2203 databases (hmm only one, so far) doesn't support subselects
2204 look for "I can't believe it's not a toy RDBMS" in the mysql
2205 backend.
2206 """
2207 return '_%s.id not in (select nodeid from %s)'%(classname,
2208 multilink_table)
2210 # Some DBs order NULL values last. Set this variable in the backend
2211 # for prepending an order by clause for each attribute that causes
2212 # correct sort order for NULLs. Examples:
2213 # order_by_null_values = '(%s is not NULL)'
2214 # order_by_null_values = 'notnull(%s)'
2215 # The format parameter is replaced with the attribute.
2216 order_by_null_values = None
2218 def supports_subselects(self):
2219 '''Assuming DBs can do subselects, overwrite if they cannot.
2220 '''
2221 return True
2223 def _filter_multilink_expression_fallback(
2224 self, classname, multilink_table, expr):
2225 '''This is a fallback for database that do not support
2226 subselects.'''
2228 is_valid = expr.evaluate
2230 last_id, kws = None, []
2232 ids = IdListOptimizer()
2233 append = ids.append
2235 # This join and the evaluation in program space
2236 # can be expensive for larger databases!
2237 # TODO: Find a faster way to collect the data needed
2238 # to evalute the expression.
2239 # Moving the expression evaluation into the database
2240 # would be nice but this tricky: Think about the cases
2241 # where the multilink table does not have join values
2242 # needed in evaluation.
2244 stmnt = "SELECT c.id, m.linkid FROM _%s c " \
2245 "LEFT OUTER JOIN %s m " \
2246 "ON c.id = m.nodeid ORDER BY c.id" % (
2247 classname, multilink_table)
2248 self.db.sql(stmnt)
2250 # collect all multilink items for a class item
2251 for nid, kw in self.db.sql_fetchiter():
2252 if nid != last_id:
2253 if last_id is None:
2254 last_id = nid
2255 else:
2256 # we have all multilink items -> evaluate!
2257 if is_valid(kws): append(last_id)
2258 last_id, kws = nid, []
2259 if kw is not None:
2260 kws.append(kw)
2262 if last_id is not None and is_valid(kws):
2263 append(last_id)
2265 # we have ids of the classname table
2266 return ids.where("_%s.id" % classname, self.db.arg)
2268 def _filter_multilink_expression(self, classname, multilink_table, v):
2269 """ Filters out elements of the classname table that do not
2270 match the given expression.
2271 Returns tuple of 'WHERE' introns for the overall filter.
2272 """
2273 try:
2274 opcodes = [int(x) for x in v]
2275 if min(opcodes) >= -1: raise ValueError()
2277 expr = compile_expression(opcodes)
2279 if not self.supports_subselects():
2280 # We heavily rely on subselects. If there is
2281 # no decent support fall back to slower variant.
2282 return self._filter_multilink_expression_fallback(
2283 classname, multilink_table, expr)
2285 atom = \
2286 "%s IN(SELECT linkid FROM %s WHERE nodeid=a.id)" % (
2287 self.db.arg,
2288 multilink_table)
2290 intron = \
2291 "_%(classname)s.id in (SELECT id " \
2292 "FROM _%(classname)s AS a WHERE %(condition)s) " % {
2293 'classname' : classname,
2294 'condition' : expr.generate(lambda n: atom) }
2296 values = []
2297 def collect_values(n): values.append(n.x)
2298 expr.visit(collect_values)
2300 return intron, values
2301 except:
2302 # original behavior
2303 where = "%s.linkid in (%s)" % (
2304 multilink_table, ','.join([self.db.arg] * len(v)))
2305 return where, v, True # True to indicate original
2307 def _filter_sql (self, search_matches, filterspec, srt=[], grp=[], retr=0):
2308 """ Compute the proptree and the SQL/ARGS for a filter.
2309 For argument description see filter below.
2310 We return a 3-tuple, the proptree, the sql and the sql-args
2311 or None if no SQL is necessary.
2312 The flag retr serves to retrieve *all* non-Multilink properties
2313 (for filling the cache during a filter_iter)
2314 """
2315 # we can't match anything if search_matches is empty
2316 if not search_matches and search_matches is not None:
2317 return None
2319 icn = self.classname
2321 # vars to hold the components of the SQL statement
2322 frum = [] # FROM clauses
2323 loj = [] # LEFT OUTER JOIN clauses
2324 where = [] # WHERE clauses
2325 args = [] # *any* positional arguments
2326 a = self.db.arg
2328 # figure the WHERE clause from the filterspec
2329 mlfilt = 0 # are we joining with Multilink tables?
2330 sortattr = self._sortattr (group = grp, sort = srt)
2331 proptree = self._proptree(filterspec, sortattr, retr)
2332 mlseen = 0
2333 for pt in reversed(proptree.sortattr):
2334 p = pt
2335 while p.parent:
2336 if isinstance (p.propclass, Multilink):
2337 mlseen = True
2338 if mlseen:
2339 p.sort_ids_needed = True
2340 p.tree_sort_done = False
2341 p = p.parent
2342 if not mlseen:
2343 pt.attr_sort_done = pt.tree_sort_done = True
2344 proptree.compute_sort_done()
2346 cols = ['_%s.id'%icn]
2347 mlsort = []
2348 rhsnum = 0
2349 for p in proptree:
2350 rc = ac = oc = None
2351 cn = p.classname
2352 ln = p.uniqname
2353 pln = p.parent.uniqname
2354 pcn = p.parent.classname
2355 k = p.name
2356 v = p.val
2357 propclass = p.propclass
2358 if p.parent == proptree and p.name == 'id' \
2359 and 'retrieve' in p.need_for:
2360 p.sql_idx = 0
2361 if 'sort' in p.need_for or 'retrieve' in p.need_for:
2362 rc = oc = ac = '_%s._%s'%(pln, k)
2363 if isinstance(propclass, Multilink):
2364 if 'search' in p.need_for:
2365 mlfilt = 1
2366 tn = '%s_%s'%(pcn, k)
2367 if v in ('-1', ['-1'], []):
2368 # only match rows that have count(linkid)=0 in the
2369 # corresponding multilink table)
2370 where.append(self._subselect(pcn, tn))
2371 else:
2372 frum.append(tn)
2373 gen_join = True
2375 if p.has_values and isinstance(v, type([])):
2376 result = self._filter_multilink_expression(pln, tn, v)
2377 # XXX: We dont need an id join if we used the filter
2378 gen_join = len(result) == 3
2380 if gen_join:
2381 where.append('_%s.id=%s.nodeid'%(pln,tn))
2383 if p.children:
2384 frum.append('_%s as _%s' % (cn, ln))
2385 where.append('%s.linkid=_%s.id'%(tn, ln))
2387 if p.has_values:
2388 if isinstance(v, type([])):
2389 where.append(result[0])
2390 args += result[1]
2391 else:
2392 where.append('%s.linkid=%s'%(tn, a))
2393 args.append(v)
2394 if 'sort' in p.need_for:
2395 assert not p.attr_sort_done and not p.sort_ids_needed
2396 elif k == 'id':
2397 if 'search' in p.need_for:
2398 if isinstance(v, type([])):
2399 # If there are no permitted values, then the
2400 # where clause will always be false, and we
2401 # can optimize the query away.
2402 if not v:
2403 return []
2404 s = ','.join([a for x in v])
2405 where.append('_%s.%s in (%s)'%(pln, k, s))
2406 args = args + v
2407 else:
2408 where.append('_%s.%s=%s'%(pln, k, a))
2409 args.append(v)
2410 if 'sort' in p.need_for or 'retrieve' in p.need_for:
2411 rc = oc = ac = '_%s.id'%pln
2412 elif isinstance(propclass, String):
2413 if 'search' in p.need_for:
2414 if not isinstance(v, type([])):
2415 v = [v]
2417 # Quote the bits in the string that need it and then embed
2418 # in a "substring" search. Note - need to quote the '%' so
2419 # they make it through the python layer happily
2420 v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
2422 # now add to the where clause
2423 where.append('('
2424 +' and '.join(["_%s._%s LIKE '%s'"%(pln, k, s) for s in v])
2425 +')')
2426 # note: args are embedded in the query string now
2427 if 'sort' in p.need_for:
2428 oc = ac = 'lower(_%s._%s)'%(pln, k)
2429 elif isinstance(propclass, Link):
2430 if 'search' in p.need_for:
2431 if p.children:
2432 if 'sort' not in p.need_for:
2433 frum.append('_%s as _%s' % (cn, ln))
2434 where.append('_%s._%s=_%s.id'%(pln, k, ln))
2435 if p.has_values:
2436 if isinstance(v, type([])):
2437 d = {}
2438 for entry in v:
2439 if entry == '-1':
2440 entry = None
2441 d[entry] = entry
2442 l = []
2443 if None in d or not d:
2444 if None in d: del d[None]
2445 l.append('_%s._%s is NULL'%(pln, k))
2446 if d:
2447 v = list(d)
2448 s = ','.join([a for x in v])
2449 l.append('(_%s._%s in (%s))'%(pln, k, s))
2450 args = args + v
2451 if l:
2452 where.append('(' + ' or '.join(l) +')')
2453 else:
2454 if v in ('-1', None):
2455 v = None
2456 where.append('_%s._%s is NULL'%(pln, k))
2457 else:
2458 where.append('_%s._%s=%s'%(pln, k, a))
2459 args.append(v)
2460 if 'sort' in p.need_for:
2461 lp = p.cls.labelprop()
2462 oc = ac = '_%s._%s'%(pln, k)
2463 if lp != 'id':
2464 if p.tree_sort_done:
2465 loj.append(
2466 'LEFT OUTER JOIN _%s as _%s on _%s._%s=_%s.id'%(
2467 cn, ln, pln, k, ln))
2468 oc = '_%s._%s'%(ln, lp)
2469 if 'retrieve' in p.need_for:
2470 rc = '_%s._%s'%(pln, k)
2471 elif isinstance(propclass, Date) and 'search' in p.need_for:
2472 dc = self.db.to_sql_value(hyperdb.Date)
2473 if isinstance(v, type([])):
2474 s = ','.join([a for x in v])
2475 where.append('_%s._%s in (%s)'%(pln, k, s))
2476 args = args + [dc(date.Date(x)) for x in v]
2477 else:
2478 try:
2479 # Try to filter on range of dates
2480 date_rng = propclass.range_from_raw(v, self.db)
2481 if date_rng.from_value:
2482 where.append('_%s._%s >= %s'%(pln, k, a))
2483 args.append(dc(date_rng.from_value))
2484 if date_rng.to_value:
2485 where.append('_%s._%s <= %s'%(pln, k, a))
2486 args.append(dc(date_rng.to_value))
2487 except ValueError:
2488 # If range creation fails - ignore that search parameter
2489 pass
2490 elif isinstance(propclass, Interval):
2491 # filter/sort using the __<prop>_int__ column
2492 if 'search' in p.need_for:
2493 if isinstance(v, type([])):
2494 s = ','.join([a for x in v])
2495 where.append('_%s.__%s_int__ in (%s)'%(pln, k, s))
2496 args = args + [date.Interval(x).as_seconds() for x in v]
2497 else:
2498 try:
2499 # Try to filter on range of intervals
2500 date_rng = Range(v, date.Interval)
2501 if date_rng.from_value:
2502 where.append('_%s.__%s_int__ >= %s'%(pln, k, a))
2503 args.append(date_rng.from_value.as_seconds())
2504 if date_rng.to_value:
2505 where.append('_%s.__%s_int__ <= %s'%(pln, k, a))
2506 args.append(date_rng.to_value.as_seconds())
2507 except ValueError:
2508 # If range creation fails - ignore search parameter
2509 pass
2510 if 'sort' in p.need_for:
2511 oc = ac = '_%s.__%s_int__'%(pln,k)
2512 if 'retrieve' in p.need_for:
2513 rc = '_%s._%s'%(pln,k)
2514 elif isinstance(propclass, Boolean) and 'search' in p.need_for:
2515 if type(v) == type(""):
2516 v = v.split(',')
2517 if type(v) != type([]):
2518 v = [v]
2519 bv = []
2520 for val in v:
2521 if type(val) is type(''):
2522 bv.append(propclass.from_raw (val))
2523 else:
2524 bv.append(bool(val))
2525 if len(bv) == 1:
2526 where.append('_%s._%s=%s'%(pln, k, a))
2527 args = args + bv
2528 else:
2529 s = ','.join([a for x in v])
2530 where.append('_%s._%s in (%s)'%(pln, k, s))
2531 args = args + bv
2532 elif 'search' in p.need_for:
2533 if isinstance(v, type([])):
2534 s = ','.join([a for x in v])
2535 where.append('_%s._%s in (%s)'%(pln, k, s))
2536 args = args + v
2537 else:
2538 where.append('_%s._%s=%s'%(pln, k, a))
2539 args.append(v)
2540 if oc:
2541 if p.sort_ids_needed:
2542 if rc == ac:
2543 p.sql_idx = len(cols)
2544 p.auxcol = len(cols)
2545 cols.append(ac)
2546 if p.tree_sort_done and p.sort_direction:
2547 # Don't select top-level id or multilink twice
2548 if (not p.sort_ids_needed or ac != oc) and (p.name != 'id'
2549 or p.parent != proptree):
2550 if rc == oc:
2551 p.sql_idx = len(cols)
2552 cols.append(oc)
2553 desc = ['', ' desc'][p.sort_direction == '-']
2554 # Some SQL dbs sort NULL values last -- we want them first.
2555 if (self.order_by_null_values and p.name != 'id'):
2556 nv = self.order_by_null_values % oc
2557 cols.append(nv)
2558 p.orderby.append(nv + desc)
2559 p.orderby.append(oc + desc)
2560 if 'retrieve' in p.need_for and p.sql_idx is None:
2561 assert(rc)
2562 p.sql_idx = len(cols)
2563 cols.append (rc)
2565 props = self.getprops()
2567 # don't match retired nodes
2568 where.append('_%s.__retired__=0'%icn)
2570 # add results of full text search
2571 if search_matches is not None:
2572 s = ','.join([a for x in search_matches])
2573 where.append('_%s.id in (%s)'%(icn, s))
2574 args = args + [x for x in search_matches]
2576 # construct the SQL
2577 frum.append('_'+icn)
2578 frum = ','.join(frum)
2579 if where:
2580 where = ' where ' + (' and '.join(where))
2581 else:
2582 where = ''
2583 if mlfilt:
2584 # we're joining tables on the id, so we will get dupes if we
2585 # don't distinct()
2586 cols[0] = 'distinct(_%s.id)'%icn
2588 order = []
2589 # keep correct sequence of order attributes.
2590 for sa in proptree.sortattr:
2591 if not sa.attr_sort_done:
2592 continue
2593 order.extend(sa.orderby)
2594 if order:
2595 order = ' order by %s'%(','.join(order))
2596 else:
2597 order = ''
2599 cols = ','.join(cols)
2600 loj = ' '.join(loj)
2601 sql = 'select %s from %s %s %s%s'%(cols, frum, loj, where, order)
2602 args = tuple(args)
2603 __traceback_info__ = (sql, args)
2604 return proptree, sql, args
2606 def filter(self, search_matches, filterspec, sort=[], group=[]):
2607 """Return a list of the ids of the active nodes in this class that
2608 match the 'filter' spec, sorted by the group spec and then the
2609 sort spec
2611 "filterspec" is {propname: value(s)}
2613 "sort" and "group" are [(dir, prop), ...] where dir is '+', '-'
2614 or None and prop is a prop name or None. Note that for
2615 backward-compatibility reasons a single (dir, prop) tuple is
2616 also allowed.
2618 "search_matches" is a container type or None
2620 The filter must match all properties specificed. If the property
2621 value to match is a list:
2623 1. String properties must match all elements in the list, and
2624 2. Other properties must match any of the elements in the list.
2625 """
2626 if __debug__:
2627 start_t = time.time()
2629 sq = self._filter_sql (search_matches, filterspec, sort, group)
2630 # nothing to match?
2631 if sq is None:
2632 return []
2633 proptree, sql, args = sq
2635 self.db.sql(sql, args)
2636 l = self.db.sql_fetchall()
2638 # Compute values needed for sorting in proptree.sort
2639 for p in proptree:
2640 if hasattr(p, 'auxcol'):
2641 p.sort_ids = p.sort_result = [row[p.auxcol] for row in l]
2642 # return the IDs (the first column)
2643 # XXX numeric ids
2644 l = [str(row[0]) for row in l]
2645 l = proptree.sort (l)
2647 if __debug__:
2648 self.db.stats['filtering'] += (time.time() - start_t)
2649 return l
2651 def filter_iter(self, search_matches, filterspec, sort=[], group=[]):
2652 """Iterator similar to filter above with same args.
2653 Limitation: We don't sort on multilinks.
2654 This uses an optimisation: We put all nodes that are in the
2655 current row into the node cache. Then we return the node id.
2656 That way a fetch of a node won't create another sql-fetch (with
2657 a join) from the database because the nodes are already in the
2658 cache. We're using our own temporary cursor.
2659 """
2660 sq = self._filter_sql(search_matches, filterspec, sort, group, retr=1)
2661 # nothing to match?
2662 if sq is None:
2663 return
2664 proptree, sql, args = sq
2665 cursor = self.db.conn.cursor()
2666 self.db.sql(sql, args, cursor)
2667 classes = {}
2668 for p in proptree:
2669 if 'retrieve' in p.need_for:
2670 cn = p.parent.classname
2671 ptid = p.parent.id # not the nodeid!
2672 key = (cn, ptid)
2673 if key not in classes:
2674 classes[key] = {}
2675 name = p.name
2676 assert (name)
2677 classes[key][name] = p
2678 p.to_hyperdb = self.db.to_hyperdb_value(p.propclass.__class__)
2679 while True:
2680 row = cursor.fetchone()
2681 if not row: break
2682 # populate cache with current items
2683 for (classname, ptid), pt in classes.iteritems():
2684 nodeid = str(row[pt['id'].sql_idx])
2685 key = (classname, nodeid)
2686 if key in self.db.cache:
2687 self.db._cache_refresh(key)
2688 continue
2689 node = {}
2690 for propname, p in pt.iteritems():
2691 value = row[p.sql_idx]
2692 if value is not None:
2693 value = p.to_hyperdb(value)
2694 node[propname] = value
2695 self.db._cache_save(key, node)
2696 yield str(row[0])
2698 def filter_sql(self, sql):
2699 """Return a list of the ids of the items in this class that match
2700 the SQL provided. The SQL is a complete "select" statement.
2702 The SQL select must include the item id as the first column.
2704 This function DOES NOT filter out retired items, add on a where
2705 clause "__retired__=0" if you don't want retired nodes.
2706 """
2707 if __debug__:
2708 start_t = time.time()
2710 self.db.sql(sql)
2711 l = self.db.sql_fetchall()
2713 if __debug__:
2714 self.db.stats['filtering'] += (time.time() - start_t)
2715 return l
2717 def count(self):
2718 """Get the number of nodes in this class.
2720 If the returned integer is 'numnodes', the ids of all the nodes
2721 in this class run from 1 to numnodes, and numnodes+1 will be the
2722 id of the next node to be created in this class.
2723 """
2724 return self.db.countnodes(self.classname)
2726 # Manipulating properties:
2727 def getprops(self, protected=1):
2728 """Return a dictionary mapping property names to property objects.
2729 If the "protected" flag is true, we include protected properties -
2730 those which may not be modified.
2731 """
2732 d = self.properties.copy()
2733 if protected:
2734 d['id'] = String()
2735 d['creation'] = hyperdb.Date()
2736 d['activity'] = hyperdb.Date()
2737 d['creator'] = hyperdb.Link('user')
2738 d['actor'] = hyperdb.Link('user')
2739 return d
2741 def addprop(self, **properties):
2742 """Add properties to this class.
2744 The keyword arguments in 'properties' must map names to property
2745 objects, or a TypeError is raised. None of the keys in 'properties'
2746 may collide with the names of existing properties, or a ValueError
2747 is raised before any properties have been added.
2748 """
2749 for key in properties:
2750 if key in self.properties:
2751 raise ValueError(key)
2752 self.properties.update(properties)
2754 def index(self, nodeid):
2755 """Add (or refresh) the node to search indexes
2756 """
2757 # find all the String properties that have indexme
2758 for prop, propclass in self.getprops().iteritems():
2759 if isinstance(propclass, String) and propclass.indexme:
2760 self.db.indexer.add_text((self.classname, nodeid, prop),
2761 str(self.get(nodeid, prop)))
2763 #
2764 # import / export support
2765 #
2766 def export_list(self, propnames, nodeid):
2767 """ Export a node - generate a list of CSV-able data in the order
2768 specified by propnames for the given node.
2769 """
2770 properties = self.getprops()
2771 l = []
2772 for prop in propnames:
2773 proptype = properties[prop]
2774 value = self.get(nodeid, prop)
2775 # "marshal" data where needed
2776 if value is None:
2777 pass
2778 elif isinstance(proptype, hyperdb.Date):
2779 value = value.get_tuple()
2780 elif isinstance(proptype, hyperdb.Interval):
2781 value = value.get_tuple()
2782 elif isinstance(proptype, hyperdb.Password):
2783 value = str(value)
2784 l.append(repr(value))
2785 l.append(repr(self.is_retired(nodeid)))
2786 return l
2788 def import_list(self, propnames, proplist):
2789 """ Import a node - all information including "id" is present and
2790 should not be sanity checked. Triggers are not triggered. The
2791 journal should be initialised using the "creator" and "created"
2792 information.
2794 Return the nodeid of the node imported.
2795 """
2796 if self.db.journaltag is None:
2797 raise DatabaseError(_('Database open read-only'))
2798 properties = self.getprops()
2800 # make the new node's property map
2801 d = {}
2802 retire = 0
2803 if not "id" in propnames:
2804 newid = self.db.newid(self.classname)
2805 else:
2806 newid = eval(proplist[propnames.index("id")])
2807 for i in range(len(propnames)):
2808 # Use eval to reverse the repr() used to output the CSV
2809 value = eval(proplist[i])
2811 # Figure the property for this column
2812 propname = propnames[i]
2814 # "unmarshal" where necessary
2815 if propname == 'id':
2816 continue
2817 elif propname == 'is retired':
2818 # is the item retired?
2819 if int(value):
2820 retire = 1
2821 continue
2822 elif value is None:
2823 d[propname] = None
2824 continue
2826 prop = properties[propname]
2827 if value is None:
2828 # don't set Nones
2829 continue
2830 elif isinstance(prop, hyperdb.Date):
2831 value = date.Date(value)
2832 elif isinstance(prop, hyperdb.Interval):
2833 value = date.Interval(value)
2834 elif isinstance(prop, hyperdb.Password):
2835 value = password.Password(encrypted=value)
2836 elif isinstance(prop, String):
2837 if isinstance(value, unicode):
2838 value = value.encode('utf8')
2839 if not isinstance(value, str):
2840 raise TypeError('new property "%(propname)s" not a '
2841 'string: %(value)r'%locals())
2842 if prop.indexme:
2843 self.db.indexer.add_text((self.classname, newid, propname),
2844 value)
2845 d[propname] = value
2847 # get a new id if necessary
2848 if newid is None:
2849 newid = self.db.newid(self.classname)
2851 # insert new node or update existing?
2852 if not self.hasnode(newid):
2853 self.db.addnode(self.classname, newid, d) # insert
2854 else:
2855 self.db.setnode(self.classname, newid, d) # update
2857 # retire?
2858 if retire:
2859 # use the arg for __retired__ to cope with any odd database type
2860 # conversion (hello, sqlite)
2861 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
2862 self.db.arg, self.db.arg)
2863 self.db.sql(sql, (newid, newid))
2864 return newid
2866 def export_journals(self):
2867 """Export a class's journal - generate a list of lists of
2868 CSV-able data:
2870 nodeid, date, user, action, params
2872 No heading here - the columns are fixed.
2873 """
2874 properties = self.getprops()
2875 r = []
2876 for nodeid in self.getnodeids():
2877 for nodeid, date, user, action, params in self.history(nodeid):
2878 date = date.get_tuple()
2879 if action == 'set':
2880 export_data = {}
2881 for propname, value in params.iteritems():
2882 if propname not in properties:
2883 # property no longer in the schema
2884 continue
2886 prop = properties[propname]
2887 # make sure the params are eval()'able
2888 if value is None:
2889 pass
2890 elif isinstance(prop, Date):
2891 value = value.get_tuple()
2892 elif isinstance(prop, Interval):
2893 value = value.get_tuple()
2894 elif isinstance(prop, Password):
2895 value = str(value)
2896 export_data[propname] = value
2897 params = export_data
2898 elif action == 'create' and params:
2899 # old tracker with data stored in the create!
2900 params = {}
2901 l = [nodeid, date, user, action, params]
2902 r.append(list(map(repr, l)))
2903 return r
2905 class FileClass(hyperdb.FileClass, Class):
2906 """This class defines a large chunk of data. To support this, it has a
2907 mandatory String property "content" which is typically saved off
2908 externally to the hyperdb.
2910 The default MIME type of this data is defined by the
2911 "default_mime_type" class attribute, which may be overridden by each
2912 node if the class defines a "type" String property.
2913 """
2914 def __init__(self, db, classname, **properties):
2915 """The newly-created class automatically includes the "content"
2916 and "type" properties.
2917 """
2918 if 'content' not in properties:
2919 properties['content'] = hyperdb.String(indexme='yes')
2920 if 'type' not in properties:
2921 properties['type'] = hyperdb.String()
2922 Class.__init__(self, db, classname, **properties)
2924 def create(self, **propvalues):
2925 """ snaffle the file propvalue and store in a file
2926 """
2927 # we need to fire the auditors now, or the content property won't
2928 # be in propvalues for the auditors to play with
2929 self.fireAuditors('create', None, propvalues)
2931 # now remove the content property so it's not stored in the db
2932 content = propvalues['content']
2933 del propvalues['content']
2935 # do the database create
2936 newid = self.create_inner(**propvalues)
2938 # figure the mime type
2939 mime_type = propvalues.get('type', self.default_mime_type)
2941 # and index!
2942 if self.properties['content'].indexme:
2943 self.db.indexer.add_text((self.classname, newid, 'content'),
2944 content, mime_type)
2946 # store off the content as a file
2947 self.db.storefile(self.classname, newid, None, content)
2949 # fire reactors
2950 self.fireReactors('create', newid, None)
2952 return newid
2954 def get(self, nodeid, propname, default=_marker, cache=1):
2955 """ Trap the content propname and get it from the file
2957 'cache' exists for backwards compatibility, and is not used.
2958 """
2959 poss_msg = 'Possibly a access right configuration problem.'
2960 if propname == 'content':
2961 try:
2962 return self.db.getfile(self.classname, nodeid, None)
2963 except IOError, strerror:
2964 # BUG: by catching this we donot see an error in the log.
2965 return 'ERROR reading file: %s%s\n%s\n%s'%(
2966 self.classname, nodeid, poss_msg, strerror)
2967 if default is not _marker:
2968 return Class.get(self, nodeid, propname, default)
2969 else:
2970 return Class.get(self, nodeid, propname)
2972 def set(self, itemid, **propvalues):
2973 """ Snarf the "content" propvalue and update it in a file
2974 """
2975 self.fireAuditors('set', itemid, propvalues)
2976 oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid))
2978 # now remove the content property so it's not stored in the db
2979 content = None
2980 if 'content' in propvalues:
2981 content = propvalues['content']
2982 del propvalues['content']
2984 # do the database create
2985 propvalues = self.set_inner(itemid, **propvalues)
2987 # do content?
2988 if content:
2989 # store and possibly index
2990 self.db.storefile(self.classname, itemid, None, content)
2991 if self.properties['content'].indexme:
2992 mime_type = self.get(itemid, 'type', self.default_mime_type)
2993 self.db.indexer.add_text((self.classname, itemid, 'content'),
2994 content, mime_type)
2995 propvalues['content'] = content
2997 # fire reactors
2998 self.fireReactors('set', itemid, oldvalues)
2999 return propvalues
3001 def index(self, nodeid):
3002 """ Add (or refresh) the node to search indexes.
3004 Use the content-type property for the content property.
3005 """
3006 # find all the String properties that have indexme
3007 for prop, propclass in self.getprops().iteritems():
3008 if prop == 'content' and propclass.indexme:
3009 mime_type = self.get(nodeid, 'type', self.default_mime_type)
3010 self.db.indexer.add_text((self.classname, nodeid, 'content'),
3011 str(self.get(nodeid, 'content')), mime_type)
3012 elif isinstance(propclass, hyperdb.String) and propclass.indexme:
3013 # index them under (classname, nodeid, property)
3014 try:
3015 value = str(self.get(nodeid, prop))
3016 except IndexError:
3017 # node has been destroyed
3018 continue
3019 self.db.indexer.add_text((self.classname, nodeid, prop), value)
3021 # XXX deviation from spec - was called ItemClass
3022 class IssueClass(Class, roundupdb.IssueClass):
3023 # Overridden methods:
3024 def __init__(self, db, classname, **properties):
3025 """The newly-created class automatically includes the "messages",
3026 "files", "nosy", and "superseder" properties. If the 'properties'
3027 dictionary attempts to specify any of these properties or a
3028 "creation", "creator", "activity" or "actor" property, a ValueError
3029 is raised.
3030 """
3031 if 'title' not in properties:
3032 properties['title'] = hyperdb.String(indexme='yes')
3033 if 'messages' not in properties:
3034 properties['messages'] = hyperdb.Multilink("msg")
3035 if 'files' not in properties:
3036 properties['files'] = hyperdb.Multilink("file")
3037 if 'nosy' not in properties:
3038 # note: journalling is turned off as it really just wastes
3039 # space. this behaviour may be overridden in an instance
3040 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
3041 if 'superseder' not in properties:
3042 properties['superseder'] = hyperdb.Multilink(classname)
3043 Class.__init__(self, db, classname, **properties)
3045 # vim: set et sts=4 sw=4 :