9c7d7219f26cddd5fae6e8e39831ac0e8bbdcdf7
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 """ Relational database (SQL) backend common code.
20 Basics:
22 - map roundup classes to relational tables
23 - automatically detect schema changes and modify the table schemas
24 appropriately (we store the "database version" of the schema in the
25 database itself as the only row of the "schema" table)
26 - multilinks (which represent a many-to-many relationship) are handled through
27 intermediate tables
28 - journals are stored adjunct to the per-class tables
29 - table names and columns have "_" prepended so the names can't clash with
30 restricted names (like "order")
31 - retirement is determined by the __retired__ column being > 0
33 Database-specific changes may generally be pushed out to the overridable
34 sql_* methods, since everything else should be fairly generic. There's
35 probably a bit of work to be done if a database is used that actually
36 honors column typing, since the initial databases don't (sqlite stores
37 everything as a string.)
39 The schema of the hyperdb being mapped to the database is stored in the
40 database itself as a repr()'ed dictionary of information about each Class
41 that maps to a table. If that information differs from the hyperdb schema,
42 then we update it. We also store in the schema dict a version which
43 allows us to upgrade the database schema when necessary. See upgrade_db().
45 To force a unqiueness constraint on the key properties we put the item
46 id into the __retired__ column duing retirement (so it's 0 for "active"
47 items) and place a unqiueness constraint on key + __retired__. This is
48 particularly important for the users class where multiple users may
49 try to have the same username, with potentially many retired users with
50 the same name.
51 """
52 __docformat__ = 'restructuredtext'
54 # standard python modules
55 import sys, os, time, re, errno, weakref, copy, logging
57 # roundup modules
58 from roundup import hyperdb, date, password, roundupdb, security, support
59 from roundup.hyperdb import String, Password, Date, Interval, Link, \
60 Multilink, DatabaseError, Boolean, Number, Node
61 from roundup.backends import locking
62 from roundup.support import reversed
63 from roundup.i18n import _
65 # support
66 from roundup.backends.blobfiles import FileStorage
67 try:
68 from roundup.backends.indexer_xapian import Indexer
69 except ImportError:
70 from roundup.backends.indexer_rdbms import Indexer
71 from roundup.backends.sessions_rdbms import Sessions, OneTimeKeys
72 from roundup.date import Range
74 from roundup.backends.back_anydbm import compile_expression
77 # dummy value meaning "argument not passed"
78 _marker = []
80 def _num_cvt(num):
81 num = str(num)
82 try:
83 return int(num)
84 except:
85 return float(num)
87 def _bool_cvt(value):
88 if value in ('TRUE', 'FALSE'):
89 return {'TRUE': 1, 'FALSE': 0}[value]
90 # assume it's a number returned from the db API
91 return int(value)
93 def connection_dict(config, dbnamestr=None):
94 """ Used by Postgresql and MySQL to detemine the keyword args for
95 opening the database connection."""
96 d = { }
97 if dbnamestr:
98 d[dbnamestr] = config.RDBMS_NAME
99 for name in ('host', 'port', 'password', 'user', 'read_default_group',
100 'read_default_file'):
101 cvar = 'RDBMS_'+name.upper()
102 if config[cvar] is not None:
103 d[name] = config[cvar]
104 return d
107 class IdListOptimizer:
108 """ To prevent flooding the SQL parser of the underlaying
109 db engine with "x IN (1, 2, 3, ..., <large number>)" collapses
110 these cases to "x BETWEEN 1 AND <large number>".
111 """
113 def __init__(self):
114 self.ranges = []
115 self.singles = []
117 def append(self, nid):
118 """ Invariant: nid are ordered ascending """
119 if self.ranges:
120 last = self.ranges[-1]
121 if last[1] == nid-1:
122 last[1] = nid
123 return
124 if self.singles:
125 last = self.singles[-1]
126 if last == nid-1:
127 self.singles.pop()
128 self.ranges.append([last, nid])
129 return
130 self.singles.append(nid)
132 def where(self, field, placeholder):
133 ranges = self.ranges
134 singles = self.singles
136 if not singles and not ranges: return "(1=0)", []
138 if ranges:
139 between = '%s BETWEEN %s AND %s' % (
140 field, placeholder, placeholder)
141 stmnt = [between] * len(ranges)
142 else:
143 stmnt = []
144 if singles:
145 stmnt.append('%s in (%s)' % (
146 field, ','.join([placeholder]*len(singles))))
148 return '(%s)' % ' OR '.join(stmnt), sum(ranges, []) + singles
150 def __str__(self):
151 return "ranges: %r / singles: %r" % (self.ranges, self.singles)
154 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
155 """ Wrapper around an SQL database that presents a hyperdb interface.
157 - some functionality is specific to the actual SQL database, hence
158 the sql_* methods that are NotImplemented
159 - we keep a cache of the latest N row fetches (where N is configurable).
160 """
161 def __init__(self, config, journaltag=None):
162 """ Open the database and load the schema from it.
163 """
164 FileStorage.__init__(self, config.UMASK)
165 self.config, self.journaltag = config, journaltag
166 self.dir = config.DATABASE
167 self.classes = {}
168 self.indexer = Indexer(self)
169 self.security = security.Security(self)
171 # additional transaction support for external files and the like
172 self.transactions = []
174 # keep a cache of the N most recently retrieved rows of any kind
175 # (classname, nodeid) = row
176 self.cache_size = config.RDBMS_CACHE_SIZE
177 self.clearCache()
178 self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0,
179 'filtering': 0}
181 # database lock
182 self.lockfile = None
184 # open a connection to the database, creating the "conn" attribute
185 self.open_connection()
187 def clearCache(self):
188 self.cache = {}
189 self.cache_lru = []
191 def getSessionManager(self):
192 return Sessions(self)
194 def getOTKManager(self):
195 return OneTimeKeys(self)
197 def open_connection(self):
198 """ Open a connection to the database, creating it if necessary.
200 Must call self.load_dbschema()
201 """
202 raise NotImplemented
204 def sql(self, sql, args=None, cursor=None):
205 """ Execute the sql with the optional args.
206 """
207 self.log_debug('SQL %r %r'%(sql, args))
208 if not cursor:
209 cursor = self.cursor
210 if args:
211 cursor.execute(sql, args)
212 else:
213 cursor.execute(sql)
215 def sql_fetchone(self):
216 """ Fetch a single row. If there's nothing to fetch, return None.
217 """
218 return self.cursor.fetchone()
220 def sql_fetchall(self):
221 """ Fetch all rows. If there's nothing to fetch, return [].
222 """
223 return self.cursor.fetchall()
225 def sql_fetchiter(self):
226 """ Fetch all row as a generator
227 """
228 while True:
229 row = self.cursor.fetchone()
230 if not row: break
231 yield row
233 def sql_stringquote(self, value):
234 """ Quote the string so it's safe to put in the 'sql quotes'
235 """
236 return re.sub("'", "''", str(value))
238 def init_dbschema(self):
239 self.database_schema = {
240 'version': self.current_db_version,
241 'tables': {}
242 }
244 def load_dbschema(self):
245 """ Load the schema definition that the database currently implements
246 """
247 self.cursor.execute('select schema from schema')
248 schema = self.cursor.fetchone()
249 if schema:
250 self.database_schema = eval(schema[0])
251 else:
252 self.database_schema = {}
254 def save_dbschema(self):
255 """ Save the schema definition that the database currently implements
256 """
257 s = repr(self.database_schema)
258 self.sql('delete from schema')
259 self.sql('insert into schema values (%s)'%self.arg, (s,))
261 def post_init(self):
262 """ Called once the schema initialisation has finished.
264 We should now confirm that the schema defined by our "classes"
265 attribute actually matches the schema in the database.
266 """
267 save = 0
269 # handle changes in the schema
270 tables = self.database_schema['tables']
271 for classname, spec in self.classes.iteritems():
272 if classname in tables:
273 dbspec = tables[classname]
274 if self.update_class(spec, dbspec):
275 tables[classname] = spec.schema()
276 save = 1
277 else:
278 self.create_class(spec)
279 tables[classname] = spec.schema()
280 save = 1
282 for classname, spec in list(tables.items()):
283 if classname not in self.classes:
284 self.drop_class(classname, tables[classname])
285 del tables[classname]
286 save = 1
288 # now upgrade the database for column type changes, new internal
289 # tables, etc.
290 save = save | self.upgrade_db()
292 # update the database version of the schema
293 if save:
294 self.save_dbschema()
296 # reindex the db if necessary
297 if self.indexer.should_reindex():
298 self.reindex()
300 # commit
301 self.sql_commit()
303 # update this number when we need to make changes to the SQL structure
304 # of the backen database
305 current_db_version = 5
306 db_version_updated = False
307 def upgrade_db(self):
308 """ Update the SQL database to reflect changes in the backend code.
310 Return boolean whether we need to save the schema.
311 """
312 version = self.database_schema.get('version', 1)
313 if version > self.current_db_version:
314 raise DatabaseError('attempting to run rev %d DATABASE with rev '
315 '%d CODE!'%(version, self.current_db_version))
316 if version == self.current_db_version:
317 # nothing to do
318 return 0
320 if version < 2:
321 self.log_info('upgrade to version 2')
322 # change the schema structure
323 self.database_schema = {'tables': self.database_schema}
325 # version 1 didn't have the actor column (note that in
326 # MySQL this will also transition the tables to typed columns)
327 self.add_new_columns_v2()
329 # version 1 doesn't have the OTK, session and indexing in the
330 # database
331 self.create_version_2_tables()
333 if version < 3:
334 self.log_info('upgrade to version 3')
335 self.fix_version_2_tables()
337 if version < 4:
338 self.fix_version_3_tables()
340 if version < 5:
341 self.fix_version_4_tables()
343 self.database_schema['version'] = self.current_db_version
344 self.db_version_updated = True
345 return 1
347 def fix_version_3_tables(self):
348 # drop the shorter VARCHAR OTK column and add a new TEXT one
349 for name in ('otk', 'session'):
350 self.sql('DELETE FROM %ss'%name)
351 self.sql('ALTER TABLE %ss DROP %s_value'%(name, name))
352 self.sql('ALTER TABLE %ss ADD %s_value TEXT'%(name, name))
354 def fix_version_2_tables(self):
355 # Default (used by sqlite): NOOP
356 pass
358 def fix_version_4_tables(self):
359 # note this is an explicit call now
360 c = self.cursor
361 for cn, klass in self.classes.iteritems():
362 c.execute('select id from _%s where __retired__<>0'%(cn,))
363 for (id,) in c.fetchall():
364 c.execute('update _%s set __retired__=%s where id=%s'%(cn,
365 self.arg, self.arg), (id, id))
367 if klass.key:
368 self.add_class_key_required_unique_constraint(cn, klass.key)
370 def _convert_journal_tables(self):
371 """Get current journal table contents, drop the table and re-create"""
372 c = self.cursor
373 cols = ','.join('nodeid date tag action params'.split())
374 for klass in self.classes.itervalues():
375 # slurp and drop
376 sql = 'select %s from %s__journal order by date'%(cols,
377 klass.classname)
378 c.execute(sql)
379 contents = c.fetchall()
380 self.drop_journal_table_indexes(klass.classname)
381 c.execute('drop table %s__journal'%klass.classname)
383 # re-create and re-populate
384 self.create_journal_table(klass)
385 a = self.arg
386 sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(
387 klass.classname, cols, a, a, a, a, a)
388 for row in contents:
389 # no data conversion needed
390 self.cursor.execute(sql, row)
392 def _convert_string_properties(self):
393 """Get current Class tables that contain String properties, and
394 convert the VARCHAR columns to TEXT"""
395 c = self.cursor
396 for klass in self.classes.itervalues():
397 # slurp and drop
398 cols, mls = self.determine_columns(list(klass.properties.iteritems()))
399 scols = ','.join([i[0] for i in cols])
400 sql = 'select id,%s from _%s'%(scols, klass.classname)
401 c.execute(sql)
402 contents = c.fetchall()
403 self.drop_class_table_indexes(klass.classname, klass.getkey())
404 c.execute('drop table _%s'%klass.classname)
406 # re-create and re-populate
407 self.create_class_table(klass, create_sequence=0)
408 a = ','.join([self.arg for i in range(len(cols)+1)])
409 sql = 'insert into _%s (id,%s) values (%s)'%(klass.classname,
410 scols, a)
411 for row in contents:
412 l = []
413 for entry in row:
414 # mysql will already be a string - psql needs "help"
415 if entry is not None and not isinstance(entry, type('')):
416 entry = str(entry)
417 l.append(entry)
418 self.cursor.execute(sql, l)
420 def refresh_database(self):
421 self.post_init()
424 def reindex(self, classname=None, show_progress=False):
425 if classname:
426 classes = [self.getclass(classname)]
427 else:
428 classes = list(self.classes.itervalues())
429 for klass in classes:
430 if show_progress:
431 for nodeid in support.Progress('Reindex %s'%klass.classname,
432 klass.list()):
433 klass.index(nodeid)
434 else:
435 for nodeid in klass.list():
436 klass.index(nodeid)
437 self.indexer.save_index()
439 hyperdb_to_sql_datatypes = {
440 hyperdb.String : 'TEXT',
441 hyperdb.Date : 'TIMESTAMP',
442 hyperdb.Link : 'INTEGER',
443 hyperdb.Interval : 'VARCHAR(255)',
444 hyperdb.Password : 'VARCHAR(255)',
445 hyperdb.Boolean : 'BOOLEAN',
446 hyperdb.Number : 'REAL',
447 }
449 def hyperdb_to_sql_datatype(self, propclass):
451 datatype = self.hyperdb_to_sql_datatypes.get(propclass)
452 if datatype:
453 return datatype
455 for k, v in self.hyperdb_to_sql_datatypes.iteritems():
456 if issubclass(propclass, k):
457 return v
459 raise ValueError('%r is not a hyperdb property class' % propclass)
461 def determine_columns(self, properties):
462 """ Figure the column names and multilink properties from the spec
464 "properties" is a list of (name, prop) where prop may be an
465 instance of a hyperdb "type" _or_ a string repr of that type.
466 """
467 cols = [
468 ('_actor', self.hyperdb_to_sql_datatype(hyperdb.Link)),
469 ('_activity', self.hyperdb_to_sql_datatype(hyperdb.Date)),
470 ('_creator', self.hyperdb_to_sql_datatype(hyperdb.Link)),
471 ('_creation', self.hyperdb_to_sql_datatype(hyperdb.Date)),
472 ]
473 mls = []
474 # add the multilinks separately
475 for col, prop in properties:
476 if isinstance(prop, Multilink):
477 mls.append(col)
478 continue
480 if isinstance(prop, type('')):
481 raise ValueError("string property spec!")
482 #and prop.find('Multilink') != -1:
483 #mls.append(col)
485 datatype = self.hyperdb_to_sql_datatype(prop.__class__)
486 cols.append(('_'+col, datatype))
488 # Intervals stored as two columns
489 if isinstance(prop, Interval):
490 cols.append(('__'+col+'_int__', 'BIGINT'))
492 cols.sort()
493 return cols, mls
495 def update_class(self, spec, old_spec, force=0):
496 """ Determine the differences between the current spec and the
497 database version of the spec, and update where necessary.
499 If 'force' is true, update the database anyway.
500 """
501 new_spec = spec.schema()
502 new_spec[1].sort()
503 old_spec[1].sort()
504 if not force and new_spec == old_spec:
505 # no changes
506 return 0
508 logger = logging.getLogger('roundup.hyperdb')
509 logger.info('update_class %s'%spec.classname)
511 logger.debug('old_spec %r'%(old_spec,))
512 logger.debug('new_spec %r'%(new_spec,))
514 # detect key prop change for potential index change
515 keyprop_changes = {}
516 if new_spec[0] != old_spec[0]:
517 if old_spec[0]:
518 keyprop_changes['remove'] = old_spec[0]
519 if new_spec[0]:
520 keyprop_changes['add'] = new_spec[0]
522 # detect multilinks that have been removed, and drop their table
523 old_has = {}
524 for name, prop in old_spec[1]:
525 old_has[name] = 1
526 if name in spec.properties:
527 continue
529 if prop.find('Multilink to') != -1:
530 # first drop indexes.
531 self.drop_multilink_table_indexes(spec.classname, name)
533 # now the multilink table itself
534 sql = 'drop table %s_%s'%(spec.classname, name)
535 else:
536 # if this is the key prop, drop the index first
537 if old_spec[0] == prop:
538 self.drop_class_table_key_index(spec.classname, name)
539 del keyprop_changes['remove']
541 # drop the column
542 sql = 'alter table _%s drop column _%s'%(spec.classname, name)
544 self.sql(sql)
546 # if we didn't remove the key prop just then, but the key prop has
547 # changed, we still need to remove the old index
548 if 'remove' in keyprop_changes:
549 self.drop_class_table_key_index(spec.classname,
550 keyprop_changes['remove'])
552 # add new columns
553 for propname, prop in new_spec[1]:
554 if propname in old_has:
555 continue
556 prop = spec.properties[propname]
557 if isinstance(prop, Multilink):
558 self.create_multilink_table(spec, propname)
559 else:
560 # add the column
561 coltype = self.hyperdb_to_sql_datatype(prop.__class__)
562 sql = 'alter table _%s add column _%s %s'%(
563 spec.classname, propname, coltype)
564 self.sql(sql)
566 # extra Interval column
567 if isinstance(prop, Interval):
568 sql = 'alter table _%s add column __%s_int__ BIGINT'%(
569 spec.classname, propname)
570 self.sql(sql)
572 # if the new column is a key prop, we need an index!
573 if new_spec[0] == propname:
574 self.create_class_table_key_index(spec.classname, propname)
575 del keyprop_changes['add']
577 # if we didn't add the key prop just then, but the key prop has
578 # changed, we still need to add the new index
579 if 'add' in keyprop_changes:
580 self.create_class_table_key_index(spec.classname,
581 keyprop_changes['add'])
583 return 1
585 def determine_all_columns(self, spec):
586 """Figure out the columns from the spec and also add internal columns
588 """
589 cols, mls = self.determine_columns(list(spec.properties.iteritems()))
591 # add on our special columns
592 cols.append(('id', 'INTEGER PRIMARY KEY'))
593 cols.append(('__retired__', 'INTEGER DEFAULT 0'))
594 return cols, mls
596 def create_class_table(self, spec):
597 """Create the class table for the given Class "spec". Creates the
598 indexes too."""
599 cols, mls = self.determine_all_columns(spec)
601 # create the base table
602 scols = ','.join(['%s %s'%x for x in cols])
603 sql = 'create table _%s (%s)'%(spec.classname, scols)
604 self.sql(sql)
606 self.create_class_table_indexes(spec)
608 return cols, mls
610 def create_class_table_indexes(self, spec):
611 """ create the class table for the given spec
612 """
613 # create __retired__ index
614 index_sql2 = 'create index _%s_retired_idx on _%s(__retired__)'%(
615 spec.classname, spec.classname)
616 self.sql(index_sql2)
618 # create index for key property
619 if spec.key:
620 index_sql3 = 'create index _%s_%s_idx on _%s(_%s)'%(
621 spec.classname, spec.key,
622 spec.classname, spec.key)
623 self.sql(index_sql3)
625 # and the unique index for key / retired(id)
626 self.add_class_key_required_unique_constraint(spec.classname,
627 spec.key)
629 # TODO: create indexes on (selected?) Link property columns, as
630 # they're more likely to be used for lookup
632 def add_class_key_required_unique_constraint(self, cn, key):
633 sql = '''create unique index _%s_key_retired_idx
634 on _%s(__retired__, _%s)'''%(cn, cn, key)
635 self.sql(sql)
637 def drop_class_table_indexes(self, cn, key):
638 # drop the old table indexes first
639 l = ['_%s_id_idx'%cn, '_%s_retired_idx'%cn]
640 if key:
641 l.append('_%s_%s_idx'%(cn, key))
643 table_name = '_%s'%cn
644 for index_name in l:
645 if not self.sql_index_exists(table_name, index_name):
646 continue
647 index_sql = 'drop index '+index_name
648 self.sql(index_sql)
650 def create_class_table_key_index(self, cn, key):
651 """ create the class table for the given spec
652 """
653 sql = 'create index _%s_%s_idx on _%s(_%s)'%(cn, key, cn, key)
654 self.sql(sql)
656 def drop_class_table_key_index(self, cn, key):
657 table_name = '_%s'%cn
658 index_name = '_%s_%s_idx'%(cn, key)
659 if self.sql_index_exists(table_name, index_name):
660 sql = 'drop index '+index_name
661 self.sql(sql)
663 # and now the retired unique index too
664 index_name = '_%s_key_retired_idx'%cn
665 if self.sql_index_exists(table_name, index_name):
666 sql = 'drop index '+index_name
667 self.sql(sql)
669 def create_journal_table(self, spec):
670 """ create the journal table for a class given the spec and
671 already-determined cols
672 """
673 # journal table
674 cols = ','.join(['%s varchar'%x
675 for x in 'nodeid date tag action params'.split()])
676 sql = """create table %s__journal (
677 nodeid integer, date %s, tag varchar(255),
678 action varchar(255), params text)""" % (spec.classname,
679 self.hyperdb_to_sql_datatype(hyperdb.Date))
680 self.sql(sql)
681 self.create_journal_table_indexes(spec)
683 def create_journal_table_indexes(self, spec):
684 # index on nodeid
685 sql = 'create index %s_journ_idx on %s__journal(nodeid)'%(
686 spec.classname, spec.classname)
687 self.sql(sql)
689 def drop_journal_table_indexes(self, classname):
690 index_name = '%s_journ_idx'%classname
691 if not self.sql_index_exists('%s__journal'%classname, index_name):
692 return
693 index_sql = 'drop index '+index_name
694 self.sql(index_sql)
696 def create_multilink_table(self, spec, ml):
697 """ Create a multilink table for the "ml" property of the class
698 given by the spec
699 """
700 # create the table
701 sql = 'create table %s_%s (linkid INTEGER, nodeid INTEGER)'%(
702 spec.classname, ml)
703 self.sql(sql)
704 self.create_multilink_table_indexes(spec, ml)
706 def create_multilink_table_indexes(self, spec, ml):
707 # create index on linkid
708 index_sql = 'create index %s_%s_l_idx on %s_%s(linkid)'%(
709 spec.classname, ml, spec.classname, ml)
710 self.sql(index_sql)
712 # create index on nodeid
713 index_sql = 'create index %s_%s_n_idx on %s_%s(nodeid)'%(
714 spec.classname, ml, spec.classname, ml)
715 self.sql(index_sql)
717 def drop_multilink_table_indexes(self, classname, ml):
718 l = [
719 '%s_%s_l_idx'%(classname, ml),
720 '%s_%s_n_idx'%(classname, ml)
721 ]
722 table_name = '%s_%s'%(classname, ml)
723 for index_name in l:
724 if not self.sql_index_exists(table_name, index_name):
725 continue
726 index_sql = 'drop index %s'%index_name
727 self.sql(index_sql)
729 def create_class(self, spec):
730 """ Create a database table according to the given spec.
731 """
732 cols, mls = self.create_class_table(spec)
733 self.create_journal_table(spec)
735 # now create the multilink tables
736 for ml in mls:
737 self.create_multilink_table(spec, ml)
739 def drop_class(self, cn, spec):
740 """ Drop the given table from the database.
742 Drop the journal and multilink tables too.
743 """
744 properties = spec[1]
745 # figure the multilinks
746 mls = []
747 for propname, prop in properties:
748 if isinstance(prop, Multilink):
749 mls.append(propname)
751 # drop class table and indexes
752 self.drop_class_table_indexes(cn, spec[0])
754 self.drop_class_table(cn)
756 # drop journal table and indexes
757 self.drop_journal_table_indexes(cn)
758 sql = 'drop table %s__journal'%cn
759 self.sql(sql)
761 for ml in mls:
762 # drop multilink table and indexes
763 self.drop_multilink_table_indexes(cn, ml)
764 sql = 'drop table %s_%s'%(spec.classname, ml)
765 self.sql(sql)
767 def drop_class_table(self, cn):
768 sql = 'drop table _%s'%cn
769 self.sql(sql)
771 #
772 # Classes
773 #
774 def __getattr__(self, classname):
775 """ A convenient way of calling self.getclass(classname).
776 """
777 if classname in self.classes:
778 return self.classes[classname]
779 raise AttributeError(classname)
781 def addclass(self, cl):
782 """ Add a Class to the hyperdatabase.
783 """
784 cn = cl.classname
785 if cn in self.classes:
786 raise ValueError(cn)
787 self.classes[cn] = cl
789 # add default Edit and View permissions
790 self.security.addPermission(name="Create", klass=cn,
791 description="User is allowed to create "+cn)
792 self.security.addPermission(name="Edit", klass=cn,
793 description="User is allowed to edit "+cn)
794 self.security.addPermission(name="View", klass=cn,
795 description="User is allowed to access "+cn)
797 def getclasses(self):
798 """ Return a list of the names of all existing classes.
799 """
800 return sorted(self.classes)
802 def getclass(self, classname):
803 """Get the Class object representing a particular class.
805 If 'classname' is not a valid class name, a KeyError is raised.
806 """
807 try:
808 return self.classes[classname]
809 except KeyError:
810 raise KeyError('There is no class called "%s"'%classname)
812 def clear(self):
813 """Delete all database contents.
815 Note: I don't commit here, which is different behaviour to the
816 "nuke from orbit" behaviour in the dbs.
817 """
818 logging.getLogger('roundup.hyperdb').info('clear')
819 for cn in self.classes:
820 sql = 'delete from _%s'%cn
821 self.sql(sql)
823 #
824 # Nodes
825 #
827 hyperdb_to_sql_value = {
828 hyperdb.String : str,
829 # fractional seconds by default
830 hyperdb.Date : lambda x: x.formal(sep=' ', sec='%06.3f'),
831 hyperdb.Link : int,
832 hyperdb.Interval : str,
833 hyperdb.Password : str,
834 hyperdb.Boolean : lambda x: x and 'TRUE' or 'FALSE',
835 hyperdb.Number : lambda x: x,
836 hyperdb.Multilink : lambda x: x, # used in journal marshalling
837 }
839 def to_sql_value(self, propklass):
841 fn = self.hyperdb_to_sql_value.get(propklass)
842 if fn:
843 return fn
845 for k, v in self.hyperdb_to_sql_value.iteritems():
846 if issubclass(propklass, k):
847 return v
849 raise ValueError('%r is not a hyperdb property class' % propklass)
851 def _cache_del(self, key):
852 del self.cache[key]
853 self.cache_lru.remove(key)
855 def _cache_refresh(self, key):
856 self.cache_lru.remove(key)
857 self.cache_lru.insert(0, key)
859 def _cache_save(self, key, node):
860 self.cache[key] = node
861 # update the LRU
862 self.cache_lru.insert(0, key)
863 if len(self.cache_lru) > self.cache_size:
864 del self.cache[self.cache_lru.pop()]
866 def addnode(self, classname, nodeid, node):
867 """ Add the specified node to its class's db.
868 """
869 self.log_debug('addnode %s%s %r'%(classname,
870 nodeid, node))
872 # determine the column definitions and multilink tables
873 cl = self.classes[classname]
874 cols, mls = self.determine_columns(list(cl.properties.iteritems()))
876 # we'll be supplied these props if we're doing an import
877 values = node.copy()
878 if 'creator' not in values:
879 # add in the "calculated" properties (dupe so we don't affect
880 # calling code's node assumptions)
881 values['creation'] = values['activity'] = date.Date()
882 values['actor'] = values['creator'] = self.getuid()
884 cl = self.classes[classname]
885 props = cl.getprops(protected=1)
886 del props['id']
888 # default the non-multilink columns
889 for col, prop in props.iteritems():
890 if col not in values:
891 if isinstance(prop, Multilink):
892 values[col] = []
893 else:
894 values[col] = None
896 # clear this node out of the cache if it's in there
897 key = (classname, nodeid)
898 if key in self.cache:
899 self._cache_del(key)
901 # figure the values to insert
902 vals = []
903 for col,dt in cols:
904 # this is somewhat dodgy....
905 if col.endswith('_int__'):
906 # XXX eugh, this test suxxors
907 value = values[col[2:-6]]
908 # this is an Interval special "int" column
909 if value is not None:
910 vals.append(value.as_seconds())
911 else:
912 vals.append(value)
913 continue
915 prop = props[col[1:]]
916 value = values[col[1:]]
917 if value is not None:
918 value = self.to_sql_value(prop.__class__)(value)
919 vals.append(value)
920 vals.append(nodeid)
921 vals = tuple(vals)
923 # make sure the ordering is correct for column name -> column value
924 s = ','.join([self.arg for x in cols]) + ',%s'%self.arg
925 cols = ','.join([col for col,dt in cols]) + ',id'
927 # perform the inserts
928 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
929 self.sql(sql, vals)
931 # insert the multilink rows
932 for col in mls:
933 t = '%s_%s'%(classname, col)
934 for entry in node[col]:
935 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
936 self.arg, self.arg)
937 self.sql(sql, (entry, nodeid))
939 def setnode(self, classname, nodeid, values, multilink_changes={}):
940 """ Change the specified node.
941 """
942 self.log_debug('setnode %s%s %r'
943 % (classname, nodeid, values))
945 # clear this node out of the cache if it's in there
946 key = (classname, nodeid)
947 if key in self.cache:
948 self._cache_del(key)
950 cl = self.classes[classname]
951 props = cl.getprops()
953 cols = []
954 mls = []
955 # add the multilinks separately
956 for col in values:
957 prop = props[col]
958 if isinstance(prop, Multilink):
959 mls.append(col)
960 elif isinstance(prop, Interval):
961 # Intervals store the seconds value too
962 cols.append(col)
963 # extra leading '_' added by code below
964 cols.append('_' +col + '_int__')
965 else:
966 cols.append(col)
967 cols.sort()
969 # figure the values to insert
970 vals = []
971 for col in cols:
972 if col.endswith('_int__'):
973 # XXX eugh, this test suxxors
974 # Intervals store the seconds value too
975 col = col[1:-6]
976 prop = props[col]
977 value = values[col]
978 if value is None:
979 vals.append(None)
980 else:
981 vals.append(value.as_seconds())
982 else:
983 prop = props[col]
984 value = values[col]
985 if value is None:
986 e = None
987 else:
988 e = self.to_sql_value(prop.__class__)(value)
989 vals.append(e)
991 vals.append(int(nodeid))
992 vals = tuple(vals)
994 # if there's any updates to regular columns, do them
995 if cols:
996 # make sure the ordering is correct for column name -> column value
997 s = ','.join(['_%s=%s'%(x, self.arg) for x in cols])
998 cols = ','.join(cols)
1000 # perform the update
1001 sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
1002 self.sql(sql, vals)
1004 # we're probably coming from an import, not a change
1005 if not multilink_changes:
1006 for name in mls:
1007 prop = props[name]
1008 value = values[name]
1010 t = '%s_%s'%(classname, name)
1012 # clear out previous values for this node
1013 # XXX numeric ids
1014 self.sql('delete from %s where nodeid=%s'%(t, self.arg),
1015 (nodeid,))
1017 # insert the values for this node
1018 for entry in values[name]:
1019 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
1020 self.arg, self.arg)
1021 # XXX numeric ids
1022 self.sql(sql, (entry, nodeid))
1024 # we have multilink changes to apply
1025 for col, (add, remove) in multilink_changes.iteritems():
1026 tn = '%s_%s'%(classname, col)
1027 if add:
1028 sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
1029 self.arg, self.arg)
1030 for addid in add:
1031 # XXX numeric ids
1032 self.sql(sql, (int(nodeid), int(addid)))
1033 if remove:
1034 s = ','.join([self.arg]*len(remove))
1035 sql = 'delete from %s where nodeid=%s and linkid in (%s)'%(tn,
1036 self.arg, s)
1037 # XXX numeric ids
1038 self.sql(sql, [int(nodeid)] + remove)
1040 sql_to_hyperdb_value = {
1041 hyperdb.String : str,
1042 hyperdb.Date : lambda x:date.Date(str(x).replace(' ', '.')),
1043 # hyperdb.Link : int, # XXX numeric ids
1044 hyperdb.Link : str,
1045 hyperdb.Interval : date.Interval,
1046 hyperdb.Password : lambda x: password.Password(encrypted=x),
1047 hyperdb.Boolean : _bool_cvt,
1048 hyperdb.Number : _num_cvt,
1049 hyperdb.Multilink : lambda x: x, # used in journal marshalling
1050 }
1052 def to_hyperdb_value(self, propklass):
1054 fn = self.sql_to_hyperdb_value.get(propklass)
1055 if fn:
1056 return fn
1058 for k, v in self.sql_to_hyperdb_value.iteritems():
1059 if issubclass(propklass, k):
1060 return v
1062 raise ValueError('%r is not a hyperdb property class' % propklass)
1064 def getnode(self, classname, nodeid):
1065 """ Get a node from the database.
1066 """
1067 # see if we have this node cached
1068 key = (classname, nodeid)
1069 if key in self.cache:
1070 # push us back to the top of the LRU
1071 self._cache_refresh(key)
1072 if __debug__:
1073 self.stats['cache_hits'] += 1
1074 # return the cached information
1075 return self.cache[key]
1077 if __debug__:
1078 self.stats['cache_misses'] += 1
1079 start_t = time.time()
1081 # figure the columns we're fetching
1082 cl = self.classes[classname]
1083 cols, mls = self.determine_columns(list(cl.properties.iteritems()))
1084 scols = ','.join([col for col,dt in cols])
1086 # perform the basic property fetch
1087 sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
1088 self.sql(sql, (nodeid,))
1090 values = self.sql_fetchone()
1091 if values is None:
1092 raise IndexError('no such %s node %s'%(classname, nodeid))
1094 # make up the node
1095 node = {}
1096 props = cl.getprops(protected=1)
1097 for col in range(len(cols)):
1098 name = cols[col][0][1:]
1099 if name.endswith('_int__'):
1100 # XXX eugh, this test suxxors
1101 # ignore the special Interval-as-seconds column
1102 continue
1103 value = values[col]
1104 if value is not None:
1105 value = self.to_hyperdb_value(props[name].__class__)(value)
1106 node[name] = value
1108 # save off in the cache
1109 key = (classname, nodeid)
1110 self._cache_save(key, node)
1112 if __debug__:
1113 self.stats['get_items'] += (time.time() - start_t)
1115 return node
1117 def destroynode(self, classname, nodeid):
1118 """Remove a node from the database. Called exclusively by the
1119 destroy() method on Class.
1120 """
1121 logging.getLogger('roundup.hyperdb').info('destroynode %s%s'%(
1122 classname, nodeid))
1124 # make sure the node exists
1125 if not self.hasnode(classname, nodeid):
1126 raise IndexError('%s has no node %s'%(classname, nodeid))
1128 # see if we have this node cached
1129 if (classname, nodeid) in self.cache:
1130 del self.cache[(classname, nodeid)]
1132 # see if there's any obvious commit actions that we should get rid of
1133 for entry in self.transactions[:]:
1134 if entry[1][:2] == (classname, nodeid):
1135 self.transactions.remove(entry)
1137 # now do the SQL
1138 sql = 'delete from _%s where id=%s'%(classname, self.arg)
1139 self.sql(sql, (nodeid,))
1141 # remove from multilnks
1142 cl = self.getclass(classname)
1143 x, mls = self.determine_columns(list(cl.properties.iteritems()))
1144 for col in mls:
1145 # get the link ids
1146 sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
1147 self.sql(sql, (nodeid,))
1149 # remove journal entries
1150 sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
1151 self.sql(sql, (nodeid,))
1153 # cleanup any blob filestorage when we commit
1154 self.transactions.append((FileStorage.destroy, (self, classname, nodeid)))
1156 def hasnode(self, classname, nodeid):
1157 """ Determine if the database has a given node.
1158 """
1159 # If this node is in the cache, then we do not need to go to
1160 # the database. (We don't consider this an LRU hit, though.)
1161 if (classname, nodeid) in self.cache:
1162 # Return 1, not True, to match the type of the result of
1163 # the SQL operation below.
1164 return 1
1165 sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
1166 self.sql(sql, (nodeid,))
1167 return int(self.cursor.fetchone()[0])
1169 def countnodes(self, classname):
1170 """ Count the number of nodes that exist for a particular Class.
1171 """
1172 sql = 'select count(*) from _%s'%classname
1173 self.sql(sql)
1174 return self.cursor.fetchone()[0]
1176 def addjournal(self, classname, nodeid, action, params, creator=None,
1177 creation=None):
1178 """ Journal the Action
1179 'action' may be:
1181 'create' or 'set' -- 'params' is a dictionary of property values
1182 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
1183 'retire' -- 'params' is None
1184 """
1185 # handle supply of the special journalling parameters (usually
1186 # supplied on importing an existing database)
1187 if creator:
1188 journaltag = creator
1189 else:
1190 journaltag = self.getuid()
1191 if creation:
1192 journaldate = creation
1193 else:
1194 journaldate = date.Date()
1196 # create the journal entry
1197 cols = 'nodeid,date,tag,action,params'
1199 self.log_debug('addjournal %s%s %r %s %s %r'%(classname,
1200 nodeid, journaldate, journaltag, action, params))
1202 # make the journalled data marshallable
1203 if isinstance(params, type({})):
1204 self._journal_marshal(params, classname)
1206 params = repr(params)
1208 dc = self.to_sql_value(hyperdb.Date)
1209 journaldate = dc(journaldate)
1211 self.save_journal(classname, cols, nodeid, journaldate,
1212 journaltag, action, params)
1214 def setjournal(self, classname, nodeid, journal):
1215 """Set the journal to the "journal" list."""
1216 # clear out any existing entries
1217 self.sql('delete from %s__journal where nodeid=%s'%(classname,
1218 self.arg), (nodeid,))
1220 # create the journal entry
1221 cols = 'nodeid,date,tag,action,params'
1223 dc = self.to_sql_value(hyperdb.Date)
1224 for nodeid, journaldate, journaltag, action, params in journal:
1225 self.log_debug('addjournal %s%s %r %s %s %r'%(
1226 classname, nodeid, journaldate, journaltag, action,
1227 params))
1229 # make the journalled data marshallable
1230 if isinstance(params, type({})):
1231 self._journal_marshal(params, classname)
1232 params = repr(params)
1234 self.save_journal(classname, cols, nodeid, dc(journaldate),
1235 journaltag, action, params)
1237 def _journal_marshal(self, params, classname):
1238 """Convert the journal params values into safely repr'able and
1239 eval'able values."""
1240 properties = self.getclass(classname).getprops()
1241 for param, value in params.iteritems():
1242 if not value:
1243 continue
1244 property = properties[param]
1245 cvt = self.to_sql_value(property.__class__)
1246 if isinstance(property, Password):
1247 params[param] = cvt(value)
1248 elif isinstance(property, Date):
1249 params[param] = cvt(value)
1250 elif isinstance(property, Interval):
1251 params[param] = cvt(value)
1252 elif isinstance(property, Boolean):
1253 params[param] = cvt(value)
1255 def getjournal(self, classname, nodeid):
1256 """ get the journal for id
1257 """
1258 # make sure the node exists
1259 if not self.hasnode(classname, nodeid):
1260 raise IndexError('%s has no node %s'%(classname, nodeid))
1262 cols = ','.join('nodeid date tag action params'.split())
1263 journal = self.load_journal(classname, cols, nodeid)
1265 # now unmarshal the data
1266 dc = self.to_hyperdb_value(hyperdb.Date)
1267 res = []
1268 properties = self.getclass(classname).getprops()
1269 for nodeid, date_stamp, user, action, params in journal:
1270 params = eval(params)
1271 if isinstance(params, type({})):
1272 for param, value in params.iteritems():
1273 if not value:
1274 continue
1275 property = properties.get(param, None)
1276 if property is None:
1277 # deleted property
1278 continue
1279 cvt = self.to_hyperdb_value(property.__class__)
1280 if isinstance(property, Password):
1281 params[param] = cvt(value)
1282 elif isinstance(property, Date):
1283 params[param] = cvt(value)
1284 elif isinstance(property, Interval):
1285 params[param] = cvt(value)
1286 elif isinstance(property, Boolean):
1287 params[param] = cvt(value)
1288 # XXX numeric ids
1289 res.append((str(nodeid), dc(date_stamp), user, action, params))
1290 return res
1292 def save_journal(self, classname, cols, nodeid, journaldate,
1293 journaltag, action, params):
1294 """ Save the journal entry to the database
1295 """
1296 entry = (nodeid, journaldate, journaltag, action, params)
1298 # do the insert
1299 a = self.arg
1300 sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(
1301 classname, cols, a, a, a, a, a)
1302 self.sql(sql, entry)
1304 def load_journal(self, classname, cols, nodeid):
1305 """ Load the journal from the database
1306 """
1307 # now get the journal entries
1308 sql = 'select %s from %s__journal where nodeid=%s order by date'%(
1309 cols, classname, self.arg)
1310 self.sql(sql, (nodeid,))
1311 return self.cursor.fetchall()
1313 def pack(self, pack_before):
1314 """ Delete all journal entries except "create" before 'pack_before'.
1315 """
1316 date_stamp = self.to_sql_value(Date)(pack_before)
1318 # do the delete
1319 for classname in self.classes:
1320 sql = "delete from %s__journal where date<%s and "\
1321 "action<>'create'"%(classname, self.arg)
1322 self.sql(sql, (date_stamp,))
1324 def sql_commit(self, fail_ok=False):
1325 """ Actually commit to the database.
1326 """
1327 logging.getLogger('roundup.hyperdb').info('commit')
1329 self.conn.commit()
1331 # open a new cursor for subsequent work
1332 self.cursor = self.conn.cursor()
1334 def commit(self, fail_ok=False):
1335 """ Commit the current transactions.
1337 Save all data changed since the database was opened or since the
1338 last commit() or rollback().
1340 fail_ok indicates that the commit is allowed to fail. This is used
1341 in the web interface when committing cleaning of the session
1342 database. We don't care if there's a concurrency issue there.
1344 The only backend this seems to affect is postgres.
1345 """
1346 # commit the database
1347 self.sql_commit(fail_ok)
1349 # now, do all the other transaction stuff
1350 for method, args in self.transactions:
1351 method(*args)
1353 # save the indexer
1354 self.indexer.save_index()
1356 # clear out the transactions
1357 self.transactions = []
1359 # clear the cache: Don't carry over cached values from one
1360 # transaction to the next (there may be other changes from other
1361 # transactions)
1362 self.clearCache()
1364 def sql_rollback(self):
1365 self.conn.rollback()
1367 def rollback(self):
1368 """ Reverse all actions from the current transaction.
1370 Undo all the changes made since the database was opened or the last
1371 commit() or rollback() was performed.
1372 """
1373 logging.getLogger('roundup.hyperdb').info('rollback')
1375 self.sql_rollback()
1377 # roll back "other" transaction stuff
1378 for method, args in self.transactions:
1379 # delete temporary files
1380 if method == self.doStoreFile:
1381 self.rollbackStoreFile(*args)
1382 self.transactions = []
1384 # clear the cache
1385 self.clearCache()
1387 def sql_close(self):
1388 logging.getLogger('roundup.hyperdb').info('close')
1389 self.conn.close()
1391 def close(self):
1392 """ Close off the connection.
1393 """
1394 self.indexer.close()
1395 self.sql_close()
1397 #
1398 # The base Class class
1399 #
1400 class Class(hyperdb.Class):
1401 """ The handle to a particular class of nodes in a hyperdatabase.
1403 All methods except __repr__ and getnode must be implemented by a
1404 concrete backend Class.
1405 """
1407 def schema(self):
1408 """ A dumpable version of the schema that we can store in the
1409 database
1410 """
1411 return (self.key, [(x, repr(y)) for x,y in self.properties.iteritems()])
1413 def enableJournalling(self):
1414 """Turn journalling on for this class
1415 """
1416 self.do_journal = 1
1418 def disableJournalling(self):
1419 """Turn journalling off for this class
1420 """
1421 self.do_journal = 0
1423 # Editing nodes:
1424 def create(self, **propvalues):
1425 """ Create a new node of this class and return its id.
1427 The keyword arguments in 'propvalues' map property names to values.
1429 The values of arguments must be acceptable for the types of their
1430 corresponding properties or a TypeError is raised.
1432 If this class has a key property, it must be present and its value
1433 must not collide with other key strings or a ValueError is raised.
1435 Any other properties on this class that are missing from the
1436 'propvalues' dictionary are set to None.
1438 If an id in a link or multilink property does not refer to a valid
1439 node, an IndexError is raised.
1440 """
1441 self.fireAuditors('create', None, propvalues)
1442 newid = self.create_inner(**propvalues)
1443 self.fireReactors('create', newid, None)
1444 return newid
1446 def create_inner(self, **propvalues):
1447 """ Called by create, in-between the audit and react calls.
1448 """
1449 if 'id' in propvalues:
1450 raise KeyError('"id" is reserved')
1452 if self.db.journaltag is None:
1453 raise DatabaseError(_('Database open read-only'))
1455 if ('creator' in propvalues or 'actor' in propvalues or
1456 'creation' in propvalues or 'activity' in propvalues):
1457 raise KeyError('"creator", "actor", "creation" and '
1458 '"activity" are reserved')
1460 # new node's id
1461 newid = self.db.newid(self.classname)
1463 # validate propvalues
1464 num_re = re.compile('^\d+$')
1465 for key, value in propvalues.iteritems():
1466 if key == self.key:
1467 try:
1468 self.lookup(value)
1469 except KeyError:
1470 pass
1471 else:
1472 raise ValueError('node with key "%s" exists'%value)
1474 # try to handle this property
1475 try:
1476 prop = self.properties[key]
1477 except KeyError:
1478 raise KeyError('"%s" has no property "%s"'%(self.classname,
1479 key))
1481 if value is not None and isinstance(prop, Link):
1482 if type(value) != type(''):
1483 raise ValueError('link value must be String')
1484 link_class = self.properties[key].classname
1485 # if it isn't a number, it's a key
1486 if not num_re.match(value):
1487 try:
1488 value = self.db.classes[link_class].lookup(value)
1489 except (TypeError, KeyError):
1490 raise IndexError('new property "%s": %s not a %s'%(
1491 key, value, link_class))
1492 elif not self.db.getclass(link_class).hasnode(value):
1493 raise IndexError('%s has no node %s'%(link_class,
1494 value))
1496 # save off the value
1497 propvalues[key] = value
1499 # register the link with the newly linked node
1500 if self.do_journal and self.properties[key].do_journal:
1501 self.db.addjournal(link_class, value, 'link',
1502 (self.classname, newid, key))
1504 elif isinstance(prop, Multilink):
1505 if value is None:
1506 value = []
1507 if not hasattr(value, '__iter__'):
1508 raise TypeError('new property "%s" not an iterable of ids'%key)
1509 # clean up and validate the list of links
1510 link_class = self.properties[key].classname
1511 l = []
1512 for entry in value:
1513 if type(entry) != type(''):
1514 raise ValueError('"%s" multilink value (%r) '
1515 'must contain Strings'%(key, value))
1516 # if it isn't a number, it's a key
1517 if not num_re.match(entry):
1518 try:
1519 entry = self.db.classes[link_class].lookup(entry)
1520 except (TypeError, KeyError):
1521 raise IndexError('new property "%s": %s not a %s'%(
1522 key, entry, self.properties[key].classname))
1523 l.append(entry)
1524 value = l
1525 propvalues[key] = value
1527 # handle additions
1528 for nodeid in value:
1529 if not self.db.getclass(link_class).hasnode(nodeid):
1530 raise IndexError('%s has no node %s'%(link_class,
1531 nodeid))
1532 # register the link with the newly linked node
1533 if self.do_journal and self.properties[key].do_journal:
1534 self.db.addjournal(link_class, nodeid, 'link',
1535 (self.classname, newid, key))
1537 elif isinstance(prop, String):
1538 if type(value) != type('') and type(value) != type(u''):
1539 raise TypeError('new property "%s" not a string'%key)
1540 if prop.indexme:
1541 self.db.indexer.add_text((self.classname, newid, key),
1542 value)
1544 elif isinstance(prop, Password):
1545 if not isinstance(value, password.Password):
1546 raise TypeError('new property "%s" not a Password'%key)
1548 elif isinstance(prop, Date):
1549 if value is not None and not isinstance(value, date.Date):
1550 raise TypeError('new property "%s" not a Date'%key)
1552 elif isinstance(prop, Interval):
1553 if value is not None and not isinstance(value, date.Interval):
1554 raise TypeError('new property "%s" not an Interval'%key)
1556 elif value is not None and isinstance(prop, Number):
1557 try:
1558 float(value)
1559 except ValueError:
1560 raise TypeError('new property "%s" not numeric'%key)
1562 elif value is not None and isinstance(prop, Boolean):
1563 try:
1564 int(value)
1565 except ValueError:
1566 raise TypeError('new property "%s" not boolean'%key)
1568 # make sure there's data where there needs to be
1569 for key, prop in self.properties.iteritems():
1570 if key in propvalues:
1571 continue
1572 if key == self.key:
1573 raise ValueError('key property "%s" is required'%key)
1574 if isinstance(prop, Multilink):
1575 propvalues[key] = []
1576 else:
1577 propvalues[key] = None
1579 # done
1580 self.db.addnode(self.classname, newid, propvalues)
1581 if self.do_journal:
1582 self.db.addjournal(self.classname, newid, ''"create", {})
1584 # XXX numeric ids
1585 return str(newid)
1587 def get(self, nodeid, propname, default=_marker, cache=1):
1588 """Get the value of a property on an existing node of this class.
1590 'nodeid' must be the id of an existing node of this class or an
1591 IndexError is raised. 'propname' must be the name of a property
1592 of this class or a KeyError is raised.
1594 'cache' exists for backwards compatibility, and is not used.
1595 """
1596 if propname == 'id':
1597 return nodeid
1599 # get the node's dict
1600 d = self.db.getnode(self.classname, nodeid)
1602 if propname == 'creation':
1603 if 'creation' in d:
1604 return d['creation']
1605 else:
1606 return date.Date()
1607 if propname == 'activity':
1608 if 'activity' in d:
1609 return d['activity']
1610 else:
1611 return date.Date()
1612 if propname == 'creator':
1613 if 'creator' in d:
1614 return d['creator']
1615 else:
1616 return self.db.getuid()
1617 if propname == 'actor':
1618 if 'actor' in d:
1619 return d['actor']
1620 else:
1621 return self.db.getuid()
1623 # get the property (raises KeyError if invalid)
1624 prop = self.properties[propname]
1626 # lazy evaluation of Multilink
1627 if propname not in d and isinstance(prop, Multilink):
1628 sql = 'select linkid from %s_%s where nodeid=%s'%(self.classname,
1629 propname, self.db.arg)
1630 self.db.sql(sql, (nodeid,))
1631 # extract the first column from the result
1632 # XXX numeric ids
1633 items = [int(x[0]) for x in self.db.cursor.fetchall()]
1634 items.sort ()
1635 d[propname] = [str(x) for x in items]
1637 # handle there being no value in the table for the property
1638 if propname not in d or d[propname] is None:
1639 if default is _marker:
1640 if isinstance(prop, Multilink):
1641 return []
1642 else:
1643 return None
1644 else:
1645 return default
1647 # don't pass our list to other code
1648 if isinstance(prop, Multilink):
1649 return d[propname][:]
1651 return d[propname]
1653 def set(self, nodeid, **propvalues):
1654 """Modify a property on an existing node of this class.
1656 'nodeid' must be the id of an existing node of this class or an
1657 IndexError is raised.
1659 Each key in 'propvalues' must be the name of a property of this
1660 class or a KeyError is raised.
1662 All values in 'propvalues' must be acceptable types for their
1663 corresponding properties or a TypeError is raised.
1665 If the value of the key property is set, it must not collide with
1666 other key strings or a ValueError is raised.
1668 If the value of a Link or Multilink property contains an invalid
1669 node id, a ValueError is raised.
1670 """
1671 self.fireAuditors('set', nodeid, propvalues)
1672 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1673 propvalues = self.set_inner(nodeid, **propvalues)
1674 self.fireReactors('set', nodeid, oldvalues)
1675 return propvalues
1677 def set_inner(self, nodeid, **propvalues):
1678 """ Called by set, in-between the audit and react calls.
1679 """
1680 if not propvalues:
1681 return propvalues
1683 if ('creator' in propvalues or 'actor' in propvalues or
1684 'creation' in propvalues or 'activity' in propvalues):
1685 raise KeyError('"creator", "actor", "creation" and '
1686 '"activity" are reserved')
1688 if 'id' in propvalues:
1689 raise KeyError('"id" is reserved')
1691 if self.db.journaltag is None:
1692 raise DatabaseError(_('Database open read-only'))
1694 node = self.db.getnode(self.classname, nodeid)
1695 if self.is_retired(nodeid):
1696 raise IndexError('Requested item is retired')
1697 num_re = re.compile('^\d+$')
1699 # make a copy of the values dictionary - we'll modify the contents
1700 propvalues = propvalues.copy()
1702 # if the journal value is to be different, store it in here
1703 journalvalues = {}
1705 # remember the add/remove stuff for multilinks, making it easier
1706 # for the Database layer to do its stuff
1707 multilink_changes = {}
1709 for propname, value in list(propvalues.items()):
1710 # check to make sure we're not duplicating an existing key
1711 if propname == self.key and node[propname] != value:
1712 try:
1713 self.lookup(value)
1714 except KeyError:
1715 pass
1716 else:
1717 raise ValueError('node with key "%s" exists'%value)
1719 # this will raise the KeyError if the property isn't valid
1720 # ... we don't use getprops() here because we only care about
1721 # the writeable properties.
1722 try:
1723 prop = self.properties[propname]
1724 except KeyError:
1725 raise KeyError('"%s" has no property named "%s"'%(
1726 self.classname, propname))
1728 # if the value's the same as the existing value, no sense in
1729 # doing anything
1730 current = node.get(propname, None)
1731 if value == current:
1732 del propvalues[propname]
1733 continue
1734 journalvalues[propname] = current
1736 # do stuff based on the prop type
1737 if isinstance(prop, Link):
1738 link_class = prop.classname
1739 # if it isn't a number, it's a key
1740 if value is not None and not isinstance(value, type('')):
1741 raise ValueError('property "%s" link value be a string'%(
1742 propname))
1743 if isinstance(value, type('')) and not num_re.match(value):
1744 try:
1745 value = self.db.classes[link_class].lookup(value)
1746 except (TypeError, KeyError):
1747 raise IndexError('new property "%s": %s not a %s'%(
1748 propname, value, prop.classname))
1750 if (value is not None and
1751 not self.db.getclass(link_class).hasnode(value)):
1752 raise IndexError('%s has no node %s'%(link_class,
1753 value))
1755 if self.do_journal and prop.do_journal:
1756 # register the unlink with the old linked node
1757 if node[propname] is not None:
1758 self.db.addjournal(link_class, node[propname],
1759 ''"unlink", (self.classname, nodeid, propname))
1761 # register the link with the newly linked node
1762 if value is not None:
1763 self.db.addjournal(link_class, value, ''"link",
1764 (self.classname, nodeid, propname))
1766 elif isinstance(prop, Multilink):
1767 if value is None:
1768 value = []
1769 if not hasattr(value, '__iter__'):
1770 raise TypeError('new property "%s" not an iterable of'
1771 ' ids'%propname)
1772 link_class = self.properties[propname].classname
1773 l = []
1774 for entry in value:
1775 # if it isn't a number, it's a key
1776 if type(entry) != type(''):
1777 raise ValueError('new property "%s" link value '
1778 'must be a string'%propname)
1779 if not num_re.match(entry):
1780 try:
1781 entry = self.db.classes[link_class].lookup(entry)
1782 except (TypeError, KeyError):
1783 raise IndexError('new property "%s": %s not a %s'%(
1784 propname, entry,
1785 self.properties[propname].classname))
1786 l.append(entry)
1787 value = l
1788 propvalues[propname] = value
1790 # figure the journal entry for this property
1791 add = []
1792 remove = []
1794 # handle removals
1795 if propname in node:
1796 l = node[propname]
1797 else:
1798 l = []
1799 for id in l[:]:
1800 if id in value:
1801 continue
1802 # register the unlink with the old linked node
1803 if self.do_journal and self.properties[propname].do_journal:
1804 self.db.addjournal(link_class, id, 'unlink',
1805 (self.classname, nodeid, propname))
1806 l.remove(id)
1807 remove.append(id)
1809 # handle additions
1810 for id in value:
1811 if id in l:
1812 continue
1813 # We can safely check this condition after
1814 # checking that this is an addition to the
1815 # multilink since the condition was checked for
1816 # existing entries at the point they were added to
1817 # the multilink. Since the hasnode call will
1818 # result in a SQL query, it is more efficient to
1819 # avoid the check if possible.
1820 if not self.db.getclass(link_class).hasnode(id):
1821 raise IndexError('%s has no node %s'%(link_class,
1822 id))
1823 # register the link with the newly linked node
1824 if self.do_journal and self.properties[propname].do_journal:
1825 self.db.addjournal(link_class, id, 'link',
1826 (self.classname, nodeid, propname))
1827 l.append(id)
1828 add.append(id)
1830 # figure the journal entry
1831 l = []
1832 if add:
1833 l.append(('+', add))
1834 if remove:
1835 l.append(('-', remove))
1836 multilink_changes[propname] = (add, remove)
1837 if l:
1838 journalvalues[propname] = tuple(l)
1840 elif isinstance(prop, String):
1841 if value is not None and type(value) != type('') and type(value) != type(u''):
1842 raise TypeError('new property "%s" not a string'%propname)
1843 if prop.indexme:
1844 if value is None: value = ''
1845 self.db.indexer.add_text((self.classname, nodeid, propname),
1846 value)
1848 elif isinstance(prop, Password):
1849 if not isinstance(value, password.Password):
1850 raise TypeError('new property "%s" not a Password'%propname)
1851 propvalues[propname] = value
1853 elif value is not None and isinstance(prop, Date):
1854 if not isinstance(value, date.Date):
1855 raise TypeError('new property "%s" not a Date'% propname)
1856 propvalues[propname] = value
1858 elif value is not None and isinstance(prop, Interval):
1859 if not isinstance(value, date.Interval):
1860 raise TypeError('new property "%s" not an '
1861 'Interval'%propname)
1862 propvalues[propname] = value
1864 elif value is not None and isinstance(prop, Number):
1865 try:
1866 float(value)
1867 except ValueError:
1868 raise TypeError('new property "%s" not numeric'%propname)
1870 elif value is not None and isinstance(prop, Boolean):
1871 try:
1872 int(value)
1873 except ValueError:
1874 raise TypeError('new property "%s" not boolean'%propname)
1876 # nothing to do?
1877 if not propvalues:
1878 return propvalues
1880 # update the activity time
1881 propvalues['activity'] = date.Date()
1882 propvalues['actor'] = self.db.getuid()
1884 # do the set
1885 self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
1887 # remove the activity props now they're handled
1888 del propvalues['activity']
1889 del propvalues['actor']
1891 # journal the set
1892 if self.do_journal:
1893 self.db.addjournal(self.classname, nodeid, ''"set", journalvalues)
1895 return propvalues
1897 def retire(self, nodeid):
1898 """Retire a node.
1900 The properties on the node remain available from the get() method,
1901 and the node's id is never reused.
1903 Retired nodes are not returned by the find(), list(), or lookup()
1904 methods, and other nodes may reuse the values of their key properties.
1905 """
1906 if self.db.journaltag is None:
1907 raise DatabaseError(_('Database open read-only'))
1909 self.fireAuditors('retire', nodeid, None)
1911 # use the arg for __retired__ to cope with any odd database type
1912 # conversion (hello, sqlite)
1913 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1914 self.db.arg, self.db.arg)
1915 self.db.sql(sql, (nodeid, nodeid))
1916 if self.do_journal:
1917 self.db.addjournal(self.classname, nodeid, ''"retired", None)
1919 self.fireReactors('retire', nodeid, None)
1921 def restore(self, nodeid):
1922 """Restore a retired node.
1924 Make node available for all operations like it was before retirement.
1925 """
1926 if self.db.journaltag is None:
1927 raise DatabaseError(_('Database open read-only'))
1929 node = self.db.getnode(self.classname, nodeid)
1930 # check if key property was overrided
1931 key = self.getkey()
1932 try:
1933 id = self.lookup(node[key])
1934 except KeyError:
1935 pass
1936 else:
1937 raise KeyError("Key property (%s) of retired node clashes "
1938 "with existing one (%s)" % (key, node[key]))
1940 self.fireAuditors('restore', nodeid, None)
1941 # use the arg for __retired__ to cope with any odd database type
1942 # conversion (hello, sqlite)
1943 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1944 self.db.arg, self.db.arg)
1945 self.db.sql(sql, (0, nodeid))
1946 if self.do_journal:
1947 self.db.addjournal(self.classname, nodeid, ''"restored", None)
1949 self.fireReactors('restore', nodeid, None)
1951 def is_retired(self, nodeid):
1952 """Return true if the node is rerired
1953 """
1954 sql = 'select __retired__ from _%s where id=%s'%(self.classname,
1955 self.db.arg)
1956 self.db.sql(sql, (nodeid,))
1957 return int(self.db.sql_fetchone()[0]) > 0
1959 def destroy(self, nodeid):
1960 """Destroy a node.
1962 WARNING: this method should never be used except in extremely rare
1963 situations where there could never be links to the node being
1964 deleted
1966 WARNING: use retire() instead
1968 WARNING: the properties of this node will not be available ever again
1970 WARNING: really, use retire() instead
1972 Well, I think that's enough warnings. This method exists mostly to
1973 support the session storage of the cgi interface.
1975 The node is completely removed from the hyperdb, including all journal
1976 entries. It will no longer be available, and will generally break code
1977 if there are any references to the node.
1978 """
1979 if self.db.journaltag is None:
1980 raise DatabaseError(_('Database open read-only'))
1981 self.db.destroynode(self.classname, nodeid)
1983 def history(self, nodeid):
1984 """Retrieve the journal of edits on a particular node.
1986 'nodeid' must be the id of an existing node of this class or an
1987 IndexError is raised.
1989 The returned list contains tuples of the form
1991 (nodeid, date, tag, action, params)
1993 'date' is a Timestamp object specifying the time of the change and
1994 'tag' is the journaltag specified when the database was opened.
1995 """
1996 if not self.do_journal:
1997 raise ValueError('Journalling is disabled for this class')
1998 return self.db.getjournal(self.classname, nodeid)
2000 # Locating nodes:
2001 def hasnode(self, nodeid):
2002 """Determine if the given nodeid actually exists
2003 """
2004 return self.db.hasnode(self.classname, nodeid)
2006 def setkey(self, propname):
2007 """Select a String property of this class to be the key property.
2009 'propname' must be the name of a String property of this class or
2010 None, or a TypeError is raised. The values of the key property on
2011 all existing nodes must be unique or a ValueError is raised.
2012 """
2013 prop = self.getprops()[propname]
2014 if not isinstance(prop, String):
2015 raise TypeError('key properties must be String')
2016 self.key = propname
2018 def getkey(self):
2019 """Return the name of the key property for this class or None."""
2020 return self.key
2022 def lookup(self, keyvalue):
2023 """Locate a particular node by its key property and return its id.
2025 If this class has no key property, a TypeError is raised. If the
2026 'keyvalue' matches one of the values for the key property among
2027 the nodes in this class, the matching node's id is returned;
2028 otherwise a KeyError is raised.
2029 """
2030 if not self.key:
2031 raise TypeError('No key property set for class %s'%self.classname)
2033 # use the arg to handle any odd database type conversion (hello,
2034 # sqlite)
2035 sql = "select id from _%s where _%s=%s and __retired__=%s"%(
2036 self.classname, self.key, self.db.arg, self.db.arg)
2037 self.db.sql(sql, (str(keyvalue), 0))
2039 # see if there was a result that's not retired
2040 row = self.db.sql_fetchone()
2041 if not row:
2042 raise KeyError('No key (%s) value "%s" for "%s"'%(self.key,
2043 keyvalue, self.classname))
2045 # return the id
2046 # XXX numeric ids
2047 return str(row[0])
2049 def find(self, **propspec):
2050 """Get the ids of nodes in this class which link to the given nodes.
2052 'propspec' consists of keyword args propname=nodeid or
2053 propname={nodeid:1, }
2054 'propname' must be the name of a property in this class, or a
2055 KeyError is raised. That property must be a Link or
2056 Multilink property, or a TypeError is raised.
2058 Any node in this class whose 'propname' property links to any of
2059 the nodeids will be returned. Examples::
2061 db.issue.find(messages='1')
2062 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
2063 """
2064 # shortcut
2065 if not propspec:
2066 return []
2068 # validate the args
2069 props = self.getprops()
2070 for propname, nodeids in propspec.iteritems():
2071 # check the prop is OK
2072 prop = props[propname]
2073 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
2074 raise TypeError("'%s' not a Link/Multilink property"%propname)
2076 # first, links
2077 a = self.db.arg
2078 allvalues = ()
2079 sql = []
2080 where = []
2081 for prop, values in propspec.iteritems():
2082 if not isinstance(props[prop], hyperdb.Link):
2083 continue
2084 if type(values) is type({}) and len(values) == 1:
2085 values = list(values)[0]
2086 if type(values) is type(''):
2087 allvalues += (values,)
2088 where.append('_%s = %s'%(prop, a))
2089 elif values is None:
2090 where.append('_%s is NULL'%prop)
2091 else:
2092 values = list(values)
2093 s = ''
2094 if None in values:
2095 values.remove(None)
2096 s = '_%s is NULL or '%prop
2097 allvalues += tuple(values)
2098 s += '_%s in (%s)'%(prop, ','.join([a]*len(values)))
2099 where.append('(' + s +')')
2100 if where:
2101 allvalues = (0, ) + allvalues
2102 sql.append("""select id from _%s where __retired__=%s
2103 and %s"""%(self.classname, a, ' and '.join(where)))
2105 # now multilinks
2106 for prop, values in propspec.iteritems():
2107 if not isinstance(props[prop], hyperdb.Multilink):
2108 continue
2109 if not values:
2110 continue
2111 allvalues += (0, )
2112 if type(values) is type(''):
2113 allvalues += (values,)
2114 s = a
2115 else:
2116 allvalues += tuple(values)
2117 s = ','.join([a]*len(values))
2118 tn = '%s_%s'%(self.classname, prop)
2119 sql.append("""select id from _%s, %s where __retired__=%s
2120 and id = %s.nodeid and %s.linkid in (%s)"""%(self.classname,
2121 tn, a, tn, tn, s))
2123 if not sql:
2124 return []
2125 sql = ' union '.join(sql)
2126 self.db.sql(sql, allvalues)
2127 # XXX numeric ids
2128 l = [str(x[0]) for x in self.db.sql_fetchall()]
2129 return l
2131 def stringFind(self, **requirements):
2132 """Locate a particular node by matching a set of its String
2133 properties in a caseless search.
2135 If the property is not a String property, a TypeError is raised.
2137 The return is a list of the id of all nodes that match.
2138 """
2139 where = []
2140 args = []
2141 for propname in requirements:
2142 prop = self.properties[propname]
2143 if not isinstance(prop, String):
2144 raise TypeError("'%s' not a String property"%propname)
2145 where.append(propname)
2146 args.append(requirements[propname].lower())
2148 # generate the where clause
2149 s = ' and '.join(['lower(_%s)=%s'%(col, self.db.arg) for col in where])
2150 sql = 'select id from _%s where %s and __retired__=%s'%(
2151 self.classname, s, self.db.arg)
2152 args.append(0)
2153 self.db.sql(sql, tuple(args))
2154 # XXX numeric ids
2155 l = [str(x[0]) for x in self.db.sql_fetchall()]
2156 return l
2158 def list(self):
2159 """ Return a list of the ids of the active nodes in this class.
2160 """
2161 return self.getnodeids(retired=0)
2163 def getnodeids(self, retired=None):
2164 """ Retrieve all the ids of the nodes for a particular Class.
2166 Set retired=None to get all nodes. Otherwise it'll get all the
2167 retired or non-retired nodes, depending on the flag.
2168 """
2169 # flip the sense of the 'retired' flag if we don't want all of them
2170 if retired is not None:
2171 args = (0, )
2172 if retired:
2173 compare = '>'
2174 else:
2175 compare = '='
2176 sql = 'select id from _%s where __retired__%s%s'%(self.classname,
2177 compare, self.db.arg)
2178 else:
2179 args = ()
2180 sql = 'select id from _%s'%self.classname
2181 self.db.sql(sql, args)
2182 # XXX numeric ids
2183 ids = [str(x[0]) for x in self.db.cursor.fetchall()]
2184 return ids
2186 def _subselect(self, classname, multilink_table):
2187 """Create a subselect. This is factored out because some
2188 databases (hmm only one, so far) doesn't support subselects
2189 look for "I can't believe it's not a toy RDBMS" in the mysql
2190 backend.
2191 """
2192 return '_%s.id not in (select nodeid from %s)'%(classname,
2193 multilink_table)
2195 # Some DBs order NULL values last. Set this variable in the backend
2196 # for prepending an order by clause for each attribute that causes
2197 # correct sort order for NULLs. Examples:
2198 # order_by_null_values = '(%s is not NULL)'
2199 # order_by_null_values = 'notnull(%s)'
2200 # The format parameter is replaced with the attribute.
2201 order_by_null_values = None
2203 def supports_subselects(self):
2204 '''Assuming DBs can do subselects, overwrite if they cannot.
2205 '''
2206 return True
2208 def _filter_multilink_expression_fallback(
2209 self, classname, multilink_table, expr):
2210 '''This is a fallback for database that do not support
2211 subselects.'''
2213 is_valid = expr.evaluate
2215 last_id, kws = None, []
2217 ids = IdListOptimizer()
2218 append = ids.append
2220 # This join and the evaluation in program space
2221 # can be expensive for larger databases!
2222 # TODO: Find a faster way to collect the data needed
2223 # to evalute the expression.
2224 # Moving the expression evaluation into the database
2225 # would be nice but this tricky: Think about the cases
2226 # where the multilink table does not have join values
2227 # needed in evaluation.
2229 stmnt = "SELECT c.id, m.linkid FROM _%s c " \
2230 "LEFT OUTER JOIN %s m " \
2231 "ON c.id = m.nodeid ORDER BY c.id" % (
2232 classname, multilink_table)
2233 self.db.sql(stmnt)
2235 # collect all multilink items for a class item
2236 for nid, kw in self.db.sql_fetchiter():
2237 if nid != last_id:
2238 if last_id is None:
2239 last_id = nid
2240 else:
2241 # we have all multilink items -> evaluate!
2242 if is_valid(kws): append(last_id)
2243 last_id, kws = nid, []
2244 if kw is not None:
2245 kws.append(kw)
2247 if last_id is not None and is_valid(kws):
2248 append(last_id)
2250 # we have ids of the classname table
2251 return ids.where("_%s.id" % classname, self.db.arg)
2253 def _filter_multilink_expression(self, classname, multilink_table, v):
2254 """ Filters out elements of the classname table that do not
2255 match the given expression.
2256 Returns tuple of 'WHERE' introns for the overall filter.
2257 """
2258 try:
2259 opcodes = [int(x) for x in v]
2260 if min(opcodes) >= -1: raise ValueError()
2262 expr = compile_expression(opcodes)
2264 if not self.supports_subselects():
2265 # We heavily rely on subselects. If there is
2266 # no decent support fall back to slower variant.
2267 return self._filter_multilink_expression_fallback(
2268 classname, multilink_table, expr)
2270 atom = \
2271 "%s IN(SELECT linkid FROM %s WHERE nodeid=a.id)" % (
2272 self.db.arg,
2273 multilink_table)
2275 intron = \
2276 "_%(classname)s.id in (SELECT id " \
2277 "FROM _%(classname)s AS a WHERE %(condition)s) " % {
2278 'classname' : classname,
2279 'condition' : expr.generate(lambda n: atom) }
2281 values = []
2282 def collect_values(n): values.append(n.x)
2283 expr.visit(collect_values)
2285 return intron, values
2286 except:
2287 # original behavior
2288 where = "%s.linkid in (%s)" % (
2289 multilink_table, ','.join([self.db.arg] * len(v)))
2290 return where, v, True # True to indicate original
2292 def _filter_sql (self, search_matches, filterspec, srt=[], grp=[], retr=0):
2293 """ Compute the proptree and the SQL/ARGS for a filter.
2294 For argument description see filter below.
2295 We return a 3-tuple, the proptree, the sql and the sql-args
2296 or None if no SQL is necessary.
2297 The flag retr serves to retrieve *all* non-Multilink properties
2298 (for filling the cache during a filter_iter)
2299 """
2300 # we can't match anything if search_matches is empty
2301 if not search_matches and search_matches is not None:
2302 return None
2304 icn = self.classname
2306 # vars to hold the components of the SQL statement
2307 frum = [] # FROM clauses
2308 loj = [] # LEFT OUTER JOIN clauses
2309 where = [] # WHERE clauses
2310 args = [] # *any* positional arguments
2311 a = self.db.arg
2313 # figure the WHERE clause from the filterspec
2314 mlfilt = 0 # are we joining with Multilink tables?
2315 sortattr = self._sortattr (group = grp, sort = srt)
2316 proptree = self._proptree(filterspec, sortattr, retr)
2317 mlseen = 0
2318 for pt in reversed(proptree.sortattr):
2319 p = pt
2320 while p.parent:
2321 if isinstance (p.propclass, Multilink):
2322 mlseen = True
2323 if mlseen:
2324 p.sort_ids_needed = True
2325 p.tree_sort_done = False
2326 p = p.parent
2327 if not mlseen:
2328 pt.attr_sort_done = pt.tree_sort_done = True
2329 proptree.compute_sort_done()
2331 cols = ['_%s.id'%icn]
2332 mlsort = []
2333 rhsnum = 0
2334 for p in proptree:
2335 rc = ac = oc = None
2336 cn = p.classname
2337 ln = p.uniqname
2338 pln = p.parent.uniqname
2339 pcn = p.parent.classname
2340 k = p.name
2341 v = p.val
2342 propclass = p.propclass
2343 if p.parent == proptree and p.name == 'id' \
2344 and 'retrieve' in p.need_for:
2345 p.sql_idx = 0
2346 if 'sort' in p.need_for or 'retrieve' in p.need_for:
2347 rc = oc = ac = '_%s._%s'%(pln, k)
2348 if isinstance(propclass, Multilink):
2349 if 'search' in p.need_for:
2350 mlfilt = 1
2351 tn = '%s_%s'%(pcn, k)
2352 if v in ('-1', ['-1'], []):
2353 # only match rows that have count(linkid)=0 in the
2354 # corresponding multilink table)
2355 where.append(self._subselect(pcn, tn))
2356 else:
2357 frum.append(tn)
2358 gen_join = True
2360 if p.has_values and isinstance(v, type([])):
2361 result = self._filter_multilink_expression(pln, tn, v)
2362 # XXX: We dont need an id join if we used the filter
2363 gen_join = len(result) == 3
2365 if gen_join:
2366 where.append('_%s.id=%s.nodeid'%(pln,tn))
2368 if p.children:
2369 frum.append('_%s as _%s' % (cn, ln))
2370 where.append('%s.linkid=_%s.id'%(tn, ln))
2372 if p.has_values:
2373 if isinstance(v, type([])):
2374 where.append(result[0])
2375 args += result[1]
2376 else:
2377 where.append('%s.linkid=%s'%(tn, a))
2378 args.append(v)
2379 if 'sort' in p.need_for:
2380 assert not p.attr_sort_done and not p.sort_ids_needed
2381 elif k == 'id':
2382 if 'search' in p.need_for:
2383 if isinstance(v, type([])):
2384 # If there are no permitted values, then the
2385 # where clause will always be false, and we
2386 # can optimize the query away.
2387 if not v:
2388 return []
2389 s = ','.join([a for x in v])
2390 where.append('_%s.%s in (%s)'%(pln, k, s))
2391 args = args + v
2392 else:
2393 where.append('_%s.%s=%s'%(pln, k, a))
2394 args.append(v)
2395 if 'sort' in p.need_for or 'retrieve' in p.need_for:
2396 rc = oc = ac = '_%s.id'%pln
2397 elif isinstance(propclass, String):
2398 if 'search' in p.need_for:
2399 if not isinstance(v, type([])):
2400 v = [v]
2402 # Quote the bits in the string that need it and then embed
2403 # in a "substring" search. Note - need to quote the '%' so
2404 # they make it through the python layer happily
2405 v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
2407 # now add to the where clause
2408 where.append('('
2409 +' and '.join(["_%s._%s LIKE '%s'"%(pln, k, s) for s in v])
2410 +')')
2411 # note: args are embedded in the query string now
2412 if 'sort' in p.need_for:
2413 oc = ac = 'lower(_%s._%s)'%(pln, k)
2414 elif isinstance(propclass, Link):
2415 if 'search' in p.need_for:
2416 if p.children:
2417 if 'sort' not in p.need_for:
2418 frum.append('_%s as _%s' % (cn, ln))
2419 where.append('_%s._%s=_%s.id'%(pln, k, ln))
2420 if p.has_values:
2421 if isinstance(v, type([])):
2422 d = {}
2423 for entry in v:
2424 if entry == '-1':
2425 entry = None
2426 d[entry] = entry
2427 l = []
2428 if None in d or not d:
2429 if None in d: del d[None]
2430 l.append('_%s._%s is NULL'%(pln, k))
2431 if d:
2432 v = list(d)
2433 s = ','.join([a for x in v])
2434 l.append('(_%s._%s in (%s))'%(pln, k, s))
2435 args = args + v
2436 if l:
2437 where.append('(' + ' or '.join(l) +')')
2438 else:
2439 if v in ('-1', None):
2440 v = None
2441 where.append('_%s._%s is NULL'%(pln, k))
2442 else:
2443 where.append('_%s._%s=%s'%(pln, k, a))
2444 args.append(v)
2445 if 'sort' in p.need_for:
2446 lp = p.cls.labelprop()
2447 oc = ac = '_%s._%s'%(pln, k)
2448 if lp != 'id':
2449 if p.tree_sort_done:
2450 loj.append(
2451 'LEFT OUTER JOIN _%s as _%s on _%s._%s=_%s.id'%(
2452 cn, ln, pln, k, ln))
2453 oc = '_%s._%s'%(ln, lp)
2454 if 'retrieve' in p.need_for:
2455 rc = '_%s._%s'%(pln, k)
2456 elif isinstance(propclass, Date) and 'search' in p.need_for:
2457 dc = self.db.to_sql_value(hyperdb.Date)
2458 if isinstance(v, type([])):
2459 s = ','.join([a for x in v])
2460 where.append('_%s._%s in (%s)'%(pln, k, s))
2461 args = args + [dc(date.Date(x)) for x in v]
2462 else:
2463 try:
2464 # Try to filter on range of dates
2465 date_rng = propclass.range_from_raw(v, self.db)
2466 if date_rng.from_value:
2467 where.append('_%s._%s >= %s'%(pln, k, a))
2468 args.append(dc(date_rng.from_value))
2469 if date_rng.to_value:
2470 where.append('_%s._%s <= %s'%(pln, k, a))
2471 args.append(dc(date_rng.to_value))
2472 except ValueError:
2473 # If range creation fails - ignore that search parameter
2474 pass
2475 elif isinstance(propclass, Interval):
2476 # filter/sort using the __<prop>_int__ column
2477 if 'search' in p.need_for:
2478 if isinstance(v, type([])):
2479 s = ','.join([a for x in v])
2480 where.append('_%s.__%s_int__ in (%s)'%(pln, k, s))
2481 args = args + [date.Interval(x).as_seconds() for x in v]
2482 else:
2483 try:
2484 # Try to filter on range of intervals
2485 date_rng = Range(v, date.Interval)
2486 if date_rng.from_value:
2487 where.append('_%s.__%s_int__ >= %s'%(pln, k, a))
2488 args.append(date_rng.from_value.as_seconds())
2489 if date_rng.to_value:
2490 where.append('_%s.__%s_int__ <= %s'%(pln, k, a))
2491 args.append(date_rng.to_value.as_seconds())
2492 except ValueError:
2493 # If range creation fails - ignore search parameter
2494 pass
2495 if 'sort' in p.need_for:
2496 oc = ac = '_%s.__%s_int__'%(pln,k)
2497 if 'retrieve' in p.need_for:
2498 rc = '_%s._%s'%(pln,k)
2499 elif isinstance(propclass, Boolean) and 'search' in p.need_for:
2500 if type(v) == type(""):
2501 v = v.split(',')
2502 if type(v) != type([]):
2503 v = [v]
2504 bv = []
2505 for val in v:
2506 if type(val) is type(''):
2507 bv.append(propclass.from_raw (val))
2508 else:
2509 bv.append(bool(val))
2510 if len(bv) == 1:
2511 where.append('_%s._%s=%s'%(pln, k, a))
2512 args = args + bv
2513 else:
2514 s = ','.join([a for x in v])
2515 where.append('_%s._%s in (%s)'%(pln, k, s))
2516 args = args + bv
2517 elif 'search' in p.need_for:
2518 if isinstance(v, type([])):
2519 s = ','.join([a for x in v])
2520 where.append('_%s._%s in (%s)'%(pln, k, s))
2521 args = args + v
2522 else:
2523 where.append('_%s._%s=%s'%(pln, k, a))
2524 args.append(v)
2525 if oc:
2526 if p.sort_ids_needed:
2527 if rc == ac:
2528 p.sql_idx = len(cols)
2529 p.auxcol = len(cols)
2530 cols.append(ac)
2531 if p.tree_sort_done and p.sort_direction:
2532 # Don't select top-level id or multilink twice
2533 if (not p.sort_ids_needed or ac != oc) and (p.name != 'id'
2534 or p.parent != proptree):
2535 if rc == oc:
2536 p.sql_idx = len(cols)
2537 cols.append(oc)
2538 desc = ['', ' desc'][p.sort_direction == '-']
2539 # Some SQL dbs sort NULL values last -- we want them first.
2540 if (self.order_by_null_values and p.name != 'id'):
2541 nv = self.order_by_null_values % oc
2542 cols.append(nv)
2543 p.orderby.append(nv + desc)
2544 p.orderby.append(oc + desc)
2545 if 'retrieve' in p.need_for and p.sql_idx is None:
2546 assert(rc)
2547 p.sql_idx = len(cols)
2548 cols.append (rc)
2550 props = self.getprops()
2552 # don't match retired nodes
2553 where.append('_%s.__retired__=0'%icn)
2555 # add results of full text search
2556 if search_matches is not None:
2557 s = ','.join([a for x in search_matches])
2558 where.append('_%s.id in (%s)'%(icn, s))
2559 args = args + [x for x in search_matches]
2561 # construct the SQL
2562 frum.append('_'+icn)
2563 frum = ','.join(frum)
2564 if where:
2565 where = ' where ' + (' and '.join(where))
2566 else:
2567 where = ''
2568 if mlfilt:
2569 # we're joining tables on the id, so we will get dupes if we
2570 # don't distinct()
2571 cols[0] = 'distinct(_%s.id)'%icn
2573 order = []
2574 # keep correct sequence of order attributes.
2575 for sa in proptree.sortattr:
2576 if not sa.attr_sort_done:
2577 continue
2578 order.extend(sa.orderby)
2579 if order:
2580 order = ' order by %s'%(','.join(order))
2581 else:
2582 order = ''
2584 cols = ','.join(cols)
2585 loj = ' '.join(loj)
2586 sql = 'select %s from %s %s %s%s'%(cols, frum, loj, where, order)
2587 args = tuple(args)
2588 __traceback_info__ = (sql, args)
2589 return proptree, sql, args
2591 def filter(self, search_matches, filterspec, sort=[], group=[]):
2592 """Return a list of the ids of the active nodes in this class that
2593 match the 'filter' spec, sorted by the group spec and then the
2594 sort spec
2596 "filterspec" is {propname: value(s)}
2598 "sort" and "group" are [(dir, prop), ...] where dir is '+', '-'
2599 or None and prop is a prop name or None. Note that for
2600 backward-compatibility reasons a single (dir, prop) tuple is
2601 also allowed.
2603 "search_matches" is a container type or None
2605 The filter must match all properties specificed. If the property
2606 value to match is a list:
2608 1. String properties must match all elements in the list, and
2609 2. Other properties must match any of the elements in the list.
2610 """
2611 if __debug__:
2612 start_t = time.time()
2614 sq = self._filter_sql (search_matches, filterspec, sort, group)
2615 # nothing to match?
2616 if sq is None:
2617 return []
2618 proptree, sql, args = sq
2620 self.db.sql(sql, args)
2621 l = self.db.sql_fetchall()
2623 # Compute values needed for sorting in proptree.sort
2624 for p in proptree:
2625 if hasattr(p, 'auxcol'):
2626 p.sort_ids = p.sort_result = [row[p.auxcol] for row in l]
2627 # return the IDs (the first column)
2628 # XXX numeric ids
2629 l = [str(row[0]) for row in l]
2630 l = proptree.sort (l)
2632 if __debug__:
2633 self.db.stats['filtering'] += (time.time() - start_t)
2634 return l
2636 def filter_iter(self, search_matches, filterspec, sort=[], group=[]):
2637 """Iterator similar to filter above with same args.
2638 Limitation: We don't sort on multilinks.
2639 This uses an optimisation: We put all nodes that are in the
2640 current row into the node cache. Then we return the node id.
2641 That way a fetch of a node won't create another sql-fetch (with
2642 a join) from the database because the nodes are already in the
2643 cache. We're using our own temporary cursor.
2644 """
2645 sq = self._filter_sql(search_matches, filterspec, sort, group, retr=1)
2646 # nothing to match?
2647 if sq is None:
2648 return
2649 proptree, sql, args = sq
2650 cursor = self.db.conn.cursor()
2651 self.db.sql(sql, args, cursor)
2652 classes = {}
2653 for p in proptree:
2654 if 'retrieve' in p.need_for:
2655 cn = p.parent.classname
2656 ptid = p.parent.id # not the nodeid!
2657 key = (cn, ptid)
2658 if key not in classes:
2659 classes[key] = {}
2660 name = p.name
2661 assert (name)
2662 classes[key][name] = p
2663 while True:
2664 row = cursor.fetchone()
2665 if not row: break
2666 # populate cache with current items
2667 for (classname, ptid), pt in classes.iteritems():
2668 nodeid = str(row[pt['id'].sql_idx])
2669 key = (classname, nodeid)
2670 if key in self.db.cache:
2671 self.db._cache_refresh(key)
2672 continue
2673 node = {}
2674 for propname, p in pt.iteritems():
2675 value = row[p.sql_idx]
2676 if value is not None:
2677 cls = p.propclass.__class__
2678 value = self.db.to_hyperdb_value(cls)(value)
2679 node[propname] = value
2680 self.db._cache_save(key, node)
2681 yield str(row[0])
2683 def filter_sql(self, sql):
2684 """Return a list of the ids of the items in this class that match
2685 the SQL provided. The SQL is a complete "select" statement.
2687 The SQL select must include the item id as the first column.
2689 This function DOES NOT filter out retired items, add on a where
2690 clause "__retired__=0" if you don't want retired nodes.
2691 """
2692 if __debug__:
2693 start_t = time.time()
2695 self.db.sql(sql)
2696 l = self.db.sql_fetchall()
2698 if __debug__:
2699 self.db.stats['filtering'] += (time.time() - start_t)
2700 return l
2702 def count(self):
2703 """Get the number of nodes in this class.
2705 If the returned integer is 'numnodes', the ids of all the nodes
2706 in this class run from 1 to numnodes, and numnodes+1 will be the
2707 id of the next node to be created in this class.
2708 """
2709 return self.db.countnodes(self.classname)
2711 # Manipulating properties:
2712 def getprops(self, protected=1):
2713 """Return a dictionary mapping property names to property objects.
2714 If the "protected" flag is true, we include protected properties -
2715 those which may not be modified.
2716 """
2717 d = self.properties.copy()
2718 if protected:
2719 d['id'] = String()
2720 d['creation'] = hyperdb.Date()
2721 d['activity'] = hyperdb.Date()
2722 d['creator'] = hyperdb.Link('user')
2723 d['actor'] = hyperdb.Link('user')
2724 return d
2726 def addprop(self, **properties):
2727 """Add properties to this class.
2729 The keyword arguments in 'properties' must map names to property
2730 objects, or a TypeError is raised. None of the keys in 'properties'
2731 may collide with the names of existing properties, or a ValueError
2732 is raised before any properties have been added.
2733 """
2734 for key in properties:
2735 if key in self.properties:
2736 raise ValueError(key)
2737 self.properties.update(properties)
2739 def index(self, nodeid):
2740 """Add (or refresh) the node to search indexes
2741 """
2742 # find all the String properties that have indexme
2743 for prop, propclass in self.getprops().iteritems():
2744 if isinstance(propclass, String) and propclass.indexme:
2745 self.db.indexer.add_text((self.classname, nodeid, prop),
2746 str(self.get(nodeid, prop)))
2748 #
2749 # import / export support
2750 #
2751 def export_list(self, propnames, nodeid):
2752 """ Export a node - generate a list of CSV-able data in the order
2753 specified by propnames for the given node.
2754 """
2755 properties = self.getprops()
2756 l = []
2757 for prop in propnames:
2758 proptype = properties[prop]
2759 value = self.get(nodeid, prop)
2760 # "marshal" data where needed
2761 if value is None:
2762 pass
2763 elif isinstance(proptype, hyperdb.Date):
2764 value = value.get_tuple()
2765 elif isinstance(proptype, hyperdb.Interval):
2766 value = value.get_tuple()
2767 elif isinstance(proptype, hyperdb.Password):
2768 value = str(value)
2769 l.append(repr(value))
2770 l.append(repr(self.is_retired(nodeid)))
2771 return l
2773 def import_list(self, propnames, proplist):
2774 """ Import a node - all information including "id" is present and
2775 should not be sanity checked. Triggers are not triggered. The
2776 journal should be initialised using the "creator" and "created"
2777 information.
2779 Return the nodeid of the node imported.
2780 """
2781 if self.db.journaltag is None:
2782 raise DatabaseError(_('Database open read-only'))
2783 properties = self.getprops()
2785 # make the new node's property map
2786 d = {}
2787 retire = 0
2788 if not "id" in propnames:
2789 newid = self.db.newid(self.classname)
2790 else:
2791 newid = eval(proplist[propnames.index("id")])
2792 for i in range(len(propnames)):
2793 # Use eval to reverse the repr() used to output the CSV
2794 value = eval(proplist[i])
2796 # Figure the property for this column
2797 propname = propnames[i]
2799 # "unmarshal" where necessary
2800 if propname == 'id':
2801 continue
2802 elif propname == 'is retired':
2803 # is the item retired?
2804 if int(value):
2805 retire = 1
2806 continue
2807 elif value is None:
2808 d[propname] = None
2809 continue
2811 prop = properties[propname]
2812 if value is None:
2813 # don't set Nones
2814 continue
2815 elif isinstance(prop, hyperdb.Date):
2816 value = date.Date(value)
2817 elif isinstance(prop, hyperdb.Interval):
2818 value = date.Interval(value)
2819 elif isinstance(prop, hyperdb.Password):
2820 pwd = password.Password()
2821 pwd.unpack(value)
2822 value = pwd
2823 elif isinstance(prop, String):
2824 if isinstance(value, unicode):
2825 value = value.encode('utf8')
2826 if not isinstance(value, str):
2827 raise TypeError('new property "%(propname)s" not a '
2828 'string: %(value)r'%locals())
2829 if prop.indexme:
2830 self.db.indexer.add_text((self.classname, newid, propname),
2831 value)
2832 d[propname] = value
2834 # get a new id if necessary
2835 if newid is None:
2836 newid = self.db.newid(self.classname)
2838 # insert new node or update existing?
2839 if not self.hasnode(newid):
2840 self.db.addnode(self.classname, newid, d) # insert
2841 else:
2842 self.db.setnode(self.classname, newid, d) # update
2844 # retire?
2845 if retire:
2846 # use the arg for __retired__ to cope with any odd database type
2847 # conversion (hello, sqlite)
2848 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
2849 self.db.arg, self.db.arg)
2850 self.db.sql(sql, (newid, newid))
2851 return newid
2853 def export_journals(self):
2854 """Export a class's journal - generate a list of lists of
2855 CSV-able data:
2857 nodeid, date, user, action, params
2859 No heading here - the columns are fixed.
2860 """
2861 properties = self.getprops()
2862 r = []
2863 for nodeid in self.getnodeids():
2864 for nodeid, date, user, action, params in self.history(nodeid):
2865 date = date.get_tuple()
2866 if action == 'set':
2867 export_data = {}
2868 for propname, value in params.iteritems():
2869 if propname not in properties:
2870 # property no longer in the schema
2871 continue
2873 prop = properties[propname]
2874 # make sure the params are eval()'able
2875 if value is None:
2876 pass
2877 elif isinstance(prop, Date):
2878 value = value.get_tuple()
2879 elif isinstance(prop, Interval):
2880 value = value.get_tuple()
2881 elif isinstance(prop, Password):
2882 value = str(value)
2883 export_data[propname] = value
2884 params = export_data
2885 elif action == 'create' and params:
2886 # old tracker with data stored in the create!
2887 params = {}
2888 l = [nodeid, date, user, action, params]
2889 r.append(list(map(repr, l)))
2890 return r
2892 class FileClass(hyperdb.FileClass, Class):
2893 """This class defines a large chunk of data. To support this, it has a
2894 mandatory String property "content" which is typically saved off
2895 externally to the hyperdb.
2897 The default MIME type of this data is defined by the
2898 "default_mime_type" class attribute, which may be overridden by each
2899 node if the class defines a "type" String property.
2900 """
2901 def __init__(self, db, classname, **properties):
2902 """The newly-created class automatically includes the "content"
2903 and "type" properties.
2904 """
2905 if 'content' not in properties:
2906 properties['content'] = hyperdb.String(indexme='yes')
2907 if 'type' not in properties:
2908 properties['type'] = hyperdb.String()
2909 Class.__init__(self, db, classname, **properties)
2911 def create(self, **propvalues):
2912 """ snaffle the file propvalue and store in a file
2913 """
2914 # we need to fire the auditors now, or the content property won't
2915 # be in propvalues for the auditors to play with
2916 self.fireAuditors('create', None, propvalues)
2918 # now remove the content property so it's not stored in the db
2919 content = propvalues['content']
2920 del propvalues['content']
2922 # do the database create
2923 newid = self.create_inner(**propvalues)
2925 # figure the mime type
2926 mime_type = propvalues.get('type', self.default_mime_type)
2928 # and index!
2929 if self.properties['content'].indexme:
2930 self.db.indexer.add_text((self.classname, newid, 'content'),
2931 content, mime_type)
2933 # store off the content as a file
2934 self.db.storefile(self.classname, newid, None, content)
2936 # fire reactors
2937 self.fireReactors('create', newid, None)
2939 return newid
2941 def get(self, nodeid, propname, default=_marker, cache=1):
2942 """ Trap the content propname and get it from the file
2944 'cache' exists for backwards compatibility, and is not used.
2945 """
2946 poss_msg = 'Possibly a access right configuration problem.'
2947 if propname == 'content':
2948 try:
2949 return self.db.getfile(self.classname, nodeid, None)
2950 except IOError, strerror:
2951 # BUG: by catching this we donot see an error in the log.
2952 return 'ERROR reading file: %s%s\n%s\n%s'%(
2953 self.classname, nodeid, poss_msg, strerror)
2954 if default is not _marker:
2955 return Class.get(self, nodeid, propname, default)
2956 else:
2957 return Class.get(self, nodeid, propname)
2959 def set(self, itemid, **propvalues):
2960 """ Snarf the "content" propvalue and update it in a file
2961 """
2962 self.fireAuditors('set', itemid, propvalues)
2963 oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid))
2965 # now remove the content property so it's not stored in the db
2966 content = None
2967 if 'content' in propvalues:
2968 content = propvalues['content']
2969 del propvalues['content']
2971 # do the database create
2972 propvalues = self.set_inner(itemid, **propvalues)
2974 # do content?
2975 if content:
2976 # store and possibly index
2977 self.db.storefile(self.classname, itemid, None, content)
2978 if self.properties['content'].indexme:
2979 mime_type = self.get(itemid, 'type', self.default_mime_type)
2980 self.db.indexer.add_text((self.classname, itemid, 'content'),
2981 content, mime_type)
2982 propvalues['content'] = content
2984 # fire reactors
2985 self.fireReactors('set', itemid, oldvalues)
2986 return propvalues
2988 def index(self, nodeid):
2989 """ Add (or refresh) the node to search indexes.
2991 Use the content-type property for the content property.
2992 """
2993 # find all the String properties that have indexme
2994 for prop, propclass in self.getprops().iteritems():
2995 if prop == 'content' and propclass.indexme:
2996 mime_type = self.get(nodeid, 'type', self.default_mime_type)
2997 self.db.indexer.add_text((self.classname, nodeid, 'content'),
2998 str(self.get(nodeid, 'content')), mime_type)
2999 elif isinstance(propclass, hyperdb.String) and propclass.indexme:
3000 # index them under (classname, nodeid, property)
3001 try:
3002 value = str(self.get(nodeid, prop))
3003 except IndexError:
3004 # node has been destroyed
3005 continue
3006 self.db.indexer.add_text((self.classname, nodeid, prop), value)
3008 # XXX deviation from spec - was called ItemClass
3009 class IssueClass(Class, roundupdb.IssueClass):
3010 # Overridden methods:
3011 def __init__(self, db, classname, **properties):
3012 """The newly-created class automatically includes the "messages",
3013 "files", "nosy", and "superseder" properties. If the 'properties'
3014 dictionary attempts to specify any of these properties or a
3015 "creation", "creator", "activity" or "actor" property, a ValueError
3016 is raised.
3017 """
3018 if 'title' not in properties:
3019 properties['title'] = hyperdb.String(indexme='yes')
3020 if 'messages' not in properties:
3021 properties['messages'] = hyperdb.Multilink("msg")
3022 if 'files' not in properties:
3023 properties['files'] = hyperdb.Multilink("file")
3024 if 'nosy' not in properties:
3025 # note: journalling is turned off as it really just wastes
3026 # space. this behaviour may be overridden in an instance
3027 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
3028 if 'superseder' not in properties:
3029 properties['superseder'] = hyperdb.Multilink(classname)
3030 Class.__init__(self, db, classname, **properties)
3032 # vim: set et sts=4 sw=4 :