76243d2249d48b0d80c72f78fb4c4af774f19573
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 """ Relational database (SQL) backend common code.
20 Basics:
22 - map roundup classes to relational tables
23 - automatically detect schema changes and modify the table schemas
24 appropriately (we store the "database version" of the schema in the
25 database itself as the only row of the "schema" table)
26 - multilinks (which represent a many-to-many relationship) are handled through
27 intermediate tables
28 - journals are stored adjunct to the per-class tables
29 - table names and columns have "_" prepended so the names can't clash with
30 restricted names (like "order")
31 - retirement is determined by the __retired__ column being > 0
33 Database-specific changes may generally be pushed out to the overridable
34 sql_* methods, since everything else should be fairly generic. There's
35 probably a bit of work to be done if a database is used that actually
36 honors column typing, since the initial databases don't (sqlite stores
37 everything as a string.)
39 The schema of the hyperdb being mapped to the database is stored in the
40 database itself as a repr()'ed dictionary of information about each Class
41 that maps to a table. If that information differs from the hyperdb schema,
42 then we update it. We also store in the schema dict a version which
43 allows us to upgrade the database schema when necessary. See upgrade_db().
45 To force a unqiueness constraint on the key properties we put the item
46 id into the __retired__ column duing retirement (so it's 0 for "active"
47 items) and place a unqiueness constraint on key + __retired__. This is
48 particularly important for the users class where multiple users may
49 try to have the same username, with potentially many retired users with
50 the same name.
51 """
52 __docformat__ = 'restructuredtext'
54 # standard python modules
55 import sys, os, time, re, errno, weakref, copy, logging
57 # roundup modules
58 from roundup import hyperdb, date, password, roundupdb, security, support
59 from roundup.hyperdb import String, Password, Date, Interval, Link, \
60 Multilink, DatabaseError, Boolean, Number, Node
61 from roundup.backends import locking
62 from roundup.support import reversed
63 from roundup.i18n import _
65 # support
66 from blobfiles import FileStorage
67 try:
68 from indexer_xapian import Indexer
69 except ImportError:
70 from indexer_rdbms import Indexer
71 from sessions_rdbms import Sessions, OneTimeKeys
72 from roundup.date import Range
74 # number of rows to keep in memory
75 ROW_CACHE_SIZE = 100
77 # dummy value meaning "argument not passed"
78 _marker = []
80 def _num_cvt(num):
81 num = str(num)
82 try:
83 return int(num)
84 except:
85 return float(num)
87 def _bool_cvt(value):
88 if value in ('TRUE', 'FALSE'):
89 return {'TRUE': 1, 'FALSE': 0}[value]
90 # assume it's a number returned from the db API
91 return int(value)
93 def connection_dict(config, dbnamestr=None):
94 """ Used by Postgresql and MySQL to detemine the keyword args for
95 opening the database connection."""
96 d = { }
97 if dbnamestr:
98 d[dbnamestr] = config.RDBMS_NAME
99 for name in ('host', 'port', 'password', 'user', 'read_default_group',
100 'read_default_file'):
101 cvar = 'RDBMS_'+name.upper()
102 if config[cvar] is not None:
103 d[name] = config[cvar]
104 return d
106 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
107 """ Wrapper around an SQL database that presents a hyperdb interface.
109 - some functionality is specific to the actual SQL database, hence
110 the sql_* methods that are NotImplemented
111 - we keep a cache of the latest ROW_CACHE_SIZE row fetches.
112 """
113 def __init__(self, config, journaltag=None):
114 """ Open the database and load the schema from it.
115 """
116 FileStorage.__init__(self, config.UMASK)
117 self.config, self.journaltag = config, journaltag
118 self.dir = config.DATABASE
119 self.classes = {}
120 self.indexer = Indexer(self)
121 self.security = security.Security(self)
123 # additional transaction support for external files and the like
124 self.transactions = []
126 # keep a cache of the N most recently retrieved rows of any kind
127 # (classname, nodeid) = row
128 self.cache = {}
129 self.cache_lru = []
130 self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0,
131 'filtering': 0}
133 # database lock
134 self.lockfile = None
136 # open a connection to the database, creating the "conn" attribute
137 self.open_connection()
139 def clearCache(self):
140 self.cache = {}
141 self.cache_lru = []
143 def getSessionManager(self):
144 return Sessions(self)
146 def getOTKManager(self):
147 return OneTimeKeys(self)
149 def open_connection(self):
150 """ Open a connection to the database, creating it if necessary.
152 Must call self.load_dbschema()
153 """
154 raise NotImplemented
156 def sql(self, sql, args=None):
157 """ Execute the sql with the optional args.
158 """
159 self.log_debug('SQL %r %r'%(sql, args))
160 if args:
161 self.cursor.execute(sql, args)
162 else:
163 self.cursor.execute(sql)
165 def sql_fetchone(self):
166 """ Fetch a single row. If there's nothing to fetch, return None.
167 """
168 return self.cursor.fetchone()
170 def sql_fetchall(self):
171 """ Fetch all rows. If there's nothing to fetch, return [].
172 """
173 return self.cursor.fetchall()
175 def sql_stringquote(self, value):
176 """ Quote the string so it's safe to put in the 'sql quotes'
177 """
178 return re.sub("'", "''", str(value))
180 def init_dbschema(self):
181 self.database_schema = {
182 'version': self.current_db_version,
183 'tables': {}
184 }
186 def load_dbschema(self):
187 """ Load the schema definition that the database currently implements
188 """
189 self.cursor.execute('select schema from schema')
190 schema = self.cursor.fetchone()
191 if schema:
192 self.database_schema = eval(schema[0])
193 else:
194 self.database_schema = {}
196 def save_dbschema(self):
197 """ Save the schema definition that the database currently implements
198 """
199 s = repr(self.database_schema)
200 self.sql('delete from schema')
201 self.sql('insert into schema values (%s)'%self.arg, (s,))
203 def post_init(self):
204 """ Called once the schema initialisation has finished.
206 We should now confirm that the schema defined by our "classes"
207 attribute actually matches the schema in the database.
208 """
209 save = 0
211 # handle changes in the schema
212 tables = self.database_schema['tables']
213 for classname, spec in self.classes.items():
214 if tables.has_key(classname):
215 dbspec = tables[classname]
216 if self.update_class(spec, dbspec):
217 tables[classname] = spec.schema()
218 save = 1
219 else:
220 self.create_class(spec)
221 tables[classname] = spec.schema()
222 save = 1
224 for classname, spec in tables.items():
225 if not self.classes.has_key(classname):
226 self.drop_class(classname, tables[classname])
227 del tables[classname]
228 save = 1
230 # now upgrade the database for column type changes, new internal
231 # tables, etc.
232 save = save | self.upgrade_db()
234 # update the database version of the schema
235 if save:
236 self.save_dbschema()
238 # reindex the db if necessary
239 if self.indexer.should_reindex():
240 self.reindex()
242 # commit
243 self.sql_commit()
245 # update this number when we need to make changes to the SQL structure
246 # of the backen database
247 current_db_version = 5
248 db_version_updated = False
249 def upgrade_db(self):
250 """ Update the SQL database to reflect changes in the backend code.
252 Return boolean whether we need to save the schema.
253 """
254 version = self.database_schema.get('version', 1)
255 if version > self.current_db_version:
256 raise DatabaseError('attempting to run rev %d DATABASE with rev '
257 '%d CODE!'%(version, self.current_db_version))
258 if version == self.current_db_version:
259 # nothing to do
260 return 0
262 if version < 2:
263 self.log_info('upgrade to version 2')
264 # change the schema structure
265 self.database_schema = {'tables': self.database_schema}
267 # version 1 didn't have the actor column (note that in
268 # MySQL this will also transition the tables to typed columns)
269 self.add_new_columns_v2()
271 # version 1 doesn't have the OTK, session and indexing in the
272 # database
273 self.create_version_2_tables()
275 if version < 3:
276 self.log_info('upgrade to version 3')
277 self.fix_version_2_tables()
279 if version < 4:
280 self.fix_version_3_tables()
282 if version < 5:
283 self.fix_version_4_tables()
285 self.database_schema['version'] = self.current_db_version
286 self.db_version_updated = True
287 return 1
289 def fix_version_3_tables(self):
290 # drop the shorter VARCHAR OTK column and add a new TEXT one
291 for name in ('otk', 'session'):
292 self.sql('DELETE FROM %ss'%name)
293 self.sql('ALTER TABLE %ss DROP %s_value'%(name, name))
294 self.sql('ALTER TABLE %ss ADD %s_value TEXT'%(name, name))
296 def fix_version_2_tables(self):
297 # Default (used by sqlite): NOOP
298 pass
300 def fix_version_4_tables(self):
301 # note this is an explicit call now
302 c = self.cursor
303 for cn, klass in self.classes.items():
304 c.execute('select id from _%s where __retired__<>0'%(cn,))
305 for (id,) in c.fetchall():
306 c.execute('update _%s set __retired__=%s where id=%s'%(cn,
307 self.arg, self.arg), (id, id))
309 if klass.key:
310 self.add_class_key_required_unique_constraint(cn, klass.key)
312 def _convert_journal_tables(self):
313 """Get current journal table contents, drop the table and re-create"""
314 c = self.cursor
315 cols = ','.join('nodeid date tag action params'.split())
316 for klass in self.classes.values():
317 # slurp and drop
318 sql = 'select %s from %s__journal order by date'%(cols,
319 klass.classname)
320 c.execute(sql)
321 contents = c.fetchall()
322 self.drop_journal_table_indexes(klass.classname)
323 c.execute('drop table %s__journal'%klass.classname)
325 # re-create and re-populate
326 self.create_journal_table(klass)
327 a = self.arg
328 sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(
329 klass.classname, cols, a, a, a, a, a)
330 for row in contents:
331 # no data conversion needed
332 self.cursor.execute(sql, row)
334 def _convert_string_properties(self):
335 """Get current Class tables that contain String properties, and
336 convert the VARCHAR columns to TEXT"""
337 c = self.cursor
338 for klass in self.classes.values():
339 # slurp and drop
340 cols, mls = self.determine_columns(klass.properties.items())
341 scols = ','.join([i[0] for i in cols])
342 sql = 'select id,%s from _%s'%(scols, klass.classname)
343 c.execute(sql)
344 contents = c.fetchall()
345 self.drop_class_table_indexes(klass.classname, klass.getkey())
346 c.execute('drop table _%s'%klass.classname)
348 # re-create and re-populate
349 self.create_class_table(klass, create_sequence=0)
350 a = ','.join([self.arg for i in range(len(cols)+1)])
351 sql = 'insert into _%s (id,%s) values (%s)'%(klass.classname,
352 scols, a)
353 for row in contents:
354 l = []
355 for entry in row:
356 # mysql will already be a string - psql needs "help"
357 if entry is not None and not isinstance(entry, type('')):
358 entry = str(entry)
359 l.append(entry)
360 self.cursor.execute(sql, l)
362 def refresh_database(self):
363 self.post_init()
366 def reindex(self, classname=None, show_progress=False):
367 if classname:
368 classes = [self.getclass(classname)]
369 else:
370 classes = self.classes.values()
371 for klass in classes:
372 if show_progress:
373 for nodeid in support.Progress('Reindex %s'%klass.classname,
374 klass.list()):
375 klass.index(nodeid)
376 else:
377 for nodeid in klass.list():
378 klass.index(nodeid)
379 self.indexer.save_index()
381 hyperdb_to_sql_datatypes = {
382 hyperdb.String : 'TEXT',
383 hyperdb.Date : 'TIMESTAMP',
384 hyperdb.Link : 'INTEGER',
385 hyperdb.Interval : 'VARCHAR(255)',
386 hyperdb.Password : 'VARCHAR(255)',
387 hyperdb.Boolean : 'BOOLEAN',
388 hyperdb.Number : 'REAL',
389 }
391 def hyperdb_to_sql_datatype(self, propclass):
393 datatype = self.hyperdb_to_sql_datatypes.get(propclass)
394 if datatype:
395 return datatype
397 for k, v in self.hyperdb_to_sql_datatypes.iteritems():
398 if issubclass(propclass, k):
399 return v
401 raise ValueError, '%r is not a hyperdb property class' % propclass
403 def determine_columns(self, properties):
404 """ Figure the column names and multilink properties from the spec
406 "properties" is a list of (name, prop) where prop may be an
407 instance of a hyperdb "type" _or_ a string repr of that type.
408 """
409 cols = [
410 ('_actor', self.hyperdb_to_sql_datatype(hyperdb.Link)),
411 ('_activity', self.hyperdb_to_sql_datatype(hyperdb.Date)),
412 ('_creator', self.hyperdb_to_sql_datatype(hyperdb.Link)),
413 ('_creation', self.hyperdb_to_sql_datatype(hyperdb.Date)),
414 ]
415 mls = []
416 # add the multilinks separately
417 for col, prop in properties:
418 if isinstance(prop, Multilink):
419 mls.append(col)
420 continue
422 if isinstance(prop, type('')):
423 raise ValueError, "string property spec!"
424 #and prop.find('Multilink') != -1:
425 #mls.append(col)
427 datatype = self.hyperdb_to_sql_datatype(prop.__class__)
428 cols.append(('_'+col, datatype))
430 # Intervals stored as two columns
431 if isinstance(prop, Interval):
432 cols.append(('__'+col+'_int__', 'BIGINT'))
434 cols.sort()
435 return cols, mls
437 def update_class(self, spec, old_spec, force=0):
438 """ Determine the differences between the current spec and the
439 database version of the spec, and update where necessary.
441 If 'force' is true, update the database anyway.
442 """
443 new_has = spec.properties.has_key
444 new_spec = spec.schema()
445 new_spec[1].sort()
446 old_spec[1].sort()
447 if not force and new_spec == old_spec:
448 # no changes
449 return 0
451 logger = logging.getLogger('hyperdb')
452 logger.info('update_class %s'%spec.classname)
454 logger.debug('old_spec %r'%(old_spec,))
455 logger.debug('new_spec %r'%(new_spec,))
457 # detect key prop change for potential index change
458 keyprop_changes = {}
459 if new_spec[0] != old_spec[0]:
460 if old_spec[0]:
461 keyprop_changes['remove'] = old_spec[0]
462 if new_spec[0]:
463 keyprop_changes['add'] = new_spec[0]
465 # detect multilinks that have been removed, and drop their table
466 old_has = {}
467 for name, prop in old_spec[1]:
468 old_has[name] = 1
469 if new_has(name):
470 continue
472 if prop.find('Multilink to') != -1:
473 # first drop indexes.
474 self.drop_multilink_table_indexes(spec.classname, name)
476 # now the multilink table itself
477 sql = 'drop table %s_%s'%(spec.classname, name)
478 else:
479 # if this is the key prop, drop the index first
480 if old_spec[0] == prop:
481 self.drop_class_table_key_index(spec.classname, name)
482 del keyprop_changes['remove']
484 # drop the column
485 sql = 'alter table _%s drop column _%s'%(spec.classname, name)
487 self.sql(sql)
488 old_has = old_has.has_key
490 # if we didn't remove the key prop just then, but the key prop has
491 # changed, we still need to remove the old index
492 if keyprop_changes.has_key('remove'):
493 self.drop_class_table_key_index(spec.classname,
494 keyprop_changes['remove'])
496 # add new columns
497 for propname, prop in new_spec[1]:
498 if old_has(propname):
499 continue
500 prop = spec.properties[propname]
501 if isinstance(prop, Multilink):
502 self.create_multilink_table(spec, propname)
503 else:
504 # add the column
505 coltype = self.hyperdb_to_sql_datatype(prop.__class__)
506 sql = 'alter table _%s add column _%s %s'%(
507 spec.classname, propname, coltype)
508 self.sql(sql)
510 # extra Interval column
511 if isinstance(prop, Interval):
512 sql = 'alter table _%s add column __%s_int__ BIGINT'%(
513 spec.classname, propname)
514 self.sql(sql)
516 # if the new column is a key prop, we need an index!
517 if new_spec[0] == propname:
518 self.create_class_table_key_index(spec.classname, propname)
519 del keyprop_changes['add']
521 # if we didn't add the key prop just then, but the key prop has
522 # changed, we still need to add the new index
523 if keyprop_changes.has_key('add'):
524 self.create_class_table_key_index(spec.classname,
525 keyprop_changes['add'])
527 return 1
529 def determine_all_columns(self, spec):
530 """Figure out the columns from the spec and also add internal columns
532 """
533 cols, mls = self.determine_columns(spec.properties.items())
535 # add on our special columns
536 cols.append(('id', 'INTEGER PRIMARY KEY'))
537 cols.append(('__retired__', 'INTEGER DEFAULT 0'))
538 return cols, mls
540 def create_class_table(self, spec):
541 """Create the class table for the given Class "spec". Creates the
542 indexes too."""
543 cols, mls = self.determine_all_columns(spec)
545 # create the base table
546 scols = ','.join(['%s %s'%x for x in cols])
547 sql = 'create table _%s (%s)'%(spec.classname, scols)
548 self.sql(sql)
550 self.create_class_table_indexes(spec)
552 return cols, mls
554 def create_class_table_indexes(self, spec):
555 """ create the class table for the given spec
556 """
557 # create __retired__ index
558 index_sql2 = 'create index _%s_retired_idx on _%s(__retired__)'%(
559 spec.classname, spec.classname)
560 self.sql(index_sql2)
562 # create index for key property
563 if spec.key:
564 index_sql3 = 'create index _%s_%s_idx on _%s(_%s)'%(
565 spec.classname, spec.key,
566 spec.classname, spec.key)
567 self.sql(index_sql3)
569 # and the unique index for key / retired(id)
570 self.add_class_key_required_unique_constraint(spec.classname,
571 spec.key)
573 # TODO: create indexes on (selected?) Link property columns, as
574 # they're more likely to be used for lookup
576 def add_class_key_required_unique_constraint(self, cn, key):
577 sql = '''create unique index _%s_key_retired_idx
578 on _%s(__retired__, _%s)'''%(cn, cn, key)
579 self.sql(sql)
581 def drop_class_table_indexes(self, cn, key):
582 # drop the old table indexes first
583 l = ['_%s_id_idx'%cn, '_%s_retired_idx'%cn]
584 if key:
585 l.append('_%s_%s_idx'%(cn, key))
587 table_name = '_%s'%cn
588 for index_name in l:
589 if not self.sql_index_exists(table_name, index_name):
590 continue
591 index_sql = 'drop index '+index_name
592 self.sql(index_sql)
594 def create_class_table_key_index(self, cn, key):
595 """ create the class table for the given spec
596 """
597 sql = 'create index _%s_%s_idx on _%s(_%s)'%(cn, key, cn, key)
598 self.sql(sql)
600 def drop_class_table_key_index(self, cn, key):
601 table_name = '_%s'%cn
602 index_name = '_%s_%s_idx'%(cn, key)
603 if self.sql_index_exists(table_name, index_name):
604 sql = 'drop index '+index_name
605 self.sql(sql)
607 # and now the retired unique index too
608 index_name = '_%s_key_retired_idx'%cn
609 if self.sql_index_exists(table_name, index_name):
610 sql = 'drop index '+index_name
611 self.sql(sql)
613 def create_journal_table(self, spec):
614 """ create the journal table for a class given the spec and
615 already-determined cols
616 """
617 # journal table
618 cols = ','.join(['%s varchar'%x
619 for x in 'nodeid date tag action params'.split()])
620 sql = """create table %s__journal (
621 nodeid integer, date %s, tag varchar(255),
622 action varchar(255), params text)""" % (spec.classname,
623 self.hyperdb_to_sql_datatype(hyperdb.Date))
624 self.sql(sql)
625 self.create_journal_table_indexes(spec)
627 def create_journal_table_indexes(self, spec):
628 # index on nodeid
629 sql = 'create index %s_journ_idx on %s__journal(nodeid)'%(
630 spec.classname, spec.classname)
631 self.sql(sql)
633 def drop_journal_table_indexes(self, classname):
634 index_name = '%s_journ_idx'%classname
635 if not self.sql_index_exists('%s__journal'%classname, index_name):
636 return
637 index_sql = 'drop index '+index_name
638 self.sql(index_sql)
640 def create_multilink_table(self, spec, ml):
641 """ Create a multilink table for the "ml" property of the class
642 given by the spec
643 """
644 # create the table
645 sql = 'create table %s_%s (linkid INTEGER, nodeid INTEGER)'%(
646 spec.classname, ml)
647 self.sql(sql)
648 self.create_multilink_table_indexes(spec, ml)
650 def create_multilink_table_indexes(self, spec, ml):
651 # create index on linkid
652 index_sql = 'create index %s_%s_l_idx on %s_%s(linkid)'%(
653 spec.classname, ml, spec.classname, ml)
654 self.sql(index_sql)
656 # create index on nodeid
657 index_sql = 'create index %s_%s_n_idx on %s_%s(nodeid)'%(
658 spec.classname, ml, spec.classname, ml)
659 self.sql(index_sql)
661 def drop_multilink_table_indexes(self, classname, ml):
662 l = [
663 '%s_%s_l_idx'%(classname, ml),
664 '%s_%s_n_idx'%(classname, ml)
665 ]
666 table_name = '%s_%s'%(classname, ml)
667 for index_name in l:
668 if not self.sql_index_exists(table_name, index_name):
669 continue
670 index_sql = 'drop index %s'%index_name
671 self.sql(index_sql)
673 def create_class(self, spec):
674 """ Create a database table according to the given spec.
675 """
676 cols, mls = self.create_class_table(spec)
677 self.create_journal_table(spec)
679 # now create the multilink tables
680 for ml in mls:
681 self.create_multilink_table(spec, ml)
683 def drop_class(self, cn, spec):
684 """ Drop the given table from the database.
686 Drop the journal and multilink tables too.
687 """
688 properties = spec[1]
689 # figure the multilinks
690 mls = []
691 for propname, prop in properties:
692 if isinstance(prop, Multilink):
693 mls.append(propname)
695 # drop class table and indexes
696 self.drop_class_table_indexes(cn, spec[0])
698 self.drop_class_table(cn)
700 # drop journal table and indexes
701 self.drop_journal_table_indexes(cn)
702 sql = 'drop table %s__journal'%cn
703 self.sql(sql)
705 for ml in mls:
706 # drop multilink table and indexes
707 self.drop_multilink_table_indexes(cn, ml)
708 sql = 'drop table %s_%s'%(spec.classname, ml)
709 self.sql(sql)
711 def drop_class_table(self, cn):
712 sql = 'drop table _%s'%cn
713 self.sql(sql)
715 #
716 # Classes
717 #
718 def __getattr__(self, classname):
719 """ A convenient way of calling self.getclass(classname).
720 """
721 if self.classes.has_key(classname):
722 return self.classes[classname]
723 raise AttributeError, classname
725 def addclass(self, cl):
726 """ Add a Class to the hyperdatabase.
727 """
728 cn = cl.classname
729 if self.classes.has_key(cn):
730 raise ValueError, cn
731 self.classes[cn] = cl
733 # add default Edit and View permissions
734 self.security.addPermission(name="Create", klass=cn,
735 description="User is allowed to create "+cn)
736 self.security.addPermission(name="Edit", klass=cn,
737 description="User is allowed to edit "+cn)
738 self.security.addPermission(name="View", klass=cn,
739 description="User is allowed to access "+cn)
741 def getclasses(self):
742 """ Return a list of the names of all existing classes.
743 """
744 l = self.classes.keys()
745 l.sort()
746 return l
748 def getclass(self, classname):
749 """Get the Class object representing a particular class.
751 If 'classname' is not a valid class name, a KeyError is raised.
752 """
753 try:
754 return self.classes[classname]
755 except KeyError:
756 raise KeyError, 'There is no class called "%s"'%classname
758 def clear(self):
759 """Delete all database contents.
761 Note: I don't commit here, which is different behaviour to the
762 "nuke from orbit" behaviour in the dbs.
763 """
764 logging.getLogger('hyperdb').info('clear')
765 for cn in self.classes.keys():
766 sql = 'delete from _%s'%cn
767 self.sql(sql)
769 #
770 # Nodes
771 #
773 hyperdb_to_sql_value = {
774 hyperdb.String : str,
775 # fractional seconds by default
776 hyperdb.Date : lambda x: x.formal(sep=' ', sec='%06.3f'),
777 hyperdb.Link : int,
778 hyperdb.Interval : str,
779 hyperdb.Password : str,
780 hyperdb.Boolean : lambda x: x and 'TRUE' or 'FALSE',
781 hyperdb.Number : lambda x: x,
782 hyperdb.Multilink : lambda x: x, # used in journal marshalling
783 }
785 def to_sql_value(self, propklass):
787 fn = self.hyperdb_to_sql_value.get(propklass)
788 if fn:
789 return fn
791 for k, v in self.hyperdb_to_sql_value.iteritems():
792 if issubclass(propklass, k):
793 return v
795 raise ValueError, '%r is not a hyperdb property class' % propklass
797 def addnode(self, classname, nodeid, node):
798 """ Add the specified node to its class's db.
799 """
800 self.log_debug('addnode %s%s %r'%(classname,
801 nodeid, node))
803 # determine the column definitions and multilink tables
804 cl = self.classes[classname]
805 cols, mls = self.determine_columns(cl.properties.items())
807 # we'll be supplied these props if we're doing an import
808 values = node.copy()
809 if not values.has_key('creator'):
810 # add in the "calculated" properties (dupe so we don't affect
811 # calling code's node assumptions)
812 values['creation'] = values['activity'] = date.Date()
813 values['actor'] = values['creator'] = self.getuid()
815 cl = self.classes[classname]
816 props = cl.getprops(protected=1)
817 del props['id']
819 # default the non-multilink columns
820 for col, prop in props.items():
821 if not values.has_key(col):
822 if isinstance(prop, Multilink):
823 values[col] = []
824 else:
825 values[col] = None
827 # clear this node out of the cache if it's in there
828 key = (classname, nodeid)
829 if self.cache.has_key(key):
830 del self.cache[key]
831 self.cache_lru.remove(key)
833 # figure the values to insert
834 vals = []
835 for col,dt in cols:
836 # this is somewhat dodgy....
837 if col.endswith('_int__'):
838 # XXX eugh, this test suxxors
839 value = values[col[2:-6]]
840 # this is an Interval special "int" column
841 if value is not None:
842 vals.append(value.as_seconds())
843 else:
844 vals.append(value)
845 continue
847 prop = props[col[1:]]
848 value = values[col[1:]]
849 if value is not None:
850 value = self.to_sql_value(prop.__class__)(value)
851 vals.append(value)
852 vals.append(nodeid)
853 vals = tuple(vals)
855 # make sure the ordering is correct for column name -> column value
856 s = ','.join([self.arg for x in cols]) + ',%s'%self.arg
857 cols = ','.join([col for col,dt in cols]) + ',id'
859 # perform the inserts
860 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
861 self.sql(sql, vals)
863 # insert the multilink rows
864 for col in mls:
865 t = '%s_%s'%(classname, col)
866 for entry in node[col]:
867 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
868 self.arg, self.arg)
869 self.sql(sql, (entry, nodeid))
871 def setnode(self, classname, nodeid, values, multilink_changes={}):
872 """ Change the specified node.
873 """
874 self.log_debug('setnode %s%s %r'
875 % (classname, nodeid, values))
877 # clear this node out of the cache if it's in there
878 key = (classname, nodeid)
879 if self.cache.has_key(key):
880 del self.cache[key]
881 self.cache_lru.remove(key)
883 cl = self.classes[classname]
884 props = cl.getprops()
886 cols = []
887 mls = []
888 # add the multilinks separately
889 for col in values.keys():
890 prop = props[col]
891 if isinstance(prop, Multilink):
892 mls.append(col)
893 elif isinstance(prop, Interval):
894 # Intervals store the seconds value too
895 cols.append(col)
896 # extra leading '_' added by code below
897 cols.append('_' +col + '_int__')
898 else:
899 cols.append(col)
900 cols.sort()
902 # figure the values to insert
903 vals = []
904 for col in cols:
905 if col.endswith('_int__'):
906 # XXX eugh, this test suxxors
907 # Intervals store the seconds value too
908 col = col[1:-6]
909 prop = props[col]
910 value = values[col]
911 if value is None:
912 vals.append(None)
913 else:
914 vals.append(value.as_seconds())
915 else:
916 prop = props[col]
917 value = values[col]
918 if value is None:
919 e = None
920 else:
921 e = self.to_sql_value(prop.__class__)(value)
922 vals.append(e)
924 vals.append(int(nodeid))
925 vals = tuple(vals)
927 # if there's any updates to regular columns, do them
928 if cols:
929 # make sure the ordering is correct for column name -> column value
930 s = ','.join(['_%s=%s'%(x, self.arg) for x in cols])
931 cols = ','.join(cols)
933 # perform the update
934 sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
935 self.sql(sql, vals)
937 # we're probably coming from an import, not a change
938 if not multilink_changes:
939 for name in mls:
940 prop = props[name]
941 value = values[name]
943 t = '%s_%s'%(classname, name)
945 # clear out previous values for this node
946 # XXX numeric ids
947 self.sql('delete from %s where nodeid=%s'%(t, self.arg),
948 (nodeid,))
950 # insert the values for this node
951 for entry in values[name]:
952 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
953 self.arg, self.arg)
954 # XXX numeric ids
955 self.sql(sql, (entry, nodeid))
957 # we have multilink changes to apply
958 for col, (add, remove) in multilink_changes.items():
959 tn = '%s_%s'%(classname, col)
960 if add:
961 sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
962 self.arg, self.arg)
963 for addid in add:
964 # XXX numeric ids
965 self.sql(sql, (int(nodeid), int(addid)))
966 if remove:
967 s = ','.join([self.arg]*len(remove))
968 sql = 'delete from %s where nodeid=%s and linkid in (%s)'%(tn,
969 self.arg, s)
970 # XXX numeric ids
971 self.sql(sql, [int(nodeid)] + remove)
973 sql_to_hyperdb_value = {
974 hyperdb.String : str,
975 hyperdb.Date : lambda x:date.Date(str(x).replace(' ', '.')),
976 # hyperdb.Link : int, # XXX numeric ids
977 hyperdb.Link : str,
978 hyperdb.Interval : date.Interval,
979 hyperdb.Password : lambda x: password.Password(encrypted=x),
980 hyperdb.Boolean : _bool_cvt,
981 hyperdb.Number : _num_cvt,
982 hyperdb.Multilink : lambda x: x, # used in journal marshalling
983 }
985 def to_hyperdb_value(self, propklass):
987 fn = self.sql_to_hyperdb_value.get(propklass)
988 if fn:
989 return fn
991 for k, v in self.sql_to_hyperdb_value.iteritems():
992 if issubclass(propklass, k):
993 return v
995 raise ValueError, '%r is not a hyperdb property class' % propklass
997 def getnode(self, classname, nodeid):
998 """ Get a node from the database.
999 """
1000 # see if we have this node cached
1001 key = (classname, nodeid)
1002 if self.cache.has_key(key):
1003 # push us back to the top of the LRU
1004 self.cache_lru.remove(key)
1005 self.cache_lru.insert(0, key)
1006 if __debug__:
1007 self.stats['cache_hits'] += 1
1008 # return the cached information
1009 return self.cache[key]
1011 if __debug__:
1012 self.stats['cache_misses'] += 1
1013 start_t = time.time()
1015 # figure the columns we're fetching
1016 cl = self.classes[classname]
1017 cols, mls = self.determine_columns(cl.properties.items())
1018 scols = ','.join([col for col,dt in cols])
1020 # perform the basic property fetch
1021 sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
1022 self.sql(sql, (nodeid,))
1024 values = self.sql_fetchone()
1025 if values is None:
1026 raise IndexError, 'no such %s node %s'%(classname, nodeid)
1028 # make up the node
1029 node = {}
1030 props = cl.getprops(protected=1)
1031 for col in range(len(cols)):
1032 name = cols[col][0][1:]
1033 if name.endswith('_int__'):
1034 # XXX eugh, this test suxxors
1035 # ignore the special Interval-as-seconds column
1036 continue
1037 value = values[col]
1038 if value is not None:
1039 value = self.to_hyperdb_value(props[name].__class__)(value)
1040 node[name] = value
1043 # now the multilinks
1044 for col in mls:
1045 # get the link ids
1046 sql = 'select linkid from %s_%s where nodeid=%s'%(classname, col,
1047 self.arg)
1048 self.sql(sql, (nodeid,))
1049 # extract the first column from the result
1050 # XXX numeric ids
1051 items = [int(x[0]) for x in self.cursor.fetchall()]
1052 items.sort ()
1053 node[col] = [str(x) for x in items]
1055 # save off in the cache
1056 key = (classname, nodeid)
1057 self.cache[key] = node
1058 # update the LRU
1059 self.cache_lru.insert(0, key)
1060 if len(self.cache_lru) > ROW_CACHE_SIZE:
1061 del self.cache[self.cache_lru.pop()]
1063 if __debug__:
1064 self.stats['get_items'] += (time.time() - start_t)
1066 return node
1068 def destroynode(self, classname, nodeid):
1069 """Remove a node from the database. Called exclusively by the
1070 destroy() method on Class.
1071 """
1072 logging.getLogger('hyperdb').info('destroynode %s%s'%(classname, nodeid))
1074 # make sure the node exists
1075 if not self.hasnode(classname, nodeid):
1076 raise IndexError, '%s has no node %s'%(classname, nodeid)
1078 # see if we have this node cached
1079 if self.cache.has_key((classname, nodeid)):
1080 del self.cache[(classname, nodeid)]
1082 # see if there's any obvious commit actions that we should get rid of
1083 for entry in self.transactions[:]:
1084 if entry[1][:2] == (classname, nodeid):
1085 self.transactions.remove(entry)
1087 # now do the SQL
1088 sql = 'delete from _%s where id=%s'%(classname, self.arg)
1089 self.sql(sql, (nodeid,))
1091 # remove from multilnks
1092 cl = self.getclass(classname)
1093 x, mls = self.determine_columns(cl.properties.items())
1094 for col in mls:
1095 # get the link ids
1096 sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
1097 self.sql(sql, (nodeid,))
1099 # remove journal entries
1100 sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
1101 self.sql(sql, (nodeid,))
1103 # cleanup any blob filestorage when we commit
1104 self.transactions.append((FileStorage.destroy, (self, classname, nodeid)))
1106 def hasnode(self, classname, nodeid):
1107 """ Determine if the database has a given node.
1108 """
1109 # If this node is in the cache, then we do not need to go to
1110 # the database. (We don't consider this an LRU hit, though.)
1111 if self.cache.has_key((classname, nodeid)):
1112 # Return 1, not True, to match the type of the result of
1113 # the SQL operation below.
1114 return 1
1115 sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
1116 self.sql(sql, (nodeid,))
1117 return int(self.cursor.fetchone()[0])
1119 def countnodes(self, classname):
1120 """ Count the number of nodes that exist for a particular Class.
1121 """
1122 sql = 'select count(*) from _%s'%classname
1123 self.sql(sql)
1124 return self.cursor.fetchone()[0]
1126 def addjournal(self, classname, nodeid, action, params, creator=None,
1127 creation=None):
1128 """ Journal the Action
1129 'action' may be:
1131 'create' or 'set' -- 'params' is a dictionary of property values
1132 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
1133 'retire' -- 'params' is None
1134 """
1135 # handle supply of the special journalling parameters (usually
1136 # supplied on importing an existing database)
1137 if creator:
1138 journaltag = creator
1139 else:
1140 journaltag = self.getuid()
1141 if creation:
1142 journaldate = creation
1143 else:
1144 journaldate = date.Date()
1146 # create the journal entry
1147 cols = 'nodeid,date,tag,action,params'
1149 self.log_debug('addjournal %s%s %r %s %s %r'%(classname,
1150 nodeid, journaldate, journaltag, action, params))
1152 # make the journalled data marshallable
1153 if isinstance(params, type({})):
1154 self._journal_marshal(params, classname)
1156 params = repr(params)
1158 dc = self.to_sql_value(hyperdb.Date)
1159 journaldate = dc(journaldate)
1161 self.save_journal(classname, cols, nodeid, journaldate,
1162 journaltag, action, params)
1164 def setjournal(self, classname, nodeid, journal):
1165 """Set the journal to the "journal" list."""
1166 # clear out any existing entries
1167 self.sql('delete from %s__journal where nodeid=%s'%(classname,
1168 self.arg), (nodeid,))
1170 # create the journal entry
1171 cols = 'nodeid,date,tag,action,params'
1173 dc = self.to_sql_value(hyperdb.Date)
1174 for nodeid, journaldate, journaltag, action, params in journal:
1175 self.log_debug('addjournal %s%s %r %s %s %r'%(
1176 classname, nodeid, journaldate, journaltag, action,
1177 params))
1179 # make the journalled data marshallable
1180 if isinstance(params, type({})):
1181 self._journal_marshal(params, classname)
1182 params = repr(params)
1184 self.save_journal(classname, cols, nodeid, dc(journaldate),
1185 journaltag, action, params)
1187 def _journal_marshal(self, params, classname):
1188 """Convert the journal params values into safely repr'able and
1189 eval'able values."""
1190 properties = self.getclass(classname).getprops()
1191 for param, value in params.items():
1192 if not value:
1193 continue
1194 property = properties[param]
1195 cvt = self.to_sql_value(property.__class__)
1196 if isinstance(property, Password):
1197 params[param] = cvt(value)
1198 elif isinstance(property, Date):
1199 params[param] = cvt(value)
1200 elif isinstance(property, Interval):
1201 params[param] = cvt(value)
1202 elif isinstance(property, Boolean):
1203 params[param] = cvt(value)
1205 def getjournal(self, classname, nodeid):
1206 """ get the journal for id
1207 """
1208 # make sure the node exists
1209 if not self.hasnode(classname, nodeid):
1210 raise IndexError, '%s has no node %s'%(classname, nodeid)
1212 cols = ','.join('nodeid date tag action params'.split())
1213 journal = self.load_journal(classname, cols, nodeid)
1215 # now unmarshal the data
1216 dc = self.to_hyperdb_value(hyperdb.Date)
1217 res = []
1218 properties = self.getclass(classname).getprops()
1219 for nodeid, date_stamp, user, action, params in journal:
1220 params = eval(params)
1221 if isinstance(params, type({})):
1222 for param, value in params.items():
1223 if not value:
1224 continue
1225 property = properties.get(param, None)
1226 if property is None:
1227 # deleted property
1228 continue
1229 cvt = self.to_hyperdb_value(property.__class__)
1230 if isinstance(property, Password):
1231 params[param] = cvt(value)
1232 elif isinstance(property, Date):
1233 params[param] = cvt(value)
1234 elif isinstance(property, Interval):
1235 params[param] = cvt(value)
1236 elif isinstance(property, Boolean):
1237 params[param] = cvt(value)
1238 # XXX numeric ids
1239 res.append((str(nodeid), dc(date_stamp), user, action, params))
1240 return res
1242 def save_journal(self, classname, cols, nodeid, journaldate,
1243 journaltag, action, params):
1244 """ Save the journal entry to the database
1245 """
1246 entry = (nodeid, journaldate, journaltag, action, params)
1248 # do the insert
1249 a = self.arg
1250 sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(
1251 classname, cols, a, a, a, a, a)
1252 self.sql(sql, entry)
1254 def load_journal(self, classname, cols, nodeid):
1255 """ Load the journal from the database
1256 """
1257 # now get the journal entries
1258 sql = 'select %s from %s__journal where nodeid=%s order by date'%(
1259 cols, classname, self.arg)
1260 self.sql(sql, (nodeid,))
1261 return self.cursor.fetchall()
1263 def pack(self, pack_before):
1264 """ Delete all journal entries except "create" before 'pack_before'.
1265 """
1266 date_stamp = self.to_sql_value(Date)(pack_before)
1268 # do the delete
1269 for classname in self.classes.keys():
1270 sql = "delete from %s__journal where date<%s and "\
1271 "action<>'create'"%(classname, self.arg)
1272 self.sql(sql, (date_stamp,))
1274 def sql_commit(self, fail_ok=False):
1275 """ Actually commit to the database.
1276 """
1277 logging.getLogger('hyperdb').info('commit')
1279 self.conn.commit()
1281 # open a new cursor for subsequent work
1282 self.cursor = self.conn.cursor()
1284 def commit(self, fail_ok=False):
1285 """ Commit the current transactions.
1287 Save all data changed since the database was opened or since the
1288 last commit() or rollback().
1290 fail_ok indicates that the commit is allowed to fail. This is used
1291 in the web interface when committing cleaning of the session
1292 database. We don't care if there's a concurrency issue there.
1294 The only backend this seems to affect is postgres.
1295 """
1296 # commit the database
1297 self.sql_commit(fail_ok)
1299 # now, do all the other transaction stuff
1300 for method, args in self.transactions:
1301 method(*args)
1303 # save the indexer
1304 self.indexer.save_index()
1306 # clear out the transactions
1307 self.transactions = []
1309 def sql_rollback(self):
1310 self.conn.rollback()
1312 def rollback(self):
1313 """ Reverse all actions from the current transaction.
1315 Undo all the changes made since the database was opened or the last
1316 commit() or rollback() was performed.
1317 """
1318 logging.getLogger('hyperdb').info('rollback')
1320 self.sql_rollback()
1322 # roll back "other" transaction stuff
1323 for method, args in self.transactions:
1324 # delete temporary files
1325 if method == self.doStoreFile:
1326 self.rollbackStoreFile(*args)
1327 self.transactions = []
1329 # clear the cache
1330 self.clearCache()
1332 def sql_close(self):
1333 logging.getLogger('hyperdb').info('close')
1334 self.conn.close()
1336 def close(self):
1337 """ Close off the connection.
1338 """
1339 self.indexer.close()
1340 self.sql_close()
1342 #
1343 # The base Class class
1344 #
1345 class Class(hyperdb.Class):
1346 """ The handle to a particular class of nodes in a hyperdatabase.
1348 All methods except __repr__ and getnode must be implemented by a
1349 concrete backend Class.
1350 """
1352 def schema(self):
1353 """ A dumpable version of the schema that we can store in the
1354 database
1355 """
1356 return (self.key, [(x, repr(y)) for x,y in self.properties.items()])
1358 def enableJournalling(self):
1359 """Turn journalling on for this class
1360 """
1361 self.do_journal = 1
1363 def disableJournalling(self):
1364 """Turn journalling off for this class
1365 """
1366 self.do_journal = 0
1368 # Editing nodes:
1369 def create(self, **propvalues):
1370 """ Create a new node of this class and return its id.
1372 The keyword arguments in 'propvalues' map property names to values.
1374 The values of arguments must be acceptable for the types of their
1375 corresponding properties or a TypeError is raised.
1377 If this class has a key property, it must be present and its value
1378 must not collide with other key strings or a ValueError is raised.
1380 Any other properties on this class that are missing from the
1381 'propvalues' dictionary are set to None.
1383 If an id in a link or multilink property does not refer to a valid
1384 node, an IndexError is raised.
1385 """
1386 self.fireAuditors('create', None, propvalues)
1387 newid = self.create_inner(**propvalues)
1388 self.fireReactors('create', newid, None)
1389 return newid
1391 def create_inner(self, **propvalues):
1392 """ Called by create, in-between the audit and react calls.
1393 """
1394 if propvalues.has_key('id'):
1395 raise KeyError, '"id" is reserved'
1397 if self.db.journaltag is None:
1398 raise DatabaseError, _('Database open read-only')
1400 if propvalues.has_key('creator') or propvalues.has_key('actor') or \
1401 propvalues.has_key('creation') or propvalues.has_key('activity'):
1402 raise KeyError, '"creator", "actor", "creation" and '\
1403 '"activity" are reserved'
1405 # new node's id
1406 newid = self.db.newid(self.classname)
1408 # validate propvalues
1409 num_re = re.compile('^\d+$')
1410 for key, value in propvalues.items():
1411 if key == self.key:
1412 try:
1413 self.lookup(value)
1414 except KeyError:
1415 pass
1416 else:
1417 raise ValueError, 'node with key "%s" exists'%value
1419 # try to handle this property
1420 try:
1421 prop = self.properties[key]
1422 except KeyError:
1423 raise KeyError, '"%s" has no property "%s"'%(self.classname,
1424 key)
1426 if value is not None and isinstance(prop, Link):
1427 if type(value) != type(''):
1428 raise ValueError, 'link value must be String'
1429 link_class = self.properties[key].classname
1430 # if it isn't a number, it's a key
1431 if not num_re.match(value):
1432 try:
1433 value = self.db.classes[link_class].lookup(value)
1434 except (TypeError, KeyError):
1435 raise IndexError, 'new property "%s": %s not a %s'%(
1436 key, value, link_class)
1437 elif not self.db.getclass(link_class).hasnode(value):
1438 raise IndexError, '%s has no node %s'%(link_class, value)
1440 # save off the value
1441 propvalues[key] = value
1443 # register the link with the newly linked node
1444 if self.do_journal and self.properties[key].do_journal:
1445 self.db.addjournal(link_class, value, 'link',
1446 (self.classname, newid, key))
1448 elif isinstance(prop, Multilink):
1449 if value is None:
1450 value = []
1451 if not hasattr(value, '__iter__'):
1452 raise TypeError, 'new property "%s" not an iterable of ids'%key
1454 # clean up and validate the list of links
1455 link_class = self.properties[key].classname
1456 l = []
1457 for entry in value:
1458 if type(entry) != type(''):
1459 raise ValueError, '"%s" multilink value (%r) '\
1460 'must contain Strings'%(key, value)
1461 # if it isn't a number, it's a key
1462 if not num_re.match(entry):
1463 try:
1464 entry = self.db.classes[link_class].lookup(entry)
1465 except (TypeError, KeyError):
1466 raise IndexError, 'new property "%s": %s not a %s'%(
1467 key, entry, self.properties[key].classname)
1468 l.append(entry)
1469 value = l
1470 propvalues[key] = value
1472 # handle additions
1473 for nodeid in value:
1474 if not self.db.getclass(link_class).hasnode(nodeid):
1475 raise IndexError, '%s has no node %s'%(link_class,
1476 nodeid)
1477 # register the link with the newly linked node
1478 if self.do_journal and self.properties[key].do_journal:
1479 self.db.addjournal(link_class, nodeid, 'link',
1480 (self.classname, newid, key))
1482 elif isinstance(prop, String):
1483 if type(value) != type('') and type(value) != type(u''):
1484 raise TypeError, 'new property "%s" not a string'%key
1485 if prop.indexme:
1486 self.db.indexer.add_text((self.classname, newid, key),
1487 value)
1489 elif isinstance(prop, Password):
1490 if not isinstance(value, password.Password):
1491 raise TypeError, 'new property "%s" not a Password'%key
1493 elif isinstance(prop, Date):
1494 if value is not None and not isinstance(value, date.Date):
1495 raise TypeError, 'new property "%s" not a Date'%key
1497 elif isinstance(prop, Interval):
1498 if value is not None and not isinstance(value, date.Interval):
1499 raise TypeError, 'new property "%s" not an Interval'%key
1501 elif value is not None and isinstance(prop, Number):
1502 try:
1503 float(value)
1504 except ValueError:
1505 raise TypeError, 'new property "%s" not numeric'%key
1507 elif value is not None and isinstance(prop, Boolean):
1508 try:
1509 int(value)
1510 except ValueError:
1511 raise TypeError, 'new property "%s" not boolean'%key
1513 # make sure there's data where there needs to be
1514 for key, prop in self.properties.items():
1515 if propvalues.has_key(key):
1516 continue
1517 if key == self.key:
1518 raise ValueError, 'key property "%s" is required'%key
1519 if isinstance(prop, Multilink):
1520 propvalues[key] = []
1521 else:
1522 propvalues[key] = None
1524 # done
1525 self.db.addnode(self.classname, newid, propvalues)
1526 if self.do_journal:
1527 self.db.addjournal(self.classname, newid, ''"create", {})
1529 # XXX numeric ids
1530 return str(newid)
1532 def get(self, nodeid, propname, default=_marker, cache=1):
1533 """Get the value of a property on an existing node of this class.
1535 'nodeid' must be the id of an existing node of this class or an
1536 IndexError is raised. 'propname' must be the name of a property
1537 of this class or a KeyError is raised.
1539 'cache' exists for backwards compatibility, and is not used.
1540 """
1541 if propname == 'id':
1542 return nodeid
1544 # get the node's dict
1545 d = self.db.getnode(self.classname, nodeid)
1547 if propname == 'creation':
1548 if d.has_key('creation'):
1549 return d['creation']
1550 else:
1551 return date.Date()
1552 if propname == 'activity':
1553 if d.has_key('activity'):
1554 return d['activity']
1555 else:
1556 return date.Date()
1557 if propname == 'creator':
1558 if d.has_key('creator'):
1559 return d['creator']
1560 else:
1561 return self.db.getuid()
1562 if propname == 'actor':
1563 if d.has_key('actor'):
1564 return d['actor']
1565 else:
1566 return self.db.getuid()
1568 # get the property (raises KeyErorr if invalid)
1569 prop = self.properties[propname]
1571 # XXX may it be that propname is valid property name
1572 # (above error is not raised) and not d.has_key(propname)???
1573 if (not d.has_key(propname)) or (d[propname] is None):
1574 if default is _marker:
1575 if isinstance(prop, Multilink):
1576 return []
1577 else:
1578 return None
1579 else:
1580 return default
1582 # don't pass our list to other code
1583 if isinstance(prop, Multilink):
1584 return d[propname][:]
1586 return d[propname]
1588 def set(self, nodeid, **propvalues):
1589 """Modify a property on an existing node of this class.
1591 'nodeid' must be the id of an existing node of this class or an
1592 IndexError is raised.
1594 Each key in 'propvalues' must be the name of a property of this
1595 class or a KeyError is raised.
1597 All values in 'propvalues' must be acceptable types for their
1598 corresponding properties or a TypeError is raised.
1600 If the value of the key property is set, it must not collide with
1601 other key strings or a ValueError is raised.
1603 If the value of a Link or Multilink property contains an invalid
1604 node id, a ValueError is raised.
1605 """
1606 self.fireAuditors('set', nodeid, propvalues)
1607 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1608 propvalues = self.set_inner(nodeid, **propvalues)
1609 self.fireReactors('set', nodeid, oldvalues)
1610 return propvalues
1612 def set_inner(self, nodeid, **propvalues):
1613 """ Called by set, in-between the audit and react calls.
1614 """
1615 if not propvalues:
1616 return propvalues
1618 if propvalues.has_key('creation') or propvalues.has_key('creator') or \
1619 propvalues.has_key('actor') or propvalues.has_key('activity'):
1620 raise KeyError, '"creation", "creator", "actor" and '\
1621 '"activity" are reserved'
1623 if propvalues.has_key('id'):
1624 raise KeyError, '"id" is reserved'
1626 if self.db.journaltag is None:
1627 raise DatabaseError, _('Database open read-only')
1629 node = self.db.getnode(self.classname, nodeid)
1630 if self.is_retired(nodeid):
1631 raise IndexError, 'Requested item is retired'
1632 num_re = re.compile('^\d+$')
1634 # make a copy of the values dictionary - we'll modify the contents
1635 propvalues = propvalues.copy()
1637 # if the journal value is to be different, store it in here
1638 journalvalues = {}
1640 # remember the add/remove stuff for multilinks, making it easier
1641 # for the Database layer to do its stuff
1642 multilink_changes = {}
1644 for propname, value in propvalues.items():
1645 # check to make sure we're not duplicating an existing key
1646 if propname == self.key and node[propname] != value:
1647 try:
1648 self.lookup(value)
1649 except KeyError:
1650 pass
1651 else:
1652 raise ValueError, 'node with key "%s" exists'%value
1654 # this will raise the KeyError if the property isn't valid
1655 # ... we don't use getprops() here because we only care about
1656 # the writeable properties.
1657 try:
1658 prop = self.properties[propname]
1659 except KeyError:
1660 raise KeyError, '"%s" has no property named "%s"'%(
1661 self.classname, propname)
1663 # if the value's the same as the existing value, no sense in
1664 # doing anything
1665 current = node.get(propname, None)
1666 if value == current:
1667 del propvalues[propname]
1668 continue
1669 journalvalues[propname] = current
1671 # do stuff based on the prop type
1672 if isinstance(prop, Link):
1673 link_class = prop.classname
1674 # if it isn't a number, it's a key
1675 if value is not None and not isinstance(value, type('')):
1676 raise ValueError, 'property "%s" link value be a string'%(
1677 propname)
1678 if isinstance(value, type('')) and not num_re.match(value):
1679 try:
1680 value = self.db.classes[link_class].lookup(value)
1681 except (TypeError, KeyError):
1682 raise IndexError, 'new property "%s": %s not a %s'%(
1683 propname, value, prop.classname)
1685 if (value is not None and
1686 not self.db.getclass(link_class).hasnode(value)):
1687 raise IndexError, '%s has no node %s'%(link_class, value)
1689 if self.do_journal and prop.do_journal:
1690 # register the unlink with the old linked node
1691 if node[propname] is not None:
1692 self.db.addjournal(link_class, node[propname],
1693 ''"unlink", (self.classname, nodeid, propname))
1695 # register the link with the newly linked node
1696 if value is not None:
1697 self.db.addjournal(link_class, value, ''"link",
1698 (self.classname, nodeid, propname))
1700 elif isinstance(prop, Multilink):
1701 if value is None:
1702 value = []
1703 if not hasattr(value, '__iter__'):
1704 raise TypeError, 'new property "%s" not an iterable of'\
1705 ' ids'%propname
1706 link_class = self.properties[propname].classname
1707 l = []
1708 for entry in value:
1709 # if it isn't a number, it's a key
1710 if type(entry) != type(''):
1711 raise ValueError, 'new property "%s" link value ' \
1712 'must be a string'%propname
1713 if not num_re.match(entry):
1714 try:
1715 entry = self.db.classes[link_class].lookup(entry)
1716 except (TypeError, KeyError):
1717 raise IndexError, 'new property "%s": %s not a %s'%(
1718 propname, entry,
1719 self.properties[propname].classname)
1720 l.append(entry)
1721 value = l
1722 propvalues[propname] = value
1724 # figure the journal entry for this property
1725 add = []
1726 remove = []
1728 # handle removals
1729 if node.has_key(propname):
1730 l = node[propname]
1731 else:
1732 l = []
1733 for id in l[:]:
1734 if id in value:
1735 continue
1736 # register the unlink with the old linked node
1737 if self.do_journal and self.properties[propname].do_journal:
1738 self.db.addjournal(link_class, id, 'unlink',
1739 (self.classname, nodeid, propname))
1740 l.remove(id)
1741 remove.append(id)
1743 # handle additions
1744 for id in value:
1745 # If this node is in the cache, then we do not need to go to
1746 # the database. (We don't consider this an LRU hit, though.)
1747 if self.cache.has_key((classname, nodeid)):
1748 # Return 1, not True, to match the type of the result of
1749 # the SQL operation below.
1750 return 1
1751 if not self.db.getclass(link_class).hasnode(id):
1752 raise IndexError, '%s has no node %s'%(link_class, id)
1753 if id in l:
1754 continue
1755 # register the link with the newly linked node
1756 if self.do_journal and self.properties[propname].do_journal:
1757 self.db.addjournal(link_class, id, 'link',
1758 (self.classname, nodeid, propname))
1759 l.append(id)
1760 add.append(id)
1762 # figure the journal entry
1763 l = []
1764 if add:
1765 l.append(('+', add))
1766 if remove:
1767 l.append(('-', remove))
1768 multilink_changes[propname] = (add, remove)
1769 if l:
1770 journalvalues[propname] = tuple(l)
1772 elif isinstance(prop, String):
1773 if value is not None and type(value) != type('') and type(value) != type(u''):
1774 raise TypeError, 'new property "%s" not a string'%propname
1775 if prop.indexme:
1776 if value is None: value = ''
1777 self.db.indexer.add_text((self.classname, nodeid, propname),
1778 value)
1780 elif isinstance(prop, Password):
1781 if not isinstance(value, password.Password):
1782 raise TypeError, 'new property "%s" not a Password'%propname
1783 propvalues[propname] = value
1785 elif value is not None and isinstance(prop, Date):
1786 if not isinstance(value, date.Date):
1787 raise TypeError, 'new property "%s" not a Date'% propname
1788 propvalues[propname] = value
1790 elif value is not None and isinstance(prop, Interval):
1791 if not isinstance(value, date.Interval):
1792 raise TypeError, 'new property "%s" not an '\
1793 'Interval'%propname
1794 propvalues[propname] = value
1796 elif value is not None and isinstance(prop, Number):
1797 try:
1798 float(value)
1799 except ValueError:
1800 raise TypeError, 'new property "%s" not numeric'%propname
1802 elif value is not None and isinstance(prop, Boolean):
1803 try:
1804 int(value)
1805 except ValueError:
1806 raise TypeError, 'new property "%s" not boolean'%propname
1808 # nothing to do?
1809 if not propvalues:
1810 return propvalues
1812 # update the activity time
1813 propvalues['activity'] = date.Date()
1814 propvalues['actor'] = self.db.getuid()
1816 # do the set
1817 self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
1819 # remove the activity props now they're handled
1820 del propvalues['activity']
1821 del propvalues['actor']
1823 # journal the set
1824 if self.do_journal:
1825 self.db.addjournal(self.classname, nodeid, ''"set", journalvalues)
1827 return propvalues
1829 def retire(self, nodeid):
1830 """Retire a node.
1832 The properties on the node remain available from the get() method,
1833 and the node's id is never reused.
1835 Retired nodes are not returned by the find(), list(), or lookup()
1836 methods, and other nodes may reuse the values of their key properties.
1837 """
1838 if self.db.journaltag is None:
1839 raise DatabaseError, _('Database open read-only')
1841 self.fireAuditors('retire', nodeid, None)
1843 # use the arg for __retired__ to cope with any odd database type
1844 # conversion (hello, sqlite)
1845 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1846 self.db.arg, self.db.arg)
1847 self.db.sql(sql, (nodeid, nodeid))
1848 if self.do_journal:
1849 self.db.addjournal(self.classname, nodeid, ''"retired", None)
1851 self.fireReactors('retire', nodeid, None)
1853 def restore(self, nodeid):
1854 """Restore a retired node.
1856 Make node available for all operations like it was before retirement.
1857 """
1858 if self.db.journaltag is None:
1859 raise DatabaseError, _('Database open read-only')
1861 node = self.db.getnode(self.classname, nodeid)
1862 # check if key property was overrided
1863 key = self.getkey()
1864 try:
1865 id = self.lookup(node[key])
1866 except KeyError:
1867 pass
1868 else:
1869 raise KeyError, "Key property (%s) of retired node clashes with \
1870 existing one (%s)" % (key, node[key])
1872 self.fireAuditors('restore', nodeid, None)
1873 # use the arg for __retired__ to cope with any odd database type
1874 # conversion (hello, sqlite)
1875 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1876 self.db.arg, self.db.arg)
1877 self.db.sql(sql, (0, nodeid))
1878 if self.do_journal:
1879 self.db.addjournal(self.classname, nodeid, ''"restored", None)
1881 self.fireReactors('restore', nodeid, None)
1883 def is_retired(self, nodeid):
1884 """Return true if the node is rerired
1885 """
1886 sql = 'select __retired__ from _%s where id=%s'%(self.classname,
1887 self.db.arg)
1888 self.db.sql(sql, (nodeid,))
1889 return int(self.db.sql_fetchone()[0]) > 0
1891 def destroy(self, nodeid):
1892 """Destroy a node.
1894 WARNING: this method should never be used except in extremely rare
1895 situations where there could never be links to the node being
1896 deleted
1898 WARNING: use retire() instead
1900 WARNING: the properties of this node will not be available ever again
1902 WARNING: really, use retire() instead
1904 Well, I think that's enough warnings. This method exists mostly to
1905 support the session storage of the cgi interface.
1907 The node is completely removed from the hyperdb, including all journal
1908 entries. It will no longer be available, and will generally break code
1909 if there are any references to the node.
1910 """
1911 if self.db.journaltag is None:
1912 raise DatabaseError, _('Database open read-only')
1913 self.db.destroynode(self.classname, nodeid)
1915 def history(self, nodeid):
1916 """Retrieve the journal of edits on a particular node.
1918 'nodeid' must be the id of an existing node of this class or an
1919 IndexError is raised.
1921 The returned list contains tuples of the form
1923 (nodeid, date, tag, action, params)
1925 'date' is a Timestamp object specifying the time of the change and
1926 'tag' is the journaltag specified when the database was opened.
1927 """
1928 if not self.do_journal:
1929 raise ValueError, 'Journalling is disabled for this class'
1930 return self.db.getjournal(self.classname, nodeid)
1932 # Locating nodes:
1933 def hasnode(self, nodeid):
1934 """Determine if the given nodeid actually exists
1935 """
1936 return self.db.hasnode(self.classname, nodeid)
1938 def setkey(self, propname):
1939 """Select a String property of this class to be the key property.
1941 'propname' must be the name of a String property of this class or
1942 None, or a TypeError is raised. The values of the key property on
1943 all existing nodes must be unique or a ValueError is raised.
1944 """
1945 prop = self.getprops()[propname]
1946 if not isinstance(prop, String):
1947 raise TypeError, 'key properties must be String'
1948 self.key = propname
1950 def getkey(self):
1951 """Return the name of the key property for this class or None."""
1952 return self.key
1954 def lookup(self, keyvalue):
1955 """Locate a particular node by its key property and return its id.
1957 If this class has no key property, a TypeError is raised. If the
1958 'keyvalue' matches one of the values for the key property among
1959 the nodes in this class, the matching node's id is returned;
1960 otherwise a KeyError is raised.
1961 """
1962 if not self.key:
1963 raise TypeError, 'No key property set for class %s'%self.classname
1965 # use the arg to handle any odd database type conversion (hello,
1966 # sqlite)
1967 sql = "select id from _%s where _%s=%s and __retired__=%s"%(
1968 self.classname, self.key, self.db.arg, self.db.arg)
1969 self.db.sql(sql, (str(keyvalue), 0))
1971 # see if there was a result that's not retired
1972 row = self.db.sql_fetchone()
1973 if not row:
1974 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1975 keyvalue, self.classname)
1977 # return the id
1978 # XXX numeric ids
1979 return str(row[0])
1981 def find(self, **propspec):
1982 """Get the ids of nodes in this class which link to the given nodes.
1984 'propspec' consists of keyword args propname=nodeid or
1985 propname={nodeid:1, }
1986 'propname' must be the name of a property in this class, or a
1987 KeyError is raised. That property must be a Link or
1988 Multilink property, or a TypeError is raised.
1990 Any node in this class whose 'propname' property links to any of
1991 the nodeids will be returned. Examples::
1993 db.issue.find(messages='1')
1994 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1995 """
1996 # shortcut
1997 if not propspec:
1998 return []
2000 # validate the args
2001 props = self.getprops()
2002 propspec = propspec.items()
2003 for propname, nodeids in propspec:
2004 # check the prop is OK
2005 prop = props[propname]
2006 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
2007 raise TypeError, "'%s' not a Link/Multilink property"%propname
2009 # first, links
2010 a = self.db.arg
2011 allvalues = ()
2012 sql = []
2013 where = []
2014 for prop, values in propspec:
2015 if not isinstance(props[prop], hyperdb.Link):
2016 continue
2017 if type(values) is type({}) and len(values) == 1:
2018 values = values.keys()[0]
2019 if type(values) is type(''):
2020 allvalues += (values,)
2021 where.append('_%s = %s'%(prop, a))
2022 elif values is None:
2023 where.append('_%s is NULL'%prop)
2024 else:
2025 values = values.keys()
2026 s = ''
2027 if None in values:
2028 values.remove(None)
2029 s = '_%s is NULL or '%prop
2030 allvalues += tuple(values)
2031 s += '_%s in (%s)'%(prop, ','.join([a]*len(values)))
2032 where.append('(' + s +')')
2033 if where:
2034 allvalues = (0, ) + allvalues
2035 sql.append("""select id from _%s where __retired__=%s
2036 and %s"""%(self.classname, a, ' and '.join(where)))
2038 # now multilinks
2039 for prop, values in propspec:
2040 if not isinstance(props[prop], hyperdb.Multilink):
2041 continue
2042 if not values:
2043 continue
2044 allvalues += (0, )
2045 if type(values) is type(''):
2046 allvalues += (values,)
2047 s = a
2048 else:
2049 allvalues += tuple(values.keys())
2050 s = ','.join([a]*len(values))
2051 tn = '%s_%s'%(self.classname, prop)
2052 sql.append("""select id from _%s, %s where __retired__=%s
2053 and id = %s.nodeid and %s.linkid in (%s)"""%(self.classname,
2054 tn, a, tn, tn, s))
2056 if not sql:
2057 return []
2058 sql = ' union '.join(sql)
2059 self.db.sql(sql, allvalues)
2060 # XXX numeric ids
2061 l = [str(x[0]) for x in self.db.sql_fetchall()]
2062 return l
2064 def stringFind(self, **requirements):
2065 """Locate a particular node by matching a set of its String
2066 properties in a caseless search.
2068 If the property is not a String property, a TypeError is raised.
2070 The return is a list of the id of all nodes that match.
2071 """
2072 where = []
2073 args = []
2074 for propname in requirements.keys():
2075 prop = self.properties[propname]
2076 if not isinstance(prop, String):
2077 raise TypeError, "'%s' not a String property"%propname
2078 where.append(propname)
2079 args.append(requirements[propname].lower())
2081 # generate the where clause
2082 s = ' and '.join(['lower(_%s)=%s'%(col, self.db.arg) for col in where])
2083 sql = 'select id from _%s where %s and __retired__=%s'%(
2084 self.classname, s, self.db.arg)
2085 args.append(0)
2086 self.db.sql(sql, tuple(args))
2087 # XXX numeric ids
2088 l = [str(x[0]) for x in self.db.sql_fetchall()]
2089 return l
2091 def list(self):
2092 """ Return a list of the ids of the active nodes in this class.
2093 """
2094 return self.getnodeids(retired=0)
2096 def getnodeids(self, retired=None):
2097 """ Retrieve all the ids of the nodes for a particular Class.
2099 Set retired=None to get all nodes. Otherwise it'll get all the
2100 retired or non-retired nodes, depending on the flag.
2101 """
2102 # flip the sense of the 'retired' flag if we don't want all of them
2103 if retired is not None:
2104 args = (0, )
2105 if retired:
2106 compare = '>'
2107 else:
2108 compare = '='
2109 sql = 'select id from _%s where __retired__%s%s'%(self.classname,
2110 compare, self.db.arg)
2111 else:
2112 args = ()
2113 sql = 'select id from _%s'%self.classname
2114 self.db.sql(sql, args)
2115 # XXX numeric ids
2116 ids = [str(x[0]) for x in self.db.cursor.fetchall()]
2117 return ids
2119 def _subselect(self, classname, multilink_table):
2120 """Create a subselect. This is factored out because some
2121 databases (hmm only one, so far) doesn't support subselects
2122 look for "I can't believe it's not a toy RDBMS" in the mysql
2123 backend.
2124 """
2125 return '_%s.id not in (select nodeid from %s)'%(classname,
2126 multilink_table)
2128 # Some DBs order NULL values last. Set this variable in the backend
2129 # for prepending an order by clause for each attribute that causes
2130 # correct sort order for NULLs. Examples:
2131 # order_by_null_values = '(%s is not NULL)'
2132 # order_by_null_values = 'notnull(%s)'
2133 # The format parameter is replaced with the attribute.
2134 order_by_null_values = None
2136 def filter(self, search_matches, filterspec, sort=[], group=[]):
2137 """Return a list of the ids of the active nodes in this class that
2138 match the 'filter' spec, sorted by the group spec and then the
2139 sort spec
2141 "filterspec" is {propname: value(s)}
2143 "sort" and "group" are [(dir, prop), ...] where dir is '+', '-'
2144 or None and prop is a prop name or None. Note that for
2145 backward-compatibility reasons a single (dir, prop) tuple is
2146 also allowed.
2148 "search_matches" is a container type or None
2150 The filter must match all properties specificed. If the property
2151 value to match is a list:
2153 1. String properties must match all elements in the list, and
2154 2. Other properties must match any of the elements in the list.
2155 """
2156 # we can't match anything if search_matches is empty
2157 if not search_matches and search_matches is not None:
2158 return []
2160 if __debug__:
2161 start_t = time.time()
2163 icn = self.classname
2165 # vars to hold the components of the SQL statement
2166 frum = [] # FROM clauses
2167 loj = [] # LEFT OUTER JOIN clauses
2168 where = [] # WHERE clauses
2169 args = [] # *any* positional arguments
2170 a = self.db.arg
2172 # figure the WHERE clause from the filterspec
2173 mlfilt = 0 # are we joining with Multilink tables?
2174 sortattr = self._sortattr (group = group, sort = sort)
2175 proptree = self._proptree(filterspec, sortattr)
2176 mlseen = 0
2177 for pt in reversed(proptree.sortattr):
2178 p = pt
2179 while p.parent:
2180 if isinstance (p.propclass, Multilink):
2181 mlseen = True
2182 if mlseen:
2183 p.sort_ids_needed = True
2184 p.tree_sort_done = False
2185 p = p.parent
2186 if not mlseen:
2187 pt.attr_sort_done = pt.tree_sort_done = True
2188 proptree.compute_sort_done()
2190 ordercols = []
2191 auxcols = {}
2192 mlsort = []
2193 rhsnum = 0
2194 for p in proptree:
2195 oc = None
2196 cn = p.classname
2197 ln = p.uniqname
2198 pln = p.parent.uniqname
2199 pcn = p.parent.classname
2200 k = p.name
2201 v = p.val
2202 propclass = p.propclass
2203 if p.sort_type > 0:
2204 oc = ac = '_%s._%s'%(pln, k)
2205 if isinstance(propclass, Multilink):
2206 if p.sort_type < 2:
2207 mlfilt = 1
2208 tn = '%s_%s'%(pcn, k)
2209 if v in ('-1', ['-1'], []):
2210 # only match rows that have count(linkid)=0 in the
2211 # corresponding multilink table)
2212 where.append(self._subselect(pcn, tn))
2213 else:
2214 frum.append(tn)
2215 where.append('_%s.id=%s.nodeid'%(pln,tn))
2216 if p.children:
2217 frum.append('_%s as _%s' % (cn, ln))
2218 where.append('%s.linkid=_%s.id'%(tn, ln))
2219 if p.has_values:
2220 if isinstance(v, type([])):
2221 s = ','.join([a for x in v])
2222 where.append('%s.linkid in (%s)'%(tn, s))
2223 args = args + v
2224 else:
2225 where.append('%s.linkid=%s'%(tn, a))
2226 args.append(v)
2227 if p.sort_type > 0:
2228 assert not p.attr_sort_done and not p.sort_ids_needed
2229 elif k == 'id':
2230 if p.sort_type < 2:
2231 if isinstance(v, type([])):
2232 s = ','.join([a for x in v])
2233 where.append('_%s.%s in (%s)'%(pln, k, s))
2234 args = args + v
2235 else:
2236 where.append('_%s.%s=%s'%(pln, k, a))
2237 args.append(v)
2238 if p.sort_type > 0:
2239 oc = ac = '_%s.id'%pln
2240 elif isinstance(propclass, String):
2241 if p.sort_type < 2:
2242 if not isinstance(v, type([])):
2243 v = [v]
2245 # Quote the bits in the string that need it and then embed
2246 # in a "substring" search. Note - need to quote the '%' so
2247 # they make it through the python layer happily
2248 v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
2250 # now add to the where clause
2251 where.append('('
2252 +' and '.join(["_%s._%s LIKE '%s'"%(pln, k, s) for s in v])
2253 +')')
2254 # note: args are embedded in the query string now
2255 if p.sort_type > 0:
2256 oc = ac = 'lower(_%s._%s)'%(pln, k)
2257 elif isinstance(propclass, Link):
2258 if p.sort_type < 2:
2259 if p.children:
2260 if p.sort_type == 0:
2261 frum.append('_%s as _%s' % (cn, ln))
2262 where.append('_%s._%s=_%s.id'%(pln, k, ln))
2263 if p.has_values:
2264 if isinstance(v, type([])):
2265 d = {}
2266 for entry in v:
2267 if entry == '-1':
2268 entry = None
2269 d[entry] = entry
2270 l = []
2271 if d.has_key(None) or not d:
2272 if d.has_key(None): del d[None]
2273 l.append('_%s._%s is NULL'%(pln, k))
2274 if d:
2275 v = d.keys()
2276 s = ','.join([a for x in v])
2277 l.append('(_%s._%s in (%s))'%(pln, k, s))
2278 args = args + v
2279 if l:
2280 where.append('(' + ' or '.join(l) +')')
2281 else:
2282 if v in ('-1', None):
2283 v = None
2284 where.append('_%s._%s is NULL'%(pln, k))
2285 else:
2286 where.append('_%s._%s=%s'%(pln, k, a))
2287 args.append(v)
2288 if p.sort_type > 0:
2289 lp = p.cls.labelprop()
2290 oc = ac = '_%s._%s'%(pln, k)
2291 if lp != 'id':
2292 if p.tree_sort_done and p.sort_type > 0:
2293 loj.append(
2294 'LEFT OUTER JOIN _%s as _%s on _%s._%s=_%s.id'%(
2295 cn, ln, pln, k, ln))
2296 oc = '_%s._%s'%(ln, lp)
2297 elif isinstance(propclass, Date) and p.sort_type < 2:
2298 dc = self.db.to_sql_value(hyperdb.Date)
2299 if isinstance(v, type([])):
2300 s = ','.join([a for x in v])
2301 where.append('_%s._%s in (%s)'%(pln, k, s))
2302 args = args + [dc(date.Date(x)) for x in v]
2303 else:
2304 try:
2305 # Try to filter on range of dates
2306 date_rng = propclass.range_from_raw(v, self.db)
2307 if date_rng.from_value:
2308 where.append('_%s._%s >= %s'%(pln, k, a))
2309 args.append(dc(date_rng.from_value))
2310 if date_rng.to_value:
2311 where.append('_%s._%s <= %s'%(pln, k, a))
2312 args.append(dc(date_rng.to_value))
2313 except ValueError:
2314 # If range creation fails - ignore that search parameter
2315 pass
2316 elif isinstance(propclass, Interval):
2317 # filter/sort using the __<prop>_int__ column
2318 if p.sort_type < 2:
2319 if isinstance(v, type([])):
2320 s = ','.join([a for x in v])
2321 where.append('_%s.__%s_int__ in (%s)'%(pln, k, s))
2322 args = args + [date.Interval(x).as_seconds() for x in v]
2323 else:
2324 try:
2325 # Try to filter on range of intervals
2326 date_rng = Range(v, date.Interval)
2327 if date_rng.from_value:
2328 where.append('_%s.__%s_int__ >= %s'%(pln, k, a))
2329 args.append(date_rng.from_value.as_seconds())
2330 if date_rng.to_value:
2331 where.append('_%s.__%s_int__ <= %s'%(pln, k, a))
2332 args.append(date_rng.to_value.as_seconds())
2333 except ValueError:
2334 # If range creation fails - ignore search parameter
2335 pass
2336 if p.sort_type > 0:
2337 oc = ac = '_%s.__%s_int__'%(pln,k)
2338 elif p.sort_type < 2:
2339 if isinstance(v, type([])):
2340 s = ','.join([a for x in v])
2341 where.append('_%s._%s in (%s)'%(pln, k, s))
2342 args = args + v
2343 else:
2344 where.append('_%s._%s=%s'%(pln, k, a))
2345 args.append(v)
2346 if oc:
2347 if p.sort_ids_needed:
2348 auxcols[ac] = p
2349 if p.tree_sort_done and p.sort_direction:
2350 # Don't select top-level id twice
2351 if p.name != 'id' or p.parent != proptree:
2352 ordercols.append(oc)
2353 desc = ['', ' desc'][p.sort_direction == '-']
2354 # Some SQL dbs sort NULL values last -- we want them first.
2355 if (self.order_by_null_values and p.name != 'id'):
2356 nv = self.order_by_null_values % oc
2357 ordercols.append(nv)
2358 p.orderby.append(nv + desc)
2359 p.orderby.append(oc + desc)
2361 props = self.getprops()
2363 # don't match retired nodes
2364 where.append('_%s.__retired__=0'%icn)
2366 # add results of full text search
2367 if search_matches is not None:
2368 s = ','.join([a for x in search_matches])
2369 where.append('_%s.id in (%s)'%(icn, s))
2370 args = args + v
2372 # construct the SQL
2373 frum.append('_'+icn)
2374 frum = ','.join(frum)
2375 if where:
2376 where = ' where ' + (' and '.join(where))
2377 else:
2378 where = ''
2379 if mlfilt:
2380 # we're joining tables on the id, so we will get dupes if we
2381 # don't distinct()
2382 cols = ['distinct(_%s.id)'%icn]
2383 else:
2384 cols = ['_%s.id'%icn]
2385 if ordercols:
2386 cols = cols + ordercols
2387 order = []
2388 # keep correct sequence of order attributes.
2389 for sa in proptree.sortattr:
2390 if not sa.attr_sort_done:
2391 continue
2392 order.extend(sa.orderby)
2393 if order:
2394 order = ' order by %s'%(','.join(order))
2395 else:
2396 order = ''
2397 for o, p in auxcols.iteritems ():
2398 cols.append (o)
2399 p.auxcol = len (cols) - 1
2401 cols = ','.join(cols)
2402 loj = ' '.join(loj)
2403 sql = 'select %s from %s %s %s%s'%(cols, frum, loj, where, order)
2404 args = tuple(args)
2405 __traceback_info__ = (sql, args)
2406 self.db.sql(sql, args)
2407 l = self.db.sql_fetchall()
2409 # Compute values needed for sorting in proptree.sort
2410 for p in auxcols.itervalues():
2411 p.sort_ids = p.sort_result = [row[p.auxcol] for row in l]
2412 # return the IDs (the first column)
2413 # XXX numeric ids
2414 l = [str(row[0]) for row in l]
2415 l = proptree.sort (l)
2417 if __debug__:
2418 self.db.stats['filtering'] += (time.time() - start_t)
2419 return l
2421 def filter_sql(self, sql):
2422 """Return a list of the ids of the items in this class that match
2423 the SQL provided. The SQL is a complete "select" statement.
2425 The SQL select must include the item id as the first column.
2427 This function DOES NOT filter out retired items, add on a where
2428 clause "__retired__=0" if you don't want retired nodes.
2429 """
2430 if __debug__:
2431 start_t = time.time()
2433 self.db.sql(sql)
2434 l = self.db.sql_fetchall()
2436 if __debug__:
2437 self.db.stats['filtering'] += (time.time() - start_t)
2438 return l
2440 def count(self):
2441 """Get the number of nodes in this class.
2443 If the returned integer is 'numnodes', the ids of all the nodes
2444 in this class run from 1 to numnodes, and numnodes+1 will be the
2445 id of the next node to be created in this class.
2446 """
2447 return self.db.countnodes(self.classname)
2449 # Manipulating properties:
2450 def getprops(self, protected=1):
2451 """Return a dictionary mapping property names to property objects.
2452 If the "protected" flag is true, we include protected properties -
2453 those which may not be modified.
2454 """
2455 d = self.properties.copy()
2456 if protected:
2457 d['id'] = String()
2458 d['creation'] = hyperdb.Date()
2459 d['activity'] = hyperdb.Date()
2460 d['creator'] = hyperdb.Link('user')
2461 d['actor'] = hyperdb.Link('user')
2462 return d
2464 def addprop(self, **properties):
2465 """Add properties to this class.
2467 The keyword arguments in 'properties' must map names to property
2468 objects, or a TypeError is raised. None of the keys in 'properties'
2469 may collide with the names of existing properties, or a ValueError
2470 is raised before any properties have been added.
2471 """
2472 for key in properties.keys():
2473 if self.properties.has_key(key):
2474 raise ValueError, key
2475 self.properties.update(properties)
2477 def index(self, nodeid):
2478 """Add (or refresh) the node to search indexes
2479 """
2480 # find all the String properties that have indexme
2481 for prop, propclass in self.getprops().items():
2482 if isinstance(propclass, String) and propclass.indexme:
2483 self.db.indexer.add_text((self.classname, nodeid, prop),
2484 str(self.get(nodeid, prop)))
2486 #
2487 # import / export support
2488 #
2489 def export_list(self, propnames, nodeid):
2490 """ Export a node - generate a list of CSV-able data in the order
2491 specified by propnames for the given node.
2492 """
2493 properties = self.getprops()
2494 l = []
2495 for prop in propnames:
2496 proptype = properties[prop]
2497 value = self.get(nodeid, prop)
2498 # "marshal" data where needed
2499 if value is None:
2500 pass
2501 elif isinstance(proptype, hyperdb.Date):
2502 value = value.get_tuple()
2503 elif isinstance(proptype, hyperdb.Interval):
2504 value = value.get_tuple()
2505 elif isinstance(proptype, hyperdb.Password):
2506 value = str(value)
2507 l.append(repr(value))
2508 l.append(repr(self.is_retired(nodeid)))
2509 return l
2511 def import_list(self, propnames, proplist):
2512 """ Import a node - all information including "id" is present and
2513 should not be sanity checked. Triggers are not triggered. The
2514 journal should be initialised using the "creator" and "created"
2515 information.
2517 Return the nodeid of the node imported.
2518 """
2519 if self.db.journaltag is None:
2520 raise DatabaseError, _('Database open read-only')
2521 properties = self.getprops()
2523 # make the new node's property map
2524 d = {}
2525 retire = 0
2526 if not "id" in propnames:
2527 newid = self.db.newid(self.classname)
2528 else:
2529 newid = eval(proplist[propnames.index("id")])
2530 for i in range(len(propnames)):
2531 # Use eval to reverse the repr() used to output the CSV
2532 value = eval(proplist[i])
2534 # Figure the property for this column
2535 propname = propnames[i]
2537 # "unmarshal" where necessary
2538 if propname == 'id':
2539 continue
2540 elif propname == 'is retired':
2541 # is the item retired?
2542 if int(value):
2543 retire = 1
2544 continue
2545 elif value is None:
2546 d[propname] = None
2547 continue
2549 prop = properties[propname]
2550 if value is None:
2551 # don't set Nones
2552 continue
2553 elif isinstance(prop, hyperdb.Date):
2554 value = date.Date(value)
2555 elif isinstance(prop, hyperdb.Interval):
2556 value = date.Interval(value)
2557 elif isinstance(prop, hyperdb.Password):
2558 pwd = password.Password()
2559 pwd.unpack(value)
2560 value = pwd
2561 elif isinstance(prop, String):
2562 if isinstance(value, unicode):
2563 value = value.encode('utf8')
2564 if not isinstance(value, str):
2565 raise TypeError, \
2566 'new property "%(propname)s" not a string: %(value)r' \
2567 % locals()
2568 if prop.indexme:
2569 self.db.indexer.add_text((self.classname, newid, propname),
2570 value)
2571 d[propname] = value
2573 # get a new id if necessary
2574 if newid is None:
2575 newid = self.db.newid(self.classname)
2577 # insert new node or update existing?
2578 if not self.hasnode(newid):
2579 self.db.addnode(self.classname, newid, d) # insert
2580 else:
2581 self.db.setnode(self.classname, newid, d) # update
2583 # retire?
2584 if retire:
2585 # use the arg for __retired__ to cope with any odd database type
2586 # conversion (hello, sqlite)
2587 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
2588 self.db.arg, self.db.arg)
2589 self.db.sql(sql, (newid, newid))
2590 return newid
2592 def export_journals(self):
2593 """Export a class's journal - generate a list of lists of
2594 CSV-able data:
2596 nodeid, date, user, action, params
2598 No heading here - the columns are fixed.
2599 """
2600 properties = self.getprops()
2601 r = []
2602 for nodeid in self.getnodeids():
2603 for nodeid, date, user, action, params in self.history(nodeid):
2604 date = date.get_tuple()
2605 if action == 'set':
2606 export_data = {}
2607 for propname, value in params.items():
2608 if not properties.has_key(propname):
2609 # property no longer in the schema
2610 continue
2612 prop = properties[propname]
2613 # make sure the params are eval()'able
2614 if value is None:
2615 pass
2616 elif isinstance(prop, Date):
2617 value = value.get_tuple()
2618 elif isinstance(prop, Interval):
2619 value = value.get_tuple()
2620 elif isinstance(prop, Password):
2621 value = str(value)
2622 export_data[propname] = value
2623 params = export_data
2624 elif action == 'create' and params:
2625 # old tracker with data stored in the create!
2626 params = {}
2627 l = [nodeid, date, user, action, params]
2628 r.append(map(repr, l))
2629 return r
2631 def import_journals(self, entries):
2632 """Import a class's journal.
2634 Uses setjournal() to set the journal for each item."""
2635 properties = self.getprops()
2636 d = {}
2637 for l in entries:
2638 l = map(eval, l)
2639 nodeid, jdate, user, action, params = l
2640 r = d.setdefault(nodeid, [])
2641 if action == 'set':
2642 for propname, value in params.items():
2643 prop = properties[propname]
2644 if value is None:
2645 pass
2646 elif isinstance(prop, Date):
2647 value = date.Date(value)
2648 elif isinstance(prop, Interval):
2649 value = date.Interval(value)
2650 elif isinstance(prop, Password):
2651 pwd = password.Password()
2652 pwd.unpack(value)
2653 value = pwd
2654 params[propname] = value
2655 elif action == 'create' and params:
2656 # old tracker with data stored in the create!
2657 params = {}
2658 r.append((nodeid, date.Date(jdate), user, action, params))
2660 for nodeid, l in d.items():
2661 self.db.setjournal(self.classname, nodeid, l)
2663 class FileClass(hyperdb.FileClass, Class):
2664 """This class defines a large chunk of data. To support this, it has a
2665 mandatory String property "content" which is typically saved off
2666 externally to the hyperdb.
2668 The default MIME type of this data is defined by the
2669 "default_mime_type" class attribute, which may be overridden by each
2670 node if the class defines a "type" String property.
2671 """
2672 def __init__(self, db, classname, **properties):
2673 """The newly-created class automatically includes the "content"
2674 and "type" properties.
2675 """
2676 if not properties.has_key('content'):
2677 properties['content'] = hyperdb.String(indexme='yes')
2678 if not properties.has_key('type'):
2679 properties['type'] = hyperdb.String()
2680 Class.__init__(self, db, classname, **properties)
2682 def create(self, **propvalues):
2683 """ snaffle the file propvalue and store in a file
2684 """
2685 # we need to fire the auditors now, or the content property won't
2686 # be in propvalues for the auditors to play with
2687 self.fireAuditors('create', None, propvalues)
2689 # now remove the content property so it's not stored in the db
2690 content = propvalues['content']
2691 del propvalues['content']
2693 # do the database create
2694 newid = self.create_inner(**propvalues)
2696 # figure the mime type
2697 mime_type = propvalues.get('type', self.default_mime_type)
2699 # and index!
2700 if self.properties['content'].indexme:
2701 self.db.indexer.add_text((self.classname, newid, 'content'),
2702 content, mime_type)
2704 # store off the content as a file
2705 self.db.storefile(self.classname, newid, None, content)
2707 # fire reactors
2708 self.fireReactors('create', newid, None)
2710 return newid
2712 def get(self, nodeid, propname, default=_marker, cache=1):
2713 """ Trap the content propname and get it from the file
2715 'cache' exists for backwards compatibility, and is not used.
2716 """
2717 poss_msg = 'Possibly a access right configuration problem.'
2718 if propname == 'content':
2719 try:
2720 return self.db.getfile(self.classname, nodeid, None)
2721 except IOError, (strerror):
2722 # BUG: by catching this we donot see an error in the log.
2723 return 'ERROR reading file: %s%s\n%s\n%s'%(
2724 self.classname, nodeid, poss_msg, strerror)
2725 if default is not _marker:
2726 return Class.get(self, nodeid, propname, default)
2727 else:
2728 return Class.get(self, nodeid, propname)
2730 def set(self, itemid, **propvalues):
2731 """ Snarf the "content" propvalue and update it in a file
2732 """
2733 self.fireAuditors('set', itemid, propvalues)
2734 oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid))
2736 # now remove the content property so it's not stored in the db
2737 content = None
2738 if propvalues.has_key('content'):
2739 content = propvalues['content']
2740 del propvalues['content']
2742 # do the database create
2743 propvalues = self.set_inner(itemid, **propvalues)
2745 # do content?
2746 if content:
2747 # store and possibly index
2748 self.db.storefile(self.classname, itemid, None, content)
2749 if self.properties['content'].indexme:
2750 mime_type = self.get(itemid, 'type', self.default_mime_type)
2751 self.db.indexer.add_text((self.classname, itemid, 'content'),
2752 content, mime_type)
2753 propvalues['content'] = content
2755 # fire reactors
2756 self.fireReactors('set', itemid, oldvalues)
2757 return propvalues
2759 def index(self, nodeid):
2760 """ Add (or refresh) the node to search indexes.
2762 Use the content-type property for the content property.
2763 """
2764 # find all the String properties that have indexme
2765 for prop, propclass in self.getprops().items():
2766 if prop == 'content' and propclass.indexme:
2767 mime_type = self.get(nodeid, 'type', self.default_mime_type)
2768 self.db.indexer.add_text((self.classname, nodeid, 'content'),
2769 str(self.get(nodeid, 'content')), mime_type)
2770 elif isinstance(propclass, hyperdb.String) and propclass.indexme:
2771 # index them under (classname, nodeid, property)
2772 try:
2773 value = str(self.get(nodeid, prop))
2774 except IndexError:
2775 # node has been destroyed
2776 continue
2777 self.db.indexer.add_text((self.classname, nodeid, prop), value)
2779 # XXX deviation from spec - was called ItemClass
2780 class IssueClass(Class, roundupdb.IssueClass):
2781 # Overridden methods:
2782 def __init__(self, db, classname, **properties):
2783 """The newly-created class automatically includes the "messages",
2784 "files", "nosy", and "superseder" properties. If the 'properties'
2785 dictionary attempts to specify any of these properties or a
2786 "creation", "creator", "activity" or "actor" property, a ValueError
2787 is raised.
2788 """
2789 if not properties.has_key('title'):
2790 properties['title'] = hyperdb.String(indexme='yes')
2791 if not properties.has_key('messages'):
2792 properties['messages'] = hyperdb.Multilink("msg")
2793 if not properties.has_key('files'):
2794 properties['files'] = hyperdb.Multilink("file")
2795 if not properties.has_key('nosy'):
2796 # note: journalling is turned off as it really just wastes
2797 # space. this behaviour may be overridden in an instance
2798 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2799 if not properties.has_key('superseder'):
2800 properties['superseder'] = hyperdb.Multilink(classname)
2801 Class.__init__(self, db, classname, **properties)
2803 # vim: set et sts=4 sw=4 :