0aa09f0a659352a9ff1a4b022bc4549c4949eba3
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 """ Relational database (SQL) backend common code.
20 Basics:
22 - map roundup classes to relational tables
23 - automatically detect schema changes and modify the table schemas
24 appropriately (we store the "database version" of the schema in the
25 database itself as the only row of the "schema" table)
26 - multilinks (which represent a many-to-many relationship) are handled through
27 intermediate tables
28 - journals are stored adjunct to the per-class tables
29 - table names and columns have "_" prepended so the names can't clash with
30 restricted names (like "order")
31 - retirement is determined by the __retired__ column being > 0
33 Database-specific changes may generally be pushed out to the overridable
34 sql_* methods, since everything else should be fairly generic. There's
35 probably a bit of work to be done if a database is used that actually
36 honors column typing, since the initial databases don't (sqlite stores
37 everything as a string.)
39 The schema of the hyperdb being mapped to the database is stored in the
40 database itself as a repr()'ed dictionary of information about each Class
41 that maps to a table. If that information differs from the hyperdb schema,
42 then we update it. We also store in the schema dict a version which
43 allows us to upgrade the database schema when necessary. See upgrade_db().
45 To force a unqiueness constraint on the key properties we put the item
46 id into the __retired__ column duing retirement (so it's 0 for "active"
47 items) and place a unqiueness constraint on key + __retired__. This is
48 particularly important for the users class where multiple users may
49 try to have the same username, with potentially many retired users with
50 the same name.
51 """
52 __docformat__ = 'restructuredtext'
54 # standard python modules
55 import sys, os, time, re, errno, weakref, copy, logging
57 # roundup modules
58 from roundup import hyperdb, date, password, roundupdb, security, support
59 from roundup.hyperdb import String, Password, Date, Interval, Link, \
60 Multilink, DatabaseError, Boolean, Number, Node
61 from roundup.backends import locking
62 from roundup.support import reversed
63 from roundup.i18n import _
65 # support
66 from blobfiles import FileStorage
67 try:
68 from indexer_xapian import Indexer
69 except ImportError:
70 from indexer_rdbms import Indexer
71 from sessions_rdbms import Sessions, OneTimeKeys
72 from roundup.date import Range
74 # dummy value meaning "argument not passed"
75 _marker = []
77 def _num_cvt(num):
78 num = str(num)
79 try:
80 return int(num)
81 except:
82 return float(num)
84 def _bool_cvt(value):
85 if value in ('TRUE', 'FALSE'):
86 return {'TRUE': 1, 'FALSE': 0}[value]
87 # assume it's a number returned from the db API
88 return int(value)
90 def connection_dict(config, dbnamestr=None):
91 """ Used by Postgresql and MySQL to detemine the keyword args for
92 opening the database connection."""
93 d = { }
94 if dbnamestr:
95 d[dbnamestr] = config.RDBMS_NAME
96 for name in ('host', 'port', 'password', 'user', 'read_default_group',
97 'read_default_file'):
98 cvar = 'RDBMS_'+name.upper()
99 if config[cvar] is not None:
100 d[name] = config[cvar]
101 return d
103 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
104 """ Wrapper around an SQL database that presents a hyperdb interface.
106 - some functionality is specific to the actual SQL database, hence
107 the sql_* methods that are NotImplemented
108 - we keep a cache of the latest N row fetches (where N is configurable).
109 """
110 def __init__(self, config, journaltag=None):
111 """ Open the database and load the schema from it.
112 """
113 FileStorage.__init__(self, config.UMASK)
114 self.config, self.journaltag = config, journaltag
115 self.dir = config.DATABASE
116 self.classes = {}
117 self.indexer = Indexer(self)
118 self.security = security.Security(self)
120 # additional transaction support for external files and the like
121 self.transactions = []
123 # keep a cache of the N most recently retrieved rows of any kind
124 # (classname, nodeid) = row
125 self.cache_size = config.RDBMS_CACHE_SIZE
126 self.cache = {}
127 self.cache_lru = []
128 self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0,
129 'filtering': 0}
131 # database lock
132 self.lockfile = None
134 # open a connection to the database, creating the "conn" attribute
135 self.open_connection()
137 def clearCache(self):
138 self.cache = {}
139 self.cache_lru = []
141 def getSessionManager(self):
142 return Sessions(self)
144 def getOTKManager(self):
145 return OneTimeKeys(self)
147 def open_connection(self):
148 """ Open a connection to the database, creating it if necessary.
150 Must call self.load_dbschema()
151 """
152 raise NotImplemented
154 def sql(self, sql, args=None):
155 """ Execute the sql with the optional args.
156 """
157 self.log_debug('SQL %r %r'%(sql, args))
158 if args:
159 self.cursor.execute(sql, args)
160 else:
161 self.cursor.execute(sql)
163 def sql_fetchone(self):
164 """ Fetch a single row. If there's nothing to fetch, return None.
165 """
166 return self.cursor.fetchone()
168 def sql_fetchall(self):
169 """ Fetch all rows. If there's nothing to fetch, return [].
170 """
171 return self.cursor.fetchall()
173 def sql_stringquote(self, value):
174 """ Quote the string so it's safe to put in the 'sql quotes'
175 """
176 return re.sub("'", "''", str(value))
178 def init_dbschema(self):
179 self.database_schema = {
180 'version': self.current_db_version,
181 'tables': {}
182 }
184 def load_dbschema(self):
185 """ Load the schema definition that the database currently implements
186 """
187 self.cursor.execute('select schema from schema')
188 schema = self.cursor.fetchone()
189 if schema:
190 self.database_schema = eval(schema[0])
191 else:
192 self.database_schema = {}
194 def save_dbschema(self):
195 """ Save the schema definition that the database currently implements
196 """
197 s = repr(self.database_schema)
198 self.sql('delete from schema')
199 self.sql('insert into schema values (%s)'%self.arg, (s,))
201 def post_init(self):
202 """ Called once the schema initialisation has finished.
204 We should now confirm that the schema defined by our "classes"
205 attribute actually matches the schema in the database.
206 """
207 save = 0
209 # handle changes in the schema
210 tables = self.database_schema['tables']
211 for classname, spec in self.classes.items():
212 if tables.has_key(classname):
213 dbspec = tables[classname]
214 if self.update_class(spec, dbspec):
215 tables[classname] = spec.schema()
216 save = 1
217 else:
218 self.create_class(spec)
219 tables[classname] = spec.schema()
220 save = 1
222 for classname, spec in tables.items():
223 if not self.classes.has_key(classname):
224 self.drop_class(classname, tables[classname])
225 del tables[classname]
226 save = 1
228 # now upgrade the database for column type changes, new internal
229 # tables, etc.
230 save = save | self.upgrade_db()
232 # update the database version of the schema
233 if save:
234 self.save_dbschema()
236 # reindex the db if necessary
237 if self.indexer.should_reindex():
238 self.reindex()
240 # commit
241 self.sql_commit()
243 # update this number when we need to make changes to the SQL structure
244 # of the backen database
245 current_db_version = 5
246 db_version_updated = False
247 def upgrade_db(self):
248 """ Update the SQL database to reflect changes in the backend code.
250 Return boolean whether we need to save the schema.
251 """
252 version = self.database_schema.get('version', 1)
253 if version > self.current_db_version:
254 raise DatabaseError('attempting to run rev %d DATABASE with rev '
255 '%d CODE!'%(version, self.current_db_version))
256 if version == self.current_db_version:
257 # nothing to do
258 return 0
260 if version < 2:
261 self.log_info('upgrade to version 2')
262 # change the schema structure
263 self.database_schema = {'tables': self.database_schema}
265 # version 1 didn't have the actor column (note that in
266 # MySQL this will also transition the tables to typed columns)
267 self.add_new_columns_v2()
269 # version 1 doesn't have the OTK, session and indexing in the
270 # database
271 self.create_version_2_tables()
273 if version < 3:
274 self.log_info('upgrade to version 3')
275 self.fix_version_2_tables()
277 if version < 4:
278 self.fix_version_3_tables()
280 if version < 5:
281 self.fix_version_4_tables()
283 self.database_schema['version'] = self.current_db_version
284 self.db_version_updated = True
285 return 1
287 def fix_version_3_tables(self):
288 # drop the shorter VARCHAR OTK column and add a new TEXT one
289 for name in ('otk', 'session'):
290 self.sql('DELETE FROM %ss'%name)
291 self.sql('ALTER TABLE %ss DROP %s_value'%(name, name))
292 self.sql('ALTER TABLE %ss ADD %s_value TEXT'%(name, name))
294 def fix_version_2_tables(self):
295 # Default (used by sqlite): NOOP
296 pass
298 def fix_version_4_tables(self):
299 # note this is an explicit call now
300 c = self.cursor
301 for cn, klass in self.classes.items():
302 c.execute('select id from _%s where __retired__<>0'%(cn,))
303 for (id,) in c.fetchall():
304 c.execute('update _%s set __retired__=%s where id=%s'%(cn,
305 self.arg, self.arg), (id, id))
307 if klass.key:
308 self.add_class_key_required_unique_constraint(cn, klass.key)
310 def _convert_journal_tables(self):
311 """Get current journal table contents, drop the table and re-create"""
312 c = self.cursor
313 cols = ','.join('nodeid date tag action params'.split())
314 for klass in self.classes.values():
315 # slurp and drop
316 sql = 'select %s from %s__journal order by date'%(cols,
317 klass.classname)
318 c.execute(sql)
319 contents = c.fetchall()
320 self.drop_journal_table_indexes(klass.classname)
321 c.execute('drop table %s__journal'%klass.classname)
323 # re-create and re-populate
324 self.create_journal_table(klass)
325 a = self.arg
326 sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(
327 klass.classname, cols, a, a, a, a, a)
328 for row in contents:
329 # no data conversion needed
330 self.cursor.execute(sql, row)
332 def _convert_string_properties(self):
333 """Get current Class tables that contain String properties, and
334 convert the VARCHAR columns to TEXT"""
335 c = self.cursor
336 for klass in self.classes.values():
337 # slurp and drop
338 cols, mls = self.determine_columns(klass.properties.items())
339 scols = ','.join([i[0] for i in cols])
340 sql = 'select id,%s from _%s'%(scols, klass.classname)
341 c.execute(sql)
342 contents = c.fetchall()
343 self.drop_class_table_indexes(klass.classname, klass.getkey())
344 c.execute('drop table _%s'%klass.classname)
346 # re-create and re-populate
347 self.create_class_table(klass, create_sequence=0)
348 a = ','.join([self.arg for i in range(len(cols)+1)])
349 sql = 'insert into _%s (id,%s) values (%s)'%(klass.classname,
350 scols, a)
351 for row in contents:
352 l = []
353 for entry in row:
354 # mysql will already be a string - psql needs "help"
355 if entry is not None and not isinstance(entry, type('')):
356 entry = str(entry)
357 l.append(entry)
358 self.cursor.execute(sql, l)
360 def refresh_database(self):
361 self.post_init()
364 def reindex(self, classname=None, show_progress=False):
365 if classname:
366 classes = [self.getclass(classname)]
367 else:
368 classes = self.classes.values()
369 for klass in classes:
370 if show_progress:
371 for nodeid in support.Progress('Reindex %s'%klass.classname,
372 klass.list()):
373 klass.index(nodeid)
374 else:
375 for nodeid in klass.list():
376 klass.index(nodeid)
377 self.indexer.save_index()
379 hyperdb_to_sql_datatypes = {
380 hyperdb.String : 'TEXT',
381 hyperdb.Date : 'TIMESTAMP',
382 hyperdb.Link : 'INTEGER',
383 hyperdb.Interval : 'VARCHAR(255)',
384 hyperdb.Password : 'VARCHAR(255)',
385 hyperdb.Boolean : 'BOOLEAN',
386 hyperdb.Number : 'REAL',
387 }
389 def hyperdb_to_sql_datatype(self, propclass):
391 datatype = self.hyperdb_to_sql_datatypes.get(propclass)
392 if datatype:
393 return datatype
395 for k, v in self.hyperdb_to_sql_datatypes.iteritems():
396 if issubclass(propclass, k):
397 return v
399 raise ValueError, '%r is not a hyperdb property class' % propclass
401 def determine_columns(self, properties):
402 """ Figure the column names and multilink properties from the spec
404 "properties" is a list of (name, prop) where prop may be an
405 instance of a hyperdb "type" _or_ a string repr of that type.
406 """
407 cols = [
408 ('_actor', self.hyperdb_to_sql_datatype(hyperdb.Link)),
409 ('_activity', self.hyperdb_to_sql_datatype(hyperdb.Date)),
410 ('_creator', self.hyperdb_to_sql_datatype(hyperdb.Link)),
411 ('_creation', self.hyperdb_to_sql_datatype(hyperdb.Date)),
412 ]
413 mls = []
414 # add the multilinks separately
415 for col, prop in properties:
416 if isinstance(prop, Multilink):
417 mls.append(col)
418 continue
420 if isinstance(prop, type('')):
421 raise ValueError, "string property spec!"
422 #and prop.find('Multilink') != -1:
423 #mls.append(col)
425 datatype = self.hyperdb_to_sql_datatype(prop.__class__)
426 cols.append(('_'+col, datatype))
428 # Intervals stored as two columns
429 if isinstance(prop, Interval):
430 cols.append(('__'+col+'_int__', 'BIGINT'))
432 cols.sort()
433 return cols, mls
435 def update_class(self, spec, old_spec, force=0):
436 """ Determine the differences between the current spec and the
437 database version of the spec, and update where necessary.
439 If 'force' is true, update the database anyway.
440 """
441 new_has = spec.properties.has_key
442 new_spec = spec.schema()
443 new_spec[1].sort()
444 old_spec[1].sort()
445 if not force and new_spec == old_spec:
446 # no changes
447 return 0
449 logger = logging.getLogger('hyperdb')
450 logger.info('update_class %s'%spec.classname)
452 logger.debug('old_spec %r'%(old_spec,))
453 logger.debug('new_spec %r'%(new_spec,))
455 # detect key prop change for potential index change
456 keyprop_changes = {}
457 if new_spec[0] != old_spec[0]:
458 if old_spec[0]:
459 keyprop_changes['remove'] = old_spec[0]
460 if new_spec[0]:
461 keyprop_changes['add'] = new_spec[0]
463 # detect multilinks that have been removed, and drop their table
464 old_has = {}
465 for name, prop in old_spec[1]:
466 old_has[name] = 1
467 if new_has(name):
468 continue
470 if prop.find('Multilink to') != -1:
471 # first drop indexes.
472 self.drop_multilink_table_indexes(spec.classname, name)
474 # now the multilink table itself
475 sql = 'drop table %s_%s'%(spec.classname, name)
476 else:
477 # if this is the key prop, drop the index first
478 if old_spec[0] == prop:
479 self.drop_class_table_key_index(spec.classname, name)
480 del keyprop_changes['remove']
482 # drop the column
483 sql = 'alter table _%s drop column _%s'%(spec.classname, name)
485 self.sql(sql)
486 old_has = old_has.has_key
488 # if we didn't remove the key prop just then, but the key prop has
489 # changed, we still need to remove the old index
490 if keyprop_changes.has_key('remove'):
491 self.drop_class_table_key_index(spec.classname,
492 keyprop_changes['remove'])
494 # add new columns
495 for propname, prop in new_spec[1]:
496 if old_has(propname):
497 continue
498 prop = spec.properties[propname]
499 if isinstance(prop, Multilink):
500 self.create_multilink_table(spec, propname)
501 else:
502 # add the column
503 coltype = self.hyperdb_to_sql_datatype(prop.__class__)
504 sql = 'alter table _%s add column _%s %s'%(
505 spec.classname, propname, coltype)
506 self.sql(sql)
508 # extra Interval column
509 if isinstance(prop, Interval):
510 sql = 'alter table _%s add column __%s_int__ BIGINT'%(
511 spec.classname, propname)
512 self.sql(sql)
514 # if the new column is a key prop, we need an index!
515 if new_spec[0] == propname:
516 self.create_class_table_key_index(spec.classname, propname)
517 del keyprop_changes['add']
519 # if we didn't add the key prop just then, but the key prop has
520 # changed, we still need to add the new index
521 if keyprop_changes.has_key('add'):
522 self.create_class_table_key_index(spec.classname,
523 keyprop_changes['add'])
525 return 1
527 def determine_all_columns(self, spec):
528 """Figure out the columns from the spec and also add internal columns
530 """
531 cols, mls = self.determine_columns(spec.properties.items())
533 # add on our special columns
534 cols.append(('id', 'INTEGER PRIMARY KEY'))
535 cols.append(('__retired__', 'INTEGER DEFAULT 0'))
536 return cols, mls
538 def create_class_table(self, spec):
539 """Create the class table for the given Class "spec". Creates the
540 indexes too."""
541 cols, mls = self.determine_all_columns(spec)
543 # create the base table
544 scols = ','.join(['%s %s'%x for x in cols])
545 sql = 'create table _%s (%s)'%(spec.classname, scols)
546 self.sql(sql)
548 self.create_class_table_indexes(spec)
550 return cols, mls
552 def create_class_table_indexes(self, spec):
553 """ create the class table for the given spec
554 """
555 # create __retired__ index
556 index_sql2 = 'create index _%s_retired_idx on _%s(__retired__)'%(
557 spec.classname, spec.classname)
558 self.sql(index_sql2)
560 # create index for key property
561 if spec.key:
562 index_sql3 = 'create index _%s_%s_idx on _%s(_%s)'%(
563 spec.classname, spec.key,
564 spec.classname, spec.key)
565 self.sql(index_sql3)
567 # and the unique index for key / retired(id)
568 self.add_class_key_required_unique_constraint(spec.classname,
569 spec.key)
571 # TODO: create indexes on (selected?) Link property columns, as
572 # they're more likely to be used for lookup
574 def add_class_key_required_unique_constraint(self, cn, key):
575 sql = '''create unique index _%s_key_retired_idx
576 on _%s(__retired__, _%s)'''%(cn, cn, key)
577 self.sql(sql)
579 def drop_class_table_indexes(self, cn, key):
580 # drop the old table indexes first
581 l = ['_%s_id_idx'%cn, '_%s_retired_idx'%cn]
582 if key:
583 l.append('_%s_%s_idx'%(cn, key))
585 table_name = '_%s'%cn
586 for index_name in l:
587 if not self.sql_index_exists(table_name, index_name):
588 continue
589 index_sql = 'drop index '+index_name
590 self.sql(index_sql)
592 def create_class_table_key_index(self, cn, key):
593 """ create the class table for the given spec
594 """
595 sql = 'create index _%s_%s_idx on _%s(_%s)'%(cn, key, cn, key)
596 self.sql(sql)
598 def drop_class_table_key_index(self, cn, key):
599 table_name = '_%s'%cn
600 index_name = '_%s_%s_idx'%(cn, key)
601 if self.sql_index_exists(table_name, index_name):
602 sql = 'drop index '+index_name
603 self.sql(sql)
605 # and now the retired unique index too
606 index_name = '_%s_key_retired_idx'%cn
607 if self.sql_index_exists(table_name, index_name):
608 sql = 'drop index '+index_name
609 self.sql(sql)
611 def create_journal_table(self, spec):
612 """ create the journal table for a class given the spec and
613 already-determined cols
614 """
615 # journal table
616 cols = ','.join(['%s varchar'%x
617 for x in 'nodeid date tag action params'.split()])
618 sql = """create table %s__journal (
619 nodeid integer, date %s, tag varchar(255),
620 action varchar(255), params text)""" % (spec.classname,
621 self.hyperdb_to_sql_datatype(hyperdb.Date))
622 self.sql(sql)
623 self.create_journal_table_indexes(spec)
625 def create_journal_table_indexes(self, spec):
626 # index on nodeid
627 sql = 'create index %s_journ_idx on %s__journal(nodeid)'%(
628 spec.classname, spec.classname)
629 self.sql(sql)
631 def drop_journal_table_indexes(self, classname):
632 index_name = '%s_journ_idx'%classname
633 if not self.sql_index_exists('%s__journal'%classname, index_name):
634 return
635 index_sql = 'drop index '+index_name
636 self.sql(index_sql)
638 def create_multilink_table(self, spec, ml):
639 """ Create a multilink table for the "ml" property of the class
640 given by the spec
641 """
642 # create the table
643 sql = 'create table %s_%s (linkid INTEGER, nodeid INTEGER)'%(
644 spec.classname, ml)
645 self.sql(sql)
646 self.create_multilink_table_indexes(spec, ml)
648 def create_multilink_table_indexes(self, spec, ml):
649 # create index on linkid
650 index_sql = 'create index %s_%s_l_idx on %s_%s(linkid)'%(
651 spec.classname, ml, spec.classname, ml)
652 self.sql(index_sql)
654 # create index on nodeid
655 index_sql = 'create index %s_%s_n_idx on %s_%s(nodeid)'%(
656 spec.classname, ml, spec.classname, ml)
657 self.sql(index_sql)
659 def drop_multilink_table_indexes(self, classname, ml):
660 l = [
661 '%s_%s_l_idx'%(classname, ml),
662 '%s_%s_n_idx'%(classname, ml)
663 ]
664 table_name = '%s_%s'%(classname, ml)
665 for index_name in l:
666 if not self.sql_index_exists(table_name, index_name):
667 continue
668 index_sql = 'drop index %s'%index_name
669 self.sql(index_sql)
671 def create_class(self, spec):
672 """ Create a database table according to the given spec.
673 """
674 cols, mls = self.create_class_table(spec)
675 self.create_journal_table(spec)
677 # now create the multilink tables
678 for ml in mls:
679 self.create_multilink_table(spec, ml)
681 def drop_class(self, cn, spec):
682 """ Drop the given table from the database.
684 Drop the journal and multilink tables too.
685 """
686 properties = spec[1]
687 # figure the multilinks
688 mls = []
689 for propname, prop in properties:
690 if isinstance(prop, Multilink):
691 mls.append(propname)
693 # drop class table and indexes
694 self.drop_class_table_indexes(cn, spec[0])
696 self.drop_class_table(cn)
698 # drop journal table and indexes
699 self.drop_journal_table_indexes(cn)
700 sql = 'drop table %s__journal'%cn
701 self.sql(sql)
703 for ml in mls:
704 # drop multilink table and indexes
705 self.drop_multilink_table_indexes(cn, ml)
706 sql = 'drop table %s_%s'%(spec.classname, ml)
707 self.sql(sql)
709 def drop_class_table(self, cn):
710 sql = 'drop table _%s'%cn
711 self.sql(sql)
713 #
714 # Classes
715 #
716 def __getattr__(self, classname):
717 """ A convenient way of calling self.getclass(classname).
718 """
719 if self.classes.has_key(classname):
720 return self.classes[classname]
721 raise AttributeError, classname
723 def addclass(self, cl):
724 """ Add a Class to the hyperdatabase.
725 """
726 cn = cl.classname
727 if self.classes.has_key(cn):
728 raise ValueError, cn
729 self.classes[cn] = cl
731 # add default Edit and View permissions
732 self.security.addPermission(name="Create", klass=cn,
733 description="User is allowed to create "+cn)
734 self.security.addPermission(name="Edit", klass=cn,
735 description="User is allowed to edit "+cn)
736 self.security.addPermission(name="View", klass=cn,
737 description="User is allowed to access "+cn)
739 def getclasses(self):
740 """ Return a list of the names of all existing classes.
741 """
742 l = self.classes.keys()
743 l.sort()
744 return l
746 def getclass(self, classname):
747 """Get the Class object representing a particular class.
749 If 'classname' is not a valid class name, a KeyError is raised.
750 """
751 try:
752 return self.classes[classname]
753 except KeyError:
754 raise KeyError, 'There is no class called "%s"'%classname
756 def clear(self):
757 """Delete all database contents.
759 Note: I don't commit here, which is different behaviour to the
760 "nuke from orbit" behaviour in the dbs.
761 """
762 logging.getLogger('hyperdb').info('clear')
763 for cn in self.classes.keys():
764 sql = 'delete from _%s'%cn
765 self.sql(sql)
767 #
768 # Nodes
769 #
771 hyperdb_to_sql_value = {
772 hyperdb.String : str,
773 # fractional seconds by default
774 hyperdb.Date : lambda x: x.formal(sep=' ', sec='%06.3f'),
775 hyperdb.Link : int,
776 hyperdb.Interval : str,
777 hyperdb.Password : str,
778 hyperdb.Boolean : lambda x: x and 'TRUE' or 'FALSE',
779 hyperdb.Number : lambda x: x,
780 hyperdb.Multilink : lambda x: x, # used in journal marshalling
781 }
783 def to_sql_value(self, propklass):
785 fn = self.hyperdb_to_sql_value.get(propklass)
786 if fn:
787 return fn
789 for k, v in self.hyperdb_to_sql_value.iteritems():
790 if issubclass(propklass, k):
791 return v
793 raise ValueError, '%r is not a hyperdb property class' % propklass
795 def addnode(self, classname, nodeid, node):
796 """ Add the specified node to its class's db.
797 """
798 self.log_debug('addnode %s%s %r'%(classname,
799 nodeid, node))
801 # determine the column definitions and multilink tables
802 cl = self.classes[classname]
803 cols, mls = self.determine_columns(cl.properties.items())
805 # we'll be supplied these props if we're doing an import
806 values = node.copy()
807 if not values.has_key('creator'):
808 # add in the "calculated" properties (dupe so we don't affect
809 # calling code's node assumptions)
810 values['creation'] = values['activity'] = date.Date()
811 values['actor'] = values['creator'] = self.getuid()
813 cl = self.classes[classname]
814 props = cl.getprops(protected=1)
815 del props['id']
817 # default the non-multilink columns
818 for col, prop in props.items():
819 if not values.has_key(col):
820 if isinstance(prop, Multilink):
821 values[col] = []
822 else:
823 values[col] = None
825 # clear this node out of the cache if it's in there
826 key = (classname, nodeid)
827 if self.cache.has_key(key):
828 del self.cache[key]
829 self.cache_lru.remove(key)
831 # figure the values to insert
832 vals = []
833 for col,dt in cols:
834 # this is somewhat dodgy....
835 if col.endswith('_int__'):
836 # XXX eugh, this test suxxors
837 value = values[col[2:-6]]
838 # this is an Interval special "int" column
839 if value is not None:
840 vals.append(value.as_seconds())
841 else:
842 vals.append(value)
843 continue
845 prop = props[col[1:]]
846 value = values[col[1:]]
847 if value is not None:
848 value = self.to_sql_value(prop.__class__)(value)
849 vals.append(value)
850 vals.append(nodeid)
851 vals = tuple(vals)
853 # make sure the ordering is correct for column name -> column value
854 s = ','.join([self.arg for x in cols]) + ',%s'%self.arg
855 cols = ','.join([col for col,dt in cols]) + ',id'
857 # perform the inserts
858 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
859 self.sql(sql, vals)
861 # insert the multilink rows
862 for col in mls:
863 t = '%s_%s'%(classname, col)
864 for entry in node[col]:
865 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
866 self.arg, self.arg)
867 self.sql(sql, (entry, nodeid))
869 def setnode(self, classname, nodeid, values, multilink_changes={}):
870 """ Change the specified node.
871 """
872 self.log_debug('setnode %s%s %r'
873 % (classname, nodeid, values))
875 # clear this node out of the cache if it's in there
876 key = (classname, nodeid)
877 if self.cache.has_key(key):
878 del self.cache[key]
879 self.cache_lru.remove(key)
881 cl = self.classes[classname]
882 props = cl.getprops()
884 cols = []
885 mls = []
886 # add the multilinks separately
887 for col in values.keys():
888 prop = props[col]
889 if isinstance(prop, Multilink):
890 mls.append(col)
891 elif isinstance(prop, Interval):
892 # Intervals store the seconds value too
893 cols.append(col)
894 # extra leading '_' added by code below
895 cols.append('_' +col + '_int__')
896 else:
897 cols.append(col)
898 cols.sort()
900 # figure the values to insert
901 vals = []
902 for col in cols:
903 if col.endswith('_int__'):
904 # XXX eugh, this test suxxors
905 # Intervals store the seconds value too
906 col = col[1:-6]
907 prop = props[col]
908 value = values[col]
909 if value is None:
910 vals.append(None)
911 else:
912 vals.append(value.as_seconds())
913 else:
914 prop = props[col]
915 value = values[col]
916 if value is None:
917 e = None
918 else:
919 e = self.to_sql_value(prop.__class__)(value)
920 vals.append(e)
922 vals.append(int(nodeid))
923 vals = tuple(vals)
925 # if there's any updates to regular columns, do them
926 if cols:
927 # make sure the ordering is correct for column name -> column value
928 s = ','.join(['_%s=%s'%(x, self.arg) for x in cols])
929 cols = ','.join(cols)
931 # perform the update
932 sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
933 self.sql(sql, vals)
935 # we're probably coming from an import, not a change
936 if not multilink_changes:
937 for name in mls:
938 prop = props[name]
939 value = values[name]
941 t = '%s_%s'%(classname, name)
943 # clear out previous values for this node
944 # XXX numeric ids
945 self.sql('delete from %s where nodeid=%s'%(t, self.arg),
946 (nodeid,))
948 # insert the values for this node
949 for entry in values[name]:
950 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
951 self.arg, self.arg)
952 # XXX numeric ids
953 self.sql(sql, (entry, nodeid))
955 # we have multilink changes to apply
956 for col, (add, remove) in multilink_changes.items():
957 tn = '%s_%s'%(classname, col)
958 if add:
959 sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
960 self.arg, self.arg)
961 for addid in add:
962 # XXX numeric ids
963 self.sql(sql, (int(nodeid), int(addid)))
964 if remove:
965 s = ','.join([self.arg]*len(remove))
966 sql = 'delete from %s where nodeid=%s and linkid in (%s)'%(tn,
967 self.arg, s)
968 # XXX numeric ids
969 self.sql(sql, [int(nodeid)] + remove)
971 sql_to_hyperdb_value = {
972 hyperdb.String : str,
973 hyperdb.Date : lambda x:date.Date(str(x).replace(' ', '.')),
974 # hyperdb.Link : int, # XXX numeric ids
975 hyperdb.Link : str,
976 hyperdb.Interval : date.Interval,
977 hyperdb.Password : lambda x: password.Password(encrypted=x),
978 hyperdb.Boolean : _bool_cvt,
979 hyperdb.Number : _num_cvt,
980 hyperdb.Multilink : lambda x: x, # used in journal marshalling
981 }
983 def to_hyperdb_value(self, propklass):
985 fn = self.sql_to_hyperdb_value.get(propklass)
986 if fn:
987 return fn
989 for k, v in self.sql_to_hyperdb_value.iteritems():
990 if issubclass(propklass, k):
991 return v
993 raise ValueError, '%r is not a hyperdb property class' % propklass
995 def getnode(self, classname, nodeid):
996 """ Get a node from the database.
997 """
998 # see if we have this node cached
999 key = (classname, nodeid)
1000 if self.cache.has_key(key):
1001 # push us back to the top of the LRU
1002 self.cache_lru.remove(key)
1003 self.cache_lru.insert(0, key)
1004 if __debug__:
1005 self.stats['cache_hits'] += 1
1006 # return the cached information
1007 return self.cache[key]
1009 if __debug__:
1010 self.stats['cache_misses'] += 1
1011 start_t = time.time()
1013 # figure the columns we're fetching
1014 cl = self.classes[classname]
1015 cols, mls = self.determine_columns(cl.properties.items())
1016 scols = ','.join([col for col,dt in cols])
1018 # perform the basic property fetch
1019 sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
1020 self.sql(sql, (nodeid,))
1022 values = self.sql_fetchone()
1023 if values is None:
1024 raise IndexError, 'no such %s node %s'%(classname, nodeid)
1026 # make up the node
1027 node = {}
1028 props = cl.getprops(protected=1)
1029 for col in range(len(cols)):
1030 name = cols[col][0][1:]
1031 if name.endswith('_int__'):
1032 # XXX eugh, this test suxxors
1033 # ignore the special Interval-as-seconds column
1034 continue
1035 value = values[col]
1036 if value is not None:
1037 value = self.to_hyperdb_value(props[name].__class__)(value)
1038 node[name] = value
1041 # now the multilinks
1042 for col in mls:
1043 # get the link ids
1044 sql = 'select linkid from %s_%s where nodeid=%s'%(classname, col,
1045 self.arg)
1046 self.sql(sql, (nodeid,))
1047 # extract the first column from the result
1048 # XXX numeric ids
1049 items = [int(x[0]) for x in self.cursor.fetchall()]
1050 items.sort ()
1051 node[col] = [str(x) for x in items]
1053 # save off in the cache
1054 key = (classname, nodeid)
1055 self.cache[key] = node
1056 # update the LRU
1057 self.cache_lru.insert(0, key)
1058 if len(self.cache_lru) > self.cache_size:
1059 del self.cache[self.cache_lru.pop()]
1061 if __debug__:
1062 self.stats['get_items'] += (time.time() - start_t)
1064 return node
1066 def destroynode(self, classname, nodeid):
1067 """Remove a node from the database. Called exclusively by the
1068 destroy() method on Class.
1069 """
1070 logging.getLogger('hyperdb').info('destroynode %s%s'%(classname, nodeid))
1072 # make sure the node exists
1073 if not self.hasnode(classname, nodeid):
1074 raise IndexError, '%s has no node %s'%(classname, nodeid)
1076 # see if we have this node cached
1077 if self.cache.has_key((classname, nodeid)):
1078 del self.cache[(classname, nodeid)]
1080 # see if there's any obvious commit actions that we should get rid of
1081 for entry in self.transactions[:]:
1082 if entry[1][:2] == (classname, nodeid):
1083 self.transactions.remove(entry)
1085 # now do the SQL
1086 sql = 'delete from _%s where id=%s'%(classname, self.arg)
1087 self.sql(sql, (nodeid,))
1089 # remove from multilnks
1090 cl = self.getclass(classname)
1091 x, mls = self.determine_columns(cl.properties.items())
1092 for col in mls:
1093 # get the link ids
1094 sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
1095 self.sql(sql, (nodeid,))
1097 # remove journal entries
1098 sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
1099 self.sql(sql, (nodeid,))
1101 # cleanup any blob filestorage when we commit
1102 self.transactions.append((FileStorage.destroy, (self, classname, nodeid)))
1104 def hasnode(self, classname, nodeid):
1105 """ Determine if the database has a given node.
1106 """
1107 # If this node is in the cache, then we do not need to go to
1108 # the database. (We don't consider this an LRU hit, though.)
1109 if self.cache.has_key((classname, nodeid)):
1110 # Return 1, not True, to match the type of the result of
1111 # the SQL operation below.
1112 return 1
1113 sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
1114 self.sql(sql, (nodeid,))
1115 return int(self.cursor.fetchone()[0])
1117 def countnodes(self, classname):
1118 """ Count the number of nodes that exist for a particular Class.
1119 """
1120 sql = 'select count(*) from _%s'%classname
1121 self.sql(sql)
1122 return self.cursor.fetchone()[0]
1124 def addjournal(self, classname, nodeid, action, params, creator=None,
1125 creation=None):
1126 """ Journal the Action
1127 'action' may be:
1129 'create' or 'set' -- 'params' is a dictionary of property values
1130 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
1131 'retire' -- 'params' is None
1132 """
1133 # handle supply of the special journalling parameters (usually
1134 # supplied on importing an existing database)
1135 if creator:
1136 journaltag = creator
1137 else:
1138 journaltag = self.getuid()
1139 if creation:
1140 journaldate = creation
1141 else:
1142 journaldate = date.Date()
1144 # create the journal entry
1145 cols = 'nodeid,date,tag,action,params'
1147 self.log_debug('addjournal %s%s %r %s %s %r'%(classname,
1148 nodeid, journaldate, journaltag, action, params))
1150 # make the journalled data marshallable
1151 if isinstance(params, type({})):
1152 self._journal_marshal(params, classname)
1154 params = repr(params)
1156 dc = self.to_sql_value(hyperdb.Date)
1157 journaldate = dc(journaldate)
1159 self.save_journal(classname, cols, nodeid, journaldate,
1160 journaltag, action, params)
1162 def setjournal(self, classname, nodeid, journal):
1163 """Set the journal to the "journal" list."""
1164 # clear out any existing entries
1165 self.sql('delete from %s__journal where nodeid=%s'%(classname,
1166 self.arg), (nodeid,))
1168 # create the journal entry
1169 cols = 'nodeid,date,tag,action,params'
1171 dc = self.to_sql_value(hyperdb.Date)
1172 for nodeid, journaldate, journaltag, action, params in journal:
1173 self.log_debug('addjournal %s%s %r %s %s %r'%(
1174 classname, nodeid, journaldate, journaltag, action,
1175 params))
1177 # make the journalled data marshallable
1178 if isinstance(params, type({})):
1179 self._journal_marshal(params, classname)
1180 params = repr(params)
1182 self.save_journal(classname, cols, nodeid, dc(journaldate),
1183 journaltag, action, params)
1185 def _journal_marshal(self, params, classname):
1186 """Convert the journal params values into safely repr'able and
1187 eval'able values."""
1188 properties = self.getclass(classname).getprops()
1189 for param, value in params.items():
1190 if not value:
1191 continue
1192 property = properties[param]
1193 cvt = self.to_sql_value(property.__class__)
1194 if isinstance(property, Password):
1195 params[param] = cvt(value)
1196 elif isinstance(property, Date):
1197 params[param] = cvt(value)
1198 elif isinstance(property, Interval):
1199 params[param] = cvt(value)
1200 elif isinstance(property, Boolean):
1201 params[param] = cvt(value)
1203 def getjournal(self, classname, nodeid):
1204 """ get the journal for id
1205 """
1206 # make sure the node exists
1207 if not self.hasnode(classname, nodeid):
1208 raise IndexError, '%s has no node %s'%(classname, nodeid)
1210 cols = ','.join('nodeid date tag action params'.split())
1211 journal = self.load_journal(classname, cols, nodeid)
1213 # now unmarshal the data
1214 dc = self.to_hyperdb_value(hyperdb.Date)
1215 res = []
1216 properties = self.getclass(classname).getprops()
1217 for nodeid, date_stamp, user, action, params in journal:
1218 params = eval(params)
1219 if isinstance(params, type({})):
1220 for param, value in params.items():
1221 if not value:
1222 continue
1223 property = properties.get(param, None)
1224 if property is None:
1225 # deleted property
1226 continue
1227 cvt = self.to_hyperdb_value(property.__class__)
1228 if isinstance(property, Password):
1229 params[param] = cvt(value)
1230 elif isinstance(property, Date):
1231 params[param] = cvt(value)
1232 elif isinstance(property, Interval):
1233 params[param] = cvt(value)
1234 elif isinstance(property, Boolean):
1235 params[param] = cvt(value)
1236 # XXX numeric ids
1237 res.append((str(nodeid), dc(date_stamp), user, action, params))
1238 return res
1240 def save_journal(self, classname, cols, nodeid, journaldate,
1241 journaltag, action, params):
1242 """ Save the journal entry to the database
1243 """
1244 entry = (nodeid, journaldate, journaltag, action, params)
1246 # do the insert
1247 a = self.arg
1248 sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(
1249 classname, cols, a, a, a, a, a)
1250 self.sql(sql, entry)
1252 def load_journal(self, classname, cols, nodeid):
1253 """ Load the journal from the database
1254 """
1255 # now get the journal entries
1256 sql = 'select %s from %s__journal where nodeid=%s order by date'%(
1257 cols, classname, self.arg)
1258 self.sql(sql, (nodeid,))
1259 return self.cursor.fetchall()
1261 def pack(self, pack_before):
1262 """ Delete all journal entries except "create" before 'pack_before'.
1263 """
1264 date_stamp = self.to_sql_value(Date)(pack_before)
1266 # do the delete
1267 for classname in self.classes.keys():
1268 sql = "delete from %s__journal where date<%s and "\
1269 "action<>'create'"%(classname, self.arg)
1270 self.sql(sql, (date_stamp,))
1272 def sql_commit(self, fail_ok=False):
1273 """ Actually commit to the database.
1274 """
1275 logging.getLogger('hyperdb').info('commit')
1277 self.conn.commit()
1279 # open a new cursor for subsequent work
1280 self.cursor = self.conn.cursor()
1282 def commit(self, fail_ok=False):
1283 """ Commit the current transactions.
1285 Save all data changed since the database was opened or since the
1286 last commit() or rollback().
1288 fail_ok indicates that the commit is allowed to fail. This is used
1289 in the web interface when committing cleaning of the session
1290 database. We don't care if there's a concurrency issue there.
1292 The only backend this seems to affect is postgres.
1293 """
1294 # commit the database
1295 self.sql_commit(fail_ok)
1297 # now, do all the other transaction stuff
1298 for method, args in self.transactions:
1299 method(*args)
1301 # save the indexer
1302 self.indexer.save_index()
1304 # clear out the transactions
1305 self.transactions = []
1307 def sql_rollback(self):
1308 self.conn.rollback()
1310 def rollback(self):
1311 """ Reverse all actions from the current transaction.
1313 Undo all the changes made since the database was opened or the last
1314 commit() or rollback() was performed.
1315 """
1316 logging.getLogger('hyperdb').info('rollback')
1318 self.sql_rollback()
1320 # roll back "other" transaction stuff
1321 for method, args in self.transactions:
1322 # delete temporary files
1323 if method == self.doStoreFile:
1324 self.rollbackStoreFile(*args)
1325 self.transactions = []
1327 # clear the cache
1328 self.clearCache()
1330 def sql_close(self):
1331 logging.getLogger('hyperdb').info('close')
1332 self.conn.close()
1334 def close(self):
1335 """ Close off the connection.
1336 """
1337 self.indexer.close()
1338 self.sql_close()
1340 #
1341 # The base Class class
1342 #
1343 class Class(hyperdb.Class):
1344 """ The handle to a particular class of nodes in a hyperdatabase.
1346 All methods except __repr__ and getnode must be implemented by a
1347 concrete backend Class.
1348 """
1350 def schema(self):
1351 """ A dumpable version of the schema that we can store in the
1352 database
1353 """
1354 return (self.key, [(x, repr(y)) for x,y in self.properties.items()])
1356 def enableJournalling(self):
1357 """Turn journalling on for this class
1358 """
1359 self.do_journal = 1
1361 def disableJournalling(self):
1362 """Turn journalling off for this class
1363 """
1364 self.do_journal = 0
1366 # Editing nodes:
1367 def create(self, **propvalues):
1368 """ Create a new node of this class and return its id.
1370 The keyword arguments in 'propvalues' map property names to values.
1372 The values of arguments must be acceptable for the types of their
1373 corresponding properties or a TypeError is raised.
1375 If this class has a key property, it must be present and its value
1376 must not collide with other key strings or a ValueError is raised.
1378 Any other properties on this class that are missing from the
1379 'propvalues' dictionary are set to None.
1381 If an id in a link or multilink property does not refer to a valid
1382 node, an IndexError is raised.
1383 """
1384 self.fireAuditors('create', None, propvalues)
1385 newid = self.create_inner(**propvalues)
1386 self.fireReactors('create', newid, None)
1387 return newid
1389 def create_inner(self, **propvalues):
1390 """ Called by create, in-between the audit and react calls.
1391 """
1392 if propvalues.has_key('id'):
1393 raise KeyError, '"id" is reserved'
1395 if self.db.journaltag is None:
1396 raise DatabaseError, _('Database open read-only')
1398 if propvalues.has_key('creator') or propvalues.has_key('actor') or \
1399 propvalues.has_key('creation') or propvalues.has_key('activity'):
1400 raise KeyError, '"creator", "actor", "creation" and '\
1401 '"activity" are reserved'
1403 # new node's id
1404 newid = self.db.newid(self.classname)
1406 # validate propvalues
1407 num_re = re.compile('^\d+$')
1408 for key, value in propvalues.items():
1409 if key == self.key:
1410 try:
1411 self.lookup(value)
1412 except KeyError:
1413 pass
1414 else:
1415 raise ValueError, 'node with key "%s" exists'%value
1417 # try to handle this property
1418 try:
1419 prop = self.properties[key]
1420 except KeyError:
1421 raise KeyError, '"%s" has no property "%s"'%(self.classname,
1422 key)
1424 if value is not None and isinstance(prop, Link):
1425 if type(value) != type(''):
1426 raise ValueError, 'link value must be String'
1427 link_class = self.properties[key].classname
1428 # if it isn't a number, it's a key
1429 if not num_re.match(value):
1430 try:
1431 value = self.db.classes[link_class].lookup(value)
1432 except (TypeError, KeyError):
1433 raise IndexError, 'new property "%s": %s not a %s'%(
1434 key, value, link_class)
1435 elif not self.db.getclass(link_class).hasnode(value):
1436 raise IndexError, '%s has no node %s'%(link_class, value)
1438 # save off the value
1439 propvalues[key] = value
1441 # register the link with the newly linked node
1442 if self.do_journal and self.properties[key].do_journal:
1443 self.db.addjournal(link_class, value, 'link',
1444 (self.classname, newid, key))
1446 elif isinstance(prop, Multilink):
1447 if value is None:
1448 value = []
1449 if not hasattr(value, '__iter__'):
1450 raise TypeError, 'new property "%s" not an iterable of ids'%key
1452 # clean up and validate the list of links
1453 link_class = self.properties[key].classname
1454 l = []
1455 for entry in value:
1456 if type(entry) != type(''):
1457 raise ValueError, '"%s" multilink value (%r) '\
1458 'must contain Strings'%(key, value)
1459 # if it isn't a number, it's a key
1460 if not num_re.match(entry):
1461 try:
1462 entry = self.db.classes[link_class].lookup(entry)
1463 except (TypeError, KeyError):
1464 raise IndexError, 'new property "%s": %s not a %s'%(
1465 key, entry, self.properties[key].classname)
1466 l.append(entry)
1467 value = l
1468 propvalues[key] = value
1470 # handle additions
1471 for nodeid in value:
1472 if not self.db.getclass(link_class).hasnode(nodeid):
1473 raise IndexError, '%s has no node %s'%(link_class,
1474 nodeid)
1475 # register the link with the newly linked node
1476 if self.do_journal and self.properties[key].do_journal:
1477 self.db.addjournal(link_class, nodeid, 'link',
1478 (self.classname, newid, key))
1480 elif isinstance(prop, String):
1481 if type(value) != type('') and type(value) != type(u''):
1482 raise TypeError, 'new property "%s" not a string'%key
1483 if prop.indexme:
1484 self.db.indexer.add_text((self.classname, newid, key),
1485 value)
1487 elif isinstance(prop, Password):
1488 if not isinstance(value, password.Password):
1489 raise TypeError, 'new property "%s" not a Password'%key
1491 elif isinstance(prop, Date):
1492 if value is not None and not isinstance(value, date.Date):
1493 raise TypeError, 'new property "%s" not a Date'%key
1495 elif isinstance(prop, Interval):
1496 if value is not None and not isinstance(value, date.Interval):
1497 raise TypeError, 'new property "%s" not an Interval'%key
1499 elif value is not None and isinstance(prop, Number):
1500 try:
1501 float(value)
1502 except ValueError:
1503 raise TypeError, 'new property "%s" not numeric'%key
1505 elif value is not None and isinstance(prop, Boolean):
1506 try:
1507 int(value)
1508 except ValueError:
1509 raise TypeError, 'new property "%s" not boolean'%key
1511 # make sure there's data where there needs to be
1512 for key, prop in self.properties.items():
1513 if propvalues.has_key(key):
1514 continue
1515 if key == self.key:
1516 raise ValueError, 'key property "%s" is required'%key
1517 if isinstance(prop, Multilink):
1518 propvalues[key] = []
1519 else:
1520 propvalues[key] = None
1522 # done
1523 self.db.addnode(self.classname, newid, propvalues)
1524 if self.do_journal:
1525 self.db.addjournal(self.classname, newid, ''"create", {})
1527 # XXX numeric ids
1528 return str(newid)
1530 def get(self, nodeid, propname, default=_marker, cache=1):
1531 """Get the value of a property on an existing node of this class.
1533 'nodeid' must be the id of an existing node of this class or an
1534 IndexError is raised. 'propname' must be the name of a property
1535 of this class or a KeyError is raised.
1537 'cache' exists for backwards compatibility, and is not used.
1538 """
1539 if propname == 'id':
1540 return nodeid
1542 # get the node's dict
1543 d = self.db.getnode(self.classname, nodeid)
1545 if propname == 'creation':
1546 if d.has_key('creation'):
1547 return d['creation']
1548 else:
1549 return date.Date()
1550 if propname == 'activity':
1551 if d.has_key('activity'):
1552 return d['activity']
1553 else:
1554 return date.Date()
1555 if propname == 'creator':
1556 if d.has_key('creator'):
1557 return d['creator']
1558 else:
1559 return self.db.getuid()
1560 if propname == 'actor':
1561 if d.has_key('actor'):
1562 return d['actor']
1563 else:
1564 return self.db.getuid()
1566 # get the property (raises KeyErorr if invalid)
1567 prop = self.properties[propname]
1569 # XXX may it be that propname is valid property name
1570 # (above error is not raised) and not d.has_key(propname)???
1571 if (not d.has_key(propname)) or (d[propname] is None):
1572 if default is _marker:
1573 if isinstance(prop, Multilink):
1574 return []
1575 else:
1576 return None
1577 else:
1578 return default
1580 # don't pass our list to other code
1581 if isinstance(prop, Multilink):
1582 return d[propname][:]
1584 return d[propname]
1586 def set(self, nodeid, **propvalues):
1587 """Modify a property on an existing node of this class.
1589 'nodeid' must be the id of an existing node of this class or an
1590 IndexError is raised.
1592 Each key in 'propvalues' must be the name of a property of this
1593 class or a KeyError is raised.
1595 All values in 'propvalues' must be acceptable types for their
1596 corresponding properties or a TypeError is raised.
1598 If the value of the key property is set, it must not collide with
1599 other key strings or a ValueError is raised.
1601 If the value of a Link or Multilink property contains an invalid
1602 node id, a ValueError is raised.
1603 """
1604 self.fireAuditors('set', nodeid, propvalues)
1605 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1606 propvalues = self.set_inner(nodeid, **propvalues)
1607 self.fireReactors('set', nodeid, oldvalues)
1608 return propvalues
1610 def set_inner(self, nodeid, **propvalues):
1611 """ Called by set, in-between the audit and react calls.
1612 """
1613 if not propvalues:
1614 return propvalues
1616 if propvalues.has_key('creation') or propvalues.has_key('creator') or \
1617 propvalues.has_key('actor') or propvalues.has_key('activity'):
1618 raise KeyError, '"creation", "creator", "actor" and '\
1619 '"activity" are reserved'
1621 if propvalues.has_key('id'):
1622 raise KeyError, '"id" is reserved'
1624 if self.db.journaltag is None:
1625 raise DatabaseError, _('Database open read-only')
1627 node = self.db.getnode(self.classname, nodeid)
1628 if self.is_retired(nodeid):
1629 raise IndexError, 'Requested item is retired'
1630 num_re = re.compile('^\d+$')
1632 # make a copy of the values dictionary - we'll modify the contents
1633 propvalues = propvalues.copy()
1635 # if the journal value is to be different, store it in here
1636 journalvalues = {}
1638 # remember the add/remove stuff for multilinks, making it easier
1639 # for the Database layer to do its stuff
1640 multilink_changes = {}
1642 for propname, value in propvalues.items():
1643 # check to make sure we're not duplicating an existing key
1644 if propname == self.key and node[propname] != value:
1645 try:
1646 self.lookup(value)
1647 except KeyError:
1648 pass
1649 else:
1650 raise ValueError, 'node with key "%s" exists'%value
1652 # this will raise the KeyError if the property isn't valid
1653 # ... we don't use getprops() here because we only care about
1654 # the writeable properties.
1655 try:
1656 prop = self.properties[propname]
1657 except KeyError:
1658 raise KeyError, '"%s" has no property named "%s"'%(
1659 self.classname, propname)
1661 # if the value's the same as the existing value, no sense in
1662 # doing anything
1663 current = node.get(propname, None)
1664 if value == current:
1665 del propvalues[propname]
1666 continue
1667 journalvalues[propname] = current
1669 # do stuff based on the prop type
1670 if isinstance(prop, Link):
1671 link_class = prop.classname
1672 # if it isn't a number, it's a key
1673 if value is not None and not isinstance(value, type('')):
1674 raise ValueError, 'property "%s" link value be a string'%(
1675 propname)
1676 if isinstance(value, type('')) and not num_re.match(value):
1677 try:
1678 value = self.db.classes[link_class].lookup(value)
1679 except (TypeError, KeyError):
1680 raise IndexError, 'new property "%s": %s not a %s'%(
1681 propname, value, prop.classname)
1683 if (value is not None and
1684 not self.db.getclass(link_class).hasnode(value)):
1685 raise IndexError, '%s has no node %s'%(link_class, value)
1687 if self.do_journal and prop.do_journal:
1688 # register the unlink with the old linked node
1689 if node[propname] is not None:
1690 self.db.addjournal(link_class, node[propname],
1691 ''"unlink", (self.classname, nodeid, propname))
1693 # register the link with the newly linked node
1694 if value is not None:
1695 self.db.addjournal(link_class, value, ''"link",
1696 (self.classname, nodeid, propname))
1698 elif isinstance(prop, Multilink):
1699 if value is None:
1700 value = []
1701 if not hasattr(value, '__iter__'):
1702 raise TypeError, 'new property "%s" not an iterable of'\
1703 ' ids'%propname
1704 link_class = self.properties[propname].classname
1705 l = []
1706 for entry in value:
1707 # if it isn't a number, it's a key
1708 if type(entry) != type(''):
1709 raise ValueError, 'new property "%s" link value ' \
1710 'must be a string'%propname
1711 if not num_re.match(entry):
1712 try:
1713 entry = self.db.classes[link_class].lookup(entry)
1714 except (TypeError, KeyError):
1715 raise IndexError, 'new property "%s": %s not a %s'%(
1716 propname, entry,
1717 self.properties[propname].classname)
1718 l.append(entry)
1719 value = l
1720 propvalues[propname] = value
1722 # figure the journal entry for this property
1723 add = []
1724 remove = []
1726 # handle removals
1727 if node.has_key(propname):
1728 l = node[propname]
1729 else:
1730 l = []
1731 for id in l[:]:
1732 if id in value:
1733 continue
1734 # register the unlink with the old linked node
1735 if self.do_journal and self.properties[propname].do_journal:
1736 self.db.addjournal(link_class, id, 'unlink',
1737 (self.classname, nodeid, propname))
1738 l.remove(id)
1739 remove.append(id)
1741 # handle additions
1742 for id in value:
1743 if id in l:
1744 continue
1745 # We can safely check this condition after
1746 # checking that this is an addition to the
1747 # multilink since the condition was checked for
1748 # existing entries at the point they were added to
1749 # the multilink. Since the hasnode call will
1750 # result in a SQL query, it is more efficient to
1751 # avoid the check if possible.
1752 if not self.db.getclass(link_class).hasnode(id):
1753 raise IndexError, '%s has no node %s'%(link_class, id)
1754 # register the link with the newly linked node
1755 if self.do_journal and self.properties[propname].do_journal:
1756 self.db.addjournal(link_class, id, 'link',
1757 (self.classname, nodeid, propname))
1758 l.append(id)
1759 add.append(id)
1761 # figure the journal entry
1762 l = []
1763 if add:
1764 l.append(('+', add))
1765 if remove:
1766 l.append(('-', remove))
1767 multilink_changes[propname] = (add, remove)
1768 if l:
1769 journalvalues[propname] = tuple(l)
1771 elif isinstance(prop, String):
1772 if value is not None and type(value) != type('') and type(value) != type(u''):
1773 raise TypeError, 'new property "%s" not a string'%propname
1774 if prop.indexme:
1775 if value is None: value = ''
1776 self.db.indexer.add_text((self.classname, nodeid, propname),
1777 value)
1779 elif isinstance(prop, Password):
1780 if not isinstance(value, password.Password):
1781 raise TypeError, 'new property "%s" not a Password'%propname
1782 propvalues[propname] = value
1784 elif value is not None and isinstance(prop, Date):
1785 if not isinstance(value, date.Date):
1786 raise TypeError, 'new property "%s" not a Date'% propname
1787 propvalues[propname] = value
1789 elif value is not None and isinstance(prop, Interval):
1790 if not isinstance(value, date.Interval):
1791 raise TypeError, 'new property "%s" not an '\
1792 'Interval'%propname
1793 propvalues[propname] = value
1795 elif value is not None and isinstance(prop, Number):
1796 try:
1797 float(value)
1798 except ValueError:
1799 raise TypeError, 'new property "%s" not numeric'%propname
1801 elif value is not None and isinstance(prop, Boolean):
1802 try:
1803 int(value)
1804 except ValueError:
1805 raise TypeError, 'new property "%s" not boolean'%propname
1807 # nothing to do?
1808 if not propvalues:
1809 return propvalues
1811 # update the activity time
1812 propvalues['activity'] = date.Date()
1813 propvalues['actor'] = self.db.getuid()
1815 # do the set
1816 self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
1818 # remove the activity props now they're handled
1819 del propvalues['activity']
1820 del propvalues['actor']
1822 # journal the set
1823 if self.do_journal:
1824 self.db.addjournal(self.classname, nodeid, ''"set", journalvalues)
1826 return propvalues
1828 def retire(self, nodeid):
1829 """Retire a node.
1831 The properties on the node remain available from the get() method,
1832 and the node's id is never reused.
1834 Retired nodes are not returned by the find(), list(), or lookup()
1835 methods, and other nodes may reuse the values of their key properties.
1836 """
1837 if self.db.journaltag is None:
1838 raise DatabaseError, _('Database open read-only')
1840 self.fireAuditors('retire', nodeid, None)
1842 # use the arg for __retired__ to cope with any odd database type
1843 # conversion (hello, sqlite)
1844 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1845 self.db.arg, self.db.arg)
1846 self.db.sql(sql, (nodeid, nodeid))
1847 if self.do_journal:
1848 self.db.addjournal(self.classname, nodeid, ''"retired", None)
1850 self.fireReactors('retire', nodeid, None)
1852 def restore(self, nodeid):
1853 """Restore a retired node.
1855 Make node available for all operations like it was before retirement.
1856 """
1857 if self.db.journaltag is None:
1858 raise DatabaseError, _('Database open read-only')
1860 node = self.db.getnode(self.classname, nodeid)
1861 # check if key property was overrided
1862 key = self.getkey()
1863 try:
1864 id = self.lookup(node[key])
1865 except KeyError:
1866 pass
1867 else:
1868 raise KeyError, "Key property (%s) of retired node clashes with \
1869 existing one (%s)" % (key, node[key])
1871 self.fireAuditors('restore', nodeid, None)
1872 # use the arg for __retired__ to cope with any odd database type
1873 # conversion (hello, sqlite)
1874 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1875 self.db.arg, self.db.arg)
1876 self.db.sql(sql, (0, nodeid))
1877 if self.do_journal:
1878 self.db.addjournal(self.classname, nodeid, ''"restored", None)
1880 self.fireReactors('restore', nodeid, None)
1882 def is_retired(self, nodeid):
1883 """Return true if the node is rerired
1884 """
1885 sql = 'select __retired__ from _%s where id=%s'%(self.classname,
1886 self.db.arg)
1887 self.db.sql(sql, (nodeid,))
1888 return int(self.db.sql_fetchone()[0]) > 0
1890 def destroy(self, nodeid):
1891 """Destroy a node.
1893 WARNING: this method should never be used except in extremely rare
1894 situations where there could never be links to the node being
1895 deleted
1897 WARNING: use retire() instead
1899 WARNING: the properties of this node will not be available ever again
1901 WARNING: really, use retire() instead
1903 Well, I think that's enough warnings. This method exists mostly to
1904 support the session storage of the cgi interface.
1906 The node is completely removed from the hyperdb, including all journal
1907 entries. It will no longer be available, and will generally break code
1908 if there are any references to the node.
1909 """
1910 if self.db.journaltag is None:
1911 raise DatabaseError, _('Database open read-only')
1912 self.db.destroynode(self.classname, nodeid)
1914 def history(self, nodeid):
1915 """Retrieve the journal of edits on a particular node.
1917 'nodeid' must be the id of an existing node of this class or an
1918 IndexError is raised.
1920 The returned list contains tuples of the form
1922 (nodeid, date, tag, action, params)
1924 'date' is a Timestamp object specifying the time of the change and
1925 'tag' is the journaltag specified when the database was opened.
1926 """
1927 if not self.do_journal:
1928 raise ValueError, 'Journalling is disabled for this class'
1929 return self.db.getjournal(self.classname, nodeid)
1931 # Locating nodes:
1932 def hasnode(self, nodeid):
1933 """Determine if the given nodeid actually exists
1934 """
1935 return self.db.hasnode(self.classname, nodeid)
1937 def setkey(self, propname):
1938 """Select a String property of this class to be the key property.
1940 'propname' must be the name of a String property of this class or
1941 None, or a TypeError is raised. The values of the key property on
1942 all existing nodes must be unique or a ValueError is raised.
1943 """
1944 prop = self.getprops()[propname]
1945 if not isinstance(prop, String):
1946 raise TypeError, 'key properties must be String'
1947 self.key = propname
1949 def getkey(self):
1950 """Return the name of the key property for this class or None."""
1951 return self.key
1953 def lookup(self, keyvalue):
1954 """Locate a particular node by its key property and return its id.
1956 If this class has no key property, a TypeError is raised. If the
1957 'keyvalue' matches one of the values for the key property among
1958 the nodes in this class, the matching node's id is returned;
1959 otherwise a KeyError is raised.
1960 """
1961 if not self.key:
1962 raise TypeError, 'No key property set for class %s'%self.classname
1964 # use the arg to handle any odd database type conversion (hello,
1965 # sqlite)
1966 sql = "select id from _%s where _%s=%s and __retired__=%s"%(
1967 self.classname, self.key, self.db.arg, self.db.arg)
1968 self.db.sql(sql, (str(keyvalue), 0))
1970 # see if there was a result that's not retired
1971 row = self.db.sql_fetchone()
1972 if not row:
1973 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1974 keyvalue, self.classname)
1976 # return the id
1977 # XXX numeric ids
1978 return str(row[0])
1980 def find(self, **propspec):
1981 """Get the ids of nodes in this class which link to the given nodes.
1983 'propspec' consists of keyword args propname=nodeid or
1984 propname={nodeid:1, }
1985 'propname' must be the name of a property in this class, or a
1986 KeyError is raised. That property must be a Link or
1987 Multilink property, or a TypeError is raised.
1989 Any node in this class whose 'propname' property links to any of
1990 the nodeids will be returned. Examples::
1992 db.issue.find(messages='1')
1993 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1994 """
1995 # shortcut
1996 if not propspec:
1997 return []
1999 # validate the args
2000 props = self.getprops()
2001 propspec = propspec.items()
2002 for propname, nodeids in propspec:
2003 # check the prop is OK
2004 prop = props[propname]
2005 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
2006 raise TypeError, "'%s' not a Link/Multilink property"%propname
2008 # first, links
2009 a = self.db.arg
2010 allvalues = ()
2011 sql = []
2012 where = []
2013 for prop, values in propspec:
2014 if not isinstance(props[prop], hyperdb.Link):
2015 continue
2016 if type(values) is type({}) and len(values) == 1:
2017 values = values.keys()[0]
2018 if type(values) is type(''):
2019 allvalues += (values,)
2020 where.append('_%s = %s'%(prop, a))
2021 elif values is None:
2022 where.append('_%s is NULL'%prop)
2023 else:
2024 values = values.keys()
2025 s = ''
2026 if None in values:
2027 values.remove(None)
2028 s = '_%s is NULL or '%prop
2029 allvalues += tuple(values)
2030 s += '_%s in (%s)'%(prop, ','.join([a]*len(values)))
2031 where.append('(' + s +')')
2032 if where:
2033 allvalues = (0, ) + allvalues
2034 sql.append("""select id from _%s where __retired__=%s
2035 and %s"""%(self.classname, a, ' and '.join(where)))
2037 # now multilinks
2038 for prop, values in propspec:
2039 if not isinstance(props[prop], hyperdb.Multilink):
2040 continue
2041 if not values:
2042 continue
2043 allvalues += (0, )
2044 if type(values) is type(''):
2045 allvalues += (values,)
2046 s = a
2047 else:
2048 allvalues += tuple(values.keys())
2049 s = ','.join([a]*len(values))
2050 tn = '%s_%s'%(self.classname, prop)
2051 sql.append("""select id from _%s, %s where __retired__=%s
2052 and id = %s.nodeid and %s.linkid in (%s)"""%(self.classname,
2053 tn, a, tn, tn, s))
2055 if not sql:
2056 return []
2057 sql = ' union '.join(sql)
2058 self.db.sql(sql, allvalues)
2059 # XXX numeric ids
2060 l = [str(x[0]) for x in self.db.sql_fetchall()]
2061 return l
2063 def stringFind(self, **requirements):
2064 """Locate a particular node by matching a set of its String
2065 properties in a caseless search.
2067 If the property is not a String property, a TypeError is raised.
2069 The return is a list of the id of all nodes that match.
2070 """
2071 where = []
2072 args = []
2073 for propname in requirements.keys():
2074 prop = self.properties[propname]
2075 if not isinstance(prop, String):
2076 raise TypeError, "'%s' not a String property"%propname
2077 where.append(propname)
2078 args.append(requirements[propname].lower())
2080 # generate the where clause
2081 s = ' and '.join(['lower(_%s)=%s'%(col, self.db.arg) for col in where])
2082 sql = 'select id from _%s where %s and __retired__=%s'%(
2083 self.classname, s, self.db.arg)
2084 args.append(0)
2085 self.db.sql(sql, tuple(args))
2086 # XXX numeric ids
2087 l = [str(x[0]) for x in self.db.sql_fetchall()]
2088 return l
2090 def list(self):
2091 """ Return a list of the ids of the active nodes in this class.
2092 """
2093 return self.getnodeids(retired=0)
2095 def getnodeids(self, retired=None):
2096 """ Retrieve all the ids of the nodes for a particular Class.
2098 Set retired=None to get all nodes. Otherwise it'll get all the
2099 retired or non-retired nodes, depending on the flag.
2100 """
2101 # flip the sense of the 'retired' flag if we don't want all of them
2102 if retired is not None:
2103 args = (0, )
2104 if retired:
2105 compare = '>'
2106 else:
2107 compare = '='
2108 sql = 'select id from _%s where __retired__%s%s'%(self.classname,
2109 compare, self.db.arg)
2110 else:
2111 args = ()
2112 sql = 'select id from _%s'%self.classname
2113 self.db.sql(sql, args)
2114 # XXX numeric ids
2115 ids = [str(x[0]) for x in self.db.cursor.fetchall()]
2116 return ids
2118 def _subselect(self, classname, multilink_table):
2119 """Create a subselect. This is factored out because some
2120 databases (hmm only one, so far) doesn't support subselects
2121 look for "I can't believe it's not a toy RDBMS" in the mysql
2122 backend.
2123 """
2124 return '_%s.id not in (select nodeid from %s)'%(classname,
2125 multilink_table)
2127 # Some DBs order NULL values last. Set this variable in the backend
2128 # for prepending an order by clause for each attribute that causes
2129 # correct sort order for NULLs. Examples:
2130 # order_by_null_values = '(%s is not NULL)'
2131 # order_by_null_values = 'notnull(%s)'
2132 # The format parameter is replaced with the attribute.
2133 order_by_null_values = None
2135 def filter(self, search_matches, filterspec, sort=[], group=[]):
2136 """Return a list of the ids of the active nodes in this class that
2137 match the 'filter' spec, sorted by the group spec and then the
2138 sort spec
2140 "filterspec" is {propname: value(s)}
2142 "sort" and "group" are [(dir, prop), ...] where dir is '+', '-'
2143 or None and prop is a prop name or None. Note that for
2144 backward-compatibility reasons a single (dir, prop) tuple is
2145 also allowed.
2147 "search_matches" is a container type or None
2149 The filter must match all properties specificed. If the property
2150 value to match is a list:
2152 1. String properties must match all elements in the list, and
2153 2. Other properties must match any of the elements in the list.
2154 """
2155 # we can't match anything if search_matches is empty
2156 if not search_matches and search_matches is not None:
2157 return []
2159 if __debug__:
2160 start_t = time.time()
2162 icn = self.classname
2164 # vars to hold the components of the SQL statement
2165 frum = [] # FROM clauses
2166 loj = [] # LEFT OUTER JOIN clauses
2167 where = [] # WHERE clauses
2168 args = [] # *any* positional arguments
2169 a = self.db.arg
2171 # figure the WHERE clause from the filterspec
2172 mlfilt = 0 # are we joining with Multilink tables?
2173 sortattr = self._sortattr (group = group, sort = sort)
2174 proptree = self._proptree(filterspec, sortattr)
2175 mlseen = 0
2176 for pt in reversed(proptree.sortattr):
2177 p = pt
2178 while p.parent:
2179 if isinstance (p.propclass, Multilink):
2180 mlseen = True
2181 if mlseen:
2182 p.sort_ids_needed = True
2183 p.tree_sort_done = False
2184 p = p.parent
2185 if not mlseen:
2186 pt.attr_sort_done = pt.tree_sort_done = True
2187 proptree.compute_sort_done()
2189 ordercols = []
2190 auxcols = {}
2191 mlsort = []
2192 rhsnum = 0
2193 for p in proptree:
2194 oc = None
2195 cn = p.classname
2196 ln = p.uniqname
2197 pln = p.parent.uniqname
2198 pcn = p.parent.classname
2199 k = p.name
2200 v = p.val
2201 propclass = p.propclass
2202 if p.sort_type > 0:
2203 oc = ac = '_%s._%s'%(pln, k)
2204 if isinstance(propclass, Multilink):
2205 if p.sort_type < 2:
2206 mlfilt = 1
2207 tn = '%s_%s'%(pcn, k)
2208 if v in ('-1', ['-1'], []):
2209 # only match rows that have count(linkid)=0 in the
2210 # corresponding multilink table)
2211 where.append(self._subselect(pcn, tn))
2212 else:
2213 frum.append(tn)
2214 where.append('_%s.id=%s.nodeid'%(pln,tn))
2215 if p.children:
2216 frum.append('_%s as _%s' % (cn, ln))
2217 where.append('%s.linkid=_%s.id'%(tn, ln))
2218 if p.has_values:
2219 if isinstance(v, type([])):
2220 s = ','.join([a for x in v])
2221 where.append('%s.linkid in (%s)'%(tn, s))
2222 args = args + v
2223 else:
2224 where.append('%s.linkid=%s'%(tn, a))
2225 args.append(v)
2226 if p.sort_type > 0:
2227 assert not p.attr_sort_done and not p.sort_ids_needed
2228 elif k == 'id':
2229 if p.sort_type < 2:
2230 if isinstance(v, type([])):
2231 s = ','.join([a for x in v])
2232 where.append('_%s.%s in (%s)'%(pln, k, s))
2233 args = args + v
2234 else:
2235 where.append('_%s.%s=%s'%(pln, k, a))
2236 args.append(v)
2237 if p.sort_type > 0:
2238 oc = ac = '_%s.id'%pln
2239 elif isinstance(propclass, String):
2240 if p.sort_type < 2:
2241 if not isinstance(v, type([])):
2242 v = [v]
2244 # Quote the bits in the string that need it and then embed
2245 # in a "substring" search. Note - need to quote the '%' so
2246 # they make it through the python layer happily
2247 v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
2249 # now add to the where clause
2250 where.append('('
2251 +' and '.join(["_%s._%s LIKE '%s'"%(pln, k, s) for s in v])
2252 +')')
2253 # note: args are embedded in the query string now
2254 if p.sort_type > 0:
2255 oc = ac = 'lower(_%s._%s)'%(pln, k)
2256 elif isinstance(propclass, Link):
2257 if p.sort_type < 2:
2258 if p.children:
2259 if p.sort_type == 0:
2260 frum.append('_%s as _%s' % (cn, ln))
2261 where.append('_%s._%s=_%s.id'%(pln, k, ln))
2262 if p.has_values:
2263 if isinstance(v, type([])):
2264 d = {}
2265 for entry in v:
2266 if entry == '-1':
2267 entry = None
2268 d[entry] = entry
2269 l = []
2270 if d.has_key(None) or not d:
2271 if d.has_key(None): del d[None]
2272 l.append('_%s._%s is NULL'%(pln, k))
2273 if d:
2274 v = d.keys()
2275 s = ','.join([a for x in v])
2276 l.append('(_%s._%s in (%s))'%(pln, k, s))
2277 args = args + v
2278 if l:
2279 where.append('(' + ' or '.join(l) +')')
2280 else:
2281 if v in ('-1', None):
2282 v = None
2283 where.append('_%s._%s is NULL'%(pln, k))
2284 else:
2285 where.append('_%s._%s=%s'%(pln, k, a))
2286 args.append(v)
2287 if p.sort_type > 0:
2288 lp = p.cls.labelprop()
2289 oc = ac = '_%s._%s'%(pln, k)
2290 if lp != 'id':
2291 if p.tree_sort_done and p.sort_type > 0:
2292 loj.append(
2293 'LEFT OUTER JOIN _%s as _%s on _%s._%s=_%s.id'%(
2294 cn, ln, pln, k, ln))
2295 oc = '_%s._%s'%(ln, lp)
2296 elif isinstance(propclass, Date) and p.sort_type < 2:
2297 dc = self.db.to_sql_value(hyperdb.Date)
2298 if isinstance(v, type([])):
2299 s = ','.join([a for x in v])
2300 where.append('_%s._%s in (%s)'%(pln, k, s))
2301 args = args + [dc(date.Date(x)) for x in v]
2302 else:
2303 try:
2304 # Try to filter on range of dates
2305 date_rng = propclass.range_from_raw(v, self.db)
2306 if date_rng.from_value:
2307 where.append('_%s._%s >= %s'%(pln, k, a))
2308 args.append(dc(date_rng.from_value))
2309 if date_rng.to_value:
2310 where.append('_%s._%s <= %s'%(pln, k, a))
2311 args.append(dc(date_rng.to_value))
2312 except ValueError:
2313 # If range creation fails - ignore that search parameter
2314 pass
2315 elif isinstance(propclass, Interval):
2316 # filter/sort using the __<prop>_int__ column
2317 if p.sort_type < 2:
2318 if isinstance(v, type([])):
2319 s = ','.join([a for x in v])
2320 where.append('_%s.__%s_int__ in (%s)'%(pln, k, s))
2321 args = args + [date.Interval(x).as_seconds() for x in v]
2322 else:
2323 try:
2324 # Try to filter on range of intervals
2325 date_rng = Range(v, date.Interval)
2326 if date_rng.from_value:
2327 where.append('_%s.__%s_int__ >= %s'%(pln, k, a))
2328 args.append(date_rng.from_value.as_seconds())
2329 if date_rng.to_value:
2330 where.append('_%s.__%s_int__ <= %s'%(pln, k, a))
2331 args.append(date_rng.to_value.as_seconds())
2332 except ValueError:
2333 # If range creation fails - ignore search parameter
2334 pass
2335 if p.sort_type > 0:
2336 oc = ac = '_%s.__%s_int__'%(pln,k)
2337 elif p.sort_type < 2:
2338 if isinstance(v, type([])):
2339 s = ','.join([a for x in v])
2340 where.append('_%s._%s in (%s)'%(pln, k, s))
2341 args = args + v
2342 else:
2343 where.append('_%s._%s=%s'%(pln, k, a))
2344 args.append(v)
2345 if oc:
2346 if p.sort_ids_needed:
2347 auxcols[ac] = p
2348 if p.tree_sort_done and p.sort_direction:
2349 # Don't select top-level id twice
2350 if p.name != 'id' or p.parent != proptree:
2351 ordercols.append(oc)
2352 desc = ['', ' desc'][p.sort_direction == '-']
2353 # Some SQL dbs sort NULL values last -- we want them first.
2354 if (self.order_by_null_values and p.name != 'id'):
2355 nv = self.order_by_null_values % oc
2356 ordercols.append(nv)
2357 p.orderby.append(nv + desc)
2358 p.orderby.append(oc + desc)
2360 props = self.getprops()
2362 # don't match retired nodes
2363 where.append('_%s.__retired__=0'%icn)
2365 # add results of full text search
2366 if search_matches is not None:
2367 s = ','.join([a for x in search_matches])
2368 where.append('_%s.id in (%s)'%(icn, s))
2369 args = args + [x for x in search_matches]
2371 # construct the SQL
2372 frum.append('_'+icn)
2373 frum = ','.join(frum)
2374 if where:
2375 where = ' where ' + (' and '.join(where))
2376 else:
2377 where = ''
2378 if mlfilt:
2379 # we're joining tables on the id, so we will get dupes if we
2380 # don't distinct()
2381 cols = ['distinct(_%s.id)'%icn]
2382 else:
2383 cols = ['_%s.id'%icn]
2384 if ordercols:
2385 cols = cols + ordercols
2386 order = []
2387 # keep correct sequence of order attributes.
2388 for sa in proptree.sortattr:
2389 if not sa.attr_sort_done:
2390 continue
2391 order.extend(sa.orderby)
2392 if order:
2393 order = ' order by %s'%(','.join(order))
2394 else:
2395 order = ''
2396 for o, p in auxcols.iteritems ():
2397 cols.append (o)
2398 p.auxcol = len (cols) - 1
2400 cols = ','.join(cols)
2401 loj = ' '.join(loj)
2402 sql = 'select %s from %s %s %s%s'%(cols, frum, loj, where, order)
2403 args = tuple(args)
2404 __traceback_info__ = (sql, args)
2405 self.db.sql(sql, args)
2406 l = self.db.sql_fetchall()
2408 # Compute values needed for sorting in proptree.sort
2409 for p in auxcols.itervalues():
2410 p.sort_ids = p.sort_result = [row[p.auxcol] for row in l]
2411 # return the IDs (the first column)
2412 # XXX numeric ids
2413 l = [str(row[0]) for row in l]
2414 l = proptree.sort (l)
2416 if __debug__:
2417 self.db.stats['filtering'] += (time.time() - start_t)
2418 return l
2420 def filter_sql(self, sql):
2421 """Return a list of the ids of the items in this class that match
2422 the SQL provided. The SQL is a complete "select" statement.
2424 The SQL select must include the item id as the first column.
2426 This function DOES NOT filter out retired items, add on a where
2427 clause "__retired__=0" if you don't want retired nodes.
2428 """
2429 if __debug__:
2430 start_t = time.time()
2432 self.db.sql(sql)
2433 l = self.db.sql_fetchall()
2435 if __debug__:
2436 self.db.stats['filtering'] += (time.time() - start_t)
2437 return l
2439 def count(self):
2440 """Get the number of nodes in this class.
2442 If the returned integer is 'numnodes', the ids of all the nodes
2443 in this class run from 1 to numnodes, and numnodes+1 will be the
2444 id of the next node to be created in this class.
2445 """
2446 return self.db.countnodes(self.classname)
2448 # Manipulating properties:
2449 def getprops(self, protected=1):
2450 """Return a dictionary mapping property names to property objects.
2451 If the "protected" flag is true, we include protected properties -
2452 those which may not be modified.
2453 """
2454 d = self.properties.copy()
2455 if protected:
2456 d['id'] = String()
2457 d['creation'] = hyperdb.Date()
2458 d['activity'] = hyperdb.Date()
2459 d['creator'] = hyperdb.Link('user')
2460 d['actor'] = hyperdb.Link('user')
2461 return d
2463 def addprop(self, **properties):
2464 """Add properties to this class.
2466 The keyword arguments in 'properties' must map names to property
2467 objects, or a TypeError is raised. None of the keys in 'properties'
2468 may collide with the names of existing properties, or a ValueError
2469 is raised before any properties have been added.
2470 """
2471 for key in properties.keys():
2472 if self.properties.has_key(key):
2473 raise ValueError, key
2474 self.properties.update(properties)
2476 def index(self, nodeid):
2477 """Add (or refresh) the node to search indexes
2478 """
2479 # find all the String properties that have indexme
2480 for prop, propclass in self.getprops().items():
2481 if isinstance(propclass, String) and propclass.indexme:
2482 self.db.indexer.add_text((self.classname, nodeid, prop),
2483 str(self.get(nodeid, prop)))
2485 #
2486 # import / export support
2487 #
2488 def export_list(self, propnames, nodeid):
2489 """ Export a node - generate a list of CSV-able data in the order
2490 specified by propnames for the given node.
2491 """
2492 properties = self.getprops()
2493 l = []
2494 for prop in propnames:
2495 proptype = properties[prop]
2496 value = self.get(nodeid, prop)
2497 # "marshal" data where needed
2498 if value is None:
2499 pass
2500 elif isinstance(proptype, hyperdb.Date):
2501 value = value.get_tuple()
2502 elif isinstance(proptype, hyperdb.Interval):
2503 value = value.get_tuple()
2504 elif isinstance(proptype, hyperdb.Password):
2505 value = str(value)
2506 l.append(repr(value))
2507 l.append(repr(self.is_retired(nodeid)))
2508 return l
2510 def import_list(self, propnames, proplist):
2511 """ Import a node - all information including "id" is present and
2512 should not be sanity checked. Triggers are not triggered. The
2513 journal should be initialised using the "creator" and "created"
2514 information.
2516 Return the nodeid of the node imported.
2517 """
2518 if self.db.journaltag is None:
2519 raise DatabaseError, _('Database open read-only')
2520 properties = self.getprops()
2522 # make the new node's property map
2523 d = {}
2524 retire = 0
2525 if not "id" in propnames:
2526 newid = self.db.newid(self.classname)
2527 else:
2528 newid = eval(proplist[propnames.index("id")])
2529 for i in range(len(propnames)):
2530 # Use eval to reverse the repr() used to output the CSV
2531 value = eval(proplist[i])
2533 # Figure the property for this column
2534 propname = propnames[i]
2536 # "unmarshal" where necessary
2537 if propname == 'id':
2538 continue
2539 elif propname == 'is retired':
2540 # is the item retired?
2541 if int(value):
2542 retire = 1
2543 continue
2544 elif value is None:
2545 d[propname] = None
2546 continue
2548 prop = properties[propname]
2549 if value is None:
2550 # don't set Nones
2551 continue
2552 elif isinstance(prop, hyperdb.Date):
2553 value = date.Date(value)
2554 elif isinstance(prop, hyperdb.Interval):
2555 value = date.Interval(value)
2556 elif isinstance(prop, hyperdb.Password):
2557 pwd = password.Password()
2558 pwd.unpack(value)
2559 value = pwd
2560 elif isinstance(prop, String):
2561 if isinstance(value, unicode):
2562 value = value.encode('utf8')
2563 if not isinstance(value, str):
2564 raise TypeError, \
2565 'new property "%(propname)s" not a string: %(value)r' \
2566 % locals()
2567 if prop.indexme:
2568 self.db.indexer.add_text((self.classname, newid, propname),
2569 value)
2570 d[propname] = value
2572 # get a new id if necessary
2573 if newid is None:
2574 newid = self.db.newid(self.classname)
2576 # insert new node or update existing?
2577 if not self.hasnode(newid):
2578 self.db.addnode(self.classname, newid, d) # insert
2579 else:
2580 self.db.setnode(self.classname, newid, d) # update
2582 # retire?
2583 if retire:
2584 # use the arg for __retired__ to cope with any odd database type
2585 # conversion (hello, sqlite)
2586 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
2587 self.db.arg, self.db.arg)
2588 self.db.sql(sql, (newid, newid))
2589 return newid
2591 def export_journals(self):
2592 """Export a class's journal - generate a list of lists of
2593 CSV-able data:
2595 nodeid, date, user, action, params
2597 No heading here - the columns are fixed.
2598 """
2599 properties = self.getprops()
2600 r = []
2601 for nodeid in self.getnodeids():
2602 for nodeid, date, user, action, params in self.history(nodeid):
2603 date = date.get_tuple()
2604 if action == 'set':
2605 export_data = {}
2606 for propname, value in params.items():
2607 if not properties.has_key(propname):
2608 # property no longer in the schema
2609 continue
2611 prop = properties[propname]
2612 # make sure the params are eval()'able
2613 if value is None:
2614 pass
2615 elif isinstance(prop, Date):
2616 value = value.get_tuple()
2617 elif isinstance(prop, Interval):
2618 value = value.get_tuple()
2619 elif isinstance(prop, Password):
2620 value = str(value)
2621 export_data[propname] = value
2622 params = export_data
2623 elif action == 'create' and params:
2624 # old tracker with data stored in the create!
2625 params = {}
2626 l = [nodeid, date, user, action, params]
2627 r.append(map(repr, l))
2628 return r
2630 def import_journals(self, entries):
2631 """Import a class's journal.
2633 Uses setjournal() to set the journal for each item."""
2634 properties = self.getprops()
2635 d = {}
2636 for l in entries:
2637 l = map(eval, l)
2638 nodeid, jdate, user, action, params = l
2639 r = d.setdefault(nodeid, [])
2640 if action == 'set':
2641 for propname, value in params.items():
2642 prop = properties[propname]
2643 if value is None:
2644 pass
2645 elif isinstance(prop, Date):
2646 value = date.Date(value)
2647 elif isinstance(prop, Interval):
2648 value = date.Interval(value)
2649 elif isinstance(prop, Password):
2650 pwd = password.Password()
2651 pwd.unpack(value)
2652 value = pwd
2653 params[propname] = value
2654 elif action == 'create' and params:
2655 # old tracker with data stored in the create!
2656 params = {}
2657 r.append((nodeid, date.Date(jdate), user, action, params))
2659 for nodeid, l in d.items():
2660 self.db.setjournal(self.classname, nodeid, l)
2662 class FileClass(hyperdb.FileClass, Class):
2663 """This class defines a large chunk of data. To support this, it has a
2664 mandatory String property "content" which is typically saved off
2665 externally to the hyperdb.
2667 The default MIME type of this data is defined by the
2668 "default_mime_type" class attribute, which may be overridden by each
2669 node if the class defines a "type" String property.
2670 """
2671 def __init__(self, db, classname, **properties):
2672 """The newly-created class automatically includes the "content"
2673 and "type" properties.
2674 """
2675 if not properties.has_key('content'):
2676 properties['content'] = hyperdb.String(indexme='yes')
2677 if not properties.has_key('type'):
2678 properties['type'] = hyperdb.String()
2679 Class.__init__(self, db, classname, **properties)
2681 def create(self, **propvalues):
2682 """ snaffle the file propvalue and store in a file
2683 """
2684 # we need to fire the auditors now, or the content property won't
2685 # be in propvalues for the auditors to play with
2686 self.fireAuditors('create', None, propvalues)
2688 # now remove the content property so it's not stored in the db
2689 content = propvalues['content']
2690 del propvalues['content']
2692 # do the database create
2693 newid = self.create_inner(**propvalues)
2695 # figure the mime type
2696 mime_type = propvalues.get('type', self.default_mime_type)
2698 # and index!
2699 if self.properties['content'].indexme:
2700 self.db.indexer.add_text((self.classname, newid, 'content'),
2701 content, mime_type)
2703 # store off the content as a file
2704 self.db.storefile(self.classname, newid, None, content)
2706 # fire reactors
2707 self.fireReactors('create', newid, None)
2709 return newid
2711 def get(self, nodeid, propname, default=_marker, cache=1):
2712 """ Trap the content propname and get it from the file
2714 'cache' exists for backwards compatibility, and is not used.
2715 """
2716 poss_msg = 'Possibly a access right configuration problem.'
2717 if propname == 'content':
2718 try:
2719 return self.db.getfile(self.classname, nodeid, None)
2720 except IOError, (strerror):
2721 # BUG: by catching this we donot see an error in the log.
2722 return 'ERROR reading file: %s%s\n%s\n%s'%(
2723 self.classname, nodeid, poss_msg, strerror)
2724 if default is not _marker:
2725 return Class.get(self, nodeid, propname, default)
2726 else:
2727 return Class.get(self, nodeid, propname)
2729 def set(self, itemid, **propvalues):
2730 """ Snarf the "content" propvalue and update it in a file
2731 """
2732 self.fireAuditors('set', itemid, propvalues)
2733 oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid))
2735 # now remove the content property so it's not stored in the db
2736 content = None
2737 if propvalues.has_key('content'):
2738 content = propvalues['content']
2739 del propvalues['content']
2741 # do the database create
2742 propvalues = self.set_inner(itemid, **propvalues)
2744 # do content?
2745 if content:
2746 # store and possibly index
2747 self.db.storefile(self.classname, itemid, None, content)
2748 if self.properties['content'].indexme:
2749 mime_type = self.get(itemid, 'type', self.default_mime_type)
2750 self.db.indexer.add_text((self.classname, itemid, 'content'),
2751 content, mime_type)
2752 propvalues['content'] = content
2754 # fire reactors
2755 self.fireReactors('set', itemid, oldvalues)
2756 return propvalues
2758 def index(self, nodeid):
2759 """ Add (or refresh) the node to search indexes.
2761 Use the content-type property for the content property.
2762 """
2763 # find all the String properties that have indexme
2764 for prop, propclass in self.getprops().items():
2765 if prop == 'content' and propclass.indexme:
2766 mime_type = self.get(nodeid, 'type', self.default_mime_type)
2767 self.db.indexer.add_text((self.classname, nodeid, 'content'),
2768 str(self.get(nodeid, 'content')), mime_type)
2769 elif isinstance(propclass, hyperdb.String) and propclass.indexme:
2770 # index them under (classname, nodeid, property)
2771 try:
2772 value = str(self.get(nodeid, prop))
2773 except IndexError:
2774 # node has been destroyed
2775 continue
2776 self.db.indexer.add_text((self.classname, nodeid, prop), value)
2778 # XXX deviation from spec - was called ItemClass
2779 class IssueClass(Class, roundupdb.IssueClass):
2780 # Overridden methods:
2781 def __init__(self, db, classname, **properties):
2782 """The newly-created class automatically includes the "messages",
2783 "files", "nosy", and "superseder" properties. If the 'properties'
2784 dictionary attempts to specify any of these properties or a
2785 "creation", "creator", "activity" or "actor" property, a ValueError
2786 is raised.
2787 """
2788 if not properties.has_key('title'):
2789 properties['title'] = hyperdb.String(indexme='yes')
2790 if not properties.has_key('messages'):
2791 properties['messages'] = hyperdb.Multilink("msg")
2792 if not properties.has_key('files'):
2793 properties['files'] = hyperdb.Multilink("file")
2794 if not properties.has_key('nosy'):
2795 # note: journalling is turned off as it really just wastes
2796 # space. this behaviour may be overridden in an instance
2797 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2798 if not properties.has_key('superseder'):
2799 properties['superseder'] = hyperdb.Multilink(classname)
2800 Class.__init__(self, db, classname, **properties)
2802 # vim: set et sts=4 sw=4 :