1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 """ Relational database (SQL) backend common code.
20 Basics:
22 - map roundup classes to relational tables
23 - automatically detect schema changes and modify the table schemas
24 appropriately (we store the "database version" of the schema in the
25 database itself as the only row of the "schema" table)
26 - multilinks (which represent a many-to-many relationship) are handled through
27 intermediate tables
28 - journals are stored adjunct to the per-class tables
29 - table names and columns have "_" prepended so the names can't clash with
30 restricted names (like "order")
31 - retirement is determined by the __retired__ column being > 0
33 Database-specific changes may generally be pushed out to the overridable
34 sql_* methods, since everything else should be fairly generic. There's
35 probably a bit of work to be done if a database is used that actually
36 honors column typing, since the initial databases don't (sqlite stores
37 everything as a string.)
39 The schema of the hyperdb being mapped to the database is stored in the
40 database itself as a repr()'ed dictionary of information about each Class
41 that maps to a table. If that information differs from the hyperdb schema,
42 then we update it. We also store in the schema dict a version which
43 allows us to upgrade the database schema when necessary. See upgrade_db().
45 To force a unqiueness constraint on the key properties we put the item
46 id into the __retired__ column duing retirement (so it's 0 for "active"
47 items) and place a unqiueness constraint on key + __retired__. This is
48 particularly important for the users class where multiple users may
49 try to have the same username, with potentially many retired users with
50 the same name.
51 """
52 __docformat__ = 'restructuredtext'
54 # standard python modules
55 import sys, os, time, re, errno, weakref, copy, logging
57 # roundup modules
58 from roundup import hyperdb, date, password, roundupdb, security, support
59 from roundup.hyperdb import String, Password, Date, Interval, Link, \
60 Multilink, DatabaseError, Boolean, Number, Node
61 from roundup.backends import locking
62 from roundup.support import reversed
63 from roundup.i18n import _
65 # support
66 from blobfiles import FileStorage
67 try:
68 from indexer_xapian import Indexer
69 except ImportError:
70 from indexer_rdbms import Indexer
71 from sessions_rdbms import Sessions, OneTimeKeys
72 from roundup.date import Range
74 # dummy value meaning "argument not passed"
75 _marker = []
77 def _num_cvt(num):
78 num = str(num)
79 try:
80 return int(num)
81 except:
82 return float(num)
84 def _bool_cvt(value):
85 if value in ('TRUE', 'FALSE'):
86 return {'TRUE': 1, 'FALSE': 0}[value]
87 # assume it's a number returned from the db API
88 return int(value)
90 def connection_dict(config, dbnamestr=None):
91 """ Used by Postgresql and MySQL to detemine the keyword args for
92 opening the database connection."""
93 d = { }
94 if dbnamestr:
95 d[dbnamestr] = config.RDBMS_NAME
96 for name in ('host', 'port', 'password', 'user', 'read_default_group',
97 'read_default_file'):
98 cvar = 'RDBMS_'+name.upper()
99 if config[cvar] is not None:
100 d[name] = config[cvar]
101 return d
103 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
104 """ Wrapper around an SQL database that presents a hyperdb interface.
106 - some functionality is specific to the actual SQL database, hence
107 the sql_* methods that are NotImplemented
108 - we keep a cache of the latest N row fetches (where N is configurable).
109 """
110 def __init__(self, config, journaltag=None):
111 """ Open the database and load the schema from it.
112 """
113 FileStorage.__init__(self, config.UMASK)
114 self.config, self.journaltag = config, journaltag
115 self.dir = config.DATABASE
116 self.classes = {}
117 self.indexer = Indexer(self)
118 self.security = security.Security(self)
120 # additional transaction support for external files and the like
121 self.transactions = []
123 # keep a cache of the N most recently retrieved rows of any kind
124 # (classname, nodeid) = row
125 self.cache_size = config.RDBMS_CACHE_SIZE
126 self.cache = {}
127 self.cache_lru = []
128 self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0,
129 'filtering': 0}
131 # database lock
132 self.lockfile = None
134 # open a connection to the database, creating the "conn" attribute
135 self.open_connection()
137 def clearCache(self):
138 self.cache = {}
139 self.cache_lru = []
141 def getSessionManager(self):
142 return Sessions(self)
144 def getOTKManager(self):
145 return OneTimeKeys(self)
147 def open_connection(self):
148 """ Open a connection to the database, creating it if necessary.
150 Must call self.load_dbschema()
151 """
152 raise NotImplemented
154 def sql(self, sql, args=None):
155 """ Execute the sql with the optional args.
156 """
157 self.log_debug('SQL %r %r'%(sql, args))
158 if args:
159 self.cursor.execute(sql, args)
160 else:
161 self.cursor.execute(sql)
163 def sql_fetchone(self):
164 """ Fetch a single row. If there's nothing to fetch, return None.
165 """
166 return self.cursor.fetchone()
168 def sql_fetchall(self):
169 """ Fetch all rows. If there's nothing to fetch, return [].
170 """
171 return self.cursor.fetchall()
173 def sql_stringquote(self, value):
174 """ Quote the string so it's safe to put in the 'sql quotes'
175 """
176 return re.sub("'", "''", str(value))
178 def init_dbschema(self):
179 self.database_schema = {
180 'version': self.current_db_version,
181 'tables': {}
182 }
184 def load_dbschema(self):
185 """ Load the schema definition that the database currently implements
186 """
187 self.cursor.execute('select schema from schema')
188 schema = self.cursor.fetchone()
189 if schema:
190 self.database_schema = eval(schema[0])
191 else:
192 self.database_schema = {}
194 def save_dbschema(self):
195 """ Save the schema definition that the database currently implements
196 """
197 s = repr(self.database_schema)
198 self.sql('delete from schema')
199 self.sql('insert into schema values (%s)'%self.arg, (s,))
201 def post_init(self):
202 """ Called once the schema initialisation has finished.
204 We should now confirm that the schema defined by our "classes"
205 attribute actually matches the schema in the database.
206 """
207 save = 0
209 # handle changes in the schema
210 tables = self.database_schema['tables']
211 for classname, spec in self.classes.items():
212 if tables.has_key(classname):
213 dbspec = tables[classname]
214 if self.update_class(spec, dbspec):
215 tables[classname] = spec.schema()
216 save = 1
217 else:
218 self.create_class(spec)
219 tables[classname] = spec.schema()
220 save = 1
222 for classname, spec in tables.items():
223 if not self.classes.has_key(classname):
224 self.drop_class(classname, tables[classname])
225 del tables[classname]
226 save = 1
228 # now upgrade the database for column type changes, new internal
229 # tables, etc.
230 save = save | self.upgrade_db()
232 # update the database version of the schema
233 if save:
234 self.save_dbschema()
236 # reindex the db if necessary
237 if self.indexer.should_reindex():
238 self.reindex()
240 # commit
241 self.sql_commit()
243 # update this number when we need to make changes to the SQL structure
244 # of the backen database
245 current_db_version = 5
246 db_version_updated = False
247 def upgrade_db(self):
248 """ Update the SQL database to reflect changes in the backend code.
250 Return boolean whether we need to save the schema.
251 """
252 version = self.database_schema.get('version', 1)
253 if version > self.current_db_version:
254 raise DatabaseError('attempting to run rev %d DATABASE with rev '
255 '%d CODE!'%(version, self.current_db_version))
256 if version == self.current_db_version:
257 # nothing to do
258 return 0
260 if version < 2:
261 self.log_info('upgrade to version 2')
262 # change the schema structure
263 self.database_schema = {'tables': self.database_schema}
265 # version 1 didn't have the actor column (note that in
266 # MySQL this will also transition the tables to typed columns)
267 self.add_new_columns_v2()
269 # version 1 doesn't have the OTK, session and indexing in the
270 # database
271 self.create_version_2_tables()
273 if version < 3:
274 self.log_info('upgrade to version 3')
275 self.fix_version_2_tables()
277 if version < 4:
278 self.fix_version_3_tables()
280 if version < 5:
281 self.fix_version_4_tables()
283 self.database_schema['version'] = self.current_db_version
284 self.db_version_updated = True
285 return 1
287 def fix_version_3_tables(self):
288 # drop the shorter VARCHAR OTK column and add a new TEXT one
289 for name in ('otk', 'session'):
290 self.sql('DELETE FROM %ss'%name)
291 self.sql('ALTER TABLE %ss DROP %s_value'%(name, name))
292 self.sql('ALTER TABLE %ss ADD %s_value TEXT'%(name, name))
294 def fix_version_2_tables(self):
295 # Default (used by sqlite): NOOP
296 pass
298 def fix_version_4_tables(self):
299 # note this is an explicit call now
300 c = self.cursor
301 for cn, klass in self.classes.items():
302 c.execute('select id from _%s where __retired__<>0'%(cn,))
303 for (id,) in c.fetchall():
304 c.execute('update _%s set __retired__=%s where id=%s'%(cn,
305 self.arg, self.arg), (id, id))
307 if klass.key:
308 self.add_class_key_required_unique_constraint(cn, klass.key)
310 def _convert_journal_tables(self):
311 """Get current journal table contents, drop the table and re-create"""
312 c = self.cursor
313 cols = ','.join('nodeid date tag action params'.split())
314 for klass in self.classes.values():
315 # slurp and drop
316 sql = 'select %s from %s__journal order by date'%(cols,
317 klass.classname)
318 c.execute(sql)
319 contents = c.fetchall()
320 self.drop_journal_table_indexes(klass.classname)
321 c.execute('drop table %s__journal'%klass.classname)
323 # re-create and re-populate
324 self.create_journal_table(klass)
325 a = self.arg
326 sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(
327 klass.classname, cols, a, a, a, a, a)
328 for row in contents:
329 # no data conversion needed
330 self.cursor.execute(sql, row)
332 def _convert_string_properties(self):
333 """Get current Class tables that contain String properties, and
334 convert the VARCHAR columns to TEXT"""
335 c = self.cursor
336 for klass in self.classes.values():
337 # slurp and drop
338 cols, mls = self.determine_columns(klass.properties.items())
339 scols = ','.join([i[0] for i in cols])
340 sql = 'select id,%s from _%s'%(scols, klass.classname)
341 c.execute(sql)
342 contents = c.fetchall()
343 self.drop_class_table_indexes(klass.classname, klass.getkey())
344 c.execute('drop table _%s'%klass.classname)
346 # re-create and re-populate
347 self.create_class_table(klass, create_sequence=0)
348 a = ','.join([self.arg for i in range(len(cols)+1)])
349 sql = 'insert into _%s (id,%s) values (%s)'%(klass.classname,
350 scols, a)
351 for row in contents:
352 l = []
353 for entry in row:
354 # mysql will already be a string - psql needs "help"
355 if entry is not None and not isinstance(entry, type('')):
356 entry = str(entry)
357 l.append(entry)
358 self.cursor.execute(sql, l)
360 def refresh_database(self):
361 self.post_init()
364 def reindex(self, classname=None, show_progress=False):
365 if classname:
366 classes = [self.getclass(classname)]
367 else:
368 classes = self.classes.values()
369 for klass in classes:
370 if show_progress:
371 for nodeid in support.Progress('Reindex %s'%klass.classname,
372 klass.list()):
373 klass.index(nodeid)
374 else:
375 for nodeid in klass.list():
376 klass.index(nodeid)
377 self.indexer.save_index()
379 hyperdb_to_sql_datatypes = {
380 hyperdb.String : 'TEXT',
381 hyperdb.Date : 'TIMESTAMP',
382 hyperdb.Link : 'INTEGER',
383 hyperdb.Interval : 'VARCHAR(255)',
384 hyperdb.Password : 'VARCHAR(255)',
385 hyperdb.Boolean : 'BOOLEAN',
386 hyperdb.Number : 'REAL',
387 }
389 def hyperdb_to_sql_datatype(self, propclass):
391 datatype = self.hyperdb_to_sql_datatypes.get(propclass)
392 if datatype:
393 return datatype
395 for k, v in self.hyperdb_to_sql_datatypes.iteritems():
396 if issubclass(propclass, k):
397 return v
399 raise ValueError, '%r is not a hyperdb property class' % propclass
401 def determine_columns(self, properties):
402 """ Figure the column names and multilink properties from the spec
404 "properties" is a list of (name, prop) where prop may be an
405 instance of a hyperdb "type" _or_ a string repr of that type.
406 """
407 cols = [
408 ('_actor', self.hyperdb_to_sql_datatype(hyperdb.Link)),
409 ('_activity', self.hyperdb_to_sql_datatype(hyperdb.Date)),
410 ('_creator', self.hyperdb_to_sql_datatype(hyperdb.Link)),
411 ('_creation', self.hyperdb_to_sql_datatype(hyperdb.Date)),
412 ]
413 mls = []
414 # add the multilinks separately
415 for col, prop in properties:
416 if isinstance(prop, Multilink):
417 mls.append(col)
418 continue
420 if isinstance(prop, type('')):
421 raise ValueError, "string property spec!"
422 #and prop.find('Multilink') != -1:
423 #mls.append(col)
425 datatype = self.hyperdb_to_sql_datatype(prop.__class__)
426 cols.append(('_'+col, datatype))
428 # Intervals stored as two columns
429 if isinstance(prop, Interval):
430 cols.append(('__'+col+'_int__', 'BIGINT'))
432 cols.sort()
433 return cols, mls
435 def update_class(self, spec, old_spec, force=0):
436 """ Determine the differences between the current spec and the
437 database version of the spec, and update where necessary.
439 If 'force' is true, update the database anyway.
440 """
441 new_has = spec.properties.has_key
442 new_spec = spec.schema()
443 new_spec[1].sort()
444 old_spec[1].sort()
445 if not force and new_spec == old_spec:
446 # no changes
447 return 0
449 logger = logging.getLogger('hyperdb')
450 logger.info('update_class %s'%spec.classname)
452 logger.debug('old_spec %r'%(old_spec,))
453 logger.debug('new_spec %r'%(new_spec,))
455 # detect key prop change for potential index change
456 keyprop_changes = {}
457 if new_spec[0] != old_spec[0]:
458 if old_spec[0]:
459 keyprop_changes['remove'] = old_spec[0]
460 if new_spec[0]:
461 keyprop_changes['add'] = new_spec[0]
463 # detect multilinks that have been removed, and drop their table
464 old_has = {}
465 for name, prop in old_spec[1]:
466 old_has[name] = 1
467 if new_has(name):
468 continue
470 if prop.find('Multilink to') != -1:
471 # first drop indexes.
472 self.drop_multilink_table_indexes(spec.classname, name)
474 # now the multilink table itself
475 sql = 'drop table %s_%s'%(spec.classname, name)
476 else:
477 # if this is the key prop, drop the index first
478 if old_spec[0] == prop:
479 self.drop_class_table_key_index(spec.classname, name)
480 del keyprop_changes['remove']
482 # drop the column
483 sql = 'alter table _%s drop column _%s'%(spec.classname, name)
485 self.sql(sql)
486 old_has = old_has.has_key
488 # if we didn't remove the key prop just then, but the key prop has
489 # changed, we still need to remove the old index
490 if keyprop_changes.has_key('remove'):
491 self.drop_class_table_key_index(spec.classname,
492 keyprop_changes['remove'])
494 # add new columns
495 for propname, prop in new_spec[1]:
496 if old_has(propname):
497 continue
498 prop = spec.properties[propname]
499 if isinstance(prop, Multilink):
500 self.create_multilink_table(spec, propname)
501 else:
502 # add the column
503 coltype = self.hyperdb_to_sql_datatype(prop.__class__)
504 sql = 'alter table _%s add column _%s %s'%(
505 spec.classname, propname, coltype)
506 self.sql(sql)
508 # extra Interval column
509 if isinstance(prop, Interval):
510 sql = 'alter table _%s add column __%s_int__ BIGINT'%(
511 spec.classname, propname)
512 self.sql(sql)
514 # if the new column is a key prop, we need an index!
515 if new_spec[0] == propname:
516 self.create_class_table_key_index(spec.classname, propname)
517 del keyprop_changes['add']
519 # if we didn't add the key prop just then, but the key prop has
520 # changed, we still need to add the new index
521 if keyprop_changes.has_key('add'):
522 self.create_class_table_key_index(spec.classname,
523 keyprop_changes['add'])
525 return 1
527 def determine_all_columns(self, spec):
528 """Figure out the columns from the spec and also add internal columns
530 """
531 cols, mls = self.determine_columns(spec.properties.items())
533 # add on our special columns
534 cols.append(('id', 'INTEGER PRIMARY KEY'))
535 cols.append(('__retired__', 'INTEGER DEFAULT 0'))
536 return cols, mls
538 def create_class_table(self, spec):
539 """Create the class table for the given Class "spec". Creates the
540 indexes too."""
541 cols, mls = self.determine_all_columns(spec)
543 # create the base table
544 scols = ','.join(['%s %s'%x for x in cols])
545 sql = 'create table _%s (%s)'%(spec.classname, scols)
546 self.sql(sql)
548 self.create_class_table_indexes(spec)
550 return cols, mls
552 def create_class_table_indexes(self, spec):
553 """ create the class table for the given spec
554 """
555 # create __retired__ index
556 index_sql2 = 'create index _%s_retired_idx on _%s(__retired__)'%(
557 spec.classname, spec.classname)
558 self.sql(index_sql2)
560 # create index for key property
561 if spec.key:
562 index_sql3 = 'create index _%s_%s_idx on _%s(_%s)'%(
563 spec.classname, spec.key,
564 spec.classname, spec.key)
565 self.sql(index_sql3)
567 # and the unique index for key / retired(id)
568 self.add_class_key_required_unique_constraint(spec.classname,
569 spec.key)
571 # TODO: create indexes on (selected?) Link property columns, as
572 # they're more likely to be used for lookup
574 def add_class_key_required_unique_constraint(self, cn, key):
575 sql = '''create unique index _%s_key_retired_idx
576 on _%s(__retired__, _%s)'''%(cn, cn, key)
577 self.sql(sql)
579 def drop_class_table_indexes(self, cn, key):
580 # drop the old table indexes first
581 l = ['_%s_id_idx'%cn, '_%s_retired_idx'%cn]
582 if key:
583 l.append('_%s_%s_idx'%(cn, key))
585 table_name = '_%s'%cn
586 for index_name in l:
587 if not self.sql_index_exists(table_name, index_name):
588 continue
589 index_sql = 'drop index '+index_name
590 self.sql(index_sql)
592 def create_class_table_key_index(self, cn, key):
593 """ create the class table for the given spec
594 """
595 sql = 'create index _%s_%s_idx on _%s(_%s)'%(cn, key, cn, key)
596 self.sql(sql)
598 def drop_class_table_key_index(self, cn, key):
599 table_name = '_%s'%cn
600 index_name = '_%s_%s_idx'%(cn, key)
601 if self.sql_index_exists(table_name, index_name):
602 sql = 'drop index '+index_name
603 self.sql(sql)
605 # and now the retired unique index too
606 index_name = '_%s_key_retired_idx'%cn
607 if self.sql_index_exists(table_name, index_name):
608 sql = 'drop index '+index_name
609 self.sql(sql)
611 def create_journal_table(self, spec):
612 """ create the journal table for a class given the spec and
613 already-determined cols
614 """
615 # journal table
616 cols = ','.join(['%s varchar'%x
617 for x in 'nodeid date tag action params'.split()])
618 sql = """create table %s__journal (
619 nodeid integer, date %s, tag varchar(255),
620 action varchar(255), params text)""" % (spec.classname,
621 self.hyperdb_to_sql_datatype(hyperdb.Date))
622 self.sql(sql)
623 self.create_journal_table_indexes(spec)
625 def create_journal_table_indexes(self, spec):
626 # index on nodeid
627 sql = 'create index %s_journ_idx on %s__journal(nodeid)'%(
628 spec.classname, spec.classname)
629 self.sql(sql)
631 def drop_journal_table_indexes(self, classname):
632 index_name = '%s_journ_idx'%classname
633 if not self.sql_index_exists('%s__journal'%classname, index_name):
634 return
635 index_sql = 'drop index '+index_name
636 self.sql(index_sql)
638 def create_multilink_table(self, spec, ml):
639 """ Create a multilink table for the "ml" property of the class
640 given by the spec
641 """
642 # create the table
643 sql = 'create table %s_%s (linkid INTEGER, nodeid INTEGER)'%(
644 spec.classname, ml)
645 self.sql(sql)
646 self.create_multilink_table_indexes(spec, ml)
648 def create_multilink_table_indexes(self, spec, ml):
649 # create index on linkid
650 index_sql = 'create index %s_%s_l_idx on %s_%s(linkid)'%(
651 spec.classname, ml, spec.classname, ml)
652 self.sql(index_sql)
654 # create index on nodeid
655 index_sql = 'create index %s_%s_n_idx on %s_%s(nodeid)'%(
656 spec.classname, ml, spec.classname, ml)
657 self.sql(index_sql)
659 def drop_multilink_table_indexes(self, classname, ml):
660 l = [
661 '%s_%s_l_idx'%(classname, ml),
662 '%s_%s_n_idx'%(classname, ml)
663 ]
664 table_name = '%s_%s'%(classname, ml)
665 for index_name in l:
666 if not self.sql_index_exists(table_name, index_name):
667 continue
668 index_sql = 'drop index %s'%index_name
669 self.sql(index_sql)
671 def create_class(self, spec):
672 """ Create a database table according to the given spec.
673 """
674 cols, mls = self.create_class_table(spec)
675 self.create_journal_table(spec)
677 # now create the multilink tables
678 for ml in mls:
679 self.create_multilink_table(spec, ml)
681 def drop_class(self, cn, spec):
682 """ Drop the given table from the database.
684 Drop the journal and multilink tables too.
685 """
686 properties = spec[1]
687 # figure the multilinks
688 mls = []
689 for propname, prop in properties:
690 if isinstance(prop, Multilink):
691 mls.append(propname)
693 # drop class table and indexes
694 self.drop_class_table_indexes(cn, spec[0])
696 self.drop_class_table(cn)
698 # drop journal table and indexes
699 self.drop_journal_table_indexes(cn)
700 sql = 'drop table %s__journal'%cn
701 self.sql(sql)
703 for ml in mls:
704 # drop multilink table and indexes
705 self.drop_multilink_table_indexes(cn, ml)
706 sql = 'drop table %s_%s'%(spec.classname, ml)
707 self.sql(sql)
709 def drop_class_table(self, cn):
710 sql = 'drop table _%s'%cn
711 self.sql(sql)
713 #
714 # Classes
715 #
716 def __getattr__(self, classname):
717 """ A convenient way of calling self.getclass(classname).
718 """
719 if self.classes.has_key(classname):
720 return self.classes[classname]
721 raise AttributeError, classname
723 def addclass(self, cl):
724 """ Add a Class to the hyperdatabase.
725 """
726 cn = cl.classname
727 if self.classes.has_key(cn):
728 raise ValueError, cn
729 self.classes[cn] = cl
731 # add default Edit and View permissions
732 self.security.addPermission(name="Create", klass=cn,
733 description="User is allowed to create "+cn)
734 self.security.addPermission(name="Edit", klass=cn,
735 description="User is allowed to edit "+cn)
736 self.security.addPermission(name="View", klass=cn,
737 description="User is allowed to access "+cn)
739 def getclasses(self):
740 """ Return a list of the names of all existing classes.
741 """
742 l = self.classes.keys()
743 l.sort()
744 return l
746 def getclass(self, classname):
747 """Get the Class object representing a particular class.
749 If 'classname' is not a valid class name, a KeyError is raised.
750 """
751 try:
752 return self.classes[classname]
753 except KeyError:
754 raise KeyError, 'There is no class called "%s"'%classname
756 def clear(self):
757 """Delete all database contents.
759 Note: I don't commit here, which is different behaviour to the
760 "nuke from orbit" behaviour in the dbs.
761 """
762 logging.getLogger('hyperdb').info('clear')
763 for cn in self.classes.keys():
764 sql = 'delete from _%s'%cn
765 self.sql(sql)
767 #
768 # Nodes
769 #
771 hyperdb_to_sql_value = {
772 hyperdb.String : str,
773 # fractional seconds by default
774 hyperdb.Date : lambda x: x.formal(sep=' ', sec='%06.3f'),
775 hyperdb.Link : int,
776 hyperdb.Interval : str,
777 hyperdb.Password : str,
778 hyperdb.Boolean : lambda x: x and 'TRUE' or 'FALSE',
779 hyperdb.Number : lambda x: x,
780 hyperdb.Multilink : lambda x: x, # used in journal marshalling
781 }
783 def to_sql_value(self, propklass):
785 fn = self.hyperdb_to_sql_value.get(propklass)
786 if fn:
787 return fn
789 for k, v in self.hyperdb_to_sql_value.iteritems():
790 if issubclass(propklass, k):
791 return v
793 raise ValueError, '%r is not a hyperdb property class' % propklass
795 def addnode(self, classname, nodeid, node):
796 """ Add the specified node to its class's db.
797 """
798 self.log_debug('addnode %s%s %r'%(classname,
799 nodeid, node))
801 # determine the column definitions and multilink tables
802 cl = self.classes[classname]
803 cols, mls = self.determine_columns(cl.properties.items())
805 # we'll be supplied these props if we're doing an import
806 values = node.copy()
807 if not values.has_key('creator'):
808 # add in the "calculated" properties (dupe so we don't affect
809 # calling code's node assumptions)
810 values['creation'] = values['activity'] = date.Date()
811 values['actor'] = values['creator'] = self.getuid()
813 cl = self.classes[classname]
814 props = cl.getprops(protected=1)
815 del props['id']
817 # default the non-multilink columns
818 for col, prop in props.items():
819 if not values.has_key(col):
820 if isinstance(prop, Multilink):
821 values[col] = []
822 else:
823 values[col] = None
825 # clear this node out of the cache if it's in there
826 key = (classname, nodeid)
827 if self.cache.has_key(key):
828 del self.cache[key]
829 self.cache_lru.remove(key)
831 # figure the values to insert
832 vals = []
833 for col,dt in cols:
834 # this is somewhat dodgy....
835 if col.endswith('_int__'):
836 # XXX eugh, this test suxxors
837 value = values[col[2:-6]]
838 # this is an Interval special "int" column
839 if value is not None:
840 vals.append(value.as_seconds())
841 else:
842 vals.append(value)
843 continue
845 prop = props[col[1:]]
846 value = values[col[1:]]
847 if value is not None:
848 value = self.to_sql_value(prop.__class__)(value)
849 vals.append(value)
850 vals.append(nodeid)
851 vals = tuple(vals)
853 # make sure the ordering is correct for column name -> column value
854 s = ','.join([self.arg for x in cols]) + ',%s'%self.arg
855 cols = ','.join([col for col,dt in cols]) + ',id'
857 # perform the inserts
858 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
859 self.sql(sql, vals)
861 # insert the multilink rows
862 for col in mls:
863 t = '%s_%s'%(classname, col)
864 for entry in node[col]:
865 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
866 self.arg, self.arg)
867 self.sql(sql, (entry, nodeid))
869 def setnode(self, classname, nodeid, values, multilink_changes={}):
870 """ Change the specified node.
871 """
872 self.log_debug('setnode %s%s %r'
873 % (classname, nodeid, values))
875 # clear this node out of the cache if it's in there
876 key = (classname, nodeid)
877 if self.cache.has_key(key):
878 del self.cache[key]
879 self.cache_lru.remove(key)
881 cl = self.classes[classname]
882 props = cl.getprops()
884 cols = []
885 mls = []
886 # add the multilinks separately
887 for col in values.keys():
888 prop = props[col]
889 if isinstance(prop, Multilink):
890 mls.append(col)
891 elif isinstance(prop, Interval):
892 # Intervals store the seconds value too
893 cols.append(col)
894 # extra leading '_' added by code below
895 cols.append('_' +col + '_int__')
896 else:
897 cols.append(col)
898 cols.sort()
900 # figure the values to insert
901 vals = []
902 for col in cols:
903 if col.endswith('_int__'):
904 # XXX eugh, this test suxxors
905 # Intervals store the seconds value too
906 col = col[1:-6]
907 prop = props[col]
908 value = values[col]
909 if value is None:
910 vals.append(None)
911 else:
912 vals.append(value.as_seconds())
913 else:
914 prop = props[col]
915 value = values[col]
916 if value is None:
917 e = None
918 else:
919 e = self.to_sql_value(prop.__class__)(value)
920 vals.append(e)
922 vals.append(int(nodeid))
923 vals = tuple(vals)
925 # if there's any updates to regular columns, do them
926 if cols:
927 # make sure the ordering is correct for column name -> column value
928 s = ','.join(['_%s=%s'%(x, self.arg) for x in cols])
929 cols = ','.join(cols)
931 # perform the update
932 sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
933 self.sql(sql, vals)
935 # we're probably coming from an import, not a change
936 if not multilink_changes:
937 for name in mls:
938 prop = props[name]
939 value = values[name]
941 t = '%s_%s'%(classname, name)
943 # clear out previous values for this node
944 # XXX numeric ids
945 self.sql('delete from %s where nodeid=%s'%(t, self.arg),
946 (nodeid,))
948 # insert the values for this node
949 for entry in values[name]:
950 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
951 self.arg, self.arg)
952 # XXX numeric ids
953 self.sql(sql, (entry, nodeid))
955 # we have multilink changes to apply
956 for col, (add, remove) in multilink_changes.items():
957 tn = '%s_%s'%(classname, col)
958 if add:
959 sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
960 self.arg, self.arg)
961 for addid in add:
962 # XXX numeric ids
963 self.sql(sql, (int(nodeid), int(addid)))
964 if remove:
965 s = ','.join([self.arg]*len(remove))
966 sql = 'delete from %s where nodeid=%s and linkid in (%s)'%(tn,
967 self.arg, s)
968 # XXX numeric ids
969 self.sql(sql, [int(nodeid)] + remove)
971 sql_to_hyperdb_value = {
972 hyperdb.String : str,
973 hyperdb.Date : lambda x:date.Date(str(x).replace(' ', '.')),
974 # hyperdb.Link : int, # XXX numeric ids
975 hyperdb.Link : str,
976 hyperdb.Interval : date.Interval,
977 hyperdb.Password : lambda x: password.Password(encrypted=x),
978 hyperdb.Boolean : _bool_cvt,
979 hyperdb.Number : _num_cvt,
980 hyperdb.Multilink : lambda x: x, # used in journal marshalling
981 }
983 def to_hyperdb_value(self, propklass):
985 fn = self.sql_to_hyperdb_value.get(propklass)
986 if fn:
987 return fn
989 for k, v in self.sql_to_hyperdb_value.iteritems():
990 if issubclass(propklass, k):
991 return v
993 raise ValueError, '%r is not a hyperdb property class' % propklass
995 def getnode(self, classname, nodeid):
996 """ Get a node from the database.
997 """
998 # see if we have this node cached
999 key = (classname, nodeid)
1000 if self.cache.has_key(key):
1001 # push us back to the top of the LRU
1002 self.cache_lru.remove(key)
1003 self.cache_lru.insert(0, key)
1004 if __debug__:
1005 self.stats['cache_hits'] += 1
1006 # return the cached information
1007 return self.cache[key]
1009 if __debug__:
1010 self.stats['cache_misses'] += 1
1011 start_t = time.time()
1013 # figure the columns we're fetching
1014 cl = self.classes[classname]
1015 cols, mls = self.determine_columns(cl.properties.items())
1016 scols = ','.join([col for col,dt in cols])
1018 # perform the basic property fetch
1019 sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
1020 self.sql(sql, (nodeid,))
1022 values = self.sql_fetchone()
1023 if values is None:
1024 raise IndexError, 'no such %s node %s'%(classname, nodeid)
1026 # make up the node
1027 node = {}
1028 props = cl.getprops(protected=1)
1029 for col in range(len(cols)):
1030 name = cols[col][0][1:]
1031 if name.endswith('_int__'):
1032 # XXX eugh, this test suxxors
1033 # ignore the special Interval-as-seconds column
1034 continue
1035 value = values[col]
1036 if value is not None:
1037 value = self.to_hyperdb_value(props[name].__class__)(value)
1038 node[name] = value
1041 # now the multilinks
1042 for col in mls:
1043 # get the link ids
1044 sql = 'select linkid from %s_%s where nodeid=%s'%(classname, col,
1045 self.arg)
1046 self.sql(sql, (nodeid,))
1047 # extract the first column from the result
1048 # XXX numeric ids
1049 items = [int(x[0]) for x in self.cursor.fetchall()]
1050 items.sort ()
1051 node[col] = [str(x) for x in items]
1053 # save off in the cache
1054 key = (classname, nodeid)
1055 self.cache[key] = node
1056 # update the LRU
1057 self.cache_lru.insert(0, key)
1058 if len(self.cache_lru) > self.cache_size:
1059 del self.cache[self.cache_lru.pop()]
1061 if __debug__:
1062 self.stats['get_items'] += (time.time() - start_t)
1064 return node
1066 def destroynode(self, classname, nodeid):
1067 """Remove a node from the database. Called exclusively by the
1068 destroy() method on Class.
1069 """
1070 logging.getLogger('hyperdb').info('destroynode %s%s'%(classname, nodeid))
1072 # make sure the node exists
1073 if not self.hasnode(classname, nodeid):
1074 raise IndexError, '%s has no node %s'%(classname, nodeid)
1076 # see if we have this node cached
1077 if self.cache.has_key((classname, nodeid)):
1078 del self.cache[(classname, nodeid)]
1080 # see if there's any obvious commit actions that we should get rid of
1081 for entry in self.transactions[:]:
1082 if entry[1][:2] == (classname, nodeid):
1083 self.transactions.remove(entry)
1085 # now do the SQL
1086 sql = 'delete from _%s where id=%s'%(classname, self.arg)
1087 self.sql(sql, (nodeid,))
1089 # remove from multilnks
1090 cl = self.getclass(classname)
1091 x, mls = self.determine_columns(cl.properties.items())
1092 for col in mls:
1093 # get the link ids
1094 sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
1095 self.sql(sql, (nodeid,))
1097 # remove journal entries
1098 sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
1099 self.sql(sql, (nodeid,))
1101 # cleanup any blob filestorage when we commit
1102 self.transactions.append((FileStorage.destroy, (self, classname, nodeid)))
1104 def hasnode(self, classname, nodeid):
1105 """ Determine if the database has a given node.
1106 """
1107 # If this node is in the cache, then we do not need to go to
1108 # the database. (We don't consider this an LRU hit, though.)
1109 if self.cache.has_key((classname, nodeid)):
1110 # Return 1, not True, to match the type of the result of
1111 # the SQL operation below.
1112 return 1
1113 sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
1114 self.sql(sql, (nodeid,))
1115 return int(self.cursor.fetchone()[0])
1117 def countnodes(self, classname):
1118 """ Count the number of nodes that exist for a particular Class.
1119 """
1120 sql = 'select count(*) from _%s'%classname
1121 self.sql(sql)
1122 return self.cursor.fetchone()[0]
1124 def addjournal(self, classname, nodeid, action, params, creator=None,
1125 creation=None):
1126 """ Journal the Action
1127 'action' may be:
1129 'create' or 'set' -- 'params' is a dictionary of property values
1130 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
1131 'retire' -- 'params' is None
1132 """
1133 # handle supply of the special journalling parameters (usually
1134 # supplied on importing an existing database)
1135 if creator:
1136 journaltag = creator
1137 else:
1138 journaltag = self.getuid()
1139 if creation:
1140 journaldate = creation
1141 else:
1142 journaldate = date.Date()
1144 # create the journal entry
1145 cols = 'nodeid,date,tag,action,params'
1147 self.log_debug('addjournal %s%s %r %s %s %r'%(classname,
1148 nodeid, journaldate, journaltag, action, params))
1150 # make the journalled data marshallable
1151 if isinstance(params, type({})):
1152 self._journal_marshal(params, classname)
1154 params = repr(params)
1156 dc = self.to_sql_value(hyperdb.Date)
1157 journaldate = dc(journaldate)
1159 self.save_journal(classname, cols, nodeid, journaldate,
1160 journaltag, action, params)
1162 def setjournal(self, classname, nodeid, journal):
1163 """Set the journal to the "journal" list."""
1164 # clear out any existing entries
1165 self.sql('delete from %s__journal where nodeid=%s'%(classname,
1166 self.arg), (nodeid,))
1168 # create the journal entry
1169 cols = 'nodeid,date,tag,action,params'
1171 dc = self.to_sql_value(hyperdb.Date)
1172 for nodeid, journaldate, journaltag, action, params in journal:
1173 self.log_debug('addjournal %s%s %r %s %s %r'%(
1174 classname, nodeid, journaldate, journaltag, action,
1175 params))
1177 # make the journalled data marshallable
1178 if isinstance(params, type({})):
1179 self._journal_marshal(params, classname)
1180 params = repr(params)
1182 self.save_journal(classname, cols, nodeid, dc(journaldate),
1183 journaltag, action, params)
1185 def _journal_marshal(self, params, classname):
1186 """Convert the journal params values into safely repr'able and
1187 eval'able values."""
1188 properties = self.getclass(classname).getprops()
1189 for param, value in params.items():
1190 if not value:
1191 continue
1192 property = properties[param]
1193 cvt = self.to_sql_value(property.__class__)
1194 if isinstance(property, Password):
1195 params[param] = cvt(value)
1196 elif isinstance(property, Date):
1197 params[param] = cvt(value)
1198 elif isinstance(property, Interval):
1199 params[param] = cvt(value)
1200 elif isinstance(property, Boolean):
1201 params[param] = cvt(value)
1203 def getjournal(self, classname, nodeid):
1204 """ get the journal for id
1205 """
1206 # make sure the node exists
1207 if not self.hasnode(classname, nodeid):
1208 raise IndexError, '%s has no node %s'%(classname, nodeid)
1210 cols = ','.join('nodeid date tag action params'.split())
1211 journal = self.load_journal(classname, cols, nodeid)
1213 # now unmarshal the data
1214 dc = self.to_hyperdb_value(hyperdb.Date)
1215 res = []
1216 properties = self.getclass(classname).getprops()
1217 for nodeid, date_stamp, user, action, params in journal:
1218 params = eval(params)
1219 if isinstance(params, type({})):
1220 for param, value in params.items():
1221 if not value:
1222 continue
1223 property = properties.get(param, None)
1224 if property is None:
1225 # deleted property
1226 continue
1227 cvt = self.to_hyperdb_value(property.__class__)
1228 if isinstance(property, Password):
1229 params[param] = cvt(value)
1230 elif isinstance(property, Date):
1231 params[param] = cvt(value)
1232 elif isinstance(property, Interval):
1233 params[param] = cvt(value)
1234 elif isinstance(property, Boolean):
1235 params[param] = cvt(value)
1236 # XXX numeric ids
1237 res.append((str(nodeid), dc(date_stamp), user, action, params))
1238 return res
1240 def save_journal(self, classname, cols, nodeid, journaldate,
1241 journaltag, action, params):
1242 """ Save the journal entry to the database
1243 """
1244 entry = (nodeid, journaldate, journaltag, action, params)
1246 # do the insert
1247 a = self.arg
1248 sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(
1249 classname, cols, a, a, a, a, a)
1250 self.sql(sql, entry)
1252 def load_journal(self, classname, cols, nodeid):
1253 """ Load the journal from the database
1254 """
1255 # now get the journal entries
1256 sql = 'select %s from %s__journal where nodeid=%s order by date'%(
1257 cols, classname, self.arg)
1258 self.sql(sql, (nodeid,))
1259 return self.cursor.fetchall()
1261 def pack(self, pack_before):
1262 """ Delete all journal entries except "create" before 'pack_before'.
1263 """
1264 date_stamp = self.to_sql_value(Date)(pack_before)
1266 # do the delete
1267 for classname in self.classes.keys():
1268 sql = "delete from %s__journal where date<%s and "\
1269 "action<>'create'"%(classname, self.arg)
1270 self.sql(sql, (date_stamp,))
1272 def sql_commit(self, fail_ok=False):
1273 """ Actually commit to the database.
1274 """
1275 logging.getLogger('hyperdb').info('commit')
1277 self.conn.commit()
1279 # open a new cursor for subsequent work
1280 self.cursor = self.conn.cursor()
1282 def commit(self, fail_ok=False):
1283 """ Commit the current transactions.
1285 Save all data changed since the database was opened or since the
1286 last commit() or rollback().
1288 fail_ok indicates that the commit is allowed to fail. This is used
1289 in the web interface when committing cleaning of the session
1290 database. We don't care if there's a concurrency issue there.
1292 The only backend this seems to affect is postgres.
1293 """
1294 # commit the database
1295 self.sql_commit(fail_ok)
1297 # now, do all the other transaction stuff
1298 for method, args in self.transactions:
1299 method(*args)
1301 # save the indexer
1302 self.indexer.save_index()
1304 # clear out the transactions
1305 self.transactions = []
1307 def sql_rollback(self):
1308 self.conn.rollback()
1310 def rollback(self):
1311 """ Reverse all actions from the current transaction.
1313 Undo all the changes made since the database was opened or the last
1314 commit() or rollback() was performed.
1315 """
1316 logging.getLogger('hyperdb').info('rollback')
1318 self.sql_rollback()
1320 # roll back "other" transaction stuff
1321 for method, args in self.transactions:
1322 # delete temporary files
1323 if method == self.doStoreFile:
1324 self.rollbackStoreFile(*args)
1325 self.transactions = []
1327 # clear the cache
1328 self.clearCache()
1330 def sql_close(self):
1331 logging.getLogger('hyperdb').info('close')
1332 self.conn.close()
1334 def close(self):
1335 """ Close off the connection.
1336 """
1337 self.indexer.close()
1338 self.sql_close()
1340 #
1341 # The base Class class
1342 #
1343 class Class(hyperdb.Class):
1344 """ The handle to a particular class of nodes in a hyperdatabase.
1346 All methods except __repr__ and getnode must be implemented by a
1347 concrete backend Class.
1348 """
1350 def schema(self):
1351 """ A dumpable version of the schema that we can store in the
1352 database
1353 """
1354 return (self.key, [(x, repr(y)) for x,y in self.properties.items()])
1356 def enableJournalling(self):
1357 """Turn journalling on for this class
1358 """
1359 self.do_journal = 1
1361 def disableJournalling(self):
1362 """Turn journalling off for this class
1363 """
1364 self.do_journal = 0
1366 # Editing nodes:
1367 def create(self, **propvalues):
1368 """ Create a new node of this class and return its id.
1370 The keyword arguments in 'propvalues' map property names to values.
1372 The values of arguments must be acceptable for the types of their
1373 corresponding properties or a TypeError is raised.
1375 If this class has a key property, it must be present and its value
1376 must not collide with other key strings or a ValueError is raised.
1378 Any other properties on this class that are missing from the
1379 'propvalues' dictionary are set to None.
1381 If an id in a link or multilink property does not refer to a valid
1382 node, an IndexError is raised.
1383 """
1384 self.fireAuditors('create', None, propvalues)
1385 newid = self.create_inner(**propvalues)
1386 self.fireReactors('create', newid, None)
1387 return newid
1389 def create_inner(self, **propvalues):
1390 """ Called by create, in-between the audit and react calls.
1391 """
1392 if propvalues.has_key('id'):
1393 raise KeyError, '"id" is reserved'
1395 if self.db.journaltag is None:
1396 raise DatabaseError, _('Database open read-only')
1398 if propvalues.has_key('creator') or propvalues.has_key('actor') or \
1399 propvalues.has_key('creation') or propvalues.has_key('activity'):
1400 raise KeyError, '"creator", "actor", "creation" and '\
1401 '"activity" are reserved'
1403 # new node's id
1404 newid = self.db.newid(self.classname)
1406 # validate propvalues
1407 num_re = re.compile('^\d+$')
1408 for key, value in propvalues.items():
1409 if key == self.key:
1410 try:
1411 self.lookup(value)
1412 except KeyError:
1413 pass
1414 else:
1415 raise ValueError, 'node with key "%s" exists'%value
1417 # try to handle this property
1418 try:
1419 prop = self.properties[key]
1420 except KeyError:
1421 raise KeyError, '"%s" has no property "%s"'%(self.classname,
1422 key)
1424 if value is not None and isinstance(prop, Link):
1425 if type(value) != type(''):
1426 raise ValueError, 'link value must be String'
1427 link_class = self.properties[key].classname
1428 # if it isn't a number, it's a key
1429 if not num_re.match(value):
1430 try:
1431 value = self.db.classes[link_class].lookup(value)
1432 except (TypeError, KeyError):
1433 raise IndexError, 'new property "%s": %s not a %s'%(
1434 key, value, link_class)
1435 elif not self.db.getclass(link_class).hasnode(value):
1436 raise IndexError, '%s has no node %s'%(link_class, value)
1438 # save off the value
1439 propvalues[key] = value
1441 # register the link with the newly linked node
1442 if self.do_journal and self.properties[key].do_journal:
1443 self.db.addjournal(link_class, value, 'link',
1444 (self.classname, newid, key))
1446 elif isinstance(prop, Multilink):
1447 if value is None:
1448 value = []
1449 if not hasattr(value, '__iter__'):
1450 raise TypeError, 'new property "%s" not an iterable of ids'%key
1452 # clean up and validate the list of links
1453 link_class = self.properties[key].classname
1454 l = []
1455 for entry in value:
1456 if type(entry) != type(''):
1457 raise ValueError, '"%s" multilink value (%r) '\
1458 'must contain Strings'%(key, value)
1459 # if it isn't a number, it's a key
1460 if not num_re.match(entry):
1461 try:
1462 entry = self.db.classes[link_class].lookup(entry)
1463 except (TypeError, KeyError):
1464 raise IndexError, 'new property "%s": %s not a %s'%(
1465 key, entry, self.properties[key].classname)
1466 l.append(entry)
1467 value = l
1468 propvalues[key] = value
1470 # handle additions
1471 for nodeid in value:
1472 if not self.db.getclass(link_class).hasnode(nodeid):
1473 raise IndexError, '%s has no node %s'%(link_class,
1474 nodeid)
1475 # register the link with the newly linked node
1476 if self.do_journal and self.properties[key].do_journal:
1477 self.db.addjournal(link_class, nodeid, 'link',
1478 (self.classname, newid, key))
1480 elif isinstance(prop, String):
1481 if type(value) != type('') and type(value) != type(u''):
1482 raise TypeError, 'new property "%s" not a string'%key
1483 if prop.indexme:
1484 self.db.indexer.add_text((self.classname, newid, key),
1485 value)
1487 elif isinstance(prop, Password):
1488 if not isinstance(value, password.Password):
1489 raise TypeError, 'new property "%s" not a Password'%key
1491 elif isinstance(prop, Date):
1492 if value is not None and not isinstance(value, date.Date):
1493 raise TypeError, 'new property "%s" not a Date'%key
1495 elif isinstance(prop, Interval):
1496 if value is not None and not isinstance(value, date.Interval):
1497 raise TypeError, 'new property "%s" not an Interval'%key
1499 elif value is not None and isinstance(prop, Number):
1500 try:
1501 float(value)
1502 except ValueError:
1503 raise TypeError, 'new property "%s" not numeric'%key
1505 elif value is not None and isinstance(prop, Boolean):
1506 try:
1507 int(value)
1508 except ValueError:
1509 raise TypeError, 'new property "%s" not boolean'%key
1511 # make sure there's data where there needs to be
1512 for key, prop in self.properties.items():
1513 if propvalues.has_key(key):
1514 continue
1515 if key == self.key:
1516 raise ValueError, 'key property "%s" is required'%key
1517 if isinstance(prop, Multilink):
1518 propvalues[key] = []
1519 else:
1520 propvalues[key] = None
1522 # done
1523 self.db.addnode(self.classname, newid, propvalues)
1524 if self.do_journal:
1525 self.db.addjournal(self.classname, newid, ''"create", {})
1527 # XXX numeric ids
1528 return str(newid)
1530 def get(self, nodeid, propname, default=_marker, cache=1):
1531 """Get the value of a property on an existing node of this class.
1533 'nodeid' must be the id of an existing node of this class or an
1534 IndexError is raised. 'propname' must be the name of a property
1535 of this class or a KeyError is raised.
1537 'cache' exists for backwards compatibility, and is not used.
1538 """
1539 if propname == 'id':
1540 return nodeid
1542 # get the node's dict
1543 d = self.db.getnode(self.classname, nodeid)
1545 if propname == 'creation':
1546 if d.has_key('creation'):
1547 return d['creation']
1548 else:
1549 return date.Date()
1550 if propname == 'activity':
1551 if d.has_key('activity'):
1552 return d['activity']
1553 else:
1554 return date.Date()
1555 if propname == 'creator':
1556 if d.has_key('creator'):
1557 return d['creator']
1558 else:
1559 return self.db.getuid()
1560 if propname == 'actor':
1561 if d.has_key('actor'):
1562 return d['actor']
1563 else:
1564 return self.db.getuid()
1566 # get the property (raises KeyErorr if invalid)
1567 prop = self.properties[propname]
1569 # XXX may it be that propname is valid property name
1570 # (above error is not raised) and not d.has_key(propname)???
1571 if (not d.has_key(propname)) or (d[propname] is None):
1572 if default is _marker:
1573 if isinstance(prop, Multilink):
1574 return []
1575 else:
1576 return None
1577 else:
1578 return default
1580 # don't pass our list to other code
1581 if isinstance(prop, Multilink):
1582 return d[propname][:]
1584 return d[propname]
1586 def set(self, nodeid, **propvalues):
1587 """Modify a property on an existing node of this class.
1589 'nodeid' must be the id of an existing node of this class or an
1590 IndexError is raised.
1592 Each key in 'propvalues' must be the name of a property of this
1593 class or a KeyError is raised.
1595 All values in 'propvalues' must be acceptable types for their
1596 corresponding properties or a TypeError is raised.
1598 If the value of the key property is set, it must not collide with
1599 other key strings or a ValueError is raised.
1601 If the value of a Link or Multilink property contains an invalid
1602 node id, a ValueError is raised.
1603 """
1604 self.fireAuditors('set', nodeid, propvalues)
1605 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1606 propvalues = self.set_inner(nodeid, **propvalues)
1607 self.fireReactors('set', nodeid, oldvalues)
1608 return propvalues
1610 def set_inner(self, nodeid, **propvalues):
1611 """ Called by set, in-between the audit and react calls.
1612 """
1613 if not propvalues:
1614 return propvalues
1616 if propvalues.has_key('creation') or propvalues.has_key('creator') or \
1617 propvalues.has_key('actor') or propvalues.has_key('activity'):
1618 raise KeyError, '"creation", "creator", "actor" and '\
1619 '"activity" are reserved'
1621 if propvalues.has_key('id'):
1622 raise KeyError, '"id" is reserved'
1624 if self.db.journaltag is None:
1625 raise DatabaseError, _('Database open read-only')
1627 node = self.db.getnode(self.classname, nodeid)
1628 if self.is_retired(nodeid):
1629 raise IndexError, 'Requested item is retired'
1630 num_re = re.compile('^\d+$')
1632 # make a copy of the values dictionary - we'll modify the contents
1633 propvalues = propvalues.copy()
1635 # if the journal value is to be different, store it in here
1636 journalvalues = {}
1638 # remember the add/remove stuff for multilinks, making it easier
1639 # for the Database layer to do its stuff
1640 multilink_changes = {}
1642 for propname, value in propvalues.items():
1643 # check to make sure we're not duplicating an existing key
1644 if propname == self.key and node[propname] != value:
1645 try:
1646 self.lookup(value)
1647 except KeyError:
1648 pass
1649 else:
1650 raise ValueError, 'node with key "%s" exists'%value
1652 # this will raise the KeyError if the property isn't valid
1653 # ... we don't use getprops() here because we only care about
1654 # the writeable properties.
1655 try:
1656 prop = self.properties[propname]
1657 except KeyError:
1658 raise KeyError, '"%s" has no property named "%s"'%(
1659 self.classname, propname)
1661 # if the value's the same as the existing value, no sense in
1662 # doing anything
1663 current = node.get(propname, None)
1664 if value == current:
1665 del propvalues[propname]
1666 continue
1667 journalvalues[propname] = current
1669 # do stuff based on the prop type
1670 if isinstance(prop, Link):
1671 link_class = prop.classname
1672 # if it isn't a number, it's a key
1673 if value is not None and not isinstance(value, type('')):
1674 raise ValueError, 'property "%s" link value be a string'%(
1675 propname)
1676 if isinstance(value, type('')) and not num_re.match(value):
1677 try:
1678 value = self.db.classes[link_class].lookup(value)
1679 except (TypeError, KeyError):
1680 raise IndexError, 'new property "%s": %s not a %s'%(
1681 propname, value, prop.classname)
1683 if (value is not None and
1684 not self.db.getclass(link_class).hasnode(value)):
1685 raise IndexError, '%s has no node %s'%(link_class, value)
1687 if self.do_journal and prop.do_journal:
1688 # register the unlink with the old linked node
1689 if node[propname] is not None:
1690 self.db.addjournal(link_class, node[propname],
1691 ''"unlink", (self.classname, nodeid, propname))
1693 # register the link with the newly linked node
1694 if value is not None:
1695 self.db.addjournal(link_class, value, ''"link",
1696 (self.classname, nodeid, propname))
1698 elif isinstance(prop, Multilink):
1699 if value is None:
1700 value = []
1701 if not hasattr(value, '__iter__'):
1702 raise TypeError, 'new property "%s" not an iterable of'\
1703 ' ids'%propname
1704 link_class = self.properties[propname].classname
1705 l = []
1706 for entry in value:
1707 # if it isn't a number, it's a key
1708 if type(entry) != type(''):
1709 raise ValueError, 'new property "%s" link value ' \
1710 'must be a string'%propname
1711 if not num_re.match(entry):
1712 try:
1713 entry = self.db.classes[link_class].lookup(entry)
1714 except (TypeError, KeyError):
1715 raise IndexError, 'new property "%s": %s not a %s'%(
1716 propname, entry,
1717 self.properties[propname].classname)
1718 l.append(entry)
1719 value = l
1720 propvalues[propname] = value
1722 # figure the journal entry for this property
1723 add = []
1724 remove = []
1726 # handle removals
1727 if node.has_key(propname):
1728 l = node[propname]
1729 else:
1730 l = []
1731 for id in l[:]:
1732 if id in value:
1733 continue
1734 # register the unlink with the old linked node
1735 if self.do_journal and self.properties[propname].do_journal:
1736 self.db.addjournal(link_class, id, 'unlink',
1737 (self.classname, nodeid, propname))
1738 l.remove(id)
1739 remove.append(id)
1741 # handle additions
1742 for id in value:
1743 # If this node is in the cache, then we do not need to go to
1744 # the database. (We don't consider this an LRU hit, though.)
1745 if self.cache.has_key((classname, nodeid)):
1746 # Return 1, not True, to match the type of the result of
1747 # the SQL operation below.
1748 return 1
1749 if not self.db.getclass(link_class).hasnode(id):
1750 raise IndexError, '%s has no node %s'%(link_class, id)
1751 if id in l:
1752 continue
1753 # register the link with the newly linked node
1754 if self.do_journal and self.properties[propname].do_journal:
1755 self.db.addjournal(link_class, id, 'link',
1756 (self.classname, nodeid, propname))
1757 l.append(id)
1758 add.append(id)
1760 # figure the journal entry
1761 l = []
1762 if add:
1763 l.append(('+', add))
1764 if remove:
1765 l.append(('-', remove))
1766 multilink_changes[propname] = (add, remove)
1767 if l:
1768 journalvalues[propname] = tuple(l)
1770 elif isinstance(prop, String):
1771 if value is not None and type(value) != type('') and type(value) != type(u''):
1772 raise TypeError, 'new property "%s" not a string'%propname
1773 if prop.indexme:
1774 if value is None: value = ''
1775 self.db.indexer.add_text((self.classname, nodeid, propname),
1776 value)
1778 elif isinstance(prop, Password):
1779 if not isinstance(value, password.Password):
1780 raise TypeError, 'new property "%s" not a Password'%propname
1781 propvalues[propname] = value
1783 elif value is not None and isinstance(prop, Date):
1784 if not isinstance(value, date.Date):
1785 raise TypeError, 'new property "%s" not a Date'% propname
1786 propvalues[propname] = value
1788 elif value is not None and isinstance(prop, Interval):
1789 if not isinstance(value, date.Interval):
1790 raise TypeError, 'new property "%s" not an '\
1791 'Interval'%propname
1792 propvalues[propname] = value
1794 elif value is not None and isinstance(prop, Number):
1795 try:
1796 float(value)
1797 except ValueError:
1798 raise TypeError, 'new property "%s" not numeric'%propname
1800 elif value is not None and isinstance(prop, Boolean):
1801 try:
1802 int(value)
1803 except ValueError:
1804 raise TypeError, 'new property "%s" not boolean'%propname
1806 # nothing to do?
1807 if not propvalues:
1808 return propvalues
1810 # update the activity time
1811 propvalues['activity'] = date.Date()
1812 propvalues['actor'] = self.db.getuid()
1814 # do the set
1815 self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
1817 # remove the activity props now they're handled
1818 del propvalues['activity']
1819 del propvalues['actor']
1821 # journal the set
1822 if self.do_journal:
1823 self.db.addjournal(self.classname, nodeid, ''"set", journalvalues)
1825 return propvalues
1827 def retire(self, nodeid):
1828 """Retire a node.
1830 The properties on the node remain available from the get() method,
1831 and the node's id is never reused.
1833 Retired nodes are not returned by the find(), list(), or lookup()
1834 methods, and other nodes may reuse the values of their key properties.
1835 """
1836 if self.db.journaltag is None:
1837 raise DatabaseError, _('Database open read-only')
1839 self.fireAuditors('retire', nodeid, None)
1841 # use the arg for __retired__ to cope with any odd database type
1842 # conversion (hello, sqlite)
1843 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1844 self.db.arg, self.db.arg)
1845 self.db.sql(sql, (nodeid, nodeid))
1846 if self.do_journal:
1847 self.db.addjournal(self.classname, nodeid, ''"retired", None)
1849 self.fireReactors('retire', nodeid, None)
1851 def restore(self, nodeid):
1852 """Restore a retired node.
1854 Make node available for all operations like it was before retirement.
1855 """
1856 if self.db.journaltag is None:
1857 raise DatabaseError, _('Database open read-only')
1859 node = self.db.getnode(self.classname, nodeid)
1860 # check if key property was overrided
1861 key = self.getkey()
1862 try:
1863 id = self.lookup(node[key])
1864 except KeyError:
1865 pass
1866 else:
1867 raise KeyError, "Key property (%s) of retired node clashes with \
1868 existing one (%s)" % (key, node[key])
1870 self.fireAuditors('restore', nodeid, None)
1871 # use the arg for __retired__ to cope with any odd database type
1872 # conversion (hello, sqlite)
1873 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1874 self.db.arg, self.db.arg)
1875 self.db.sql(sql, (0, nodeid))
1876 if self.do_journal:
1877 self.db.addjournal(self.classname, nodeid, ''"restored", None)
1879 self.fireReactors('restore', nodeid, None)
1881 def is_retired(self, nodeid):
1882 """Return true if the node is rerired
1883 """
1884 sql = 'select __retired__ from _%s where id=%s'%(self.classname,
1885 self.db.arg)
1886 self.db.sql(sql, (nodeid,))
1887 return int(self.db.sql_fetchone()[0]) > 0
1889 def destroy(self, nodeid):
1890 """Destroy a node.
1892 WARNING: this method should never be used except in extremely rare
1893 situations where there could never be links to the node being
1894 deleted
1896 WARNING: use retire() instead
1898 WARNING: the properties of this node will not be available ever again
1900 WARNING: really, use retire() instead
1902 Well, I think that's enough warnings. This method exists mostly to
1903 support the session storage of the cgi interface.
1905 The node is completely removed from the hyperdb, including all journal
1906 entries. It will no longer be available, and will generally break code
1907 if there are any references to the node.
1908 """
1909 if self.db.journaltag is None:
1910 raise DatabaseError, _('Database open read-only')
1911 self.db.destroynode(self.classname, nodeid)
1913 def history(self, nodeid):
1914 """Retrieve the journal of edits on a particular node.
1916 'nodeid' must be the id of an existing node of this class or an
1917 IndexError is raised.
1919 The returned list contains tuples of the form
1921 (nodeid, date, tag, action, params)
1923 'date' is a Timestamp object specifying the time of the change and
1924 'tag' is the journaltag specified when the database was opened.
1925 """
1926 if not self.do_journal:
1927 raise ValueError, 'Journalling is disabled for this class'
1928 return self.db.getjournal(self.classname, nodeid)
1930 # Locating nodes:
1931 def hasnode(self, nodeid):
1932 """Determine if the given nodeid actually exists
1933 """
1934 return self.db.hasnode(self.classname, nodeid)
1936 def setkey(self, propname):
1937 """Select a String property of this class to be the key property.
1939 'propname' must be the name of a String property of this class or
1940 None, or a TypeError is raised. The values of the key property on
1941 all existing nodes must be unique or a ValueError is raised.
1942 """
1943 prop = self.getprops()[propname]
1944 if not isinstance(prop, String):
1945 raise TypeError, 'key properties must be String'
1946 self.key = propname
1948 def getkey(self):
1949 """Return the name of the key property for this class or None."""
1950 return self.key
1952 def lookup(self, keyvalue):
1953 """Locate a particular node by its key property and return its id.
1955 If this class has no key property, a TypeError is raised. If the
1956 'keyvalue' matches one of the values for the key property among
1957 the nodes in this class, the matching node's id is returned;
1958 otherwise a KeyError is raised.
1959 """
1960 if not self.key:
1961 raise TypeError, 'No key property set for class %s'%self.classname
1963 # use the arg to handle any odd database type conversion (hello,
1964 # sqlite)
1965 sql = "select id from _%s where _%s=%s and __retired__=%s"%(
1966 self.classname, self.key, self.db.arg, self.db.arg)
1967 self.db.sql(sql, (str(keyvalue), 0))
1969 # see if there was a result that's not retired
1970 row = self.db.sql_fetchone()
1971 if not row:
1972 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1973 keyvalue, self.classname)
1975 # return the id
1976 # XXX numeric ids
1977 return str(row[0])
1979 def find(self, **propspec):
1980 """Get the ids of nodes in this class which link to the given nodes.
1982 'propspec' consists of keyword args propname=nodeid or
1983 propname={nodeid:1, }
1984 'propname' must be the name of a property in this class, or a
1985 KeyError is raised. That property must be a Link or
1986 Multilink property, or a TypeError is raised.
1988 Any node in this class whose 'propname' property links to any of
1989 the nodeids will be returned. Examples::
1991 db.issue.find(messages='1')
1992 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1993 """
1994 # shortcut
1995 if not propspec:
1996 return []
1998 # validate the args
1999 props = self.getprops()
2000 propspec = propspec.items()
2001 for propname, nodeids in propspec:
2002 # check the prop is OK
2003 prop = props[propname]
2004 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
2005 raise TypeError, "'%s' not a Link/Multilink property"%propname
2007 # first, links
2008 a = self.db.arg
2009 allvalues = ()
2010 sql = []
2011 where = []
2012 for prop, values in propspec:
2013 if not isinstance(props[prop], hyperdb.Link):
2014 continue
2015 if type(values) is type({}) and len(values) == 1:
2016 values = values.keys()[0]
2017 if type(values) is type(''):
2018 allvalues += (values,)
2019 where.append('_%s = %s'%(prop, a))
2020 elif values is None:
2021 where.append('_%s is NULL'%prop)
2022 else:
2023 values = values.keys()
2024 s = ''
2025 if None in values:
2026 values.remove(None)
2027 s = '_%s is NULL or '%prop
2028 allvalues += tuple(values)
2029 s += '_%s in (%s)'%(prop, ','.join([a]*len(values)))
2030 where.append('(' + s +')')
2031 if where:
2032 allvalues = (0, ) + allvalues
2033 sql.append("""select id from _%s where __retired__=%s
2034 and %s"""%(self.classname, a, ' and '.join(where)))
2036 # now multilinks
2037 for prop, values in propspec:
2038 if not isinstance(props[prop], hyperdb.Multilink):
2039 continue
2040 if not values:
2041 continue
2042 allvalues += (0, )
2043 if type(values) is type(''):
2044 allvalues += (values,)
2045 s = a
2046 else:
2047 allvalues += tuple(values.keys())
2048 s = ','.join([a]*len(values))
2049 tn = '%s_%s'%(self.classname, prop)
2050 sql.append("""select id from _%s, %s where __retired__=%s
2051 and id = %s.nodeid and %s.linkid in (%s)"""%(self.classname,
2052 tn, a, tn, tn, s))
2054 if not sql:
2055 return []
2056 sql = ' union '.join(sql)
2057 self.db.sql(sql, allvalues)
2058 # XXX numeric ids
2059 l = [str(x[0]) for x in self.db.sql_fetchall()]
2060 return l
2062 def stringFind(self, **requirements):
2063 """Locate a particular node by matching a set of its String
2064 properties in a caseless search.
2066 If the property is not a String property, a TypeError is raised.
2068 The return is a list of the id of all nodes that match.
2069 """
2070 where = []
2071 args = []
2072 for propname in requirements.keys():
2073 prop = self.properties[propname]
2074 if not isinstance(prop, String):
2075 raise TypeError, "'%s' not a String property"%propname
2076 where.append(propname)
2077 args.append(requirements[propname].lower())
2079 # generate the where clause
2080 s = ' and '.join(['lower(_%s)=%s'%(col, self.db.arg) for col in where])
2081 sql = 'select id from _%s where %s and __retired__=%s'%(
2082 self.classname, s, self.db.arg)
2083 args.append(0)
2084 self.db.sql(sql, tuple(args))
2085 # XXX numeric ids
2086 l = [str(x[0]) for x in self.db.sql_fetchall()]
2087 return l
2089 def list(self):
2090 """ Return a list of the ids of the active nodes in this class.
2091 """
2092 return self.getnodeids(retired=0)
2094 def getnodeids(self, retired=None):
2095 """ Retrieve all the ids of the nodes for a particular Class.
2097 Set retired=None to get all nodes. Otherwise it'll get all the
2098 retired or non-retired nodes, depending on the flag.
2099 """
2100 # flip the sense of the 'retired' flag if we don't want all of them
2101 if retired is not None:
2102 args = (0, )
2103 if retired:
2104 compare = '>'
2105 else:
2106 compare = '='
2107 sql = 'select id from _%s where __retired__%s%s'%(self.classname,
2108 compare, self.db.arg)
2109 else:
2110 args = ()
2111 sql = 'select id from _%s'%self.classname
2112 self.db.sql(sql, args)
2113 # XXX numeric ids
2114 ids = [str(x[0]) for x in self.db.cursor.fetchall()]
2115 return ids
2117 def _subselect(self, classname, multilink_table):
2118 """Create a subselect. This is factored out because some
2119 databases (hmm only one, so far) doesn't support subselects
2120 look for "I can't believe it's not a toy RDBMS" in the mysql
2121 backend.
2122 """
2123 return '_%s.id not in (select nodeid from %s)'%(classname,
2124 multilink_table)
2126 # Some DBs order NULL values last. Set this variable in the backend
2127 # for prepending an order by clause for each attribute that causes
2128 # correct sort order for NULLs. Examples:
2129 # order_by_null_values = '(%s is not NULL)'
2130 # order_by_null_values = 'notnull(%s)'
2131 # The format parameter is replaced with the attribute.
2132 order_by_null_values = None
2134 def filter(self, search_matches, filterspec, sort=[], group=[]):
2135 """Return a list of the ids of the active nodes in this class that
2136 match the 'filter' spec, sorted by the group spec and then the
2137 sort spec
2139 "filterspec" is {propname: value(s)}
2141 "sort" and "group" are [(dir, prop), ...] where dir is '+', '-'
2142 or None and prop is a prop name or None. Note that for
2143 backward-compatibility reasons a single (dir, prop) tuple is
2144 also allowed.
2146 "search_matches" is a container type or None
2148 The filter must match all properties specificed. If the property
2149 value to match is a list:
2151 1. String properties must match all elements in the list, and
2152 2. Other properties must match any of the elements in the list.
2153 """
2154 # we can't match anything if search_matches is empty
2155 if not search_matches and search_matches is not None:
2156 return []
2158 if __debug__:
2159 start_t = time.time()
2161 icn = self.classname
2163 # vars to hold the components of the SQL statement
2164 frum = [] # FROM clauses
2165 loj = [] # LEFT OUTER JOIN clauses
2166 where = [] # WHERE clauses
2167 args = [] # *any* positional arguments
2168 a = self.db.arg
2170 # figure the WHERE clause from the filterspec
2171 mlfilt = 0 # are we joining with Multilink tables?
2172 sortattr = self._sortattr (group = group, sort = sort)
2173 proptree = self._proptree(filterspec, sortattr)
2174 mlseen = 0
2175 for pt in reversed(proptree.sortattr):
2176 p = pt
2177 while p.parent:
2178 if isinstance (p.propclass, Multilink):
2179 mlseen = True
2180 if mlseen:
2181 p.sort_ids_needed = True
2182 p.tree_sort_done = False
2183 p = p.parent
2184 if not mlseen:
2185 pt.attr_sort_done = pt.tree_sort_done = True
2186 proptree.compute_sort_done()
2188 ordercols = []
2189 auxcols = {}
2190 mlsort = []
2191 rhsnum = 0
2192 for p in proptree:
2193 oc = None
2194 cn = p.classname
2195 ln = p.uniqname
2196 pln = p.parent.uniqname
2197 pcn = p.parent.classname
2198 k = p.name
2199 v = p.val
2200 propclass = p.propclass
2201 if p.sort_type > 0:
2202 oc = ac = '_%s._%s'%(pln, k)
2203 if isinstance(propclass, Multilink):
2204 if p.sort_type < 2:
2205 mlfilt = 1
2206 tn = '%s_%s'%(pcn, k)
2207 if v in ('-1', ['-1'], []):
2208 # only match rows that have count(linkid)=0 in the
2209 # corresponding multilink table)
2210 where.append(self._subselect(pcn, tn))
2211 else:
2212 frum.append(tn)
2213 where.append('_%s.id=%s.nodeid'%(pln,tn))
2214 if p.children:
2215 frum.append('_%s as _%s' % (cn, ln))
2216 where.append('%s.linkid=_%s.id'%(tn, ln))
2217 if p.has_values:
2218 if isinstance(v, type([])):
2219 s = ','.join([a for x in v])
2220 where.append('%s.linkid in (%s)'%(tn, s))
2221 args = args + v
2222 else:
2223 where.append('%s.linkid=%s'%(tn, a))
2224 args.append(v)
2225 if p.sort_type > 0:
2226 assert not p.attr_sort_done and not p.sort_ids_needed
2227 elif k == 'id':
2228 if p.sort_type < 2:
2229 if isinstance(v, type([])):
2230 s = ','.join([a for x in v])
2231 where.append('_%s.%s in (%s)'%(pln, k, s))
2232 args = args + v
2233 else:
2234 where.append('_%s.%s=%s'%(pln, k, a))
2235 args.append(v)
2236 if p.sort_type > 0:
2237 oc = ac = '_%s.id'%pln
2238 elif isinstance(propclass, String):
2239 if p.sort_type < 2:
2240 if not isinstance(v, type([])):
2241 v = [v]
2243 # Quote the bits in the string that need it and then embed
2244 # in a "substring" search. Note - need to quote the '%' so
2245 # they make it through the python layer happily
2246 v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
2248 # now add to the where clause
2249 where.append('('
2250 +' and '.join(["_%s._%s LIKE '%s'"%(pln, k, s) for s in v])
2251 +')')
2252 # note: args are embedded in the query string now
2253 if p.sort_type > 0:
2254 oc = ac = 'lower(_%s._%s)'%(pln, k)
2255 elif isinstance(propclass, Link):
2256 if p.sort_type < 2:
2257 if p.children:
2258 if p.sort_type == 0:
2259 frum.append('_%s as _%s' % (cn, ln))
2260 where.append('_%s._%s=_%s.id'%(pln, k, ln))
2261 if p.has_values:
2262 if isinstance(v, type([])):
2263 d = {}
2264 for entry in v:
2265 if entry == '-1':
2266 entry = None
2267 d[entry] = entry
2268 l = []
2269 if d.has_key(None) or not d:
2270 if d.has_key(None): del d[None]
2271 l.append('_%s._%s is NULL'%(pln, k))
2272 if d:
2273 v = d.keys()
2274 s = ','.join([a for x in v])
2275 l.append('(_%s._%s in (%s))'%(pln, k, s))
2276 args = args + v
2277 if l:
2278 where.append('(' + ' or '.join(l) +')')
2279 else:
2280 if v in ('-1', None):
2281 v = None
2282 where.append('_%s._%s is NULL'%(pln, k))
2283 else:
2284 where.append('_%s._%s=%s'%(pln, k, a))
2285 args.append(v)
2286 if p.sort_type > 0:
2287 lp = p.cls.labelprop()
2288 oc = ac = '_%s._%s'%(pln, k)
2289 if lp != 'id':
2290 if p.tree_sort_done and p.sort_type > 0:
2291 loj.append(
2292 'LEFT OUTER JOIN _%s as _%s on _%s._%s=_%s.id'%(
2293 cn, ln, pln, k, ln))
2294 oc = '_%s._%s'%(ln, lp)
2295 elif isinstance(propclass, Date) and p.sort_type < 2:
2296 dc = self.db.to_sql_value(hyperdb.Date)
2297 if isinstance(v, type([])):
2298 s = ','.join([a for x in v])
2299 where.append('_%s._%s in (%s)'%(pln, k, s))
2300 args = args + [dc(date.Date(x)) for x in v]
2301 else:
2302 try:
2303 # Try to filter on range of dates
2304 date_rng = propclass.range_from_raw(v, self.db)
2305 if date_rng.from_value:
2306 where.append('_%s._%s >= %s'%(pln, k, a))
2307 args.append(dc(date_rng.from_value))
2308 if date_rng.to_value:
2309 where.append('_%s._%s <= %s'%(pln, k, a))
2310 args.append(dc(date_rng.to_value))
2311 except ValueError:
2312 # If range creation fails - ignore that search parameter
2313 pass
2314 elif isinstance(propclass, Interval):
2315 # filter/sort using the __<prop>_int__ column
2316 if p.sort_type < 2:
2317 if isinstance(v, type([])):
2318 s = ','.join([a for x in v])
2319 where.append('_%s.__%s_int__ in (%s)'%(pln, k, s))
2320 args = args + [date.Interval(x).as_seconds() for x in v]
2321 else:
2322 try:
2323 # Try to filter on range of intervals
2324 date_rng = Range(v, date.Interval)
2325 if date_rng.from_value:
2326 where.append('_%s.__%s_int__ >= %s'%(pln, k, a))
2327 args.append(date_rng.from_value.as_seconds())
2328 if date_rng.to_value:
2329 where.append('_%s.__%s_int__ <= %s'%(pln, k, a))
2330 args.append(date_rng.to_value.as_seconds())
2331 except ValueError:
2332 # If range creation fails - ignore search parameter
2333 pass
2334 if p.sort_type > 0:
2335 oc = ac = '_%s.__%s_int__'%(pln,k)
2336 elif p.sort_type < 2:
2337 if isinstance(v, type([])):
2338 s = ','.join([a for x in v])
2339 where.append('_%s._%s in (%s)'%(pln, k, s))
2340 args = args + v
2341 else:
2342 where.append('_%s._%s=%s'%(pln, k, a))
2343 args.append(v)
2344 if oc:
2345 if p.sort_ids_needed:
2346 auxcols[ac] = p
2347 if p.tree_sort_done and p.sort_direction:
2348 # Don't select top-level id twice
2349 if p.name != 'id' or p.parent != proptree:
2350 ordercols.append(oc)
2351 desc = ['', ' desc'][p.sort_direction == '-']
2352 # Some SQL dbs sort NULL values last -- we want them first.
2353 if (self.order_by_null_values and p.name != 'id'):
2354 nv = self.order_by_null_values % oc
2355 ordercols.append(nv)
2356 p.orderby.append(nv + desc)
2357 p.orderby.append(oc + desc)
2359 props = self.getprops()
2361 # don't match retired nodes
2362 where.append('_%s.__retired__=0'%icn)
2364 # add results of full text search
2365 if search_matches is not None:
2366 s = ','.join([a for x in search_matches])
2367 where.append('_%s.id in (%s)'%(icn, s))
2368 args = args + v
2370 # construct the SQL
2371 frum.append('_'+icn)
2372 frum = ','.join(frum)
2373 if where:
2374 where = ' where ' + (' and '.join(where))
2375 else:
2376 where = ''
2377 if mlfilt:
2378 # we're joining tables on the id, so we will get dupes if we
2379 # don't distinct()
2380 cols = ['distinct(_%s.id)'%icn]
2381 else:
2382 cols = ['_%s.id'%icn]
2383 if ordercols:
2384 cols = cols + ordercols
2385 order = []
2386 # keep correct sequence of order attributes.
2387 for sa in proptree.sortattr:
2388 if not sa.attr_sort_done:
2389 continue
2390 order.extend(sa.orderby)
2391 if order:
2392 order = ' order by %s'%(','.join(order))
2393 else:
2394 order = ''
2395 for o, p in auxcols.iteritems ():
2396 cols.append (o)
2397 p.auxcol = len (cols) - 1
2399 cols = ','.join(cols)
2400 loj = ' '.join(loj)
2401 sql = 'select %s from %s %s %s%s'%(cols, frum, loj, where, order)
2402 args = tuple(args)
2403 __traceback_info__ = (sql, args)
2404 self.db.sql(sql, args)
2405 l = self.db.sql_fetchall()
2407 # Compute values needed for sorting in proptree.sort
2408 for p in auxcols.itervalues():
2409 p.sort_ids = p.sort_result = [row[p.auxcol] for row in l]
2410 # return the IDs (the first column)
2411 # XXX numeric ids
2412 l = [str(row[0]) for row in l]
2413 l = proptree.sort (l)
2415 if __debug__:
2416 self.db.stats['filtering'] += (time.time() - start_t)
2417 return l
2419 def filter_sql(self, sql):
2420 """Return a list of the ids of the items in this class that match
2421 the SQL provided. The SQL is a complete "select" statement.
2423 The SQL select must include the item id as the first column.
2425 This function DOES NOT filter out retired items, add on a where
2426 clause "__retired__=0" if you don't want retired nodes.
2427 """
2428 if __debug__:
2429 start_t = time.time()
2431 self.db.sql(sql)
2432 l = self.db.sql_fetchall()
2434 if __debug__:
2435 self.db.stats['filtering'] += (time.time() - start_t)
2436 return l
2438 def count(self):
2439 """Get the number of nodes in this class.
2441 If the returned integer is 'numnodes', the ids of all the nodes
2442 in this class run from 1 to numnodes, and numnodes+1 will be the
2443 id of the next node to be created in this class.
2444 """
2445 return self.db.countnodes(self.classname)
2447 # Manipulating properties:
2448 def getprops(self, protected=1):
2449 """Return a dictionary mapping property names to property objects.
2450 If the "protected" flag is true, we include protected properties -
2451 those which may not be modified.
2452 """
2453 d = self.properties.copy()
2454 if protected:
2455 d['id'] = String()
2456 d['creation'] = hyperdb.Date()
2457 d['activity'] = hyperdb.Date()
2458 d['creator'] = hyperdb.Link('user')
2459 d['actor'] = hyperdb.Link('user')
2460 return d
2462 def addprop(self, **properties):
2463 """Add properties to this class.
2465 The keyword arguments in 'properties' must map names to property
2466 objects, or a TypeError is raised. None of the keys in 'properties'
2467 may collide with the names of existing properties, or a ValueError
2468 is raised before any properties have been added.
2469 """
2470 for key in properties.keys():
2471 if self.properties.has_key(key):
2472 raise ValueError, key
2473 self.properties.update(properties)
2475 def index(self, nodeid):
2476 """Add (or refresh) the node to search indexes
2477 """
2478 # find all the String properties that have indexme
2479 for prop, propclass in self.getprops().items():
2480 if isinstance(propclass, String) and propclass.indexme:
2481 self.db.indexer.add_text((self.classname, nodeid, prop),
2482 str(self.get(nodeid, prop)))
2484 #
2485 # import / export support
2486 #
2487 def export_list(self, propnames, nodeid):
2488 """ Export a node - generate a list of CSV-able data in the order
2489 specified by propnames for the given node.
2490 """
2491 properties = self.getprops()
2492 l = []
2493 for prop in propnames:
2494 proptype = properties[prop]
2495 value = self.get(nodeid, prop)
2496 # "marshal" data where needed
2497 if value is None:
2498 pass
2499 elif isinstance(proptype, hyperdb.Date):
2500 value = value.get_tuple()
2501 elif isinstance(proptype, hyperdb.Interval):
2502 value = value.get_tuple()
2503 elif isinstance(proptype, hyperdb.Password):
2504 value = str(value)
2505 l.append(repr(value))
2506 l.append(repr(self.is_retired(nodeid)))
2507 return l
2509 def import_list(self, propnames, proplist):
2510 """ Import a node - all information including "id" is present and
2511 should not be sanity checked. Triggers are not triggered. The
2512 journal should be initialised using the "creator" and "created"
2513 information.
2515 Return the nodeid of the node imported.
2516 """
2517 if self.db.journaltag is None:
2518 raise DatabaseError, _('Database open read-only')
2519 properties = self.getprops()
2521 # make the new node's property map
2522 d = {}
2523 retire = 0
2524 if not "id" in propnames:
2525 newid = self.db.newid(self.classname)
2526 else:
2527 newid = eval(proplist[propnames.index("id")])
2528 for i in range(len(propnames)):
2529 # Use eval to reverse the repr() used to output the CSV
2530 value = eval(proplist[i])
2532 # Figure the property for this column
2533 propname = propnames[i]
2535 # "unmarshal" where necessary
2536 if propname == 'id':
2537 continue
2538 elif propname == 'is retired':
2539 # is the item retired?
2540 if int(value):
2541 retire = 1
2542 continue
2543 elif value is None:
2544 d[propname] = None
2545 continue
2547 prop = properties[propname]
2548 if value is None:
2549 # don't set Nones
2550 continue
2551 elif isinstance(prop, hyperdb.Date):
2552 value = date.Date(value)
2553 elif isinstance(prop, hyperdb.Interval):
2554 value = date.Interval(value)
2555 elif isinstance(prop, hyperdb.Password):
2556 pwd = password.Password()
2557 pwd.unpack(value)
2558 value = pwd
2559 elif isinstance(prop, String):
2560 if isinstance(value, unicode):
2561 value = value.encode('utf8')
2562 if not isinstance(value, str):
2563 raise TypeError, \
2564 'new property "%(propname)s" not a string: %(value)r' \
2565 % locals()
2566 if prop.indexme:
2567 self.db.indexer.add_text((self.classname, newid, propname),
2568 value)
2569 d[propname] = value
2571 # get a new id if necessary
2572 if newid is None:
2573 newid = self.db.newid(self.classname)
2575 # insert new node or update existing?
2576 if not self.hasnode(newid):
2577 self.db.addnode(self.classname, newid, d) # insert
2578 else:
2579 self.db.setnode(self.classname, newid, d) # update
2581 # retire?
2582 if retire:
2583 # use the arg for __retired__ to cope with any odd database type
2584 # conversion (hello, sqlite)
2585 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
2586 self.db.arg, self.db.arg)
2587 self.db.sql(sql, (newid, newid))
2588 return newid
2590 def export_journals(self):
2591 """Export a class's journal - generate a list of lists of
2592 CSV-able data:
2594 nodeid, date, user, action, params
2596 No heading here - the columns are fixed.
2597 """
2598 properties = self.getprops()
2599 r = []
2600 for nodeid in self.getnodeids():
2601 for nodeid, date, user, action, params in self.history(nodeid):
2602 date = date.get_tuple()
2603 if action == 'set':
2604 export_data = {}
2605 for propname, value in params.items():
2606 if not properties.has_key(propname):
2607 # property no longer in the schema
2608 continue
2610 prop = properties[propname]
2611 # make sure the params are eval()'able
2612 if value is None:
2613 pass
2614 elif isinstance(prop, Date):
2615 value = value.get_tuple()
2616 elif isinstance(prop, Interval):
2617 value = value.get_tuple()
2618 elif isinstance(prop, Password):
2619 value = str(value)
2620 export_data[propname] = value
2621 params = export_data
2622 elif action == 'create' and params:
2623 # old tracker with data stored in the create!
2624 params = {}
2625 l = [nodeid, date, user, action, params]
2626 r.append(map(repr, l))
2627 return r
2629 def import_journals(self, entries):
2630 """Import a class's journal.
2632 Uses setjournal() to set the journal for each item."""
2633 properties = self.getprops()
2634 d = {}
2635 for l in entries:
2636 l = map(eval, l)
2637 nodeid, jdate, user, action, params = l
2638 r = d.setdefault(nodeid, [])
2639 if action == 'set':
2640 for propname, value in params.items():
2641 prop = properties[propname]
2642 if value is None:
2643 pass
2644 elif isinstance(prop, Date):
2645 value = date.Date(value)
2646 elif isinstance(prop, Interval):
2647 value = date.Interval(value)
2648 elif isinstance(prop, Password):
2649 pwd = password.Password()
2650 pwd.unpack(value)
2651 value = pwd
2652 params[propname] = value
2653 elif action == 'create' and params:
2654 # old tracker with data stored in the create!
2655 params = {}
2656 r.append((nodeid, date.Date(jdate), user, action, params))
2658 for nodeid, l in d.items():
2659 self.db.setjournal(self.classname, nodeid, l)
2661 class FileClass(hyperdb.FileClass, Class):
2662 """This class defines a large chunk of data. To support this, it has a
2663 mandatory String property "content" which is typically saved off
2664 externally to the hyperdb.
2666 The default MIME type of this data is defined by the
2667 "default_mime_type" class attribute, which may be overridden by each
2668 node if the class defines a "type" String property.
2669 """
2670 def __init__(self, db, classname, **properties):
2671 """The newly-created class automatically includes the "content"
2672 and "type" properties.
2673 """
2674 if not properties.has_key('content'):
2675 properties['content'] = hyperdb.String(indexme='yes')
2676 if not properties.has_key('type'):
2677 properties['type'] = hyperdb.String()
2678 Class.__init__(self, db, classname, **properties)
2680 def create(self, **propvalues):
2681 """ snaffle the file propvalue and store in a file
2682 """
2683 # we need to fire the auditors now, or the content property won't
2684 # be in propvalues for the auditors to play with
2685 self.fireAuditors('create', None, propvalues)
2687 # now remove the content property so it's not stored in the db
2688 content = propvalues['content']
2689 del propvalues['content']
2691 # do the database create
2692 newid = self.create_inner(**propvalues)
2694 # figure the mime type
2695 mime_type = propvalues.get('type', self.default_mime_type)
2697 # and index!
2698 if self.properties['content'].indexme:
2699 self.db.indexer.add_text((self.classname, newid, 'content'),
2700 content, mime_type)
2702 # store off the content as a file
2703 self.db.storefile(self.classname, newid, None, content)
2705 # fire reactors
2706 self.fireReactors('create', newid, None)
2708 return newid
2710 def get(self, nodeid, propname, default=_marker, cache=1):
2711 """ Trap the content propname and get it from the file
2713 'cache' exists for backwards compatibility, and is not used.
2714 """
2715 poss_msg = 'Possibly a access right configuration problem.'
2716 if propname == 'content':
2717 try:
2718 return self.db.getfile(self.classname, nodeid, None)
2719 except IOError, (strerror):
2720 # BUG: by catching this we donot see an error in the log.
2721 return 'ERROR reading file: %s%s\n%s\n%s'%(
2722 self.classname, nodeid, poss_msg, strerror)
2723 if default is not _marker:
2724 return Class.get(self, nodeid, propname, default)
2725 else:
2726 return Class.get(self, nodeid, propname)
2728 def set(self, itemid, **propvalues):
2729 """ Snarf the "content" propvalue and update it in a file
2730 """
2731 self.fireAuditors('set', itemid, propvalues)
2732 oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid))
2734 # now remove the content property so it's not stored in the db
2735 content = None
2736 if propvalues.has_key('content'):
2737 content = propvalues['content']
2738 del propvalues['content']
2740 # do the database create
2741 propvalues = self.set_inner(itemid, **propvalues)
2743 # do content?
2744 if content:
2745 # store and possibly index
2746 self.db.storefile(self.classname, itemid, None, content)
2747 if self.properties['content'].indexme:
2748 mime_type = self.get(itemid, 'type', self.default_mime_type)
2749 self.db.indexer.add_text((self.classname, itemid, 'content'),
2750 content, mime_type)
2751 propvalues['content'] = content
2753 # fire reactors
2754 self.fireReactors('set', itemid, oldvalues)
2755 return propvalues
2757 def index(self, nodeid):
2758 """ Add (or refresh) the node to search indexes.
2760 Use the content-type property for the content property.
2761 """
2762 # find all the String properties that have indexme
2763 for prop, propclass in self.getprops().items():
2764 if prop == 'content' and propclass.indexme:
2765 mime_type = self.get(nodeid, 'type', self.default_mime_type)
2766 self.db.indexer.add_text((self.classname, nodeid, 'content'),
2767 str(self.get(nodeid, 'content')), mime_type)
2768 elif isinstance(propclass, hyperdb.String) and propclass.indexme:
2769 # index them under (classname, nodeid, property)
2770 try:
2771 value = str(self.get(nodeid, prop))
2772 except IndexError:
2773 # node has been destroyed
2774 continue
2775 self.db.indexer.add_text((self.classname, nodeid, prop), value)
2777 # XXX deviation from spec - was called ItemClass
2778 class IssueClass(Class, roundupdb.IssueClass):
2779 # Overridden methods:
2780 def __init__(self, db, classname, **properties):
2781 """The newly-created class automatically includes the "messages",
2782 "files", "nosy", and "superseder" properties. If the 'properties'
2783 dictionary attempts to specify any of these properties or a
2784 "creation", "creator", "activity" or "actor" property, a ValueError
2785 is raised.
2786 """
2787 if not properties.has_key('title'):
2788 properties['title'] = hyperdb.String(indexme='yes')
2789 if not properties.has_key('messages'):
2790 properties['messages'] = hyperdb.Multilink("msg")
2791 if not properties.has_key('files'):
2792 properties['files'] = hyperdb.Multilink("file")
2793 if not properties.has_key('nosy'):
2794 # note: journalling is turned off as it really just wastes
2795 # space. this behaviour may be overridden in an instance
2796 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2797 if not properties.has_key('superseder'):
2798 properties['superseder'] = hyperdb.Multilink(classname)
2799 Class.__init__(self, db, classname, **properties)
2801 # vim: set et sts=4 sw=4 :