1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 """ Relational database (SQL) backend common code.
20 Basics:
22 - map roundup classes to relational tables
23 - automatically detect schema changes and modify the table schemas
24 appropriately (we store the "database version" of the schema in the
25 database itself as the only row of the "schema" table)
26 - multilinks (which represent a many-to-many relationship) are handled through
27 intermediate tables
28 - journals are stored adjunct to the per-class tables
29 - table names and columns have "_" prepended so the names can't clash with
30 restricted names (like "order")
31 - retirement is determined by the __retired__ column being > 0
33 Database-specific changes may generally be pushed out to the overridable
34 sql_* methods, since everything else should be fairly generic. There's
35 probably a bit of work to be done if a database is used that actually
36 honors column typing, since the initial databases don't (sqlite stores
37 everything as a string.)
39 The schema of the hyperdb being mapped to the database is stored in the
40 database itself as a repr()'ed dictionary of information about each Class
41 that maps to a table. If that information differs from the hyperdb schema,
42 then we update it. We also store in the schema dict a version which
43 allows us to upgrade the database schema when necessary. See upgrade_db().
45 To force a unqiueness constraint on the key properties we put the item
46 id into the __retired__ column duing retirement (so it's 0 for "active"
47 items) and place a unqiueness constraint on key + __retired__. This is
48 particularly important for the users class where multiple users may
49 try to have the same username, with potentially many retired users with
50 the same name.
51 """
52 __docformat__ = 'restructuredtext'
54 # standard python modules
55 import sys, os, time, re, errno, weakref, copy, logging
57 # roundup modules
58 from roundup import hyperdb, date, password, roundupdb, security, support
59 from roundup.hyperdb import String, Password, Date, Interval, Link, \
60 Multilink, DatabaseError, Boolean, Number, Node
61 from roundup.backends import locking
62 from roundup.support import reversed
63 from roundup.i18n import _
65 # support
66 from blobfiles import FileStorage
67 try:
68 from indexer_xapian import Indexer
69 except ImportError:
70 from indexer_rdbms import Indexer
71 from sessions_rdbms import Sessions, OneTimeKeys
72 from roundup.date import Range
74 # dummy value meaning "argument not passed"
75 _marker = []
77 def _num_cvt(num):
78 num = str(num)
79 try:
80 return int(num)
81 except:
82 return float(num)
84 def _bool_cvt(value):
85 if value in ('TRUE', 'FALSE'):
86 return {'TRUE': 1, 'FALSE': 0}[value]
87 # assume it's a number returned from the db API
88 return int(value)
90 def connection_dict(config, dbnamestr=None):
91 """ Used by Postgresql and MySQL to detemine the keyword args for
92 opening the database connection."""
93 d = { }
94 if dbnamestr:
95 d[dbnamestr] = config.RDBMS_NAME
96 for name in ('host', 'port', 'password', 'user', 'read_default_group',
97 'read_default_file'):
98 cvar = 'RDBMS_'+name.upper()
99 if config[cvar] is not None:
100 d[name] = config[cvar]
101 return d
103 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
104 """ Wrapper around an SQL database that presents a hyperdb interface.
106 - some functionality is specific to the actual SQL database, hence
107 the sql_* methods that are NotImplemented
108 - we keep a cache of the latest N row fetches (where N is configurable).
109 """
110 def __init__(self, config, journaltag=None):
111 """ Open the database and load the schema from it.
112 """
113 FileStorage.__init__(self, config.UMASK)
114 self.config, self.journaltag = config, journaltag
115 self.dir = config.DATABASE
116 self.classes = {}
117 self.indexer = Indexer(self)
118 self.security = security.Security(self)
120 # additional transaction support for external files and the like
121 self.transactions = []
123 # keep a cache of the N most recently retrieved rows of any kind
124 # (classname, nodeid) = row
125 self.cache_size = config.RDBMS_CACHE_SIZE
126 self.cache = {}
127 self.cache_lru = []
128 self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0,
129 'filtering': 0}
131 # database lock
132 self.lockfile = None
134 # open a connection to the database, creating the "conn" attribute
135 self.open_connection()
137 def clearCache(self):
138 self.cache = {}
139 self.cache_lru = []
141 def getSessionManager(self):
142 return Sessions(self)
144 def getOTKManager(self):
145 return OneTimeKeys(self)
147 def open_connection(self):
148 """ Open a connection to the database, creating it if necessary.
150 Must call self.load_dbschema()
151 """
152 raise NotImplemented
154 def sql(self, sql, args=None):
155 """ Execute the sql with the optional args.
156 """
157 self.log_debug('SQL %r %r'%(sql, args))
158 if args:
159 self.cursor.execute(sql, args)
160 else:
161 self.cursor.execute(sql)
163 def sql_fetchone(self):
164 """ Fetch a single row. If there's nothing to fetch, return None.
165 """
166 return self.cursor.fetchone()
168 def sql_fetchall(self):
169 """ Fetch all rows. If there's nothing to fetch, return [].
170 """
171 return self.cursor.fetchall()
173 def sql_stringquote(self, value):
174 """ Quote the string so it's safe to put in the 'sql quotes'
175 """
176 return re.sub("'", "''", str(value))
178 def init_dbschema(self):
179 self.database_schema = {
180 'version': self.current_db_version,
181 'tables': {}
182 }
184 def load_dbschema(self):
185 """ Load the schema definition that the database currently implements
186 """
187 self.cursor.execute('select schema from schema')
188 schema = self.cursor.fetchone()
189 if schema:
190 self.database_schema = eval(schema[0])
191 else:
192 self.database_schema = {}
194 def save_dbschema(self):
195 """ Save the schema definition that the database currently implements
196 """
197 s = repr(self.database_schema)
198 self.sql('delete from schema')
199 self.sql('insert into schema values (%s)'%self.arg, (s,))
201 def post_init(self):
202 """ Called once the schema initialisation has finished.
204 We should now confirm that the schema defined by our "classes"
205 attribute actually matches the schema in the database.
206 """
207 save = 0
209 # handle changes in the schema
210 tables = self.database_schema['tables']
211 for classname, spec in self.classes.items():
212 if tables.has_key(classname):
213 dbspec = tables[classname]
214 if self.update_class(spec, dbspec):
215 tables[classname] = spec.schema()
216 save = 1
217 else:
218 self.create_class(spec)
219 tables[classname] = spec.schema()
220 save = 1
222 for classname, spec in tables.items():
223 if not self.classes.has_key(classname):
224 self.drop_class(classname, tables[classname])
225 del tables[classname]
226 save = 1
228 # now upgrade the database for column type changes, new internal
229 # tables, etc.
230 save = save | self.upgrade_db()
232 # update the database version of the schema
233 if save:
234 self.save_dbschema()
236 # reindex the db if necessary
237 if self.indexer.should_reindex():
238 self.reindex()
240 # commit
241 self.sql_commit()
243 # update this number when we need to make changes to the SQL structure
244 # of the backen database
245 current_db_version = 5
246 db_version_updated = False
247 def upgrade_db(self):
248 """ Update the SQL database to reflect changes in the backend code.
250 Return boolean whether we need to save the schema.
251 """
252 version = self.database_schema.get('version', 1)
253 if version > self.current_db_version:
254 raise DatabaseError('attempting to run rev %d DATABASE with rev '
255 '%d CODE!'%(version, self.current_db_version))
256 if version == self.current_db_version:
257 # nothing to do
258 return 0
260 if version < 2:
261 self.log_info('upgrade to version 2')
262 # change the schema structure
263 self.database_schema = {'tables': self.database_schema}
265 # version 1 didn't have the actor column (note that in
266 # MySQL this will also transition the tables to typed columns)
267 self.add_new_columns_v2()
269 # version 1 doesn't have the OTK, session and indexing in the
270 # database
271 self.create_version_2_tables()
273 if version < 3:
274 self.log_info('upgrade to version 3')
275 self.fix_version_2_tables()
277 if version < 4:
278 self.fix_version_3_tables()
280 if version < 5:
281 self.fix_version_4_tables()
283 self.database_schema['version'] = self.current_db_version
284 self.db_version_updated = True
285 return 1
287 def fix_version_3_tables(self):
288 # drop the shorter VARCHAR OTK column and add a new TEXT one
289 for name in ('otk', 'session'):
290 self.sql('DELETE FROM %ss'%name)
291 self.sql('ALTER TABLE %ss DROP %s_value'%(name, name))
292 self.sql('ALTER TABLE %ss ADD %s_value TEXT'%(name, name))
294 def fix_version_2_tables(self):
295 # Default (used by sqlite): NOOP
296 pass
298 def fix_version_4_tables(self):
299 # note this is an explicit call now
300 c = self.cursor
301 for cn, klass in self.classes.items():
302 c.execute('select id from _%s where __retired__<>0'%(cn,))
303 for (id,) in c.fetchall():
304 c.execute('update _%s set __retired__=%s where id=%s'%(cn,
305 self.arg, self.arg), (id, id))
307 if klass.key:
308 self.add_class_key_required_unique_constraint(cn, klass.key)
310 def _convert_journal_tables(self):
311 """Get current journal table contents, drop the table and re-create"""
312 c = self.cursor
313 cols = ','.join('nodeid date tag action params'.split())
314 for klass in self.classes.values():
315 # slurp and drop
316 sql = 'select %s from %s__journal order by date'%(cols,
317 klass.classname)
318 c.execute(sql)
319 contents = c.fetchall()
320 self.drop_journal_table_indexes(klass.classname)
321 c.execute('drop table %s__journal'%klass.classname)
323 # re-create and re-populate
324 self.create_journal_table(klass)
325 a = self.arg
326 sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(
327 klass.classname, cols, a, a, a, a, a)
328 for row in contents:
329 # no data conversion needed
330 self.cursor.execute(sql, row)
332 def _convert_string_properties(self):
333 """Get current Class tables that contain String properties, and
334 convert the VARCHAR columns to TEXT"""
335 c = self.cursor
336 for klass in self.classes.values():
337 # slurp and drop
338 cols, mls = self.determine_columns(klass.properties.items())
339 scols = ','.join([i[0] for i in cols])
340 sql = 'select id,%s from _%s'%(scols, klass.classname)
341 c.execute(sql)
342 contents = c.fetchall()
343 self.drop_class_table_indexes(klass.classname, klass.getkey())
344 c.execute('drop table _%s'%klass.classname)
346 # re-create and re-populate
347 self.create_class_table(klass, create_sequence=0)
348 a = ','.join([self.arg for i in range(len(cols)+1)])
349 sql = 'insert into _%s (id,%s) values (%s)'%(klass.classname,
350 scols, a)
351 for row in contents:
352 l = []
353 for entry in row:
354 # mysql will already be a string - psql needs "help"
355 if entry is not None and not isinstance(entry, type('')):
356 entry = str(entry)
357 l.append(entry)
358 self.cursor.execute(sql, l)
360 def refresh_database(self):
361 self.post_init()
364 def reindex(self, classname=None, show_progress=False):
365 if classname:
366 classes = [self.getclass(classname)]
367 else:
368 classes = self.classes.values()
369 for klass in classes:
370 if show_progress:
371 for nodeid in support.Progress('Reindex %s'%klass.classname,
372 klass.list()):
373 klass.index(nodeid)
374 else:
375 for nodeid in klass.list():
376 klass.index(nodeid)
377 self.indexer.save_index()
379 hyperdb_to_sql_datatypes = {
380 hyperdb.String : 'TEXT',
381 hyperdb.Date : 'TIMESTAMP',
382 hyperdb.Link : 'INTEGER',
383 hyperdb.Interval : 'VARCHAR(255)',
384 hyperdb.Password : 'VARCHAR(255)',
385 hyperdb.Boolean : 'BOOLEAN',
386 hyperdb.Number : 'REAL',
387 }
389 def hyperdb_to_sql_datatype(self, propclass):
391 datatype = self.hyperdb_to_sql_datatypes.get(propclass)
392 if datatype:
393 return datatype
395 for k, v in self.hyperdb_to_sql_datatypes.iteritems():
396 if issubclass(propclass, k):
397 return v
399 raise ValueError, '%r is not a hyperdb property class' % propclass
401 def determine_columns(self, properties):
402 """ Figure the column names and multilink properties from the spec
404 "properties" is a list of (name, prop) where prop may be an
405 instance of a hyperdb "type" _or_ a string repr of that type.
406 """
407 cols = [
408 ('_actor', self.hyperdb_to_sql_datatype(hyperdb.Link)),
409 ('_activity', self.hyperdb_to_sql_datatype(hyperdb.Date)),
410 ('_creator', self.hyperdb_to_sql_datatype(hyperdb.Link)),
411 ('_creation', self.hyperdb_to_sql_datatype(hyperdb.Date)),
412 ]
413 mls = []
414 # add the multilinks separately
415 for col, prop in properties:
416 if isinstance(prop, Multilink):
417 mls.append(col)
418 continue
420 if isinstance(prop, type('')):
421 raise ValueError, "string property spec!"
422 #and prop.find('Multilink') != -1:
423 #mls.append(col)
425 datatype = self.hyperdb_to_sql_datatype(prop.__class__)
426 cols.append(('_'+col, datatype))
428 # Intervals stored as two columns
429 if isinstance(prop, Interval):
430 cols.append(('__'+col+'_int__', 'BIGINT'))
432 cols.sort()
433 return cols, mls
435 def update_class(self, spec, old_spec, force=0):
436 """ Determine the differences between the current spec and the
437 database version of the spec, and update where necessary.
439 If 'force' is true, update the database anyway.
440 """
441 new_has = spec.properties.has_key
442 new_spec = spec.schema()
443 new_spec[1].sort()
444 old_spec[1].sort()
445 if not force and new_spec == old_spec:
446 # no changes
447 return 0
449 logger = logging.getLogger('hyperdb')
450 logger.info('update_class %s'%spec.classname)
452 logger.debug('old_spec %r'%(old_spec,))
453 logger.debug('new_spec %r'%(new_spec,))
455 # detect key prop change for potential index change
456 keyprop_changes = {}
457 if new_spec[0] != old_spec[0]:
458 if old_spec[0]:
459 keyprop_changes['remove'] = old_spec[0]
460 if new_spec[0]:
461 keyprop_changes['add'] = new_spec[0]
463 # detect multilinks that have been removed, and drop their table
464 old_has = {}
465 for name, prop in old_spec[1]:
466 old_has[name] = 1
467 if new_has(name):
468 continue
470 if prop.find('Multilink to') != -1:
471 # first drop indexes.
472 self.drop_multilink_table_indexes(spec.classname, name)
474 # now the multilink table itself
475 sql = 'drop table %s_%s'%(spec.classname, name)
476 else:
477 # if this is the key prop, drop the index first
478 if old_spec[0] == prop:
479 self.drop_class_table_key_index(spec.classname, name)
480 del keyprop_changes['remove']
482 # drop the column
483 sql = 'alter table _%s drop column _%s'%(spec.classname, name)
485 self.sql(sql)
486 old_has = old_has.has_key
488 # if we didn't remove the key prop just then, but the key prop has
489 # changed, we still need to remove the old index
490 if keyprop_changes.has_key('remove'):
491 self.drop_class_table_key_index(spec.classname,
492 keyprop_changes['remove'])
494 # add new columns
495 for propname, prop in new_spec[1]:
496 if old_has(propname):
497 continue
498 prop = spec.properties[propname]
499 if isinstance(prop, Multilink):
500 self.create_multilink_table(spec, propname)
501 else:
502 # add the column
503 coltype = self.hyperdb_to_sql_datatype(prop.__class__)
504 sql = 'alter table _%s add column _%s %s'%(
505 spec.classname, propname, coltype)
506 self.sql(sql)
508 # extra Interval column
509 if isinstance(prop, Interval):
510 sql = 'alter table _%s add column __%s_int__ BIGINT'%(
511 spec.classname, propname)
512 self.sql(sql)
514 # if the new column is a key prop, we need an index!
515 if new_spec[0] == propname:
516 self.create_class_table_key_index(spec.classname, propname)
517 del keyprop_changes['add']
519 # if we didn't add the key prop just then, but the key prop has
520 # changed, we still need to add the new index
521 if keyprop_changes.has_key('add'):
522 self.create_class_table_key_index(spec.classname,
523 keyprop_changes['add'])
525 return 1
527 def determine_all_columns(self, spec):
528 """Figure out the columns from the spec and also add internal columns
530 """
531 cols, mls = self.determine_columns(spec.properties.items())
533 # add on our special columns
534 cols.append(('id', 'INTEGER PRIMARY KEY'))
535 cols.append(('__retired__', 'INTEGER DEFAULT 0'))
536 return cols, mls
538 def create_class_table(self, spec):
539 """Create the class table for the given Class "spec". Creates the
540 indexes too."""
541 cols, mls = self.determine_all_columns(spec)
543 # create the base table
544 scols = ','.join(['%s %s'%x for x in cols])
545 sql = 'create table _%s (%s)'%(spec.classname, scols)
546 self.sql(sql)
548 self.create_class_table_indexes(spec)
550 return cols, mls
552 def create_class_table_indexes(self, spec):
553 """ create the class table for the given spec
554 """
555 # create __retired__ index
556 index_sql2 = 'create index _%s_retired_idx on _%s(__retired__)'%(
557 spec.classname, spec.classname)
558 self.sql(index_sql2)
560 # create index for key property
561 if spec.key:
562 index_sql3 = 'create index _%s_%s_idx on _%s(_%s)'%(
563 spec.classname, spec.key,
564 spec.classname, spec.key)
565 self.sql(index_sql3)
567 # and the unique index for key / retired(id)
568 self.add_class_key_required_unique_constraint(spec.classname,
569 spec.key)
571 # TODO: create indexes on (selected?) Link property columns, as
572 # they're more likely to be used for lookup
574 def add_class_key_required_unique_constraint(self, cn, key):
575 sql = '''create unique index _%s_key_retired_idx
576 on _%s(__retired__, _%s)'''%(cn, cn, key)
577 self.sql(sql)
579 def drop_class_table_indexes(self, cn, key):
580 # drop the old table indexes first
581 l = ['_%s_id_idx'%cn, '_%s_retired_idx'%cn]
582 if key:
583 l.append('_%s_%s_idx'%(cn, key))
585 table_name = '_%s'%cn
586 for index_name in l:
587 if not self.sql_index_exists(table_name, index_name):
588 continue
589 index_sql = 'drop index '+index_name
590 self.sql(index_sql)
592 def create_class_table_key_index(self, cn, key):
593 """ create the class table for the given spec
594 """
595 sql = 'create index _%s_%s_idx on _%s(_%s)'%(cn, key, cn, key)
596 self.sql(sql)
598 def drop_class_table_key_index(self, cn, key):
599 table_name = '_%s'%cn
600 index_name = '_%s_%s_idx'%(cn, key)
601 if self.sql_index_exists(table_name, index_name):
602 sql = 'drop index '+index_name
603 self.sql(sql)
605 # and now the retired unique index too
606 index_name = '_%s_key_retired_idx'%cn
607 if self.sql_index_exists(table_name, index_name):
608 sql = 'drop index '+index_name
609 self.sql(sql)
611 def create_journal_table(self, spec):
612 """ create the journal table for a class given the spec and
613 already-determined cols
614 """
615 # journal table
616 cols = ','.join(['%s varchar'%x
617 for x in 'nodeid date tag action params'.split()])
618 sql = """create table %s__journal (
619 nodeid integer, date %s, tag varchar(255),
620 action varchar(255), params text)""" % (spec.classname,
621 self.hyperdb_to_sql_datatype(hyperdb.Date))
622 self.sql(sql)
623 self.create_journal_table_indexes(spec)
625 def create_journal_table_indexes(self, spec):
626 # index on nodeid
627 sql = 'create index %s_journ_idx on %s__journal(nodeid)'%(
628 spec.classname, spec.classname)
629 self.sql(sql)
631 def drop_journal_table_indexes(self, classname):
632 index_name = '%s_journ_idx'%classname
633 if not self.sql_index_exists('%s__journal'%classname, index_name):
634 return
635 index_sql = 'drop index '+index_name
636 self.sql(index_sql)
638 def create_multilink_table(self, spec, ml):
639 """ Create a multilink table for the "ml" property of the class
640 given by the spec
641 """
642 # create the table
643 sql = 'create table %s_%s (linkid INTEGER, nodeid INTEGER)'%(
644 spec.classname, ml)
645 self.sql(sql)
646 self.create_multilink_table_indexes(spec, ml)
648 def create_multilink_table_indexes(self, spec, ml):
649 # create index on linkid
650 index_sql = 'create index %s_%s_l_idx on %s_%s(linkid)'%(
651 spec.classname, ml, spec.classname, ml)
652 self.sql(index_sql)
654 # create index on nodeid
655 index_sql = 'create index %s_%s_n_idx on %s_%s(nodeid)'%(
656 spec.classname, ml, spec.classname, ml)
657 self.sql(index_sql)
659 def drop_multilink_table_indexes(self, classname, ml):
660 l = [
661 '%s_%s_l_idx'%(classname, ml),
662 '%s_%s_n_idx'%(classname, ml)
663 ]
664 table_name = '%s_%s'%(classname, ml)
665 for index_name in l:
666 if not self.sql_index_exists(table_name, index_name):
667 continue
668 index_sql = 'drop index %s'%index_name
669 self.sql(index_sql)
671 def create_class(self, spec):
672 """ Create a database table according to the given spec.
673 """
674 cols, mls = self.create_class_table(spec)
675 self.create_journal_table(spec)
677 # now create the multilink tables
678 for ml in mls:
679 self.create_multilink_table(spec, ml)
681 def drop_class(self, cn, spec):
682 """ Drop the given table from the database.
684 Drop the journal and multilink tables too.
685 """
686 properties = spec[1]
687 # figure the multilinks
688 mls = []
689 for propname, prop in properties:
690 if isinstance(prop, Multilink):
691 mls.append(propname)
693 # drop class table and indexes
694 self.drop_class_table_indexes(cn, spec[0])
696 self.drop_class_table(cn)
698 # drop journal table and indexes
699 self.drop_journal_table_indexes(cn)
700 sql = 'drop table %s__journal'%cn
701 self.sql(sql)
703 for ml in mls:
704 # drop multilink table and indexes
705 self.drop_multilink_table_indexes(cn, ml)
706 sql = 'drop table %s_%s'%(spec.classname, ml)
707 self.sql(sql)
709 def drop_class_table(self, cn):
710 sql = 'drop table _%s'%cn
711 self.sql(sql)
713 #
714 # Classes
715 #
716 def __getattr__(self, classname):
717 """ A convenient way of calling self.getclass(classname).
718 """
719 if self.classes.has_key(classname):
720 return self.classes[classname]
721 raise AttributeError, classname
723 def addclass(self, cl):
724 """ Add a Class to the hyperdatabase.
725 """
726 cn = cl.classname
727 if self.classes.has_key(cn):
728 raise ValueError, cn
729 self.classes[cn] = cl
731 # add default Edit and View permissions
732 self.security.addPermission(name="Create", klass=cn,
733 description="User is allowed to create "+cn)
734 self.security.addPermission(name="Edit", klass=cn,
735 description="User is allowed to edit "+cn)
736 self.security.addPermission(name="View", klass=cn,
737 description="User is allowed to access "+cn)
739 def getclasses(self):
740 """ Return a list of the names of all existing classes.
741 """
742 l = self.classes.keys()
743 l.sort()
744 return l
746 def getclass(self, classname):
747 """Get the Class object representing a particular class.
749 If 'classname' is not a valid class name, a KeyError is raised.
750 """
751 try:
752 return self.classes[classname]
753 except KeyError:
754 raise KeyError, 'There is no class called "%s"'%classname
756 def clear(self):
757 """Delete all database contents.
759 Note: I don't commit here, which is different behaviour to the
760 "nuke from orbit" behaviour in the dbs.
761 """
762 logging.getLogger('hyperdb').info('clear')
763 for cn in self.classes.keys():
764 sql = 'delete from _%s'%cn
765 self.sql(sql)
767 #
768 # Nodes
769 #
771 hyperdb_to_sql_value = {
772 hyperdb.String : str,
773 # fractional seconds by default
774 hyperdb.Date : lambda x: x.formal(sep=' ', sec='%06.3f'),
775 hyperdb.Link : int,
776 hyperdb.Interval : str,
777 hyperdb.Password : str,
778 hyperdb.Boolean : lambda x: x and 'TRUE' or 'FALSE',
779 hyperdb.Number : lambda x: x,
780 hyperdb.Multilink : lambda x: x, # used in journal marshalling
781 }
783 def to_sql_value(self, propklass):
785 fn = self.hyperdb_to_sql_value.get(propklass)
786 if fn:
787 return fn
789 for k, v in self.hyperdb_to_sql_value.iteritems():
790 if issubclass(propklass, k):
791 return v
793 raise ValueError, '%r is not a hyperdb property class' % propklass
795 def addnode(self, classname, nodeid, node):
796 """ Add the specified node to its class's db.
797 """
798 self.log_debug('addnode %s%s %r'%(classname,
799 nodeid, node))
801 # determine the column definitions and multilink tables
802 cl = self.classes[classname]
803 cols, mls = self.determine_columns(cl.properties.items())
805 # we'll be supplied these props if we're doing an import
806 values = node.copy()
807 if not values.has_key('creator'):
808 # add in the "calculated" properties (dupe so we don't affect
809 # calling code's node assumptions)
810 values['creation'] = values['activity'] = date.Date()
811 values['actor'] = values['creator'] = self.getuid()
813 cl = self.classes[classname]
814 props = cl.getprops(protected=1)
815 del props['id']
817 # default the non-multilink columns
818 for col, prop in props.items():
819 if not values.has_key(col):
820 if isinstance(prop, Multilink):
821 values[col] = []
822 else:
823 values[col] = None
825 # clear this node out of the cache if it's in there
826 key = (classname, nodeid)
827 if self.cache.has_key(key):
828 del self.cache[key]
829 self.cache_lru.remove(key)
831 # figure the values to insert
832 vals = []
833 for col,dt in cols:
834 # this is somewhat dodgy....
835 if col.endswith('_int__'):
836 # XXX eugh, this test suxxors
837 value = values[col[2:-6]]
838 # this is an Interval special "int" column
839 if value is not None:
840 vals.append(value.as_seconds())
841 else:
842 vals.append(value)
843 continue
845 prop = props[col[1:]]
846 value = values[col[1:]]
847 if value is not None:
848 value = self.to_sql_value(prop.__class__)(value)
849 vals.append(value)
850 vals.append(nodeid)
851 vals = tuple(vals)
853 # make sure the ordering is correct for column name -> column value
854 s = ','.join([self.arg for x in cols]) + ',%s'%self.arg
855 cols = ','.join([col for col,dt in cols]) + ',id'
857 # perform the inserts
858 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
859 self.sql(sql, vals)
861 # insert the multilink rows
862 for col in mls:
863 t = '%s_%s'%(classname, col)
864 for entry in node[col]:
865 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
866 self.arg, self.arg)
867 self.sql(sql, (entry, nodeid))
869 def setnode(self, classname, nodeid, values, multilink_changes={}):
870 """ Change the specified node.
871 """
872 self.log_debug('setnode %s%s %r'
873 % (classname, nodeid, values))
875 # clear this node out of the cache if it's in there
876 key = (classname, nodeid)
877 if self.cache.has_key(key):
878 del self.cache[key]
879 self.cache_lru.remove(key)
881 cl = self.classes[classname]
882 props = cl.getprops()
884 cols = []
885 mls = []
886 # add the multilinks separately
887 for col in values.keys():
888 prop = props[col]
889 if isinstance(prop, Multilink):
890 mls.append(col)
891 elif isinstance(prop, Interval):
892 # Intervals store the seconds value too
893 cols.append(col)
894 # extra leading '_' added by code below
895 cols.append('_' +col + '_int__')
896 else:
897 cols.append(col)
898 cols.sort()
900 # figure the values to insert
901 vals = []
902 for col in cols:
903 if col.endswith('_int__'):
904 # XXX eugh, this test suxxors
905 # Intervals store the seconds value too
906 col = col[1:-6]
907 prop = props[col]
908 value = values[col]
909 if value is None:
910 vals.append(None)
911 else:
912 vals.append(value.as_seconds())
913 else:
914 prop = props[col]
915 value = values[col]
916 if value is None:
917 e = None
918 else:
919 e = self.to_sql_value(prop.__class__)(value)
920 vals.append(e)
922 vals.append(int(nodeid))
923 vals = tuple(vals)
925 # if there's any updates to regular columns, do them
926 if cols:
927 # make sure the ordering is correct for column name -> column value
928 s = ','.join(['_%s=%s'%(x, self.arg) for x in cols])
929 cols = ','.join(cols)
931 # perform the update
932 sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
933 self.sql(sql, vals)
935 # we're probably coming from an import, not a change
936 if not multilink_changes:
937 for name in mls:
938 prop = props[name]
939 value = values[name]
941 t = '%s_%s'%(classname, name)
943 # clear out previous values for this node
944 # XXX numeric ids
945 self.sql('delete from %s where nodeid=%s'%(t, self.arg),
946 (nodeid,))
948 # insert the values for this node
949 for entry in values[name]:
950 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
951 self.arg, self.arg)
952 # XXX numeric ids
953 self.sql(sql, (entry, nodeid))
955 # we have multilink changes to apply
956 for col, (add, remove) in multilink_changes.items():
957 tn = '%s_%s'%(classname, col)
958 if add:
959 sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
960 self.arg, self.arg)
961 for addid in add:
962 # XXX numeric ids
963 self.sql(sql, (int(nodeid), int(addid)))
964 if remove:
965 s = ','.join([self.arg]*len(remove))
966 sql = 'delete from %s where nodeid=%s and linkid in (%s)'%(tn,
967 self.arg, s)
968 # XXX numeric ids
969 self.sql(sql, [int(nodeid)] + remove)
971 sql_to_hyperdb_value = {
972 hyperdb.String : str,
973 hyperdb.Date : lambda x:date.Date(str(x).replace(' ', '.')),
974 # hyperdb.Link : int, # XXX numeric ids
975 hyperdb.Link : str,
976 hyperdb.Interval : date.Interval,
977 hyperdb.Password : lambda x: password.Password(encrypted=x),
978 hyperdb.Boolean : _bool_cvt,
979 hyperdb.Number : _num_cvt,
980 hyperdb.Multilink : lambda x: x, # used in journal marshalling
981 }
983 def to_hyperdb_value(self, propklass):
985 fn = self.sql_to_hyperdb_value.get(propklass)
986 if fn:
987 return fn
989 for k, v in self.sql_to_hyperdb_value.iteritems():
990 if issubclass(propklass, k):
991 return v
993 raise ValueError, '%r is not a hyperdb property class' % propklass
995 def getnode(self, classname, nodeid):
996 """ Get a node from the database.
997 """
998 # see if we have this node cached
999 key = (classname, nodeid)
1000 if self.cache.has_key(key):
1001 # push us back to the top of the LRU
1002 self.cache_lru.remove(key)
1003 self.cache_lru.insert(0, key)
1004 if __debug__:
1005 self.stats['cache_hits'] += 1
1006 # return the cached information
1007 return self.cache[key]
1009 if __debug__:
1010 self.stats['cache_misses'] += 1
1011 start_t = time.time()
1013 # figure the columns we're fetching
1014 cl = self.classes[classname]
1015 cols, mls = self.determine_columns(cl.properties.items())
1016 scols = ','.join([col for col,dt in cols])
1018 # perform the basic property fetch
1019 sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
1020 self.sql(sql, (nodeid,))
1022 values = self.sql_fetchone()
1023 if values is None:
1024 raise IndexError, 'no such %s node %s'%(classname, nodeid)
1026 # make up the node
1027 node = {}
1028 props = cl.getprops(protected=1)
1029 for col in range(len(cols)):
1030 name = cols[col][0][1:]
1031 if name.endswith('_int__'):
1032 # XXX eugh, this test suxxors
1033 # ignore the special Interval-as-seconds column
1034 continue
1035 value = values[col]
1036 if value is not None:
1037 value = self.to_hyperdb_value(props[name].__class__)(value)
1038 node[name] = value
1041 # now the multilinks
1042 for col in mls:
1043 # get the link ids
1044 sql = 'select linkid from %s_%s where nodeid=%s'%(classname, col,
1045 self.arg)
1046 self.sql(sql, (nodeid,))
1047 # extract the first column from the result
1048 # XXX numeric ids
1049 items = [int(x[0]) for x in self.cursor.fetchall()]
1050 items.sort ()
1051 node[col] = [str(x) for x in items]
1053 # save off in the cache
1054 key = (classname, nodeid)
1055 self.cache[key] = node
1056 # update the LRU
1057 self.cache_lru.insert(0, key)
1058 if len(self.cache_lru) > self.cache_size:
1059 del self.cache[self.cache_lru.pop()]
1061 if __debug__:
1062 self.stats['get_items'] += (time.time() - start_t)
1064 return node
1066 def destroynode(self, classname, nodeid):
1067 """Remove a node from the database. Called exclusively by the
1068 destroy() method on Class.
1069 """
1070 logging.getLogger('hyperdb').info('destroynode %s%s'%(classname, nodeid))
1072 # make sure the node exists
1073 if not self.hasnode(classname, nodeid):
1074 raise IndexError, '%s has no node %s'%(classname, nodeid)
1076 # see if we have this node cached
1077 if self.cache.has_key((classname, nodeid)):
1078 del self.cache[(classname, nodeid)]
1080 # see if there's any obvious commit actions that we should get rid of
1081 for entry in self.transactions[:]:
1082 if entry[1][:2] == (classname, nodeid):
1083 self.transactions.remove(entry)
1085 # now do the SQL
1086 sql = 'delete from _%s where id=%s'%(classname, self.arg)
1087 self.sql(sql, (nodeid,))
1089 # remove from multilnks
1090 cl = self.getclass(classname)
1091 x, mls = self.determine_columns(cl.properties.items())
1092 for col in mls:
1093 # get the link ids
1094 sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
1095 self.sql(sql, (nodeid,))
1097 # remove journal entries
1098 sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
1099 self.sql(sql, (nodeid,))
1101 # cleanup any blob filestorage when we commit
1102 self.transactions.append((FileStorage.destroy, (self, classname, nodeid)))
1104 def hasnode(self, classname, nodeid):
1105 """ Determine if the database has a given node.
1106 """
1107 # If this node is in the cache, then we do not need to go to
1108 # the database. (We don't consider this an LRU hit, though.)
1109 if self.cache.has_key((classname, nodeid)):
1110 # Return 1, not True, to match the type of the result of
1111 # the SQL operation below.
1112 return 1
1113 sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
1114 self.sql(sql, (nodeid,))
1115 return int(self.cursor.fetchone()[0])
1117 def countnodes(self, classname):
1118 """ Count the number of nodes that exist for a particular Class.
1119 """
1120 sql = 'select count(*) from _%s'%classname
1121 self.sql(sql)
1122 return self.cursor.fetchone()[0]
1124 def addjournal(self, classname, nodeid, action, params, creator=None,
1125 creation=None):
1126 """ Journal the Action
1127 'action' may be:
1129 'create' or 'set' -- 'params' is a dictionary of property values
1130 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
1131 'retire' -- 'params' is None
1132 """
1133 # handle supply of the special journalling parameters (usually
1134 # supplied on importing an existing database)
1135 if creator:
1136 journaltag = creator
1137 else:
1138 journaltag = self.getuid()
1139 if creation:
1140 journaldate = creation
1141 else:
1142 journaldate = date.Date()
1144 # create the journal entry
1145 cols = 'nodeid,date,tag,action,params'
1147 self.log_debug('addjournal %s%s %r %s %s %r'%(classname,
1148 nodeid, journaldate, journaltag, action, params))
1150 # make the journalled data marshallable
1151 if isinstance(params, type({})):
1152 self._journal_marshal(params, classname)
1154 params = repr(params)
1156 dc = self.to_sql_value(hyperdb.Date)
1157 journaldate = dc(journaldate)
1159 self.save_journal(classname, cols, nodeid, journaldate,
1160 journaltag, action, params)
1162 def setjournal(self, classname, nodeid, journal):
1163 """Set the journal to the "journal" list."""
1164 # clear out any existing entries
1165 self.sql('delete from %s__journal where nodeid=%s'%(classname,
1166 self.arg), (nodeid,))
1168 # create the journal entry
1169 cols = 'nodeid,date,tag,action,params'
1171 dc = self.to_sql_value(hyperdb.Date)
1172 for nodeid, journaldate, journaltag, action, params in journal:
1173 self.log_debug('addjournal %s%s %r %s %s %r'%(
1174 classname, nodeid, journaldate, journaltag, action,
1175 params))
1177 # make the journalled data marshallable
1178 if isinstance(params, type({})):
1179 self._journal_marshal(params, classname)
1180 params = repr(params)
1182 self.save_journal(classname, cols, nodeid, dc(journaldate),
1183 journaltag, action, params)
1185 def _journal_marshal(self, params, classname):
1186 """Convert the journal params values into safely repr'able and
1187 eval'able values."""
1188 properties = self.getclass(classname).getprops()
1189 for param, value in params.items():
1190 if not value:
1191 continue
1192 property = properties[param]
1193 cvt = self.to_sql_value(property.__class__)
1194 if isinstance(property, Password):
1195 params[param] = cvt(value)
1196 elif isinstance(property, Date):
1197 params[param] = cvt(value)
1198 elif isinstance(property, Interval):
1199 params[param] = cvt(value)
1200 elif isinstance(property, Boolean):
1201 params[param] = cvt(value)
1203 def getjournal(self, classname, nodeid):
1204 """ get the journal for id
1205 """
1206 # make sure the node exists
1207 if not self.hasnode(classname, nodeid):
1208 raise IndexError, '%s has no node %s'%(classname, nodeid)
1210 cols = ','.join('nodeid date tag action params'.split())
1211 journal = self.load_journal(classname, cols, nodeid)
1213 # now unmarshal the data
1214 dc = self.to_hyperdb_value(hyperdb.Date)
1215 res = []
1216 properties = self.getclass(classname).getprops()
1217 for nodeid, date_stamp, user, action, params in journal:
1218 params = eval(params)
1219 if isinstance(params, type({})):
1220 for param, value in params.items():
1221 if not value:
1222 continue
1223 property = properties.get(param, None)
1224 if property is None:
1225 # deleted property
1226 continue
1227 cvt = self.to_hyperdb_value(property.__class__)
1228 if isinstance(property, Password):
1229 params[param] = cvt(value)
1230 elif isinstance(property, Date):
1231 params[param] = cvt(value)
1232 elif isinstance(property, Interval):
1233 params[param] = cvt(value)
1234 elif isinstance(property, Boolean):
1235 params[param] = cvt(value)
1236 # XXX numeric ids
1237 res.append((str(nodeid), dc(date_stamp), user, action, params))
1238 return res
1240 def save_journal(self, classname, cols, nodeid, journaldate,
1241 journaltag, action, params):
1242 """ Save the journal entry to the database
1243 """
1244 entry = (nodeid, journaldate, journaltag, action, params)
1246 # do the insert
1247 a = self.arg
1248 sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(
1249 classname, cols, a, a, a, a, a)
1250 self.sql(sql, entry)
1252 def load_journal(self, classname, cols, nodeid):
1253 """ Load the journal from the database
1254 """
1255 # now get the journal entries
1256 sql = 'select %s from %s__journal where nodeid=%s order by date'%(
1257 cols, classname, self.arg)
1258 self.sql(sql, (nodeid,))
1259 return self.cursor.fetchall()
1261 def pack(self, pack_before):
1262 """ Delete all journal entries except "create" before 'pack_before'.
1263 """
1264 date_stamp = self.to_sql_value(Date)(pack_before)
1266 # do the delete
1267 for classname in self.classes.keys():
1268 sql = "delete from %s__journal where date<%s and "\
1269 "action<>'create'"%(classname, self.arg)
1270 self.sql(sql, (date_stamp,))
1272 def sql_commit(self, fail_ok=False):
1273 """ Actually commit to the database.
1274 """
1275 logging.getLogger('hyperdb').info('commit')
1277 self.conn.commit()
1279 # open a new cursor for subsequent work
1280 self.cursor = self.conn.cursor()
1282 def commit(self, fail_ok=False):
1283 """ Commit the current transactions.
1285 Save all data changed since the database was opened or since the
1286 last commit() or rollback().
1288 fail_ok indicates that the commit is allowed to fail. This is used
1289 in the web interface when committing cleaning of the session
1290 database. We don't care if there's a concurrency issue there.
1292 The only backend this seems to affect is postgres.
1293 """
1294 # commit the database
1295 self.sql_commit(fail_ok)
1297 # now, do all the other transaction stuff
1298 for method, args in self.transactions:
1299 method(*args)
1301 # save the indexer
1302 self.indexer.save_index()
1304 # clear out the transactions
1305 self.transactions = []
1307 def sql_rollback(self):
1308 self.conn.rollback()
1310 def rollback(self):
1311 """ Reverse all actions from the current transaction.
1313 Undo all the changes made since the database was opened or the last
1314 commit() or rollback() was performed.
1315 """
1316 logging.getLogger('hyperdb').info('rollback')
1318 self.sql_rollback()
1320 # roll back "other" transaction stuff
1321 for method, args in self.transactions:
1322 # delete temporary files
1323 if method == self.doStoreFile:
1324 self.rollbackStoreFile(*args)
1325 self.transactions = []
1327 # clear the cache
1328 self.clearCache()
1330 def sql_close(self):
1331 logging.getLogger('hyperdb').info('close')
1332 self.conn.close()
1334 def close(self):
1335 """ Close off the connection.
1336 """
1337 self.indexer.close()
1338 self.sql_close()
1340 #
1341 # The base Class class
1342 #
1343 class Class(hyperdb.Class):
1344 """ The handle to a particular class of nodes in a hyperdatabase.
1346 All methods except __repr__ and getnode must be implemented by a
1347 concrete backend Class.
1348 """
1350 def schema(self):
1351 """ A dumpable version of the schema that we can store in the
1352 database
1353 """
1354 return (self.key, [(x, repr(y)) for x,y in self.properties.items()])
1356 def enableJournalling(self):
1357 """Turn journalling on for this class
1358 """
1359 self.do_journal = 1
1361 def disableJournalling(self):
1362 """Turn journalling off for this class
1363 """
1364 self.do_journal = 0
1366 # Editing nodes:
1367 def create(self, **propvalues):
1368 """ Create a new node of this class and return its id.
1370 The keyword arguments in 'propvalues' map property names to values.
1372 The values of arguments must be acceptable for the types of their
1373 corresponding properties or a TypeError is raised.
1375 If this class has a key property, it must be present and its value
1376 must not collide with other key strings or a ValueError is raised.
1378 Any other properties on this class that are missing from the
1379 'propvalues' dictionary are set to None.
1381 If an id in a link or multilink property does not refer to a valid
1382 node, an IndexError is raised.
1383 """
1384 self.fireAuditors('create', None, propvalues)
1385 newid = self.create_inner(**propvalues)
1386 self.fireReactors('create', newid, None)
1387 return newid
1389 def create_inner(self, **propvalues):
1390 """ Called by create, in-between the audit and react calls.
1391 """
1392 if propvalues.has_key('id'):
1393 raise KeyError, '"id" is reserved'
1395 if self.db.journaltag is None:
1396 raise DatabaseError, _('Database open read-only')
1398 if propvalues.has_key('creator') or propvalues.has_key('actor') or \
1399 propvalues.has_key('creation') or propvalues.has_key('activity'):
1400 raise KeyError, '"creator", "actor", "creation" and '\
1401 '"activity" are reserved'
1403 # new node's id
1404 newid = self.db.newid(self.classname)
1406 # validate propvalues
1407 num_re = re.compile('^\d+$')
1408 for key, value in propvalues.items():
1409 if key == self.key:
1410 try:
1411 self.lookup(value)
1412 except KeyError:
1413 pass
1414 else:
1415 raise ValueError, 'node with key "%s" exists'%value
1417 # try to handle this property
1418 try:
1419 prop = self.properties[key]
1420 except KeyError:
1421 raise KeyError, '"%s" has no property "%s"'%(self.classname,
1422 key)
1424 if value is not None and isinstance(prop, Link):
1425 if type(value) != type(''):
1426 raise ValueError, 'link value must be String'
1427 link_class = self.properties[key].classname
1428 # if it isn't a number, it's a key
1429 if not num_re.match(value):
1430 try:
1431 value = self.db.classes[link_class].lookup(value)
1432 except (TypeError, KeyError):
1433 raise IndexError, 'new property "%s": %s not a %s'%(
1434 key, value, link_class)
1435 elif not self.db.getclass(link_class).hasnode(value):
1436 raise IndexError, '%s has no node %s'%(link_class, value)
1438 # save off the value
1439 propvalues[key] = value
1441 # register the link with the newly linked node
1442 if self.do_journal and self.properties[key].do_journal:
1443 self.db.addjournal(link_class, value, 'link',
1444 (self.classname, newid, key))
1446 elif isinstance(prop, Multilink):
1447 if value is None:
1448 value = []
1449 if not hasattr(value, '__iter__'):
1450 raise TypeError, 'new property "%s" not an iterable of ids'%key
1452 # clean up and validate the list of links
1453 link_class = self.properties[key].classname
1454 l = []
1455 for entry in value:
1456 if type(entry) != type(''):
1457 raise ValueError, '"%s" multilink value (%r) '\
1458 'must contain Strings'%(key, value)
1459 # if it isn't a number, it's a key
1460 if not num_re.match(entry):
1461 try:
1462 entry = self.db.classes[link_class].lookup(entry)
1463 except (TypeError, KeyError):
1464 raise IndexError, 'new property "%s": %s not a %s'%(
1465 key, entry, self.properties[key].classname)
1466 l.append(entry)
1467 value = l
1468 propvalues[key] = value
1470 # handle additions
1471 for nodeid in value:
1472 if not self.db.getclass(link_class).hasnode(nodeid):
1473 raise IndexError, '%s has no node %s'%(link_class,
1474 nodeid)
1475 # register the link with the newly linked node
1476 if self.do_journal and self.properties[key].do_journal:
1477 self.db.addjournal(link_class, nodeid, 'link',
1478 (self.classname, newid, key))
1480 elif isinstance(prop, String):
1481 if type(value) != type('') and type(value) != type(u''):
1482 raise TypeError, 'new property "%s" not a string'%key
1483 if prop.indexme:
1484 self.db.indexer.add_text((self.classname, newid, key),
1485 value)
1487 elif isinstance(prop, Password):
1488 if not isinstance(value, password.Password):
1489 raise TypeError, 'new property "%s" not a Password'%key
1491 elif isinstance(prop, Date):
1492 if value is not None and not isinstance(value, date.Date):
1493 raise TypeError, 'new property "%s" not a Date'%key
1495 elif isinstance(prop, Interval):
1496 if value is not None and not isinstance(value, date.Interval):
1497 raise TypeError, 'new property "%s" not an Interval'%key
1499 elif value is not None and isinstance(prop, Number):
1500 try:
1501 float(value)
1502 except ValueError:
1503 raise TypeError, 'new property "%s" not numeric'%key
1505 elif value is not None and isinstance(prop, Boolean):
1506 try:
1507 int(value)
1508 except ValueError:
1509 raise TypeError, 'new property "%s" not boolean'%key
1511 # make sure there's data where there needs to be
1512 for key, prop in self.properties.items():
1513 if propvalues.has_key(key):
1514 continue
1515 if key == self.key:
1516 raise ValueError, 'key property "%s" is required'%key
1517 if isinstance(prop, Multilink):
1518 propvalues[key] = []
1519 else:
1520 propvalues[key] = None
1522 # done
1523 self.db.addnode(self.classname, newid, propvalues)
1524 if self.do_journal:
1525 self.db.addjournal(self.classname, newid, ''"create", {})
1527 # XXX numeric ids
1528 return str(newid)
1530 def get(self, nodeid, propname, default=_marker, cache=1):
1531 """Get the value of a property on an existing node of this class.
1533 'nodeid' must be the id of an existing node of this class or an
1534 IndexError is raised. 'propname' must be the name of a property
1535 of this class or a KeyError is raised.
1537 'cache' exists for backwards compatibility, and is not used.
1538 """
1539 if propname == 'id':
1540 return nodeid
1542 # get the node's dict
1543 d = self.db.getnode(self.classname, nodeid)
1545 if propname == 'creation':
1546 if d.has_key('creation'):
1547 return d['creation']
1548 else:
1549 return date.Date()
1550 if propname == 'activity':
1551 if d.has_key('activity'):
1552 return d['activity']
1553 else:
1554 return date.Date()
1555 if propname == 'creator':
1556 if d.has_key('creator'):
1557 return d['creator']
1558 else:
1559 return self.db.getuid()
1560 if propname == 'actor':
1561 if d.has_key('actor'):
1562 return d['actor']
1563 else:
1564 return self.db.getuid()
1566 # get the property (raises KeyErorr if invalid)
1567 prop = self.properties[propname]
1569 # XXX may it be that propname is valid property name
1570 # (above error is not raised) and not d.has_key(propname)???
1571 if (not d.has_key(propname)) or (d[propname] is None):
1572 if default is _marker:
1573 if isinstance(prop, Multilink):
1574 return []
1575 else:
1576 return None
1577 else:
1578 return default
1580 # don't pass our list to other code
1581 if isinstance(prop, Multilink):
1582 return d[propname][:]
1584 return d[propname]
1586 def set(self, nodeid, **propvalues):
1587 """Modify a property on an existing node of this class.
1589 'nodeid' must be the id of an existing node of this class or an
1590 IndexError is raised.
1592 Each key in 'propvalues' must be the name of a property of this
1593 class or a KeyError is raised.
1595 All values in 'propvalues' must be acceptable types for their
1596 corresponding properties or a TypeError is raised.
1598 If the value of the key property is set, it must not collide with
1599 other key strings or a ValueError is raised.
1601 If the value of a Link or Multilink property contains an invalid
1602 node id, a ValueError is raised.
1603 """
1604 self.fireAuditors('set', nodeid, propvalues)
1605 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1606 propvalues = self.set_inner(nodeid, **propvalues)
1607 self.fireReactors('set', nodeid, oldvalues)
1608 return propvalues
1610 def set_inner(self, nodeid, **propvalues):
1611 """ Called by set, in-between the audit and react calls.
1612 """
1613 if not propvalues:
1614 return propvalues
1616 if propvalues.has_key('creation') or propvalues.has_key('creator') or \
1617 propvalues.has_key('actor') or propvalues.has_key('activity'):
1618 raise KeyError, '"creation", "creator", "actor" and '\
1619 '"activity" are reserved'
1621 if propvalues.has_key('id'):
1622 raise KeyError, '"id" is reserved'
1624 if self.db.journaltag is None:
1625 raise DatabaseError, _('Database open read-only')
1627 node = self.db.getnode(self.classname, nodeid)
1628 if self.is_retired(nodeid):
1629 raise IndexError, 'Requested item is retired'
1630 num_re = re.compile('^\d+$')
1632 # make a copy of the values dictionary - we'll modify the contents
1633 propvalues = propvalues.copy()
1635 # if the journal value is to be different, store it in here
1636 journalvalues = {}
1638 # remember the add/remove stuff for multilinks, making it easier
1639 # for the Database layer to do its stuff
1640 multilink_changes = {}
1642 for propname, value in propvalues.items():
1643 # check to make sure we're not duplicating an existing key
1644 if propname == self.key and node[propname] != value:
1645 try:
1646 self.lookup(value)
1647 except KeyError:
1648 pass
1649 else:
1650 raise ValueError, 'node with key "%s" exists'%value
1652 # this will raise the KeyError if the property isn't valid
1653 # ... we don't use getprops() here because we only care about
1654 # the writeable properties.
1655 try:
1656 prop = self.properties[propname]
1657 except KeyError:
1658 raise KeyError, '"%s" has no property named "%s"'%(
1659 self.classname, propname)
1661 # if the value's the same as the existing value, no sense in
1662 # doing anything
1663 current = node.get(propname, None)
1664 if value == current:
1665 del propvalues[propname]
1666 continue
1667 journalvalues[propname] = current
1669 # do stuff based on the prop type
1670 if isinstance(prop, Link):
1671 link_class = prop.classname
1672 # if it isn't a number, it's a key
1673 if value is not None and not isinstance(value, type('')):
1674 raise ValueError, 'property "%s" link value be a string'%(
1675 propname)
1676 if isinstance(value, type('')) and not num_re.match(value):
1677 try:
1678 value = self.db.classes[link_class].lookup(value)
1679 except (TypeError, KeyError):
1680 raise IndexError, 'new property "%s": %s not a %s'%(
1681 propname, value, prop.classname)
1683 if (value is not None and
1684 not self.db.getclass(link_class).hasnode(value)):
1685 raise IndexError, '%s has no node %s'%(link_class, value)
1687 if self.do_journal and prop.do_journal:
1688 # register the unlink with the old linked node
1689 if node[propname] is not None:
1690 self.db.addjournal(link_class, node[propname],
1691 ''"unlink", (self.classname, nodeid, propname))
1693 # register the link with the newly linked node
1694 if value is not None:
1695 self.db.addjournal(link_class, value, ''"link",
1696 (self.classname, nodeid, propname))
1698 elif isinstance(prop, Multilink):
1699 if value is None:
1700 value = []
1701 if not hasattr(value, '__iter__'):
1702 raise TypeError, 'new property "%s" not an iterable of'\
1703 ' ids'%propname
1704 link_class = self.properties[propname].classname
1705 l = []
1706 for entry in value:
1707 # if it isn't a number, it's a key
1708 if type(entry) != type(''):
1709 raise ValueError, 'new property "%s" link value ' \
1710 'must be a string'%propname
1711 if not num_re.match(entry):
1712 try:
1713 entry = self.db.classes[link_class].lookup(entry)
1714 except (TypeError, KeyError):
1715 raise IndexError, 'new property "%s": %s not a %s'%(
1716 propname, entry,
1717 self.properties[propname].classname)
1718 l.append(entry)
1719 value = l
1720 propvalues[propname] = value
1722 # figure the journal entry for this property
1723 add = []
1724 remove = []
1726 # handle removals
1727 if node.has_key(propname):
1728 l = node[propname]
1729 else:
1730 l = []
1731 for id in l[:]:
1732 if id in value:
1733 continue
1734 # register the unlink with the old linked node
1735 if self.do_journal and self.properties[propname].do_journal:
1736 self.db.addjournal(link_class, id, 'unlink',
1737 (self.classname, nodeid, propname))
1738 l.remove(id)
1739 remove.append(id)
1741 # handle additions
1742 for id in value:
1743 if not self.db.getclass(link_class).hasnode(id):
1744 raise IndexError, '%s has no node %s'%(link_class, id)
1745 if id in l:
1746 continue
1747 # register the link with the newly linked node
1748 if self.do_journal and self.properties[propname].do_journal:
1749 self.db.addjournal(link_class, id, 'link',
1750 (self.classname, nodeid, propname))
1751 l.append(id)
1752 add.append(id)
1754 # figure the journal entry
1755 l = []
1756 if add:
1757 l.append(('+', add))
1758 if remove:
1759 l.append(('-', remove))
1760 multilink_changes[propname] = (add, remove)
1761 if l:
1762 journalvalues[propname] = tuple(l)
1764 elif isinstance(prop, String):
1765 if value is not None and type(value) != type('') and type(value) != type(u''):
1766 raise TypeError, 'new property "%s" not a string'%propname
1767 if prop.indexme:
1768 if value is None: value = ''
1769 self.db.indexer.add_text((self.classname, nodeid, propname),
1770 value)
1772 elif isinstance(prop, Password):
1773 if not isinstance(value, password.Password):
1774 raise TypeError, 'new property "%s" not a Password'%propname
1775 propvalues[propname] = value
1777 elif value is not None and isinstance(prop, Date):
1778 if not isinstance(value, date.Date):
1779 raise TypeError, 'new property "%s" not a Date'% propname
1780 propvalues[propname] = value
1782 elif value is not None and isinstance(prop, Interval):
1783 if not isinstance(value, date.Interval):
1784 raise TypeError, 'new property "%s" not an '\
1785 'Interval'%propname
1786 propvalues[propname] = value
1788 elif value is not None and isinstance(prop, Number):
1789 try:
1790 float(value)
1791 except ValueError:
1792 raise TypeError, 'new property "%s" not numeric'%propname
1794 elif value is not None and isinstance(prop, Boolean):
1795 try:
1796 int(value)
1797 except ValueError:
1798 raise TypeError, 'new property "%s" not boolean'%propname
1800 # nothing to do?
1801 if not propvalues:
1802 return propvalues
1804 # update the activity time
1805 propvalues['activity'] = date.Date()
1806 propvalues['actor'] = self.db.getuid()
1808 # do the set
1809 self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
1811 # remove the activity props now they're handled
1812 del propvalues['activity']
1813 del propvalues['actor']
1815 # journal the set
1816 if self.do_journal:
1817 self.db.addjournal(self.classname, nodeid, ''"set", journalvalues)
1819 return propvalues
1821 def retire(self, nodeid):
1822 """Retire a node.
1824 The properties on the node remain available from the get() method,
1825 and the node's id is never reused.
1827 Retired nodes are not returned by the find(), list(), or lookup()
1828 methods, and other nodes may reuse the values of their key properties.
1829 """
1830 if self.db.journaltag is None:
1831 raise DatabaseError, _('Database open read-only')
1833 self.fireAuditors('retire', nodeid, None)
1835 # use the arg for __retired__ to cope with any odd database type
1836 # conversion (hello, sqlite)
1837 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1838 self.db.arg, self.db.arg)
1839 self.db.sql(sql, (nodeid, nodeid))
1840 if self.do_journal:
1841 self.db.addjournal(self.classname, nodeid, ''"retired", None)
1843 self.fireReactors('retire', nodeid, None)
1845 def restore(self, nodeid):
1846 """Restore a retired node.
1848 Make node available for all operations like it was before retirement.
1849 """
1850 if self.db.journaltag is None:
1851 raise DatabaseError, _('Database open read-only')
1853 node = self.db.getnode(self.classname, nodeid)
1854 # check if key property was overrided
1855 key = self.getkey()
1856 try:
1857 id = self.lookup(node[key])
1858 except KeyError:
1859 pass
1860 else:
1861 raise KeyError, "Key property (%s) of retired node clashes with \
1862 existing one (%s)" % (key, node[key])
1864 self.fireAuditors('restore', nodeid, None)
1865 # use the arg for __retired__ to cope with any odd database type
1866 # conversion (hello, sqlite)
1867 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1868 self.db.arg, self.db.arg)
1869 self.db.sql(sql, (0, nodeid))
1870 if self.do_journal:
1871 self.db.addjournal(self.classname, nodeid, ''"restored", None)
1873 self.fireReactors('restore', nodeid, None)
1875 def is_retired(self, nodeid):
1876 """Return true if the node is rerired
1877 """
1878 sql = 'select __retired__ from _%s where id=%s'%(self.classname,
1879 self.db.arg)
1880 self.db.sql(sql, (nodeid,))
1881 return int(self.db.sql_fetchone()[0]) > 0
1883 def destroy(self, nodeid):
1884 """Destroy a node.
1886 WARNING: this method should never be used except in extremely rare
1887 situations where there could never be links to the node being
1888 deleted
1890 WARNING: use retire() instead
1892 WARNING: the properties of this node will not be available ever again
1894 WARNING: really, use retire() instead
1896 Well, I think that's enough warnings. This method exists mostly to
1897 support the session storage of the cgi interface.
1899 The node is completely removed from the hyperdb, including all journal
1900 entries. It will no longer be available, and will generally break code
1901 if there are any references to the node.
1902 """
1903 if self.db.journaltag is None:
1904 raise DatabaseError, _('Database open read-only')
1905 self.db.destroynode(self.classname, nodeid)
1907 def history(self, nodeid):
1908 """Retrieve the journal of edits on a particular node.
1910 'nodeid' must be the id of an existing node of this class or an
1911 IndexError is raised.
1913 The returned list contains tuples of the form
1915 (nodeid, date, tag, action, params)
1917 'date' is a Timestamp object specifying the time of the change and
1918 'tag' is the journaltag specified when the database was opened.
1919 """
1920 if not self.do_journal:
1921 raise ValueError, 'Journalling is disabled for this class'
1922 return self.db.getjournal(self.classname, nodeid)
1924 # Locating nodes:
1925 def hasnode(self, nodeid):
1926 """Determine if the given nodeid actually exists
1927 """
1928 return self.db.hasnode(self.classname, nodeid)
1930 def setkey(self, propname):
1931 """Select a String property of this class to be the key property.
1933 'propname' must be the name of a String property of this class or
1934 None, or a TypeError is raised. The values of the key property on
1935 all existing nodes must be unique or a ValueError is raised.
1936 """
1937 prop = self.getprops()[propname]
1938 if not isinstance(prop, String):
1939 raise TypeError, 'key properties must be String'
1940 self.key = propname
1942 def getkey(self):
1943 """Return the name of the key property for this class or None."""
1944 return self.key
1946 def lookup(self, keyvalue):
1947 """Locate a particular node by its key property and return its id.
1949 If this class has no key property, a TypeError is raised. If the
1950 'keyvalue' matches one of the values for the key property among
1951 the nodes in this class, the matching node's id is returned;
1952 otherwise a KeyError is raised.
1953 """
1954 if not self.key:
1955 raise TypeError, 'No key property set for class %s'%self.classname
1957 # use the arg to handle any odd database type conversion (hello,
1958 # sqlite)
1959 sql = "select id from _%s where _%s=%s and __retired__=%s"%(
1960 self.classname, self.key, self.db.arg, self.db.arg)
1961 self.db.sql(sql, (str(keyvalue), 0))
1963 # see if there was a result that's not retired
1964 row = self.db.sql_fetchone()
1965 if not row:
1966 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1967 keyvalue, self.classname)
1969 # return the id
1970 # XXX numeric ids
1971 return str(row[0])
1973 def find(self, **propspec):
1974 """Get the ids of nodes in this class which link to the given nodes.
1976 'propspec' consists of keyword args propname=nodeid or
1977 propname={nodeid:1, }
1978 'propname' must be the name of a property in this class, or a
1979 KeyError is raised. That property must be a Link or
1980 Multilink property, or a TypeError is raised.
1982 Any node in this class whose 'propname' property links to any of
1983 the nodeids will be returned. Examples::
1985 db.issue.find(messages='1')
1986 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1987 """
1988 # shortcut
1989 if not propspec:
1990 return []
1992 # validate the args
1993 props = self.getprops()
1994 propspec = propspec.items()
1995 for propname, nodeids in propspec:
1996 # check the prop is OK
1997 prop = props[propname]
1998 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1999 raise TypeError, "'%s' not a Link/Multilink property"%propname
2001 # first, links
2002 a = self.db.arg
2003 allvalues = ()
2004 sql = []
2005 where = []
2006 for prop, values in propspec:
2007 if not isinstance(props[prop], hyperdb.Link):
2008 continue
2009 if type(values) is type({}) and len(values) == 1:
2010 values = values.keys()[0]
2011 if type(values) is type(''):
2012 allvalues += (values,)
2013 where.append('_%s = %s'%(prop, a))
2014 elif values is None:
2015 where.append('_%s is NULL'%prop)
2016 else:
2017 values = values.keys()
2018 s = ''
2019 if None in values:
2020 values.remove(None)
2021 s = '_%s is NULL or '%prop
2022 allvalues += tuple(values)
2023 s += '_%s in (%s)'%(prop, ','.join([a]*len(values)))
2024 where.append('(' + s +')')
2025 if where:
2026 allvalues = (0, ) + allvalues
2027 sql.append("""select id from _%s where __retired__=%s
2028 and %s"""%(self.classname, a, ' and '.join(where)))
2030 # now multilinks
2031 for prop, values in propspec:
2032 if not isinstance(props[prop], hyperdb.Multilink):
2033 continue
2034 if not values:
2035 continue
2036 allvalues += (0, )
2037 if type(values) is type(''):
2038 allvalues += (values,)
2039 s = a
2040 else:
2041 allvalues += tuple(values.keys())
2042 s = ','.join([a]*len(values))
2043 tn = '%s_%s'%(self.classname, prop)
2044 sql.append("""select id from _%s, %s where __retired__=%s
2045 and id = %s.nodeid and %s.linkid in (%s)"""%(self.classname,
2046 tn, a, tn, tn, s))
2048 if not sql:
2049 return []
2050 sql = ' union '.join(sql)
2051 self.db.sql(sql, allvalues)
2052 # XXX numeric ids
2053 l = [str(x[0]) for x in self.db.sql_fetchall()]
2054 return l
2056 def stringFind(self, **requirements):
2057 """Locate a particular node by matching a set of its String
2058 properties in a caseless search.
2060 If the property is not a String property, a TypeError is raised.
2062 The return is a list of the id of all nodes that match.
2063 """
2064 where = []
2065 args = []
2066 for propname in requirements.keys():
2067 prop = self.properties[propname]
2068 if not isinstance(prop, String):
2069 raise TypeError, "'%s' not a String property"%propname
2070 where.append(propname)
2071 args.append(requirements[propname].lower())
2073 # generate the where clause
2074 s = ' and '.join(['lower(_%s)=%s'%(col, self.db.arg) for col in where])
2075 sql = 'select id from _%s where %s and __retired__=%s'%(
2076 self.classname, s, self.db.arg)
2077 args.append(0)
2078 self.db.sql(sql, tuple(args))
2079 # XXX numeric ids
2080 l = [str(x[0]) for x in self.db.sql_fetchall()]
2081 return l
2083 def list(self):
2084 """ Return a list of the ids of the active nodes in this class.
2085 """
2086 return self.getnodeids(retired=0)
2088 def getnodeids(self, retired=None):
2089 """ Retrieve all the ids of the nodes for a particular Class.
2091 Set retired=None to get all nodes. Otherwise it'll get all the
2092 retired or non-retired nodes, depending on the flag.
2093 """
2094 # flip the sense of the 'retired' flag if we don't want all of them
2095 if retired is not None:
2096 args = (0, )
2097 if retired:
2098 compare = '>'
2099 else:
2100 compare = '='
2101 sql = 'select id from _%s where __retired__%s%s'%(self.classname,
2102 compare, self.db.arg)
2103 else:
2104 args = ()
2105 sql = 'select id from _%s'%self.classname
2106 self.db.sql(sql, args)
2107 # XXX numeric ids
2108 ids = [str(x[0]) for x in self.db.cursor.fetchall()]
2109 return ids
2111 def _subselect(self, classname, multilink_table):
2112 """Create a subselect. This is factored out because some
2113 databases (hmm only one, so far) doesn't support subselects
2114 look for "I can't believe it's not a toy RDBMS" in the mysql
2115 backend.
2116 """
2117 return '_%s.id not in (select nodeid from %s)'%(classname,
2118 multilink_table)
2120 # Some DBs order NULL values last. Set this variable in the backend
2121 # for prepending an order by clause for each attribute that causes
2122 # correct sort order for NULLs. Examples:
2123 # order_by_null_values = '(%s is not NULL)'
2124 # order_by_null_values = 'notnull(%s)'
2125 # The format parameter is replaced with the attribute.
2126 order_by_null_values = None
2128 def filter(self, search_matches, filterspec, sort=[], group=[]):
2129 """Return a list of the ids of the active nodes in this class that
2130 match the 'filter' spec, sorted by the group spec and then the
2131 sort spec
2133 "filterspec" is {propname: value(s)}
2135 "sort" and "group" are [(dir, prop), ...] where dir is '+', '-'
2136 or None and prop is a prop name or None. Note that for
2137 backward-compatibility reasons a single (dir, prop) tuple is
2138 also allowed.
2140 "search_matches" is a container type or None
2142 The filter must match all properties specificed. If the property
2143 value to match is a list:
2145 1. String properties must match all elements in the list, and
2146 2. Other properties must match any of the elements in the list.
2147 """
2148 # we can't match anything if search_matches is empty
2149 if not search_matches and search_matches is not None:
2150 return []
2152 if __debug__:
2153 start_t = time.time()
2155 icn = self.classname
2157 # vars to hold the components of the SQL statement
2158 frum = [] # FROM clauses
2159 loj = [] # LEFT OUTER JOIN clauses
2160 where = [] # WHERE clauses
2161 args = [] # *any* positional arguments
2162 a = self.db.arg
2164 # figure the WHERE clause from the filterspec
2165 mlfilt = 0 # are we joining with Multilink tables?
2166 sortattr = self._sortattr (group = group, sort = sort)
2167 proptree = self._proptree(filterspec, sortattr)
2168 mlseen = 0
2169 for pt in reversed(proptree.sortattr):
2170 p = pt
2171 while p.parent:
2172 if isinstance (p.propclass, Multilink):
2173 mlseen = True
2174 if mlseen:
2175 p.sort_ids_needed = True
2176 p.tree_sort_done = False
2177 p = p.parent
2178 if not mlseen:
2179 pt.attr_sort_done = pt.tree_sort_done = True
2180 proptree.compute_sort_done()
2182 ordercols = []
2183 auxcols = {}
2184 mlsort = []
2185 rhsnum = 0
2186 for p in proptree:
2187 oc = None
2188 cn = p.classname
2189 ln = p.uniqname
2190 pln = p.parent.uniqname
2191 pcn = p.parent.classname
2192 k = p.name
2193 v = p.val
2194 propclass = p.propclass
2195 if p.sort_type > 0:
2196 oc = ac = '_%s._%s'%(pln, k)
2197 if isinstance(propclass, Multilink):
2198 if p.sort_type < 2:
2199 mlfilt = 1
2200 tn = '%s_%s'%(pcn, k)
2201 if v in ('-1', ['-1'], []):
2202 # only match rows that have count(linkid)=0 in the
2203 # corresponding multilink table)
2204 where.append(self._subselect(pcn, tn))
2205 else:
2206 frum.append(tn)
2207 where.append('_%s.id=%s.nodeid'%(pln,tn))
2208 if p.children:
2209 frum.append('_%s as _%s' % (cn, ln))
2210 where.append('%s.linkid=_%s.id'%(tn, ln))
2211 if p.has_values:
2212 if isinstance(v, type([])):
2213 s = ','.join([a for x in v])
2214 where.append('%s.linkid in (%s)'%(tn, s))
2215 args = args + v
2216 else:
2217 where.append('%s.linkid=%s'%(tn, a))
2218 args.append(v)
2219 if p.sort_type > 0:
2220 assert not p.attr_sort_done and not p.sort_ids_needed
2221 elif k == 'id':
2222 if p.sort_type < 2:
2223 if isinstance(v, type([])):
2224 s = ','.join([a for x in v])
2225 where.append('_%s.%s in (%s)'%(pln, k, s))
2226 args = args + v
2227 else:
2228 where.append('_%s.%s=%s'%(pln, k, a))
2229 args.append(v)
2230 if p.sort_type > 0:
2231 oc = ac = '_%s.id'%pln
2232 elif isinstance(propclass, String):
2233 if p.sort_type < 2:
2234 if not isinstance(v, type([])):
2235 v = [v]
2237 # Quote the bits in the string that need it and then embed
2238 # in a "substring" search. Note - need to quote the '%' so
2239 # they make it through the python layer happily
2240 v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
2242 # now add to the where clause
2243 where.append('('
2244 +' and '.join(["_%s._%s LIKE '%s'"%(pln, k, s) for s in v])
2245 +')')
2246 # note: args are embedded in the query string now
2247 if p.sort_type > 0:
2248 oc = ac = 'lower(_%s._%s)'%(pln, k)
2249 elif isinstance(propclass, Link):
2250 if p.sort_type < 2:
2251 if p.children:
2252 if p.sort_type == 0:
2253 frum.append('_%s as _%s' % (cn, ln))
2254 where.append('_%s._%s=_%s.id'%(pln, k, ln))
2255 if p.has_values:
2256 if isinstance(v, type([])):
2257 d = {}
2258 for entry in v:
2259 if entry == '-1':
2260 entry = None
2261 d[entry] = entry
2262 l = []
2263 if d.has_key(None) or not d:
2264 if d.has_key(None): del d[None]
2265 l.append('_%s._%s is NULL'%(pln, k))
2266 if d:
2267 v = d.keys()
2268 s = ','.join([a for x in v])
2269 l.append('(_%s._%s in (%s))'%(pln, k, s))
2270 args = args + v
2271 if l:
2272 where.append('(' + ' or '.join(l) +')')
2273 else:
2274 if v in ('-1', None):
2275 v = None
2276 where.append('_%s._%s is NULL'%(pln, k))
2277 else:
2278 where.append('_%s._%s=%s'%(pln, k, a))
2279 args.append(v)
2280 if p.sort_type > 0:
2281 lp = p.cls.labelprop()
2282 oc = ac = '_%s._%s'%(pln, k)
2283 if lp != 'id':
2284 if p.tree_sort_done and p.sort_type > 0:
2285 loj.append(
2286 'LEFT OUTER JOIN _%s as _%s on _%s._%s=_%s.id'%(
2287 cn, ln, pln, k, ln))
2288 oc = '_%s._%s'%(ln, lp)
2289 elif isinstance(propclass, Date) and p.sort_type < 2:
2290 dc = self.db.to_sql_value(hyperdb.Date)
2291 if isinstance(v, type([])):
2292 s = ','.join([a for x in v])
2293 where.append('_%s._%s in (%s)'%(pln, k, s))
2294 args = args + [dc(date.Date(x)) for x in v]
2295 else:
2296 try:
2297 # Try to filter on range of dates
2298 date_rng = propclass.range_from_raw(v, self.db)
2299 if date_rng.from_value:
2300 where.append('_%s._%s >= %s'%(pln, k, a))
2301 args.append(dc(date_rng.from_value))
2302 if date_rng.to_value:
2303 where.append('_%s._%s <= %s'%(pln, k, a))
2304 args.append(dc(date_rng.to_value))
2305 except ValueError:
2306 # If range creation fails - ignore that search parameter
2307 pass
2308 elif isinstance(propclass, Interval):
2309 # filter/sort using the __<prop>_int__ column
2310 if p.sort_type < 2:
2311 if isinstance(v, type([])):
2312 s = ','.join([a for x in v])
2313 where.append('_%s.__%s_int__ in (%s)'%(pln, k, s))
2314 args = args + [date.Interval(x).as_seconds() for x in v]
2315 else:
2316 try:
2317 # Try to filter on range of intervals
2318 date_rng = Range(v, date.Interval)
2319 if date_rng.from_value:
2320 where.append('_%s.__%s_int__ >= %s'%(pln, k, a))
2321 args.append(date_rng.from_value.as_seconds())
2322 if date_rng.to_value:
2323 where.append('_%s.__%s_int__ <= %s'%(pln, k, a))
2324 args.append(date_rng.to_value.as_seconds())
2325 except ValueError:
2326 # If range creation fails - ignore search parameter
2327 pass
2328 if p.sort_type > 0:
2329 oc = ac = '_%s.__%s_int__'%(pln,k)
2330 elif p.sort_type < 2:
2331 if isinstance(v, type([])):
2332 s = ','.join([a for x in v])
2333 where.append('_%s._%s in (%s)'%(pln, k, s))
2334 args = args + v
2335 else:
2336 where.append('_%s._%s=%s'%(pln, k, a))
2337 args.append(v)
2338 if oc:
2339 if p.sort_ids_needed:
2340 auxcols[ac] = p
2341 if p.tree_sort_done and p.sort_direction:
2342 # Don't select top-level id twice
2343 if p.name != 'id' or p.parent != proptree:
2344 ordercols.append(oc)
2345 desc = ['', ' desc'][p.sort_direction == '-']
2346 # Some SQL dbs sort NULL values last -- we want them first.
2347 if (self.order_by_null_values and p.name != 'id'):
2348 nv = self.order_by_null_values % oc
2349 ordercols.append(nv)
2350 p.orderby.append(nv + desc)
2351 p.orderby.append(oc + desc)
2353 props = self.getprops()
2355 # don't match retired nodes
2356 where.append('_%s.__retired__=0'%icn)
2358 # add results of full text search
2359 if search_matches is not None:
2360 s = ','.join([a for x in search_matches])
2361 where.append('_%s.id in (%s)'%(icn, s))
2362 args = args + v
2364 # construct the SQL
2365 frum.append('_'+icn)
2366 frum = ','.join(frum)
2367 if where:
2368 where = ' where ' + (' and '.join(where))
2369 else:
2370 where = ''
2371 if mlfilt:
2372 # we're joining tables on the id, so we will get dupes if we
2373 # don't distinct()
2374 cols = ['distinct(_%s.id)'%icn]
2375 else:
2376 cols = ['_%s.id'%icn]
2377 if ordercols:
2378 cols = cols + ordercols
2379 order = []
2380 # keep correct sequence of order attributes.
2381 for sa in proptree.sortattr:
2382 if not sa.attr_sort_done:
2383 continue
2384 order.extend(sa.orderby)
2385 if order:
2386 order = ' order by %s'%(','.join(order))
2387 else:
2388 order = ''
2389 for o, p in auxcols.iteritems ():
2390 cols.append (o)
2391 p.auxcol = len (cols) - 1
2393 cols = ','.join(cols)
2394 loj = ' '.join(loj)
2395 sql = 'select %s from %s %s %s%s'%(cols, frum, loj, where, order)
2396 args = tuple(args)
2397 __traceback_info__ = (sql, args)
2398 self.db.sql(sql, args)
2399 l = self.db.sql_fetchall()
2401 # Compute values needed for sorting in proptree.sort
2402 for p in auxcols.itervalues():
2403 p.sort_ids = p.sort_result = [row[p.auxcol] for row in l]
2404 # return the IDs (the first column)
2405 # XXX numeric ids
2406 l = [str(row[0]) for row in l]
2407 l = proptree.sort (l)
2409 if __debug__:
2410 self.db.stats['filtering'] += (time.time() - start_t)
2411 return l
2413 def filter_sql(self, sql):
2414 """Return a list of the ids of the items in this class that match
2415 the SQL provided. The SQL is a complete "select" statement.
2417 The SQL select must include the item id as the first column.
2419 This function DOES NOT filter out retired items, add on a where
2420 clause "__retired__=0" if you don't want retired nodes.
2421 """
2422 if __debug__:
2423 start_t = time.time()
2425 self.db.sql(sql)
2426 l = self.db.sql_fetchall()
2428 if __debug__:
2429 self.db.stats['filtering'] += (time.time() - start_t)
2430 return l
2432 def count(self):
2433 """Get the number of nodes in this class.
2435 If the returned integer is 'numnodes', the ids of all the nodes
2436 in this class run from 1 to numnodes, and numnodes+1 will be the
2437 id of the next node to be created in this class.
2438 """
2439 return self.db.countnodes(self.classname)
2441 # Manipulating properties:
2442 def getprops(self, protected=1):
2443 """Return a dictionary mapping property names to property objects.
2444 If the "protected" flag is true, we include protected properties -
2445 those which may not be modified.
2446 """
2447 d = self.properties.copy()
2448 if protected:
2449 d['id'] = String()
2450 d['creation'] = hyperdb.Date()
2451 d['activity'] = hyperdb.Date()
2452 d['creator'] = hyperdb.Link('user')
2453 d['actor'] = hyperdb.Link('user')
2454 return d
2456 def addprop(self, **properties):
2457 """Add properties to this class.
2459 The keyword arguments in 'properties' must map names to property
2460 objects, or a TypeError is raised. None of the keys in 'properties'
2461 may collide with the names of existing properties, or a ValueError
2462 is raised before any properties have been added.
2463 """
2464 for key in properties.keys():
2465 if self.properties.has_key(key):
2466 raise ValueError, key
2467 self.properties.update(properties)
2469 def index(self, nodeid):
2470 """Add (or refresh) the node to search indexes
2471 """
2472 # find all the String properties that have indexme
2473 for prop, propclass in self.getprops().items():
2474 if isinstance(propclass, String) and propclass.indexme:
2475 self.db.indexer.add_text((self.classname, nodeid, prop),
2476 str(self.get(nodeid, prop)))
2478 #
2479 # import / export support
2480 #
2481 def export_list(self, propnames, nodeid):
2482 """ Export a node - generate a list of CSV-able data in the order
2483 specified by propnames for the given node.
2484 """
2485 properties = self.getprops()
2486 l = []
2487 for prop in propnames:
2488 proptype = properties[prop]
2489 value = self.get(nodeid, prop)
2490 # "marshal" data where needed
2491 if value is None:
2492 pass
2493 elif isinstance(proptype, hyperdb.Date):
2494 value = value.get_tuple()
2495 elif isinstance(proptype, hyperdb.Interval):
2496 value = value.get_tuple()
2497 elif isinstance(proptype, hyperdb.Password):
2498 value = str(value)
2499 l.append(repr(value))
2500 l.append(repr(self.is_retired(nodeid)))
2501 return l
2503 def import_list(self, propnames, proplist):
2504 """ Import a node - all information including "id" is present and
2505 should not be sanity checked. Triggers are not triggered. The
2506 journal should be initialised using the "creator" and "created"
2507 information.
2509 Return the nodeid of the node imported.
2510 """
2511 if self.db.journaltag is None:
2512 raise DatabaseError, _('Database open read-only')
2513 properties = self.getprops()
2515 # make the new node's property map
2516 d = {}
2517 retire = 0
2518 if not "id" in propnames:
2519 newid = self.db.newid(self.classname)
2520 else:
2521 newid = eval(proplist[propnames.index("id")])
2522 for i in range(len(propnames)):
2523 # Use eval to reverse the repr() used to output the CSV
2524 value = eval(proplist[i])
2526 # Figure the property for this column
2527 propname = propnames[i]
2529 # "unmarshal" where necessary
2530 if propname == 'id':
2531 continue
2532 elif propname == 'is retired':
2533 # is the item retired?
2534 if int(value):
2535 retire = 1
2536 continue
2537 elif value is None:
2538 d[propname] = None
2539 continue
2541 prop = properties[propname]
2542 if value is None:
2543 # don't set Nones
2544 continue
2545 elif isinstance(prop, hyperdb.Date):
2546 value = date.Date(value)
2547 elif isinstance(prop, hyperdb.Interval):
2548 value = date.Interval(value)
2549 elif isinstance(prop, hyperdb.Password):
2550 pwd = password.Password()
2551 pwd.unpack(value)
2552 value = pwd
2553 elif isinstance(prop, String):
2554 if isinstance(value, unicode):
2555 value = value.encode('utf8')
2556 if not isinstance(value, str):
2557 raise TypeError, \
2558 'new property "%(propname)s" not a string: %(value)r' \
2559 % locals()
2560 if prop.indexme:
2561 self.db.indexer.add_text((self.classname, newid, propname),
2562 value)
2563 d[propname] = value
2565 # get a new id if necessary
2566 if newid is None:
2567 newid = self.db.newid(self.classname)
2569 # insert new node or update existing?
2570 if not self.hasnode(newid):
2571 self.db.addnode(self.classname, newid, d) # insert
2572 else:
2573 self.db.setnode(self.classname, newid, d) # update
2575 # retire?
2576 if retire:
2577 # use the arg for __retired__ to cope with any odd database type
2578 # conversion (hello, sqlite)
2579 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
2580 self.db.arg, self.db.arg)
2581 self.db.sql(sql, (newid, newid))
2582 return newid
2584 def export_journals(self):
2585 """Export a class's journal - generate a list of lists of
2586 CSV-able data:
2588 nodeid, date, user, action, params
2590 No heading here - the columns are fixed.
2591 """
2592 properties = self.getprops()
2593 r = []
2594 for nodeid in self.getnodeids():
2595 for nodeid, date, user, action, params in self.history(nodeid):
2596 date = date.get_tuple()
2597 if action == 'set':
2598 export_data = {}
2599 for propname, value in params.items():
2600 if not properties.has_key(propname):
2601 # property no longer in the schema
2602 continue
2604 prop = properties[propname]
2605 # make sure the params are eval()'able
2606 if value is None:
2607 pass
2608 elif isinstance(prop, Date):
2609 value = value.get_tuple()
2610 elif isinstance(prop, Interval):
2611 value = value.get_tuple()
2612 elif isinstance(prop, Password):
2613 value = str(value)
2614 export_data[propname] = value
2615 params = export_data
2616 elif action == 'create' and params:
2617 # old tracker with data stored in the create!
2618 params = {}
2619 l = [nodeid, date, user, action, params]
2620 r.append(map(repr, l))
2621 return r
2623 def import_journals(self, entries):
2624 """Import a class's journal.
2626 Uses setjournal() to set the journal for each item."""
2627 properties = self.getprops()
2628 d = {}
2629 for l in entries:
2630 l = map(eval, l)
2631 nodeid, jdate, user, action, params = l
2632 r = d.setdefault(nodeid, [])
2633 if action == 'set':
2634 for propname, value in params.items():
2635 prop = properties[propname]
2636 if value is None:
2637 pass
2638 elif isinstance(prop, Date):
2639 value = date.Date(value)
2640 elif isinstance(prop, Interval):
2641 value = date.Interval(value)
2642 elif isinstance(prop, Password):
2643 pwd = password.Password()
2644 pwd.unpack(value)
2645 value = pwd
2646 params[propname] = value
2647 elif action == 'create' and params:
2648 # old tracker with data stored in the create!
2649 params = {}
2650 r.append((nodeid, date.Date(jdate), user, action, params))
2652 for nodeid, l in d.items():
2653 self.db.setjournal(self.classname, nodeid, l)
2655 class FileClass(hyperdb.FileClass, Class):
2656 """This class defines a large chunk of data. To support this, it has a
2657 mandatory String property "content" which is typically saved off
2658 externally to the hyperdb.
2660 The default MIME type of this data is defined by the
2661 "default_mime_type" class attribute, which may be overridden by each
2662 node if the class defines a "type" String property.
2663 """
2664 def __init__(self, db, classname, **properties):
2665 """The newly-created class automatically includes the "content"
2666 and "type" properties.
2667 """
2668 if not properties.has_key('content'):
2669 properties['content'] = hyperdb.String(indexme='yes')
2670 if not properties.has_key('type'):
2671 properties['type'] = hyperdb.String()
2672 Class.__init__(self, db, classname, **properties)
2674 def create(self, **propvalues):
2675 """ snaffle the file propvalue and store in a file
2676 """
2677 # we need to fire the auditors now, or the content property won't
2678 # be in propvalues for the auditors to play with
2679 self.fireAuditors('create', None, propvalues)
2681 # now remove the content property so it's not stored in the db
2682 content = propvalues['content']
2683 del propvalues['content']
2685 # do the database create
2686 newid = self.create_inner(**propvalues)
2688 # figure the mime type
2689 mime_type = propvalues.get('type', self.default_mime_type)
2691 # and index!
2692 if self.properties['content'].indexme:
2693 self.db.indexer.add_text((self.classname, newid, 'content'),
2694 content, mime_type)
2696 # store off the content as a file
2697 self.db.storefile(self.classname, newid, None, content)
2699 # fire reactors
2700 self.fireReactors('create', newid, None)
2702 return newid
2704 def get(self, nodeid, propname, default=_marker, cache=1):
2705 """ Trap the content propname and get it from the file
2707 'cache' exists for backwards compatibility, and is not used.
2708 """
2709 poss_msg = 'Possibly a access right configuration problem.'
2710 if propname == 'content':
2711 try:
2712 return self.db.getfile(self.classname, nodeid, None)
2713 except IOError, (strerror):
2714 # BUG: by catching this we donot see an error in the log.
2715 return 'ERROR reading file: %s%s\n%s\n%s'%(
2716 self.classname, nodeid, poss_msg, strerror)
2717 if default is not _marker:
2718 return Class.get(self, nodeid, propname, default)
2719 else:
2720 return Class.get(self, nodeid, propname)
2722 def set(self, itemid, **propvalues):
2723 """ Snarf the "content" propvalue and update it in a file
2724 """
2725 self.fireAuditors('set', itemid, propvalues)
2726 oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid))
2728 # now remove the content property so it's not stored in the db
2729 content = None
2730 if propvalues.has_key('content'):
2731 content = propvalues['content']
2732 del propvalues['content']
2734 # do the database create
2735 propvalues = self.set_inner(itemid, **propvalues)
2737 # do content?
2738 if content:
2739 # store and possibly index
2740 self.db.storefile(self.classname, itemid, None, content)
2741 if self.properties['content'].indexme:
2742 mime_type = self.get(itemid, 'type', self.default_mime_type)
2743 self.db.indexer.add_text((self.classname, itemid, 'content'),
2744 content, mime_type)
2745 propvalues['content'] = content
2747 # fire reactors
2748 self.fireReactors('set', itemid, oldvalues)
2749 return propvalues
2751 def index(self, nodeid):
2752 """ Add (or refresh) the node to search indexes.
2754 Use the content-type property for the content property.
2755 """
2756 # find all the String properties that have indexme
2757 for prop, propclass in self.getprops().items():
2758 if prop == 'content' and propclass.indexme:
2759 mime_type = self.get(nodeid, 'type', self.default_mime_type)
2760 self.db.indexer.add_text((self.classname, nodeid, 'content'),
2761 str(self.get(nodeid, 'content')), mime_type)
2762 elif isinstance(propclass, hyperdb.String) and propclass.indexme:
2763 # index them under (classname, nodeid, property)
2764 try:
2765 value = str(self.get(nodeid, prop))
2766 except IndexError:
2767 # node has been destroyed
2768 continue
2769 self.db.indexer.add_text((self.classname, nodeid, prop), value)
2771 # XXX deviation from spec - was called ItemClass
2772 class IssueClass(Class, roundupdb.IssueClass):
2773 # Overridden methods:
2774 def __init__(self, db, classname, **properties):
2775 """The newly-created class automatically includes the "messages",
2776 "files", "nosy", and "superseder" properties. If the 'properties'
2777 dictionary attempts to specify any of these properties or a
2778 "creation", "creator", "activity" or "actor" property, a ValueError
2779 is raised.
2780 """
2781 if not properties.has_key('title'):
2782 properties['title'] = hyperdb.String(indexme='yes')
2783 if not properties.has_key('messages'):
2784 properties['messages'] = hyperdb.Multilink("msg")
2785 if not properties.has_key('files'):
2786 properties['files'] = hyperdb.Multilink("file")
2787 if not properties.has_key('nosy'):
2788 # note: journalling is turned off as it really just wastes
2789 # space. this behaviour may be overridden in an instance
2790 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2791 if not properties.has_key('superseder'):
2792 properties['superseder'] = hyperdb.Multilink(classname)
2793 Class.__init__(self, db, classname, **properties)
2795 # vim: set et sts=4 sw=4 :