1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 """ Relational database (SQL) backend common code.
20 Basics:
22 - map roundup classes to relational tables
23 - automatically detect schema changes and modify the table schemas
24 appropriately (we store the "database version" of the schema in the
25 database itself as the only row of the "schema" table)
26 - multilinks (which represent a many-to-many relationship) are handled through
27 intermediate tables
28 - journals are stored adjunct to the per-class tables
29 - table names and columns have "_" prepended so the names can't clash with
30 restricted names (like "order")
31 - retirement is determined by the __retired__ column being > 0
33 Database-specific changes may generally be pushed out to the overridable
34 sql_* methods, since everything else should be fairly generic. There's
35 probably a bit of work to be done if a database is used that actually
36 honors column typing, since the initial databases don't (sqlite stores
37 everything as a string.)
39 The schema of the hyperdb being mapped to the database is stored in the
40 database itself as a repr()'ed dictionary of information about each Class
41 that maps to a table. If that information differs from the hyperdb schema,
42 then we update it. We also store in the schema dict a version which
43 allows us to upgrade the database schema when necessary. See upgrade_db().
45 To force a unqiueness constraint on the key properties we put the item
46 id into the __retired__ column duing retirement (so it's 0 for "active"
47 items) and place a unqiueness constraint on key + __retired__. This is
48 particularly important for the users class where multiple users may
49 try to have the same username, with potentially many retired users with
50 the same name.
51 """
52 __docformat__ = 'restructuredtext'
54 # standard python modules
55 import sys, os, time, re, errno, weakref, copy, logging
57 # roundup modules
58 from roundup import hyperdb, date, password, roundupdb, security, support
59 from roundup.hyperdb import String, Password, Date, Interval, Link, \
60 Multilink, DatabaseError, Boolean, Number, Node
61 from roundup.backends import locking
62 from roundup.support import reversed
63 from roundup.i18n import _
65 # support
66 from blobfiles import FileStorage
67 try:
68 from indexer_xapian import Indexer
69 except ImportError:
70 from indexer_rdbms import Indexer
71 from sessions_rdbms import Sessions, OneTimeKeys
72 from roundup.date import Range
74 # number of rows to keep in memory
75 ROW_CACHE_SIZE = 100
77 # dummy value meaning "argument not passed"
78 _marker = []
80 def _num_cvt(num):
81 num = str(num)
82 try:
83 return int(num)
84 except:
85 return float(num)
87 def _bool_cvt(value):
88 if value in ('TRUE', 'FALSE'):
89 return {'TRUE': 1, 'FALSE': 0}[value]
90 # assume it's a number returned from the db API
91 return int(value)
93 def connection_dict(config, dbnamestr=None):
94 """ Used by Postgresql and MySQL to detemine the keyword args for
95 opening the database connection."""
96 d = { }
97 if dbnamestr:
98 d[dbnamestr] = config.RDBMS_NAME
99 for name in ('host', 'port', 'password', 'user', 'read_default_group',
100 'read_default_file'):
101 cvar = 'RDBMS_'+name.upper()
102 if config[cvar] is not None:
103 d[name] = config[cvar]
104 return d
106 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
107 """ Wrapper around an SQL database that presents a hyperdb interface.
109 - some functionality is specific to the actual SQL database, hence
110 the sql_* methods that are NotImplemented
111 - we keep a cache of the latest ROW_CACHE_SIZE row fetches.
112 """
113 def __init__(self, config, journaltag=None):
114 """ Open the database and load the schema from it.
115 """
116 FileStorage.__init__(self, config.UMASK)
117 self.config, self.journaltag = config, journaltag
118 self.dir = config.DATABASE
119 self.classes = {}
120 self.indexer = Indexer(self)
121 self.security = security.Security(self)
123 # additional transaction support for external files and the like
124 self.transactions = []
126 # keep a cache of the N most recently retrieved rows of any kind
127 # (classname, nodeid) = row
128 self.cache = {}
129 self.cache_lru = []
130 self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0,
131 'filtering': 0}
133 # database lock
134 self.lockfile = None
136 # open a connection to the database, creating the "conn" attribute
137 self.open_connection()
139 def clearCache(self):
140 self.cache = {}
141 self.cache_lru = []
143 def getSessionManager(self):
144 return Sessions(self)
146 def getOTKManager(self):
147 return OneTimeKeys(self)
149 def open_connection(self):
150 """ Open a connection to the database, creating it if necessary.
152 Must call self.load_dbschema()
153 """
154 raise NotImplemented
156 def sql(self, sql, args=None):
157 """ Execute the sql with the optional args.
158 """
159 self.log_debug('SQL %r %r'%(sql, args))
160 if args:
161 self.cursor.execute(sql, args)
162 else:
163 self.cursor.execute(sql)
165 def sql_fetchone(self):
166 """ Fetch a single row. If there's nothing to fetch, return None.
167 """
168 return self.cursor.fetchone()
170 def sql_fetchall(self):
171 """ Fetch all rows. If there's nothing to fetch, return [].
172 """
173 return self.cursor.fetchall()
175 def sql_stringquote(self, value):
176 """ Quote the string so it's safe to put in the 'sql quotes'
177 """
178 return re.sub("'", "''", str(value))
180 def init_dbschema(self):
181 self.database_schema = {
182 'version': self.current_db_version,
183 'tables': {}
184 }
186 def load_dbschema(self):
187 """ Load the schema definition that the database currently implements
188 """
189 self.cursor.execute('select schema from schema')
190 schema = self.cursor.fetchone()
191 if schema:
192 self.database_schema = eval(schema[0])
193 else:
194 self.database_schema = {}
196 def save_dbschema(self):
197 """ Save the schema definition that the database currently implements
198 """
199 s = repr(self.database_schema)
200 self.sql('delete from schema')
201 self.sql('insert into schema values (%s)'%self.arg, (s,))
203 def post_init(self):
204 """ Called once the schema initialisation has finished.
206 We should now confirm that the schema defined by our "classes"
207 attribute actually matches the schema in the database.
208 """
209 save = 0
211 # handle changes in the schema
212 tables = self.database_schema['tables']
213 for classname, spec in self.classes.items():
214 if tables.has_key(classname):
215 dbspec = tables[classname]
216 if self.update_class(spec, dbspec):
217 tables[classname] = spec.schema()
218 save = 1
219 else:
220 self.create_class(spec)
221 tables[classname] = spec.schema()
222 save = 1
224 for classname, spec in tables.items():
225 if not self.classes.has_key(classname):
226 self.drop_class(classname, tables[classname])
227 del tables[classname]
228 save = 1
230 # now upgrade the database for column type changes, new internal
231 # tables, etc.
232 save = save | self.upgrade_db()
234 # update the database version of the schema
235 if save:
236 self.save_dbschema()
238 # reindex the db if necessary
239 if self.indexer.should_reindex():
240 self.reindex()
242 # commit
243 self.sql_commit()
245 # update this number when we need to make changes to the SQL structure
246 # of the backen database
247 current_db_version = 5
248 db_version_updated = False
249 def upgrade_db(self):
250 """ Update the SQL database to reflect changes in the backend code.
252 Return boolean whether we need to save the schema.
253 """
254 version = self.database_schema.get('version', 1)
255 if version > self.current_db_version:
256 raise DatabaseError('attempting to run rev %d DATABASE with rev '
257 '%d CODE!'%(version, self.current_db_version))
258 if version == self.current_db_version:
259 # nothing to do
260 return 0
262 if version < 2:
263 log_info('upgrade to version 2')
264 # change the schema structure
265 self.database_schema = {'tables': self.database_schema}
267 # version 1 didn't have the actor column (note that in
268 # MySQL this will also transition the tables to typed columns)
269 self.add_new_columns_v2()
271 # version 1 doesn't have the OTK, session and indexing in the
272 # database
273 self.create_version_2_tables()
275 if version < 3:
276 log_info('upgrade to version 3')
277 self.fix_version_2_tables()
279 if version < 4:
280 self.fix_version_3_tables()
282 if version < 5:
283 self.fix_version_4_tables()
285 self.database_schema['version'] = self.current_db_version
286 self.db_version_updated = True
287 return 1
289 def fix_version_3_tables(self):
290 # drop the shorter VARCHAR OTK column and add a new TEXT one
291 for name in ('otk', 'session'):
292 self.sql('DELETE FROM %ss'%name)
293 self.sql('ALTER TABLE %ss DROP %s_value'%(name, name))
294 self.sql('ALTER TABLE %ss ADD %s_value TEXT'%(name, name))
296 def fix_version_2_tables(self):
297 # Default (used by sqlite): NOOP
298 pass
300 def fix_version_4_tables(self):
301 # note this is an explicit call now
302 c = self.cursor
303 for cn, klass in self.classes.items():
304 c.execute('select id from _%s where __retired__<>0'%(cn,))
305 for (id,) in c.fetchall():
306 c.execute('update _%s set __retired__=%s where id=%s'%(cn,
307 self.arg, self.arg), (id, id))
309 if klass.key:
310 self.add_class_key_required_unique_constraint(cn, klass.key)
312 def _convert_journal_tables(self):
313 """Get current journal table contents, drop the table and re-create"""
314 c = self.cursor
315 cols = ','.join('nodeid date tag action params'.split())
316 for klass in self.classes.values():
317 # slurp and drop
318 sql = 'select %s from %s__journal order by date'%(cols,
319 klass.classname)
320 c.execute(sql)
321 contents = c.fetchall()
322 self.drop_journal_table_indexes(klass.classname)
323 c.execute('drop table %s__journal'%klass.classname)
325 # re-create and re-populate
326 self.create_journal_table(klass)
327 a = self.arg
328 sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(
329 klass.classname, cols, a, a, a, a, a)
330 for row in contents:
331 # no data conversion needed
332 self.cursor.execute(sql, row)
334 def _convert_string_properties(self):
335 """Get current Class tables that contain String properties, and
336 convert the VARCHAR columns to TEXT"""
337 c = self.cursor
338 for klass in self.classes.values():
339 # slurp and drop
340 cols, mls = self.determine_columns(klass.properties.items())
341 scols = ','.join([i[0] for i in cols])
342 sql = 'select id,%s from _%s'%(scols, klass.classname)
343 c.execute(sql)
344 contents = c.fetchall()
345 self.drop_class_table_indexes(klass.classname, klass.getkey())
346 c.execute('drop table _%s'%klass.classname)
348 # re-create and re-populate
349 self.create_class_table(klass, create_sequence=0)
350 a = ','.join([self.arg for i in range(len(cols)+1)])
351 sql = 'insert into _%s (id,%s) values (%s)'%(klass.classname,
352 scols, a)
353 for row in contents:
354 l = []
355 for entry in row:
356 # mysql will already be a string - psql needs "help"
357 if entry is not None and not isinstance(entry, type('')):
358 entry = str(entry)
359 l.append(entry)
360 self.cursor.execute(sql, l)
362 def refresh_database(self):
363 self.post_init()
366 def reindex(self, classname=None, show_progress=False):
367 if classname:
368 classes = [self.getclass(classname)]
369 else:
370 classes = self.classes.values()
371 for klass in classes:
372 if show_progress:
373 for nodeid in support.Progress('Reindex %s'%klass.classname,
374 klass.list()):
375 klass.index(nodeid)
376 else:
377 for nodeid in klass.list():
378 klass.index(nodeid)
379 self.indexer.save_index()
381 hyperdb_to_sql_datatypes = {
382 hyperdb.String : 'TEXT',
383 hyperdb.Date : 'TIMESTAMP',
384 hyperdb.Link : 'INTEGER',
385 hyperdb.Interval : 'VARCHAR(255)',
386 hyperdb.Password : 'VARCHAR(255)',
387 hyperdb.Boolean : 'BOOLEAN',
388 hyperdb.Number : 'REAL',
389 }
390 def determine_columns(self, properties):
391 """ Figure the column names and multilink properties from the spec
393 "properties" is a list of (name, prop) where prop may be an
394 instance of a hyperdb "type" _or_ a string repr of that type.
395 """
396 cols = [
397 ('_actor', self.hyperdb_to_sql_datatypes[hyperdb.Link]),
398 ('_activity', self.hyperdb_to_sql_datatypes[hyperdb.Date]),
399 ('_creator', self.hyperdb_to_sql_datatypes[hyperdb.Link]),
400 ('_creation', self.hyperdb_to_sql_datatypes[hyperdb.Date]),
401 ]
402 mls = []
403 # add the multilinks separately
404 for col, prop in properties:
405 if isinstance(prop, Multilink):
406 mls.append(col)
407 continue
409 if isinstance(prop, type('')):
410 raise ValueError, "string property spec!"
411 #and prop.find('Multilink') != -1:
412 #mls.append(col)
414 datatype = self.hyperdb_to_sql_datatypes[prop.__class__]
415 cols.append(('_'+col, datatype))
417 # Intervals stored as two columns
418 if isinstance(prop, Interval):
419 cols.append(('__'+col+'_int__', 'BIGINT'))
421 cols.sort()
422 return cols, mls
424 def update_class(self, spec, old_spec, force=0):
425 """ Determine the differences between the current spec and the
426 database version of the spec, and update where necessary.
428 If 'force' is true, update the database anyway.
429 """
430 new_has = spec.properties.has_key
431 new_spec = spec.schema()
432 new_spec[1].sort()
433 old_spec[1].sort()
434 if not force and new_spec == old_spec:
435 # no changes
436 return 0
438 logger = logging.getLogger('hyperdb')
439 logger.info('update_class %s'%spec.classname)
441 logger.debug('old_spec %r'%(old_spec,))
442 logger.debug('new_spec %r'%(new_spec,))
444 # detect key prop change for potential index change
445 keyprop_changes = {}
446 if new_spec[0] != old_spec[0]:
447 if old_spec[0]:
448 keyprop_changes['remove'] = old_spec[0]
449 if new_spec[0]:
450 keyprop_changes['add'] = new_spec[0]
452 # detect multilinks that have been removed, and drop their table
453 old_has = {}
454 for name, prop in old_spec[1]:
455 old_has[name] = 1
456 if new_has(name):
457 continue
459 if prop.find('Multilink to') != -1:
460 # first drop indexes.
461 self.drop_multilink_table_indexes(spec.classname, name)
463 # now the multilink table itself
464 sql = 'drop table %s_%s'%(spec.classname, name)
465 else:
466 # if this is the key prop, drop the index first
467 if old_spec[0] == prop:
468 self.drop_class_table_key_index(spec.classname, name)
469 del keyprop_changes['remove']
471 # drop the column
472 sql = 'alter table _%s drop column _%s'%(spec.classname, name)
474 self.sql(sql)
475 old_has = old_has.has_key
477 # if we didn't remove the key prop just then, but the key prop has
478 # changed, we still need to remove the old index
479 if keyprop_changes.has_key('remove'):
480 self.drop_class_table_key_index(spec.classname,
481 keyprop_changes['remove'])
483 # add new columns
484 for propname, prop in new_spec[1]:
485 if old_has(propname):
486 continue
487 prop = spec.properties[propname]
488 if isinstance(prop, Multilink):
489 self.create_multilink_table(spec, propname)
490 else:
491 # add the column
492 coltype = self.hyperdb_to_sql_datatypes[prop.__class__]
493 sql = 'alter table _%s add column _%s %s'%(
494 spec.classname, propname, coltype)
495 self.sql(sql)
497 # extra Interval column
498 if isinstance(prop, Interval):
499 sql = 'alter table _%s add column __%s_int__ BIGINT'%(
500 spec.classname, propname)
501 self.sql(sql)
503 # if the new column is a key prop, we need an index!
504 if new_spec[0] == propname:
505 self.create_class_table_key_index(spec.classname, propname)
506 del keyprop_changes['add']
508 # if we didn't add the key prop just then, but the key prop has
509 # changed, we still need to add the new index
510 if keyprop_changes.has_key('add'):
511 self.create_class_table_key_index(spec.classname,
512 keyprop_changes['add'])
514 return 1
516 def determine_all_columns(self, spec):
517 """Figure out the columns from the spec and also add internal columns
519 """
520 cols, mls = self.determine_columns(spec.properties.items())
522 # add on our special columns
523 cols.append(('id', 'INTEGER PRIMARY KEY'))
524 cols.append(('__retired__', 'INTEGER DEFAULT 0'))
525 return cols, mls
527 def create_class_table(self, spec):
528 """Create the class table for the given Class "spec". Creates the
529 indexes too."""
530 cols, mls = self.determine_all_columns(spec)
532 # create the base table
533 scols = ','.join(['%s %s'%x for x in cols])
534 sql = 'create table _%s (%s)'%(spec.classname, scols)
535 self.sql(sql)
537 self.create_class_table_indexes(spec)
539 return cols, mls
541 def create_class_table_indexes(self, spec):
542 """ create the class table for the given spec
543 """
544 # create __retired__ index
545 index_sql2 = 'create index _%s_retired_idx on _%s(__retired__)'%(
546 spec.classname, spec.classname)
547 self.sql(index_sql2)
549 # create index for key property
550 if spec.key:
551 index_sql3 = 'create index _%s_%s_idx on _%s(_%s)'%(
552 spec.classname, spec.key,
553 spec.classname, spec.key)
554 self.sql(index_sql3)
556 # and the unique index for key / retired(id)
557 self.add_class_key_required_unique_constraint(spec.classname,
558 spec.key)
560 # TODO: create indexes on (selected?) Link property columns, as
561 # they're more likely to be used for lookup
563 def add_class_key_required_unique_constraint(self, cn, key):
564 sql = '''create unique index _%s_key_retired_idx
565 on _%s(__retired__, _%s)'''%(cn, cn, key)
566 self.sql(sql)
568 def drop_class_table_indexes(self, cn, key):
569 # drop the old table indexes first
570 l = ['_%s_id_idx'%cn, '_%s_retired_idx'%cn]
571 if key:
572 l.append('_%s_%s_idx'%(cn, key))
574 table_name = '_%s'%cn
575 for index_name in l:
576 if not self.sql_index_exists(table_name, index_name):
577 continue
578 index_sql = 'drop index '+index_name
579 self.sql(index_sql)
581 def create_class_table_key_index(self, cn, key):
582 """ create the class table for the given spec
583 """
584 sql = 'create index _%s_%s_idx on _%s(_%s)'%(cn, key, cn, key)
585 self.sql(sql)
587 def drop_class_table_key_index(self, cn, key):
588 table_name = '_%s'%cn
589 index_name = '_%s_%s_idx'%(cn, key)
590 if self.sql_index_exists(table_name, index_name):
591 sql = 'drop index '+index_name
592 self.sql(sql)
594 # and now the retired unique index too
595 index_name = '_%s_key_retired_idx'%cn
596 if self.sql_index_exists(table_name, index_name):
597 sql = 'drop index '+index_name
598 self.sql(sql)
600 def create_journal_table(self, spec):
601 """ create the journal table for a class given the spec and
602 already-determined cols
603 """
604 # journal table
605 cols = ','.join(['%s varchar'%x
606 for x in 'nodeid date tag action params'.split()])
607 sql = """create table %s__journal (
608 nodeid integer, date %s, tag varchar(255),
609 action varchar(255), params text)""" % (spec.classname,
610 self.hyperdb_to_sql_datatypes[hyperdb.Date])
611 self.sql(sql)
612 self.create_journal_table_indexes(spec)
614 def create_journal_table_indexes(self, spec):
615 # index on nodeid
616 sql = 'create index %s_journ_idx on %s__journal(nodeid)'%(
617 spec.classname, spec.classname)
618 self.sql(sql)
620 def drop_journal_table_indexes(self, classname):
621 index_name = '%s_journ_idx'%classname
622 if not self.sql_index_exists('%s__journal'%classname, index_name):
623 return
624 index_sql = 'drop index '+index_name
625 self.sql(index_sql)
627 def create_multilink_table(self, spec, ml):
628 """ Create a multilink table for the "ml" property of the class
629 given by the spec
630 """
631 # create the table
632 sql = 'create table %s_%s (linkid INTEGER, nodeid INTEGER)'%(
633 spec.classname, ml)
634 self.sql(sql)
635 self.create_multilink_table_indexes(spec, ml)
637 def create_multilink_table_indexes(self, spec, ml):
638 # create index on linkid
639 index_sql = 'create index %s_%s_l_idx on %s_%s(linkid)'%(
640 spec.classname, ml, spec.classname, ml)
641 self.sql(index_sql)
643 # create index on nodeid
644 index_sql = 'create index %s_%s_n_idx on %s_%s(nodeid)'%(
645 spec.classname, ml, spec.classname, ml)
646 self.sql(index_sql)
648 def drop_multilink_table_indexes(self, classname, ml):
649 l = [
650 '%s_%s_l_idx'%(classname, ml),
651 '%s_%s_n_idx'%(classname, ml)
652 ]
653 table_name = '%s_%s'%(classname, ml)
654 for index_name in l:
655 if not self.sql_index_exists(table_name, index_name):
656 continue
657 index_sql = 'drop index %s'%index_name
658 self.sql(index_sql)
660 def create_class(self, spec):
661 """ Create a database table according to the given spec.
662 """
663 cols, mls = self.create_class_table(spec)
664 self.create_journal_table(spec)
666 # now create the multilink tables
667 for ml in mls:
668 self.create_multilink_table(spec, ml)
670 def drop_class(self, cn, spec):
671 """ Drop the given table from the database.
673 Drop the journal and multilink tables too.
674 """
675 properties = spec[1]
676 # figure the multilinks
677 mls = []
678 for propname, prop in properties:
679 if isinstance(prop, Multilink):
680 mls.append(propname)
682 # drop class table and indexes
683 self.drop_class_table_indexes(cn, spec[0])
685 self.drop_class_table(cn)
687 # drop journal table and indexes
688 self.drop_journal_table_indexes(cn)
689 sql = 'drop table %s__journal'%cn
690 self.sql(sql)
692 for ml in mls:
693 # drop multilink table and indexes
694 self.drop_multilink_table_indexes(cn, ml)
695 sql = 'drop table %s_%s'%(spec.classname, ml)
696 self.sql(sql)
698 def drop_class_table(self, cn):
699 sql = 'drop table _%s'%cn
700 self.sql(sql)
702 #
703 # Classes
704 #
705 def __getattr__(self, classname):
706 """ A convenient way of calling self.getclass(classname).
707 """
708 if self.classes.has_key(classname):
709 return self.classes[classname]
710 raise AttributeError, classname
712 def addclass(self, cl):
713 """ Add a Class to the hyperdatabase.
714 """
715 cn = cl.classname
716 if self.classes.has_key(cn):
717 raise ValueError, cn
718 self.classes[cn] = cl
720 # add default Edit and View permissions
721 self.security.addPermission(name="Create", klass=cn,
722 description="User is allowed to create "+cn)
723 self.security.addPermission(name="Edit", klass=cn,
724 description="User is allowed to edit "+cn)
725 self.security.addPermission(name="View", klass=cn,
726 description="User is allowed to access "+cn)
728 def getclasses(self):
729 """ Return a list of the names of all existing classes.
730 """
731 l = self.classes.keys()
732 l.sort()
733 return l
735 def getclass(self, classname):
736 """Get the Class object representing a particular class.
738 If 'classname' is not a valid class name, a KeyError is raised.
739 """
740 try:
741 return self.classes[classname]
742 except KeyError:
743 raise KeyError, 'There is no class called "%s"'%classname
745 def clear(self):
746 """Delete all database contents.
748 Note: I don't commit here, which is different behaviour to the
749 "nuke from orbit" behaviour in the dbs.
750 """
751 logging.getLogger('hyperdb').info('clear')
752 for cn in self.classes.keys():
753 sql = 'delete from _%s'%cn
754 self.sql(sql)
756 #
757 # Nodes
758 #
760 hyperdb_to_sql_value = {
761 hyperdb.String : str,
762 # fractional seconds by default
763 hyperdb.Date : lambda x: x.formal(sep=' ', sec='%06.3f'),
764 hyperdb.Link : int,
765 hyperdb.Interval : str,
766 hyperdb.Password : str,
767 hyperdb.Boolean : lambda x: x and 'TRUE' or 'FALSE',
768 hyperdb.Number : lambda x: x,
769 hyperdb.Multilink : lambda x: x, # used in journal marshalling
770 }
771 def addnode(self, classname, nodeid, node):
772 """ Add the specified node to its class's db.
773 """
774 self.log_debug('addnode %s%s %r'%(classname,
775 nodeid, node))
777 # determine the column definitions and multilink tables
778 cl = self.classes[classname]
779 cols, mls = self.determine_columns(cl.properties.items())
781 # we'll be supplied these props if we're doing an import
782 values = node.copy()
783 if not values.has_key('creator'):
784 # add in the "calculated" properties (dupe so we don't affect
785 # calling code's node assumptions)
786 values['creation'] = values['activity'] = date.Date()
787 values['actor'] = values['creator'] = self.getuid()
789 cl = self.classes[classname]
790 props = cl.getprops(protected=1)
791 del props['id']
793 # default the non-multilink columns
794 for col, prop in props.items():
795 if not values.has_key(col):
796 if isinstance(prop, Multilink):
797 values[col] = []
798 else:
799 values[col] = None
801 # clear this node out of the cache if it's in there
802 key = (classname, nodeid)
803 if self.cache.has_key(key):
804 del self.cache[key]
805 self.cache_lru.remove(key)
807 # figure the values to insert
808 vals = []
809 for col,dt in cols:
810 # this is somewhat dodgy....
811 if col.endswith('_int__'):
812 # XXX eugh, this test suxxors
813 value = values[col[2:-6]]
814 # this is an Interval special "int" column
815 if value is not None:
816 vals.append(value.as_seconds())
817 else:
818 vals.append(value)
819 continue
821 prop = props[col[1:]]
822 value = values[col[1:]]
823 if value is not None:
824 value = self.hyperdb_to_sql_value[prop.__class__](value)
825 vals.append(value)
826 vals.append(nodeid)
827 vals = tuple(vals)
829 # make sure the ordering is correct for column name -> column value
830 s = ','.join([self.arg for x in cols]) + ',%s'%self.arg
831 cols = ','.join([col for col,dt in cols]) + ',id'
833 # perform the inserts
834 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
835 self.sql(sql, vals)
837 # insert the multilink rows
838 for col in mls:
839 t = '%s_%s'%(classname, col)
840 for entry in node[col]:
841 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
842 self.arg, self.arg)
843 self.sql(sql, (entry, nodeid))
845 def setnode(self, classname, nodeid, values, multilink_changes={}):
846 """ Change the specified node.
847 """
848 self.log_debug('setnode %s%s %r'
849 % (classname, nodeid, values))
851 # clear this node out of the cache if it's in there
852 key = (classname, nodeid)
853 if self.cache.has_key(key):
854 del self.cache[key]
855 self.cache_lru.remove(key)
857 cl = self.classes[classname]
858 props = cl.getprops()
860 cols = []
861 mls = []
862 # add the multilinks separately
863 for col in values.keys():
864 prop = props[col]
865 if isinstance(prop, Multilink):
866 mls.append(col)
867 elif isinstance(prop, Interval):
868 # Intervals store the seconds value too
869 cols.append(col)
870 # extra leading '_' added by code below
871 cols.append('_' +col + '_int__')
872 else:
873 cols.append(col)
874 cols.sort()
876 # figure the values to insert
877 vals = []
878 for col in cols:
879 if col.endswith('_int__'):
880 # XXX eugh, this test suxxors
881 # Intervals store the seconds value too
882 col = col[1:-6]
883 prop = props[col]
884 value = values[col]
885 if value is None:
886 vals.append(None)
887 else:
888 vals.append(value.as_seconds())
889 else:
890 prop = props[col]
891 value = values[col]
892 if value is None:
893 e = None
894 else:
895 e = self.hyperdb_to_sql_value[prop.__class__](value)
896 vals.append(e)
898 vals.append(int(nodeid))
899 vals = tuple(vals)
901 # if there's any updates to regular columns, do them
902 if cols:
903 # make sure the ordering is correct for column name -> column value
904 s = ','.join(['_%s=%s'%(x, self.arg) for x in cols])
905 cols = ','.join(cols)
907 # perform the update
908 sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
909 self.sql(sql, vals)
911 # we're probably coming from an import, not a change
912 if not multilink_changes:
913 for name in mls:
914 prop = props[name]
915 value = values[name]
917 t = '%s_%s'%(classname, name)
919 # clear out previous values for this node
920 # XXX numeric ids
921 self.sql('delete from %s where nodeid=%s'%(t, self.arg),
922 (nodeid,))
924 # insert the values for this node
925 for entry in values[name]:
926 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
927 self.arg, self.arg)
928 # XXX numeric ids
929 self.sql(sql, (entry, nodeid))
931 # we have multilink changes to apply
932 for col, (add, remove) in multilink_changes.items():
933 tn = '%s_%s'%(classname, col)
934 if add:
935 sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
936 self.arg, self.arg)
937 for addid in add:
938 # XXX numeric ids
939 self.sql(sql, (int(nodeid), int(addid)))
940 if remove:
941 s = ','.join([self.arg]*len(remove))
942 sql = 'delete from %s where nodeid=%s and linkid in (%s)'%(tn,
943 self.arg, s)
944 # XXX numeric ids
945 self.sql(sql, [int(nodeid)] + remove)
947 sql_to_hyperdb_value = {
948 hyperdb.String : str,
949 hyperdb.Date : lambda x:date.Date(str(x).replace(' ', '.')),
950 # hyperdb.Link : int, # XXX numeric ids
951 hyperdb.Link : str,
952 hyperdb.Interval : date.Interval,
953 hyperdb.Password : lambda x: password.Password(encrypted=x),
954 hyperdb.Boolean : _bool_cvt,
955 hyperdb.Number : _num_cvt,
956 hyperdb.Multilink : lambda x: x, # used in journal marshalling
957 }
958 def getnode(self, classname, nodeid):
959 """ Get a node from the database.
960 """
961 # see if we have this node cached
962 key = (classname, nodeid)
963 if self.cache.has_key(key):
964 # push us back to the top of the LRU
965 self.cache_lru.remove(key)
966 self.cache_lru.insert(0, key)
967 if __debug__:
968 self.stats['cache_hits'] += 1
969 # return the cached information
970 return self.cache[key]
972 if __debug__:
973 self.stats['cache_misses'] += 1
974 start_t = time.time()
976 # figure the columns we're fetching
977 cl = self.classes[classname]
978 cols, mls = self.determine_columns(cl.properties.items())
979 scols = ','.join([col for col,dt in cols])
981 # perform the basic property fetch
982 sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
983 self.sql(sql, (nodeid,))
985 values = self.sql_fetchone()
986 if values is None:
987 raise IndexError, 'no such %s node %s'%(classname, nodeid)
989 # make up the node
990 node = {}
991 props = cl.getprops(protected=1)
992 for col in range(len(cols)):
993 name = cols[col][0][1:]
994 if name.endswith('_int__'):
995 # XXX eugh, this test suxxors
996 # ignore the special Interval-as-seconds column
997 continue
998 value = values[col]
999 if value is not None:
1000 value = self.sql_to_hyperdb_value[props[name].__class__](value)
1001 node[name] = value
1004 # now the multilinks
1005 for col in mls:
1006 # get the link ids
1007 sql = 'select linkid from %s_%s where nodeid=%s'%(classname, col,
1008 self.arg)
1009 self.sql(sql, (nodeid,))
1010 # extract the first column from the result
1011 # XXX numeric ids
1012 items = [int(x[0]) for x in self.cursor.fetchall()]
1013 items.sort ()
1014 node[col] = [str(x) for x in items]
1016 # save off in the cache
1017 key = (classname, nodeid)
1018 self.cache[key] = node
1019 # update the LRU
1020 self.cache_lru.insert(0, key)
1021 if len(self.cache_lru) > ROW_CACHE_SIZE:
1022 del self.cache[self.cache_lru.pop()]
1024 if __debug__:
1025 self.stats['get_items'] += (time.time() - start_t)
1027 return node
1029 def destroynode(self, classname, nodeid):
1030 """Remove a node from the database. Called exclusively by the
1031 destroy() method on Class.
1032 """
1033 logging.getLogger('hyperdb').info('destroynode %s%s'%(classname, nodeid))
1035 # make sure the node exists
1036 if not self.hasnode(classname, nodeid):
1037 raise IndexError, '%s has no node %s'%(classname, nodeid)
1039 # see if we have this node cached
1040 if self.cache.has_key((classname, nodeid)):
1041 del self.cache[(classname, nodeid)]
1043 # see if there's any obvious commit actions that we should get rid of
1044 for entry in self.transactions[:]:
1045 if entry[1][:2] == (classname, nodeid):
1046 self.transactions.remove(entry)
1048 # now do the SQL
1049 sql = 'delete from _%s where id=%s'%(classname, self.arg)
1050 self.sql(sql, (nodeid,))
1052 # remove from multilnks
1053 cl = self.getclass(classname)
1054 x, mls = self.determine_columns(cl.properties.items())
1055 for col in mls:
1056 # get the link ids
1057 sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
1058 self.sql(sql, (nodeid,))
1060 # remove journal entries
1061 sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
1062 self.sql(sql, (nodeid,))
1064 # cleanup any blob filestorage when we commit
1065 self.transactions.append((FileStorage.destroy, (self, classname, nodeid)))
1067 def hasnode(self, classname, nodeid):
1068 """ Determine if the database has a given node.
1069 """
1070 # If this node is in the cache, then we do not need to go to
1071 # the database. (We don't consider this an LRU hit, though.)
1072 if self.cache.has_key((classname, nodeid)):
1073 # Return 1, not True, to match the type of the result of
1074 # the SQL operation below.
1075 return 1
1076 sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
1077 self.sql(sql, (nodeid,))
1078 return int(self.cursor.fetchone()[0])
1080 def countnodes(self, classname):
1081 """ Count the number of nodes that exist for a particular Class.
1082 """
1083 sql = 'select count(*) from _%s'%classname
1084 self.sql(sql)
1085 return self.cursor.fetchone()[0]
1087 def addjournal(self, classname, nodeid, action, params, creator=None,
1088 creation=None):
1089 """ Journal the Action
1090 'action' may be:
1092 'create' or 'set' -- 'params' is a dictionary of property values
1093 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
1094 'retire' -- 'params' is None
1095 """
1096 # handle supply of the special journalling parameters (usually
1097 # supplied on importing an existing database)
1098 if creator:
1099 journaltag = creator
1100 else:
1101 journaltag = self.getuid()
1102 if creation:
1103 journaldate = creation
1104 else:
1105 journaldate = date.Date()
1107 # create the journal entry
1108 cols = 'nodeid,date,tag,action,params'
1110 self.log_debug('addjournal %s%s %r %s %s %r'%(classname,
1111 nodeid, journaldate, journaltag, action, params))
1113 # make the journalled data marshallable
1114 if isinstance(params, type({})):
1115 self._journal_marshal(params, classname)
1117 params = repr(params)
1119 dc = self.hyperdb_to_sql_value[hyperdb.Date]
1120 journaldate = dc(journaldate)
1122 self.save_journal(classname, cols, nodeid, journaldate,
1123 journaltag, action, params)
1125 def setjournal(self, classname, nodeid, journal):
1126 """Set the journal to the "journal" list."""
1127 # clear out any existing entries
1128 self.sql('delete from %s__journal where nodeid=%s'%(classname,
1129 self.arg), (nodeid,))
1131 # create the journal entry
1132 cols = 'nodeid,date,tag,action,params'
1134 dc = self.hyperdb_to_sql_value[hyperdb.Date]
1135 for nodeid, journaldate, journaltag, action, params in journal:
1136 self.log_debug('addjournal %s%s %r %s %s %r'%(
1137 classname, nodeid, journaldate, journaltag, action,
1138 params))
1140 # make the journalled data marshallable
1141 if isinstance(params, type({})):
1142 self._journal_marshal(params, classname)
1143 params = repr(params)
1145 self.save_journal(classname, cols, nodeid, dc(journaldate),
1146 journaltag, action, params)
1148 def _journal_marshal(self, params, classname):
1149 """Convert the journal params values into safely repr'able and
1150 eval'able values."""
1151 properties = self.getclass(classname).getprops()
1152 for param, value in params.items():
1153 if not value:
1154 continue
1155 property = properties[param]
1156 cvt = self.hyperdb_to_sql_value[property.__class__]
1157 if isinstance(property, Password):
1158 params[param] = cvt(value)
1159 elif isinstance(property, Date):
1160 params[param] = cvt(value)
1161 elif isinstance(property, Interval):
1162 params[param] = cvt(value)
1163 elif isinstance(property, Boolean):
1164 params[param] = cvt(value)
1166 def getjournal(self, classname, nodeid):
1167 """ get the journal for id
1168 """
1169 # make sure the node exists
1170 if not self.hasnode(classname, nodeid):
1171 raise IndexError, '%s has no node %s'%(classname, nodeid)
1173 cols = ','.join('nodeid date tag action params'.split())
1174 journal = self.load_journal(classname, cols, nodeid)
1176 # now unmarshal the data
1177 dc = self.sql_to_hyperdb_value[hyperdb.Date]
1178 res = []
1179 properties = self.getclass(classname).getprops()
1180 for nodeid, date_stamp, user, action, params in journal:
1181 params = eval(params)
1182 if isinstance(params, type({})):
1183 for param, value in params.items():
1184 if not value:
1185 continue
1186 property = properties.get(param, None)
1187 if property is None:
1188 # deleted property
1189 continue
1190 cvt = self.sql_to_hyperdb_value[property.__class__]
1191 if isinstance(property, Password):
1192 params[param] = cvt(value)
1193 elif isinstance(property, Date):
1194 params[param] = cvt(value)
1195 elif isinstance(property, Interval):
1196 params[param] = cvt(value)
1197 elif isinstance(property, Boolean):
1198 params[param] = cvt(value)
1199 # XXX numeric ids
1200 res.append((str(nodeid), dc(date_stamp), user, action, params))
1201 return res
1203 def save_journal(self, classname, cols, nodeid, journaldate,
1204 journaltag, action, params):
1205 """ Save the journal entry to the database
1206 """
1207 entry = (nodeid, journaldate, journaltag, action, params)
1209 # do the insert
1210 a = self.arg
1211 sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(
1212 classname, cols, a, a, a, a, a)
1213 self.sql(sql, entry)
1215 def load_journal(self, classname, cols, nodeid):
1216 """ Load the journal from the database
1217 """
1218 # now get the journal entries
1219 sql = 'select %s from %s__journal where nodeid=%s order by date'%(
1220 cols, classname, self.arg)
1221 self.sql(sql, (nodeid,))
1222 return self.cursor.fetchall()
1224 def pack(self, pack_before):
1225 """ Delete all journal entries except "create" before 'pack_before'.
1226 """
1227 date_stamp = self.hyperdb_to_sql_value[Date](pack_before)
1229 # do the delete
1230 for classname in self.classes.keys():
1231 sql = "delete from %s__journal where date<%s and "\
1232 "action<>'create'"%(classname, self.arg)
1233 self.sql(sql, (date_stamp,))
1235 def sql_commit(self, fail_ok=False):
1236 """ Actually commit to the database.
1237 """
1238 logging.getLogger('hyperdb').info('commit')
1240 self.conn.commit()
1242 # open a new cursor for subsequent work
1243 self.cursor = self.conn.cursor()
1245 def commit(self, fail_ok=False):
1246 """ Commit the current transactions.
1248 Save all data changed since the database was opened or since the
1249 last commit() or rollback().
1251 fail_ok indicates that the commit is allowed to fail. This is used
1252 in the web interface when committing cleaning of the session
1253 database. We don't care if there's a concurrency issue there.
1255 The only backend this seems to affect is postgres.
1256 """
1257 # commit the database
1258 self.sql_commit(fail_ok)
1260 # now, do all the other transaction stuff
1261 for method, args in self.transactions:
1262 method(*args)
1264 # save the indexer
1265 self.indexer.save_index()
1267 # clear out the transactions
1268 self.transactions = []
1270 def sql_rollback(self):
1271 self.conn.rollback()
1273 def rollback(self):
1274 """ Reverse all actions from the current transaction.
1276 Undo all the changes made since the database was opened or the last
1277 commit() or rollback() was performed.
1278 """
1279 logging.getLogger('hyperdb').info('rollback')
1281 self.sql_rollback()
1283 # roll back "other" transaction stuff
1284 for method, args in self.transactions:
1285 # delete temporary files
1286 if method == self.doStoreFile:
1287 self.rollbackStoreFile(*args)
1288 self.transactions = []
1290 # clear the cache
1291 self.clearCache()
1293 def sql_close(self):
1294 logging.getLogger('hyperdb').info('close')
1295 self.conn.close()
1297 def close(self):
1298 """ Close off the connection.
1299 """
1300 self.indexer.close()
1301 self.sql_close()
1303 #
1304 # The base Class class
1305 #
1306 class Class(hyperdb.Class):
1307 """ The handle to a particular class of nodes in a hyperdatabase.
1309 All methods except __repr__ and getnode must be implemented by a
1310 concrete backend Class.
1311 """
1313 def schema(self):
1314 """ A dumpable version of the schema that we can store in the
1315 database
1316 """
1317 return (self.key, [(x, repr(y)) for x,y in self.properties.items()])
1319 def enableJournalling(self):
1320 """Turn journalling on for this class
1321 """
1322 self.do_journal = 1
1324 def disableJournalling(self):
1325 """Turn journalling off for this class
1326 """
1327 self.do_journal = 0
1329 # Editing nodes:
1330 def create(self, **propvalues):
1331 """ Create a new node of this class and return its id.
1333 The keyword arguments in 'propvalues' map property names to values.
1335 The values of arguments must be acceptable for the types of their
1336 corresponding properties or a TypeError is raised.
1338 If this class has a key property, it must be present and its value
1339 must not collide with other key strings or a ValueError is raised.
1341 Any other properties on this class that are missing from the
1342 'propvalues' dictionary are set to None.
1344 If an id in a link or multilink property does not refer to a valid
1345 node, an IndexError is raised.
1346 """
1347 self.fireAuditors('create', None, propvalues)
1348 newid = self.create_inner(**propvalues)
1349 self.fireReactors('create', newid, None)
1350 return newid
1352 def create_inner(self, **propvalues):
1353 """ Called by create, in-between the audit and react calls.
1354 """
1355 if propvalues.has_key('id'):
1356 raise KeyError, '"id" is reserved'
1358 if self.db.journaltag is None:
1359 raise DatabaseError, _('Database open read-only')
1361 if propvalues.has_key('creator') or propvalues.has_key('actor') or \
1362 propvalues.has_key('creation') or propvalues.has_key('activity'):
1363 raise KeyError, '"creator", "actor", "creation" and '\
1364 '"activity" are reserved'
1366 # new node's id
1367 newid = self.db.newid(self.classname)
1369 # validate propvalues
1370 num_re = re.compile('^\d+$')
1371 for key, value in propvalues.items():
1372 if key == self.key:
1373 try:
1374 self.lookup(value)
1375 except KeyError:
1376 pass
1377 else:
1378 raise ValueError, 'node with key "%s" exists'%value
1380 # try to handle this property
1381 try:
1382 prop = self.properties[key]
1383 except KeyError:
1384 raise KeyError, '"%s" has no property "%s"'%(self.classname,
1385 key)
1387 if value is not None and isinstance(prop, Link):
1388 if type(value) != type(''):
1389 raise ValueError, 'link value must be String'
1390 link_class = self.properties[key].classname
1391 # if it isn't a number, it's a key
1392 if not num_re.match(value):
1393 try:
1394 value = self.db.classes[link_class].lookup(value)
1395 except (TypeError, KeyError):
1396 raise IndexError, 'new property "%s": %s not a %s'%(
1397 key, value, link_class)
1398 elif not self.db.getclass(link_class).hasnode(value):
1399 raise IndexError, '%s has no node %s'%(link_class, value)
1401 # save off the value
1402 propvalues[key] = value
1404 # register the link with the newly linked node
1405 if self.do_journal and self.properties[key].do_journal:
1406 self.db.addjournal(link_class, value, 'link',
1407 (self.classname, newid, key))
1409 elif isinstance(prop, Multilink):
1410 if value is None:
1411 value = []
1412 if not hasattr(value, '__iter__'):
1413 raise TypeError, 'new property "%s" not an iterable of ids'%key
1415 # clean up and validate the list of links
1416 link_class = self.properties[key].classname
1417 l = []
1418 for entry in value:
1419 if type(entry) != type(''):
1420 raise ValueError, '"%s" multilink value (%r) '\
1421 'must contain Strings'%(key, value)
1422 # if it isn't a number, it's a key
1423 if not num_re.match(entry):
1424 try:
1425 entry = self.db.classes[link_class].lookup(entry)
1426 except (TypeError, KeyError):
1427 raise IndexError, 'new property "%s": %s not a %s'%(
1428 key, entry, self.properties[key].classname)
1429 l.append(entry)
1430 value = l
1431 propvalues[key] = value
1433 # handle additions
1434 for nodeid in value:
1435 if not self.db.getclass(link_class).hasnode(nodeid):
1436 raise IndexError, '%s has no node %s'%(link_class,
1437 nodeid)
1438 # register the link with the newly linked node
1439 if self.do_journal and self.properties[key].do_journal:
1440 self.db.addjournal(link_class, nodeid, 'link',
1441 (self.classname, newid, key))
1443 elif isinstance(prop, String):
1444 if type(value) != type('') and type(value) != type(u''):
1445 raise TypeError, 'new property "%s" not a string'%key
1446 if prop.indexme:
1447 self.db.indexer.add_text((self.classname, newid, key),
1448 value)
1450 elif isinstance(prop, Password):
1451 if not isinstance(value, password.Password):
1452 raise TypeError, 'new property "%s" not a Password'%key
1454 elif isinstance(prop, Date):
1455 if value is not None and not isinstance(value, date.Date):
1456 raise TypeError, 'new property "%s" not a Date'%key
1458 elif isinstance(prop, Interval):
1459 if value is not None and not isinstance(value, date.Interval):
1460 raise TypeError, 'new property "%s" not an Interval'%key
1462 elif value is not None and isinstance(prop, Number):
1463 try:
1464 float(value)
1465 except ValueError:
1466 raise TypeError, 'new property "%s" not numeric'%key
1468 elif value is not None and isinstance(prop, Boolean):
1469 try:
1470 int(value)
1471 except ValueError:
1472 raise TypeError, 'new property "%s" not boolean'%key
1474 # make sure there's data where there needs to be
1475 for key, prop in self.properties.items():
1476 if propvalues.has_key(key):
1477 continue
1478 if key == self.key:
1479 raise ValueError, 'key property "%s" is required'%key
1480 if isinstance(prop, Multilink):
1481 propvalues[key] = []
1482 else:
1483 propvalues[key] = None
1485 # done
1486 self.db.addnode(self.classname, newid, propvalues)
1487 if self.do_journal:
1488 self.db.addjournal(self.classname, newid, ''"create", {})
1490 # XXX numeric ids
1491 return str(newid)
1493 def get(self, nodeid, propname, default=_marker, cache=1):
1494 """Get the value of a property on an existing node of this class.
1496 'nodeid' must be the id of an existing node of this class or an
1497 IndexError is raised. 'propname' must be the name of a property
1498 of this class or a KeyError is raised.
1500 'cache' exists for backwards compatibility, and is not used.
1501 """
1502 if propname == 'id':
1503 return nodeid
1505 # get the node's dict
1506 d = self.db.getnode(self.classname, nodeid)
1508 if propname == 'creation':
1509 if d.has_key('creation'):
1510 return d['creation']
1511 else:
1512 return date.Date()
1513 if propname == 'activity':
1514 if d.has_key('activity'):
1515 return d['activity']
1516 else:
1517 return date.Date()
1518 if propname == 'creator':
1519 if d.has_key('creator'):
1520 return d['creator']
1521 else:
1522 return self.db.getuid()
1523 if propname == 'actor':
1524 if d.has_key('actor'):
1525 return d['actor']
1526 else:
1527 return self.db.getuid()
1529 # get the property (raises KeyErorr if invalid)
1530 prop = self.properties[propname]
1532 # XXX may it be that propname is valid property name
1533 # (above error is not raised) and not d.has_key(propname)???
1534 if (not d.has_key(propname)) or (d[propname] is None):
1535 if default is _marker:
1536 if isinstance(prop, Multilink):
1537 return []
1538 else:
1539 return None
1540 else:
1541 return default
1543 # don't pass our list to other code
1544 if isinstance(prop, Multilink):
1545 return d[propname][:]
1547 return d[propname]
1549 def set(self, nodeid, **propvalues):
1550 """Modify a property on an existing node of this class.
1552 'nodeid' must be the id of an existing node of this class or an
1553 IndexError is raised.
1555 Each key in 'propvalues' must be the name of a property of this
1556 class or a KeyError is raised.
1558 All values in 'propvalues' must be acceptable types for their
1559 corresponding properties or a TypeError is raised.
1561 If the value of the key property is set, it must not collide with
1562 other key strings or a ValueError is raised.
1564 If the value of a Link or Multilink property contains an invalid
1565 node id, a ValueError is raised.
1566 """
1567 self.fireAuditors('set', nodeid, propvalues)
1568 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1569 propvalues = self.set_inner(nodeid, **propvalues)
1570 self.fireReactors('set', nodeid, oldvalues)
1571 return propvalues
1573 def set_inner(self, nodeid, **propvalues):
1574 """ Called by set, in-between the audit and react calls.
1575 """
1576 if not propvalues:
1577 return propvalues
1579 if propvalues.has_key('creation') or propvalues.has_key('creator') or \
1580 propvalues.has_key('actor') or propvalues.has_key('activity'):
1581 raise KeyError, '"creation", "creator", "actor" and '\
1582 '"activity" are reserved'
1584 if propvalues.has_key('id'):
1585 raise KeyError, '"id" is reserved'
1587 if self.db.journaltag is None:
1588 raise DatabaseError, _('Database open read-only')
1590 node = self.db.getnode(self.classname, nodeid)
1591 if self.is_retired(nodeid):
1592 raise IndexError, 'Requested item is retired'
1593 num_re = re.compile('^\d+$')
1595 # make a copy of the values dictionary - we'll modify the contents
1596 propvalues = propvalues.copy()
1598 # if the journal value is to be different, store it in here
1599 journalvalues = {}
1601 # remember the add/remove stuff for multilinks, making it easier
1602 # for the Database layer to do its stuff
1603 multilink_changes = {}
1605 for propname, value in propvalues.items():
1606 # check to make sure we're not duplicating an existing key
1607 if propname == self.key and node[propname] != value:
1608 try:
1609 self.lookup(value)
1610 except KeyError:
1611 pass
1612 else:
1613 raise ValueError, 'node with key "%s" exists'%value
1615 # this will raise the KeyError if the property isn't valid
1616 # ... we don't use getprops() here because we only care about
1617 # the writeable properties.
1618 try:
1619 prop = self.properties[propname]
1620 except KeyError:
1621 raise KeyError, '"%s" has no property named "%s"'%(
1622 self.classname, propname)
1624 # if the value's the same as the existing value, no sense in
1625 # doing anything
1626 current = node.get(propname, None)
1627 if value == current:
1628 del propvalues[propname]
1629 continue
1630 journalvalues[propname] = current
1632 # do stuff based on the prop type
1633 if isinstance(prop, Link):
1634 link_class = prop.classname
1635 # if it isn't a number, it's a key
1636 if value is not None and not isinstance(value, type('')):
1637 raise ValueError, 'property "%s" link value be a string'%(
1638 propname)
1639 if isinstance(value, type('')) and not num_re.match(value):
1640 try:
1641 value = self.db.classes[link_class].lookup(value)
1642 except (TypeError, KeyError):
1643 raise IndexError, 'new property "%s": %s not a %s'%(
1644 propname, value, prop.classname)
1646 if (value is not None and
1647 not self.db.getclass(link_class).hasnode(value)):
1648 raise IndexError, '%s has no node %s'%(link_class, value)
1650 if self.do_journal and prop.do_journal:
1651 # register the unlink with the old linked node
1652 if node[propname] is not None:
1653 self.db.addjournal(link_class, node[propname],
1654 ''"unlink", (self.classname, nodeid, propname))
1656 # register the link with the newly linked node
1657 if value is not None:
1658 self.db.addjournal(link_class, value, ''"link",
1659 (self.classname, nodeid, propname))
1661 elif isinstance(prop, Multilink):
1662 if value is None:
1663 value = []
1664 if not hasattr(value, '__iter__'):
1665 raise TypeError, 'new property "%s" not an iterable of'\
1666 ' ids'%propname
1667 link_class = self.properties[propname].classname
1668 l = []
1669 for entry in value:
1670 # if it isn't a number, it's a key
1671 if type(entry) != type(''):
1672 raise ValueError, 'new property "%s" link value ' \
1673 'must be a string'%propname
1674 if not num_re.match(entry):
1675 try:
1676 entry = self.db.classes[link_class].lookup(entry)
1677 except (TypeError, KeyError):
1678 raise IndexError, 'new property "%s": %s not a %s'%(
1679 propname, entry,
1680 self.properties[propname].classname)
1681 l.append(entry)
1682 value = l
1683 propvalues[propname] = value
1685 # figure the journal entry for this property
1686 add = []
1687 remove = []
1689 # handle removals
1690 if node.has_key(propname):
1691 l = node[propname]
1692 else:
1693 l = []
1694 for id in l[:]:
1695 if id in value:
1696 continue
1697 # register the unlink with the old linked node
1698 if self.do_journal and self.properties[propname].do_journal:
1699 self.db.addjournal(link_class, id, 'unlink',
1700 (self.classname, nodeid, propname))
1701 l.remove(id)
1702 remove.append(id)
1704 # handle additions
1705 for id in value:
1706 # If this node is in the cache, then we do not need to go to
1707 # the database. (We don't consider this an LRU hit, though.)
1708 if self.cache.has_key((classname, nodeid)):
1709 # Return 1, not True, to match the type of the result of
1710 # the SQL operation below.
1711 return 1
1712 if not self.db.getclass(link_class).hasnode(id):
1713 raise IndexError, '%s has no node %s'%(link_class, id)
1714 if id in l:
1715 continue
1716 # register the link with the newly linked node
1717 if self.do_journal and self.properties[propname].do_journal:
1718 self.db.addjournal(link_class, id, 'link',
1719 (self.classname, nodeid, propname))
1720 l.append(id)
1721 add.append(id)
1723 # figure the journal entry
1724 l = []
1725 if add:
1726 l.append(('+', add))
1727 if remove:
1728 l.append(('-', remove))
1729 multilink_changes[propname] = (add, remove)
1730 if l:
1731 journalvalues[propname] = tuple(l)
1733 elif isinstance(prop, String):
1734 if value is not None and type(value) != type('') and type(value) != type(u''):
1735 raise TypeError, 'new property "%s" not a string'%propname
1736 if prop.indexme:
1737 if value is None: value = ''
1738 self.db.indexer.add_text((self.classname, nodeid, propname),
1739 value)
1741 elif isinstance(prop, Password):
1742 if not isinstance(value, password.Password):
1743 raise TypeError, 'new property "%s" not a Password'%propname
1744 propvalues[propname] = value
1746 elif value is not None and isinstance(prop, Date):
1747 if not isinstance(value, date.Date):
1748 raise TypeError, 'new property "%s" not a Date'% propname
1749 propvalues[propname] = value
1751 elif value is not None and isinstance(prop, Interval):
1752 if not isinstance(value, date.Interval):
1753 raise TypeError, 'new property "%s" not an '\
1754 'Interval'%propname
1755 propvalues[propname] = value
1757 elif value is not None and isinstance(prop, Number):
1758 try:
1759 float(value)
1760 except ValueError:
1761 raise TypeError, 'new property "%s" not numeric'%propname
1763 elif value is not None and isinstance(prop, Boolean):
1764 try:
1765 int(value)
1766 except ValueError:
1767 raise TypeError, 'new property "%s" not boolean'%propname
1769 # nothing to do?
1770 if not propvalues:
1771 return propvalues
1773 # update the activity time
1774 propvalues['activity'] = date.Date()
1775 propvalues['actor'] = self.db.getuid()
1777 # do the set
1778 self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
1780 # remove the activity props now they're handled
1781 del propvalues['activity']
1782 del propvalues['actor']
1784 # journal the set
1785 if self.do_journal:
1786 self.db.addjournal(self.classname, nodeid, ''"set", journalvalues)
1788 return propvalues
1790 def retire(self, nodeid):
1791 """Retire a node.
1793 The properties on the node remain available from the get() method,
1794 and the node's id is never reused.
1796 Retired nodes are not returned by the find(), list(), or lookup()
1797 methods, and other nodes may reuse the values of their key properties.
1798 """
1799 if self.db.journaltag is None:
1800 raise DatabaseError, _('Database open read-only')
1802 self.fireAuditors('retire', nodeid, None)
1804 # use the arg for __retired__ to cope with any odd database type
1805 # conversion (hello, sqlite)
1806 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1807 self.db.arg, self.db.arg)
1808 self.db.sql(sql, (nodeid, nodeid))
1809 if self.do_journal:
1810 self.db.addjournal(self.classname, nodeid, ''"retired", None)
1812 self.fireReactors('retire', nodeid, None)
1814 def restore(self, nodeid):
1815 """Restore a retired node.
1817 Make node available for all operations like it was before retirement.
1818 """
1819 if self.db.journaltag is None:
1820 raise DatabaseError, _('Database open read-only')
1822 node = self.db.getnode(self.classname, nodeid)
1823 # check if key property was overrided
1824 key = self.getkey()
1825 try:
1826 id = self.lookup(node[key])
1827 except KeyError:
1828 pass
1829 else:
1830 raise KeyError, "Key property (%s) of retired node clashes with \
1831 existing one (%s)" % (key, node[key])
1833 self.fireAuditors('restore', nodeid, None)
1834 # use the arg for __retired__ to cope with any odd database type
1835 # conversion (hello, sqlite)
1836 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1837 self.db.arg, self.db.arg)
1838 self.db.sql(sql, (0, nodeid))
1839 if self.do_journal:
1840 self.db.addjournal(self.classname, nodeid, ''"restored", None)
1842 self.fireReactors('restore', nodeid, None)
1844 def is_retired(self, nodeid):
1845 """Return true if the node is rerired
1846 """
1847 sql = 'select __retired__ from _%s where id=%s'%(self.classname,
1848 self.db.arg)
1849 self.db.sql(sql, (nodeid,))
1850 return int(self.db.sql_fetchone()[0]) > 0
1852 def destroy(self, nodeid):
1853 """Destroy a node.
1855 WARNING: this method should never be used except in extremely rare
1856 situations where there could never be links to the node being
1857 deleted
1859 WARNING: use retire() instead
1861 WARNING: the properties of this node will not be available ever again
1863 WARNING: really, use retire() instead
1865 Well, I think that's enough warnings. This method exists mostly to
1866 support the session storage of the cgi interface.
1868 The node is completely removed from the hyperdb, including all journal
1869 entries. It will no longer be available, and will generally break code
1870 if there are any references to the node.
1871 """
1872 if self.db.journaltag is None:
1873 raise DatabaseError, _('Database open read-only')
1874 self.db.destroynode(self.classname, nodeid)
1876 def history(self, nodeid):
1877 """Retrieve the journal of edits on a particular node.
1879 'nodeid' must be the id of an existing node of this class or an
1880 IndexError is raised.
1882 The returned list contains tuples of the form
1884 (nodeid, date, tag, action, params)
1886 'date' is a Timestamp object specifying the time of the change and
1887 'tag' is the journaltag specified when the database was opened.
1888 """
1889 if not self.do_journal:
1890 raise ValueError, 'Journalling is disabled for this class'
1891 return self.db.getjournal(self.classname, nodeid)
1893 # Locating nodes:
1894 def hasnode(self, nodeid):
1895 """Determine if the given nodeid actually exists
1896 """
1897 return self.db.hasnode(self.classname, nodeid)
1899 def setkey(self, propname):
1900 """Select a String property of this class to be the key property.
1902 'propname' must be the name of a String property of this class or
1903 None, or a TypeError is raised. The values of the key property on
1904 all existing nodes must be unique or a ValueError is raised.
1905 """
1906 prop = self.getprops()[propname]
1907 if not isinstance(prop, String):
1908 raise TypeError, 'key properties must be String'
1909 self.key = propname
1911 def getkey(self):
1912 """Return the name of the key property for this class or None."""
1913 return self.key
1915 def lookup(self, keyvalue):
1916 """Locate a particular node by its key property and return its id.
1918 If this class has no key property, a TypeError is raised. If the
1919 'keyvalue' matches one of the values for the key property among
1920 the nodes in this class, the matching node's id is returned;
1921 otherwise a KeyError is raised.
1922 """
1923 if not self.key:
1924 raise TypeError, 'No key property set for class %s'%self.classname
1926 # use the arg to handle any odd database type conversion (hello,
1927 # sqlite)
1928 sql = "select id from _%s where _%s=%s and __retired__=%s"%(
1929 self.classname, self.key, self.db.arg, self.db.arg)
1930 self.db.sql(sql, (str(keyvalue), 0))
1932 # see if there was a result that's not retired
1933 row = self.db.sql_fetchone()
1934 if not row:
1935 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1936 keyvalue, self.classname)
1938 # return the id
1939 # XXX numeric ids
1940 return str(row[0])
1942 def find(self, **propspec):
1943 """Get the ids of nodes in this class which link to the given nodes.
1945 'propspec' consists of keyword args propname=nodeid or
1946 propname={nodeid:1, }
1947 'propname' must be the name of a property in this class, or a
1948 KeyError is raised. That property must be a Link or
1949 Multilink property, or a TypeError is raised.
1951 Any node in this class whose 'propname' property links to any of
1952 the nodeids will be returned. Examples::
1954 db.issue.find(messages='1')
1955 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1956 """
1957 # shortcut
1958 if not propspec:
1959 return []
1961 # validate the args
1962 props = self.getprops()
1963 propspec = propspec.items()
1964 for propname, nodeids in propspec:
1965 # check the prop is OK
1966 prop = props[propname]
1967 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1968 raise TypeError, "'%s' not a Link/Multilink property"%propname
1970 # first, links
1971 a = self.db.arg
1972 allvalues = ()
1973 sql = []
1974 where = []
1975 for prop, values in propspec:
1976 if not isinstance(props[prop], hyperdb.Link):
1977 continue
1978 if type(values) is type({}) and len(values) == 1:
1979 values = values.keys()[0]
1980 if type(values) is type(''):
1981 allvalues += (values,)
1982 where.append('_%s = %s'%(prop, a))
1983 elif values is None:
1984 where.append('_%s is NULL'%prop)
1985 else:
1986 values = values.keys()
1987 s = ''
1988 if None in values:
1989 values.remove(None)
1990 s = '_%s is NULL or '%prop
1991 allvalues += tuple(values)
1992 s += '_%s in (%s)'%(prop, ','.join([a]*len(values)))
1993 where.append('(' + s +')')
1994 if where:
1995 allvalues = (0, ) + allvalues
1996 sql.append("""select id from _%s where __retired__=%s
1997 and %s"""%(self.classname, a, ' and '.join(where)))
1999 # now multilinks
2000 for prop, values in propspec:
2001 if not isinstance(props[prop], hyperdb.Multilink):
2002 continue
2003 if not values:
2004 continue
2005 allvalues += (0, )
2006 if type(values) is type(''):
2007 allvalues += (values,)
2008 s = a
2009 else:
2010 allvalues += tuple(values.keys())
2011 s = ','.join([a]*len(values))
2012 tn = '%s_%s'%(self.classname, prop)
2013 sql.append("""select id from _%s, %s where __retired__=%s
2014 and id = %s.nodeid and %s.linkid in (%s)"""%(self.classname,
2015 tn, a, tn, tn, s))
2017 if not sql:
2018 return []
2019 sql = ' union '.join(sql)
2020 self.db.sql(sql, allvalues)
2021 # XXX numeric ids
2022 l = [str(x[0]) for x in self.db.sql_fetchall()]
2023 return l
2025 def stringFind(self, **requirements):
2026 """Locate a particular node by matching a set of its String
2027 properties in a caseless search.
2029 If the property is not a String property, a TypeError is raised.
2031 The return is a list of the id of all nodes that match.
2032 """
2033 where = []
2034 args = []
2035 for propname in requirements.keys():
2036 prop = self.properties[propname]
2037 if not isinstance(prop, String):
2038 raise TypeError, "'%s' not a String property"%propname
2039 where.append(propname)
2040 args.append(requirements[propname].lower())
2042 # generate the where clause
2043 s = ' and '.join(['lower(_%s)=%s'%(col, self.db.arg) for col in where])
2044 sql = 'select id from _%s where %s and __retired__=%s'%(
2045 self.classname, s, self.db.arg)
2046 args.append(0)
2047 self.db.sql(sql, tuple(args))
2048 # XXX numeric ids
2049 l = [str(x[0]) for x in self.db.sql_fetchall()]
2050 return l
2052 def list(self):
2053 """ Return a list of the ids of the active nodes in this class.
2054 """
2055 return self.getnodeids(retired=0)
2057 def getnodeids(self, retired=None):
2058 """ Retrieve all the ids of the nodes for a particular Class.
2060 Set retired=None to get all nodes. Otherwise it'll get all the
2061 retired or non-retired nodes, depending on the flag.
2062 """
2063 # flip the sense of the 'retired' flag if we don't want all of them
2064 if retired is not None:
2065 args = (0, )
2066 if retired:
2067 compare = '>'
2068 else:
2069 compare = '='
2070 sql = 'select id from _%s where __retired__%s%s'%(self.classname,
2071 compare, self.db.arg)
2072 else:
2073 args = ()
2074 sql = 'select id from _%s'%self.classname
2075 self.db.sql(sql, args)
2076 # XXX numeric ids
2077 ids = [str(x[0]) for x in self.db.cursor.fetchall()]
2078 return ids
2080 def _subselect(self, classname, multilink_table):
2081 """Create a subselect. This is factored out because some
2082 databases (hmm only one, so far) doesn't support subselects
2083 look for "I can't believe it's not a toy RDBMS" in the mysql
2084 backend.
2085 """
2086 return '_%s.id not in (select nodeid from %s)'%(classname,
2087 multilink_table)
2089 # Some DBs order NULL values last. Set this variable in the backend
2090 # for prepending an order by clause for each attribute that causes
2091 # correct sort order for NULLs. Examples:
2092 # order_by_null_values = '(%s is not NULL)'
2093 # order_by_null_values = 'notnull(%s)'
2094 # The format parameter is replaced with the attribute.
2095 order_by_null_values = None
2097 def filter(self, search_matches, filterspec, sort=[], group=[]):
2098 """Return a list of the ids of the active nodes in this class that
2099 match the 'filter' spec, sorted by the group spec and then the
2100 sort spec
2102 "filterspec" is {propname: value(s)}
2104 "sort" and "group" are [(dir, prop), ...] where dir is '+', '-'
2105 or None and prop is a prop name or None. Note that for
2106 backward-compatibility reasons a single (dir, prop) tuple is
2107 also allowed.
2109 "search_matches" is a container type or None
2111 The filter must match all properties specificed. If the property
2112 value to match is a list:
2114 1. String properties must match all elements in the list, and
2115 2. Other properties must match any of the elements in the list.
2116 """
2117 # we can't match anything if search_matches is empty
2118 if not search_matches and search_matches is not None:
2119 return []
2121 if __debug__:
2122 start_t = time.time()
2124 icn = self.classname
2126 # vars to hold the components of the SQL statement
2127 frum = [] # FROM clauses
2128 loj = [] # LEFT OUTER JOIN clauses
2129 where = [] # WHERE clauses
2130 args = [] # *any* positional arguments
2131 a = self.db.arg
2133 # figure the WHERE clause from the filterspec
2134 mlfilt = 0 # are we joining with Multilink tables?
2135 sortattr = self._sortattr (group = group, sort = sort)
2136 proptree = self._proptree(filterspec, sortattr)
2137 mlseen = 0
2138 for pt in reversed(proptree.sortattr):
2139 p = pt
2140 while p.parent:
2141 if isinstance (p.propclass, Multilink):
2142 mlseen = True
2143 if mlseen:
2144 p.sort_ids_needed = True
2145 p.tree_sort_done = False
2146 p = p.parent
2147 if not mlseen:
2148 pt.attr_sort_done = pt.tree_sort_done = True
2149 proptree.compute_sort_done()
2151 ordercols = []
2152 auxcols = {}
2153 mlsort = []
2154 rhsnum = 0
2155 for p in proptree:
2156 oc = None
2157 cn = p.classname
2158 ln = p.uniqname
2159 pln = p.parent.uniqname
2160 pcn = p.parent.classname
2161 k = p.name
2162 v = p.val
2163 propclass = p.propclass
2164 if p.sort_type > 0:
2165 oc = ac = '_%s._%s'%(pln, k)
2166 if isinstance(propclass, Multilink):
2167 if p.sort_type < 2:
2168 mlfilt = 1
2169 tn = '%s_%s'%(pcn, k)
2170 if v in ('-1', ['-1']):
2171 # only match rows that have count(linkid)=0 in the
2172 # corresponding multilink table)
2173 where.append(self._subselect(pcn, tn))
2174 else:
2175 frum.append(tn)
2176 where.append('_%s.id=%s.nodeid'%(pln,tn))
2177 if p.children:
2178 frum.append('_%s as _%s' % (cn, ln))
2179 where.append('%s.linkid=_%s.id'%(tn, ln))
2180 if p.has_values:
2181 if isinstance(v, type([])):
2182 s = ','.join([a for x in v])
2183 where.append('%s.linkid in (%s)'%(tn, s))
2184 args = args + v
2185 else:
2186 where.append('%s.linkid=%s'%(tn, a))
2187 args.append(v)
2188 if p.sort_type > 0:
2189 assert not p.attr_sort_done and not p.sort_ids_needed
2190 elif k == 'id':
2191 if p.sort_type < 2:
2192 if isinstance(v, type([])):
2193 s = ','.join([a for x in v])
2194 where.append('_%s.%s in (%s)'%(pln, k, s))
2195 args = args + v
2196 else:
2197 where.append('_%s.%s=%s'%(pln, k, a))
2198 args.append(v)
2199 if p.sort_type > 0:
2200 oc = ac = '_%s.id'%pln
2201 elif isinstance(propclass, String):
2202 if p.sort_type < 2:
2203 if not isinstance(v, type([])):
2204 v = [v]
2206 # Quote the bits in the string that need it and then embed
2207 # in a "substring" search. Note - need to quote the '%' so
2208 # they make it through the python layer happily
2209 v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
2211 # now add to the where clause
2212 where.append('('
2213 +' and '.join(["_%s._%s LIKE '%s'"%(pln, k, s) for s in v])
2214 +')')
2215 # note: args are embedded in the query string now
2216 if p.sort_type > 0:
2217 oc = ac = 'lower(_%s._%s)'%(pln, k)
2218 elif isinstance(propclass, Link):
2219 if p.sort_type < 2:
2220 if p.children:
2221 if p.sort_type == 0:
2222 frum.append('_%s as _%s' % (cn, ln))
2223 where.append('_%s._%s=_%s.id'%(pln, k, ln))
2224 if p.has_values:
2225 if isinstance(v, type([])):
2226 d = {}
2227 for entry in v:
2228 if entry == '-1':
2229 entry = None
2230 d[entry] = entry
2231 l = []
2232 if d.has_key(None) or not d:
2233 if d.has_key(None): del d[None]
2234 l.append('_%s._%s is NULL'%(pln, k))
2235 if d:
2236 v = d.keys()
2237 s = ','.join([a for x in v])
2238 l.append('(_%s._%s in (%s))'%(pln, k, s))
2239 args = args + v
2240 if l:
2241 where.append('(' + ' or '.join(l) +')')
2242 else:
2243 if v in ('-1', None):
2244 v = None
2245 where.append('_%s._%s is NULL'%(pln, k))
2246 else:
2247 where.append('_%s._%s=%s'%(pln, k, a))
2248 args.append(v)
2249 if p.sort_type > 0:
2250 lp = p.cls.labelprop()
2251 oc = ac = '_%s._%s'%(pln, k)
2252 if lp != 'id':
2253 if p.tree_sort_done and p.sort_type > 0:
2254 loj.append(
2255 'LEFT OUTER JOIN _%s as _%s on _%s._%s=_%s.id'%(
2256 cn, ln, pln, k, ln))
2257 oc = '_%s._%s'%(ln, lp)
2258 elif isinstance(propclass, Date) and p.sort_type < 2:
2259 dc = self.db.hyperdb_to_sql_value[hyperdb.Date]
2260 if isinstance(v, type([])):
2261 s = ','.join([a for x in v])
2262 where.append('_%s._%s in (%s)'%(pln, k, s))
2263 args = args + [dc(date.Date(x)) for x in v]
2264 else:
2265 try:
2266 # Try to filter on range of dates
2267 date_rng = propclass.range_from_raw(v, self.db)
2268 if date_rng.from_value:
2269 where.append('_%s._%s >= %s'%(pln, k, a))
2270 args.append(dc(date_rng.from_value))
2271 if date_rng.to_value:
2272 where.append('_%s._%s <= %s'%(pln, k, a))
2273 args.append(dc(date_rng.to_value))
2274 except ValueError:
2275 # If range creation fails - ignore that search parameter
2276 pass
2277 elif isinstance(propclass, Interval):
2278 # filter/sort using the __<prop>_int__ column
2279 if p.sort_type < 2:
2280 if isinstance(v, type([])):
2281 s = ','.join([a for x in v])
2282 where.append('_%s.__%s_int__ in (%s)'%(pln, k, s))
2283 args = args + [date.Interval(x).as_seconds() for x in v]
2284 else:
2285 try:
2286 # Try to filter on range of intervals
2287 date_rng = Range(v, date.Interval)
2288 if date_rng.from_value:
2289 where.append('_%s.__%s_int__ >= %s'%(pln, k, a))
2290 args.append(date_rng.from_value.as_seconds())
2291 if date_rng.to_value:
2292 where.append('_%s.__%s_int__ <= %s'%(pln, k, a))
2293 args.append(date_rng.to_value.as_seconds())
2294 except ValueError:
2295 # If range creation fails - ignore search parameter
2296 pass
2297 if p.sort_type > 0:
2298 oc = ac = '_%s.__%s_int__'%(pln,k)
2299 elif p.sort_type < 2:
2300 if isinstance(v, type([])):
2301 s = ','.join([a for x in v])
2302 where.append('_%s._%s in (%s)'%(pln, k, s))
2303 args = args + v
2304 else:
2305 where.append('_%s._%s=%s'%(pln, k, a))
2306 args.append(v)
2307 if oc:
2308 if p.sort_ids_needed:
2309 auxcols[ac] = p
2310 if p.tree_sort_done and p.sort_direction:
2311 # Don't select top-level id twice
2312 if p.name != 'id' or p.parent != proptree:
2313 ordercols.append(oc)
2314 desc = ['', ' desc'][p.sort_direction == '-']
2315 # Some SQL dbs sort NULL values last -- we want them first.
2316 if (self.order_by_null_values and p.name != 'id'):
2317 nv = self.order_by_null_values % oc
2318 ordercols.append(nv)
2319 p.orderby.append(nv + desc)
2320 p.orderby.append(oc + desc)
2322 props = self.getprops()
2324 # don't match retired nodes
2325 where.append('_%s.__retired__=0'%icn)
2327 # add results of full text search
2328 if search_matches is not None:
2329 s = ','.join([a for x in search_matches])
2330 where.append('_%s.id in (%s)'%(icn, s))
2331 args = args + v
2333 # construct the SQL
2334 frum.append('_'+icn)
2335 frum = ','.join(frum)
2336 if where:
2337 where = ' where ' + (' and '.join(where))
2338 else:
2339 where = ''
2340 if mlfilt:
2341 # we're joining tables on the id, so we will get dupes if we
2342 # don't distinct()
2343 cols = ['distinct(_%s.id)'%icn]
2344 else:
2345 cols = ['_%s.id'%icn]
2346 if ordercols:
2347 cols = cols + ordercols
2348 order = []
2349 # keep correct sequence of order attributes.
2350 for sa in proptree.sortattr:
2351 if not sa.attr_sort_done:
2352 continue
2353 order.extend(sa.orderby)
2354 if order:
2355 order = ' order by %s'%(','.join(order))
2356 else:
2357 order = ''
2358 for o, p in auxcols.iteritems ():
2359 cols.append (o)
2360 p.auxcol = len (cols) - 1
2362 cols = ','.join(cols)
2363 loj = ' '.join(loj)
2364 sql = 'select %s from %s %s %s%s'%(cols, frum, loj, where, order)
2365 args = tuple(args)
2366 __traceback_info__ = (sql, args)
2367 self.db.sql(sql, args)
2368 l = self.db.sql_fetchall()
2370 # Compute values needed for sorting in proptree.sort
2371 for p in auxcols.itervalues():
2372 p.sort_ids = p.sort_result = [row[p.auxcol] for row in l]
2373 # return the IDs (the first column)
2374 # XXX numeric ids
2375 l = [str(row[0]) for row in l]
2376 l = proptree.sort (l)
2378 if __debug__:
2379 self.db.stats['filtering'] += (time.time() - start_t)
2380 return l
2382 def filter_sql(self, sql):
2383 """Return a list of the ids of the items in this class that match
2384 the SQL provided. The SQL is a complete "select" statement.
2386 The SQL select must include the item id as the first column.
2388 This function DOES NOT filter out retired items, add on a where
2389 clause "__retired__=0" if you don't want retired nodes.
2390 """
2391 if __debug__:
2392 start_t = time.time()
2394 self.db.sql(sql)
2395 l = self.db.sql_fetchall()
2397 if __debug__:
2398 self.db.stats['filtering'] += (time.time() - start_t)
2399 return l
2401 def count(self):
2402 """Get the number of nodes in this class.
2404 If the returned integer is 'numnodes', the ids of all the nodes
2405 in this class run from 1 to numnodes, and numnodes+1 will be the
2406 id of the next node to be created in this class.
2407 """
2408 return self.db.countnodes(self.classname)
2410 # Manipulating properties:
2411 def getprops(self, protected=1):
2412 """Return a dictionary mapping property names to property objects.
2413 If the "protected" flag is true, we include protected properties -
2414 those which may not be modified.
2415 """
2416 d = self.properties.copy()
2417 if protected:
2418 d['id'] = String()
2419 d['creation'] = hyperdb.Date()
2420 d['activity'] = hyperdb.Date()
2421 d['creator'] = hyperdb.Link('user')
2422 d['actor'] = hyperdb.Link('user')
2423 return d
2425 def addprop(self, **properties):
2426 """Add properties to this class.
2428 The keyword arguments in 'properties' must map names to property
2429 objects, or a TypeError is raised. None of the keys in 'properties'
2430 may collide with the names of existing properties, or a ValueError
2431 is raised before any properties have been added.
2432 """
2433 for key in properties.keys():
2434 if self.properties.has_key(key):
2435 raise ValueError, key
2436 self.properties.update(properties)
2438 def index(self, nodeid):
2439 """Add (or refresh) the node to search indexes
2440 """
2441 # find all the String properties that have indexme
2442 for prop, propclass in self.getprops().items():
2443 if isinstance(propclass, String) and propclass.indexme:
2444 self.db.indexer.add_text((self.classname, nodeid, prop),
2445 str(self.get(nodeid, prop)))
2447 #
2448 # import / export support
2449 #
2450 def export_list(self, propnames, nodeid):
2451 """ Export a node - generate a list of CSV-able data in the order
2452 specified by propnames for the given node.
2453 """
2454 properties = self.getprops()
2455 l = []
2456 for prop in propnames:
2457 proptype = properties[prop]
2458 value = self.get(nodeid, prop)
2459 # "marshal" data where needed
2460 if value is None:
2461 pass
2462 elif isinstance(proptype, hyperdb.Date):
2463 value = value.get_tuple()
2464 elif isinstance(proptype, hyperdb.Interval):
2465 value = value.get_tuple()
2466 elif isinstance(proptype, hyperdb.Password):
2467 value = str(value)
2468 l.append(repr(value))
2469 l.append(repr(self.is_retired(nodeid)))
2470 return l
2472 def import_list(self, propnames, proplist):
2473 """ Import a node - all information including "id" is present and
2474 should not be sanity checked. Triggers are not triggered. The
2475 journal should be initialised using the "creator" and "created"
2476 information.
2478 Return the nodeid of the node imported.
2479 """
2480 if self.db.journaltag is None:
2481 raise DatabaseError, _('Database open read-only')
2482 properties = self.getprops()
2484 # make the new node's property map
2485 d = {}
2486 retire = 0
2487 if not "id" in propnames:
2488 newid = self.db.newid(self.classname)
2489 else:
2490 newid = eval(proplist[propnames.index("id")])
2491 for i in range(len(propnames)):
2492 # Use eval to reverse the repr() used to output the CSV
2493 value = eval(proplist[i])
2495 # Figure the property for this column
2496 propname = propnames[i]
2498 # "unmarshal" where necessary
2499 if propname == 'id':
2500 continue
2501 elif propname == 'is retired':
2502 # is the item retired?
2503 if int(value):
2504 retire = 1
2505 continue
2506 elif value is None:
2507 d[propname] = None
2508 continue
2510 prop = properties[propname]
2511 if value is None:
2512 # don't set Nones
2513 continue
2514 elif isinstance(prop, hyperdb.Date):
2515 value = date.Date(value)
2516 elif isinstance(prop, hyperdb.Interval):
2517 value = date.Interval(value)
2518 elif isinstance(prop, hyperdb.Password):
2519 pwd = password.Password()
2520 pwd.unpack(value)
2521 value = pwd
2522 elif isinstance(prop, String):
2523 if isinstance(value, unicode):
2524 value = value.encode('utf8')
2525 if not isinstance(value, str):
2526 raise TypeError, \
2527 'new property "%(propname)s" not a string: %(value)r' \
2528 % locals()
2529 if prop.indexme:
2530 self.db.indexer.add_text((self.classname, newid, propname),
2531 value)
2532 d[propname] = value
2534 # get a new id if necessary
2535 if newid is None:
2536 newid = self.db.newid(self.classname)
2538 # insert new node or update existing?
2539 if not self.hasnode(newid):
2540 self.db.addnode(self.classname, newid, d) # insert
2541 else:
2542 self.db.setnode(self.classname, newid, d) # update
2544 # retire?
2545 if retire:
2546 # use the arg for __retired__ to cope with any odd database type
2547 # conversion (hello, sqlite)
2548 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
2549 self.db.arg, self.db.arg)
2550 self.db.sql(sql, (newid, newid))
2551 return newid
2553 def export_journals(self):
2554 """Export a class's journal - generate a list of lists of
2555 CSV-able data:
2557 nodeid, date, user, action, params
2559 No heading here - the columns are fixed.
2560 """
2561 properties = self.getprops()
2562 r = []
2563 for nodeid in self.getnodeids():
2564 for nodeid, date, user, action, params in self.history(nodeid):
2565 date = date.get_tuple()
2566 if action == 'set':
2567 export_data = {}
2568 for propname, value in params.items():
2569 if not properties.has_key(propname):
2570 # property no longer in the schema
2571 continue
2573 prop = properties[propname]
2574 # make sure the params are eval()'able
2575 if value is None:
2576 pass
2577 elif isinstance(prop, Date):
2578 value = value.get_tuple()
2579 elif isinstance(prop, Interval):
2580 value = value.get_tuple()
2581 elif isinstance(prop, Password):
2582 value = str(value)
2583 export_data[propname] = value
2584 params = export_data
2585 elif action == 'create' and params:
2586 # old tracker with data stored in the create!
2587 params = {}
2588 l = [nodeid, date, user, action, params]
2589 r.append(map(repr, l))
2590 return r
2592 def import_journals(self, entries):
2593 """Import a class's journal.
2595 Uses setjournal() to set the journal for each item."""
2596 properties = self.getprops()
2597 d = {}
2598 for l in entries:
2599 l = map(eval, l)
2600 nodeid, jdate, user, action, params = l
2601 r = d.setdefault(nodeid, [])
2602 if action == 'set':
2603 for propname, value in params.items():
2604 prop = properties[propname]
2605 if value is None:
2606 pass
2607 elif isinstance(prop, Date):
2608 value = date.Date(value)
2609 elif isinstance(prop, Interval):
2610 value = date.Interval(value)
2611 elif isinstance(prop, Password):
2612 pwd = password.Password()
2613 pwd.unpack(value)
2614 value = pwd
2615 params[propname] = value
2616 elif action == 'create' and params:
2617 # old tracker with data stored in the create!
2618 params = {}
2619 r.append((nodeid, date.Date(jdate), user, action, params))
2621 for nodeid, l in d.items():
2622 self.db.setjournal(self.classname, nodeid, l)
2624 class FileClass(hyperdb.FileClass, Class):
2625 """This class defines a large chunk of data. To support this, it has a
2626 mandatory String property "content" which is typically saved off
2627 externally to the hyperdb.
2629 The default MIME type of this data is defined by the
2630 "default_mime_type" class attribute, which may be overridden by each
2631 node if the class defines a "type" String property.
2632 """
2633 def __init__(self, db, classname, **properties):
2634 """The newly-created class automatically includes the "content"
2635 and "type" properties.
2636 """
2637 if not properties.has_key('content'):
2638 properties['content'] = hyperdb.String(indexme='yes')
2639 if not properties.has_key('type'):
2640 properties['type'] = hyperdb.String()
2641 Class.__init__(self, db, classname, **properties)
2643 def create(self, **propvalues):
2644 """ snaffle the file propvalue and store in a file
2645 """
2646 # we need to fire the auditors now, or the content property won't
2647 # be in propvalues for the auditors to play with
2648 self.fireAuditors('create', None, propvalues)
2650 # now remove the content property so it's not stored in the db
2651 content = propvalues['content']
2652 del propvalues['content']
2654 # do the database create
2655 newid = self.create_inner(**propvalues)
2657 # figure the mime type
2658 mime_type = propvalues.get('type', self.default_mime_type)
2660 # and index!
2661 if self.properties['content'].indexme:
2662 self.db.indexer.add_text((self.classname, newid, 'content'),
2663 content, mime_type)
2665 # store off the content as a file
2666 self.db.storefile(self.classname, newid, None, content)
2668 # fire reactors
2669 self.fireReactors('create', newid, None)
2671 return newid
2673 def get(self, nodeid, propname, default=_marker, cache=1):
2674 """ Trap the content propname and get it from the file
2676 'cache' exists for backwards compatibility, and is not used.
2677 """
2678 poss_msg = 'Possibly a access right configuration problem.'
2679 if propname == 'content':
2680 try:
2681 return self.db.getfile(self.classname, nodeid, None)
2682 except IOError, (strerror):
2683 # BUG: by catching this we donot see an error in the log.
2684 return 'ERROR reading file: %s%s\n%s\n%s'%(
2685 self.classname, nodeid, poss_msg, strerror)
2686 if default is not _marker:
2687 return Class.get(self, nodeid, propname, default)
2688 else:
2689 return Class.get(self, nodeid, propname)
2691 def set(self, itemid, **propvalues):
2692 """ Snarf the "content" propvalue and update it in a file
2693 """
2694 self.fireAuditors('set', itemid, propvalues)
2695 oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid))
2697 # now remove the content property so it's not stored in the db
2698 content = None
2699 if propvalues.has_key('content'):
2700 content = propvalues['content']
2701 del propvalues['content']
2703 # do the database create
2704 propvalues = self.set_inner(itemid, **propvalues)
2706 # do content?
2707 if content:
2708 # store and possibly index
2709 self.db.storefile(self.classname, itemid, None, content)
2710 if self.properties['content'].indexme:
2711 mime_type = self.get(itemid, 'type', self.default_mime_type)
2712 self.db.indexer.add_text((self.classname, itemid, 'content'),
2713 content, mime_type)
2714 propvalues['content'] = content
2716 # fire reactors
2717 self.fireReactors('set', itemid, oldvalues)
2718 return propvalues
2720 def index(self, nodeid):
2721 """ Add (or refresh) the node to search indexes.
2723 Use the content-type property for the content property.
2724 """
2725 # find all the String properties that have indexme
2726 for prop, propclass in self.getprops().items():
2727 if prop == 'content' and propclass.indexme:
2728 mime_type = self.get(nodeid, 'type', self.default_mime_type)
2729 self.db.indexer.add_text((self.classname, nodeid, 'content'),
2730 str(self.get(nodeid, 'content')), mime_type)
2731 elif isinstance(propclass, hyperdb.String) and propclass.indexme:
2732 # index them under (classname, nodeid, property)
2733 try:
2734 value = str(self.get(nodeid, prop))
2735 except IndexError:
2736 # node has been destroyed
2737 continue
2738 self.db.indexer.add_text((self.classname, nodeid, prop), value)
2740 # XXX deviation from spec - was called ItemClass
2741 class IssueClass(Class, roundupdb.IssueClass):
2742 # Overridden methods:
2743 def __init__(self, db, classname, **properties):
2744 """The newly-created class automatically includes the "messages",
2745 "files", "nosy", and "superseder" properties. If the 'properties'
2746 dictionary attempts to specify any of these properties or a
2747 "creation", "creator", "activity" or "actor" property, a ValueError
2748 is raised.
2749 """
2750 if not properties.has_key('title'):
2751 properties['title'] = hyperdb.String(indexme='yes')
2752 if not properties.has_key('messages'):
2753 properties['messages'] = hyperdb.Multilink("msg")
2754 if not properties.has_key('files'):
2755 properties['files'] = hyperdb.Multilink("file")
2756 if not properties.has_key('nosy'):
2757 # note: journalling is turned off as it really just wastes
2758 # space. this behaviour may be overridden in an instance
2759 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2760 if not properties.has_key('superseder'):
2761 properties['superseder'] = hyperdb.Multilink(classname)
2762 Class.__init__(self, db, classname, **properties)
2764 # vim: set et sts=4 sw=4 :