5168367c96d9762da080b75aed9784580b65a4f5
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: rdbms_common.py,v 1.199 2008-08-18 06:25:47 richard Exp $
19 """ Relational database (SQL) backend common code.
21 Basics:
23 - map roundup classes to relational tables
24 - automatically detect schema changes and modify the table schemas
25 appropriately (we store the "database version" of the schema in the
26 database itself as the only row of the "schema" table)
27 - multilinks (which represent a many-to-many relationship) are handled through
28 intermediate tables
29 - journals are stored adjunct to the per-class tables
30 - table names and columns have "_" prepended so the names can't clash with
31 restricted names (like "order")
32 - retirement is determined by the __retired__ column being > 0
34 Database-specific changes may generally be pushed out to the overridable
35 sql_* methods, since everything else should be fairly generic. There's
36 probably a bit of work to be done if a database is used that actually
37 honors column typing, since the initial databases don't (sqlite stores
38 everything as a string.)
40 The schema of the hyperdb being mapped to the database is stored in the
41 database itself as a repr()'ed dictionary of information about each Class
42 that maps to a table. If that information differs from the hyperdb schema,
43 then we update it. We also store in the schema dict a version which
44 allows us to upgrade the database schema when necessary. See upgrade_db().
46 To force a unqiueness constraint on the key properties we put the item
47 id into the __retired__ column duing retirement (so it's 0 for "active"
48 items) and place a unqiueness constraint on key + __retired__. This is
49 particularly important for the users class where multiple users may
50 try to have the same username, with potentially many retired users with
51 the same name.
52 """
53 __docformat__ = 'restructuredtext'
55 # standard python modules
56 import sys, os, time, re, errno, weakref, copy, logging
58 # roundup modules
59 from roundup import hyperdb, date, password, roundupdb, security, support
60 from roundup.hyperdb import String, Password, Date, Interval, Link, \
61 Multilink, DatabaseError, Boolean, Number, Node
62 from roundup.backends import locking
63 from roundup.support import reversed
64 from roundup.i18n import _
66 # support
67 from blobfiles import FileStorage
68 try:
69 from indexer_xapian import Indexer
70 except ImportError:
71 from indexer_rdbms import Indexer
72 from sessions_rdbms import Sessions, OneTimeKeys
73 from roundup.date import Range
75 # number of rows to keep in memory
76 ROW_CACHE_SIZE = 100
78 # dummy value meaning "argument not passed"
79 _marker = []
81 def _num_cvt(num):
82 num = str(num)
83 try:
84 return int(num)
85 except:
86 return float(num)
88 def _bool_cvt(value):
89 if value in ('TRUE', 'FALSE'):
90 return {'TRUE': 1, 'FALSE': 0}[value]
91 # assume it's a number returned from the db API
92 return int(value)
94 def connection_dict(config, dbnamestr=None):
95 """ Used by Postgresql and MySQL to detemine the keyword args for
96 opening the database connection."""
97 d = { }
98 if dbnamestr:
99 d[dbnamestr] = config.RDBMS_NAME
100 for name in ('host', 'port', 'password', 'user', 'read_default_group',
101 'read_default_file'):
102 cvar = 'RDBMS_'+name.upper()
103 if config[cvar] is not None:
104 d[name] = config[cvar]
105 return d
107 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
108 """ Wrapper around an SQL database that presents a hyperdb interface.
110 - some functionality is specific to the actual SQL database, hence
111 the sql_* methods that are NotImplemented
112 - we keep a cache of the latest ROW_CACHE_SIZE row fetches.
113 """
114 def __init__(self, config, journaltag=None):
115 """ Open the database and load the schema from it.
116 """
117 FileStorage.__init__(self, config.UMASK)
118 self.config, self.journaltag = config, journaltag
119 self.dir = config.DATABASE
120 self.classes = {}
121 self.indexer = Indexer(self)
122 self.security = security.Security(self)
124 # additional transaction support for external files and the like
125 self.transactions = []
127 # keep a cache of the N most recently retrieved rows of any kind
128 # (classname, nodeid) = row
129 self.cache = {}
130 self.cache_lru = []
131 self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0,
132 'filtering': 0}
134 # database lock
135 self.lockfile = None
137 # open a connection to the database, creating the "conn" attribute
138 self.open_connection()
140 def clearCache(self):
141 self.cache = {}
142 self.cache_lru = []
144 def getSessionManager(self):
145 return Sessions(self)
147 def getOTKManager(self):
148 return OneTimeKeys(self)
150 def open_connection(self):
151 """ Open a connection to the database, creating it if necessary.
153 Must call self.load_dbschema()
154 """
155 raise NotImplemented
157 def sql(self, sql, args=None):
158 """ Execute the sql with the optional args.
159 """
160 if __debug__:
161 logging.getLogger('hyperdb').debug('SQL %r %r'%(sql, args))
162 if args:
163 self.cursor.execute(sql, args)
164 else:
165 self.cursor.execute(sql)
167 def sql_fetchone(self):
168 """ Fetch a single row. If there's nothing to fetch, return None.
169 """
170 return self.cursor.fetchone()
172 def sql_fetchall(self):
173 """ Fetch all rows. If there's nothing to fetch, return [].
174 """
175 return self.cursor.fetchall()
177 def sql_stringquote(self, value):
178 """ Quote the string so it's safe to put in the 'sql quotes'
179 """
180 return re.sub("'", "''", str(value))
182 def init_dbschema(self):
183 self.database_schema = {
184 'version': self.current_db_version,
185 'tables': {}
186 }
188 def load_dbschema(self):
189 """ Load the schema definition that the database currently implements
190 """
191 self.cursor.execute('select schema from schema')
192 schema = self.cursor.fetchone()
193 if schema:
194 self.database_schema = eval(schema[0])
195 else:
196 self.database_schema = {}
198 def save_dbschema(self):
199 """ Save the schema definition that the database currently implements
200 """
201 s = repr(self.database_schema)
202 self.sql('delete from schema')
203 self.sql('insert into schema values (%s)'%self.arg, (s,))
205 def post_init(self):
206 """ Called once the schema initialisation has finished.
208 We should now confirm that the schema defined by our "classes"
209 attribute actually matches the schema in the database.
210 """
211 save = 0
213 # handle changes in the schema
214 tables = self.database_schema['tables']
215 for classname, spec in self.classes.items():
216 if tables.has_key(classname):
217 dbspec = tables[classname]
218 if self.update_class(spec, dbspec):
219 tables[classname] = spec.schema()
220 save = 1
221 else:
222 self.create_class(spec)
223 tables[classname] = spec.schema()
224 save = 1
226 for classname, spec in tables.items():
227 if not self.classes.has_key(classname):
228 self.drop_class(classname, tables[classname])
229 del tables[classname]
230 save = 1
232 # now upgrade the database for column type changes, new internal
233 # tables, etc.
234 save = save | self.upgrade_db()
236 # update the database version of the schema
237 if save:
238 self.save_dbschema()
240 # reindex the db if necessary
241 if self.indexer.should_reindex():
242 self.reindex()
244 # commit
245 self.sql_commit()
247 # update this number when we need to make changes to the SQL structure
248 # of the backen database
249 current_db_version = 5
250 db_version_updated = False
251 def upgrade_db(self):
252 """ Update the SQL database to reflect changes in the backend code.
254 Return boolean whether we need to save the schema.
255 """
256 version = self.database_schema.get('version', 1)
257 if version > self.current_db_version:
258 raise DatabaseError('attempting to run rev %d DATABASE with rev '
259 '%d CODE!'%(version, self.current_db_version))
260 if version == self.current_db_version:
261 # nothing to do
262 return 0
264 if version < 2:
265 if __debug__:
266 logging.getLogger('hyperdb').info('upgrade to version 2')
267 # change the schema structure
268 self.database_schema = {'tables': self.database_schema}
270 # version 1 didn't have the actor column (note that in
271 # MySQL this will also transition the tables to typed columns)
272 self.add_new_columns_v2()
274 # version 1 doesn't have the OTK, session and indexing in the
275 # database
276 self.create_version_2_tables()
278 if version < 3:
279 if __debug__:
280 logging.getLogger('hyperdb').info('upgrade to version 3')
281 self.fix_version_2_tables()
283 if version < 4:
284 self.fix_version_3_tables()
286 if version < 5:
287 self.fix_version_4_tables()
289 self.database_schema['version'] = self.current_db_version
290 self.db_version_updated = True
291 return 1
293 def fix_version_3_tables(self):
294 # drop the shorter VARCHAR OTK column and add a new TEXT one
295 for name in ('otk', 'session'):
296 self.sql('DELETE FROM %ss'%name)
297 self.sql('ALTER TABLE %ss DROP %s_value'%(name, name))
298 self.sql('ALTER TABLE %ss ADD %s_value TEXT'%(name, name))
300 def fix_version_2_tables(self):
301 # Default (used by sqlite): NOOP
302 pass
304 def fix_version_4_tables(self):
305 # note this is an explicit call now
306 c = self.cursor
307 for cn, klass in self.classes.items():
308 c.execute('select id from _%s where __retired__<>0'%(cn,))
309 for (id,) in c.fetchall():
310 c.execute('update _%s set __retired__=%s where id=%s'%(cn,
311 self.arg, self.arg), (id, id))
313 if klass.key:
314 self.add_class_key_required_unique_constraint(cn, klass.key)
316 def _convert_journal_tables(self):
317 """Get current journal table contents, drop the table and re-create"""
318 c = self.cursor
319 cols = ','.join('nodeid date tag action params'.split())
320 for klass in self.classes.values():
321 # slurp and drop
322 sql = 'select %s from %s__journal order by date'%(cols,
323 klass.classname)
324 c.execute(sql)
325 contents = c.fetchall()
326 self.drop_journal_table_indexes(klass.classname)
327 c.execute('drop table %s__journal'%klass.classname)
329 # re-create and re-populate
330 self.create_journal_table(klass)
331 a = self.arg
332 sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(
333 klass.classname, cols, a, a, a, a, a)
334 for row in contents:
335 # no data conversion needed
336 self.cursor.execute(sql, row)
338 def _convert_string_properties(self):
339 """Get current Class tables that contain String properties, and
340 convert the VARCHAR columns to TEXT"""
341 c = self.cursor
342 for klass in self.classes.values():
343 # slurp and drop
344 cols, mls = self.determine_columns(klass.properties.items())
345 scols = ','.join([i[0] for i in cols])
346 sql = 'select id,%s from _%s'%(scols, klass.classname)
347 c.execute(sql)
348 contents = c.fetchall()
349 self.drop_class_table_indexes(klass.classname, klass.getkey())
350 c.execute('drop table _%s'%klass.classname)
352 # re-create and re-populate
353 self.create_class_table(klass, create_sequence=0)
354 a = ','.join([self.arg for i in range(len(cols)+1)])
355 sql = 'insert into _%s (id,%s) values (%s)'%(klass.classname,
356 scols, a)
357 for row in contents:
358 l = []
359 for entry in row:
360 # mysql will already be a string - psql needs "help"
361 if entry is not None and not isinstance(entry, type('')):
362 entry = str(entry)
363 l.append(entry)
364 self.cursor.execute(sql, l)
366 def refresh_database(self):
367 self.post_init()
370 def reindex(self, classname=None, show_progress=False):
371 if classname:
372 classes = [self.getclass(classname)]
373 else:
374 classes = self.classes.values()
375 for klass in classes:
376 if show_progress:
377 for nodeid in support.Progress('Reindex %s'%klass.classname,
378 klass.list()):
379 klass.index(nodeid)
380 else:
381 for nodeid in klass.list():
382 klass.index(nodeid)
383 self.indexer.save_index()
385 hyperdb_to_sql_datatypes = {
386 hyperdb.String : 'TEXT',
387 hyperdb.Date : 'TIMESTAMP',
388 hyperdb.Link : 'INTEGER',
389 hyperdb.Interval : 'VARCHAR(255)',
390 hyperdb.Password : 'VARCHAR(255)',
391 hyperdb.Boolean : 'BOOLEAN',
392 hyperdb.Number : 'REAL',
393 }
394 def determine_columns(self, properties):
395 """ Figure the column names and multilink properties from the spec
397 "properties" is a list of (name, prop) where prop may be an
398 instance of a hyperdb "type" _or_ a string repr of that type.
399 """
400 cols = [
401 ('_actor', self.hyperdb_to_sql_datatypes[hyperdb.Link]),
402 ('_activity', self.hyperdb_to_sql_datatypes[hyperdb.Date]),
403 ('_creator', self.hyperdb_to_sql_datatypes[hyperdb.Link]),
404 ('_creation', self.hyperdb_to_sql_datatypes[hyperdb.Date]),
405 ]
406 mls = []
407 # add the multilinks separately
408 for col, prop in properties:
409 if isinstance(prop, Multilink):
410 mls.append(col)
411 continue
413 if isinstance(prop, type('')):
414 raise ValueError, "string property spec!"
415 #and prop.find('Multilink') != -1:
416 #mls.append(col)
418 datatype = self.hyperdb_to_sql_datatypes[prop.__class__]
419 cols.append(('_'+col, datatype))
421 # Intervals stored as two columns
422 if isinstance(prop, Interval):
423 cols.append(('__'+col+'_int__', 'BIGINT'))
425 cols.sort()
426 return cols, mls
428 def update_class(self, spec, old_spec, force=0):
429 """ Determine the differences between the current spec and the
430 database version of the spec, and update where necessary.
432 If 'force' is true, update the database anyway.
433 """
434 new_has = spec.properties.has_key
435 new_spec = spec.schema()
436 new_spec[1].sort()
437 old_spec[1].sort()
438 if not force and new_spec == old_spec:
439 # no changes
440 return 0
442 logger = logging.getLogger('hyperdb')
443 logger.info('update_class %s'%spec.classname)
445 logger.debug('old_spec %r'%(old_spec,))
446 logger.debug('new_spec %r'%(new_spec,))
448 # detect key prop change for potential index change
449 keyprop_changes = {}
450 if new_spec[0] != old_spec[0]:
451 if old_spec[0]:
452 keyprop_changes['remove'] = old_spec[0]
453 if new_spec[0]:
454 keyprop_changes['add'] = new_spec[0]
456 # detect multilinks that have been removed, and drop their table
457 old_has = {}
458 for name, prop in old_spec[1]:
459 old_has[name] = 1
460 if new_has(name):
461 continue
463 if prop.find('Multilink to') != -1:
464 # first drop indexes.
465 self.drop_multilink_table_indexes(spec.classname, name)
467 # now the multilink table itself
468 sql = 'drop table %s_%s'%(spec.classname, name)
469 else:
470 # if this is the key prop, drop the index first
471 if old_spec[0] == prop:
472 self.drop_class_table_key_index(spec.classname, name)
473 del keyprop_changes['remove']
475 # drop the column
476 sql = 'alter table _%s drop column _%s'%(spec.classname, name)
478 self.sql(sql)
479 old_has = old_has.has_key
481 # if we didn't remove the key prop just then, but the key prop has
482 # changed, we still need to remove the old index
483 if keyprop_changes.has_key('remove'):
484 self.drop_class_table_key_index(spec.classname,
485 keyprop_changes['remove'])
487 # add new columns
488 for propname, prop in new_spec[1]:
489 if old_has(propname):
490 continue
491 prop = spec.properties[propname]
492 if isinstance(prop, Multilink):
493 self.create_multilink_table(spec, propname)
494 else:
495 # add the column
496 coltype = self.hyperdb_to_sql_datatypes[prop.__class__]
497 sql = 'alter table _%s add column _%s %s'%(
498 spec.classname, propname, coltype)
499 self.sql(sql)
501 # extra Interval column
502 if isinstance(prop, Interval):
503 sql = 'alter table _%s add column __%s_int__ BIGINT'%(
504 spec.classname, propname)
505 self.sql(sql)
507 # if the new column is a key prop, we need an index!
508 if new_spec[0] == propname:
509 self.create_class_table_key_index(spec.classname, propname)
510 del keyprop_changes['add']
512 # if we didn't add the key prop just then, but the key prop has
513 # changed, we still need to add the new index
514 if keyprop_changes.has_key('add'):
515 self.create_class_table_key_index(spec.classname,
516 keyprop_changes['add'])
518 return 1
520 def determine_all_columns(self, spec):
521 """Figure out the columns from the spec and also add internal columns
523 """
524 cols, mls = self.determine_columns(spec.properties.items())
526 # add on our special columns
527 cols.append(('id', 'INTEGER PRIMARY KEY'))
528 cols.append(('__retired__', 'INTEGER DEFAULT 0'))
529 return cols, mls
531 def create_class_table(self, spec):
532 """Create the class table for the given Class "spec". Creates the
533 indexes too."""
534 cols, mls = self.determine_all_columns(spec)
536 # create the base table
537 scols = ','.join(['%s %s'%x for x in cols])
538 sql = 'create table _%s (%s)'%(spec.classname, scols)
539 self.sql(sql)
541 self.create_class_table_indexes(spec)
543 return cols, mls
545 def create_class_table_indexes(self, spec):
546 """ create the class table for the given spec
547 """
548 # create __retired__ index
549 index_sql2 = 'create index _%s_retired_idx on _%s(__retired__)'%(
550 spec.classname, spec.classname)
551 self.sql(index_sql2)
553 # create index for key property
554 if spec.key:
555 index_sql3 = 'create index _%s_%s_idx on _%s(_%s)'%(
556 spec.classname, spec.key,
557 spec.classname, spec.key)
558 self.sql(index_sql3)
560 # and the unique index for key / retired(id)
561 self.add_class_key_required_unique_constraint(spec.classname,
562 spec.key)
564 # TODO: create indexes on (selected?) Link property columns, as
565 # they're more likely to be used for lookup
567 def add_class_key_required_unique_constraint(self, cn, key):
568 sql = '''create unique index _%s_key_retired_idx
569 on _%s(__retired__, _%s)'''%(cn, cn, key)
570 self.sql(sql)
572 def drop_class_table_indexes(self, cn, key):
573 # drop the old table indexes first
574 l = ['_%s_id_idx'%cn, '_%s_retired_idx'%cn]
575 if key:
576 l.append('_%s_%s_idx'%(cn, key))
578 table_name = '_%s'%cn
579 for index_name in l:
580 if not self.sql_index_exists(table_name, index_name):
581 continue
582 index_sql = 'drop index '+index_name
583 self.sql(index_sql)
585 def create_class_table_key_index(self, cn, key):
586 """ create the class table for the given spec
587 """
588 sql = 'create index _%s_%s_idx on _%s(_%s)'%(cn, key, cn, key)
589 self.sql(sql)
591 def drop_class_table_key_index(self, cn, key):
592 table_name = '_%s'%cn
593 index_name = '_%s_%s_idx'%(cn, key)
594 if self.sql_index_exists(table_name, index_name):
595 sql = 'drop index '+index_name
596 self.sql(sql)
598 # and now the retired unique index too
599 index_name = '_%s_key_retired_idx'%cn
600 if self.sql_index_exists(table_name, index_name):
601 sql = 'drop index '+index_name
602 self.sql(sql)
604 def create_journal_table(self, spec):
605 """ create the journal table for a class given the spec and
606 already-determined cols
607 """
608 # journal table
609 cols = ','.join(['%s varchar'%x
610 for x in 'nodeid date tag action params'.split()])
611 sql = """create table %s__journal (
612 nodeid integer, date %s, tag varchar(255),
613 action varchar(255), params text)""" % (spec.classname,
614 self.hyperdb_to_sql_datatypes[hyperdb.Date])
615 self.sql(sql)
616 self.create_journal_table_indexes(spec)
618 def create_journal_table_indexes(self, spec):
619 # index on nodeid
620 sql = 'create index %s_journ_idx on %s__journal(nodeid)'%(
621 spec.classname, spec.classname)
622 self.sql(sql)
624 def drop_journal_table_indexes(self, classname):
625 index_name = '%s_journ_idx'%classname
626 if not self.sql_index_exists('%s__journal'%classname, index_name):
627 return
628 index_sql = 'drop index '+index_name
629 self.sql(index_sql)
631 def create_multilink_table(self, spec, ml):
632 """ Create a multilink table for the "ml" property of the class
633 given by the spec
634 """
635 # create the table
636 sql = 'create table %s_%s (linkid INTEGER, nodeid INTEGER)'%(
637 spec.classname, ml)
638 self.sql(sql)
639 self.create_multilink_table_indexes(spec, ml)
641 def create_multilink_table_indexes(self, spec, ml):
642 # create index on linkid
643 index_sql = 'create index %s_%s_l_idx on %s_%s(linkid)'%(
644 spec.classname, ml, spec.classname, ml)
645 self.sql(index_sql)
647 # create index on nodeid
648 index_sql = 'create index %s_%s_n_idx on %s_%s(nodeid)'%(
649 spec.classname, ml, spec.classname, ml)
650 self.sql(index_sql)
652 def drop_multilink_table_indexes(self, classname, ml):
653 l = [
654 '%s_%s_l_idx'%(classname, ml),
655 '%s_%s_n_idx'%(classname, ml)
656 ]
657 table_name = '%s_%s'%(classname, ml)
658 for index_name in l:
659 if not self.sql_index_exists(table_name, index_name):
660 continue
661 index_sql = 'drop index %s'%index_name
662 self.sql(index_sql)
664 def create_class(self, spec):
665 """ Create a database table according to the given spec.
666 """
667 cols, mls = self.create_class_table(spec)
668 self.create_journal_table(spec)
670 # now create the multilink tables
671 for ml in mls:
672 self.create_multilink_table(spec, ml)
674 def drop_class(self, cn, spec):
675 """ Drop the given table from the database.
677 Drop the journal and multilink tables too.
678 """
679 properties = spec[1]
680 # figure the multilinks
681 mls = []
682 for propname, prop in properties:
683 if isinstance(prop, Multilink):
684 mls.append(propname)
686 # drop class table and indexes
687 self.drop_class_table_indexes(cn, spec[0])
689 self.drop_class_table(cn)
691 # drop journal table and indexes
692 self.drop_journal_table_indexes(cn)
693 sql = 'drop table %s__journal'%cn
694 self.sql(sql)
696 for ml in mls:
697 # drop multilink table and indexes
698 self.drop_multilink_table_indexes(cn, ml)
699 sql = 'drop table %s_%s'%(spec.classname, ml)
700 self.sql(sql)
702 def drop_class_table(self, cn):
703 sql = 'drop table _%s'%cn
704 self.sql(sql)
706 #
707 # Classes
708 #
709 def __getattr__(self, classname):
710 """ A convenient way of calling self.getclass(classname).
711 """
712 if self.classes.has_key(classname):
713 return self.classes[classname]
714 raise AttributeError, classname
716 def addclass(self, cl):
717 """ Add a Class to the hyperdatabase.
718 """
719 cn = cl.classname
720 if self.classes.has_key(cn):
721 raise ValueError, cn
722 self.classes[cn] = cl
724 # add default Edit and View permissions
725 self.security.addPermission(name="Create", klass=cn,
726 description="User is allowed to create "+cn)
727 self.security.addPermission(name="Edit", klass=cn,
728 description="User is allowed to edit "+cn)
729 self.security.addPermission(name="View", klass=cn,
730 description="User is allowed to access "+cn)
732 def getclasses(self):
733 """ Return a list of the names of all existing classes.
734 """
735 l = self.classes.keys()
736 l.sort()
737 return l
739 def getclass(self, classname):
740 """Get the Class object representing a particular class.
742 If 'classname' is not a valid class name, a KeyError is raised.
743 """
744 try:
745 return self.classes[classname]
746 except KeyError:
747 raise KeyError, 'There is no class called "%s"'%classname
749 def clear(self):
750 """Delete all database contents.
752 Note: I don't commit here, which is different behaviour to the
753 "nuke from orbit" behaviour in the dbs.
754 """
755 logging.getLogger('hyperdb').info('clear')
756 for cn in self.classes.keys():
757 sql = 'delete from _%s'%cn
758 self.sql(sql)
760 #
761 # Nodes
762 #
764 hyperdb_to_sql_value = {
765 hyperdb.String : str,
766 # fractional seconds by default
767 hyperdb.Date : lambda x: x.formal(sep=' ', sec='%06.3f'),
768 hyperdb.Link : int,
769 hyperdb.Interval : str,
770 hyperdb.Password : str,
771 hyperdb.Boolean : lambda x: x and 'TRUE' or 'FALSE',
772 hyperdb.Number : lambda x: x,
773 hyperdb.Multilink : lambda x: x, # used in journal marshalling
774 }
775 def addnode(self, classname, nodeid, node):
776 """ Add the specified node to its class's db.
777 """
778 if __debug__:
779 logging.getLogger('hyperdb').debug('addnode %s%s %r'%(classname,
780 nodeid, node))
782 # determine the column definitions and multilink tables
783 cl = self.classes[classname]
784 cols, mls = self.determine_columns(cl.properties.items())
786 # we'll be supplied these props if we're doing an import
787 values = node.copy()
788 if not values.has_key('creator'):
789 # add in the "calculated" properties (dupe so we don't affect
790 # calling code's node assumptions)
791 values['creation'] = values['activity'] = date.Date()
792 values['actor'] = values['creator'] = self.getuid()
794 cl = self.classes[classname]
795 props = cl.getprops(protected=1)
796 del props['id']
798 # default the non-multilink columns
799 for col, prop in props.items():
800 if not values.has_key(col):
801 if isinstance(prop, Multilink):
802 values[col] = []
803 else:
804 values[col] = None
806 # clear this node out of the cache if it's in there
807 key = (classname, nodeid)
808 if self.cache.has_key(key):
809 del self.cache[key]
810 self.cache_lru.remove(key)
812 # figure the values to insert
813 vals = []
814 for col,dt in cols:
815 # this is somewhat dodgy....
816 if col.endswith('_int__'):
817 # XXX eugh, this test suxxors
818 value = values[col[2:-6]]
819 # this is an Interval special "int" column
820 if value is not None:
821 vals.append(value.as_seconds())
822 else:
823 vals.append(value)
824 continue
826 prop = props[col[1:]]
827 value = values[col[1:]]
828 if value is not None:
829 value = self.hyperdb_to_sql_value[prop.__class__](value)
830 vals.append(value)
831 vals.append(nodeid)
832 vals = tuple(vals)
834 # make sure the ordering is correct for column name -> column value
835 s = ','.join([self.arg for x in cols]) + ',%s'%self.arg
836 cols = ','.join([col for col,dt in cols]) + ',id'
838 # perform the inserts
839 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
840 self.sql(sql, vals)
842 # insert the multilink rows
843 for col in mls:
844 t = '%s_%s'%(classname, col)
845 for entry in node[col]:
846 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
847 self.arg, self.arg)
848 self.sql(sql, (entry, nodeid))
850 def setnode(self, classname, nodeid, values, multilink_changes={}):
851 """ Change the specified node.
852 """
853 if __debug__:
854 logging.getLogger('hyperdb').debug('setnode %s%s %r'
855 % (classname, nodeid, values))
857 # clear this node out of the cache if it's in there
858 key = (classname, nodeid)
859 if self.cache.has_key(key):
860 del self.cache[key]
861 self.cache_lru.remove(key)
863 cl = self.classes[classname]
864 props = cl.getprops()
866 cols = []
867 mls = []
868 # add the multilinks separately
869 for col in values.keys():
870 prop = props[col]
871 if isinstance(prop, Multilink):
872 mls.append(col)
873 elif isinstance(prop, Interval):
874 # Intervals store the seconds value too
875 cols.append(col)
876 # extra leading '_' added by code below
877 cols.append('_' +col + '_int__')
878 else:
879 cols.append(col)
880 cols.sort()
882 # figure the values to insert
883 vals = []
884 for col in cols:
885 if col.endswith('_int__'):
886 # XXX eugh, this test suxxors
887 # Intervals store the seconds value too
888 col = col[1:-6]
889 prop = props[col]
890 value = values[col]
891 if value is None:
892 vals.append(None)
893 else:
894 vals.append(value.as_seconds())
895 else:
896 prop = props[col]
897 value = values[col]
898 if value is None:
899 e = None
900 else:
901 e = self.hyperdb_to_sql_value[prop.__class__](value)
902 vals.append(e)
904 vals.append(int(nodeid))
905 vals = tuple(vals)
907 # if there's any updates to regular columns, do them
908 if cols:
909 # make sure the ordering is correct for column name -> column value
910 s = ','.join(['_%s=%s'%(x, self.arg) for x in cols])
911 cols = ','.join(cols)
913 # perform the update
914 sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
915 self.sql(sql, vals)
917 # we're probably coming from an import, not a change
918 if not multilink_changes:
919 for name in mls:
920 prop = props[name]
921 value = values[name]
923 t = '%s_%s'%(classname, name)
925 # clear out previous values for this node
926 # XXX numeric ids
927 self.sql('delete from %s where nodeid=%s'%(t, self.arg),
928 (nodeid,))
930 # insert the values for this node
931 for entry in values[name]:
932 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
933 self.arg, self.arg)
934 # XXX numeric ids
935 self.sql(sql, (entry, nodeid))
937 # we have multilink changes to apply
938 for col, (add, remove) in multilink_changes.items():
939 tn = '%s_%s'%(classname, col)
940 if add:
941 sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
942 self.arg, self.arg)
943 for addid in add:
944 # XXX numeric ids
945 self.sql(sql, (int(nodeid), int(addid)))
946 if remove:
947 sql = 'delete from %s where nodeid=%s and linkid=%s'%(tn,
948 self.arg, self.arg)
949 for removeid in remove:
950 # XXX numeric ids
951 self.sql(sql, (int(nodeid), int(removeid)))
953 sql_to_hyperdb_value = {
954 hyperdb.String : str,
955 hyperdb.Date : lambda x:date.Date(str(x).replace(' ', '.')),
956 # hyperdb.Link : int, # XXX numeric ids
957 hyperdb.Link : str,
958 hyperdb.Interval : date.Interval,
959 hyperdb.Password : lambda x: password.Password(encrypted=x),
960 hyperdb.Boolean : _bool_cvt,
961 hyperdb.Number : _num_cvt,
962 hyperdb.Multilink : lambda x: x, # used in journal marshalling
963 }
964 def getnode(self, classname, nodeid):
965 """ Get a node from the database.
966 """
967 # see if we have this node cached
968 key = (classname, nodeid)
969 if self.cache.has_key(key):
970 # push us back to the top of the LRU
971 self.cache_lru.remove(key)
972 self.cache_lru.insert(0, key)
973 if __debug__:
974 self.stats['cache_hits'] += 1
975 # return the cached information
976 return self.cache[key]
978 if __debug__:
979 self.stats['cache_misses'] += 1
980 start_t = time.time()
982 # figure the columns we're fetching
983 cl = self.classes[classname]
984 cols, mls = self.determine_columns(cl.properties.items())
985 scols = ','.join([col for col,dt in cols])
987 # perform the basic property fetch
988 sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
989 self.sql(sql, (nodeid,))
991 values = self.sql_fetchone()
992 if values is None:
993 raise IndexError, 'no such %s node %s'%(classname, nodeid)
995 # make up the node
996 node = {}
997 props = cl.getprops(protected=1)
998 for col in range(len(cols)):
999 name = cols[col][0][1:]
1000 if name.endswith('_int__'):
1001 # XXX eugh, this test suxxors
1002 # ignore the special Interval-as-seconds column
1003 continue
1004 value = values[col]
1005 if value is not None:
1006 value = self.sql_to_hyperdb_value[props[name].__class__](value)
1007 node[name] = value
1010 # now the multilinks
1011 for col in mls:
1012 # get the link ids
1013 sql = 'select linkid from %s_%s where nodeid=%s'%(classname, col,
1014 self.arg)
1015 self.cursor.execute(sql, (nodeid,))
1016 # extract the first column from the result
1017 # XXX numeric ids
1018 items = [int(x[0]) for x in self.cursor.fetchall()]
1019 items.sort ()
1020 node[col] = [str(x) for x in items]
1022 # save off in the cache
1023 key = (classname, nodeid)
1024 self.cache[key] = node
1025 # update the LRU
1026 self.cache_lru.insert(0, key)
1027 if len(self.cache_lru) > ROW_CACHE_SIZE:
1028 del self.cache[self.cache_lru.pop()]
1030 if __debug__:
1031 self.stats['get_items'] += (time.time() - start_t)
1033 return node
1035 def destroynode(self, classname, nodeid):
1036 """Remove a node from the database. Called exclusively by the
1037 destroy() method on Class.
1038 """
1039 logging.getLogger('hyperdb').info('destroynode %s%s'%(classname, nodeid))
1041 # make sure the node exists
1042 if not self.hasnode(classname, nodeid):
1043 raise IndexError, '%s has no node %s'%(classname, nodeid)
1045 # see if we have this node cached
1046 if self.cache.has_key((classname, nodeid)):
1047 del self.cache[(classname, nodeid)]
1049 # see if there's any obvious commit actions that we should get rid of
1050 for entry in self.transactions[:]:
1051 if entry[1][:2] == (classname, nodeid):
1052 self.transactions.remove(entry)
1054 # now do the SQL
1055 sql = 'delete from _%s where id=%s'%(classname, self.arg)
1056 self.sql(sql, (nodeid,))
1058 # remove from multilnks
1059 cl = self.getclass(classname)
1060 x, mls = self.determine_columns(cl.properties.items())
1061 for col in mls:
1062 # get the link ids
1063 sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
1064 self.sql(sql, (nodeid,))
1066 # remove journal entries
1067 sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
1068 self.sql(sql, (nodeid,))
1070 # cleanup any blob filestorage when we commit
1071 self.transactions.append((FileStorage.destroy, (self, classname, nodeid)))
1073 def hasnode(self, classname, nodeid):
1074 """ Determine if the database has a given node.
1075 """
1076 sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
1077 self.sql(sql, (nodeid,))
1078 return int(self.cursor.fetchone()[0])
1080 def countnodes(self, classname):
1081 """ Count the number of nodes that exist for a particular Class.
1082 """
1083 sql = 'select count(*) from _%s'%classname
1084 self.sql(sql)
1085 return self.cursor.fetchone()[0]
1087 def addjournal(self, classname, nodeid, action, params, creator=None,
1088 creation=None):
1089 """ Journal the Action
1090 'action' may be:
1092 'create' or 'set' -- 'params' is a dictionary of property values
1093 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
1094 'retire' -- 'params' is None
1095 """
1096 # handle supply of the special journalling parameters (usually
1097 # supplied on importing an existing database)
1098 if creator:
1099 journaltag = creator
1100 else:
1101 journaltag = self.getuid()
1102 if creation:
1103 journaldate = creation
1104 else:
1105 journaldate = date.Date()
1107 # create the journal entry
1108 cols = 'nodeid,date,tag,action,params'
1110 if __debug__:
1111 logging.getLogger('hyperdb').debug('addjournal %s%s %r %s %s %r'%(classname,
1112 nodeid, journaldate, journaltag, action, params))
1114 # make the journalled data marshallable
1115 if isinstance(params, type({})):
1116 self._journal_marshal(params, classname)
1118 params = repr(params)
1120 dc = self.hyperdb_to_sql_value[hyperdb.Date]
1121 journaldate = dc(journaldate)
1123 self.save_journal(classname, cols, nodeid, journaldate,
1124 journaltag, action, params)
1126 def setjournal(self, classname, nodeid, journal):
1127 """Set the journal to the "journal" list."""
1128 # clear out any existing entries
1129 self.sql('delete from %s__journal where nodeid=%s'%(classname,
1130 self.arg), (nodeid,))
1132 # create the journal entry
1133 cols = 'nodeid,date,tag,action,params'
1135 dc = self.hyperdb_to_sql_value[hyperdb.Date]
1136 for nodeid, journaldate, journaltag, action, params in journal:
1137 if __debug__:
1138 logging.getLogger('hyperdb').debug('addjournal %s%s %r %s %s %r'%(
1139 classname, nodeid, journaldate, journaltag, action,
1140 params))
1142 # make the journalled data marshallable
1143 if isinstance(params, type({})):
1144 self._journal_marshal(params, classname)
1145 params = repr(params)
1147 self.save_journal(classname, cols, nodeid, dc(journaldate),
1148 journaltag, action, params)
1150 def _journal_marshal(self, params, classname):
1151 """Convert the journal params values into safely repr'able and
1152 eval'able values."""
1153 properties = self.getclass(classname).getprops()
1154 for param, value in params.items():
1155 if not value:
1156 continue
1157 property = properties[param]
1158 cvt = self.hyperdb_to_sql_value[property.__class__]
1159 if isinstance(property, Password):
1160 params[param] = cvt(value)
1161 elif isinstance(property, Date):
1162 params[param] = cvt(value)
1163 elif isinstance(property, Interval):
1164 params[param] = cvt(value)
1165 elif isinstance(property, Boolean):
1166 params[param] = cvt(value)
1168 def getjournal(self, classname, nodeid):
1169 """ get the journal for id
1170 """
1171 # make sure the node exists
1172 if not self.hasnode(classname, nodeid):
1173 raise IndexError, '%s has no node %s'%(classname, nodeid)
1175 cols = ','.join('nodeid date tag action params'.split())
1176 journal = self.load_journal(classname, cols, nodeid)
1178 # now unmarshal the data
1179 dc = self.sql_to_hyperdb_value[hyperdb.Date]
1180 res = []
1181 properties = self.getclass(classname).getprops()
1182 for nodeid, date_stamp, user, action, params in journal:
1183 params = eval(params)
1184 if isinstance(params, type({})):
1185 for param, value in params.items():
1186 if not value:
1187 continue
1188 property = properties.get(param, None)
1189 if property is None:
1190 # deleted property
1191 continue
1192 cvt = self.sql_to_hyperdb_value[property.__class__]
1193 if isinstance(property, Password):
1194 params[param] = cvt(value)
1195 elif isinstance(property, Date):
1196 params[param] = cvt(value)
1197 elif isinstance(property, Interval):
1198 params[param] = cvt(value)
1199 elif isinstance(property, Boolean):
1200 params[param] = cvt(value)
1201 # XXX numeric ids
1202 res.append((str(nodeid), dc(date_stamp), user, action, params))
1203 return res
1205 def save_journal(self, classname, cols, nodeid, journaldate,
1206 journaltag, action, params):
1207 """ Save the journal entry to the database
1208 """
1209 entry = (nodeid, journaldate, journaltag, action, params)
1211 # do the insert
1212 a = self.arg
1213 sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(
1214 classname, cols, a, a, a, a, a)
1215 self.sql(sql, entry)
1217 def load_journal(self, classname, cols, nodeid):
1218 """ Load the journal from the database
1219 """
1220 # now get the journal entries
1221 sql = 'select %s from %s__journal where nodeid=%s order by date'%(
1222 cols, classname, self.arg)
1223 self.sql(sql, (nodeid,))
1224 return self.cursor.fetchall()
1226 def pack(self, pack_before):
1227 """ Delete all journal entries except "create" before 'pack_before'.
1228 """
1229 date_stamp = self.hyperdb_to_sql_value[Date](pack_before)
1231 # do the delete
1232 for classname in self.classes.keys():
1233 sql = "delete from %s__journal where date<%s and "\
1234 "action<>'create'"%(classname, self.arg)
1235 self.sql(sql, (date_stamp,))
1237 def sql_commit(self, fail_ok=False):
1238 """ Actually commit to the database.
1239 """
1240 logging.getLogger('hyperdb').info('commit')
1242 self.conn.commit()
1244 # open a new cursor for subsequent work
1245 self.cursor = self.conn.cursor()
1247 def commit(self, fail_ok=False):
1248 """ Commit the current transactions.
1250 Save all data changed since the database was opened or since the
1251 last commit() or rollback().
1253 fail_ok indicates that the commit is allowed to fail. This is used
1254 in the web interface when committing cleaning of the session
1255 database. We don't care if there's a concurrency issue there.
1257 The only backend this seems to affect is postgres.
1258 """
1259 # commit the database
1260 self.sql_commit(fail_ok)
1262 # now, do all the other transaction stuff
1263 for method, args in self.transactions:
1264 method(*args)
1266 # save the indexer
1267 self.indexer.save_index()
1269 # clear out the transactions
1270 self.transactions = []
1272 def sql_rollback(self):
1273 self.conn.rollback()
1275 def rollback(self):
1276 """ Reverse all actions from the current transaction.
1278 Undo all the changes made since the database was opened or the last
1279 commit() or rollback() was performed.
1280 """
1281 logging.getLogger('hyperdb').info('rollback')
1283 self.sql_rollback()
1285 # roll back "other" transaction stuff
1286 for method, args in self.transactions:
1287 # delete temporary files
1288 if method == self.doStoreFile:
1289 self.rollbackStoreFile(*args)
1290 self.transactions = []
1292 # clear the cache
1293 self.clearCache()
1295 def sql_close(self):
1296 logging.getLogger('hyperdb').info('close')
1297 self.conn.close()
1299 def close(self):
1300 """ Close off the connection.
1301 """
1302 self.indexer.close()
1303 self.sql_close()
1305 #
1306 # The base Class class
1307 #
1308 class Class(hyperdb.Class):
1309 """ The handle to a particular class of nodes in a hyperdatabase.
1311 All methods except __repr__ and getnode must be implemented by a
1312 concrete backend Class.
1313 """
1315 def schema(self):
1316 """ A dumpable version of the schema that we can store in the
1317 database
1318 """
1319 return (self.key, [(x, repr(y)) for x,y in self.properties.items()])
1321 def enableJournalling(self):
1322 """Turn journalling on for this class
1323 """
1324 self.do_journal = 1
1326 def disableJournalling(self):
1327 """Turn journalling off for this class
1328 """
1329 self.do_journal = 0
1331 # Editing nodes:
1332 def create(self, **propvalues):
1333 """ Create a new node of this class and return its id.
1335 The keyword arguments in 'propvalues' map property names to values.
1337 The values of arguments must be acceptable for the types of their
1338 corresponding properties or a TypeError is raised.
1340 If this class has a key property, it must be present and its value
1341 must not collide with other key strings or a ValueError is raised.
1343 Any other properties on this class that are missing from the
1344 'propvalues' dictionary are set to None.
1346 If an id in a link or multilink property does not refer to a valid
1347 node, an IndexError is raised.
1348 """
1349 self.fireAuditors('create', None, propvalues)
1350 newid = self.create_inner(**propvalues)
1351 self.fireReactors('create', newid, None)
1352 return newid
1354 def create_inner(self, **propvalues):
1355 """ Called by create, in-between the audit and react calls.
1356 """
1357 if propvalues.has_key('id'):
1358 raise KeyError, '"id" is reserved'
1360 if self.db.journaltag is None:
1361 raise DatabaseError, _('Database open read-only')
1363 if propvalues.has_key('creator') or propvalues.has_key('actor') or \
1364 propvalues.has_key('creation') or propvalues.has_key('activity'):
1365 raise KeyError, '"creator", "actor", "creation" and '\
1366 '"activity" are reserved'
1368 # new node's id
1369 newid = self.db.newid(self.classname)
1371 # validate propvalues
1372 num_re = re.compile('^\d+$')
1373 for key, value in propvalues.items():
1374 if key == self.key:
1375 try:
1376 self.lookup(value)
1377 except KeyError:
1378 pass
1379 else:
1380 raise ValueError, 'node with key "%s" exists'%value
1382 # try to handle this property
1383 try:
1384 prop = self.properties[key]
1385 except KeyError:
1386 raise KeyError, '"%s" has no property "%s"'%(self.classname,
1387 key)
1389 if value is not None and isinstance(prop, Link):
1390 if type(value) != type(''):
1391 raise ValueError, 'link value must be String'
1392 link_class = self.properties[key].classname
1393 # if it isn't a number, it's a key
1394 if not num_re.match(value):
1395 try:
1396 value = self.db.classes[link_class].lookup(value)
1397 except (TypeError, KeyError):
1398 raise IndexError, 'new property "%s": %s not a %s'%(
1399 key, value, link_class)
1400 elif not self.db.getclass(link_class).hasnode(value):
1401 raise IndexError, '%s has no node %s'%(link_class, value)
1403 # save off the value
1404 propvalues[key] = value
1406 # register the link with the newly linked node
1407 if self.do_journal and self.properties[key].do_journal:
1408 self.db.addjournal(link_class, value, 'link',
1409 (self.classname, newid, key))
1411 elif isinstance(prop, Multilink):
1412 if value is None:
1413 value = []
1414 if not hasattr(value, '__iter__'):
1415 raise TypeError, 'new property "%s" not an iterable of ids'%key
1417 # clean up and validate the list of links
1418 link_class = self.properties[key].classname
1419 l = []
1420 for entry in value:
1421 if type(entry) != type(''):
1422 raise ValueError, '"%s" multilink value (%r) '\
1423 'must contain Strings'%(key, value)
1424 # if it isn't a number, it's a key
1425 if not num_re.match(entry):
1426 try:
1427 entry = self.db.classes[link_class].lookup(entry)
1428 except (TypeError, KeyError):
1429 raise IndexError, 'new property "%s": %s not a %s'%(
1430 key, entry, self.properties[key].classname)
1431 l.append(entry)
1432 value = l
1433 propvalues[key] = value
1435 # handle additions
1436 for nodeid in value:
1437 if not self.db.getclass(link_class).hasnode(nodeid):
1438 raise IndexError, '%s has no node %s'%(link_class,
1439 nodeid)
1440 # register the link with the newly linked node
1441 if self.do_journal and self.properties[key].do_journal:
1442 self.db.addjournal(link_class, nodeid, 'link',
1443 (self.classname, newid, key))
1445 elif isinstance(prop, String):
1446 if type(value) != type('') and type(value) != type(u''):
1447 raise TypeError, 'new property "%s" not a string'%key
1448 if prop.indexme:
1449 self.db.indexer.add_text((self.classname, newid, key),
1450 value)
1452 elif isinstance(prop, Password):
1453 if not isinstance(value, password.Password):
1454 raise TypeError, 'new property "%s" not a Password'%key
1456 elif isinstance(prop, Date):
1457 if value is not None and not isinstance(value, date.Date):
1458 raise TypeError, 'new property "%s" not a Date'%key
1460 elif isinstance(prop, Interval):
1461 if value is not None and not isinstance(value, date.Interval):
1462 raise TypeError, 'new property "%s" not an Interval'%key
1464 elif value is not None and isinstance(prop, Number):
1465 try:
1466 float(value)
1467 except ValueError:
1468 raise TypeError, 'new property "%s" not numeric'%key
1470 elif value is not None and isinstance(prop, Boolean):
1471 try:
1472 int(value)
1473 except ValueError:
1474 raise TypeError, 'new property "%s" not boolean'%key
1476 # make sure there's data where there needs to be
1477 for key, prop in self.properties.items():
1478 if propvalues.has_key(key):
1479 continue
1480 if key == self.key:
1481 raise ValueError, 'key property "%s" is required'%key
1482 if isinstance(prop, Multilink):
1483 propvalues[key] = []
1484 else:
1485 propvalues[key] = None
1487 # done
1488 self.db.addnode(self.classname, newid, propvalues)
1489 if self.do_journal:
1490 self.db.addjournal(self.classname, newid, ''"create", {})
1492 # XXX numeric ids
1493 return str(newid)
1495 def get(self, nodeid, propname, default=_marker, cache=1):
1496 """Get the value of a property on an existing node of this class.
1498 'nodeid' must be the id of an existing node of this class or an
1499 IndexError is raised. 'propname' must be the name of a property
1500 of this class or a KeyError is raised.
1502 'cache' exists for backwards compatibility, and is not used.
1503 """
1504 if propname == 'id':
1505 return nodeid
1507 # get the node's dict
1508 d = self.db.getnode(self.classname, nodeid)
1510 if propname == 'creation':
1511 if d.has_key('creation'):
1512 return d['creation']
1513 else:
1514 return date.Date()
1515 if propname == 'activity':
1516 if d.has_key('activity'):
1517 return d['activity']
1518 else:
1519 return date.Date()
1520 if propname == 'creator':
1521 if d.has_key('creator'):
1522 return d['creator']
1523 else:
1524 return self.db.getuid()
1525 if propname == 'actor':
1526 if d.has_key('actor'):
1527 return d['actor']
1528 else:
1529 return self.db.getuid()
1531 # get the property (raises KeyErorr if invalid)
1532 prop = self.properties[propname]
1534 # XXX may it be that propname is valid property name
1535 # (above error is not raised) and not d.has_key(propname)???
1536 if (not d.has_key(propname)) or (d[propname] is None):
1537 if default is _marker:
1538 if isinstance(prop, Multilink):
1539 return []
1540 else:
1541 return None
1542 else:
1543 return default
1545 # don't pass our list to other code
1546 if isinstance(prop, Multilink):
1547 return d[propname][:]
1549 return d[propname]
1551 def set(self, nodeid, **propvalues):
1552 """Modify a property on an existing node of this class.
1554 'nodeid' must be the id of an existing node of this class or an
1555 IndexError is raised.
1557 Each key in 'propvalues' must be the name of a property of this
1558 class or a KeyError is raised.
1560 All values in 'propvalues' must be acceptable types for their
1561 corresponding properties or a TypeError is raised.
1563 If the value of the key property is set, it must not collide with
1564 other key strings or a ValueError is raised.
1566 If the value of a Link or Multilink property contains an invalid
1567 node id, a ValueError is raised.
1568 """
1569 self.fireAuditors('set', nodeid, propvalues)
1570 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1571 propvalues = self.set_inner(nodeid, **propvalues)
1572 self.fireReactors('set', nodeid, oldvalues)
1573 return propvalues
1575 def set_inner(self, nodeid, **propvalues):
1576 """ Called by set, in-between the audit and react calls.
1577 """
1578 if not propvalues:
1579 return propvalues
1581 if propvalues.has_key('creation') or propvalues.has_key('creator') or \
1582 propvalues.has_key('actor') or propvalues.has_key('activity'):
1583 raise KeyError, '"creation", "creator", "actor" and '\
1584 '"activity" are reserved'
1586 if propvalues.has_key('id'):
1587 raise KeyError, '"id" is reserved'
1589 if self.db.journaltag is None:
1590 raise DatabaseError, _('Database open read-only')
1592 node = self.db.getnode(self.classname, nodeid)
1593 if self.is_retired(nodeid):
1594 raise IndexError, 'Requested item is retired'
1595 num_re = re.compile('^\d+$')
1597 # make a copy of the values dictionary - we'll modify the contents
1598 propvalues = propvalues.copy()
1600 # if the journal value is to be different, store it in here
1601 journalvalues = {}
1603 # remember the add/remove stuff for multilinks, making it easier
1604 # for the Database layer to do its stuff
1605 multilink_changes = {}
1607 for propname, value in propvalues.items():
1608 # check to make sure we're not duplicating an existing key
1609 if propname == self.key and node[propname] != value:
1610 try:
1611 self.lookup(value)
1612 except KeyError:
1613 pass
1614 else:
1615 raise ValueError, 'node with key "%s" exists'%value
1617 # this will raise the KeyError if the property isn't valid
1618 # ... we don't use getprops() here because we only care about
1619 # the writeable properties.
1620 try:
1621 prop = self.properties[propname]
1622 except KeyError:
1623 raise KeyError, '"%s" has no property named "%s"'%(
1624 self.classname, propname)
1626 # if the value's the same as the existing value, no sense in
1627 # doing anything
1628 current = node.get(propname, None)
1629 if value == current:
1630 del propvalues[propname]
1631 continue
1632 journalvalues[propname] = current
1634 # do stuff based on the prop type
1635 if isinstance(prop, Link):
1636 link_class = prop.classname
1637 # if it isn't a number, it's a key
1638 if value is not None and not isinstance(value, type('')):
1639 raise ValueError, 'property "%s" link value be a string'%(
1640 propname)
1641 if isinstance(value, type('')) and not num_re.match(value):
1642 try:
1643 value = self.db.classes[link_class].lookup(value)
1644 except (TypeError, KeyError):
1645 raise IndexError, 'new property "%s": %s not a %s'%(
1646 propname, value, prop.classname)
1648 if (value is not None and
1649 not self.db.getclass(link_class).hasnode(value)):
1650 raise IndexError, '%s has no node %s'%(link_class, value)
1652 if self.do_journal and prop.do_journal:
1653 # register the unlink with the old linked node
1654 if node[propname] is not None:
1655 self.db.addjournal(link_class, node[propname],
1656 ''"unlink", (self.classname, nodeid, propname))
1658 # register the link with the newly linked node
1659 if value is not None:
1660 self.db.addjournal(link_class, value, ''"link",
1661 (self.classname, nodeid, propname))
1663 elif isinstance(prop, Multilink):
1664 if value is None:
1665 value = []
1666 if not hasattr(value, '__iter__'):
1667 raise TypeError, 'new property "%s" not an iterable of'\
1668 ' ids'%propname
1669 link_class = self.properties[propname].classname
1670 l = []
1671 for entry in value:
1672 # if it isn't a number, it's a key
1673 if type(entry) != type(''):
1674 raise ValueError, 'new property "%s" link value ' \
1675 'must be a string'%propname
1676 if not num_re.match(entry):
1677 try:
1678 entry = self.db.classes[link_class].lookup(entry)
1679 except (TypeError, KeyError):
1680 raise IndexError, 'new property "%s": %s not a %s'%(
1681 propname, entry,
1682 self.properties[propname].classname)
1683 l.append(entry)
1684 value = l
1685 propvalues[propname] = value
1687 # figure the journal entry for this property
1688 add = []
1689 remove = []
1691 # handle removals
1692 if node.has_key(propname):
1693 l = node[propname]
1694 else:
1695 l = []
1696 for id in l[:]:
1697 if id in value:
1698 continue
1699 # register the unlink with the old linked node
1700 if self.do_journal and self.properties[propname].do_journal:
1701 self.db.addjournal(link_class, id, 'unlink',
1702 (self.classname, nodeid, propname))
1703 l.remove(id)
1704 remove.append(id)
1706 # handle additions
1707 for id in value:
1708 if not self.db.getclass(link_class).hasnode(id):
1709 raise IndexError, '%s has no node %s'%(link_class, id)
1710 if id in l:
1711 continue
1712 # register the link with the newly linked node
1713 if self.do_journal and self.properties[propname].do_journal:
1714 self.db.addjournal(link_class, id, 'link',
1715 (self.classname, nodeid, propname))
1716 l.append(id)
1717 add.append(id)
1719 # figure the journal entry
1720 l = []
1721 if add:
1722 l.append(('+', add))
1723 if remove:
1724 l.append(('-', remove))
1725 multilink_changes[propname] = (add, remove)
1726 if l:
1727 journalvalues[propname] = tuple(l)
1729 elif isinstance(prop, String):
1730 if value is not None and type(value) != type('') and type(value) != type(u''):
1731 raise TypeError, 'new property "%s" not a string'%propname
1732 if prop.indexme:
1733 if value is None: value = ''
1734 self.db.indexer.add_text((self.classname, nodeid, propname),
1735 value)
1737 elif isinstance(prop, Password):
1738 if not isinstance(value, password.Password):
1739 raise TypeError, 'new property "%s" not a Password'%propname
1740 propvalues[propname] = value
1742 elif value is not None and isinstance(prop, Date):
1743 if not isinstance(value, date.Date):
1744 raise TypeError, 'new property "%s" not a Date'% propname
1745 propvalues[propname] = value
1747 elif value is not None and isinstance(prop, Interval):
1748 if not isinstance(value, date.Interval):
1749 raise TypeError, 'new property "%s" not an '\
1750 'Interval'%propname
1751 propvalues[propname] = value
1753 elif value is not None and isinstance(prop, Number):
1754 try:
1755 float(value)
1756 except ValueError:
1757 raise TypeError, 'new property "%s" not numeric'%propname
1759 elif value is not None and isinstance(prop, Boolean):
1760 try:
1761 int(value)
1762 except ValueError:
1763 raise TypeError, 'new property "%s" not boolean'%propname
1765 # nothing to do?
1766 if not propvalues:
1767 return propvalues
1769 # update the activity time
1770 propvalues['activity'] = date.Date()
1771 propvalues['actor'] = self.db.getuid()
1773 # do the set
1774 self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
1776 # remove the activity props now they're handled
1777 del propvalues['activity']
1778 del propvalues['actor']
1780 # journal the set
1781 if self.do_journal:
1782 self.db.addjournal(self.classname, nodeid, ''"set", journalvalues)
1784 return propvalues
1786 def retire(self, nodeid):
1787 """Retire a node.
1789 The properties on the node remain available from the get() method,
1790 and the node's id is never reused.
1792 Retired nodes are not returned by the find(), list(), or lookup()
1793 methods, and other nodes may reuse the values of their key properties.
1794 """
1795 if self.db.journaltag is None:
1796 raise DatabaseError, _('Database open read-only')
1798 self.fireAuditors('retire', nodeid, None)
1800 # use the arg for __retired__ to cope with any odd database type
1801 # conversion (hello, sqlite)
1802 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1803 self.db.arg, self.db.arg)
1804 self.db.sql(sql, (nodeid, nodeid))
1805 if self.do_journal:
1806 self.db.addjournal(self.classname, nodeid, ''"retired", None)
1808 self.fireReactors('retire', nodeid, None)
1810 def restore(self, nodeid):
1811 """Restore a retired node.
1813 Make node available for all operations like it was before retirement.
1814 """
1815 if self.db.journaltag is None:
1816 raise DatabaseError, _('Database open read-only')
1818 node = self.db.getnode(self.classname, nodeid)
1819 # check if key property was overrided
1820 key = self.getkey()
1821 try:
1822 id = self.lookup(node[key])
1823 except KeyError:
1824 pass
1825 else:
1826 raise KeyError, "Key property (%s) of retired node clashes with \
1827 existing one (%s)" % (key, node[key])
1829 self.fireAuditors('restore', nodeid, None)
1830 # use the arg for __retired__ to cope with any odd database type
1831 # conversion (hello, sqlite)
1832 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1833 self.db.arg, self.db.arg)
1834 self.db.sql(sql, (0, nodeid))
1835 if self.do_journal:
1836 self.db.addjournal(self.classname, nodeid, ''"restored", None)
1838 self.fireReactors('restore', nodeid, None)
1840 def is_retired(self, nodeid):
1841 """Return true if the node is rerired
1842 """
1843 sql = 'select __retired__ from _%s where id=%s'%(self.classname,
1844 self.db.arg)
1845 self.db.sql(sql, (nodeid,))
1846 return int(self.db.sql_fetchone()[0]) > 0
1848 def destroy(self, nodeid):
1849 """Destroy a node.
1851 WARNING: this method should never be used except in extremely rare
1852 situations where there could never be links to the node being
1853 deleted
1855 WARNING: use retire() instead
1857 WARNING: the properties of this node will not be available ever again
1859 WARNING: really, use retire() instead
1861 Well, I think that's enough warnings. This method exists mostly to
1862 support the session storage of the cgi interface.
1864 The node is completely removed from the hyperdb, including all journal
1865 entries. It will no longer be available, and will generally break code
1866 if there are any references to the node.
1867 """
1868 if self.db.journaltag is None:
1869 raise DatabaseError, _('Database open read-only')
1870 self.db.destroynode(self.classname, nodeid)
1872 def history(self, nodeid):
1873 """Retrieve the journal of edits on a particular node.
1875 'nodeid' must be the id of an existing node of this class or an
1876 IndexError is raised.
1878 The returned list contains tuples of the form
1880 (nodeid, date, tag, action, params)
1882 'date' is a Timestamp object specifying the time of the change and
1883 'tag' is the journaltag specified when the database was opened.
1884 """
1885 if not self.do_journal:
1886 raise ValueError, 'Journalling is disabled for this class'
1887 return self.db.getjournal(self.classname, nodeid)
1889 # Locating nodes:
1890 def hasnode(self, nodeid):
1891 """Determine if the given nodeid actually exists
1892 """
1893 return self.db.hasnode(self.classname, nodeid)
1895 def setkey(self, propname):
1896 """Select a String property of this class to be the key property.
1898 'propname' must be the name of a String property of this class or
1899 None, or a TypeError is raised. The values of the key property on
1900 all existing nodes must be unique or a ValueError is raised.
1901 """
1902 prop = self.getprops()[propname]
1903 if not isinstance(prop, String):
1904 raise TypeError, 'key properties must be String'
1905 self.key = propname
1907 def getkey(self):
1908 """Return the name of the key property for this class or None."""
1909 return self.key
1911 def lookup(self, keyvalue):
1912 """Locate a particular node by its key property and return its id.
1914 If this class has no key property, a TypeError is raised. If the
1915 'keyvalue' matches one of the values for the key property among
1916 the nodes in this class, the matching node's id is returned;
1917 otherwise a KeyError is raised.
1918 """
1919 if not self.key:
1920 raise TypeError, 'No key property set for class %s'%self.classname
1922 # use the arg to handle any odd database type conversion (hello,
1923 # sqlite)
1924 sql = "select id from _%s where _%s=%s and __retired__=%s"%(
1925 self.classname, self.key, self.db.arg, self.db.arg)
1926 self.db.sql(sql, (str(keyvalue), 0))
1928 # see if there was a result that's not retired
1929 row = self.db.sql_fetchone()
1930 if not row:
1931 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1932 keyvalue, self.classname)
1934 # return the id
1935 # XXX numeric ids
1936 return str(row[0])
1938 def find(self, **propspec):
1939 """Get the ids of nodes in this class which link to the given nodes.
1941 'propspec' consists of keyword args propname=nodeid or
1942 propname={nodeid:1, }
1943 'propname' must be the name of a property in this class, or a
1944 KeyError is raised. That property must be a Link or
1945 Multilink property, or a TypeError is raised.
1947 Any node in this class whose 'propname' property links to any of
1948 the nodeids will be returned. Examples::
1950 db.issue.find(messages='1')
1951 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1952 """
1953 # shortcut
1954 if not propspec:
1955 return []
1957 # validate the args
1958 props = self.getprops()
1959 propspec = propspec.items()
1960 for propname, nodeids in propspec:
1961 # check the prop is OK
1962 prop = props[propname]
1963 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1964 raise TypeError, "'%s' not a Link/Multilink property"%propname
1966 # first, links
1967 a = self.db.arg
1968 allvalues = ()
1969 sql = []
1970 where = []
1971 for prop, values in propspec:
1972 if not isinstance(props[prop], hyperdb.Link):
1973 continue
1974 if type(values) is type({}) and len(values) == 1:
1975 values = values.keys()[0]
1976 if type(values) is type(''):
1977 allvalues += (values,)
1978 where.append('_%s = %s'%(prop, a))
1979 elif values is None:
1980 where.append('_%s is NULL'%prop)
1981 else:
1982 values = values.keys()
1983 s = ''
1984 if None in values:
1985 values.remove(None)
1986 s = '_%s is NULL or '%prop
1987 allvalues += tuple(values)
1988 s += '_%s in (%s)'%(prop, ','.join([a]*len(values)))
1989 where.append('(' + s +')')
1990 if where:
1991 allvalues = (0, ) + allvalues
1992 sql.append("""select id from _%s where __retired__=%s
1993 and %s"""%(self.classname, a, ' and '.join(where)))
1995 # now multilinks
1996 for prop, values in propspec:
1997 if not isinstance(props[prop], hyperdb.Multilink):
1998 continue
1999 if not values:
2000 continue
2001 allvalues += (0, )
2002 if type(values) is type(''):
2003 allvalues += (values,)
2004 s = a
2005 else:
2006 allvalues += tuple(values.keys())
2007 s = ','.join([a]*len(values))
2008 tn = '%s_%s'%(self.classname, prop)
2009 sql.append("""select id from _%s, %s where __retired__=%s
2010 and id = %s.nodeid and %s.linkid in (%s)"""%(self.classname,
2011 tn, a, tn, tn, s))
2013 if not sql:
2014 return []
2015 sql = ' union '.join(sql)
2016 self.db.sql(sql, allvalues)
2017 # XXX numeric ids
2018 l = [str(x[0]) for x in self.db.sql_fetchall()]
2019 return l
2021 def stringFind(self, **requirements):
2022 """Locate a particular node by matching a set of its String
2023 properties in a caseless search.
2025 If the property is not a String property, a TypeError is raised.
2027 The return is a list of the id of all nodes that match.
2028 """
2029 where = []
2030 args = []
2031 for propname in requirements.keys():
2032 prop = self.properties[propname]
2033 if not isinstance(prop, String):
2034 raise TypeError, "'%s' not a String property"%propname
2035 where.append(propname)
2036 args.append(requirements[propname].lower())
2038 # generate the where clause
2039 s = ' and '.join(['lower(_%s)=%s'%(col, self.db.arg) for col in where])
2040 sql = 'select id from _%s where %s and __retired__=%s'%(
2041 self.classname, s, self.db.arg)
2042 args.append(0)
2043 self.db.sql(sql, tuple(args))
2044 # XXX numeric ids
2045 l = [str(x[0]) for x in self.db.sql_fetchall()]
2046 return l
2048 def list(self):
2049 """ Return a list of the ids of the active nodes in this class.
2050 """
2051 return self.getnodeids(retired=0)
2053 def getnodeids(self, retired=None):
2054 """ Retrieve all the ids of the nodes for a particular Class.
2056 Set retired=None to get all nodes. Otherwise it'll get all the
2057 retired or non-retired nodes, depending on the flag.
2058 """
2059 # flip the sense of the 'retired' flag if we don't want all of them
2060 if retired is not None:
2061 args = (0, )
2062 if retired:
2063 compare = '>'
2064 else:
2065 compare = '='
2066 sql = 'select id from _%s where __retired__%s%s'%(self.classname,
2067 compare, self.db.arg)
2068 else:
2069 args = ()
2070 sql = 'select id from _%s'%self.classname
2071 self.db.sql(sql, args)
2072 # XXX numeric ids
2073 ids = [str(x[0]) for x in self.db.cursor.fetchall()]
2074 return ids
2076 def _subselect(self, classname, multilink_table):
2077 """Create a subselect. This is factored out because some
2078 databases (hmm only one, so far) doesn't support subselects
2079 look for "I can't believe it's not a toy RDBMS" in the mysql
2080 backend.
2081 """
2082 return '_%s.id not in (select nodeid from %s)'%(classname,
2083 multilink_table)
2085 # Some DBs order NULL values last. Set this variable in the backend
2086 # for prepending an order by clause for each attribute that causes
2087 # correct sort order for NULLs. Examples:
2088 # order_by_null_values = '(%s is not NULL)'
2089 # order_by_null_values = 'notnull(%s)'
2090 # The format parameter is replaced with the attribute.
2091 order_by_null_values = None
2093 def filter(self, search_matches, filterspec, sort=[], group=[]):
2094 """Return a list of the ids of the active nodes in this class that
2095 match the 'filter' spec, sorted by the group spec and then the
2096 sort spec
2098 "filterspec" is {propname: value(s)}
2100 "sort" and "group" are [(dir, prop), ...] where dir is '+', '-'
2101 or None and prop is a prop name or None. Note that for
2102 backward-compatibility reasons a single (dir, prop) tuple is
2103 also allowed.
2105 "search_matches" is {nodeid: marker} or None
2107 The filter must match all properties specificed. If the property
2108 value to match is a list:
2110 1. String properties must match all elements in the list, and
2111 2. Other properties must match any of the elements in the list.
2112 """
2113 # we can't match anything if search_matches is empty
2114 if search_matches == {}:
2115 return []
2117 if __debug__:
2118 start_t = time.time()
2120 icn = self.classname
2122 # vars to hold the components of the SQL statement
2123 frum = [] # FROM clauses
2124 loj = [] # LEFT OUTER JOIN clauses
2125 where = [] # WHERE clauses
2126 args = [] # *any* positional arguments
2127 a = self.db.arg
2129 # figure the WHERE clause from the filterspec
2130 mlfilt = 0 # are we joining with Multilink tables?
2131 sortattr = self._sortattr (group = group, sort = sort)
2132 proptree = self._proptree(filterspec, sortattr)
2133 mlseen = 0
2134 for pt in reversed(proptree.sortattr):
2135 p = pt
2136 while p.parent:
2137 if isinstance (p.propclass, Multilink):
2138 mlseen = True
2139 if mlseen:
2140 p.sort_ids_needed = True
2141 p.tree_sort_done = False
2142 p = p.parent
2143 if not mlseen:
2144 pt.attr_sort_done = pt.tree_sort_done = True
2145 proptree.compute_sort_done()
2147 ordercols = []
2148 auxcols = {}
2149 mlsort = []
2150 rhsnum = 0
2151 for p in proptree:
2152 oc = None
2153 cn = p.classname
2154 ln = p.uniqname
2155 pln = p.parent.uniqname
2156 pcn = p.parent.classname
2157 k = p.name
2158 v = p.val
2159 propclass = p.propclass
2160 if p.sort_type > 0:
2161 oc = ac = '_%s._%s'%(pln, k)
2162 if isinstance(propclass, Multilink):
2163 if p.sort_type < 2:
2164 mlfilt = 1
2165 tn = '%s_%s'%(pcn, k)
2166 if v in ('-1', ['-1']):
2167 # only match rows that have count(linkid)=0 in the
2168 # corresponding multilink table)
2169 where.append(self._subselect(pcn, tn))
2170 else:
2171 frum.append(tn)
2172 where.append('_%s.id=%s.nodeid'%(pln,tn))
2173 if p.children:
2174 frum.append('_%s as _%s' % (cn, ln))
2175 where.append('%s.linkid=_%s.id'%(tn, ln))
2176 if p.has_values:
2177 if isinstance(v, type([])):
2178 s = ','.join([a for x in v])
2179 where.append('%s.linkid in (%s)'%(tn, s))
2180 args = args + v
2181 else:
2182 where.append('%s.linkid=%s'%(tn, a))
2183 args.append(v)
2184 if p.sort_type > 0:
2185 assert not p.attr_sort_done and not p.sort_ids_needed
2186 elif k == 'id':
2187 if p.sort_type < 2:
2188 if isinstance(v, type([])):
2189 s = ','.join([a for x in v])
2190 where.append('_%s.%s in (%s)'%(pln, k, s))
2191 args = args + v
2192 else:
2193 where.append('_%s.%s=%s'%(pln, k, a))
2194 args.append(v)
2195 if p.sort_type > 0:
2196 oc = ac = '_%s.id'%pln
2197 elif isinstance(propclass, String):
2198 if p.sort_type < 2:
2199 if not isinstance(v, type([])):
2200 v = [v]
2202 # Quote the bits in the string that need it and then embed
2203 # in a "substring" search. Note - need to quote the '%' so
2204 # they make it through the python layer happily
2205 v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
2207 # now add to the where clause
2208 where.append('('
2209 +' and '.join(["_%s._%s LIKE '%s'"%(pln, k, s) for s in v])
2210 +')')
2211 # note: args are embedded in the query string now
2212 if p.sort_type > 0:
2213 oc = ac = 'lower(_%s._%s)'%(pln, k)
2214 elif isinstance(propclass, Link):
2215 if p.sort_type < 2:
2216 if p.children:
2217 if p.sort_type == 0:
2218 frum.append('_%s as _%s' % (cn, ln))
2219 where.append('_%s._%s=_%s.id'%(pln, k, ln))
2220 if p.has_values:
2221 if isinstance(v, type([])):
2222 d = {}
2223 for entry in v:
2224 if entry == '-1':
2225 entry = None
2226 d[entry] = entry
2227 l = []
2228 if d.has_key(None) or not d:
2229 if d.has_key(None): del d[None]
2230 l.append('_%s._%s is NULL'%(pln, k))
2231 if d:
2232 v = d.keys()
2233 s = ','.join([a for x in v])
2234 l.append('(_%s._%s in (%s))'%(pln, k, s))
2235 args = args + v
2236 if l:
2237 where.append('(' + ' or '.join(l) +')')
2238 else:
2239 if v in ('-1', None):
2240 v = None
2241 where.append('_%s._%s is NULL'%(pln, k))
2242 else:
2243 where.append('_%s._%s=%s'%(pln, k, a))
2244 args.append(v)
2245 if p.sort_type > 0:
2246 lp = p.cls.labelprop()
2247 oc = ac = '_%s._%s'%(pln, k)
2248 if lp != 'id':
2249 if p.tree_sort_done and p.sort_type > 0:
2250 loj.append(
2251 'LEFT OUTER JOIN _%s as _%s on _%s._%s=_%s.id'%(
2252 cn, ln, pln, k, ln))
2253 oc = '_%s._%s'%(ln, lp)
2254 elif isinstance(propclass, Date) and p.sort_type < 2:
2255 dc = self.db.hyperdb_to_sql_value[hyperdb.Date]
2256 if isinstance(v, type([])):
2257 s = ','.join([a for x in v])
2258 where.append('_%s._%s in (%s)'%(pln, k, s))
2259 args = args + [dc(date.Date(x)) for x in v]
2260 else:
2261 try:
2262 # Try to filter on range of dates
2263 date_rng = propclass.range_from_raw(v, self.db)
2264 if date_rng.from_value:
2265 where.append('_%s._%s >= %s'%(pln, k, a))
2266 args.append(dc(date_rng.from_value))
2267 if date_rng.to_value:
2268 where.append('_%s._%s <= %s'%(pln, k, a))
2269 args.append(dc(date_rng.to_value))
2270 except ValueError:
2271 # If range creation fails - ignore that search parameter
2272 pass
2273 elif isinstance(propclass, Interval):
2274 # filter/sort using the __<prop>_int__ column
2275 if p.sort_type < 2:
2276 if isinstance(v, type([])):
2277 s = ','.join([a for x in v])
2278 where.append('_%s.__%s_int__ in (%s)'%(pln, k, s))
2279 args = args + [date.Interval(x).as_seconds() for x in v]
2280 else:
2281 try:
2282 # Try to filter on range of intervals
2283 date_rng = Range(v, date.Interval)
2284 if date_rng.from_value:
2285 where.append('_%s.__%s_int__ >= %s'%(pln, k, a))
2286 args.append(date_rng.from_value.as_seconds())
2287 if date_rng.to_value:
2288 where.append('_%s.__%s_int__ <= %s'%(pln, k, a))
2289 args.append(date_rng.to_value.as_seconds())
2290 except ValueError:
2291 # If range creation fails - ignore search parameter
2292 pass
2293 if p.sort_type > 0:
2294 oc = ac = '_%s.__%s_int__'%(pln,k)
2295 elif p.sort_type < 2:
2296 if isinstance(v, type([])):
2297 s = ','.join([a for x in v])
2298 where.append('_%s._%s in (%s)'%(pln, k, s))
2299 args = args + v
2300 else:
2301 where.append('_%s._%s=%s'%(pln, k, a))
2302 args.append(v)
2303 if oc:
2304 if p.sort_ids_needed:
2305 auxcols[ac] = p
2306 if p.tree_sort_done and p.sort_direction:
2307 # Don't select top-level id twice
2308 if p.name != 'id' or p.parent != proptree:
2309 ordercols.append(oc)
2310 desc = ['', ' desc'][p.sort_direction == '-']
2311 # Some SQL dbs sort NULL values last -- we want them first.
2312 if (self.order_by_null_values and p.name != 'id'):
2313 nv = self.order_by_null_values % oc
2314 ordercols.append(nv)
2315 p.orderby.append(nv + desc)
2316 p.orderby.append(oc + desc)
2318 props = self.getprops()
2320 # don't match retired nodes
2321 where.append('_%s.__retired__=0'%icn)
2323 # add results of full text search
2324 if search_matches is not None:
2325 v = search_matches.keys()
2326 s = ','.join([a for x in v])
2327 where.append('_%s.id in (%s)'%(icn, s))
2328 args = args + v
2330 # construct the SQL
2331 frum.append('_'+icn)
2332 frum = ','.join(frum)
2333 if where:
2334 where = ' where ' + (' and '.join(where))
2335 else:
2336 where = ''
2337 if mlfilt:
2338 # we're joining tables on the id, so we will get dupes if we
2339 # don't distinct()
2340 cols = ['distinct(_%s.id)'%icn]
2341 else:
2342 cols = ['_%s.id'%icn]
2343 if ordercols:
2344 cols = cols + ordercols
2345 order = []
2346 # keep correct sequence of order attributes.
2347 for sa in proptree.sortattr:
2348 if not sa.attr_sort_done:
2349 continue
2350 order.extend(sa.orderby)
2351 if order:
2352 order = ' order by %s'%(','.join(order))
2353 else:
2354 order = ''
2355 for o, p in auxcols.iteritems ():
2356 cols.append (o)
2357 p.auxcol = len (cols) - 1
2359 cols = ','.join(cols)
2360 loj = ' '.join(loj)
2361 sql = 'select %s from %s %s %s%s'%(cols, frum, loj, where, order)
2362 args = tuple(args)
2363 __traceback_info__ = (sql, args)
2364 self.db.sql(sql, args)
2365 l = self.db.sql_fetchall()
2367 # Compute values needed for sorting in proptree.sort
2368 for p in auxcols.itervalues():
2369 p.sort_ids = p.sort_result = [row[p.auxcol] for row in l]
2370 # return the IDs (the first column)
2371 # XXX numeric ids
2372 l = [str(row[0]) for row in l]
2373 l = proptree.sort (l)
2375 if __debug__:
2376 self.db.stats['filtering'] += (time.time() - start_t)
2377 return l
2379 def filter_sql(self, sql):
2380 """Return a list of the ids of the items in this class that match
2381 the SQL provided. The SQL is a complete "select" statement.
2383 The SQL select must include the item id as the first column.
2385 This function DOES NOT filter out retired items, add on a where
2386 clause "__retired__=0" if you don't want retired nodes.
2387 """
2388 if __debug__:
2389 start_t = time.time()
2391 self.db.sql(sql)
2392 l = self.db.sql_fetchall()
2394 if __debug__:
2395 self.db.stats['filtering'] += (time.time() - start_t)
2396 return l
2398 def count(self):
2399 """Get the number of nodes in this class.
2401 If the returned integer is 'numnodes', the ids of all the nodes
2402 in this class run from 1 to numnodes, and numnodes+1 will be the
2403 id of the next node to be created in this class.
2404 """
2405 return self.db.countnodes(self.classname)
2407 # Manipulating properties:
2408 def getprops(self, protected=1):
2409 """Return a dictionary mapping property names to property objects.
2410 If the "protected" flag is true, we include protected properties -
2411 those which may not be modified.
2412 """
2413 d = self.properties.copy()
2414 if protected:
2415 d['id'] = String()
2416 d['creation'] = hyperdb.Date()
2417 d['activity'] = hyperdb.Date()
2418 d['creator'] = hyperdb.Link('user')
2419 d['actor'] = hyperdb.Link('user')
2420 return d
2422 def addprop(self, **properties):
2423 """Add properties to this class.
2425 The keyword arguments in 'properties' must map names to property
2426 objects, or a TypeError is raised. None of the keys in 'properties'
2427 may collide with the names of existing properties, or a ValueError
2428 is raised before any properties have been added.
2429 """
2430 for key in properties.keys():
2431 if self.properties.has_key(key):
2432 raise ValueError, key
2433 self.properties.update(properties)
2435 def index(self, nodeid):
2436 """Add (or refresh) the node to search indexes
2437 """
2438 # find all the String properties that have indexme
2439 for prop, propclass in self.getprops().items():
2440 if isinstance(propclass, String) and propclass.indexme:
2441 self.db.indexer.add_text((self.classname, nodeid, prop),
2442 str(self.get(nodeid, prop)))
2444 #
2445 # import / export support
2446 #
2447 def export_list(self, propnames, nodeid):
2448 """ Export a node - generate a list of CSV-able data in the order
2449 specified by propnames for the given node.
2450 """
2451 properties = self.getprops()
2452 l = []
2453 for prop in propnames:
2454 proptype = properties[prop]
2455 value = self.get(nodeid, prop)
2456 # "marshal" data where needed
2457 if value is None:
2458 pass
2459 elif isinstance(proptype, hyperdb.Date):
2460 value = value.get_tuple()
2461 elif isinstance(proptype, hyperdb.Interval):
2462 value = value.get_tuple()
2463 elif isinstance(proptype, hyperdb.Password):
2464 value = str(value)
2465 l.append(repr(value))
2466 l.append(repr(self.is_retired(nodeid)))
2467 return l
2469 def import_list(self, propnames, proplist):
2470 """ Import a node - all information including "id" is present and
2471 should not be sanity checked. Triggers are not triggered. The
2472 journal should be initialised using the "creator" and "created"
2473 information.
2475 Return the nodeid of the node imported.
2476 """
2477 if self.db.journaltag is None:
2478 raise DatabaseError, _('Database open read-only')
2479 properties = self.getprops()
2481 # make the new node's property map
2482 d = {}
2483 retire = 0
2484 if not "id" in propnames:
2485 newid = self.db.newid(self.classname)
2486 else:
2487 newid = eval(proplist[propnames.index("id")])
2488 for i in range(len(propnames)):
2489 # Use eval to reverse the repr() used to output the CSV
2490 value = eval(proplist[i])
2492 # Figure the property for this column
2493 propname = propnames[i]
2495 # "unmarshal" where necessary
2496 if propname == 'id':
2497 continue
2498 elif propname == 'is retired':
2499 # is the item retired?
2500 if int(value):
2501 retire = 1
2502 continue
2503 elif value is None:
2504 d[propname] = None
2505 continue
2507 prop = properties[propname]
2508 if value is None:
2509 # don't set Nones
2510 continue
2511 elif isinstance(prop, hyperdb.Date):
2512 value = date.Date(value)
2513 elif isinstance(prop, hyperdb.Interval):
2514 value = date.Interval(value)
2515 elif isinstance(prop, hyperdb.Password):
2516 pwd = password.Password()
2517 pwd.unpack(value)
2518 value = pwd
2519 elif isinstance(prop, String):
2520 if isinstance(value, unicode):
2521 value = value.encode('utf8')
2522 if not isinstance(value, str):
2523 raise TypeError, \
2524 'new property "%(propname)s" not a string: %(value)r' \
2525 % locals()
2526 if prop.indexme:
2527 self.db.indexer.add_text((self.classname, newid, propname),
2528 value)
2529 d[propname] = value
2531 # get a new id if necessary
2532 if newid is None:
2533 newid = self.db.newid(self.classname)
2535 # insert new node or update existing?
2536 if not self.hasnode(newid):
2537 self.db.addnode(self.classname, newid, d) # insert
2538 else:
2539 self.db.setnode(self.classname, newid, d) # update
2541 # retire?
2542 if retire:
2543 # use the arg for __retired__ to cope with any odd database type
2544 # conversion (hello, sqlite)
2545 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
2546 self.db.arg, self.db.arg)
2547 self.db.sql(sql, (newid, newid))
2548 return newid
2550 def export_journals(self):
2551 """Export a class's journal - generate a list of lists of
2552 CSV-able data:
2554 nodeid, date, user, action, params
2556 No heading here - the columns are fixed.
2557 """
2558 properties = self.getprops()
2559 r = []
2560 for nodeid in self.getnodeids():
2561 for nodeid, date, user, action, params in self.history(nodeid):
2562 date = date.get_tuple()
2563 if action == 'set':
2564 export_data = {}
2565 for propname, value in params.items():
2566 if not properties.has_key(propname):
2567 # property no longer in the schema
2568 continue
2570 prop = properties[propname]
2571 # make sure the params are eval()'able
2572 if value is None:
2573 pass
2574 elif isinstance(prop, Date):
2575 value = value.get_tuple()
2576 elif isinstance(prop, Interval):
2577 value = value.get_tuple()
2578 elif isinstance(prop, Password):
2579 value = str(value)
2580 export_data[propname] = value
2581 params = export_data
2582 elif action == 'create' and params:
2583 # old tracker with data stored in the create!
2584 params = {}
2585 l = [nodeid, date, user, action, params]
2586 r.append(map(repr, l))
2587 return r
2589 def import_journals(self, entries):
2590 """Import a class's journal.
2592 Uses setjournal() to set the journal for each item."""
2593 properties = self.getprops()
2594 d = {}
2595 for l in entries:
2596 l = map(eval, l)
2597 nodeid, jdate, user, action, params = l
2598 r = d.setdefault(nodeid, [])
2599 if action == 'set':
2600 for propname, value in params.items():
2601 prop = properties[propname]
2602 if value is None:
2603 pass
2604 elif isinstance(prop, Date):
2605 value = date.Date(value)
2606 elif isinstance(prop, Interval):
2607 value = date.Interval(value)
2608 elif isinstance(prop, Password):
2609 pwd = password.Password()
2610 pwd.unpack(value)
2611 value = pwd
2612 params[propname] = value
2613 elif action == 'create' and params:
2614 # old tracker with data stored in the create!
2615 params = {}
2616 r.append((nodeid, date.Date(jdate), user, action, params))
2618 for nodeid, l in d.items():
2619 self.db.setjournal(self.classname, nodeid, l)
2621 class FileClass(hyperdb.FileClass, Class):
2622 """This class defines a large chunk of data. To support this, it has a
2623 mandatory String property "content" which is typically saved off
2624 externally to the hyperdb.
2626 The default MIME type of this data is defined by the
2627 "default_mime_type" class attribute, which may be overridden by each
2628 node if the class defines a "type" String property.
2629 """
2630 def __init__(self, db, classname, **properties):
2631 """The newly-created class automatically includes the "content"
2632 and "type" properties.
2633 """
2634 if not properties.has_key('content'):
2635 properties['content'] = hyperdb.String(indexme='yes')
2636 if not properties.has_key('type'):
2637 properties['type'] = hyperdb.String()
2638 Class.__init__(self, db, classname, **properties)
2640 def create(self, **propvalues):
2641 """ snaffle the file propvalue and store in a file
2642 """
2643 # we need to fire the auditors now, or the content property won't
2644 # be in propvalues for the auditors to play with
2645 self.fireAuditors('create', None, propvalues)
2647 # now remove the content property so it's not stored in the db
2648 content = propvalues['content']
2649 del propvalues['content']
2651 # do the database create
2652 newid = self.create_inner(**propvalues)
2654 # figure the mime type
2655 mime_type = propvalues.get('type', self.default_mime_type)
2657 # and index!
2658 if self.properties['content'].indexme:
2659 self.db.indexer.add_text((self.classname, newid, 'content'),
2660 content, mime_type)
2662 # store off the content as a file
2663 self.db.storefile(self.classname, newid, None, content)
2665 # fire reactors
2666 self.fireReactors('create', newid, None)
2668 return newid
2670 def get(self, nodeid, propname, default=_marker, cache=1):
2671 """ Trap the content propname and get it from the file
2673 'cache' exists for backwards compatibility, and is not used.
2674 """
2675 poss_msg = 'Possibly a access right configuration problem.'
2676 if propname == 'content':
2677 try:
2678 return self.db.getfile(self.classname, nodeid, None)
2679 except IOError, (strerror):
2680 # BUG: by catching this we donot see an error in the log.
2681 return 'ERROR reading file: %s%s\n%s\n%s'%(
2682 self.classname, nodeid, poss_msg, strerror)
2683 if default is not _marker:
2684 return Class.get(self, nodeid, propname, default)
2685 else:
2686 return Class.get(self, nodeid, propname)
2688 def set(self, itemid, **propvalues):
2689 """ Snarf the "content" propvalue and update it in a file
2690 """
2691 self.fireAuditors('set', itemid, propvalues)
2692 oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid))
2694 # now remove the content property so it's not stored in the db
2695 content = None
2696 if propvalues.has_key('content'):
2697 content = propvalues['content']
2698 del propvalues['content']
2700 # do the database create
2701 propvalues = self.set_inner(itemid, **propvalues)
2703 # do content?
2704 if content:
2705 # store and possibly index
2706 self.db.storefile(self.classname, itemid, None, content)
2707 if self.properties['content'].indexme:
2708 mime_type = self.get(itemid, 'type', self.default_mime_type)
2709 self.db.indexer.add_text((self.classname, itemid, 'content'),
2710 content, mime_type)
2711 propvalues['content'] = content
2713 # fire reactors
2714 self.fireReactors('set', itemid, oldvalues)
2715 return propvalues
2717 def index(self, nodeid):
2718 """ Add (or refresh) the node to search indexes.
2720 Use the content-type property for the content property.
2721 """
2722 # find all the String properties that have indexme
2723 for prop, propclass in self.getprops().items():
2724 if prop == 'content' and propclass.indexme:
2725 mime_type = self.get(nodeid, 'type', self.default_mime_type)
2726 self.db.indexer.add_text((self.classname, nodeid, 'content'),
2727 str(self.get(nodeid, 'content')), mime_type)
2728 elif isinstance(propclass, hyperdb.String) and propclass.indexme:
2729 # index them under (classname, nodeid, property)
2730 try:
2731 value = str(self.get(nodeid, prop))
2732 except IndexError:
2733 # node has been destroyed
2734 continue
2735 self.db.indexer.add_text((self.classname, nodeid, prop), value)
2737 # XXX deviation from spec - was called ItemClass
2738 class IssueClass(Class, roundupdb.IssueClass):
2739 # Overridden methods:
2740 def __init__(self, db, classname, **properties):
2741 """The newly-created class automatically includes the "messages",
2742 "files", "nosy", and "superseder" properties. If the 'properties'
2743 dictionary attempts to specify any of these properties or a
2744 "creation", "creator", "activity" or "actor" property, a ValueError
2745 is raised.
2746 """
2747 if not properties.has_key('title'):
2748 properties['title'] = hyperdb.String(indexme='yes')
2749 if not properties.has_key('messages'):
2750 properties['messages'] = hyperdb.Multilink("msg")
2751 if not properties.has_key('files'):
2752 properties['files'] = hyperdb.Multilink("file")
2753 if not properties.has_key('nosy'):
2754 # note: journalling is turned off as it really just wastes
2755 # space. this behaviour may be overridden in an instance
2756 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2757 if not properties.has_key('superseder'):
2758 properties['superseder'] = hyperdb.Multilink(classname)
2759 Class.__init__(self, db, classname, **properties)
2761 # vim: set et sts=4 sw=4 :