1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: rdbms_common.py,v 1.199 2008-08-18 06:25:47 richard Exp $
19 """ Relational database (SQL) backend common code.
21 Basics:
23 - map roundup classes to relational tables
24 - automatically detect schema changes and modify the table schemas
25 appropriately (we store the "database version" of the schema in the
26 database itself as the only row of the "schema" table)
27 - multilinks (which represent a many-to-many relationship) are handled through
28 intermediate tables
29 - journals are stored adjunct to the per-class tables
30 - table names and columns have "_" prepended so the names can't clash with
31 restricted names (like "order")
32 - retirement is determined by the __retired__ column being > 0
34 Database-specific changes may generally be pushed out to the overridable
35 sql_* methods, since everything else should be fairly generic. There's
36 probably a bit of work to be done if a database is used that actually
37 honors column typing, since the initial databases don't (sqlite stores
38 everything as a string.)
40 The schema of the hyperdb being mapped to the database is stored in the
41 database itself as a repr()'ed dictionary of information about each Class
42 that maps to a table. If that information differs from the hyperdb schema,
43 then we update it. We also store in the schema dict a version which
44 allows us to upgrade the database schema when necessary. See upgrade_db().
46 To force a unqiueness constraint on the key properties we put the item
47 id into the __retired__ column duing retirement (so it's 0 for "active"
48 items) and place a unqiueness constraint on key + __retired__. This is
49 particularly important for the users class where multiple users may
50 try to have the same username, with potentially many retired users with
51 the same name.
52 """
53 __docformat__ = 'restructuredtext'
55 # standard python modules
56 import sys, os, time, re, errno, weakref, copy, logging
58 # roundup modules
59 from roundup import hyperdb, date, password, roundupdb, security, support
60 from roundup.hyperdb import String, Password, Date, Interval, Link, \
61 Multilink, DatabaseError, Boolean, Number, Node
62 from roundup.backends import locking
63 from roundup.support import reversed
64 from roundup.i18n import _
66 # support
67 from blobfiles import FileStorage
68 try:
69 from indexer_xapian import Indexer
70 except ImportError:
71 from indexer_rdbms import Indexer
72 from sessions_rdbms import Sessions, OneTimeKeys
73 from roundup.date import Range
75 # number of rows to keep in memory
76 ROW_CACHE_SIZE = 100
78 # dummy value meaning "argument not passed"
79 _marker = []
81 def _num_cvt(num):
82 num = str(num)
83 try:
84 return int(num)
85 except:
86 return float(num)
88 def _bool_cvt(value):
89 if value in ('TRUE', 'FALSE'):
90 return {'TRUE': 1, 'FALSE': 0}[value]
91 # assume it's a number returned from the db API
92 return int(value)
94 def connection_dict(config, dbnamestr=None):
95 """ Used by Postgresql and MySQL to detemine the keyword args for
96 opening the database connection."""
97 d = { }
98 if dbnamestr:
99 d[dbnamestr] = config.RDBMS_NAME
100 for name in ('host', 'port', 'password', 'user', 'read_default_group',
101 'read_default_file'):
102 cvar = 'RDBMS_'+name.upper()
103 if config[cvar] is not None:
104 d[name] = config[cvar]
105 return d
107 class Database(FileStorage, hyperdb.Database, roundupdb.Database):
108 """ Wrapper around an SQL database that presents a hyperdb interface.
110 - some functionality is specific to the actual SQL database, hence
111 the sql_* methods that are NotImplemented
112 - we keep a cache of the latest ROW_CACHE_SIZE row fetches.
113 """
114 def __init__(self, config, journaltag=None):
115 """ Open the database and load the schema from it.
116 """
117 FileStorage.__init__(self, config.UMASK)
118 self.config, self.journaltag = config, journaltag
119 self.dir = config.DATABASE
120 self.classes = {}
121 self.indexer = Indexer(self)
122 self.security = security.Security(self)
124 # additional transaction support for external files and the like
125 self.transactions = []
127 # keep a cache of the N most recently retrieved rows of any kind
128 # (classname, nodeid) = row
129 self.cache = {}
130 self.cache_lru = []
131 self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0,
132 'filtering': 0}
134 # database lock
135 self.lockfile = None
137 # open a connection to the database, creating the "conn" attribute
138 self.open_connection()
140 def clearCache(self):
141 self.cache = {}
142 self.cache_lru = []
144 def getSessionManager(self):
145 return Sessions(self)
147 def getOTKManager(self):
148 return OneTimeKeys(self)
150 def open_connection(self):
151 """ Open a connection to the database, creating it if necessary.
153 Must call self.load_dbschema()
154 """
155 raise NotImplemented
157 def sql(self, sql, args=None):
158 """ Execute the sql with the optional args.
159 """
160 if __debug__:
161 logging.getLogger('hyperdb').debug('SQL %r %r'%(sql, args))
162 if args:
163 self.cursor.execute(sql, args)
164 else:
165 self.cursor.execute(sql)
167 def sql_fetchone(self):
168 """ Fetch a single row. If there's nothing to fetch, return None.
169 """
170 return self.cursor.fetchone()
172 def sql_fetchall(self):
173 """ Fetch all rows. If there's nothing to fetch, return [].
174 """
175 return self.cursor.fetchall()
177 def sql_stringquote(self, value):
178 """ Quote the string so it's safe to put in the 'sql quotes'
179 """
180 return re.sub("'", "''", str(value))
182 def init_dbschema(self):
183 self.database_schema = {
184 'version': self.current_db_version,
185 'tables': {}
186 }
188 def load_dbschema(self):
189 """ Load the schema definition that the database currently implements
190 """
191 self.cursor.execute('select schema from schema')
192 schema = self.cursor.fetchone()
193 if schema:
194 self.database_schema = eval(schema[0])
195 else:
196 self.database_schema = {}
198 def save_dbschema(self):
199 """ Save the schema definition that the database currently implements
200 """
201 s = repr(self.database_schema)
202 self.sql('delete from schema')
203 self.sql('insert into schema values (%s)'%self.arg, (s,))
205 def post_init(self):
206 """ Called once the schema initialisation has finished.
208 We should now confirm that the schema defined by our "classes"
209 attribute actually matches the schema in the database.
210 """
211 save = 0
213 # handle changes in the schema
214 tables = self.database_schema['tables']
215 for classname, spec in self.classes.items():
216 if tables.has_key(classname):
217 dbspec = tables[classname]
218 if self.update_class(spec, dbspec):
219 tables[classname] = spec.schema()
220 save = 1
221 else:
222 self.create_class(spec)
223 tables[classname] = spec.schema()
224 save = 1
226 for classname, spec in tables.items():
227 if not self.classes.has_key(classname):
228 self.drop_class(classname, tables[classname])
229 del tables[classname]
230 save = 1
232 # now upgrade the database for column type changes, new internal
233 # tables, etc.
234 save = save | self.upgrade_db()
236 # update the database version of the schema
237 if save:
238 self.save_dbschema()
240 # reindex the db if necessary
241 if self.indexer.should_reindex():
242 self.reindex()
244 # commit
245 self.sql_commit()
247 # update this number when we need to make changes to the SQL structure
248 # of the backen database
249 current_db_version = 5
250 db_version_updated = False
251 def upgrade_db(self):
252 """ Update the SQL database to reflect changes in the backend code.
254 Return boolean whether we need to save the schema.
255 """
256 version = self.database_schema.get('version', 1)
257 if version > self.current_db_version:
258 raise DatabaseError('attempting to run rev %d DATABASE with rev '
259 '%d CODE!'%(version, self.current_db_version))
260 if version == self.current_db_version:
261 # nothing to do
262 return 0
264 if version < 2:
265 if __debug__:
266 logging.getLogger('hyperdb').info('upgrade to version 2')
267 # change the schema structure
268 self.database_schema = {'tables': self.database_schema}
270 # version 1 didn't have the actor column (note that in
271 # MySQL this will also transition the tables to typed columns)
272 self.add_new_columns_v2()
274 # version 1 doesn't have the OTK, session and indexing in the
275 # database
276 self.create_version_2_tables()
278 if version < 3:
279 if __debug__:
280 logging.getLogger('hyperdb').info('upgrade to version 3')
281 self.fix_version_2_tables()
283 if version < 4:
284 self.fix_version_3_tables()
286 if version < 5:
287 self.fix_version_4_tables()
289 self.database_schema['version'] = self.current_db_version
290 self.db_version_updated = True
291 return 1
293 def fix_version_3_tables(self):
294 # drop the shorter VARCHAR OTK column and add a new TEXT one
295 for name in ('otk', 'session'):
296 self.sql('DELETE FROM %ss'%name)
297 self.sql('ALTER TABLE %ss DROP %s_value'%(name, name))
298 self.sql('ALTER TABLE %ss ADD %s_value TEXT'%(name, name))
300 def fix_version_2_tables(self):
301 # Default (used by sqlite): NOOP
302 pass
304 def fix_version_4_tables(self):
305 # note this is an explicit call now
306 c = self.cursor
307 for cn, klass in self.classes.items():
308 c.execute('select id from _%s where __retired__<>0'%(cn,))
309 for (id,) in c.fetchall():
310 c.execute('update _%s set __retired__=%s where id=%s'%(cn,
311 self.arg, self.arg), (id, id))
313 if klass.key:
314 self.add_class_key_required_unique_constraint(cn, klass.key)
316 def _convert_journal_tables(self):
317 """Get current journal table contents, drop the table and re-create"""
318 c = self.cursor
319 cols = ','.join('nodeid date tag action params'.split())
320 for klass in self.classes.values():
321 # slurp and drop
322 sql = 'select %s from %s__journal order by date'%(cols,
323 klass.classname)
324 c.execute(sql)
325 contents = c.fetchall()
326 self.drop_journal_table_indexes(klass.classname)
327 c.execute('drop table %s__journal'%klass.classname)
329 # re-create and re-populate
330 self.create_journal_table(klass)
331 a = self.arg
332 sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(
333 klass.classname, cols, a, a, a, a, a)
334 for row in contents:
335 # no data conversion needed
336 self.cursor.execute(sql, row)
338 def _convert_string_properties(self):
339 """Get current Class tables that contain String properties, and
340 convert the VARCHAR columns to TEXT"""
341 c = self.cursor
342 for klass in self.classes.values():
343 # slurp and drop
344 cols, mls = self.determine_columns(klass.properties.items())
345 scols = ','.join([i[0] for i in cols])
346 sql = 'select id,%s from _%s'%(scols, klass.classname)
347 c.execute(sql)
348 contents = c.fetchall()
349 self.drop_class_table_indexes(klass.classname, klass.getkey())
350 c.execute('drop table _%s'%klass.classname)
352 # re-create and re-populate
353 self.create_class_table(klass, create_sequence=0)
354 a = ','.join([self.arg for i in range(len(cols)+1)])
355 sql = 'insert into _%s (id,%s) values (%s)'%(klass.classname,
356 scols, a)
357 for row in contents:
358 l = []
359 for entry in row:
360 # mysql will already be a string - psql needs "help"
361 if entry is not None and not isinstance(entry, type('')):
362 entry = str(entry)
363 l.append(entry)
364 self.cursor.execute(sql, l)
366 def refresh_database(self):
367 self.post_init()
370 def reindex(self, classname=None, show_progress=False):
371 if classname:
372 classes = [self.getclass(classname)]
373 else:
374 classes = self.classes.values()
375 for klass in classes:
376 if show_progress:
377 for nodeid in support.Progress('Reindex %s'%klass.classname,
378 klass.list()):
379 klass.index(nodeid)
380 else:
381 for nodeid in klass.list():
382 klass.index(nodeid)
383 self.indexer.save_index()
385 hyperdb_to_sql_datatypes = {
386 hyperdb.String : 'TEXT',
387 hyperdb.Date : 'TIMESTAMP',
388 hyperdb.Link : 'INTEGER',
389 hyperdb.Interval : 'VARCHAR(255)',
390 hyperdb.Password : 'VARCHAR(255)',
391 hyperdb.Boolean : 'BOOLEAN',
392 hyperdb.Number : 'REAL',
393 }
394 def determine_columns(self, properties):
395 """ Figure the column names and multilink properties from the spec
397 "properties" is a list of (name, prop) where prop may be an
398 instance of a hyperdb "type" _or_ a string repr of that type.
399 """
400 cols = [
401 ('_actor', self.hyperdb_to_sql_datatypes[hyperdb.Link]),
402 ('_activity', self.hyperdb_to_sql_datatypes[hyperdb.Date]),
403 ('_creator', self.hyperdb_to_sql_datatypes[hyperdb.Link]),
404 ('_creation', self.hyperdb_to_sql_datatypes[hyperdb.Date]),
405 ]
406 mls = []
407 # add the multilinks separately
408 for col, prop in properties:
409 if isinstance(prop, Multilink):
410 mls.append(col)
411 continue
413 if isinstance(prop, type('')):
414 raise ValueError, "string property spec!"
415 #and prop.find('Multilink') != -1:
416 #mls.append(col)
418 datatype = self.hyperdb_to_sql_datatypes[prop.__class__]
419 cols.append(('_'+col, datatype))
421 # Intervals stored as two columns
422 if isinstance(prop, Interval):
423 cols.append(('__'+col+'_int__', 'BIGINT'))
425 cols.sort()
426 return cols, mls
428 def update_class(self, spec, old_spec, force=0):
429 """ Determine the differences between the current spec and the
430 database version of the spec, and update where necessary.
432 If 'force' is true, update the database anyway.
433 """
434 new_has = spec.properties.has_key
435 new_spec = spec.schema()
436 new_spec[1].sort()
437 old_spec[1].sort()
438 if not force and new_spec == old_spec:
439 # no changes
440 return 0
442 logger = logging.getLogger('hyperdb')
443 logger.info('update_class %s'%spec.classname)
445 logger.debug('old_spec %r'%(old_spec,))
446 logger.debug('new_spec %r'%(new_spec,))
448 # detect key prop change for potential index change
449 keyprop_changes = {}
450 if new_spec[0] != old_spec[0]:
451 if old_spec[0]:
452 keyprop_changes['remove'] = old_spec[0]
453 if new_spec[0]:
454 keyprop_changes['add'] = new_spec[0]
456 # detect multilinks that have been removed, and drop their table
457 old_has = {}
458 for name, prop in old_spec[1]:
459 old_has[name] = 1
460 if new_has(name):
461 continue
463 if prop.find('Multilink to') != -1:
464 # first drop indexes.
465 self.drop_multilink_table_indexes(spec.classname, name)
467 # now the multilink table itself
468 sql = 'drop table %s_%s'%(spec.classname, name)
469 else:
470 # if this is the key prop, drop the index first
471 if old_spec[0] == prop:
472 self.drop_class_table_key_index(spec.classname, name)
473 del keyprop_changes['remove']
475 # drop the column
476 sql = 'alter table _%s drop column _%s'%(spec.classname, name)
478 self.sql(sql)
479 old_has = old_has.has_key
481 # if we didn't remove the key prop just then, but the key prop has
482 # changed, we still need to remove the old index
483 if keyprop_changes.has_key('remove'):
484 self.drop_class_table_key_index(spec.classname,
485 keyprop_changes['remove'])
487 # add new columns
488 for propname, prop in new_spec[1]:
489 if old_has(propname):
490 continue
491 prop = spec.properties[propname]
492 if isinstance(prop, Multilink):
493 self.create_multilink_table(spec, propname)
494 else:
495 # add the column
496 coltype = self.hyperdb_to_sql_datatypes[prop.__class__]
497 sql = 'alter table _%s add column _%s %s'%(
498 spec.classname, propname, coltype)
499 self.sql(sql)
501 # extra Interval column
502 if isinstance(prop, Interval):
503 sql = 'alter table _%s add column __%s_int__ BIGINT'%(
504 spec.classname, propname)
505 self.sql(sql)
507 # if the new column is a key prop, we need an index!
508 if new_spec[0] == propname:
509 self.create_class_table_key_index(spec.classname, propname)
510 del keyprop_changes['add']
512 # if we didn't add the key prop just then, but the key prop has
513 # changed, we still need to add the new index
514 if keyprop_changes.has_key('add'):
515 self.create_class_table_key_index(spec.classname,
516 keyprop_changes['add'])
518 return 1
520 def determine_all_columns(self, spec):
521 """Figure out the columns from the spec and also add internal columns
523 """
524 cols, mls = self.determine_columns(spec.properties.items())
526 # add on our special columns
527 cols.append(('id', 'INTEGER PRIMARY KEY'))
528 cols.append(('__retired__', 'INTEGER DEFAULT 0'))
529 return cols, mls
531 def create_class_table(self, spec):
532 """Create the class table for the given Class "spec". Creates the
533 indexes too."""
534 cols, mls = self.determine_all_columns(spec)
536 # create the base table
537 scols = ','.join(['%s %s'%x for x in cols])
538 sql = 'create table _%s (%s)'%(spec.classname, scols)
539 self.sql(sql)
541 self.create_class_table_indexes(spec)
543 return cols, mls
545 def create_class_table_indexes(self, spec):
546 """ create the class table for the given spec
547 """
548 # create __retired__ index
549 index_sql2 = 'create index _%s_retired_idx on _%s(__retired__)'%(
550 spec.classname, spec.classname)
551 self.sql(index_sql2)
553 # create index for key property
554 if spec.key:
555 index_sql3 = 'create index _%s_%s_idx on _%s(_%s)'%(
556 spec.classname, spec.key,
557 spec.classname, spec.key)
558 self.sql(index_sql3)
560 # and the unique index for key / retired(id)
561 self.add_class_key_required_unique_constraint(spec.classname,
562 spec.key)
564 # TODO: create indexes on (selected?) Link property columns, as
565 # they're more likely to be used for lookup
567 def add_class_key_required_unique_constraint(self, cn, key):
568 sql = '''create unique index _%s_key_retired_idx
569 on _%s(__retired__, _%s)'''%(cn, cn, key)
570 self.sql(sql)
572 def drop_class_table_indexes(self, cn, key):
573 # drop the old table indexes first
574 l = ['_%s_id_idx'%cn, '_%s_retired_idx'%cn]
575 if key:
576 l.append('_%s_%s_idx'%(cn, key))
578 table_name = '_%s'%cn
579 for index_name in l:
580 if not self.sql_index_exists(table_name, index_name):
581 continue
582 index_sql = 'drop index '+index_name
583 self.sql(index_sql)
585 def create_class_table_key_index(self, cn, key):
586 """ create the class table for the given spec
587 """
588 sql = 'create index _%s_%s_idx on _%s(_%s)'%(cn, key, cn, key)
589 self.sql(sql)
591 def drop_class_table_key_index(self, cn, key):
592 table_name = '_%s'%cn
593 index_name = '_%s_%s_idx'%(cn, key)
594 if self.sql_index_exists(table_name, index_name):
595 sql = 'drop index '+index_name
596 self.sql(sql)
598 # and now the retired unique index too
599 index_name = '_%s_key_retired_idx'%cn
600 if self.sql_index_exists(table_name, index_name):
601 sql = 'drop index '+index_name
602 self.sql(sql)
604 def create_journal_table(self, spec):
605 """ create the journal table for a class given the spec and
606 already-determined cols
607 """
608 # journal table
609 cols = ','.join(['%s varchar'%x
610 for x in 'nodeid date tag action params'.split()])
611 sql = """create table %s__journal (
612 nodeid integer, date %s, tag varchar(255),
613 action varchar(255), params text)""" % (spec.classname,
614 self.hyperdb_to_sql_datatypes[hyperdb.Date])
615 self.sql(sql)
616 self.create_journal_table_indexes(spec)
618 def create_journal_table_indexes(self, spec):
619 # index on nodeid
620 sql = 'create index %s_journ_idx on %s__journal(nodeid)'%(
621 spec.classname, spec.classname)
622 self.sql(sql)
624 def drop_journal_table_indexes(self, classname):
625 index_name = '%s_journ_idx'%classname
626 if not self.sql_index_exists('%s__journal'%classname, index_name):
627 return
628 index_sql = 'drop index '+index_name
629 self.sql(index_sql)
631 def create_multilink_table(self, spec, ml):
632 """ Create a multilink table for the "ml" property of the class
633 given by the spec
634 """
635 # create the table
636 sql = 'create table %s_%s (linkid INTEGER, nodeid INTEGER)'%(
637 spec.classname, ml)
638 self.sql(sql)
639 self.create_multilink_table_indexes(spec, ml)
641 def create_multilink_table_indexes(self, spec, ml):
642 # create index on linkid
643 index_sql = 'create index %s_%s_l_idx on %s_%s(linkid)'%(
644 spec.classname, ml, spec.classname, ml)
645 self.sql(index_sql)
647 # create index on nodeid
648 index_sql = 'create index %s_%s_n_idx on %s_%s(nodeid)'%(
649 spec.classname, ml, spec.classname, ml)
650 self.sql(index_sql)
652 def drop_multilink_table_indexes(self, classname, ml):
653 l = [
654 '%s_%s_l_idx'%(classname, ml),
655 '%s_%s_n_idx'%(classname, ml)
656 ]
657 table_name = '%s_%s'%(classname, ml)
658 for index_name in l:
659 if not self.sql_index_exists(table_name, index_name):
660 continue
661 index_sql = 'drop index %s'%index_name
662 self.sql(index_sql)
664 def create_class(self, spec):
665 """ Create a database table according to the given spec.
666 """
667 cols, mls = self.create_class_table(spec)
668 self.create_journal_table(spec)
670 # now create the multilink tables
671 for ml in mls:
672 self.create_multilink_table(spec, ml)
674 def drop_class(self, cn, spec):
675 """ Drop the given table from the database.
677 Drop the journal and multilink tables too.
678 """
679 properties = spec[1]
680 # figure the multilinks
681 mls = []
682 for propname, prop in properties:
683 if isinstance(prop, Multilink):
684 mls.append(propname)
686 # drop class table and indexes
687 self.drop_class_table_indexes(cn, spec[0])
689 self.drop_class_table(cn)
691 # drop journal table and indexes
692 self.drop_journal_table_indexes(cn)
693 sql = 'drop table %s__journal'%cn
694 self.sql(sql)
696 for ml in mls:
697 # drop multilink table and indexes
698 self.drop_multilink_table_indexes(cn, ml)
699 sql = 'drop table %s_%s'%(spec.classname, ml)
700 self.sql(sql)
702 def drop_class_table(self, cn):
703 sql = 'drop table _%s'%cn
704 self.sql(sql)
706 #
707 # Classes
708 #
709 def __getattr__(self, classname):
710 """ A convenient way of calling self.getclass(classname).
711 """
712 if self.classes.has_key(classname):
713 return self.classes[classname]
714 raise AttributeError, classname
716 def addclass(self, cl):
717 """ Add a Class to the hyperdatabase.
718 """
719 cn = cl.classname
720 if self.classes.has_key(cn):
721 raise ValueError, cn
722 self.classes[cn] = cl
724 # add default Edit and View permissions
725 self.security.addPermission(name="Create", klass=cn,
726 description="User is allowed to create "+cn)
727 self.security.addPermission(name="Edit", klass=cn,
728 description="User is allowed to edit "+cn)
729 self.security.addPermission(name="View", klass=cn,
730 description="User is allowed to access "+cn)
732 def getclasses(self):
733 """ Return a list of the names of all existing classes.
734 """
735 l = self.classes.keys()
736 l.sort()
737 return l
739 def getclass(self, classname):
740 """Get the Class object representing a particular class.
742 If 'classname' is not a valid class name, a KeyError is raised.
743 """
744 try:
745 return self.classes[classname]
746 except KeyError:
747 raise KeyError, 'There is no class called "%s"'%classname
749 def clear(self):
750 """Delete all database contents.
752 Note: I don't commit here, which is different behaviour to the
753 "nuke from orbit" behaviour in the dbs.
754 """
755 logging.getLogger('hyperdb').info('clear')
756 for cn in self.classes.keys():
757 sql = 'delete from _%s'%cn
758 self.sql(sql)
760 #
761 # Nodes
762 #
764 hyperdb_to_sql_value = {
765 hyperdb.String : str,
766 # fractional seconds by default
767 hyperdb.Date : lambda x: x.formal(sep=' ', sec='%06.3f'),
768 hyperdb.Link : int,
769 hyperdb.Interval : str,
770 hyperdb.Password : str,
771 hyperdb.Boolean : lambda x: x and 'TRUE' or 'FALSE',
772 hyperdb.Number : lambda x: x,
773 hyperdb.Multilink : lambda x: x, # used in journal marshalling
774 }
775 def addnode(self, classname, nodeid, node):
776 """ Add the specified node to its class's db.
777 """
778 if __debug__:
779 logging.getLogger('hyperdb').debug('addnode %s%s %r'%(classname,
780 nodeid, node))
782 # determine the column definitions and multilink tables
783 cl = self.classes[classname]
784 cols, mls = self.determine_columns(cl.properties.items())
786 # we'll be supplied these props if we're doing an import
787 values = node.copy()
788 if not values.has_key('creator'):
789 # add in the "calculated" properties (dupe so we don't affect
790 # calling code's node assumptions)
791 values['creation'] = values['activity'] = date.Date()
792 values['actor'] = values['creator'] = self.getuid()
794 cl = self.classes[classname]
795 props = cl.getprops(protected=1)
796 del props['id']
798 # default the non-multilink columns
799 for col, prop in props.items():
800 if not values.has_key(col):
801 if isinstance(prop, Multilink):
802 values[col] = []
803 else:
804 values[col] = None
806 # clear this node out of the cache if it's in there
807 key = (classname, nodeid)
808 if self.cache.has_key(key):
809 del self.cache[key]
810 self.cache_lru.remove(key)
812 # figure the values to insert
813 vals = []
814 for col,dt in cols:
815 # this is somewhat dodgy....
816 if col.endswith('_int__'):
817 # XXX eugh, this test suxxors
818 value = values[col[2:-6]]
819 # this is an Interval special "int" column
820 if value is not None:
821 vals.append(value.as_seconds())
822 else:
823 vals.append(value)
824 continue
826 prop = props[col[1:]]
827 value = values[col[1:]]
828 if value is not None:
829 value = self.hyperdb_to_sql_value[prop.__class__](value)
830 vals.append(value)
831 vals.append(nodeid)
832 vals = tuple(vals)
834 # make sure the ordering is correct for column name -> column value
835 s = ','.join([self.arg for x in cols]) + ',%s'%self.arg
836 cols = ','.join([col for col,dt in cols]) + ',id'
838 # perform the inserts
839 sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
840 self.sql(sql, vals)
842 # insert the multilink rows
843 for col in mls:
844 t = '%s_%s'%(classname, col)
845 for entry in node[col]:
846 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
847 self.arg, self.arg)
848 self.sql(sql, (entry, nodeid))
850 def setnode(self, classname, nodeid, values, multilink_changes={}):
851 """ Change the specified node.
852 """
853 if __debug__:
854 logging.getLogger('hyperdb').debug('setnode %s%s %r'
855 % (classname, nodeid, values))
857 # clear this node out of the cache if it's in there
858 key = (classname, nodeid)
859 if self.cache.has_key(key):
860 del self.cache[key]
861 self.cache_lru.remove(key)
863 cl = self.classes[classname]
864 props = cl.getprops()
866 cols = []
867 mls = []
868 # add the multilinks separately
869 for col in values.keys():
870 prop = props[col]
871 if isinstance(prop, Multilink):
872 mls.append(col)
873 elif isinstance(prop, Interval):
874 # Intervals store the seconds value too
875 cols.append(col)
876 # extra leading '_' added by code below
877 cols.append('_' +col + '_int__')
878 else:
879 cols.append(col)
880 cols.sort()
882 # figure the values to insert
883 vals = []
884 for col in cols:
885 if col.endswith('_int__'):
886 # XXX eugh, this test suxxors
887 # Intervals store the seconds value too
888 col = col[1:-6]
889 prop = props[col]
890 value = values[col]
891 if value is None:
892 vals.append(None)
893 else:
894 vals.append(value.as_seconds())
895 else:
896 prop = props[col]
897 value = values[col]
898 if value is None:
899 e = None
900 else:
901 e = self.hyperdb_to_sql_value[prop.__class__](value)
902 vals.append(e)
904 vals.append(int(nodeid))
905 vals = tuple(vals)
907 # if there's any updates to regular columns, do them
908 if cols:
909 # make sure the ordering is correct for column name -> column value
910 s = ','.join(['_%s=%s'%(x, self.arg) for x in cols])
911 cols = ','.join(cols)
913 # perform the update
914 sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
915 self.sql(sql, vals)
917 # we're probably coming from an import, not a change
918 if not multilink_changes:
919 for name in mls:
920 prop = props[name]
921 value = values[name]
923 t = '%s_%s'%(classname, name)
925 # clear out previous values for this node
926 # XXX numeric ids
927 self.sql('delete from %s where nodeid=%s'%(t, self.arg),
928 (nodeid,))
930 # insert the values for this node
931 for entry in values[name]:
932 sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
933 self.arg, self.arg)
934 # XXX numeric ids
935 self.sql(sql, (entry, nodeid))
937 # we have multilink changes to apply
938 for col, (add, remove) in multilink_changes.items():
939 tn = '%s_%s'%(classname, col)
940 if add:
941 sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
942 self.arg, self.arg)
943 for addid in add:
944 # XXX numeric ids
945 self.sql(sql, (int(nodeid), int(addid)))
946 if remove:
947 s = ','.join([self.arg]*len(remove))
948 sql = 'delete from %s where nodeid=%s and linkid in (%s)'%(tn,
949 self.arg, s)
950 # XXX numeric ids
951 self.sql(sql, [int(nodeid)] + remove)
953 sql_to_hyperdb_value = {
954 hyperdb.String : str,
955 hyperdb.Date : lambda x:date.Date(str(x).replace(' ', '.')),
956 # hyperdb.Link : int, # XXX numeric ids
957 hyperdb.Link : str,
958 hyperdb.Interval : date.Interval,
959 hyperdb.Password : lambda x: password.Password(encrypted=x),
960 hyperdb.Boolean : _bool_cvt,
961 hyperdb.Number : _num_cvt,
962 hyperdb.Multilink : lambda x: x, # used in journal marshalling
963 }
964 def getnode(self, classname, nodeid):
965 """ Get a node from the database.
966 """
967 # see if we have this node cached
968 key = (classname, nodeid)
969 if self.cache.has_key(key):
970 # push us back to the top of the LRU
971 self.cache_lru.remove(key)
972 self.cache_lru.insert(0, key)
973 if __debug__:
974 self.stats['cache_hits'] += 1
975 # return the cached information
976 return self.cache[key]
978 if __debug__:
979 self.stats['cache_misses'] += 1
980 start_t = time.time()
982 # figure the columns we're fetching
983 cl = self.classes[classname]
984 cols, mls = self.determine_columns(cl.properties.items())
985 scols = ','.join([col for col,dt in cols])
987 # perform the basic property fetch
988 sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
989 self.sql(sql, (nodeid,))
991 values = self.sql_fetchone()
992 if values is None:
993 raise IndexError, 'no such %s node %s'%(classname, nodeid)
995 # make up the node
996 node = {}
997 props = cl.getprops(protected=1)
998 for col in range(len(cols)):
999 name = cols[col][0][1:]
1000 if name.endswith('_int__'):
1001 # XXX eugh, this test suxxors
1002 # ignore the special Interval-as-seconds column
1003 continue
1004 value = values[col]
1005 if value is not None:
1006 value = self.sql_to_hyperdb_value[props[name].__class__](value)
1007 node[name] = value
1010 # now the multilinks
1011 for col in mls:
1012 # get the link ids
1013 sql = 'select linkid from %s_%s where nodeid=%s'%(classname, col,
1014 self.arg)
1015 self.sql(sql, (nodeid,))
1016 # extract the first column from the result
1017 # XXX numeric ids
1018 items = [int(x[0]) for x in self.cursor.fetchall()]
1019 items.sort ()
1020 node[col] = [str(x) for x in items]
1022 # save off in the cache
1023 key = (classname, nodeid)
1024 self.cache[key] = node
1025 # update the LRU
1026 self.cache_lru.insert(0, key)
1027 if len(self.cache_lru) > ROW_CACHE_SIZE:
1028 del self.cache[self.cache_lru.pop()]
1030 if __debug__:
1031 self.stats['get_items'] += (time.time() - start_t)
1033 return node
1035 def destroynode(self, classname, nodeid):
1036 """Remove a node from the database. Called exclusively by the
1037 destroy() method on Class.
1038 """
1039 logging.getLogger('hyperdb').info('destroynode %s%s'%(classname, nodeid))
1041 # make sure the node exists
1042 if not self.hasnode(classname, nodeid):
1043 raise IndexError, '%s has no node %s'%(classname, nodeid)
1045 # see if we have this node cached
1046 if self.cache.has_key((classname, nodeid)):
1047 del self.cache[(classname, nodeid)]
1049 # see if there's any obvious commit actions that we should get rid of
1050 for entry in self.transactions[:]:
1051 if entry[1][:2] == (classname, nodeid):
1052 self.transactions.remove(entry)
1054 # now do the SQL
1055 sql = 'delete from _%s where id=%s'%(classname, self.arg)
1056 self.sql(sql, (nodeid,))
1058 # remove from multilnks
1059 cl = self.getclass(classname)
1060 x, mls = self.determine_columns(cl.properties.items())
1061 for col in mls:
1062 # get the link ids
1063 sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
1064 self.sql(sql, (nodeid,))
1066 # remove journal entries
1067 sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
1068 self.sql(sql, (nodeid,))
1070 # cleanup any blob filestorage when we commit
1071 self.transactions.append((FileStorage.destroy, (self, classname, nodeid)))
1073 def hasnode(self, classname, nodeid):
1074 """ Determine if the database has a given node.
1075 """
1076 # If this node is in the cache, then we do not need to go to
1077 # the database. (We don't consider this an LRU hit, though.)
1078 if self.cache.has_key((classname, nodeid)):
1079 # Return 1, not True, to match the type of the result of
1080 # the SQL operation below.
1081 return 1
1082 sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
1083 self.sql(sql, (nodeid,))
1084 return int(self.cursor.fetchone()[0])
1086 def countnodes(self, classname):
1087 """ Count the number of nodes that exist for a particular Class.
1088 """
1089 sql = 'select count(*) from _%s'%classname
1090 self.sql(sql)
1091 return self.cursor.fetchone()[0]
1093 def addjournal(self, classname, nodeid, action, params, creator=None,
1094 creation=None):
1095 """ Journal the Action
1096 'action' may be:
1098 'create' or 'set' -- 'params' is a dictionary of property values
1099 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
1100 'retire' -- 'params' is None
1101 """
1102 # handle supply of the special journalling parameters (usually
1103 # supplied on importing an existing database)
1104 if creator:
1105 journaltag = creator
1106 else:
1107 journaltag = self.getuid()
1108 if creation:
1109 journaldate = creation
1110 else:
1111 journaldate = date.Date()
1113 # create the journal entry
1114 cols = 'nodeid,date,tag,action,params'
1116 if __debug__:
1117 logging.getLogger('hyperdb').debug('addjournal %s%s %r %s %s %r'%(classname,
1118 nodeid, journaldate, journaltag, action, params))
1120 # make the journalled data marshallable
1121 if isinstance(params, type({})):
1122 self._journal_marshal(params, classname)
1124 params = repr(params)
1126 dc = self.hyperdb_to_sql_value[hyperdb.Date]
1127 journaldate = dc(journaldate)
1129 self.save_journal(classname, cols, nodeid, journaldate,
1130 journaltag, action, params)
1132 def setjournal(self, classname, nodeid, journal):
1133 """Set the journal to the "journal" list."""
1134 # clear out any existing entries
1135 self.sql('delete from %s__journal where nodeid=%s'%(classname,
1136 self.arg), (nodeid,))
1138 # create the journal entry
1139 cols = 'nodeid,date,tag,action,params'
1141 dc = self.hyperdb_to_sql_value[hyperdb.Date]
1142 for nodeid, journaldate, journaltag, action, params in journal:
1143 if __debug__:
1144 logging.getLogger('hyperdb').debug('addjournal %s%s %r %s %s %r'%(
1145 classname, nodeid, journaldate, journaltag, action,
1146 params))
1148 # make the journalled data marshallable
1149 if isinstance(params, type({})):
1150 self._journal_marshal(params, classname)
1151 params = repr(params)
1153 self.save_journal(classname, cols, nodeid, dc(journaldate),
1154 journaltag, action, params)
1156 def _journal_marshal(self, params, classname):
1157 """Convert the journal params values into safely repr'able and
1158 eval'able values."""
1159 properties = self.getclass(classname).getprops()
1160 for param, value in params.items():
1161 if not value:
1162 continue
1163 property = properties[param]
1164 cvt = self.hyperdb_to_sql_value[property.__class__]
1165 if isinstance(property, Password):
1166 params[param] = cvt(value)
1167 elif isinstance(property, Date):
1168 params[param] = cvt(value)
1169 elif isinstance(property, Interval):
1170 params[param] = cvt(value)
1171 elif isinstance(property, Boolean):
1172 params[param] = cvt(value)
1174 def getjournal(self, classname, nodeid):
1175 """ get the journal for id
1176 """
1177 # make sure the node exists
1178 if not self.hasnode(classname, nodeid):
1179 raise IndexError, '%s has no node %s'%(classname, nodeid)
1181 cols = ','.join('nodeid date tag action params'.split())
1182 journal = self.load_journal(classname, cols, nodeid)
1184 # now unmarshal the data
1185 dc = self.sql_to_hyperdb_value[hyperdb.Date]
1186 res = []
1187 properties = self.getclass(classname).getprops()
1188 for nodeid, date_stamp, user, action, params in journal:
1189 params = eval(params)
1190 if isinstance(params, type({})):
1191 for param, value in params.items():
1192 if not value:
1193 continue
1194 property = properties.get(param, None)
1195 if property is None:
1196 # deleted property
1197 continue
1198 cvt = self.sql_to_hyperdb_value[property.__class__]
1199 if isinstance(property, Password):
1200 params[param] = cvt(value)
1201 elif isinstance(property, Date):
1202 params[param] = cvt(value)
1203 elif isinstance(property, Interval):
1204 params[param] = cvt(value)
1205 elif isinstance(property, Boolean):
1206 params[param] = cvt(value)
1207 # XXX numeric ids
1208 res.append((str(nodeid), dc(date_stamp), user, action, params))
1209 return res
1211 def save_journal(self, classname, cols, nodeid, journaldate,
1212 journaltag, action, params):
1213 """ Save the journal entry to the database
1214 """
1215 entry = (nodeid, journaldate, journaltag, action, params)
1217 # do the insert
1218 a = self.arg
1219 sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(
1220 classname, cols, a, a, a, a, a)
1221 self.sql(sql, entry)
1223 def load_journal(self, classname, cols, nodeid):
1224 """ Load the journal from the database
1225 """
1226 # now get the journal entries
1227 sql = 'select %s from %s__journal where nodeid=%s order by date'%(
1228 cols, classname, self.arg)
1229 self.sql(sql, (nodeid,))
1230 return self.cursor.fetchall()
1232 def pack(self, pack_before):
1233 """ Delete all journal entries except "create" before 'pack_before'.
1234 """
1235 date_stamp = self.hyperdb_to_sql_value[Date](pack_before)
1237 # do the delete
1238 for classname in self.classes.keys():
1239 sql = "delete from %s__journal where date<%s and "\
1240 "action<>'create'"%(classname, self.arg)
1241 self.sql(sql, (date_stamp,))
1243 def sql_commit(self, fail_ok=False):
1244 """ Actually commit to the database.
1245 """
1246 logging.getLogger('hyperdb').info('commit')
1248 self.conn.commit()
1250 # open a new cursor for subsequent work
1251 self.cursor = self.conn.cursor()
1253 def commit(self, fail_ok=False):
1254 """ Commit the current transactions.
1256 Save all data changed since the database was opened or since the
1257 last commit() or rollback().
1259 fail_ok indicates that the commit is allowed to fail. This is used
1260 in the web interface when committing cleaning of the session
1261 database. We don't care if there's a concurrency issue there.
1263 The only backend this seems to affect is postgres.
1264 """
1265 # commit the database
1266 self.sql_commit(fail_ok)
1268 # now, do all the other transaction stuff
1269 for method, args in self.transactions:
1270 method(*args)
1272 # save the indexer
1273 self.indexer.save_index()
1275 # clear out the transactions
1276 self.transactions = []
1278 def sql_rollback(self):
1279 self.conn.rollback()
1281 def rollback(self):
1282 """ Reverse all actions from the current transaction.
1284 Undo all the changes made since the database was opened or the last
1285 commit() or rollback() was performed.
1286 """
1287 logging.getLogger('hyperdb').info('rollback')
1289 self.sql_rollback()
1291 # roll back "other" transaction stuff
1292 for method, args in self.transactions:
1293 # delete temporary files
1294 if method == self.doStoreFile:
1295 self.rollbackStoreFile(*args)
1296 self.transactions = []
1298 # clear the cache
1299 self.clearCache()
1301 def sql_close(self):
1302 logging.getLogger('hyperdb').info('close')
1303 self.conn.close()
1305 def close(self):
1306 """ Close off the connection.
1307 """
1308 self.indexer.close()
1309 self.sql_close()
1311 #
1312 # The base Class class
1313 #
1314 class Class(hyperdb.Class):
1315 """ The handle to a particular class of nodes in a hyperdatabase.
1317 All methods except __repr__ and getnode must be implemented by a
1318 concrete backend Class.
1319 """
1321 def schema(self):
1322 """ A dumpable version of the schema that we can store in the
1323 database
1324 """
1325 return (self.key, [(x, repr(y)) for x,y in self.properties.items()])
1327 def enableJournalling(self):
1328 """Turn journalling on for this class
1329 """
1330 self.do_journal = 1
1332 def disableJournalling(self):
1333 """Turn journalling off for this class
1334 """
1335 self.do_journal = 0
1337 # Editing nodes:
1338 def create(self, **propvalues):
1339 """ Create a new node of this class and return its id.
1341 The keyword arguments in 'propvalues' map property names to values.
1343 The values of arguments must be acceptable for the types of their
1344 corresponding properties or a TypeError is raised.
1346 If this class has a key property, it must be present and its value
1347 must not collide with other key strings or a ValueError is raised.
1349 Any other properties on this class that are missing from the
1350 'propvalues' dictionary are set to None.
1352 If an id in a link or multilink property does not refer to a valid
1353 node, an IndexError is raised.
1354 """
1355 self.fireAuditors('create', None, propvalues)
1356 newid = self.create_inner(**propvalues)
1357 self.fireReactors('create', newid, None)
1358 return newid
1360 def create_inner(self, **propvalues):
1361 """ Called by create, in-between the audit and react calls.
1362 """
1363 if propvalues.has_key('id'):
1364 raise KeyError, '"id" is reserved'
1366 if self.db.journaltag is None:
1367 raise DatabaseError, _('Database open read-only')
1369 if propvalues.has_key('creator') or propvalues.has_key('actor') or \
1370 propvalues.has_key('creation') or propvalues.has_key('activity'):
1371 raise KeyError, '"creator", "actor", "creation" and '\
1372 '"activity" are reserved'
1374 # new node's id
1375 newid = self.db.newid(self.classname)
1377 # validate propvalues
1378 num_re = re.compile('^\d+$')
1379 for key, value in propvalues.items():
1380 if key == self.key:
1381 try:
1382 self.lookup(value)
1383 except KeyError:
1384 pass
1385 else:
1386 raise ValueError, 'node with key "%s" exists'%value
1388 # try to handle this property
1389 try:
1390 prop = self.properties[key]
1391 except KeyError:
1392 raise KeyError, '"%s" has no property "%s"'%(self.classname,
1393 key)
1395 if value is not None and isinstance(prop, Link):
1396 if type(value) != type(''):
1397 raise ValueError, 'link value must be String'
1398 link_class = self.properties[key].classname
1399 # if it isn't a number, it's a key
1400 if not num_re.match(value):
1401 try:
1402 value = self.db.classes[link_class].lookup(value)
1403 except (TypeError, KeyError):
1404 raise IndexError, 'new property "%s": %s not a %s'%(
1405 key, value, link_class)
1406 elif not self.db.getclass(link_class).hasnode(value):
1407 raise IndexError, '%s has no node %s'%(link_class, value)
1409 # save off the value
1410 propvalues[key] = value
1412 # register the link with the newly linked node
1413 if self.do_journal and self.properties[key].do_journal:
1414 self.db.addjournal(link_class, value, 'link',
1415 (self.classname, newid, key))
1417 elif isinstance(prop, Multilink):
1418 if value is None:
1419 value = []
1420 if not hasattr(value, '__iter__'):
1421 raise TypeError, 'new property "%s" not an iterable of ids'%key
1423 # clean up and validate the list of links
1424 link_class = self.properties[key].classname
1425 l = []
1426 for entry in value:
1427 if type(entry) != type(''):
1428 raise ValueError, '"%s" multilink value (%r) '\
1429 'must contain Strings'%(key, value)
1430 # if it isn't a number, it's a key
1431 if not num_re.match(entry):
1432 try:
1433 entry = self.db.classes[link_class].lookup(entry)
1434 except (TypeError, KeyError):
1435 raise IndexError, 'new property "%s": %s not a %s'%(
1436 key, entry, self.properties[key].classname)
1437 l.append(entry)
1438 value = l
1439 propvalues[key] = value
1441 # handle additions
1442 for nodeid in value:
1443 if not self.db.getclass(link_class).hasnode(nodeid):
1444 raise IndexError, '%s has no node %s'%(link_class,
1445 nodeid)
1446 # register the link with the newly linked node
1447 if self.do_journal and self.properties[key].do_journal:
1448 self.db.addjournal(link_class, nodeid, 'link',
1449 (self.classname, newid, key))
1451 elif isinstance(prop, String):
1452 if type(value) != type('') and type(value) != type(u''):
1453 raise TypeError, 'new property "%s" not a string'%key
1454 if prop.indexme:
1455 self.db.indexer.add_text((self.classname, newid, key),
1456 value)
1458 elif isinstance(prop, Password):
1459 if not isinstance(value, password.Password):
1460 raise TypeError, 'new property "%s" not a Password'%key
1462 elif isinstance(prop, Date):
1463 if value is not None and not isinstance(value, date.Date):
1464 raise TypeError, 'new property "%s" not a Date'%key
1466 elif isinstance(prop, Interval):
1467 if value is not None and not isinstance(value, date.Interval):
1468 raise TypeError, 'new property "%s" not an Interval'%key
1470 elif value is not None and isinstance(prop, Number):
1471 try:
1472 float(value)
1473 except ValueError:
1474 raise TypeError, 'new property "%s" not numeric'%key
1476 elif value is not None and isinstance(prop, Boolean):
1477 try:
1478 int(value)
1479 except ValueError:
1480 raise TypeError, 'new property "%s" not boolean'%key
1482 # make sure there's data where there needs to be
1483 for key, prop in self.properties.items():
1484 if propvalues.has_key(key):
1485 continue
1486 if key == self.key:
1487 raise ValueError, 'key property "%s" is required'%key
1488 if isinstance(prop, Multilink):
1489 propvalues[key] = []
1490 else:
1491 propvalues[key] = None
1493 # done
1494 self.db.addnode(self.classname, newid, propvalues)
1495 if self.do_journal:
1496 self.db.addjournal(self.classname, newid, ''"create", {})
1498 # XXX numeric ids
1499 return str(newid)
1501 def get(self, nodeid, propname, default=_marker, cache=1):
1502 """Get the value of a property on an existing node of this class.
1504 'nodeid' must be the id of an existing node of this class or an
1505 IndexError is raised. 'propname' must be the name of a property
1506 of this class or a KeyError is raised.
1508 'cache' exists for backwards compatibility, and is not used.
1509 """
1510 if propname == 'id':
1511 return nodeid
1513 # get the node's dict
1514 d = self.db.getnode(self.classname, nodeid)
1516 if propname == 'creation':
1517 if d.has_key('creation'):
1518 return d['creation']
1519 else:
1520 return date.Date()
1521 if propname == 'activity':
1522 if d.has_key('activity'):
1523 return d['activity']
1524 else:
1525 return date.Date()
1526 if propname == 'creator':
1527 if d.has_key('creator'):
1528 return d['creator']
1529 else:
1530 return self.db.getuid()
1531 if propname == 'actor':
1532 if d.has_key('actor'):
1533 return d['actor']
1534 else:
1535 return self.db.getuid()
1537 # get the property (raises KeyErorr if invalid)
1538 prop = self.properties[propname]
1540 # XXX may it be that propname is valid property name
1541 # (above error is not raised) and not d.has_key(propname)???
1542 if (not d.has_key(propname)) or (d[propname] is None):
1543 if default is _marker:
1544 if isinstance(prop, Multilink):
1545 return []
1546 else:
1547 return None
1548 else:
1549 return default
1551 # don't pass our list to other code
1552 if isinstance(prop, Multilink):
1553 return d[propname][:]
1555 return d[propname]
1557 def set(self, nodeid, **propvalues):
1558 """Modify a property on an existing node of this class.
1560 'nodeid' must be the id of an existing node of this class or an
1561 IndexError is raised.
1563 Each key in 'propvalues' must be the name of a property of this
1564 class or a KeyError is raised.
1566 All values in 'propvalues' must be acceptable types for their
1567 corresponding properties or a TypeError is raised.
1569 If the value of the key property is set, it must not collide with
1570 other key strings or a ValueError is raised.
1572 If the value of a Link or Multilink property contains an invalid
1573 node id, a ValueError is raised.
1574 """
1575 self.fireAuditors('set', nodeid, propvalues)
1576 oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
1577 propvalues = self.set_inner(nodeid, **propvalues)
1578 self.fireReactors('set', nodeid, oldvalues)
1579 return propvalues
1581 def set_inner(self, nodeid, **propvalues):
1582 """ Called by set, in-between the audit and react calls.
1583 """
1584 if not propvalues:
1585 return propvalues
1587 if propvalues.has_key('creation') or propvalues.has_key('creator') or \
1588 propvalues.has_key('actor') or propvalues.has_key('activity'):
1589 raise KeyError, '"creation", "creator", "actor" and '\
1590 '"activity" are reserved'
1592 if propvalues.has_key('id'):
1593 raise KeyError, '"id" is reserved'
1595 if self.db.journaltag is None:
1596 raise DatabaseError, _('Database open read-only')
1598 node = self.db.getnode(self.classname, nodeid)
1599 if self.is_retired(nodeid):
1600 raise IndexError, 'Requested item is retired'
1601 num_re = re.compile('^\d+$')
1603 # make a copy of the values dictionary - we'll modify the contents
1604 propvalues = propvalues.copy()
1606 # if the journal value is to be different, store it in here
1607 journalvalues = {}
1609 # remember the add/remove stuff for multilinks, making it easier
1610 # for the Database layer to do its stuff
1611 multilink_changes = {}
1613 for propname, value in propvalues.items():
1614 # check to make sure we're not duplicating an existing key
1615 if propname == self.key and node[propname] != value:
1616 try:
1617 self.lookup(value)
1618 except KeyError:
1619 pass
1620 else:
1621 raise ValueError, 'node with key "%s" exists'%value
1623 # this will raise the KeyError if the property isn't valid
1624 # ... we don't use getprops() here because we only care about
1625 # the writeable properties.
1626 try:
1627 prop = self.properties[propname]
1628 except KeyError:
1629 raise KeyError, '"%s" has no property named "%s"'%(
1630 self.classname, propname)
1632 # if the value's the same as the existing value, no sense in
1633 # doing anything
1634 current = node.get(propname, None)
1635 if value == current:
1636 del propvalues[propname]
1637 continue
1638 journalvalues[propname] = current
1640 # do stuff based on the prop type
1641 if isinstance(prop, Link):
1642 link_class = prop.classname
1643 # if it isn't a number, it's a key
1644 if value is not None and not isinstance(value, type('')):
1645 raise ValueError, 'property "%s" link value be a string'%(
1646 propname)
1647 if isinstance(value, type('')) and not num_re.match(value):
1648 try:
1649 value = self.db.classes[link_class].lookup(value)
1650 except (TypeError, KeyError):
1651 raise IndexError, 'new property "%s": %s not a %s'%(
1652 propname, value, prop.classname)
1654 if (value is not None and
1655 not self.db.getclass(link_class).hasnode(value)):
1656 raise IndexError, '%s has no node %s'%(link_class, value)
1658 if self.do_journal and prop.do_journal:
1659 # register the unlink with the old linked node
1660 if node[propname] is not None:
1661 self.db.addjournal(link_class, node[propname],
1662 ''"unlink", (self.classname, nodeid, propname))
1664 # register the link with the newly linked node
1665 if value is not None:
1666 self.db.addjournal(link_class, value, ''"link",
1667 (self.classname, nodeid, propname))
1669 elif isinstance(prop, Multilink):
1670 if value is None:
1671 value = []
1672 if not hasattr(value, '__iter__'):
1673 raise TypeError, 'new property "%s" not an iterable of'\
1674 ' ids'%propname
1675 link_class = self.properties[propname].classname
1676 l = []
1677 for entry in value:
1678 # if it isn't a number, it's a key
1679 if type(entry) != type(''):
1680 raise ValueError, 'new property "%s" link value ' \
1681 'must be a string'%propname
1682 if not num_re.match(entry):
1683 try:
1684 entry = self.db.classes[link_class].lookup(entry)
1685 except (TypeError, KeyError):
1686 raise IndexError, 'new property "%s": %s not a %s'%(
1687 propname, entry,
1688 self.properties[propname].classname)
1689 l.append(entry)
1690 value = l
1691 propvalues[propname] = value
1693 # figure the journal entry for this property
1694 add = []
1695 remove = []
1697 # handle removals
1698 if node.has_key(propname):
1699 l = node[propname]
1700 else:
1701 l = []
1702 for id in l[:]:
1703 if id in value:
1704 continue
1705 # register the unlink with the old linked node
1706 if self.do_journal and self.properties[propname].do_journal:
1707 self.db.addjournal(link_class, id, 'unlink',
1708 (self.classname, nodeid, propname))
1709 l.remove(id)
1710 remove.append(id)
1712 # handle additions
1713 for id in value:
1714 # If this node is in the cache, then we do not need to go to
1715 # the database. (We don't consider this an LRU hit, though.)
1716 if self.cache.has_key((classname, nodeid)):
1717 # Return 1, not True, to match the type of the result of
1718 # the SQL operation below.
1719 return 1
1720 if not self.db.getclass(link_class).hasnode(id):
1721 raise IndexError, '%s has no node %s'%(link_class, id)
1722 if id in l:
1723 continue
1724 # register the link with the newly linked node
1725 if self.do_journal and self.properties[propname].do_journal:
1726 self.db.addjournal(link_class, id, 'link',
1727 (self.classname, nodeid, propname))
1728 l.append(id)
1729 add.append(id)
1731 # figure the journal entry
1732 l = []
1733 if add:
1734 l.append(('+', add))
1735 if remove:
1736 l.append(('-', remove))
1737 multilink_changes[propname] = (add, remove)
1738 if l:
1739 journalvalues[propname] = tuple(l)
1741 elif isinstance(prop, String):
1742 if value is not None and type(value) != type('') and type(value) != type(u''):
1743 raise TypeError, 'new property "%s" not a string'%propname
1744 if prop.indexme:
1745 if value is None: value = ''
1746 self.db.indexer.add_text((self.classname, nodeid, propname),
1747 value)
1749 elif isinstance(prop, Password):
1750 if not isinstance(value, password.Password):
1751 raise TypeError, 'new property "%s" not a Password'%propname
1752 propvalues[propname] = value
1754 elif value is not None and isinstance(prop, Date):
1755 if not isinstance(value, date.Date):
1756 raise TypeError, 'new property "%s" not a Date'% propname
1757 propvalues[propname] = value
1759 elif value is not None and isinstance(prop, Interval):
1760 if not isinstance(value, date.Interval):
1761 raise TypeError, 'new property "%s" not an '\
1762 'Interval'%propname
1763 propvalues[propname] = value
1765 elif value is not None and isinstance(prop, Number):
1766 try:
1767 float(value)
1768 except ValueError:
1769 raise TypeError, 'new property "%s" not numeric'%propname
1771 elif value is not None and isinstance(prop, Boolean):
1772 try:
1773 int(value)
1774 except ValueError:
1775 raise TypeError, 'new property "%s" not boolean'%propname
1777 # nothing to do?
1778 if not propvalues:
1779 return propvalues
1781 # update the activity time
1782 propvalues['activity'] = date.Date()
1783 propvalues['actor'] = self.db.getuid()
1785 # do the set
1786 self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
1788 # remove the activity props now they're handled
1789 del propvalues['activity']
1790 del propvalues['actor']
1792 # journal the set
1793 if self.do_journal:
1794 self.db.addjournal(self.classname, nodeid, ''"set", journalvalues)
1796 return propvalues
1798 def retire(self, nodeid):
1799 """Retire a node.
1801 The properties on the node remain available from the get() method,
1802 and the node's id is never reused.
1804 Retired nodes are not returned by the find(), list(), or lookup()
1805 methods, and other nodes may reuse the values of their key properties.
1806 """
1807 if self.db.journaltag is None:
1808 raise DatabaseError, _('Database open read-only')
1810 self.fireAuditors('retire', nodeid, None)
1812 # use the arg for __retired__ to cope with any odd database type
1813 # conversion (hello, sqlite)
1814 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1815 self.db.arg, self.db.arg)
1816 self.db.sql(sql, (nodeid, nodeid))
1817 if self.do_journal:
1818 self.db.addjournal(self.classname, nodeid, ''"retired", None)
1820 self.fireReactors('retire', nodeid, None)
1822 def restore(self, nodeid):
1823 """Restore a retired node.
1825 Make node available for all operations like it was before retirement.
1826 """
1827 if self.db.journaltag is None:
1828 raise DatabaseError, _('Database open read-only')
1830 node = self.db.getnode(self.classname, nodeid)
1831 # check if key property was overrided
1832 key = self.getkey()
1833 try:
1834 id = self.lookup(node[key])
1835 except KeyError:
1836 pass
1837 else:
1838 raise KeyError, "Key property (%s) of retired node clashes with \
1839 existing one (%s)" % (key, node[key])
1841 self.fireAuditors('restore', nodeid, None)
1842 # use the arg for __retired__ to cope with any odd database type
1843 # conversion (hello, sqlite)
1844 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
1845 self.db.arg, self.db.arg)
1846 self.db.sql(sql, (0, nodeid))
1847 if self.do_journal:
1848 self.db.addjournal(self.classname, nodeid, ''"restored", None)
1850 self.fireReactors('restore', nodeid, None)
1852 def is_retired(self, nodeid):
1853 """Return true if the node is rerired
1854 """
1855 sql = 'select __retired__ from _%s where id=%s'%(self.classname,
1856 self.db.arg)
1857 self.db.sql(sql, (nodeid,))
1858 return int(self.db.sql_fetchone()[0]) > 0
1860 def destroy(self, nodeid):
1861 """Destroy a node.
1863 WARNING: this method should never be used except in extremely rare
1864 situations where there could never be links to the node being
1865 deleted
1867 WARNING: use retire() instead
1869 WARNING: the properties of this node will not be available ever again
1871 WARNING: really, use retire() instead
1873 Well, I think that's enough warnings. This method exists mostly to
1874 support the session storage of the cgi interface.
1876 The node is completely removed from the hyperdb, including all journal
1877 entries. It will no longer be available, and will generally break code
1878 if there are any references to the node.
1879 """
1880 if self.db.journaltag is None:
1881 raise DatabaseError, _('Database open read-only')
1882 self.db.destroynode(self.classname, nodeid)
1884 def history(self, nodeid):
1885 """Retrieve the journal of edits on a particular node.
1887 'nodeid' must be the id of an existing node of this class or an
1888 IndexError is raised.
1890 The returned list contains tuples of the form
1892 (nodeid, date, tag, action, params)
1894 'date' is a Timestamp object specifying the time of the change and
1895 'tag' is the journaltag specified when the database was opened.
1896 """
1897 if not self.do_journal:
1898 raise ValueError, 'Journalling is disabled for this class'
1899 return self.db.getjournal(self.classname, nodeid)
1901 # Locating nodes:
1902 def hasnode(self, nodeid):
1903 """Determine if the given nodeid actually exists
1904 """
1905 return self.db.hasnode(self.classname, nodeid)
1907 def setkey(self, propname):
1908 """Select a String property of this class to be the key property.
1910 'propname' must be the name of a String property of this class or
1911 None, or a TypeError is raised. The values of the key property on
1912 all existing nodes must be unique or a ValueError is raised.
1913 """
1914 prop = self.getprops()[propname]
1915 if not isinstance(prop, String):
1916 raise TypeError, 'key properties must be String'
1917 self.key = propname
1919 def getkey(self):
1920 """Return the name of the key property for this class or None."""
1921 return self.key
1923 def lookup(self, keyvalue):
1924 """Locate a particular node by its key property and return its id.
1926 If this class has no key property, a TypeError is raised. If the
1927 'keyvalue' matches one of the values for the key property among
1928 the nodes in this class, the matching node's id is returned;
1929 otherwise a KeyError is raised.
1930 """
1931 if not self.key:
1932 raise TypeError, 'No key property set for class %s'%self.classname
1934 # use the arg to handle any odd database type conversion (hello,
1935 # sqlite)
1936 sql = "select id from _%s where _%s=%s and __retired__=%s"%(
1937 self.classname, self.key, self.db.arg, self.db.arg)
1938 self.db.sql(sql, (str(keyvalue), 0))
1940 # see if there was a result that's not retired
1941 row = self.db.sql_fetchone()
1942 if not row:
1943 raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
1944 keyvalue, self.classname)
1946 # return the id
1947 # XXX numeric ids
1948 return str(row[0])
1950 def find(self, **propspec):
1951 """Get the ids of nodes in this class which link to the given nodes.
1953 'propspec' consists of keyword args propname=nodeid or
1954 propname={nodeid:1, }
1955 'propname' must be the name of a property in this class, or a
1956 KeyError is raised. That property must be a Link or
1957 Multilink property, or a TypeError is raised.
1959 Any node in this class whose 'propname' property links to any of
1960 the nodeids will be returned. Examples::
1962 db.issue.find(messages='1')
1963 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1964 """
1965 # shortcut
1966 if not propspec:
1967 return []
1969 # validate the args
1970 props = self.getprops()
1971 propspec = propspec.items()
1972 for propname, nodeids in propspec:
1973 # check the prop is OK
1974 prop = props[propname]
1975 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
1976 raise TypeError, "'%s' not a Link/Multilink property"%propname
1978 # first, links
1979 a = self.db.arg
1980 allvalues = ()
1981 sql = []
1982 where = []
1983 for prop, values in propspec:
1984 if not isinstance(props[prop], hyperdb.Link):
1985 continue
1986 if type(values) is type({}) and len(values) == 1:
1987 values = values.keys()[0]
1988 if type(values) is type(''):
1989 allvalues += (values,)
1990 where.append('_%s = %s'%(prop, a))
1991 elif values is None:
1992 where.append('_%s is NULL'%prop)
1993 else:
1994 values = values.keys()
1995 s = ''
1996 if None in values:
1997 values.remove(None)
1998 s = '_%s is NULL or '%prop
1999 allvalues += tuple(values)
2000 s += '_%s in (%s)'%(prop, ','.join([a]*len(values)))
2001 where.append('(' + s +')')
2002 if where:
2003 allvalues = (0, ) + allvalues
2004 sql.append("""select id from _%s where __retired__=%s
2005 and %s"""%(self.classname, a, ' and '.join(where)))
2007 # now multilinks
2008 for prop, values in propspec:
2009 if not isinstance(props[prop], hyperdb.Multilink):
2010 continue
2011 if not values:
2012 continue
2013 allvalues += (0, )
2014 if type(values) is type(''):
2015 allvalues += (values,)
2016 s = a
2017 else:
2018 allvalues += tuple(values.keys())
2019 s = ','.join([a]*len(values))
2020 tn = '%s_%s'%(self.classname, prop)
2021 sql.append("""select id from _%s, %s where __retired__=%s
2022 and id = %s.nodeid and %s.linkid in (%s)"""%(self.classname,
2023 tn, a, tn, tn, s))
2025 if not sql:
2026 return []
2027 sql = ' union '.join(sql)
2028 self.db.sql(sql, allvalues)
2029 # XXX numeric ids
2030 l = [str(x[0]) for x in self.db.sql_fetchall()]
2031 return l
2033 def stringFind(self, **requirements):
2034 """Locate a particular node by matching a set of its String
2035 properties in a caseless search.
2037 If the property is not a String property, a TypeError is raised.
2039 The return is a list of the id of all nodes that match.
2040 """
2041 where = []
2042 args = []
2043 for propname in requirements.keys():
2044 prop = self.properties[propname]
2045 if not isinstance(prop, String):
2046 raise TypeError, "'%s' not a String property"%propname
2047 where.append(propname)
2048 args.append(requirements[propname].lower())
2050 # generate the where clause
2051 s = ' and '.join(['lower(_%s)=%s'%(col, self.db.arg) for col in where])
2052 sql = 'select id from _%s where %s and __retired__=%s'%(
2053 self.classname, s, self.db.arg)
2054 args.append(0)
2055 self.db.sql(sql, tuple(args))
2056 # XXX numeric ids
2057 l = [str(x[0]) for x in self.db.sql_fetchall()]
2058 return l
2060 def list(self):
2061 """ Return a list of the ids of the active nodes in this class.
2062 """
2063 return self.getnodeids(retired=0)
2065 def getnodeids(self, retired=None):
2066 """ Retrieve all the ids of the nodes for a particular Class.
2068 Set retired=None to get all nodes. Otherwise it'll get all the
2069 retired or non-retired nodes, depending on the flag.
2070 """
2071 # flip the sense of the 'retired' flag if we don't want all of them
2072 if retired is not None:
2073 args = (0, )
2074 if retired:
2075 compare = '>'
2076 else:
2077 compare = '='
2078 sql = 'select id from _%s where __retired__%s%s'%(self.classname,
2079 compare, self.db.arg)
2080 else:
2081 args = ()
2082 sql = 'select id from _%s'%self.classname
2083 self.db.sql(sql, args)
2084 # XXX numeric ids
2085 ids = [str(x[0]) for x in self.db.cursor.fetchall()]
2086 return ids
2088 def _subselect(self, classname, multilink_table):
2089 """Create a subselect. This is factored out because some
2090 databases (hmm only one, so far) doesn't support subselects
2091 look for "I can't believe it's not a toy RDBMS" in the mysql
2092 backend.
2093 """
2094 return '_%s.id not in (select nodeid from %s)'%(classname,
2095 multilink_table)
2097 # Some DBs order NULL values last. Set this variable in the backend
2098 # for prepending an order by clause for each attribute that causes
2099 # correct sort order for NULLs. Examples:
2100 # order_by_null_values = '(%s is not NULL)'
2101 # order_by_null_values = 'notnull(%s)'
2102 # The format parameter is replaced with the attribute.
2103 order_by_null_values = None
2105 def filter(self, search_matches, filterspec, sort=[], group=[]):
2106 """Return a list of the ids of the active nodes in this class that
2107 match the 'filter' spec, sorted by the group spec and then the
2108 sort spec
2110 "filterspec" is {propname: value(s)}
2112 "sort" and "group" are [(dir, prop), ...] where dir is '+', '-'
2113 or None and prop is a prop name or None. Note that for
2114 backward-compatibility reasons a single (dir, prop) tuple is
2115 also allowed.
2117 "search_matches" is {nodeid: marker} or None
2119 The filter must match all properties specificed. If the property
2120 value to match is a list:
2122 1. String properties must match all elements in the list, and
2123 2. Other properties must match any of the elements in the list.
2124 """
2125 # we can't match anything if search_matches is empty
2126 if search_matches == {}:
2127 return []
2129 if __debug__:
2130 start_t = time.time()
2132 icn = self.classname
2134 # vars to hold the components of the SQL statement
2135 frum = [] # FROM clauses
2136 loj = [] # LEFT OUTER JOIN clauses
2137 where = [] # WHERE clauses
2138 args = [] # *any* positional arguments
2139 a = self.db.arg
2141 # figure the WHERE clause from the filterspec
2142 mlfilt = 0 # are we joining with Multilink tables?
2143 sortattr = self._sortattr (group = group, sort = sort)
2144 proptree = self._proptree(filterspec, sortattr)
2145 mlseen = 0
2146 for pt in reversed(proptree.sortattr):
2147 p = pt
2148 while p.parent:
2149 if isinstance (p.propclass, Multilink):
2150 mlseen = True
2151 if mlseen:
2152 p.sort_ids_needed = True
2153 p.tree_sort_done = False
2154 p = p.parent
2155 if not mlseen:
2156 pt.attr_sort_done = pt.tree_sort_done = True
2157 proptree.compute_sort_done()
2159 ordercols = []
2160 auxcols = {}
2161 mlsort = []
2162 rhsnum = 0
2163 for p in proptree:
2164 oc = None
2165 cn = p.classname
2166 ln = p.uniqname
2167 pln = p.parent.uniqname
2168 pcn = p.parent.classname
2169 k = p.name
2170 v = p.val
2171 propclass = p.propclass
2172 if p.sort_type > 0:
2173 oc = ac = '_%s._%s'%(pln, k)
2174 if isinstance(propclass, Multilink):
2175 if p.sort_type < 2:
2176 mlfilt = 1
2177 tn = '%s_%s'%(pcn, k)
2178 if v in ('-1', ['-1']):
2179 # only match rows that have count(linkid)=0 in the
2180 # corresponding multilink table)
2181 where.append(self._subselect(pcn, tn))
2182 else:
2183 frum.append(tn)
2184 where.append('_%s.id=%s.nodeid'%(pln,tn))
2185 if p.children:
2186 frum.append('_%s as _%s' % (cn, ln))
2187 where.append('%s.linkid=_%s.id'%(tn, ln))
2188 if p.has_values:
2189 if isinstance(v, type([])):
2190 s = ','.join([a for x in v])
2191 where.append('%s.linkid in (%s)'%(tn, s))
2192 args = args + v
2193 else:
2194 where.append('%s.linkid=%s'%(tn, a))
2195 args.append(v)
2196 if p.sort_type > 0:
2197 assert not p.attr_sort_done and not p.sort_ids_needed
2198 elif k == 'id':
2199 if p.sort_type < 2:
2200 if isinstance(v, type([])):
2201 s = ','.join([a for x in v])
2202 where.append('_%s.%s in (%s)'%(pln, k, s))
2203 args = args + v
2204 else:
2205 where.append('_%s.%s=%s'%(pln, k, a))
2206 args.append(v)
2207 if p.sort_type > 0:
2208 oc = ac = '_%s.id'%pln
2209 elif isinstance(propclass, String):
2210 if p.sort_type < 2:
2211 if not isinstance(v, type([])):
2212 v = [v]
2214 # Quote the bits in the string that need it and then embed
2215 # in a "substring" search. Note - need to quote the '%' so
2216 # they make it through the python layer happily
2217 v = ['%%'+self.db.sql_stringquote(s)+'%%' for s in v]
2219 # now add to the where clause
2220 where.append('('
2221 +' and '.join(["_%s._%s LIKE '%s'"%(pln, k, s) for s in v])
2222 +')')
2223 # note: args are embedded in the query string now
2224 if p.sort_type > 0:
2225 oc = ac = 'lower(_%s._%s)'%(pln, k)
2226 elif isinstance(propclass, Link):
2227 if p.sort_type < 2:
2228 if p.children:
2229 if p.sort_type == 0:
2230 frum.append('_%s as _%s' % (cn, ln))
2231 where.append('_%s._%s=_%s.id'%(pln, k, ln))
2232 if p.has_values:
2233 if isinstance(v, type([])):
2234 d = {}
2235 for entry in v:
2236 if entry == '-1':
2237 entry = None
2238 d[entry] = entry
2239 l = []
2240 if d.has_key(None) or not d:
2241 if d.has_key(None): del d[None]
2242 l.append('_%s._%s is NULL'%(pln, k))
2243 if d:
2244 v = d.keys()
2245 s = ','.join([a for x in v])
2246 l.append('(_%s._%s in (%s))'%(pln, k, s))
2247 args = args + v
2248 if l:
2249 where.append('(' + ' or '.join(l) +')')
2250 else:
2251 if v in ('-1', None):
2252 v = None
2253 where.append('_%s._%s is NULL'%(pln, k))
2254 else:
2255 where.append('_%s._%s=%s'%(pln, k, a))
2256 args.append(v)
2257 if p.sort_type > 0:
2258 lp = p.cls.labelprop()
2259 oc = ac = '_%s._%s'%(pln, k)
2260 if lp != 'id':
2261 if p.tree_sort_done and p.sort_type > 0:
2262 loj.append(
2263 'LEFT OUTER JOIN _%s as _%s on _%s._%s=_%s.id'%(
2264 cn, ln, pln, k, ln))
2265 oc = '_%s._%s'%(ln, lp)
2266 elif isinstance(propclass, Date) and p.sort_type < 2:
2267 dc = self.db.hyperdb_to_sql_value[hyperdb.Date]
2268 if isinstance(v, type([])):
2269 s = ','.join([a for x in v])
2270 where.append('_%s._%s in (%s)'%(pln, k, s))
2271 args = args + [dc(date.Date(x)) for x in v]
2272 else:
2273 try:
2274 # Try to filter on range of dates
2275 date_rng = propclass.range_from_raw(v, self.db)
2276 if date_rng.from_value:
2277 where.append('_%s._%s >= %s'%(pln, k, a))
2278 args.append(dc(date_rng.from_value))
2279 if date_rng.to_value:
2280 where.append('_%s._%s <= %s'%(pln, k, a))
2281 args.append(dc(date_rng.to_value))
2282 except ValueError:
2283 # If range creation fails - ignore that search parameter
2284 pass
2285 elif isinstance(propclass, Interval):
2286 # filter/sort using the __<prop>_int__ column
2287 if p.sort_type < 2:
2288 if isinstance(v, type([])):
2289 s = ','.join([a for x in v])
2290 where.append('_%s.__%s_int__ in (%s)'%(pln, k, s))
2291 args = args + [date.Interval(x).as_seconds() for x in v]
2292 else:
2293 try:
2294 # Try to filter on range of intervals
2295 date_rng = Range(v, date.Interval)
2296 if date_rng.from_value:
2297 where.append('_%s.__%s_int__ >= %s'%(pln, k, a))
2298 args.append(date_rng.from_value.as_seconds())
2299 if date_rng.to_value:
2300 where.append('_%s.__%s_int__ <= %s'%(pln, k, a))
2301 args.append(date_rng.to_value.as_seconds())
2302 except ValueError:
2303 # If range creation fails - ignore search parameter
2304 pass
2305 if p.sort_type > 0:
2306 oc = ac = '_%s.__%s_int__'%(pln,k)
2307 elif p.sort_type < 2:
2308 if isinstance(v, type([])):
2309 s = ','.join([a for x in v])
2310 where.append('_%s._%s in (%s)'%(pln, k, s))
2311 args = args + v
2312 else:
2313 where.append('_%s._%s=%s'%(pln, k, a))
2314 args.append(v)
2315 if oc:
2316 if p.sort_ids_needed:
2317 auxcols[ac] = p
2318 if p.tree_sort_done and p.sort_direction:
2319 # Don't select top-level id twice
2320 if p.name != 'id' or p.parent != proptree:
2321 ordercols.append(oc)
2322 desc = ['', ' desc'][p.sort_direction == '-']
2323 # Some SQL dbs sort NULL values last -- we want them first.
2324 if (self.order_by_null_values and p.name != 'id'):
2325 nv = self.order_by_null_values % oc
2326 ordercols.append(nv)
2327 p.orderby.append(nv + desc)
2328 p.orderby.append(oc + desc)
2330 props = self.getprops()
2332 # don't match retired nodes
2333 where.append('_%s.__retired__=0'%icn)
2335 # add results of full text search
2336 if search_matches is not None:
2337 v = search_matches.keys()
2338 s = ','.join([a for x in v])
2339 where.append('_%s.id in (%s)'%(icn, s))
2340 args = args + v
2342 # construct the SQL
2343 frum.append('_'+icn)
2344 frum = ','.join(frum)
2345 if where:
2346 where = ' where ' + (' and '.join(where))
2347 else:
2348 where = ''
2349 if mlfilt:
2350 # we're joining tables on the id, so we will get dupes if we
2351 # don't distinct()
2352 cols = ['distinct(_%s.id)'%icn]
2353 else:
2354 cols = ['_%s.id'%icn]
2355 if ordercols:
2356 cols = cols + ordercols
2357 order = []
2358 # keep correct sequence of order attributes.
2359 for sa in proptree.sortattr:
2360 if not sa.attr_sort_done:
2361 continue
2362 order.extend(sa.orderby)
2363 if order:
2364 order = ' order by %s'%(','.join(order))
2365 else:
2366 order = ''
2367 for o, p in auxcols.iteritems ():
2368 cols.append (o)
2369 p.auxcol = len (cols) - 1
2371 cols = ','.join(cols)
2372 loj = ' '.join(loj)
2373 sql = 'select %s from %s %s %s%s'%(cols, frum, loj, where, order)
2374 args = tuple(args)
2375 __traceback_info__ = (sql, args)
2376 self.db.sql(sql, args)
2377 l = self.db.sql_fetchall()
2379 # Compute values needed for sorting in proptree.sort
2380 for p in auxcols.itervalues():
2381 p.sort_ids = p.sort_result = [row[p.auxcol] for row in l]
2382 # return the IDs (the first column)
2383 # XXX numeric ids
2384 l = [str(row[0]) for row in l]
2385 l = proptree.sort (l)
2387 if __debug__:
2388 self.db.stats['filtering'] += (time.time() - start_t)
2389 return l
2391 def filter_sql(self, sql):
2392 """Return a list of the ids of the items in this class that match
2393 the SQL provided. The SQL is a complete "select" statement.
2395 The SQL select must include the item id as the first column.
2397 This function DOES NOT filter out retired items, add on a where
2398 clause "__retired__=0" if you don't want retired nodes.
2399 """
2400 if __debug__:
2401 start_t = time.time()
2403 self.db.sql(sql)
2404 l = self.db.sql_fetchall()
2406 if __debug__:
2407 self.db.stats['filtering'] += (time.time() - start_t)
2408 return l
2410 def count(self):
2411 """Get the number of nodes in this class.
2413 If the returned integer is 'numnodes', the ids of all the nodes
2414 in this class run from 1 to numnodes, and numnodes+1 will be the
2415 id of the next node to be created in this class.
2416 """
2417 return self.db.countnodes(self.classname)
2419 # Manipulating properties:
2420 def getprops(self, protected=1):
2421 """Return a dictionary mapping property names to property objects.
2422 If the "protected" flag is true, we include protected properties -
2423 those which may not be modified.
2424 """
2425 d = self.properties.copy()
2426 if protected:
2427 d['id'] = String()
2428 d['creation'] = hyperdb.Date()
2429 d['activity'] = hyperdb.Date()
2430 d['creator'] = hyperdb.Link('user')
2431 d['actor'] = hyperdb.Link('user')
2432 return d
2434 def addprop(self, **properties):
2435 """Add properties to this class.
2437 The keyword arguments in 'properties' must map names to property
2438 objects, or a TypeError is raised. None of the keys in 'properties'
2439 may collide with the names of existing properties, or a ValueError
2440 is raised before any properties have been added.
2441 """
2442 for key in properties.keys():
2443 if self.properties.has_key(key):
2444 raise ValueError, key
2445 self.properties.update(properties)
2447 def index(self, nodeid):
2448 """Add (or refresh) the node to search indexes
2449 """
2450 # find all the String properties that have indexme
2451 for prop, propclass in self.getprops().items():
2452 if isinstance(propclass, String) and propclass.indexme:
2453 self.db.indexer.add_text((self.classname, nodeid, prop),
2454 str(self.get(nodeid, prop)))
2456 #
2457 # import / export support
2458 #
2459 def export_list(self, propnames, nodeid):
2460 """ Export a node - generate a list of CSV-able data in the order
2461 specified by propnames for the given node.
2462 """
2463 properties = self.getprops()
2464 l = []
2465 for prop in propnames:
2466 proptype = properties[prop]
2467 value = self.get(nodeid, prop)
2468 # "marshal" data where needed
2469 if value is None:
2470 pass
2471 elif isinstance(proptype, hyperdb.Date):
2472 value = value.get_tuple()
2473 elif isinstance(proptype, hyperdb.Interval):
2474 value = value.get_tuple()
2475 elif isinstance(proptype, hyperdb.Password):
2476 value = str(value)
2477 l.append(repr(value))
2478 l.append(repr(self.is_retired(nodeid)))
2479 return l
2481 def import_list(self, propnames, proplist):
2482 """ Import a node - all information including "id" is present and
2483 should not be sanity checked. Triggers are not triggered. The
2484 journal should be initialised using the "creator" and "created"
2485 information.
2487 Return the nodeid of the node imported.
2488 """
2489 if self.db.journaltag is None:
2490 raise DatabaseError, _('Database open read-only')
2491 properties = self.getprops()
2493 # make the new node's property map
2494 d = {}
2495 retire = 0
2496 if not "id" in propnames:
2497 newid = self.db.newid(self.classname)
2498 else:
2499 newid = eval(proplist[propnames.index("id")])
2500 for i in range(len(propnames)):
2501 # Use eval to reverse the repr() used to output the CSV
2502 value = eval(proplist[i])
2504 # Figure the property for this column
2505 propname = propnames[i]
2507 # "unmarshal" where necessary
2508 if propname == 'id':
2509 continue
2510 elif propname == 'is retired':
2511 # is the item retired?
2512 if int(value):
2513 retire = 1
2514 continue
2515 elif value is None:
2516 d[propname] = None
2517 continue
2519 prop = properties[propname]
2520 if value is None:
2521 # don't set Nones
2522 continue
2523 elif isinstance(prop, hyperdb.Date):
2524 value = date.Date(value)
2525 elif isinstance(prop, hyperdb.Interval):
2526 value = date.Interval(value)
2527 elif isinstance(prop, hyperdb.Password):
2528 pwd = password.Password()
2529 pwd.unpack(value)
2530 value = pwd
2531 elif isinstance(prop, String):
2532 if isinstance(value, unicode):
2533 value = value.encode('utf8')
2534 if not isinstance(value, str):
2535 raise TypeError, \
2536 'new property "%(propname)s" not a string: %(value)r' \
2537 % locals()
2538 if prop.indexme:
2539 self.db.indexer.add_text((self.classname, newid, propname),
2540 value)
2541 d[propname] = value
2543 # get a new id if necessary
2544 if newid is None:
2545 newid = self.db.newid(self.classname)
2547 # insert new node or update existing?
2548 if not self.hasnode(newid):
2549 self.db.addnode(self.classname, newid, d) # insert
2550 else:
2551 self.db.setnode(self.classname, newid, d) # update
2553 # retire?
2554 if retire:
2555 # use the arg for __retired__ to cope with any odd database type
2556 # conversion (hello, sqlite)
2557 sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
2558 self.db.arg, self.db.arg)
2559 self.db.sql(sql, (newid, newid))
2560 return newid
2562 def export_journals(self):
2563 """Export a class's journal - generate a list of lists of
2564 CSV-able data:
2566 nodeid, date, user, action, params
2568 No heading here - the columns are fixed.
2569 """
2570 properties = self.getprops()
2571 r = []
2572 for nodeid in self.getnodeids():
2573 for nodeid, date, user, action, params in self.history(nodeid):
2574 date = date.get_tuple()
2575 if action == 'set':
2576 export_data = {}
2577 for propname, value in params.items():
2578 if not properties.has_key(propname):
2579 # property no longer in the schema
2580 continue
2582 prop = properties[propname]
2583 # make sure the params are eval()'able
2584 if value is None:
2585 pass
2586 elif isinstance(prop, Date):
2587 value = value.get_tuple()
2588 elif isinstance(prop, Interval):
2589 value = value.get_tuple()
2590 elif isinstance(prop, Password):
2591 value = str(value)
2592 export_data[propname] = value
2593 params = export_data
2594 elif action == 'create' and params:
2595 # old tracker with data stored in the create!
2596 params = {}
2597 l = [nodeid, date, user, action, params]
2598 r.append(map(repr, l))
2599 return r
2601 def import_journals(self, entries):
2602 """Import a class's journal.
2604 Uses setjournal() to set the journal for each item."""
2605 properties = self.getprops()
2606 d = {}
2607 for l in entries:
2608 l = map(eval, l)
2609 nodeid, jdate, user, action, params = l
2610 r = d.setdefault(nodeid, [])
2611 if action == 'set':
2612 for propname, value in params.items():
2613 prop = properties[propname]
2614 if value is None:
2615 pass
2616 elif isinstance(prop, Date):
2617 value = date.Date(value)
2618 elif isinstance(prop, Interval):
2619 value = date.Interval(value)
2620 elif isinstance(prop, Password):
2621 pwd = password.Password()
2622 pwd.unpack(value)
2623 value = pwd
2624 params[propname] = value
2625 elif action == 'create' and params:
2626 # old tracker with data stored in the create!
2627 params = {}
2628 r.append((nodeid, date.Date(jdate), user, action, params))
2630 for nodeid, l in d.items():
2631 self.db.setjournal(self.classname, nodeid, l)
2633 class FileClass(hyperdb.FileClass, Class):
2634 """This class defines a large chunk of data. To support this, it has a
2635 mandatory String property "content" which is typically saved off
2636 externally to the hyperdb.
2638 The default MIME type of this data is defined by the
2639 "default_mime_type" class attribute, which may be overridden by each
2640 node if the class defines a "type" String property.
2641 """
2642 def __init__(self, db, classname, **properties):
2643 """The newly-created class automatically includes the "content"
2644 and "type" properties.
2645 """
2646 if not properties.has_key('content'):
2647 properties['content'] = hyperdb.String(indexme='yes')
2648 if not properties.has_key('type'):
2649 properties['type'] = hyperdb.String()
2650 Class.__init__(self, db, classname, **properties)
2652 def create(self, **propvalues):
2653 """ snaffle the file propvalue and store in a file
2654 """
2655 # we need to fire the auditors now, or the content property won't
2656 # be in propvalues for the auditors to play with
2657 self.fireAuditors('create', None, propvalues)
2659 # now remove the content property so it's not stored in the db
2660 content = propvalues['content']
2661 del propvalues['content']
2663 # do the database create
2664 newid = self.create_inner(**propvalues)
2666 # figure the mime type
2667 mime_type = propvalues.get('type', self.default_mime_type)
2669 # and index!
2670 if self.properties['content'].indexme:
2671 self.db.indexer.add_text((self.classname, newid, 'content'),
2672 content, mime_type)
2674 # store off the content as a file
2675 self.db.storefile(self.classname, newid, None, content)
2677 # fire reactors
2678 self.fireReactors('create', newid, None)
2680 return newid
2682 def get(self, nodeid, propname, default=_marker, cache=1):
2683 """ Trap the content propname and get it from the file
2685 'cache' exists for backwards compatibility, and is not used.
2686 """
2687 poss_msg = 'Possibly a access right configuration problem.'
2688 if propname == 'content':
2689 try:
2690 return self.db.getfile(self.classname, nodeid, None)
2691 except IOError, (strerror):
2692 # BUG: by catching this we donot see an error in the log.
2693 return 'ERROR reading file: %s%s\n%s\n%s'%(
2694 self.classname, nodeid, poss_msg, strerror)
2695 if default is not _marker:
2696 return Class.get(self, nodeid, propname, default)
2697 else:
2698 return Class.get(self, nodeid, propname)
2700 def set(self, itemid, **propvalues):
2701 """ Snarf the "content" propvalue and update it in a file
2702 """
2703 self.fireAuditors('set', itemid, propvalues)
2704 oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid))
2706 # now remove the content property so it's not stored in the db
2707 content = None
2708 if propvalues.has_key('content'):
2709 content = propvalues['content']
2710 del propvalues['content']
2712 # do the database create
2713 propvalues = self.set_inner(itemid, **propvalues)
2715 # do content?
2716 if content:
2717 # store and possibly index
2718 self.db.storefile(self.classname, itemid, None, content)
2719 if self.properties['content'].indexme:
2720 mime_type = self.get(itemid, 'type', self.default_mime_type)
2721 self.db.indexer.add_text((self.classname, itemid, 'content'),
2722 content, mime_type)
2723 propvalues['content'] = content
2725 # fire reactors
2726 self.fireReactors('set', itemid, oldvalues)
2727 return propvalues
2729 def index(self, nodeid):
2730 """ Add (or refresh) the node to search indexes.
2732 Use the content-type property for the content property.
2733 """
2734 # find all the String properties that have indexme
2735 for prop, propclass in self.getprops().items():
2736 if prop == 'content' and propclass.indexme:
2737 mime_type = self.get(nodeid, 'type', self.default_mime_type)
2738 self.db.indexer.add_text((self.classname, nodeid, 'content'),
2739 str(self.get(nodeid, 'content')), mime_type)
2740 elif isinstance(propclass, hyperdb.String) and propclass.indexme:
2741 # index them under (classname, nodeid, property)
2742 try:
2743 value = str(self.get(nodeid, prop))
2744 except IndexError:
2745 # node has been destroyed
2746 continue
2747 self.db.indexer.add_text((self.classname, nodeid, prop), value)
2749 # XXX deviation from spec - was called ItemClass
2750 class IssueClass(Class, roundupdb.IssueClass):
2751 # Overridden methods:
2752 def __init__(self, db, classname, **properties):
2753 """The newly-created class automatically includes the "messages",
2754 "files", "nosy", and "superseder" properties. If the 'properties'
2755 dictionary attempts to specify any of these properties or a
2756 "creation", "creator", "activity" or "actor" property, a ValueError
2757 is raised.
2758 """
2759 if not properties.has_key('title'):
2760 properties['title'] = hyperdb.String(indexme='yes')
2761 if not properties.has_key('messages'):
2762 properties['messages'] = hyperdb.Multilink("msg")
2763 if not properties.has_key('files'):
2764 properties['files'] = hyperdb.Multilink("file")
2765 if not properties.has_key('nosy'):
2766 # note: journalling is turned off as it really just wastes
2767 # space. this behaviour may be overridden in an instance
2768 properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
2769 if not properties.has_key('superseder'):
2770 properties['superseder'] = hyperdb.Multilink(classname)
2771 Class.__init__(self, db, classname, **properties)
2773 # vim: set et sts=4 sw=4 :