![[tokkee]](http://tokkee.org/images/avatar.png){kind=avatar}
index 93d3e8bc463a26455f5406d26aafcfd482071c1d..45078c44b0f3237e3bdcf31e4c4c89fdbcf53f94 100644 (file)
-# $Id: rdbms_common.py,v 1.7 2002-09-20 01:20:32 richard Exp $
+# $Id: rdbms_common.py,v 1.72 2003-12-05 09:47:46 richard Exp $
+''' Relational database (SQL) backend common code.
+
+Basics:
+
+- map roundup classes to relational tables
+- automatically detect schema changes and modify the table schemas
+ appropriately (we store the "database version" of the schema in the
+ database itself as the only row of the "schema" table)
+- multilinks (which represent a many-to-many relationship) are handled through
+ intermediate tables
+- journals are stored adjunct to the per-class tables
+- table names and columns have "_" prepended so the names can't clash with
+ restricted names (like "order")
+- retirement is determined by the __retired__ column being true
+
+Database-specific changes may generally be pushed out to the overridable
+sql_* methods, since everything else should be fairly generic. There's
+probably a bit of work to be done if a database is used that actually
+honors column typing, since the initial databases don't (sqlite stores
+everything as a string.)
+'''
# standard python modules
import sys, os, time, re, errno, weakref, copy
# standard python modules
import sys, os, time, re, errno, weakref, copy
# roundup modules
from roundup import hyperdb, date, password, roundupdb, security
from roundup.hyperdb import String, Password, Date, Interval, Link, \
# roundup modules
from roundup import hyperdb, date, password, roundupdb, security
from roundup.hyperdb import String, Password, Date, Interval, Link, \
- Multilink, DatabaseError, Boolean, Number
+ Multilink, DatabaseError, Boolean, Number, Node
+from roundup.backends import locking
# support
from blobfiles import FileStorage
from roundup.indexer import Indexer
# support
from blobfiles import FileStorage
from roundup.indexer import Indexer
-from sessions import Sessions
+from sessions import Sessions, OneTimeKeys
+from roundup.date import Range
# number of rows to keep in memory
ROW_CACHE_SIZE = 100
# number of rows to keep in memory
ROW_CACHE_SIZE = 100
self.classes = {}
self.indexer = Indexer(self.dir)
self.sessions = Sessions(self.config)
self.classes = {}
self.indexer = Indexer(self.dir)
self.sessions = Sessions(self.config)
+ self.otks = OneTimeKeys(self.config)
self.security = security.Security(self)
# additional transaction support for external files and the like
self.security = security.Security(self)
# additional transaction support for external files and the like
self.cache = {}
self.cache_lru = []
self.cache = {}
self.cache_lru = []
+ # database lock
+ self.lockfile = None
+
# open a connection to the database, creating the "conn" attribute
# open a connection to the database, creating the "conn" attribute
- self.open_connection()
+ self.sql_open_connection()
+
+ def clearCache(self):
+ self.cache = {}
+ self.cache_lru = []
- def open_connection(self):
+ def sql_open_connection(self):
''' Open a connection to the database, creating it if necessary
'''
raise NotImplemented
''' Open a connection to the database, creating it if necessary
'''
raise NotImplemented
- def sql(self, cursor, sql, args=None):
+ def sql(self, sql, args=None):
''' Execute the sql with the optional args.
'''
if __debug__:
print >>hyperdb.DEBUG, (self, sql, args)
if args:
''' Execute the sql with the optional args.
'''
if __debug__:
print >>hyperdb.DEBUG, (self, sql, args)
if args:
- cursor.execute(sql, args)
+ self.cursor.execute(sql, args)
else:
else:
- cursor.execute(sql)
+ self.cursor.execute(sql)
- def sql_fetchone(self, cursor):
+ def sql_fetchone(self):
''' Fetch a single row. If there's nothing to fetch, return None.
'''
''' Fetch a single row. If there's nothing to fetch, return None.
'''
- raise NotImplemented
+ return self.cursor.fetchone()
+
+ def sql_fetchall(self):
+ ''' Fetch all rows. If there's nothing to fetch, return [].
+ '''
+ return self.cursor.fetchall()
def sql_stringquote(self, value):
''' Quote the string so it's safe to put in the 'sql quotes'
'''
return re.sub("'", "''", str(value))
def sql_stringquote(self, value):
''' Quote the string so it's safe to put in the 'sql quotes'
'''
return re.sub("'", "''", str(value))
- def save_dbschema(self, cursor, schema):
+ def save_dbschema(self, schema):
''' Save the schema definition that the database currently implements
'''
''' Save the schema definition that the database currently implements
'''
- raise NotImplemented
+ s = repr(self.database_schema)
+ self.sql('insert into schema values (%s)', (s,))
- def load_dbschema(self, cursor):
+ def load_dbschema(self):
''' Load the schema definition that the database currently implements
'''
''' Load the schema definition that the database currently implements
'''
- raise NotImplemented
+ self.cursor.execute('select schema from schema')
+ return eval(self.cursor.fetchone()[0])
def post_init(self):
''' Called once the schema initialisation has finished.
def post_init(self):
''' Called once the schema initialisation has finished.
self.database_schema[classname] = spec.schema()
save = 1
self.database_schema[classname] = spec.schema()
save = 1
- for classname in self.database_schema.keys():
+ for classname, spec in self.database_schema.items():
if not self.classes.has_key(classname):
if not self.classes.has_key(classname):
- self.drop_class(classname)
+ self.drop_class(classname, spec)
+ del self.database_schema[classname]
+ save = 1
# update the database version of the schema
if save:
# update the database version of the schema
if save:
- cursor = self.conn.cursor()
- self.sql(cursor, 'delete from schema')
- self.save_dbschema(cursor, self.database_schema)
+ self.sql('delete from schema')
+ self.save_dbschema(self.database_schema)
# reindex the db if necessary
if self.indexer.should_reindex():
# reindex the db if necessary
if self.indexer.should_reindex():
# commit
self.conn.commit()
# commit
self.conn.commit()
+ def refresh_database(self):
+ self.post_init()
+
def reindex(self):
for klass in self.classes.values():
for nodeid in klass.list():
def reindex(self):
for klass in self.classes.values():
for nodeid in klass.list():
cols.sort()
return cols, mls
cols.sort()
return cols, mls
- def update_class(self, spec, dbspec):
+ def update_class(self, spec, old_spec, force=0):
''' Determine the differences between the current spec and the
''' Determine the differences between the current spec and the
- database version of the spec, and update where necessary
+ database version of the spec, and update where necessary.
+
+ If 'force' is true, update the database anyway.
'''
'''
- spec_schema = spec.schema()
- if spec_schema == dbspec:
- # no save needed for this one
+ new_has = spec.properties.has_key
+ new_spec = spec.schema()
+ new_spec[1].sort()
+ old_spec[1].sort()
+ if not force and new_spec == old_spec:
+ # no changes
return 0
return 0
+
if __debug__:
print >>hyperdb.DEBUG, 'update_class FIRING'
if __debug__:
print >>hyperdb.DEBUG, 'update_class FIRING'
- # key property changed?
- if dbspec[0] != spec_schema[0]:
- if __debug__:
- print >>hyperdb.DEBUG, 'update_class setting keyprop', `spec[0]`
- # XXX turn on indexing for the key property
-
- # dict 'em up
- spec_propnames,spec_props = [],{}
- for propname,prop in spec_schema[1]:
- spec_propnames.append(propname)
- spec_props[propname] = prop
- dbspec_propnames,dbspec_props = [],{}
- for propname,prop in dbspec[1]:
- dbspec_propnames.append(propname)
- dbspec_props[propname] = prop
-
- # we're going to need one of these
- cursor = self.conn.cursor()
-
- # now compare
- for propname in spec_propnames:
- prop = spec_props[propname]
- if dbspec_props.has_key(propname) and prop==dbspec_props[propname]:
+ # detect multilinks that have been removed, and drop their table
+ old_has = {}
+ for name,prop in old_spec[1]:
+ old_has[name] = 1
+ if new_has(name) or not isinstance(prop, Multilink):
continue
continue
+ # it's a multilink, and it's been removed - drop the old
+ # table. First drop indexes.
+ self.drop_multilink_table_indexes(spec.classname, ml)
+ sql = 'drop table %s_%s'%(spec.classname, prop)
if __debug__:
if __debug__:
- print >>hyperdb.DEBUG, 'update_class ADD', (propname, prop)
+ print >>hyperdb.DEBUG, 'update_class', (self, sql)
+ self.cursor.execute(sql)
+ old_has = old_has.has_key
+
+ # now figure how we populate the new table
+ fetch = ['_activity', '_creation', '_creator']
+ properties = spec.getprops()
+ for propname,x in new_spec[1]:
+ prop = properties[propname]
+ if isinstance(prop, Multilink):
+ if force or not old_has(propname):
+ # we need to create the new table
+ self.create_multilink_table(spec, propname)
+ elif old_has(propname):
+ # we copy this col over from the old table
+ fetch.append('_'+propname)
+
+ # select the data out of the old table
+ fetch.append('id')
+ fetch.append('__retired__')
+ fetchcols = ','.join(fetch)
+ cn = spec.classname
+ sql = 'select %s from _%s'%(fetchcols, cn)
+ if __debug__:
+ print >>hyperdb.DEBUG, 'update_class', (self, sql)
+ self.cursor.execute(sql)
+ olddata = self.cursor.fetchall()
- if not dbspec_props.has_key(propname):
- # add the property
- if isinstance(prop, Multilink):
- # all we have to do here is create a new table, easy!
- self.create_multilink_table(cursor, spec, propname)
- continue
-
- # no ALTER TABLE, so we:
- # 1. pull out the data, including an extra None column
- oldcols, x = self.determine_columns(dbspec[1])
- oldcols.append('id')
- oldcols.append('__retired__')
- cn = spec.classname
- sql = 'select %s,%s from _%s'%(','.join(oldcols), self.arg, cn)
- if __debug__:
- print >>hyperdb.DEBUG, 'update_class', (self, sql, None)
- cursor.execute(sql, (None,))
- olddata = cursor.fetchall()
-
- # 2. drop the old table
- cursor.execute('drop table _%s'%cn)
-
- # 3. create the new table
- cols, mls = self.create_class_table(cursor, spec)
- # ensure the new column is last
- cols.remove('_'+propname)
- assert oldcols == cols, "Column lists don't match!"
- cols.append('_'+propname)
-
- # 4. populate with the data from step one
- s = ','.join([self.arg for x in cols])
- scols = ','.join(cols)
- sql = 'insert into _%s (%s) values (%s)'%(cn, scols, s)
-
- # GAH, nothing had better go wrong from here on in... but
- # we have to commit the drop...
- # XXX this isn't necessary in sqlite :(
- self.conn.commit()
-
- # do the insert
- for row in olddata:
- self.sql(cursor, sql, tuple(row))
+ # TODO: update all the other index dropping code
+ self.drop_class_table_indexes(cn, old_spec[0])
- else:
- # modify the property
- if __debug__:
- print >>hyperdb.DEBUG, 'update_class NOOP'
- pass # NOOP in gadfly
-
- # and the other way - only worry about deletions here
- for propname in dbspec_propnames:
- prop = dbspec_props[propname]
- if spec_props.has_key(propname):
- continue
+ # drop the old table
+ self.cursor.execute('drop table _%s'%cn)
+
+ # create the new table
+ self.create_class_table(spec)
+
+ if olddata:
+ # do the insert
+ args = ','.join([self.arg for x in fetch])
+ sql = 'insert into _%s (%s) values (%s)'%(cn, fetchcols, args)
if __debug__:
if __debug__:
- print >>hyperdb.DEBUG, 'update_class REMOVE', `prop`
+ print >>hyperdb.DEBUG, 'update_class', (self, sql, olddata[0])
+ for entry in olddata:
+ self.cursor.execute(sql, tuple(entry))
- # delete the property
- if isinstance(prop, Multilink):
- sql = 'drop table %s_%s'%(spec.classname, prop)
- if __debug__:
- print >>hyperdb.DEBUG, 'update_class', (self, sql)
- cursor.execute(sql)
- else:
- # no ALTER TABLE, so we:
- # 1. pull out the data, excluding the removed column
- oldcols, x = self.determine_columns(spec.properties.items())
- oldcols.append('id')
- oldcols.append('__retired__')
- # remove the missing column
- oldcols.remove('_'+propname)
- cn = spec.classname
- sql = 'select %s from _%s'%(','.join(oldcols), cn)
- cursor.execute(sql, (None,))
- olddata = sql.fetchall()
-
- # 2. drop the old table
- cursor.execute('drop table _%s'%cn)
-
- # 3. create the new table
- cols, mls = self.create_class_table(self, cursor, spec)
- assert oldcols != cols, "Column lists don't match!"
-
- # 4. populate with the data from step one
- qs = ','.join([self.arg for x in cols])
- sql = 'insert into _%s values (%s)'%(cn, s)
- cursor.execute(sql, olddata)
return 1
return 1
- def create_class_table(self, cursor, spec):
+ def create_class_table(self, spec):
''' create the class table for the given spec
'''
cols, mls = self.determine_columns(spec.properties.items())
''' create the class table for the given spec
'''
cols, mls = self.determine_columns(spec.properties.items())
sql = 'create table _%s (%s)'%(spec.classname, scols)
if __debug__:
print >>hyperdb.DEBUG, 'create_class', (self, sql)
sql = 'create table _%s (%s)'%(spec.classname, scols)
if __debug__:
print >>hyperdb.DEBUG, 'create_class', (self, sql)
- cursor.execute(sql)
+ self.cursor.execute(sql)
+
+ self.create_class_table_indexes(spec)
return cols, mls
return cols, mls
- def create_journal_table(self, cursor, spec):
+ def create_class_table_indexes(self, spec):
+ ''' create the class table for the given spec
+ '''
+ # create id index
+ index_sql1 = 'create index _%s_id_idx on _%s(id)'%(
+ spec.classname, spec.classname)
+ if __debug__:
+ print >>hyperdb.DEBUG, 'create_index', (self, index_sql1)
+ self.cursor.execute(index_sql1)
+
+ # create __retired__ index
+ index_sql2 = 'create index _%s_retired_idx on _%s(__retired__)'%(
+ spec.classname, spec.classname)
+ if __debug__:
+ print >>hyperdb.DEBUG, 'create_index', (self, index_sql2)
+ self.cursor.execute(index_sql2)
+
+ # create index for key property
+ if spec.key:
+ if __debug__:
+ print >>hyperdb.DEBUG, 'update_class setting keyprop %r'% \
+ spec.key
+ index_sql3 = 'create index _%s_%s_idx on _%s(_%s)'%(
+ spec.classname, spec.key,
+ spec.classname, spec.key)
+ if __debug__:
+ print >>hyperdb.DEBUG, 'create_index', (self, index_sql3)
+ self.cursor.execute(index_sql3)
+
+ def drop_class_table_indexes(self, cn, key):
+ # drop the old table indexes first
+ l = ['_%s_id_idx'%cn, '_%s_retired_idx'%cn]
+ if key:
+ # key prop too?
+ l.append('_%s_%s_idx'%(cn, key))
+
+ # TODO: update all the other index dropping code
+ table_name = '_%s'%cn
+ for index_name in l:
+ if not self.sql_index_exists(table_name, index_name):
+ continue
+ index_sql = 'drop index '+index_name
+ if __debug__:
+ print >>hyperdb.DEBUG, 'drop_index', (self, index_sql)
+ self.cursor.execute(index_sql)
+
+ def create_journal_table(self, spec):
''' create the journal table for a class given the spec and
already-determined cols
'''
''' create the journal table for a class given the spec and
already-determined cols
'''
sql = 'create table %s__journal (%s)'%(spec.classname, cols)
if __debug__:
print >>hyperdb.DEBUG, 'create_class', (self, sql)
sql = 'create table %s__journal (%s)'%(spec.classname, cols)
if __debug__:
print >>hyperdb.DEBUG, 'create_class', (self, sql)
- cursor.execute(sql)
+ self.cursor.execute(sql)
+ self.create_journal_table_indexes(spec)
- def create_multilink_table(self, cursor, spec, ml):
+ def create_journal_table_indexes(self, spec):
+ # index on nodeid
+ index_sql = 'create index %s_journ_idx on %s__journal(nodeid)'%(
+ spec.classname, spec.classname)
+ if __debug__:
+ print >>hyperdb.DEBUG, 'create_index', (self, index_sql)
+ self.cursor.execute(index_sql)
+
+ def drop_journal_table_indexes(self, classname):
+ index_name = '%s_journ_idx'%classname
+ if not self.sql_index_exists('%s__journal'%classname, index_name):
+ return
+ index_sql = 'drop index '+index_name
+ if __debug__:
+ print >>hyperdb.DEBUG, 'drop_index', (self, index_sql)
+ self.cursor.execute(index_sql)
+
+ def create_multilink_table(self, spec, ml):
''' Create a multilink table for the "ml" property of the class
given by the spec
'''
''' Create a multilink table for the "ml" property of the class
given by the spec
'''
+ # create the table
sql = 'create table %s_%s (linkid varchar, nodeid varchar)'%(
spec.classname, ml)
if __debug__:
print >>hyperdb.DEBUG, 'create_class', (self, sql)
sql = 'create table %s_%s (linkid varchar, nodeid varchar)'%(
spec.classname, ml)
if __debug__:
print >>hyperdb.DEBUG, 'create_class', (self, sql)
- cursor.execute(sql)
+ self.cursor.execute(sql)
+ self.create_multilink_table_indexes(spec, ml)
+
+ def create_multilink_table_indexes(self, spec, ml):
+ # create index on linkid
+ index_sql = 'create index %s_%s_l_idx on %s_%s(linkid)'%(
+ spec.classname, ml, spec.classname, ml)
+ if __debug__:
+ print >>hyperdb.DEBUG, 'create_index', (self, index_sql)
+ self.cursor.execute(index_sql)
+
+ # create index on nodeid
+ index_sql = 'create index %s_%s_n_idx on %s_%s(nodeid)'%(
+ spec.classname, ml, spec.classname, ml)
+ if __debug__:
+ print >>hyperdb.DEBUG, 'create_index', (self, index_sql)
+ self.cursor.execute(index_sql)
+
+ def drop_multilink_table_indexes(self, classname, ml):
+ l = [
+ '%s_%s_l_idx'%(classname, ml),
+ '%s_%s_n_idx'%(classname, ml)
+ ]
+ table_name = '%s_%s'%(classname, ml)
+ for index_name in l:
+ if not self.sql_index_exists(table_name, index_name):
+ continue
+ index_sql = 'drop index %s'%index_name
+ if __debug__:
+ print >>hyperdb.DEBUG, 'drop_index', (self, index_sql)
+ self.cursor.execute(index_sql)
def create_class(self, spec):
''' Create a database table according to the given spec.
'''
def create_class(self, spec):
''' Create a database table according to the given spec.
'''
- cursor = self.conn.cursor()
- cols, mls = self.create_class_table(cursor, spec)
- self.create_journal_table(cursor, spec)
+ cols, mls = self.create_class_table(spec)
+ self.create_journal_table(spec)
# now create the multilink tables
for ml in mls:
# now create the multilink tables
for ml in mls:
- self.create_multilink_table(cursor, spec, ml)
+ self.create_multilink_table(spec, ml)
# ID counter
sql = 'insert into ids (name, num) values (%s,%s)'%(self.arg, self.arg)
vals = (spec.classname, 1)
if __debug__:
print >>hyperdb.DEBUG, 'create_class', (self, sql, vals)
# ID counter
sql = 'insert into ids (name, num) values (%s,%s)'%(self.arg, self.arg)
vals = (spec.classname, 1)
if __debug__:
print >>hyperdb.DEBUG, 'create_class', (self, sql, vals)
- cursor.execute(sql, vals)
+ self.cursor.execute(sql, vals)
- def drop_class(self, spec):
+ def drop_class(self, cn, spec):
''' Drop the given table from the database.
Drop the journal and multilink tables too.
'''
''' Drop the given table from the database.
Drop the journal and multilink tables too.
'''
+ properties = spec[1]
# figure the multilinks
mls = []
# figure the multilinks
mls = []
- for col, prop in spec.properties.items():
+ for propanme, prop in properties:
if isinstance(prop, Multilink):
if isinstance(prop, Multilink):
- mls.append(col)
- cursor = self.conn.cursor()
+ mls.append(propname)
- sql = 'drop table _%s'%spec.classname
+ # drop class table and indexes
+ self.drop_class_table_indexes(cn, spec[0])
+ sql = 'drop table _%s'%cn
if __debug__:
print >>hyperdb.DEBUG, 'drop_class', (self, sql)
if __debug__:
print >>hyperdb.DEBUG, 'drop_class', (self, sql)
- cursor.execute(sql)
+ self.cursor.execute(sql)
- sql = 'drop table %s__journal'%spec.classname
+ # drop journal table and indexes
+ self.drop_journal_table_indexes(cn)
+ sql = 'drop table %s__journal'%cn
if __debug__:
print >>hyperdb.DEBUG, 'drop_class', (self, sql)
if __debug__:
print >>hyperdb.DEBUG, 'drop_class', (self, sql)
- cursor.execute(sql)
+ self.cursor.execute(sql)
for ml in mls:
for ml in mls:
+ # drop multilink table and indexes
+ self.drop_multilink_table_indexes(cn, ml)
sql = 'drop table %s_%s'%(spec.classname, ml)
if __debug__:
print >>hyperdb.DEBUG, 'drop_class', (self, sql)
sql = 'drop table %s_%s'%(spec.classname, ml)
if __debug__:
print >>hyperdb.DEBUG, 'drop_class', (self, sql)
- cursor.execute(sql)
+ self.cursor.execute(sql)
#
# Classes
#
# Classes
'''
if __debug__:
print >>hyperdb.DEBUG, 'clear', (self,)
'''
if __debug__:
print >>hyperdb.DEBUG, 'clear', (self,)
- cursor = self.conn.cursor()
for cn in self.classes.keys():
sql = 'delete from _%s'%cn
if __debug__:
print >>hyperdb.DEBUG, 'clear', (self, sql)
for cn in self.classes.keys():
sql = 'delete from _%s'%cn
if __debug__:
print >>hyperdb.DEBUG, 'clear', (self, sql)
- cursor.execute(sql)
+ self.cursor.execute(sql)
#
# Node IDs
#
# Node IDs
''' Generate a new id for the given class
'''
# get the next ID
''' Generate a new id for the given class
'''
# get the next ID
- cursor = self.conn.cursor()
sql = 'select num from ids where name=%s'%self.arg
if __debug__:
print >>hyperdb.DEBUG, 'newid', (self, sql, classname)
sql = 'select num from ids where name=%s'%self.arg
if __debug__:
print >>hyperdb.DEBUG, 'newid', (self, sql, classname)
- cursor.execute(sql, (classname, ))
- newid = cursor.fetchone()[0]
+ self.cursor.execute(sql, (classname, ))
+ newid = self.cursor.fetchone()[0]
# update the counter
sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
vals = (int(newid)+1, classname)
if __debug__:
print >>hyperdb.DEBUG, 'newid', (self, sql, vals)
# update the counter
sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
vals = (int(newid)+1, classname)
if __debug__:
print >>hyperdb.DEBUG, 'newid', (self, sql, vals)
- cursor.execute(sql, vals)
+ self.cursor.execute(sql, vals)
# return as string
return str(newid)
# return as string
return str(newid)
def setid(self, classname, setid):
''' Set the id counter: used during import of database
'''
def setid(self, classname, setid):
''' Set the id counter: used during import of database
'''
- cursor = self.conn.cursor()
sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
vals = (setid, classname)
if __debug__:
print >>hyperdb.DEBUG, 'setid', (self, sql, vals)
sql = 'update ids set num=%s where name=%s'%(self.arg, self.arg)
vals = (setid, classname)
if __debug__:
print >>hyperdb.DEBUG, 'setid', (self, sql, vals)
- cursor.execute(sql, vals)
+ self.cursor.execute(sql, vals)
#
# Nodes
#
#
# Nodes
#
-
def addnode(self, classname, nodeid, node):
''' Add the specified node to its class's db.
'''
if __debug__:
print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
def addnode(self, classname, nodeid, node):
''' Add the specified node to its class's db.
'''
if __debug__:
print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
- # gadfly requires values for all non-multilink columns
+
+ # determine the column definitions and multilink tables
cl = self.classes[classname]
cols, mls = self.determine_columns(cl.properties.items())
cl = self.classes[classname]
cols, mls = self.determine_columns(cl.properties.items())
- # add the special props
- node = node.copy()
- node['creation'] = node['activity'] = date.Date()
- node['creator'] = self.journaltag
+ # we'll be supplied these props if we're doing an import
+ if not node.has_key('creator'):
+ # add in the "calculated" properties (dupe so we don't affect
+ # calling code's node assumptions)
+ node = node.copy()
+ node['creation'] = node['activity'] = date.Date()
+ node['creator'] = self.getuid()
# default the non-multilink columns
for col, prop in cl.properties.items():
# default the non-multilink columns
for col, prop in cl.properties.items():
- if not isinstance(col, Multilink):
- if not node.has_key(col):
+ if not node.has_key(col):
+ if isinstance(prop, Multilink):
+ node[col] = []
+ else:
node[col] = None
# clear this node out of the cache if it's in there
node[col] = None
# clear this node out of the cache if it's in there
cols = ','.join(cols) + ',id,__retired__'
# perform the inserts
cols = ','.join(cols) + ',id,__retired__'
# perform the inserts
- cursor = self.conn.cursor()
sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
if __debug__:
print >>hyperdb.DEBUG, 'addnode', (self, sql, vals)
sql = 'insert into _%s (%s) values (%s)'%(classname, cols, s)
if __debug__:
print >>hyperdb.DEBUG, 'addnode', (self, sql, vals)
- cursor.execute(sql, vals)
+ self.cursor.execute(sql, vals)
# insert the multilink rows
for col in mls:
# insert the multilink rows
for col in mls:
for entry in node[col]:
sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
self.arg, self.arg)
for entry in node[col]:
sql = 'insert into %s (linkid, nodeid) values (%s,%s)'%(t,
self.arg, self.arg)
- self.sql(cursor, sql, (entry, nodeid))
+ self.sql(sql, (entry, nodeid))
# make sure we do the commit-time extra stuff for this node
self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
# make sure we do the commit-time extra stuff for this node
self.transactions.append((self.doSaveNode, (classname, nodeid, node)))
cols.append('_'+col)
cols.sort()
cols.append('_'+col)
cols.sort()
- cursor = self.conn.cursor()
-
# if there's any updates to regular columns, do them
if cols:
# make sure the ordering is correct for column name -> column value
# if there's any updates to regular columns, do them
if cols:
# make sure the ordering is correct for column name -> column value
sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
if __debug__:
print >>hyperdb.DEBUG, 'setnode', (self, sql, sqlvals)
sql = 'update _%s set %s where id=%s'%(classname, s, self.arg)
if __debug__:
print >>hyperdb.DEBUG, 'setnode', (self, sql, sqlvals)
- cursor.execute(sql, sqlvals)
+ self.cursor.execute(sql, sqlvals)
# now the fun bit, updating the multilinks ;)
for col, (add, remove) in multilink_changes.items():
# now the fun bit, updating the multilinks ;)
for col, (add, remove) in multilink_changes.items():
sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
self.arg, self.arg)
for addid in add:
sql = 'insert into %s (nodeid, linkid) values (%s,%s)'%(tn,
self.arg, self.arg)
for addid in add:
- self.sql(cursor, sql, (nodeid, addid))
+ self.sql(sql, (nodeid, addid))
if remove:
sql = 'delete from %s where nodeid=%s and linkid=%s'%(tn,
self.arg, self.arg)
for removeid in remove:
if remove:
sql = 'delete from %s where nodeid=%s and linkid=%s'%(tn,
self.arg, self.arg)
for removeid in remove:
- self.sql(cursor, sql, (nodeid, removeid))
+ self.sql(sql, (nodeid, removeid))
# make sure we do the commit-time extra stuff for this node
self.transactions.append((self.doSaveNode, (classname, nodeid, values)))
# make sure we do the commit-time extra stuff for this node
self.transactions.append((self.doSaveNode, (classname, nodeid, values)))
if self.cache.has_key(key):
# push us back to the top of the LRU
self.cache_lru.remove(key)
if self.cache.has_key(key):
# push us back to the top of the LRU
self.cache_lru.remove(key)
- self.cache_lry.insert(0, key)
+ self.cache_lru.insert(0, key)
# return the cached information
return self.cache[key]
# return the cached information
return self.cache[key]
scols = ','.join(cols)
# perform the basic property fetch
scols = ','.join(cols)
# perform the basic property fetch
- cursor = self.conn.cursor()
sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
sql = 'select %s from _%s where id=%s'%(scols, classname, self.arg)
- self.sql(cursor, sql, (nodeid,))
+ self.sql(sql, (nodeid,))
- values = self.sql_fetchone(cursor)
+ values = self.sql_fetchone()
if values is None:
raise IndexError, 'no such %s node %s'%(classname, nodeid)
if values is None:
raise IndexError, 'no such %s node %s'%(classname, nodeid)
# get the link ids
sql = 'select linkid from %s_%s where nodeid=%s'%(classname, col,
self.arg)
# get the link ids
sql = 'select linkid from %s_%s where nodeid=%s'%(classname, col,
self.arg)
- cursor.execute(sql, (nodeid,))
+ self.cursor.execute(sql, (nodeid,))
# extract the first column from the result
# extract the first column from the result
- node[col] = [x[0] for x in cursor.fetchall()]
+ node[col] = [x[0] for x in self.cursor.fetchall()]
# un-dbificate the node data
node = self.unserialise(classname, node)
# un-dbificate the node data
node = self.unserialise(classname, node)
self.cache[key] = node
# update the LRU
self.cache_lru.insert(0, key)
self.cache[key] = node
# update the LRU
self.cache_lru.insert(0, key)
- del self.cache[self.cache_lru.pop()]
+ if len(self.cache_lru) > ROW_CACHE_SIZE:
+ del self.cache[self.cache_lru.pop()]
return node
return node
self.transactions.remove(entry)
# now do the SQL
self.transactions.remove(entry)
# now do the SQL
- cursor = self.conn.cursor()
sql = 'delete from _%s where id=%s'%(classname, self.arg)
sql = 'delete from _%s where id=%s'%(classname, self.arg)
- self.sql(cursor, sql, (nodeid,))
+ self.sql(sql, (nodeid,))
# remove from multilnks
cl = self.getclass(classname)
# remove from multilnks
cl = self.getclass(classname)
for col in mls:
# get the link ids
sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
for col in mls:
# get the link ids
sql = 'delete from %s_%s where nodeid=%s'%(classname, col, self.arg)
- cursor.execute(sql, (nodeid,))
+ self.sql(sql, (nodeid,))
# remove journal entries
sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
# remove journal entries
sql = 'delete from %s__journal where nodeid=%s'%(classname, self.arg)
- self.sql(cursor, sql, (nodeid,))
+ self.sql(sql, (nodeid,))
def serialise(self, classname, node):
'''Copy the node contents, converting non-marshallable data into
def serialise(self, classname, node):
'''Copy the node contents, converting non-marshallable data into
# get the property spec
prop = properties[k]
# get the property spec
prop = properties[k]
- if isinstance(prop, Password):
+ if isinstance(prop, Password) and v is not None:
d[k] = str(v)
elif isinstance(prop, Date) and v is not None:
d[k] = v.serialise()
d[k] = str(v)
elif isinstance(prop, Date) and v is not None:
d[k] = v.serialise()
d[k] = date.Date(v)
elif isinstance(prop, Interval) and v is not None:
d[k] = date.Interval(v)
d[k] = date.Date(v)
elif isinstance(prop, Interval) and v is not None:
d[k] = date.Interval(v)
- elif isinstance(prop, Password):
+ elif isinstance(prop, Password) and v is not None:
p = password.Password()
p.unpack(v)
d[k] = p
p = password.Password()
p.unpack(v)
d[k] = p
+ elif isinstance(prop, Boolean) and v is not None:
+ d[k] = int(v)
+ elif isinstance(prop, Number) and v is not None:
+ # try int first, then assume it's a float
+ try:
+ d[k] = int(v)
+ except ValueError:
+ d[k] = float(v)
else:
d[k] = v
return d
else:
d[k] = v
return d
def hasnode(self, classname, nodeid):
''' Determine if the database has a given node.
'''
def hasnode(self, classname, nodeid):
''' Determine if the database has a given node.
'''
- cursor = self.conn.cursor()
sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
if __debug__:
print >>hyperdb.DEBUG, 'hasnode', (self, sql, nodeid)
sql = 'select count(*) from _%s where id=%s'%(classname, self.arg)
if __debug__:
print >>hyperdb.DEBUG, 'hasnode', (self, sql, nodeid)
- cursor.execute(sql, (nodeid,))
- return int(cursor.fetchone()[0])
+ self.cursor.execute(sql, (nodeid,))
+ return int(self.cursor.fetchone()[0])
def countnodes(self, classname):
''' Count the number of nodes that exist for a particular Class.
'''
def countnodes(self, classname):
''' Count the number of nodes that exist for a particular Class.
'''
- cursor = self.conn.cursor()
sql = 'select count(*) from _%s'%classname
if __debug__:
print >>hyperdb.DEBUG, 'countnodes', (self, sql)
sql = 'select count(*) from _%s'%classname
if __debug__:
print >>hyperdb.DEBUG, 'countnodes', (self, sql)
- cursor.execute(sql)
- return cursor.fetchone()[0]
-
- def getnodeids(self, classname, retired=0):
- ''' Retrieve all the ids of the nodes for a particular Class.
-
- Set retired=None to get all nodes. Otherwise it'll get all the
- retired or non-retired nodes, depending on the flag.
- '''
- cursor = self.conn.cursor()
- # flip the sense of the flag if we don't want all of them
- if retired is not None:
- retired = not retired
- sql = 'select id from _%s where __retired__ <> %s'%(classname, self.arg)
- if __debug__:
- print >>hyperdb.DEBUG, 'getnodeids', (self, sql, retired)
- cursor.execute(sql, (retired,))
- return [x[0] for x in cursor.fetchall()]
+ self.cursor.execute(sql)
+ return self.cursor.fetchone()[0]
def addjournal(self, classname, nodeid, action, params, creator=None,
creation=None):
def addjournal(self, classname, nodeid, action, params, creator=None,
creation=None):
if creator:
journaltag = creator
else:
if creator:
journaltag = creator
else:
- journaltag = self.journaltag
+ journaltag = self.getuid()
if creation:
journaldate = creation.serialise()
else:
if creation:
journaldate = creation.serialise()
else:
print >>hyperdb.DEBUG, 'addjournal', (nodeid, journaldate,
journaltag, action, params)
print >>hyperdb.DEBUG, 'addjournal', (nodeid, journaldate,
journaltag, action, params)
- cursor = self.conn.cursor()
- self.save_journal(cursor, classname, cols, nodeid, journaldate,
+ self.save_journal(classname, cols, nodeid, journaldate,
journaltag, action, params)
journaltag, action, params)
- def save_journal(self, cursor, classname, cols, nodeid, journaldate,
- journaltag, action, params):
- ''' Save the journal entry to the database
- '''
- raise NotImplemented
-
def getjournal(self, classname, nodeid):
''' get the journal for id
'''
def getjournal(self, classname, nodeid):
''' get the journal for id
'''
if not self.hasnode(classname, nodeid):
raise IndexError, '%s has no node %s'%(classname, nodeid)
if not self.hasnode(classname, nodeid):
raise IndexError, '%s has no node %s'%(classname, nodeid)
- cursor = self.conn.cursor()
cols = ','.join('nodeid date tag action params'.split())
cols = ','.join('nodeid date tag action params'.split())
- return self.load_journal(cursor, classname, cols, nodeid)
+ return self.load_journal(classname, cols, nodeid)
+
+ def save_journal(self, classname, cols, nodeid, journaldate,
+ journaltag, action, params):
+ ''' Save the journal entry to the database
+ '''
+ # make the params db-friendly
+ params = repr(params)
+ entry = (nodeid, journaldate, journaltag, action, params)
+
+ # do the insert
+ a = self.arg
+ sql = 'insert into %s__journal (%s) values (%s,%s,%s,%s,%s)'%(classname,
+ cols, a, a, a, a, a)
+ if __debug__:
+ print >>hyperdb.DEBUG, 'addjournal', (self, sql, entry)
+ self.cursor.execute(sql, entry)
- def load_journal(self, cursor, classname, cols, nodeid):
+ def load_journal(self, classname, cols, nodeid):
''' Load the journal from the database
'''
''' Load the journal from the database
'''
- raise NotImplemented
+ # now get the journal entries
+ sql = 'select %s from %s__journal where nodeid=%s'%(cols, classname,
+ self.arg)
+ if __debug__:
+ print >>hyperdb.DEBUG, 'load_journal', (self, sql, nodeid)
+ self.cursor.execute(sql, (nodeid,))
+ res = []
+ for nodeid, date_stamp, user, action, params in self.cursor.fetchall():
+ params = eval(params)
+ res.append((nodeid, date.Date(date_stamp), user, action, params))
+ return res
def pack(self, pack_before):
''' Delete all journal entries except "create" before 'pack_before'.
def pack(self, pack_before):
''' Delete all journal entries except "create" before 'pack_before'.
date_stamp = pack_before.serialise()
# do the delete
date_stamp = pack_before.serialise()
# do the delete
- cursor = self.conn.cursor()
for classname in self.classes.keys():
sql = "delete from %s__journal where date<%s and "\
"action<>'create'"%(classname, self.arg)
if __debug__:
print >>hyperdb.DEBUG, 'pack', (self, sql, date_stamp)
for classname in self.classes.keys():
sql = "delete from %s__journal where date<%s and "\
"action<>'create'"%(classname, self.arg)
if __debug__:
print >>hyperdb.DEBUG, 'pack', (self, sql, date_stamp)
- cursor.execute(sql, (date_stamp,))
+ self.cursor.execute(sql, (date_stamp,))
def sql_commit(self):
''' Actually commit to the database.
def sql_commit(self):
''' Actually commit to the database.
# clear out the transactions
self.transactions = []
# clear out the transactions
self.transactions = []
+ def sql_rollback(self):
+ self.conn.rollback()
+
def rollback(self):
''' Reverse all actions from the current transaction.
def rollback(self):
''' Reverse all actions from the current transaction.
if __debug__:
print >>hyperdb.DEBUG, 'rollback', (self,)
if __debug__:
print >>hyperdb.DEBUG, 'rollback', (self,)
- # roll back
- self.conn.rollback()
+ self.sql_rollback()
# roll back "other" transaction stuff
for method, args in self.transactions:
# roll back "other" transaction stuff
for method, args in self.transactions:
self.rollbackStoreFile(*args)
self.transactions = []
self.rollbackStoreFile(*args)
self.transactions = []
+ # clear the cache
+ self.clearCache()
+
def doSaveNode(self, classname, nodeid, node):
''' dummy that just generates a reindex event
'''
# return the classname, nodeid so we reindex this content
return (classname, nodeid)
def doSaveNode(self, classname, nodeid, node):
''' dummy that just generates a reindex event
'''
# return the classname, nodeid so we reindex this content
return (classname, nodeid)
+ def sql_close(self):
+ self.conn.close()
+
def close(self):
''' Close off the connection.
'''
def close(self):
''' Close off the connection.
'''
- self.conn.close()
+ self.sql_close()
+ if self.lockfile is not None:
+ locking.release_lock(self.lockfile)
+ if self.lockfile is not None:
+ self.lockfile.close()
+ self.lockfile = None
#
# The base Class class
#
# The base Class class
# do the db-related init stuff
db.addclass(self)
# do the db-related init stuff
db.addclass(self)
- self.auditors = {'create': [], 'set': [], 'retire': []}
- self.reactors = {'create': [], 'set': [], 'retire': []}
+ self.auditors = {'create': [], 'set': [], 'retire': [], 'restore': []}
+ self.reactors = {'create': [], 'set': [], 'retire': [], 'restore': []}
def schema(self):
''' A dumpable version of the schema that we can store in the
def schema(self):
''' A dumpable version of the schema that we can store in the
If an id in a link or multilink property does not refer to a valid
node, an IndexError is raised.
'''
If an id in a link or multilink property does not refer to a valid
node, an IndexError is raised.
'''
+ self.fireAuditors('create', None, propvalues)
+ newid = self.create_inner(**propvalues)
+ self.fireReactors('create', newid, None)
+ return newid
+
+ def create_inner(self, **propvalues):
+ ''' Called by create, in-between the audit and react calls.
+ '''
if propvalues.has_key('id'):
raise KeyError, '"id" is reserved'
if propvalues.has_key('id'):
raise KeyError, '"id" is reserved'
if propvalues.has_key('creation') or propvalues.has_key('activity'):
raise KeyError, '"creation" and "activity" are reserved'
if propvalues.has_key('creation') or propvalues.has_key('activity'):
raise KeyError, '"creation" and "activity" are reserved'
- self.fireAuditors('create', None, propvalues)
-
# new node's id
newid = self.db.newid(self.classname)
# new node's id
newid = self.db.newid(self.classname)
(self.classname, newid, key))
elif isinstance(prop, String):
(self.classname, newid, key))
elif isinstance(prop, String):
- if type(value) != type(''):
+ if type(value) != type('') and type(value) != type(u''):
raise TypeError, 'new property "%s" not a string'%key
elif isinstance(prop, Password):
raise TypeError, 'new property "%s" not a string'%key
elif isinstance(prop, Password):
# done
self.db.addnode(self.classname, newid, propvalues)
if self.do_journal:
# done
self.db.addnode(self.classname, newid, propvalues)
if self.do_journal:
- self.db.addjournal(self.classname, newid, 'create', propvalues)
-
- self.fireReactors('create', newid, None)
+ self.db.addjournal(self.classname, newid, 'create', {})
return newid
return newid
elif isinstance(proptype, hyperdb.Password):
value = str(value)
l.append(repr(value))
elif isinstance(proptype, hyperdb.Password):
value = str(value)
l.append(repr(value))
+ l.append(repr(self.is_retired(nodeid)))
return l
def import_list(self, propnames, proplist):
return l
def import_list(self, propnames, proplist):
# make the new node's property map
d = {}
# make the new node's property map
d = {}
+ retire = 0
+ newid = None
for i in range(len(propnames)):
# Use eval to reverse the repr() used to output the CSV
value = eval(proplist[i])
# Figure the property for this column
propname = propnames[i]
for i in range(len(propnames)):
# Use eval to reverse the repr() used to output the CSV
value = eval(proplist[i])
# Figure the property for this column
propname = propnames[i]
- prop = properties[propname]
# "unmarshal" where necessary
if propname == 'id':
newid = value
continue
# "unmarshal" where necessary
if propname == 'id':
newid = value
continue
+ elif propname == 'is retired':
+ # is the item retired?
+ if int(value):
+ retire = 1
+ continue
elif value is None:
elif value is None:
+ d[propname] = None
+ continue
+
+ prop = properties[propname]
+ if value is None:
# don't set Nones
continue
elif isinstance(prop, hyperdb.Date):
# don't set Nones
continue
elif isinstance(prop, hyperdb.Date):
value = pwd
d[propname] = value
value = pwd
d[propname] = value
+ # get a new id if necessary
+ if newid is None:
+ newid = self.db.newid(self.classname)
+
+ # add the node and journal
+ self.db.addnode(self.classname, newid, d)
+
+ # retire?
+ if retire:
+ # use the arg for __retired__ to cope with any odd database type
+ # conversion (hello, sqlite)
+ sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
+ self.db.arg, self.db.arg)
+ if __debug__:
+ print >>hyperdb.DEBUG, 'retire', (self, sql, newid)
+ self.db.cursor.execute(sql, (1, newid))
+
# extract the extraneous journalling gumpf and nuke it
if d.has_key('creator'):
creator = d['creator']
del d['creator']
# extract the extraneous journalling gumpf and nuke it
if d.has_key('creator'):
creator = d['creator']
del d['creator']
+ else:
+ creator = None
if d.has_key('creation'):
creation = d['creation']
del d['creation']
if d.has_key('creation'):
creation = d['creation']
del d['creation']
+ else:
+ creation = None
if d.has_key('activity'):
del d['activity']
if d.has_key('activity'):
del d['activity']
-
- # add the node and journal
- self.db.addnode(self.classname, newid, d)
- self.db.addjournal(self.classname, newid, 'create', d, creator,
+ self.db.addjournal(self.classname, newid, 'create', {}, creator,
creation)
return newid
creation)
return newid
IndexError is raised. 'propname' must be the name of a property
of this class or a KeyError is raised.
IndexError is raised. 'propname' must be the name of a property
of this class or a KeyError is raised.
- 'cache' indicates whether the transaction cache should be queried
- for the node. If the node has been modified and you need to
- determine what its values prior to modification are, you need to
- set cache=0.
+ 'cache' exists for backwards compatibility, and is not used.
'''
if propname == 'id':
return nodeid
'''
if propname == 'id':
return nodeid
if d.has_key('creator'):
return d['creator']
else:
if d.has_key('creator'):
return d['creator']
else:
- return self.db.journaltag
+ return self.db.getuid()
# get the property (raises KeyErorr if invalid)
prop = self.properties[propname]
# get the property (raises KeyErorr if invalid)
prop = self.properties[propname]
return d[propname]
return d[propname]
- def getnode(self, nodeid, cache=1):
- ''' Return a convenience wrapper for the node.
-
- 'nodeid' must be the id of an existing node of this class or an
- IndexError is raised.
-
- 'cache' indicates whether the transaction cache should be queried
- for the node. If the node has been modified and you need to
- determine what its values prior to modification are, you need to
- set cache=0.
- '''
- return Node(self, nodeid, cache=cache)
-
def set(self, nodeid, **propvalues):
'''Modify a property on an existing node of this class.
def set(self, nodeid, **propvalues):
'''Modify a property on an existing node of this class.
# if the value's the same as the existing value, no sense in
# doing anything
# if the value's the same as the existing value, no sense in
# doing anything
- if node.has_key(propname) and value == node[propname]:
+ current = node.get(propname, None)
+ if value == current:
del propvalues[propname]
continue
del propvalues[propname]
continue
+ journalvalues[propname] = current
# do stuff based on the prop type
if isinstance(prop, Link):
# do stuff based on the prop type
if isinstance(prop, Link):
journalvalues[propname] = tuple(l)
elif isinstance(prop, String):
journalvalues[propname] = tuple(l)
elif isinstance(prop, String):
- if value is not None and type(value) != type(''):
+ if value is not None and type(value) != type('') and type(value) != type(u''):
raise TypeError, 'new property "%s" not a string'%propname
elif isinstance(prop, Password):
raise TypeError, 'new property "%s" not a string'%propname
elif isinstance(prop, Password):
self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
if self.do_journal:
self.db.setnode(self.classname, nodeid, propvalues, multilink_changes)
if self.do_journal:
- propvalues.update(journalvalues)
- self.db.addjournal(self.classname, nodeid, 'set', propvalues)
+ self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
self.fireReactors('set', nodeid, oldvalues)
self.fireReactors('set', nodeid, oldvalues)
if self.db.journaltag is None:
raise DatabaseError, 'Database open read-only'
if self.db.journaltag is None:
raise DatabaseError, 'Database open read-only'
- cursor = self.db.conn.cursor()
- sql = 'update _%s set __retired__=1 where id=%s'%(self.classname,
- self.db.arg)
+ self.fireAuditors('retire', nodeid, None)
+
+ # use the arg for __retired__ to cope with any odd database type
+ # conversion (hello, sqlite)
+ sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
+ self.db.arg, self.db.arg)
if __debug__:
print >>hyperdb.DEBUG, 'retire', (self, sql, nodeid)
if __debug__:
print >>hyperdb.DEBUG, 'retire', (self, sql, nodeid)
- cursor.execute(sql, (nodeid,))
+ self.db.cursor.execute(sql, (1, nodeid))
+ if self.do_journal:
+ self.db.addjournal(self.classname, nodeid, 'retired', None)
+
+ self.fireReactors('retire', nodeid, None)
+
+ def restore(self, nodeid):
+ '''Restore a retired node.
+
+ Make node available for all operations like it was before retirement.
+ '''
+ if self.db.journaltag is None:
+ raise DatabaseError, 'Database open read-only'
+ node = self.db.getnode(self.classname, nodeid)
+ # check if key property was overrided
+ key = self.getkey()
+ try:
+ id = self.lookup(node[key])
+ except KeyError:
+ pass
+ else:
+ raise KeyError, "Key property (%s) of retired node clashes with \
+ existing one (%s)" % (key, node[key])
+
+ self.fireAuditors('restore', nodeid, None)
+ # use the arg for __retired__ to cope with any odd database type
+ # conversion (hello, sqlite)
+ sql = 'update _%s set __retired__=%s where id=%s'%(self.classname,
+ self.db.arg, self.db.arg)
+ if __debug__:
+ print >>hyperdb.DEBUG, 'restore', (self, sql, nodeid)
+ self.db.cursor.execute(sql, (0, nodeid))
+ if self.do_journal:
+ self.db.addjournal(self.classname, nodeid, 'restored', None)
+
+ self.fireReactors('restore', nodeid, None)
+
def is_retired(self, nodeid):
'''Return true if the node is rerired
'''
def is_retired(self, nodeid):
'''Return true if the node is rerired
'''
- cursor = self.db.conn.cursor()
sql = 'select __retired__ from _%s where id=%s'%(self.classname,
self.db.arg)
if __debug__:
print >>hyperdb.DEBUG, 'is_retired', (self, sql, nodeid)
sql = 'select __retired__ from _%s where id=%s'%(self.classname,
self.db.arg)
if __debug__:
print >>hyperdb.DEBUG, 'is_retired', (self, sql, nodeid)
- cursor.execute(sql, (nodeid,))
- return int(cursor.fetchone()[0])
+ self.db.cursor.execute(sql, (nodeid,))
+ return int(self.db.sql_fetchone()[0])
def destroy(self, nodeid):
'''Destroy a node.
def destroy(self, nodeid):
'''Destroy a node.
The returned list contains tuples of the form
The returned list contains tuples of the form
- (date, tag, action, params)
+ (nodeid, date, tag, action, params)
'date' is a Timestamp object specifying the time of the change and
'tag' is the journaltag specified when the database was opened.
'date' is a Timestamp object specifying the time of the change and
'tag' is the journaltag specified when the database was opened.
None, or a TypeError is raised. The values of the key property on
all existing nodes must be unique or a ValueError is raised.
'''
None, or a TypeError is raised. The values of the key property on
all existing nodes must be unique or a ValueError is raised.
'''
- # XXX create an index on the key prop column
+ # XXX create an index on the key prop column. We should also
+ # record that we've created this index in the schema somewhere.
prop = self.getprops()[propname]
if not isinstance(prop, String):
raise TypeError, 'key properties must be String'
prop = self.getprops()[propname]
if not isinstance(prop, String):
raise TypeError, 'key properties must be String'
if not self.key:
raise TypeError, 'No key property set for class %s'%self.classname
if not self.key:
raise TypeError, 'No key property set for class %s'%self.classname
- cursor = self.db.conn.cursor()
- sql = 'select id,__retired__ from _%s where _%s=%s'%(self.classname,
- self.key, self.db.arg)
- self.db.sql(cursor, sql, (keyvalue,))
+ # use the arg to handle any odd database type conversion (hello,
+ # sqlite)
+ sql = "select id from _%s where _%s=%s and __retired__ <> %s"%(
+ self.classname, self.key, self.db.arg, self.db.arg)
+ self.db.sql(sql, (keyvalue, 1))
# see if there was a result that's not retired
# see if there was a result that's not retired
- l = cursor.fetchall()
- if not l or int(l[0][1]):
+ row = self.db.sql_fetchone()
+ if not row:
raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
keyvalue, self.classname)
# return the id
raise KeyError, 'No key (%s) value "%s" for "%s"'%(self.key,
keyvalue, self.classname)
# return the id
- return l[0][0]
+ return row[0]
def find(self, **propspec):
'''Get the ids of nodes in this class which link to the given nodes.
def find(self, **propspec):
'''Get the ids of nodes in this class which link to the given nodes.
- 'propspec' consists of keyword args propname={nodeid:1,}
+ 'propspec' consists of keyword args propname=nodeid or
+ propname={nodeid:1, }
'propname' must be the name of a property in this class, or a
'propname' must be the name of a property in this class, or a
- KeyError is raised. That property must be a Link or Multilink
- property, or a TypeError is raised.
+ KeyError is raised. That property must be a Link or
+ Multilink property, or a TypeError is raised.
Any node in this class whose 'propname' property links to any of the
nodeids will be returned. Used by the full text indexing, which knows
Any node in this class whose 'propname' property links to any of the
nodeids will be returned. Used by the full text indexing, which knows
'''
if __debug__:
print >>hyperdb.DEBUG, 'find', (self, propspec)
'''
if __debug__:
print >>hyperdb.DEBUG, 'find', (self, propspec)
+
+ # shortcut
if not propspec:
return []
if not propspec:
return []
- queries = []
+
+ # validate the args
+ props = self.getprops()
+ propspec = propspec.items()
+ for propname, nodeids in propspec:
+ # check the prop is OK
+ prop = props[propname]
+ if not isinstance(prop, Link) and not isinstance(prop, Multilink):
+ raise TypeError, "'%s' not a Link/Multilink property"%propname
+
+ # first, links
+ where = ['__retired__ = %s']
+ allvalues = (0,)
+ a = self.db.arg
+ for prop, values in propspec:
+ if not isinstance(props[prop], hyperdb.Link):
+ continue
+ if type(values) is type({}) and len(values) == 1:
+ values = values.keys()[0]
+ if type(values) is type(''):
+ allvalues += (values,)
+ where.append('_%s = %s'%(prop, a))
+ elif values is None:
+ where.append('_%s is NULL'%prop)
+ else:
+ allvalues += tuple(values.keys())
+ where.append('_%s in (%s)'%(prop, ','.join([a]*len(values))))
tables = []
tables = []
- allvalues = ()
- for prop, values in propspec.items():
- allvalues += tuple(values.keys())
- a = self.db.arg
+ if where:
+ tables.append('select id as nodeid from _%s where %s'%(
+ self.classname, ' and '.join(where)))
+
+ # now multilinks
+ for prop, values in propspec:
+ if not isinstance(props[prop], hyperdb.Multilink):
+ continue
+ if type(values) is type(''):
+ allvalues += (values,)
+ s = a
+ else:
+ allvalues += tuple(values.keys())
+ s = ','.join([a]*len(values))
tables.append('select nodeid from %s_%s where linkid in (%s)'%(
tables.append('select nodeid from %s_%s where linkid in (%s)'%(
- self.classname, prop, ','.join([a for x in values.keys()])))
- sql = '\nintersect\n'.join(tables)
+ self.classname, prop, s))
+ sql = '\nunion\n'.join(tables)
+ self.db.sql(sql, allvalues)
+ l = [x[0] for x in self.db.sql_fetchall()]
if __debug__:
if __debug__:
- print >>hyperdb.DEBUG, 'find', (self, sql, allvalues)
- cursor = self.db.conn.cursor()
- cursor.execute(sql, allvalues)
- try:
- l = [x[0] for x in cursor.fetchall()]
- except gadfly.database.error, message:
- if message == 'no more results':
- l = []
- raise
+ print >>hyperdb.DEBUG, 'find ... ', l
+ return l
+
+ def stringFind(self, **requirements):
+ '''Locate a particular node by matching a set of its String
+ properties in a caseless search.
+
+ If the property is not a String property, a TypeError is raised.
+
+ The return is a list of the id of all nodes that match.
+ '''
+ where = []
+ args = []
+ for propname in requirements.keys():
+ prop = self.properties[propname]
+ if not isinstance(prop, String):
+ raise TypeError, "'%s' not a String property"%propname
+ where.append(propname)
+ args.append(requirements[propname].lower())
+
+ # generate the where clause
+ s = ' and '.join(['lower(_%s)=%s'%(col, self.db.arg) for col in where])
+ sql = 'select id from _%s where %s and __retired__=%s'%(self.classname,
+ s, self.db.arg)
+ args.append(0)
+ self.db.sql(sql, tuple(args))
+ l = [x[0] for x in self.db.sql_fetchall()]
if __debug__:
print >>hyperdb.DEBUG, 'find ... ', l
return l
if __debug__:
print >>hyperdb.DEBUG, 'find ... ', l
return l
def list(self):
''' Return a list of the ids of the active nodes in this class.
'''
def list(self):
''' Return a list of the ids of the active nodes in this class.
'''
- return self.db.getnodeids(self.classname, retired=0)
+ return self.getnodeids(retired=0)
- def filter(self, search_matches, filterspec, sort, group):
+ def getnodeids(self, retired=None):
+ ''' Retrieve all the ids of the nodes for a particular Class.
+
+ Set retired=None to get all nodes. Otherwise it'll get all the
+ retired or non-retired nodes, depending on the flag.
+ '''
+ # flip the sense of the 'retired' flag if we don't want all of them
+ if retired is not None:
+ if retired:
+ args = (0, )
+ else:
+ args = (1, )
+ sql = 'select id from _%s where __retired__ <> %s'%(self.classname,
+ self.db.arg)
+ else:
+ args = ()
+ sql = 'select id from _%s'%self.classname
+ if __debug__:
+ print >>hyperdb.DEBUG, 'getnodeids', (self, sql, retired)
+ self.db.cursor.execute(sql, args)
+ ids = [x[0] for x in self.db.cursor.fetchall()]
+ return ids
+
+ def filter(self, search_matches, filterspec, sort=(None,None),
+ group=(None,None)):
''' Return a list of the ids of the active nodes in this class that
match the 'filter' spec, sorted by the group spec and then the
sort spec
''' Return a list of the ids of the active nodes in this class that
match the 'filter' spec, sorted by the group spec and then the
sort spec
property value to match is a list, any one of the values in the
list may match for that property to match.
'''
property value to match is a list, any one of the values in the
list may match for that property to match.
'''
+ # just don't bother if the full-text search matched diddly
+ if search_matches == {}:
+ return []
+
cn = self.classname
cn = self.classname
+ timezone = self.db.getUserTimezone()
+
# figure the WHERE clause from the filterspec
props = self.getprops()
frum = ['_'+cn]
# figure the WHERE clause from the filterspec
props = self.getprops()
frum = ['_'+cn]
# now do other where clause stuff
if isinstance(propclass, Multilink):
tn = '%s_%s'%(cn, k)
# now do other where clause stuff
if isinstance(propclass, Multilink):
tn = '%s_%s'%(cn, k)
- frum.append(tn)
- if isinstance(v, type([])):
+ if v in ('-1', ['-1']):
+ # only match rows that have count(linkid)=0 in the
+ # corresponding multilink table)
+ where.append('id not in (select nodeid from %s)'%tn)
+ elif isinstance(v, type([])):
+ frum.append(tn)
s = ','.join([a for x in v])
where.append('id=%s.nodeid and %s.linkid in (%s)'%(tn,tn,s))
args = args + v
else:
s = ','.join([a for x in v])
where.append('id=%s.nodeid and %s.linkid in (%s)'%(tn,tn,s))
args = args + v
else:
- where.append('id=%s.nodeid and %s.linkid = %s'%(tn, tn, a))
+ frum.append(tn)
+ where.append('id=%s.nodeid and %s.linkid=%s'%(tn, tn, a))
+ args.append(v)
+ elif k == 'id':
+ if isinstance(v, type([])):
+ s = ','.join([a for x in v])
+ where.append('%s in (%s)'%(k, s))
+ args = args + v
+ else:
+ where.append('%s=%s'%(k, a))
args.append(v)
elif isinstance(propclass, String):
if not isinstance(v, type([])):
args.append(v)
elif isinstance(propclass, String):
if not isinstance(v, type([])):
elif isinstance(propclass, Link):
if isinstance(v, type([])):
if '-1' in v:
elif isinstance(propclass, Link):
if isinstance(v, type([])):
if '-1' in v:
+ v = v[:]
v.remove('-1')
xtra = ' or _%s is NULL'%k
else:
xtra = ''
v.remove('-1')
xtra = ' or _%s is NULL'%k
else:
xtra = ''
- s = ','.join([a for x in v])
- where.append('(_%s in (%s)%s)'%(k, s, xtra))
- args = args + v
+ if v:
+ s = ','.join([a for x in v])
+ where.append('(_%s in (%s)%s)'%(k, s, xtra))
+ args = args + v
+ else:
+ where.append('_%s is NULL'%k)
else:
if v == '-1':
v = None
else:
if v == '-1':
v = None
else:
where.append('_%s=%s'%(k, a))
args.append(v)
else:
where.append('_%s=%s'%(k, a))
args.append(v)
+ elif isinstance(propclass, Date):
+ if isinstance(v, type([])):
+ s = ','.join([a for x in v])
+ where.append('_%s in (%s)'%(k, s))
+ args = args + [date.Date(x).serialise() for x in v]
+ else:
+ try:
+ # Try to filter on range of dates
+ date_rng = Range(v, date.Date, offset=timezone)
+ if (date_rng.from_value):
+ where.append('_%s >= %s'%(k, a))
+ args.append(date_rng.from_value.serialise())
+ if (date_rng.to_value):
+ where.append('_%s <= %s'%(k, a))
+ args.append(date_rng.to_value.serialise())
+ except ValueError:
+ # If range creation fails - ignore that search parameter
+ pass
+ elif isinstance(propclass, Interval):
+ if isinstance(v, type([])):
+ s = ','.join([a for x in v])
+ where.append('_%s in (%s)'%(k, s))
+ args = args + [date.Interval(x).serialise() for x in v]
+ else:
+ try:
+ # Try to filter on range of intervals
+ date_rng = Range(v, date.Interval)
+ if (date_rng.from_value):
+ where.append('_%s >= %s'%(k, a))
+ args.append(date_rng.from_value.serialise())
+ if (date_rng.to_value):
+ where.append('_%s <= %s'%(k, a))
+ args.append(date_rng.to_value.serialise())
+ except ValueError:
+ # If range creation fails - ignore that search parameter
+ pass
+ #where.append('_%s=%s'%(k, a))
+ #args.append(date.Interval(v).serialise())
else:
if isinstance(v, type([])):
s = ','.join([a for x in v])
else:
if isinstance(v, type([])):
s = ','.join([a for x in v])
where.append('_%s=%s'%(k, a))
args.append(v)
where.append('_%s=%s'%(k, a))
args.append(v)
+ # don't match retired nodes
+ where.append('__retired__ <> 1')
+
# add results of full text search
if search_matches is not None:
v = search_matches.keys()
# add results of full text search
if search_matches is not None:
v = search_matches.keys()
args = tuple(args)
if __debug__:
print >>hyperdb.DEBUG, 'filter', (self, sql, args)
args = tuple(args)
if __debug__:
print >>hyperdb.DEBUG, 'filter', (self, sql, args)
- cursor = self.db.conn.cursor()
- cursor.execute(sql, args)
- l = cursor.fetchall()
+ if args:
+ self.db.cursor.execute(sql, args)
+ else:
+ # psycopg doesn't like empty args
+ self.db.cursor.execute(sql)
+ l = self.db.sql_fetchall()
# return the IDs (the first column)
# return the IDs (the first column)
- # XXX The filter(None, l) bit is sqlite-specific... if there's _NO_
- # XXX matches to a fetch, it returns NULL instead of nothing!?!
- return filter(None, [row[0] for row in l])
+ return [row[0] for row in l]
def count(self):
'''Get the number of nodes in this class.
def count(self):
'''Get the number of nodes in this class.
d['id'] = String()
d['creation'] = hyperdb.Date()
d['activity'] = hyperdb.Date()
d['id'] = String()
d['creation'] = hyperdb.Date()
d['activity'] = hyperdb.Date()
- d['creator'] = hyperdb.Link("user")
+ d['creator'] = hyperdb.Link('user')
return d
def addprop(self, **properties):
return d
def addprop(self, **properties):
for react in self.reactors[action]:
react(self.db, self, nodeid, oldvalues)
for react in self.reactors[action]:
react(self.db, self, nodeid, oldvalues)
-class FileClass(Class):
+class FileClass(Class, hyperdb.FileClass):
'''This class defines a large chunk of data. To support this, it has a
mandatory String property "content" which is typically saved off
externally to the hyperdb.
'''This class defines a large chunk of data. To support this, it has a
mandatory String property "content" which is typically saved off
externally to the hyperdb.
def create(self, **propvalues):
''' snaffle the file propvalue and store in a file
'''
def create(self, **propvalues):
''' snaffle the file propvalue and store in a file
'''
+ # we need to fire the auditors now, or the content property won't
+ # be in propvalues for the auditors to play with
+ self.fireAuditors('create', None, propvalues)
+
+ # now remove the content property so it's not stored in the db
content = propvalues['content']
del propvalues['content']
content = propvalues['content']
del propvalues['content']
- newid = Class.create(self, **propvalues)
+
+ # do the database create
+ newid = Class.create_inner(self, **propvalues)
+
+ # fire reactors
+ self.fireReactors('create', newid, None)
+
+ # store off the content as a file
self.db.storefile(self.classname, newid, None, content)
return newid
self.db.storefile(self.classname, newid, None, content)
return newid
_marker = []
def get(self, nodeid, propname, default=_marker, cache=1):
_marker = []
def get(self, nodeid, propname, default=_marker, cache=1):
- ''' trap the content propname and get it from the file
- '''
+ ''' Trap the content propname and get it from the file
+ 'cache' exists for backwards compatibility, and is not used.
+ '''
poss_msg = 'Possibly a access right configuration problem.'
if propname == 'content':
try:
poss_msg = 'Possibly a access right configuration problem.'
if propname == 'content':
try:
return 'ERROR reading file: %s%s\n%s\n%s'%(
self.classname, nodeid, poss_msg, strerror)
if default is not self._marker:
return 'ERROR reading file: %s%s\n%s\n%s'%(
self.classname, nodeid, poss_msg, strerror)
if default is not self._marker:
- return Class.get(self, nodeid, propname, default, cache=cache)
+ return Class.get(self, nodeid, propname, default)
else:
else:
- return Class.get(self, nodeid, propname, cache=cache)
+ return Class.get(self, nodeid, propname)
def getprops(self, protected=1):
''' In addition to the actual properties on the node, these methods
def getprops(self, protected=1):
''' In addition to the actual properties on the node, these methods