From: richard Date: Tue, 18 Mar 2003 00:50:24 +0000 (+0000) Subject: bye bye gadfly - you served your purpose well (sf bug 701127) X-Git-Url: https://git.tokkee.org/?a=commitdiff_plain;h=227ce7c256c90bb7b11fb9ae41836e49dbb67a9d;p=roundup.git bye bye gadfly - you served your purpose well (sf bug 701127) git-svn-id: http://svn.roundup-tracker.org/svnroot/roundup/trunk@1602 57a73879-2fb5-44c3-a270-3262357dd7e2 --- diff --git a/CHANGES.txt b/CHANGES.txt index 6e2f14e..9c53e02 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -2,6 +2,11 @@ This file contains the changes to the Roundup system over time. The entries are given with the most recent entry first. 2003-??-?? 0.6.0 +Removed: +- having served its purpose as a template for other relational database + implementations, the gadfly backend has now been removed from the Roundup + distribution. + Feature: - support setting of properties on message and file through web and email interface (thanks John Rouillard) diff --git a/doc/installation.txt b/doc/installation.txt index b6eba99..7607ba0 100644 --- a/doc/installation.txt +++ b/doc/installation.txt @@ -2,7 +2,7 @@ Installing Roundup ================== -:Version: $Revision: 1.39 $ +:Version: $Revision: 1.40 $ .. contents:: @@ -222,12 +222,6 @@ There's several to choose from, each with benefits and limitations: **metakit** This backend is implemented over the metakit_ storage system, using Mk4Py as the interface. It scales much better than the dbm backends. -**gadfly** - This is a proof-of-concept relational database backend, not really intended - for actual production use, although it can be. It uses the Gadfly RDBMS - to store data. It is unable to perform string searches due to gadfly not - having a LIKE operation. It should scale well, assuming a client/server - setup is used. It's much slower than even the dbm backends. Note: you may set your tracker up with the anydbm backend (which is guaranteed to be available) and switch to one of the other backends at any time using the diff --git a/roundup/backends/__init__.py b/roundup/backends/__init__.py index 7dfe3e8..48affd1 100644 --- a/roundup/backends/__init__.py +++ b/roundup/backends/__init__.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: __init__.py,v 1.21 2003-01-12 23:53:19 richard Exp $ +# $Id: __init__.py,v 1.22 2003-03-18 00:50:24 richard Exp $ ''' Container for the hyperdb storage backend implementations. @@ -42,20 +42,6 @@ else: anydbm = back_anydbm __all__.append('anydbm') -try: - import gadfly - import gadfly.client -except ImportError, message: - if str(message) == 'No module named client': - # don't keep the old gadfly around - del gadfly - elif str(message) != 'No module named gadfly': - raise -else: - import back_gadfly - gadfly = back_gadfly - __all__.append('gadfly') - try: import MySQLdb except ImportError, message: diff --git a/roundup/backends/back_gadfly.py b/roundup/backends/back_gadfly.py deleted file mode 100644 index 363f385..0000000 --- a/roundup/backends/back_gadfly.py +++ /dev/null @@ -1,370 +0,0 @@ -# $Id: back_gadfly.py,v 1.34 2003-03-14 02:51:25 richard Exp $ -''' Gadlfy relational database hypderb backend. - -About Gadfly -============ - -Gadfly is a collection of python modules that provides relational -database functionality entirely implemented in Python. It supports a -subset of the intergalactic standard RDBMS Structured Query Language -SQL. - - -Additional Instance Requirements -================================ - -The instance configuration must specify where the database is. It does this -with GADFLY_DATABASE, which is used as the arguments to the gadfly.gadfly() -method: - -Using an on-disk database directly (not a good idea): - GADFLY_DATABASE = (database name, directory) - -Using a network database (much better idea): - GADFLY_DATABASE = (policy, password, address, port) - -Because multiple accesses directly to a gadfly database aren't handled, but -multiple network accesses are, it's strongly advised that the latter setup be -used. - -''' - -# standard python modules -import sys, os, time, re, errno, weakref, copy - -# roundup modules -from roundup import hyperdb, date, password, roundupdb, security -from roundup.hyperdb import String, Password, Date, Interval, Link, \ - Multilink, DatabaseError, Boolean, Number -from roundup.backends import locking - -# basic RDBMS backen implementation -from roundup.backends import rdbms_common - -# the all-important gadfly :) -import gadfly -import gadfly.client -import gadfly.database - -class Database(rdbms_common.Database): - # char to use for positional arguments - arg = '?' - - def open_connection(self): - db = getattr(self.config, 'GADFLY_DATABASE', ('database', self.dir)) - - # lock it - lockfilenm = os.path.join(db[1], db[0]) + '.lck' - self.lockfile = locking.acquire_lock(lockfilenm) - self.lockfile.write(str(os.getpid())) - self.lockfile.flush() - - if len(db) == 2: - # ensure files are group readable and writable - os.umask(0002) - try: - self.conn = gadfly.gadfly(*db) - except IOError, error: - if error.errno != errno.ENOENT: - raise - self.database_schema = {} - self.conn = gadfly.gadfly() - self.conn.startup(*db) - self.cursor = self.conn.cursor() - self.cursor.execute('create table schema (schema varchar)') - self.cursor.execute('create table ids (name varchar, num integer)') - else: - self.cursor = self.conn.cursor() - self.cursor.execute('select schema from schema') - self.database_schema = self.cursor.fetchone()[0] - else: - self.conn = gadfly.client.gfclient(*db) - self.database_schema = self.load_dbschema() - - def __repr__(self): - return ''%id(self) - - def sql_fetchone(self): - ''' Fetch a single row. If there's nothing to fetch, return None. - ''' - try: - return self.cursor.fetchone() - except gadfly.database.error, message: - if message == 'no more results': - return None - raise - - def sql_fetchall(self): - ''' Fetch a single row. If there's nothing to fetch, return []. - ''' - try: - return self.cursor.fetchall() - except gadfly.database.error, message: - if message == 'no more results': - return [] - raise - - def save_dbschema(self, schema): - ''' Save the schema definition that the database currently implements - ''' - self.sql('insert into schema values (?)', (self.database_schema,)) - - def load_dbschema(self): - ''' Load the schema definition that the database currently implements - ''' - self.cursor.execute('select schema from schema') - return self.cursor.fetchone()[0] - - def save_journal(self, classname, cols, nodeid, journaldate, - journaltag, action, params): - ''' Save the journal entry to the database - ''' - # nothing special to do - entry = (nodeid, journaldate, journaltag, action, params) - - # do the insert - a = self.arg - sql = 'insert into %s__journal (%s) values (?,?,?,?,?)'%(classname, - cols) - if __debug__: - print >>hyperdb.DEBUG, 'addjournal', (self, sql, entry) - self.cursor.execute(sql, entry) - - def load_journal(self, classname, cols, nodeid): - ''' Load the journal from the database - ''' - # now get the journal entries - sql = 'select %s from %s__journal where nodeid=%s'%(cols, classname, - self.arg) - if __debug__: - print >>hyperdb.DEBUG, 'getjournal', (self, sql, nodeid) - self.cursor.execute(sql, (nodeid,)) - res = [] - for nodeid, date_stamp, user, action, params in self.cursor.fetchall(): - res.append((nodeid, date.Date(date_stamp), user, action, params)) - return res - - def update_class(self, spec, old_spec): - ''' Determine the differences between the current spec and the - database version of the spec, and update where necessary - - GADFLY requires a commit after the table drop! - ''' - new_spec = spec - new_has = new_spec.properties.has_key - - new_spec = new_spec.schema() - new_spec[1].sort() - old_spec[1].sort() - if new_spec == old_spec: - # no changes - return 0 - - if __debug__: - print >>hyperdb.DEBUG, 'update_class FIRING' - - # key property changed? - if old_spec[0] != new_spec[0]: - if __debug__: - print >>hyperdb.DEBUG, 'update_class setting keyprop', `spec[0]` - # XXX turn on indexing for the key property - - # detect multilinks that have been removed, and drop their table - old_has = {} - for name,prop in old_spec[1]: - old_has[name] = 1 - if not new_has(name) and isinstance(prop, Multilink): - # it's a multilink, and it's been removed - drop the old - # table - sql = 'drop table %s_%s'%(spec.classname, prop) - if __debug__: - print >>hyperdb.DEBUG, 'update_class', (self, sql) - self.cursor.execute(sql) - continue - old_has = old_has.has_key - - # now figure how we populate the new table - fetch = ['_activity', '_creation', '_creator'] - properties = spec.getprops() - for propname,x in new_spec[1]: - prop = properties[propname] - if isinstance(prop, Multilink): - if not old_has(propname): - # we need to create the new table - self.create_multilink_table(spec, propname) - elif old_has(propname): - # we copy this col over from the old table - fetch.append('_'+propname) - - # select the data out of the old table - fetch.append('id') - fetch.append('__retired__') - fetchcols = ','.join(fetch) - cn = spec.classname - sql = 'select %s from _%s'%(fetchcols, cn) - if __debug__: - print >>hyperdb.DEBUG, 'update_class', (self, sql) - self.cursor.execute(sql) - olddata = self.cursor.fetchall() - - # drop the old table - self.cursor.execute('drop table _%s'%cn) - - # GADFLY requires a commit here, or the table spec screws up - self.conn.commit() - - # create the new table - cols, mls = self.create_class_table(spec) - - # figure the new columns - extra = 0 - for col in cols: - if col not in fetch: - fetch.append(col) - extra += 1 - - if olddata: - # do the insert - fetchcols = ','.join(fetch) - args = ','.join([self.arg for x in fetch]) - sql = 'insert into _%s (%s) values (%s)'%(cn, fetchcols, args) - if __debug__: - print >>hyperdb.DEBUG, 'update_class', (self, sql, olddata[0]) - for entry in olddata: - self.cursor.execute(sql, tuple(entry) + (None,)*extra) - - return 1 - -class GadflyClass: - def filter(self, search_matches, filterspec, sort=(None,None), - group=(None,None)): - ''' Gadfly doesn't have a LIKE predicate :( - ''' - cn = self.classname - - # figure the WHERE clause from the filterspec - props = self.getprops() - frum = ['_'+cn] - where = [] - args = [] - a = self.db.arg - for k, v in filterspec.items(): - propclass = props[k] - if isinstance(propclass, Multilink): - tn = '%s_%s'%(cn, k) - frum.append(tn) - if isinstance(v, type([])): - s = ','.join([a for x in v]) - where.append('id=%s.nodeid and %s.linkid in (%s)'%(tn,tn,s)) - args = args + v - else: - where.append('id=%s.nodeid and %s.linkid = %s'%(tn, tn, a)) - args.append(v) - elif isinstance(propclass, Date): - if isinstance(v, type([])): - s = ','.join([a for x in v]) - where.append('_%s in (%s)'%(k, s)) - args = args + [date.Date(x).serialise() for x in v] - else: - where.append('_%s=%s'%(k, a)) - args.append(date.Date(v).serialise()) - elif isinstance(propclass, Interval): - if isinstance(v, type([])): - s = ','.join([a for x in v]) - where.append('_%s in (%s)'%(k, s)) - args = args + [date.Interval(x).serialise() for x in v] - else: - where.append('_%s=%s'%(k, a)) - args.append(date.Interval(v).serialise()) - elif k == 'id': - if isinstance(v, type([])): - s = ','.join([a for x in v]) - where.append('%s in (%s)'%(k, s)) - args = args + v - else: - where.append('%s=%s'%(k, a)) - args.append(v) - else: - if isinstance(v, type([])): - s = ','.join([a for x in v]) - where.append('_%s in (%s)'%(k, s)) - args = args + v - else: - where.append('_%s=%s'%(k, a)) - args.append(v) - - # add results of full text search - if search_matches is not None: - v = search_matches.keys() - s = ','.join([a for x in v]) - where.append('id in (%s)'%s) - args = args + v - - # "grouping" is just the first-order sorting in the SQL fetch - # can modify it...) - orderby = [] - ordercols = [] - if group[0] is not None and group[1] is not None: - if group[0] != '-': - orderby.append('_'+group[1]) - ordercols.append('_'+group[1]) - else: - orderby.append('_'+group[1]+' desc') - ordercols.append('_'+group[1]) - - # now add in the sorting - group = '' - if sort[0] is not None and sort[1] is not None: - direction, colname = sort - if direction != '-': - if colname == 'id': - orderby.append(colname) - else: - orderby.append('_'+colname) - ordercols.append('_'+colname) - else: - if colname == 'id': - orderby.append(colname+' desc') - ordercols.append(colname) - else: - orderby.append('_'+colname+' desc') - ordercols.append('_'+colname) - - # construct the SQL - frum = ','.join(frum) - if where: - where = ' where ' + (' and '.join(where)) - else: - where = '' - cols = ['id'] - if orderby: - cols = cols + ordercols - order = ' order by %s'%(','.join(orderby)) - else: - order = '' - cols = ','.join(cols) - sql = 'select %s from %s %s%s%s'%(cols, frum, where, group, order) - args = tuple(args) - if __debug__: - print >>hyperdb.DEBUG, 'filter', (self, sql, args) - self.db.cursor.execute(sql, args) - l = self.db.cursor.fetchall() - - # return the IDs - return [row[0] for row in l] - - def find(self, **propspec): - ''' Overload to filter out duplicates in the result - ''' - d = {} - for k in rdbms_common.Class.find(self, **propspec): - d[k] = 1 - return d.keys() - -class Class(GadflyClass, rdbms_common.Class): - pass -class IssueClass(GadflyClass, rdbms_common.IssueClass): - pass -class FileClass(GadflyClass, rdbms_common.FileClass): - pass - diff --git a/roundup/backends/rdbms_common.py b/roundup/backends/rdbms_common.py index 27b5caf..8670cf3 100644 --- a/roundup/backends/rdbms_common.py +++ b/roundup/backends/rdbms_common.py @@ -1,4 +1,4 @@ -# $Id: rdbms_common.py,v 1.45 2003-03-17 22:03:08 kedder Exp $ +# $Id: rdbms_common.py,v 1.46 2003-03-18 00:50:24 richard Exp $ ''' Relational database (SQL) backend common code. Basics: @@ -18,7 +18,7 @@ Database-specific changes may generally be pushed out to the overridable sql_* methods, since everything else should be fairly generic. There's probably a bit of work to be done if a database is used that actually honors column typing, since the initial databases don't (sqlite stores -everything as a string, and gadfly stores anything that's marsallable). +everything as a string.) ''' # standard python modules @@ -433,13 +433,13 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): # # Nodes # - def addnode(self, classname, nodeid, node): ''' Add the specified node to its class's db. ''' if __debug__: print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node) - # gadfly requires values for all non-multilink columns + + # determine the column definitions and multilink tables cl = self.classes[classname] cols, mls = self.determine_columns(cl.properties.items()) diff --git a/test/test_db.py b/test/test_db.py index bc6720e..cb7e115 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: test_db.py,v 1.78 2003-03-17 22:03:08 kedder Exp $ +# $Id: test_db.py,v 1.79 2003-03-18 00:50:24 richard Exp $ import unittest, os, shutil, time @@ -763,41 +763,6 @@ class bsddb3ReadOnlyDBTestCase(anydbmReadOnlyDBTestCase): setupSchema(self.db, 0, bsddb3) -class gadflyDBTestCase(anydbmDBTestCase): - ''' Gadfly doesn't support multiple connections to the one local - database - ''' - def setUp(self): - from roundup.backends import gadfly - # remove previous test, ignore errors - if os.path.exists(config.DATABASE): - shutil.rmtree(config.DATABASE) - config.GADFLY_DATABASE = ('test', config.DATABASE) - os.makedirs(config.DATABASE + '/files') - self.db = gadfly.Database(config, 'admin') - setupSchema(self.db, 1, gadfly) - - def testFilteringString(self): - ae, filt = self.filteringSetup() - ae(filt(None, {'title': 'issue one'}, ('+','id'), (None,None)), ['1']) - # XXX gadfly can't do substring LIKE searches - #ae(filt(None, {'title': 'issue'}, ('+','id'), (None,None)), - # ['1','2','3']) - -class gadflyReadOnlyDBTestCase(anydbmReadOnlyDBTestCase): - def setUp(self): - from roundup.backends import gadfly - # remove previous test, ignore errors - if os.path.exists(config.DATABASE): - shutil.rmtree(config.DATABASE) - config.GADFLY_DATABASE = ('test', config.DATABASE) - os.makedirs(config.DATABASE + '/files') - db = gadfly.Database(config, 'admin') - setupSchema(db, 1, gadfly) - db.close() - self.db = gadfly.Database(config) - setupSchema(self.db, 0, gadfly) - class mysqlDBTestCase(anydbmDBTestCase): def setUp(self): from roundup.backends import mysql @@ -956,11 +921,6 @@ def suite(): l.append(unittest.makeSuite(mysqlReadOnlyDBTestCase, 'test')) #return unittest.TestSuite(l) - if hasattr(backends, 'gadfly'): - p.append('gadfly') - l.append(unittest.makeSuite(gadflyDBTestCase, 'test')) - l.append(unittest.makeSuite(gadflyReadOnlyDBTestCase, 'test')) - if hasattr(backends, 'sqlite'): p.append('sqlite') l.append(unittest.makeSuite(sqliteDBTestCase, 'test')) diff --git a/test/test_init.py b/test/test_init.py index 6f1f318..b58d0ce 100644 --- a/test/test_init.py +++ b/test/test_init.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: test_init.py,v 1.21 2003-02-28 03:33:25 richard Exp $ +# $Id: test_init.py,v 1.22 2003-03-18 00:50:24 richard Exp $ import unittest, os, shutil, errno, imp, sys @@ -78,8 +78,8 @@ class bsddb3ClassicTestCase(ClassicTestCase): class metakitClassicTestCase(ClassicTestCase): backend = 'metakit' -class gadflyClassicTestCase(ClassicTestCase): - backend = 'gadfly' +class mysqlClassicTestCase(ClassicTestCase): + backend = 'mysql' class sqliteClassicTestCase(ClassicTestCase): backend = 'sqlite' @@ -96,8 +96,8 @@ def suite(): l.append(unittest.makeSuite(bsddb3ClassicTestCase, 'test')) if hasattr(backends, 'metakit'): l.append(unittest.makeSuite(metakitClassicTestCase, 'test')) - if hasattr(backends, 'gadfly'): - l.append(unittest.makeSuite(gadflyClassicTestCase, 'test')) + if hasattr(backends, 'mysql'): + l.append(unittest.makeSuite(mysqlClassicTestCase, 'test')) if hasattr(backends, 'sqlite'): l.append(unittest.makeSuite(sqliteClassicTestCase, 'test'))