From a3cd6cd6fc4db25f4d2f7d671af777ac7fb04e21 Mon Sep 17 00:00:00 2001 From: grubert Date: Mon, 25 Feb 2002 14:34:31 +0000 Subject: [PATCH] . use blobfiles in back_anydbm which is used in back_bsddb. change test_db as dirlist does not work for subdirectories. ATTENTION: blobfiles now creates subdirectories for files. git-svn-id: http://svn.roundup-tracker.org/svnroot/roundup/trunk@658 57a73879-2fb5-44c3-a270-3262357dd7e2 --- CHANGES.txt | 3 +++ MIGRATION.txt | 10 +++++++++ README.txt | 2 -- roundup/backends/back_anydbm.py | 39 ++++++--------------------------- test/test_db.py | 34 +++++++++++++++++----------- 5 files changed, 41 insertions(+), 47 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 0f1e2dc..5e637b2 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -3,6 +3,9 @@ are given with the most recent entry first. 2002-02-?? - 0.4.1 Feature: + . use blobfiles in back_anydbm which is used in back_bsddb. + change test_db as dirlist does not work for subdirectories. + ATTENTION: blobfiles now creates subdirectories for files. . add module blobfiles in backends with file access functions. . roundup db catch only IOError in getfile. . roundup db catches retrieving not existing files. diff --git a/MIGRATION.txt b/MIGRATION.txt index 5de7618..396ade1 100644 --- a/MIGRATION.txt +++ b/MIGRATION.txt @@ -9,6 +9,16 @@ This file contains information for users upgrading from: 0.3.x -> 0.4.x 0.2.x -> 0.3.x +From CVS +======== + +Files storage +------------- + +Messages and files from newly created issues will be put into subdierectories +in thousands e.g. msg123 will be put into files/msg/0/msg123, file2003 +will go into files/file/2/file2003. Previous messages are still found, but +could be put into this structure. Migrating from 0.4.0 to 0.4.1 ============================= diff --git a/README.txt b/README.txt index 4dc1b06..57745d3 100644 --- a/README.txt +++ b/README.txt @@ -51,8 +51,6 @@ in general: . more back-ends hyperdb: . more efficient reverse lookups -roundupdb: - . split the file storage into multiple dirs? roundup-server: . check the source file timestamps before reloading cgi_client diff --git a/roundup/backends/back_anydbm.py b/roundup/backends/back_anydbm.py index 5f30b1a..7c95841 100644 --- a/roundup/backends/back_anydbm.py +++ b/roundup/backends/back_anydbm.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -#$Id: back_anydbm.py,v 1.28 2002-02-16 09:14:17 richard Exp $ +#$Id: back_anydbm.py,v 1.29 2002-02-25 14:34:31 grubert Exp $ ''' This module defines a backend that saves the hyperdatabase in a database chosen by anydbm. It is guaranteed to always be available in python @@ -25,11 +25,12 @@ serious bugs, and is not available) import whichdb, anydbm, os, marshal from roundup import hyperdb, date, password +from blobfiles import FileStorage # # Now the database # -class Database(hyperdb.Database): +class Database(FileStorage, hyperdb.Database): """A database for storing records containing flexible data types. Transaction stuff TODO: @@ -250,36 +251,7 @@ class Database(hyperdb.Database): # # Files - special node properties - # - def filename(self, classname, nodeid, property=None): - '''Determine what the filename for the given node and optionally property is. - ''' - # TODO: split into multiple files directories - if property: - return os.path.join(self.dir, 'files', '%s%s.%s'%(classname, - nodeid, property)) - else: - # roundupdb.FileClass never specified the property name, so don't include it - return os.path.join(self.dir, 'files', '%s%s'%(classname, - nodeid)) - - def storefile(self, classname, nodeid, property, content): - '''Store the content of the file in the database. The property may be None, in - which case the filename does not indicate which property is being saved. - ''' - name = self.filename(classname, nodeid, property) - open(name + '.tmp', 'wb').write(content) - self.transactions.append((self._doStoreFile, (name, ))) - - def getfile(self, classname, nodeid, property): - '''Store the content of the file in the database. - ''' - filename = self.filename(classname, nodeid, property) - try: - return open(filename, 'rb').read() - except: - return open(filename+'.tmp', 'rb').read() - + # inherited from FileStorage # # Journal @@ -453,6 +425,9 @@ class Database(hyperdb.Database): # #$Log: not supported by cvs2svn $ +#Revision 1.28 2002/02/16 09:14:17 richard +# . #514854 ] History: "User" is always ticket creator +# #Revision 1.27 2002/01/22 07:21:13 richard #. fixed back_bsddb so it passed the journal tests # diff --git a/test/test_db.py b/test/test_db.py index b96f166..dc08709 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: test_db.py,v 1.18 2002-01-22 07:21:13 richard Exp $ +# $Id: test_db.py,v 1.19 2002-02-25 14:34:31 grubert Exp $ import unittest, os, shutil @@ -99,12 +99,9 @@ class anydbmDBTestCase(MyTestCase): self.db.status.history('2') def testTransactions(self): + # remember the number of items we started num_issues = len(self.db.issue.list()) - files_dir = os.path.join('_test_dir', 'files') - if os.path.exists(files_dir): - num_files = len(os.listdir(files_dir)) - else: - num_files = 0 + num_files = self.db.numfiles() self.db.issue.create(title="don't commit me!", status='1') self.assertNotEqual(num_issues, len(self.db.issue.list())) self.db.rollback() @@ -117,15 +114,18 @@ class anydbmDBTestCase(MyTestCase): self.assertNotEqual(num_issues, len(self.db.issue.list())) self.db.file.create(name="test", type="text/plain", content="hi") self.db.rollback() - self.assertEqual(num_files, len(os.listdir(files_dir))) - self.db.file.create(name="test", type="text/plain", content="hi") - self.db.commit() - self.assertNotEqual(num_files, len(os.listdir(files_dir))) - num_files2 = len(os.listdir(files_dir)) + self.assertEqual(num_files, self.db.numfiles()) + for i in range(10): + self.db.file.create(name="test", type="text/plain", + content="hi %d"%(i)) + self.db.commit() + num_files2 = self.db.numfiles() + self.assertNotEqual(num_files, num_files2) self.db.file.create(name="test", type="text/plain", content="hi") self.db.rollback() - self.assertNotEqual(num_files, len(os.listdir(files_dir))) - self.assertEqual(num_files2, len(os.listdir(files_dir))) + self.assertNotEqual(num_files, self.db.numfiles()) + self.assertEqual(num_files2, self.db.numfiles()) + def testExceptions(self): @@ -346,6 +346,14 @@ def suite(): # # $Log: not supported by cvs2svn $ +# Revision 1.18 2002/01/22 07:21:13 richard +# . fixed back_bsddb so it passed the journal tests +# +# ... it didn't seem happy using the back_anydbm _open method, which is odd. +# Yet another occurrance of whichdb not being able to recognise older bsddb +# databases. Yadda yadda. Made the HYPERDBDEBUG stuff more sane in the +# process. +# # Revision 1.17 2002/01/22 05:06:09 rochecompaan # We need to keep the last 'set' entry in the journal to preserve # information on 'activity' for nodes. -- 2.30.2