add in-memory hyperdb implementation to speed up testing

git-svn-id: http://svn.roundup-tracker.org/svnroot/roundup/roundup/trunk@4446 57a73879-2fb5-44c3-a270-3262357dd7e2

diff --git a/roundup/backends/back_anydbm.py b/roundup/backends/back_anydbm.py
index 94f7047902880a7917b096c53c787820f6fc6b0c..d0eb1a9545150f64d20e59ca4c30206995c543f6 100644 (file)
--- a/roundup/backends/back_anydbm.py
+++ b/roundup/backends/back_anydbm.py
@@ -950,7 +950,7 @@ class Class(hyperdb.Class):
                 raise ValueError, 'Journalling is disabled for this class'
             journal = self.db.getjournal(self.classname, nodeid)
             if journal:
-                return self.db.getjournal(self.classname, nodeid)[0][1]
+                return journal[0][1]
             else:
                 # on the strange chance that there's no journal
                 return date.Date()

diff --git a/roundup/configuration.py b/roundup/configuration.py
index f15adbd46301a909204da4be4794fbf2d718aef9..3ad193d7dba9dd7fc48374e0120544dbbe842058 100644 (file)
--- a/roundup/configuration.py
+++ b/roundup/configuration.py
@@ -1249,6 +1249,14 @@ class CoreConfig(Config):
         if home_dir is None:
             self.init_logging()
 
+    def copy(self):
+        new = CoreConfig()
+        new.sections = list(self.sections)
+        new.section_descriptions = dict(self.section_descriptions)
+        new.section_options = dict(self.section_options)
+        new.options = dict(self.options)
+        return new
+
     def _get_unset_options(self):
         need_set = Config._get_unset_options(self)
         # remove MAIL_PASSWORD if MAIL_USER is empty

diff --git a/test/memorydb.py b/test/memorydb.py
new file mode 100644 (file)
index 0000000..2590121
--- /dev/null
+++ b/test/memorydb.py
@@ -0,0 +1,366 @@
+'''Implement an in-memory hyperdb for testing purposes.
+'''
+
+import shutil
+
+from roundup import hyperdb
+from roundup import roundupdb
+from roundup import security
+from roundup import password
+from roundup import configuration
+from roundup.backends import back_anydbm
+from roundup.backends import indexer_dbm
+from roundup.backends import indexer_common
+from roundup.hyperdb import *
+
+def new_config():
+    config = configuration.CoreConfig()
+    config.DATABASE = "db"
+    #config.logging = MockNull()
+    # these TRACKER_WEB and MAIL_DOMAIN values are used in mailgw tests
+    config.MAIL_DOMAIN = "your.tracker.email.domain.example"
+    config.TRACKER_WEB = "http://tracker.example/cgi-bin/roundup.cgi/bugs/"
+    return config
+
+def create(journaltag, create=True):
+    db = Database(new_config(), journaltag)
+
+    # load standard schema
+    schema = os.path.join(os.path.dirname(__file__),
+        '../share/roundup/templates/classic/schema.py')
+    vars = dict(globals())
+    vars['db'] = db
+    execfile(schema, vars)
+    initial_data = os.path.join(os.path.dirname(__file__),
+        '../share/roundup/templates/classic/initial_data.py')
+    vars = dict(db=db, admin_email='admin@test.com',
+        adminpw=password.Password('sekrit'))
+    execfile(initial_data, vars)
+
+    # load standard detectors
+    dirname = os.path.join(os.path.dirname(__file__),
+        '../share/roundup/templates/classic/detectors')
+    for fn in os.listdir(dirname):
+        if not fn.endswith('.py'): continue
+        vars = {}
+        execfile(os.path.join(dirname, fn), vars)
+        vars['init'](db)
+
+    '''
+    status = Class(db, "status", name=String())
+    status.setkey("name")
+    priority = Class(db, "priority", name=String(), order=String())
+    priority.setkey("name")
+    keyword = Class(db, "keyword", name=String(), order=String())
+    keyword.setkey("name")
+    user = Class(db, "user", username=String(), password=Password(),
+        assignable=Boolean(), age=Number(), roles=String(), address=String(),
+        supervisor=Link('user'),realname=String(),alternate_addresses=String())
+    user.setkey("username")
+    file = FileClass(db, "file", name=String(), type=String(),
+        comment=String(indexme="yes"), fooz=Password())
+    file_nidx = FileClass(db, "file_nidx", content=String(indexme='no'))
+    issue = IssueClass(db, "issue", title=String(indexme="yes"),
+        status=Link("status"), nosy=Multilink("user"), deadline=Date(),
+        foo=Interval(), files=Multilink("file"), assignedto=Link('user'),
+        priority=Link('priority'), spam=Multilink('msg'),
+        feedback=Link('msg'))
+    stuff = Class(db, "stuff", stuff=String())
+    session = Class(db, 'session', title=String())
+    msg = FileClass(db, "msg", date=Date(),
+                           author=Link("user", do_journal='no'),
+                           files=Multilink('file'), inreplyto=String(),
+                           messageid=String(), summary=String(),
+                           content=String(),
+                           recipients=Multilink("user", do_journal='no')
+                           )
+    '''
+    if create:
+        db.user.create(username="fred", roles='User',
+            password=password.Password('sekrit'), address='fred@example.com')
+
+    db.security.addPermissionToRole('User', 'Email Access')
+    '''
+    db.security.addPermission(name='Register', klass='user')
+    db.security.addPermissionToRole('User', 'Web Access')
+    db.security.addPermissionToRole('Anonymous', 'Email Access')
+    db.security.addPermissionToRole('Anonymous', 'Register', 'user')
+    for cl in 'issue', 'file', 'msg', 'keyword':
+        db.security.addPermissionToRole('User', 'View', cl)
+        db.security.addPermissionToRole('User', 'Edit', cl)
+        db.security.addPermissionToRole('User', 'Create', cl)
+    for cl in 'priority', 'status':
+        db.security.addPermissionToRole('User', 'View', cl)
+    '''
+    return db
+
+class cldb(dict):
+    def close(self):
+        pass
+
+class BasicDatabase(dict):
+    ''' Provide a nice encapsulation of an anydbm store.
+
+        Keys are id strings, values are automatically marshalled data.
+    '''
+    def __getitem__(self, key):
+        if key not in self:
+            d = self[key] = {}
+            return d
+        return super(BasicDatabase, self).__getitem__(key)
+    def exists(self, infoid):
+        return infoid in self
+    def get(self, infoid, value, default=None):
+        return self[infoid].get(value, default)
+    def getall(self, infoid):
+        return self[infoid]
+    def set(self, infoid, **newvalues):
+        self[infoid].update(newvalues)
+    def list(self):
+        return self.keys()
+    def destroy(self, infoid):
+        del self[infoid]
+    def commit(self):
+        pass
+    def close(self):
+        pass
+    def updateTimestamp(self, sessid):
+        pass
+    def clean(self):
+        pass
+
+class Sessions(BasicDatabase):
+    name = 'sessions'
+
+class OneTimeKeys(BasicDatabase):
+    name = 'otks'
+
+class Indexer(indexer_dbm.Indexer):
+    def __init__(self, db):
+        indexer_common.Indexer.__init__(self, db)
+        self.reindex = 0
+        self.quiet = 9
+        self.changed = 0
+
+    def load_index(self, reload=0, wordlist=None):
+        # Unless reload is indicated, do not load twice
+        if self.index_loaded() and not reload:
+            return 0
+        self.words = {}
+        self.files = {'_TOP':(0,None)}
+        self.fileids = {}
+        self.changed = 0
+
+    def save_index(self):
+        pass
+
+class Database(hyperdb.Database, roundupdb.Database):
+    """A database for storing records containing flexible data types.
+
+    Transaction stuff TODO:
+
+    - check the timestamp of the class file and nuke the cache if it's
+      modified. Do some sort of conflict checking on the dirty stuff.
+    - perhaps detect write collisions (related to above)?
+    """
+    def __init__(self, config, journaltag=None):
+        self.config, self.journaltag = config, journaltag
+        self.classes = {}
+        self.items = {}
+        self.ids = {}
+        self.journals = {}
+        self.files = {}
+        self.security = security.Security(self)
+        self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0,
+            'filtering': 0}
+        self.sessions = Sessions()
+        self.otks = OneTimeKeys()
+        self.indexer = Indexer(self)
+
+
+    def filename(self, classname, nodeid, property=None, create=0):
+        shutil.copyfile(__file__, __file__+'.dummy')
+        return __file__+'.dummy'
+
+    def post_init(self):
+        pass
+
+    def refresh_database(self):
+        pass
+
+    def getSessionManager(self):
+        return self.sessions
+
+    def getOTKManager(self):
+        return self.otks
+
+    def reindex(self, classname=None, show_progress=False):
+        pass
+
+    def __repr__(self):
+        return '<memorydb instance at %x>'%id(self)
+
+    def storefile(self, classname, nodeid, property, content):
+        self.files[classname, nodeid, property] = content
+
+    def getfile(self, classname, nodeid, property):
+        return self.files[classname, nodeid, property]
+
+    def numfiles(self):
+        return len(self.files)
+
+    #
+    # Classes
+    #
+    def __getattr__(self, classname):
+        """A convenient way of calling self.getclass(classname)."""
+        if self.classes.has_key(classname):
+            return self.classes[classname]
+        raise AttributeError, classname
+
+    def addclass(self, cl):
+        cn = cl.classname
+        if self.classes.has_key(cn):
+            raise ValueError, cn
+        self.classes[cn] = cl
+        self.items[cn] = cldb()
+        self.ids[cn] = 0
+
+        # add default Edit and View permissions
+        self.security.addPermission(name="Create", klass=cn,
+            description="User is allowed to create "+cn)
+        self.security.addPermission(name="Edit", klass=cn,
+            description="User is allowed to edit "+cn)
+        self.security.addPermission(name="View", klass=cn,
+            description="User is allowed to access "+cn)
+
+    def getclasses(self):
+        """Return a list of the names of all existing classes."""
+        l = self.classes.keys()
+        l.sort()
+        return l
+
+    def getclass(self, classname):
+        """Get the Class object representing a particular class.
+
+        If 'classname' is not a valid class name, a KeyError is raised.
+        """
+        try:
+            return self.classes[classname]
+        except KeyError:
+            raise KeyError, 'There is no class called "%s"'%classname
+
+    #
+    # Class DBs
+    #
+    def clear(self):
+        self.items = {}
+
+    def getclassdb(self, classname):
+        """ grab a connection to the class db that will be used for
+            multiple actions
+        """
+        return self.items[classname]
+
+    #
+    # Node IDs
+    #
+    def newid(self, classname):
+        self.ids[classname] += 1
+        return str(self.ids[classname])
+
+    #
+    # Nodes
+    #
+    def addnode(self, classname, nodeid, node):
+        self.getclassdb(classname)[nodeid] = node
+
+    def setnode(self, classname, nodeid, node):
+        self.getclassdb(classname)[nodeid] = node
+
+    def getnode(self, classname, nodeid, cldb=None):
+        if cldb is not None:
+            return cldb[nodeid]
+        return self.getclassdb(classname)[nodeid]
+
+    def destroynode(self, classname, nodeid):
+        del self.getclassdb(classname)[nodeid]
+
+    def hasnode(self, classname, nodeid):
+        return nodeid in self.getclassdb(classname)
+
+    def countnodes(self, classname, db=None):
+        return len(self.getclassdb(classname))
+
+    #
+    # Journal
+    #
+    def addjournal(self, classname, nodeid, action, params, creator=None,
+            creation=None):
+        if creator is None:
+            creator = self.getuid()
+        if creation is None:
+            creation = date.Date()
+        self.journals.setdefault(classname, {}).setdefault(nodeid,
+            []).append((nodeid, creation, creator, action, params))
+
+    def setjournal(self, classname, nodeid, journal):
+        self.journals.setdefault(classname, {})[nodeid] = journal
+
+    def getjournal(self, classname, nodeid):
+        return self.journals.get(classname, {}).get(nodeid, [])
+
+    def pack(self, pack_before):
+        TODO
+
+    #
+    # Basic transaction support
+    #
+    def commit(self, fail_ok=False):
+        pass
+
+    def rollback(self):
+        TODO
+
+    def close(self):
+        pass
+
+class Class(back_anydbm.Class):
+    def getnodeids(self, db=None, retired=None):
+        return self.db.getclassdb(self.classname).keys()
+
+class FileClass(back_anydbm.Class):
+    def __init__(self, db, classname, **properties):
+        if not properties.has_key('content'):
+            properties['content'] = hyperdb.String(indexme='yes')
+        if not properties.has_key('type'):
+            properties['type'] = hyperdb.String()
+        back_anydbm.Class.__init__(self, db, classname, **properties)
+
+    def getnodeids(self, db=None, retired=None):
+        return self.db.getclassdb(self.classname).keys()
+
+# deviation from spec - was called ItemClass
+class IssueClass(Class, roundupdb.IssueClass):
+    # Overridden methods:
+    def __init__(self, db, classname, **properties):
+        """The newly-created class automatically includes the "messages",
+        "files", "nosy", and "superseder" properties.  If the 'properties'
+        dictionary attempts to specify any of these properties or a
+        "creation" or "activity" property, a ValueError is raised.
+        """
+        if not properties.has_key('title'):
+            properties['title'] = hyperdb.String(indexme='yes')
+        if not properties.has_key('messages'):
+            properties['messages'] = hyperdb.Multilink("msg")
+        if not properties.has_key('files'):
+            properties['files'] = hyperdb.Multilink("file")
+        if not properties.has_key('nosy'):
+            # note: journalling is turned off as it really just wastes
+            # space. this behaviour may be overridden in an instance
+            properties['nosy'] = hyperdb.Multilink("user", do_journal="no")
+        if not properties.has_key('superseder'):
+            properties['superseder'] = hyperdb.Multilink(classname)
+        Class.__init__(self, db, classname, **properties)
+
+# vim: set et sts=4 sw=4 :

diff --git a/test/test_mailgw.py b/test/test_mailgw.py
index a3c95af95100dc1ec0638df8c894710e7295892b..e15e2928585e441f7fc3c6eb08000e3d6e056c6b 100644 (file)
--- a/test/test_mailgw.py
+++ b/test/test_mailgw.py
@@ -26,7 +26,8 @@ from roundup.mailgw import MailGW, Unauthorized, uidFromAddress, \
 from roundup import init, instance, password, rfc2822, __version__
 from roundup.anypy.sets_ import set
 
-import db_test_base
+#import db_test_base
+import memorydb
 
 class Message(rfc822.Message):
     """String-based Message class with equivalence test."""
@@ -37,6 +38,10 @@ class Message(rfc822.Message):
         return (self.dict == other.dict and
                 self.fp.read() == other.fp.read())
 
+class Tracker(object):
+    def open(self, journaltag):
+        return self.db
+
 class DiffHelper:
     def compareMessages(self, new, old):
         """Compare messages for semantic equivalence."""
@@ -115,12 +120,14 @@ class MailgwTestCase(unittest.TestCase, DiffHelper):
     schema = 'classic'
     def setUp(self):
         MailgwTestCase.count = MailgwTestCase.count + 1
-        self.dirname = '_test_mailgw_%s'%self.count
-        # set up and open a tracker
-        self.instance = db_test_base.setupTracker(self.dirname)
 
-        # and open the database
-        self.db = self.instance.open('admin')
+        # and open the database / "instance"
+        self.db = memorydb.create('admin')
+        self.instance = Tracker()
+        self.instance.db = self.db
+        self.instance.config = self.db.config
+        self.instance.MailGW = MailGW
+
         self.chef_id = self.db.user.create(username='Chef',
             address='chef@bork.bork.bork', realname='Bork, Chef', roles='User')
         self.richard_id = self.db.user.create(username='richard',
@@ -135,22 +142,11 @@ class MailgwTestCase(unittest.TestCase, DiffHelper):
         if os.path.exists(SENDMAILDEBUG):
             os.remove(SENDMAILDEBUG)
         self.db.close()
-        try:
-            shutil.rmtree(self.dirname)
-        except OSError, error:
-            if error.errno not in (errno.ENOENT, errno.ESRCH): raise
 
     def _handle_mail(self, message):
-        # handler will open a new db handle. On single-threaded
-        # databases we'll have to close our current connection
-        self.db.commit()
-        self.db.close()
         handler = self.instance.MailGW(self.instance)
         handler.trapExceptions = 0
-        ret = handler.main(StringIO(message))
-        # handler had its own database, open new connection
-        self.db = self.instance.open('admin')
-        return ret
+        return handler.main(StringIO(message))
 
     def _get_mail(self):
         f = open(SENDMAILDEBUG)
@@ -173,6 +169,22 @@ Subject: [issue] Testing...
         assert not os.path.exists(SENDMAILDEBUG)
         self.assertEqual(self.db.issue.get(nodeid, 'title'), 'Testing...')
 
+    def testMessageWithFromInIt(self):
+        nodeid = self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Cc: richard@test.test
+Reply-To: chef@bork.bork.bork
+Message-Id: <dummy_test_message_id>
+Subject: [issue] Testing...
+
+From here to there!
+''')
+        assert not os.path.exists(SENDMAILDEBUG)
+        msgid = self.db.issue.get(nodeid, 'msg')[0]
+        self.assertEqual(self.db.issue.get(msgid, 'content'), 'From here to there!')
+
     def doNewIssue(self):
         nodeid = self._handle_mail('''Content-Type: text/plain;
   charset="iso-8859-1"
@@ -1020,7 +1032,6 @@ Subject: [issue1] Testing... [nosy=-richard]
         assert not os.path.exists(SENDMAILDEBUG)
 
     def testNewUserAuthor(self):
-
         l = self.db.user.list()
         l.sort()
         message = '''Content-Type: text/plain;
@@ -1032,12 +1043,9 @@ Subject: [issue] Testing...
 
 This is a test submission of a new issue.
 '''
-        def hook (db, **kw):
-            ''' set up callback for db open '''
-            db.security.role['anonymous'].permissions=[]
-            anonid = db.user.lookup('anonymous')
-            db.user.set(anonid, roles='Anonymous')
-        self.instance.schema_hook = hook
+        self.db.security.role['anonymous'].permissions=[]
+        anonid = self.db.user.lookup('anonymous')
+        self.db.user.set(anonid, roles='Anonymous')
         try:
             self._handle_mail(message)
         except Unauthorized, value:
@@ -1046,23 +1054,17 @@ You are not a registered user.
 
 Unknown address: fubar@bork.bork.bork
 """)
-
             assert not body_diff, body_diff
-
         else:
             raise AssertionError, "Unathorized not raised when handling mail"
 
-
-        def hook (db, **kw):
-            ''' set up callback for db open '''
-            # Add Web Access role to anonymous, and try again to make sure
-            # we get a "please register at:" message this time.
-            p = [
-                db.security.getPermission('Register', 'user'),
-                db.security.getPermission('Web Access', None),
-            ]
-            db.security.role['anonymous'].permissions=p
-        self.instance.schema_hook = hook
+        # Add Web Access role to anonymous, and try again to make sure
+        # we get a "please register at:" message this time.
+        p = [
+            self.db.security.getPermission('Register', 'user'),
+            self.db.security.getPermission('Web Access', None),
+        ]
+        self.db.security.role['anonymous'].permissions=p
         try:
             self._handle_mail(message)
         except Unauthorized, value:
@@ -1075,9 +1077,7 @@ http://tracker.example/cgi-bin/roundup.cgi/bugs/user?template=register
 
 Unknown address: fubar@bork.bork.bork
 """)
-
             assert not body_diff, body_diff
-
         else:
             raise AssertionError, "Unathorized not raised when handling mail"
 
@@ -1086,15 +1086,12 @@ Unknown address: fubar@bork.bork.bork
         m.sort()
         self.assertEqual(l, m)
 
-        def hook (db, **kw):
-            ''' set up callback for db open '''
-            # now with the permission
-            p = [
-                db.security.getPermission('Register', 'user'),
-                db.security.getPermission('Email Access', None),
-            ]
-            db.security.role['anonymous'].permissions=p
-        self.instance.schema_hook = hook
+        # now with the permission
+        p = [
+            self.db.security.getPermission('Register', 'user'),
+            self.db.security.getPermission('Email Access', None),
+        ]
+        self.db.security.role['anonymous'].permissions=p
         self._handle_mail(message)
         m = self.db.user.list()
         m.sort()
@@ -1112,16 +1109,13 @@ Subject: [issue] Testing...
 
 This is a test submission of a new issue.
 '''
-        def hook (db, **kw):
-            ''' set up callback for db open '''
-            p = [
-                db.security.getPermission('Register', 'user'),
-                db.security.getPermission('Email Access', None),
-                db.security.getPermission('Create', 'issue'),
-                db.security.getPermission('Create', 'msg'),
-            ]
-            db.security.role['anonymous'].permissions = p
-        self.instance.schema_hook = hook
+        p = [
+            self.db.security.getPermission('Register', 'user'),
+            self.db.security.getPermission('Email Access', None),
+            self.db.security.getPermission('Create', 'issue'),
+            self.db.security.getPermission('Create', 'msg'),
+        ]
+        self.db.security.role['anonymous'].permissions = p
         self._handle_mail(message)
         m = set(self.db.user.list())
         new = list(m - l)[0]