From: richard Date: Tue, 2 Feb 2010 04:44:18 +0000 (+0000) Subject: add in-memory hyperdb implementation to speed up testing X-Git-Url: https://git.tokkee.org/?a=commitdiff_plain;h=f4d1da34f0662e6ebdd9e097ef1457f0d1db833c;p=roundup.git add in-memory hyperdb implementation to speed up testing git-svn-id: http://svn.roundup-tracker.org/svnroot/roundup/roundup/trunk@4446 57a73879-2fb5-44c3-a270-3262357dd7e2 --- diff --git a/roundup/backends/back_anydbm.py b/roundup/backends/back_anydbm.py index 94f7047..d0eb1a9 100644 --- a/roundup/backends/back_anydbm.py +++ b/roundup/backends/back_anydbm.py @@ -950,7 +950,7 @@ class Class(hyperdb.Class): raise ValueError, 'Journalling is disabled for this class' journal = self.db.getjournal(self.classname, nodeid) if journal: - return self.db.getjournal(self.classname, nodeid)[0][1] + return journal[0][1] else: # on the strange chance that there's no journal return date.Date() diff --git a/roundup/configuration.py b/roundup/configuration.py index f15adbd..3ad193d 100644 --- a/roundup/configuration.py +++ b/roundup/configuration.py @@ -1249,6 +1249,14 @@ class CoreConfig(Config): if home_dir is None: self.init_logging() + def copy(self): + new = CoreConfig() + new.sections = list(self.sections) + new.section_descriptions = dict(self.section_descriptions) + new.section_options = dict(self.section_options) + new.options = dict(self.options) + return new + def _get_unset_options(self): need_set = Config._get_unset_options(self) # remove MAIL_PASSWORD if MAIL_USER is empty diff --git a/test/memorydb.py b/test/memorydb.py new file mode 100644 index 0000000..2590121 --- /dev/null +++ b/test/memorydb.py @@ -0,0 +1,366 @@ +'''Implement an in-memory hyperdb for testing purposes. +''' + +import shutil + +from roundup import hyperdb +from roundup import roundupdb +from roundup import security +from roundup import password +from roundup import configuration +from roundup.backends import back_anydbm +from roundup.backends import indexer_dbm +from roundup.backends import indexer_common +from roundup.hyperdb import * + +def new_config(): + config = configuration.CoreConfig() + config.DATABASE = "db" + #config.logging = MockNull() + # these TRACKER_WEB and MAIL_DOMAIN values are used in mailgw tests + config.MAIL_DOMAIN = "your.tracker.email.domain.example" + config.TRACKER_WEB = "http://tracker.example/cgi-bin/roundup.cgi/bugs/" + return config + +def create(journaltag, create=True): + db = Database(new_config(), journaltag) + + # load standard schema + schema = os.path.join(os.path.dirname(__file__), + '../share/roundup/templates/classic/schema.py') + vars = dict(globals()) + vars['db'] = db + execfile(schema, vars) + initial_data = os.path.join(os.path.dirname(__file__), + '../share/roundup/templates/classic/initial_data.py') + vars = dict(db=db, admin_email='admin@test.com', + adminpw=password.Password('sekrit')) + execfile(initial_data, vars) + + # load standard detectors + dirname = os.path.join(os.path.dirname(__file__), + '../share/roundup/templates/classic/detectors') + for fn in os.listdir(dirname): + if not fn.endswith('.py'): continue + vars = {} + execfile(os.path.join(dirname, fn), vars) + vars['init'](db) + + ''' + status = Class(db, "status", name=String()) + status.setkey("name") + priority = Class(db, "priority", name=String(), order=String()) + priority.setkey("name") + keyword = Class(db, "keyword", name=String(), order=String()) + keyword.setkey("name") + user = Class(db, "user", username=String(), password=Password(), + assignable=Boolean(), age=Number(), roles=String(), address=String(), + supervisor=Link('user'),realname=String(),alternate_addresses=String()) + user.setkey("username") + file = FileClass(db, "file", name=String(), type=String(), + comment=String(indexme="yes"), fooz=Password()) + file_nidx = FileClass(db, "file_nidx", content=String(indexme='no')) + issue = IssueClass(db, "issue", title=String(indexme="yes"), + status=Link("status"), nosy=Multilink("user"), deadline=Date(), + foo=Interval(), files=Multilink("file"), assignedto=Link('user'), + priority=Link('priority'), spam=Multilink('msg'), + feedback=Link('msg')) + stuff = Class(db, "stuff", stuff=String()) + session = Class(db, 'session', title=String()) + msg = FileClass(db, "msg", date=Date(), + author=Link("user", do_journal='no'), + files=Multilink('file'), inreplyto=String(), + messageid=String(), summary=String(), + content=String(), + recipients=Multilink("user", do_journal='no') + ) + ''' + if create: + db.user.create(username="fred", roles='User', + password=password.Password('sekrit'), address='fred@example.com') + + db.security.addPermissionToRole('User', 'Email Access') + ''' + db.security.addPermission(name='Register', klass='user') + db.security.addPermissionToRole('User', 'Web Access') + db.security.addPermissionToRole('Anonymous', 'Email Access') + db.security.addPermissionToRole('Anonymous', 'Register', 'user') + for cl in 'issue', 'file', 'msg', 'keyword': + db.security.addPermissionToRole('User', 'View', cl) + db.security.addPermissionToRole('User', 'Edit', cl) + db.security.addPermissionToRole('User', 'Create', cl) + for cl in 'priority', 'status': + db.security.addPermissionToRole('User', 'View', cl) + ''' + return db + +class cldb(dict): + def close(self): + pass + +class BasicDatabase(dict): + ''' Provide a nice encapsulation of an anydbm store. + + Keys are id strings, values are automatically marshalled data. + ''' + def __getitem__(self, key): + if key not in self: + d = self[key] = {} + return d + return super(BasicDatabase, self).__getitem__(key) + def exists(self, infoid): + return infoid in self + def get(self, infoid, value, default=None): + return self[infoid].get(value, default) + def getall(self, infoid): + return self[infoid] + def set(self, infoid, **newvalues): + self[infoid].update(newvalues) + def list(self): + return self.keys() + def destroy(self, infoid): + del self[infoid] + def commit(self): + pass + def close(self): + pass + def updateTimestamp(self, sessid): + pass + def clean(self): + pass + +class Sessions(BasicDatabase): + name = 'sessions' + +class OneTimeKeys(BasicDatabase): + name = 'otks' + +class Indexer(indexer_dbm.Indexer): + def __init__(self, db): + indexer_common.Indexer.__init__(self, db) + self.reindex = 0 + self.quiet = 9 + self.changed = 0 + + def load_index(self, reload=0, wordlist=None): + # Unless reload is indicated, do not load twice + if self.index_loaded() and not reload: + return 0 + self.words = {} + self.files = {'_TOP':(0,None)} + self.fileids = {} + self.changed = 0 + + def save_index(self): + pass + +class Database(hyperdb.Database, roundupdb.Database): + """A database for storing records containing flexible data types. + + Transaction stuff TODO: + + - check the timestamp of the class file and nuke the cache if it's + modified. Do some sort of conflict checking on the dirty stuff. + - perhaps detect write collisions (related to above)? + """ + def __init__(self, config, journaltag=None): + self.config, self.journaltag = config, journaltag + self.classes = {} + self.items = {} + self.ids = {} + self.journals = {} + self.files = {} + self.security = security.Security(self) + self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0, + 'filtering': 0} + self.sessions = Sessions() + self.otks = OneTimeKeys() + self.indexer = Indexer(self) + + + def filename(self, classname, nodeid, property=None, create=0): + shutil.copyfile(__file__, __file__+'.dummy') + return __file__+'.dummy' + + def post_init(self): + pass + + def refresh_database(self): + pass + + def getSessionManager(self): + return self.sessions + + def getOTKManager(self): + return self.otks + + def reindex(self, classname=None, show_progress=False): + pass + + def __repr__(self): + return ''%id(self) + + def storefile(self, classname, nodeid, property, content): + self.files[classname, nodeid, property] = content + + def getfile(self, classname, nodeid, property): + return self.files[classname, nodeid, property] + + def numfiles(self): + return len(self.files) + + # + # Classes + # + def __getattr__(self, classname): + """A convenient way of calling self.getclass(classname).""" + if self.classes.has_key(classname): + return self.classes[classname] + raise AttributeError, classname + + def addclass(self, cl): + cn = cl.classname + if self.classes.has_key(cn): + raise ValueError, cn + self.classes[cn] = cl + self.items[cn] = cldb() + self.ids[cn] = 0 + + # add default Edit and View permissions + self.security.addPermission(name="Create", klass=cn, + description="User is allowed to create "+cn) + self.security.addPermission(name="Edit", klass=cn, + description="User is allowed to edit "+cn) + self.security.addPermission(name="View", klass=cn, + description="User is allowed to access "+cn) + + def getclasses(self): + """Return a list of the names of all existing classes.""" + l = self.classes.keys() + l.sort() + return l + + def getclass(self, classname): + """Get the Class object representing a particular class. + + If 'classname' is not a valid class name, a KeyError is raised. + """ + try: + return self.classes[classname] + except KeyError: + raise KeyError, 'There is no class called "%s"'%classname + + # + # Class DBs + # + def clear(self): + self.items = {} + + def getclassdb(self, classname): + """ grab a connection to the class db that will be used for + multiple actions + """ + return self.items[classname] + + # + # Node IDs + # + def newid(self, classname): + self.ids[classname] += 1 + return str(self.ids[classname]) + + # + # Nodes + # + def addnode(self, classname, nodeid, node): + self.getclassdb(classname)[nodeid] = node + + def setnode(self, classname, nodeid, node): + self.getclassdb(classname)[nodeid] = node + + def getnode(self, classname, nodeid, cldb=None): + if cldb is not None: + return cldb[nodeid] + return self.getclassdb(classname)[nodeid] + + def destroynode(self, classname, nodeid): + del self.getclassdb(classname)[nodeid] + + def hasnode(self, classname, nodeid): + return nodeid in self.getclassdb(classname) + + def countnodes(self, classname, db=None): + return len(self.getclassdb(classname)) + + # + # Journal + # + def addjournal(self, classname, nodeid, action, params, creator=None, + creation=None): + if creator is None: + creator = self.getuid() + if creation is None: + creation = date.Date() + self.journals.setdefault(classname, {}).setdefault(nodeid, + []).append((nodeid, creation, creator, action, params)) + + def setjournal(self, classname, nodeid, journal): + self.journals.setdefault(classname, {})[nodeid] = journal + + def getjournal(self, classname, nodeid): + return self.journals.get(classname, {}).get(nodeid, []) + + def pack(self, pack_before): + TODO + + # + # Basic transaction support + # + def commit(self, fail_ok=False): + pass + + def rollback(self): + TODO + + def close(self): + pass + +class Class(back_anydbm.Class): + def getnodeids(self, db=None, retired=None): + return self.db.getclassdb(self.classname).keys() + +class FileClass(back_anydbm.Class): + def __init__(self, db, classname, **properties): + if not properties.has_key('content'): + properties['content'] = hyperdb.String(indexme='yes') + if not properties.has_key('type'): + properties['type'] = hyperdb.String() + back_anydbm.Class.__init__(self, db, classname, **properties) + + def getnodeids(self, db=None, retired=None): + return self.db.getclassdb(self.classname).keys() + +# deviation from spec - was called ItemClass +class IssueClass(Class, roundupdb.IssueClass): + # Overridden methods: + def __init__(self, db, classname, **properties): + """The newly-created class automatically includes the "messages", + "files", "nosy", and "superseder" properties. If the 'properties' + dictionary attempts to specify any of these properties or a + "creation" or "activity" property, a ValueError is raised. + """ + if not properties.has_key('title'): + properties['title'] = hyperdb.String(indexme='yes') + if not properties.has_key('messages'): + properties['messages'] = hyperdb.Multilink("msg") + if not properties.has_key('files'): + properties['files'] = hyperdb.Multilink("file") + if not properties.has_key('nosy'): + # note: journalling is turned off as it really just wastes + # space. this behaviour may be overridden in an instance + properties['nosy'] = hyperdb.Multilink("user", do_journal="no") + if not properties.has_key('superseder'): + properties['superseder'] = hyperdb.Multilink(classname) + Class.__init__(self, db, classname, **properties) + +# vim: set et sts=4 sw=4 : diff --git a/test/test_mailgw.py b/test/test_mailgw.py index a3c95af..e15e292 100644 --- a/test/test_mailgw.py +++ b/test/test_mailgw.py @@ -26,7 +26,8 @@ from roundup.mailgw import MailGW, Unauthorized, uidFromAddress, \ from roundup import init, instance, password, rfc2822, __version__ from roundup.anypy.sets_ import set -import db_test_base +#import db_test_base +import memorydb class Message(rfc822.Message): """String-based Message class with equivalence test.""" @@ -37,6 +38,10 @@ class Message(rfc822.Message): return (self.dict == other.dict and self.fp.read() == other.fp.read()) +class Tracker(object): + def open(self, journaltag): + return self.db + class DiffHelper: def compareMessages(self, new, old): """Compare messages for semantic equivalence.""" @@ -115,12 +120,14 @@ class MailgwTestCase(unittest.TestCase, DiffHelper): schema = 'classic' def setUp(self): MailgwTestCase.count = MailgwTestCase.count + 1 - self.dirname = '_test_mailgw_%s'%self.count - # set up and open a tracker - self.instance = db_test_base.setupTracker(self.dirname) - # and open the database - self.db = self.instance.open('admin') + # and open the database / "instance" + self.db = memorydb.create('admin') + self.instance = Tracker() + self.instance.db = self.db + self.instance.config = self.db.config + self.instance.MailGW = MailGW + self.chef_id = self.db.user.create(username='Chef', address='chef@bork.bork.bork', realname='Bork, Chef', roles='User') self.richard_id = self.db.user.create(username='richard', @@ -135,22 +142,11 @@ class MailgwTestCase(unittest.TestCase, DiffHelper): if os.path.exists(SENDMAILDEBUG): os.remove(SENDMAILDEBUG) self.db.close() - try: - shutil.rmtree(self.dirname) - except OSError, error: - if error.errno not in (errno.ENOENT, errno.ESRCH): raise def _handle_mail(self, message): - # handler will open a new db handle. On single-threaded - # databases we'll have to close our current connection - self.db.commit() - self.db.close() handler = self.instance.MailGW(self.instance) handler.trapExceptions = 0 - ret = handler.main(StringIO(message)) - # handler had its own database, open new connection - self.db = self.instance.open('admin') - return ret + return handler.main(StringIO(message)) def _get_mail(self): f = open(SENDMAILDEBUG) @@ -173,6 +169,22 @@ Subject: [issue] Testing... assert not os.path.exists(SENDMAILDEBUG) self.assertEqual(self.db.issue.get(nodeid, 'title'), 'Testing...') + def testMessageWithFromInIt(self): + nodeid = self._handle_mail('''Content-Type: text/plain; + charset="iso-8859-1" +From: Chef +To: issue_tracker@your.tracker.email.domain.example +Cc: richard@test.test +Reply-To: chef@bork.bork.bork +Message-Id: +Subject: [issue] Testing... + +From here to there! +''') + assert not os.path.exists(SENDMAILDEBUG) + msgid = self.db.issue.get(nodeid, 'msg')[0] + self.assertEqual(self.db.issue.get(msgid, 'content'), 'From here to there!') + def doNewIssue(self): nodeid = self._handle_mail('''Content-Type: text/plain; charset="iso-8859-1" @@ -1020,7 +1032,6 @@ Subject: [issue1] Testing... [nosy=-richard] assert not os.path.exists(SENDMAILDEBUG) def testNewUserAuthor(self): - l = self.db.user.list() l.sort() message = '''Content-Type: text/plain; @@ -1032,12 +1043,9 @@ Subject: [issue] Testing... This is a test submission of a new issue. ''' - def hook (db, **kw): - ''' set up callback for db open ''' - db.security.role['anonymous'].permissions=[] - anonid = db.user.lookup('anonymous') - db.user.set(anonid, roles='Anonymous') - self.instance.schema_hook = hook + self.db.security.role['anonymous'].permissions=[] + anonid = self.db.user.lookup('anonymous') + self.db.user.set(anonid, roles='Anonymous') try: self._handle_mail(message) except Unauthorized, value: @@ -1046,23 +1054,17 @@ You are not a registered user. Unknown address: fubar@bork.bork.bork """) - assert not body_diff, body_diff - else: raise AssertionError, "Unathorized not raised when handling mail" - - def hook (db, **kw): - ''' set up callback for db open ''' - # Add Web Access role to anonymous, and try again to make sure - # we get a "please register at:" message this time. - p = [ - db.security.getPermission('Register', 'user'), - db.security.getPermission('Web Access', None), - ] - db.security.role['anonymous'].permissions=p - self.instance.schema_hook = hook + # Add Web Access role to anonymous, and try again to make sure + # we get a "please register at:" message this time. + p = [ + self.db.security.getPermission('Register', 'user'), + self.db.security.getPermission('Web Access', None), + ] + self.db.security.role['anonymous'].permissions=p try: self._handle_mail(message) except Unauthorized, value: @@ -1075,9 +1077,7 @@ http://tracker.example/cgi-bin/roundup.cgi/bugs/user?template=register Unknown address: fubar@bork.bork.bork """) - assert not body_diff, body_diff - else: raise AssertionError, "Unathorized not raised when handling mail" @@ -1086,15 +1086,12 @@ Unknown address: fubar@bork.bork.bork m.sort() self.assertEqual(l, m) - def hook (db, **kw): - ''' set up callback for db open ''' - # now with the permission - p = [ - db.security.getPermission('Register', 'user'), - db.security.getPermission('Email Access', None), - ] - db.security.role['anonymous'].permissions=p - self.instance.schema_hook = hook + # now with the permission + p = [ + self.db.security.getPermission('Register', 'user'), + self.db.security.getPermission('Email Access', None), + ] + self.db.security.role['anonymous'].permissions=p self._handle_mail(message) m = self.db.user.list() m.sort() @@ -1112,16 +1109,13 @@ Subject: [issue] Testing... This is a test submission of a new issue. ''' - def hook (db, **kw): - ''' set up callback for db open ''' - p = [ - db.security.getPermission('Register', 'user'), - db.security.getPermission('Email Access', None), - db.security.getPermission('Create', 'issue'), - db.security.getPermission('Create', 'msg'), - ] - db.security.role['anonymous'].permissions = p - self.instance.schema_hook = hook + p = [ + self.db.security.getPermission('Register', 'user'), + self.db.security.getPermission('Email Access', None), + self.db.security.getPermission('Create', 'issue'), + self.db.security.getPermission('Create', 'msg'), + ] + self.db.security.role['anonymous'].permissions = p self._handle_mail(message) m = set(self.db.user.list()) new = list(m - l)[0]