Code

Fix security-problem: If user hasn't permission on a message (notably
[roundup.git] / test / test_mailgw.py
index 6d531977479a332e96b65d867b3148df37480f8d..d763fb98701dccf40f699ff34b493a0d6bacb8aa 100644 (file)
@@ -1,3 +1,4 @@
+# -*- encoding: utf-8 -*-
 #
 # Copyright (c) 2001 Richard Jones, richard@bofh.asn.au.
 # This module is free software, and you may redistribute it and/or modify
@@ -8,44 +9,90 @@
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 #
-# $Id: test_mailgw.py,v 1.25 2002-07-29 00:56:06 richard Exp $
+# $Id: test_mailgw.py,v 1.96 2008-08-19 01:40:59 richard Exp $
 
-import unittest, cStringIO, tempfile, os, shutil, errno, imp, sys, difflib
+# TODO: test bcc
 
-# Note: Should parse emails according to RFC2822 instead of performing a
-# literal string comparision.  Parsing the messages allows the tests to work for
-# any legal serialization of an email.
-#try :
-#    import email
-#except ImportError :
-#    import rfc822 as email
+import unittest, tempfile, os, shutil, errno, imp, sys, difflib, rfc822, time
 
-from roundup.mailgw import MailGW, Unauthorized
-from roundup import init, instance
+from cStringIO import StringIO
+
+if not os.environ.has_key('SENDMAILDEBUG'):
+    os.environ['SENDMAILDEBUG'] = 'mail-test.log'
+SENDMAILDEBUG = os.environ['SENDMAILDEBUG']
+
+from roundup.mailgw import MailGW, Unauthorized, uidFromAddress, \
+    parseContent, IgnoreLoop, IgnoreBulk, MailUsageError, MailUsageHelp
+from roundup import init, instance, password, rfc2822, __version__
+from roundup.anypy.sets_ import set
+
+import db_test_base
+
+class Message(rfc822.Message):
+    """String-based Message class with equivalence test."""
+    def __init__(self, s):
+        rfc822.Message.__init__(self, StringIO(s.strip()))
+
+    def __eq__(self, other):
+        return (self.dict == other.dict and
+                self.fp.read() == other.fp.read())
 
-# TODO: make this output only enough equal lines for context, not all of
-# them
 class DiffHelper:
-    def compareStrings(self, s2, s1):
+    def compareMessages(self, new, old):
+        """Compare messages for semantic equivalence."""
+        new, old = Message(new), Message(old)
+
+        # all Roundup-generated messages have "Precedence: bulk"
+        old['Precedence'] = 'bulk'
+
+        # don't try to compare the date
+        del new['date'], old['date']
+
+        if not new == old:
+            res = []
+
+            replace = {}
+            for key in new.keys():
+                if key.startswith('from '):
+                    # skip the unix from line
+                    continue
+                if key.lower() == 'x-roundup-version':
+                    # version changes constantly, so handle it specially
+                    if new[key] != __version__:
+                        res.append('  %s: %r != %r' % (key, __version__,
+                            new[key]))
+                elif key.lower() == 'content-type' and 'boundary=' in new[key]:
+                    # handle mime messages
+                    newmime = new[key].split('=',1)[-1].strip('"')
+                    oldmime = old.get(key, '').split('=',1)[-1].strip('"')
+                    replace ['--' + newmime] = '--' + oldmime
+                    replace ['--' + newmime + '--'] = '--' + oldmime + '--'
+                elif new.get(key, '') != old.get(key, ''):
+                    res.append('  %s: %r != %r' % (key, old.get(key, ''),
+                        new.get(key, '')))
+
+            body_diff = self.compareStrings(new.fp.read(), old.fp.read(),
+                replace=replace)
+            if body_diff:
+                res.append('')
+                res.extend(body_diff)
+
+            if res:
+                res.insert(0, 'Generated message not correct (diff follows):')
+                raise AssertionError, '\n'.join(res)
+
+    def compareStrings(self, s2, s1, replace={}):
         '''Note the reversal of s2 and s1 - difflib.SequenceMatcher wants
            the first to be the "original" but in the calls in this file,
            the second arg is the original. Ho hum.
+           Do replacements over the replace dict -- used for mime boundary
         '''
-        if s1 == s2:
+        l1 = s1.strip().split('\n')
+        l2 = [replace.get(i,i) for i in s2.strip().split('\n')]
+        if l1 == l2:
             return
-
-        # under python2.[12] we allow a difference of one trailing empty line.
-        if sys.version_info[0:2] == (2,1):
-            if s1+'\n' == s2:
-                return
-        if sys.version_info[0:2] == (2,2):
-            if s1 == s2+'\n':
-                return
-        
-        l1=s1.split('\n')
-        l2=s2.split('\n')
         s = difflib.SequenceMatcher(None, l1, l2)
-        res = ['Generated message not correct (diff follows):']
+        res = []
         for value, s1s, s1e, s2s, s2e in s.get_opcodes():
             if value == 'equal':
                 for i in range(s1s, s1e):
@@ -61,7 +108,7 @@ class DiffHelper:
                     res.append('- %s'%l1[i])
                     res.append('+ %s'%l2[j])
 
-        raise AssertionError, '\n'.join(res)
+        return res
 
 class MailgwTestCase(unittest.TestCase, DiffHelper):
     count = 0
@@ -69,81 +116,104 @@ class MailgwTestCase(unittest.TestCase, DiffHelper):
     def setUp(self):
         MailgwTestCase.count = MailgwTestCase.count + 1
         self.dirname = '_test_mailgw_%s'%self.count
-        try:
-            shutil.rmtree(self.dirname)
-        except OSError, error:
-            if error.errno not in (errno.ENOENT, errno.ESRCH): raise
-        # create the instance
-        init.install(self.dirname, 'classic', 'anydbm')
-        init.initialise(self.dirname, 'sekrit')
-        # check we can load the package
-        self.instance = instance.open(self.dirname)
+        # set up and open a tracker
+        self.instance = db_test_base.setupTracker(self.dirname)
+
         # and open the database
-        self.db = self.instance.open('sekrit')
-        self.db.user.create(username='Chef', address='chef@bork.bork.bork')
-        self.db.user.create(username='richard', address='richard@test')
-        self.db.user.create(username='mary', address='mary@test')
-        self.db.user.create(username='john', address='john@test',
-            alternate_addresses='jondoe@test\njohn.doe@test')
+        self.db = self.instance.open('admin')
+        self.chef_id = self.db.user.create(username='Chef',
+            address='chef@bork.bork.bork', realname='Bork, Chef', roles='User')
+        self.richard_id = self.db.user.create(username='richard',
+            address='richard@test.test', roles='User')
+        self.mary_id = self.db.user.create(username='mary',
+            address='mary@test.test', roles='User', realname='Contrary, Mary')
+        self.john_id = self.db.user.create(username='john',
+            address='john@test.test', roles='User', realname='John Doe',
+            alternate_addresses='jondoe@test.test\njohn.doe@test.test')
 
     def tearDown(self):
-        if os.path.exists(os.environ['SENDMAILDEBUG']):
-            os.remove(os.environ['SENDMAILDEBUG'])
+        if os.path.exists(SENDMAILDEBUG):
+            os.remove(SENDMAILDEBUG)
+        self.db.close()
         try:
             shutil.rmtree(self.dirname)
         except OSError, error:
             if error.errno not in (errno.ENOENT, errno.ESRCH): raise
 
+    def _handle_mail(self, message):
+        # handler will open a new db handle. On single-threaded
+        # databases we'll have to close our current connection
+        self.db.commit()
+        self.db.close()
+        handler = self.instance.MailGW(self.instance)
+        handler.trapExceptions = 0
+        ret = handler.main(StringIO(message))
+        # handler had its own database, open new connection
+        self.db = self.instance.open('admin')
+        return ret
+
+    def _get_mail(self):
+        f = open(SENDMAILDEBUG)
+        try:
+            return f.read()
+        finally:
+            f.close()
+
+    def testEmptyMessage(self):
+        nodeid = self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Cc: richard@test.test
+Reply-To: chef@bork.bork.bork
+Message-Id: <dummy_test_message_id>
+Subject: [issue] Testing...
+
+''')
+        assert not os.path.exists(SENDMAILDEBUG)
+        self.assertEqual(self.db.issue.get(nodeid, 'title'), 'Testing...')
+
     def doNewIssue(self):
-        message = cStringIO.StringIO('''Content-Type: text/plain;
+        nodeid = self._handle_mail('''Content-Type: text/plain;
   charset="iso-8859-1"
 From: Chef <chef@bork.bork.bork>
 To: issue_tracker@your.tracker.email.domain.example
-Cc: richard@test
+Cc: richard@test.test
 Message-Id: <dummy_test_message_id>
 Subject: [issue] Testing...
 
 This is a test submission of a new issue.
 ''')
-        handler = self.instance.MailGW(self.instance, self.db)
-        handler.trapExceptions = 0
-        nodeid = handler.main(message)
-        if os.path.exists(os.environ['SENDMAILDEBUG']):
-            error = open(os.environ['SENDMAILDEBUG']).read()
-            self.assertEqual('no error', error)
+        assert not os.path.exists(SENDMAILDEBUG)
         l = self.db.issue.get(nodeid, 'nosy')
         l.sort()
-        self.assertEqual(l, ['3', '4'])
+        self.assertEqual(l, [self.chef_id, self.richard_id])
+        return nodeid
 
     def testNewIssue(self):
         self.doNewIssue()
 
     def testNewIssueNosy(self):
-        self.instance.ADD_AUTHOR_TO_NOSY = 'yes'
-        message = cStringIO.StringIO('''Content-Type: text/plain;
+        self.instance.config.ADD_AUTHOR_TO_NOSY = 'yes'
+        nodeid = self._handle_mail('''Content-Type: text/plain;
   charset="iso-8859-1"
 From: Chef <chef@bork.bork.bork>
 To: issue_tracker@your.tracker.email.domain.example
-Cc: richard@test
+Cc: richard@test.test
 Message-Id: <dummy_test_message_id>
 Subject: [issue] Testing...
 
 This is a test submission of a new issue.
 ''')
-        handler = self.instance.MailGW(self.instance, self.db)
-        handler.trapExceptions = 0
-        nodeid = handler.main(message)
-        if os.path.exists(os.environ['SENDMAILDEBUG']):
-            error = open(os.environ['SENDMAILDEBUG']).read()
-            self.assertEqual('no error', error)
+        assert not os.path.exists(SENDMAILDEBUG)
         l = self.db.issue.get(nodeid, 'nosy')
         l.sort()
-        self.assertEqual(l, ['3', '4'])
+        self.assertEqual(l, [self.chef_id, self.richard_id])
 
     def testAlternateAddress(self):
-        message = cStringIO.StringIO('''Content-Type: text/plain;
+        self._handle_mail('''Content-Type: text/plain;
   charset="iso-8859-1"
-From: John Doe <john.doe@test>
+From: John Doe <john.doe@test.test>
 To: issue_tracker@your.tracker.email.domain.example
 Message-Id: <dummy_test_message_id>
 Subject: [issue] Testing...
@@ -151,35 +221,27 @@ Subject: [issue] Testing...
 This is a test submission of a new issue.
 ''')
         userlist = self.db.user.list()
-        handler = self.instance.MailGW(self.instance, self.db)
-        handler.trapExceptions = 0
-        handler.main(message)
-        if os.path.exists(os.environ['SENDMAILDEBUG']):
-            error = open(os.environ['SENDMAILDEBUG']).read()
-            self.assertEqual('no error', error)
+        assert not os.path.exists(SENDMAILDEBUG)
         self.assertEqual(userlist, self.db.user.list(),
             "user created when it shouldn't have been")
 
     def testNewIssueNoClass(self):
-        message = cStringIO.StringIO('''Content-Type: text/plain;
+        self._handle_mail('''Content-Type: text/plain;
   charset="iso-8859-1"
 From: Chef <chef@bork.bork.bork>
 To: issue_tracker@your.tracker.email.domain.example
-Cc: richard@test
+Cc: richard@test.test
 Message-Id: <dummy_test_message_id>
 Subject: Testing...
 
 This is a test submission of a new issue.
 ''')
-        handler = self.instance.MailGW(self.instance, self.db)
-        handler.trapExceptions = 0
-        handler.main(message)
-        if os.path.exists(os.environ['SENDMAILDEBUG']):
-            error = open(os.environ['SENDMAILDEBUG']).read()
-            self.assertEqual('no error', error)
+        assert not os.path.exists(SENDMAILDEBUG)
 
     def testNewIssueAuthMsg(self):
-        message = cStringIO.StringIO('''Content-Type: text/plain;
+        # TODO: fix the damn config - this is apalling
+        self.db.config.MESSAGES_TO_AUTHOR = 'yes'
+        self._handle_mail('''Content-Type: text/plain;
   charset="iso-8859-1"
 From: Chef <chef@bork.bork.bork>
 To: issue_tracker@your.tracker.email.domain.example
@@ -188,30 +250,109 @@ Subject: [issue] Testing... [nosy=mary; assignedto=richard]
 
 This is a test submission of a new issue.
 ''')
-        handler = self.instance.MailGW(self.instance, self.db)
-        handler.trapExceptions = 0
-        # TODO: fix the damn config - this is apalling
-        self.db.config.MESSAGES_TO_AUTHOR = 'yes'
-        handler.main(message)
+        self.compareMessages(self._get_mail(),
+'''FROM: roundup-admin@your.tracker.email.domain.example
+TO: chef@bork.bork.bork, mary@test.test, richard@test.test
+Content-Type: text/plain; charset="utf-8"
+Subject: [issue1] Testing...
+To: chef@bork.bork.bork, mary@test.test, richard@test.test
+From: "Bork, Chef" <issue_tracker@your.tracker.email.domain.example>
+Reply-To: Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
+MIME-Version: 1.0
+Message-Id: <dummy_test_message_id>
+X-Roundup-Name: Roundup issue tracker
+X-Roundup-Loop: hello
+X-Roundup-Issue-Status: unread
+Content-Transfer-Encoding: quoted-printable
+
+
+New submission from Bork, Chef <chef@bork.bork.bork>:
+
+This is a test submission of a new issue.
+
+----------
+assignedto: richard
+messages: 1
+nosy: Chef, mary, richard
+status: unread
+title: Testing...
+
+_______________________________________________________________________
+Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
+<http://tracker.example/cgi-bin/roundup.cgi/bugs/issue1>
+_______________________________________________________________________
+''')
+
+    def testNewIssueNoAuthorInfo(self):
+        self.db.config.MAIL_ADD_AUTHORINFO = 'no'
+        self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Message-Id: <dummy_test_message_id>
+Subject: [issue] Testing... [nosy=mary; assignedto=richard]
 
-        self.compareStrings(open(os.environ['SENDMAILDEBUG']).read(),
+This is a test submission of a new issue.
+''')
+        self.compareMessages(self._get_mail(),
 '''FROM: roundup-admin@your.tracker.email.domain.example
-TO: chef@bork.bork.bork, mary@test, richard@test
-Content-Type: text/plain
+TO: chef@bork.bork.bork, mary@test.test, richard@test.test
+Content-Type: text/plain; charset="utf-8"
 Subject: [issue1] Testing...
-To: chef@bork.bork.bork, mary@test, richard@test
-From: "Chef" <issue_tracker@your.tracker.email.domain.example>
-Reply-To: "Roundup issue tracker" <issue_tracker@your.tracker.email.domain.example>
+To: mary@test.test, richard@test.test
+From: "Bork, Chef" <issue_tracker@your.tracker.email.domain.example>
+Reply-To: Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
 MIME-Version: 1.0
 Message-Id: <dummy_test_message_id>
 X-Roundup-Name: Roundup issue tracker
+X-Roundup-Loop: hello
+X-Roundup-Issue-Status: unread
 Content-Transfer-Encoding: quoted-printable
 
+This is a test submission of a new issue.
+
+----------
+assignedto: richard
+messages: 1
+nosy: Chef, mary, richard
+status: unread
+title: Testing...
+
+_______________________________________________________________________
+Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
+<http://tracker.example/cgi-bin/roundup.cgi/bugs/issue1>
+_______________________________________________________________________
+''')
 
-New submission from Chef <chef@bork.bork.bork>:
+    def testNewIssueNoAuthorEmail(self):
+        self.db.config.MAIL_ADD_AUTHOREMAIL = 'no'
+        self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Message-Id: <dummy_test_message_id>
+Subject: [issue] Testing... [nosy=mary; assignedto=richard]
 
 This is a test submission of a new issue.
+''')
+        self.compareMessages(self._get_mail(),
+'''FROM: roundup-admin@your.tracker.email.domain.example
+TO: chef@bork.bork.bork, mary@test.test, richard@test.test
+Content-Type: text/plain; charset="utf-8"
+Subject: [issue1] Testing...
+To: mary@test.test, richard@test.test
+From: "Bork, Chef" <issue_tracker@your.tracker.email.domain.example>
+Reply-To: Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
+MIME-Version: 1.0
+Message-Id: <dummy_test_message_id>
+X-Roundup-Name: Roundup issue tracker
+X-Roundup-Loop: hello
+X-Roundup-Issue-Status: unread
+Content-Transfer-Encoding: quoted-printable
+
+New submission from Bork, Chef:
 
+This is a test submission of a new issue.
 
 ----------
 assignedto: richard
@@ -219,26 +360,117 @@ messages: 1
 nosy: Chef, mary, richard
 status: unread
 title: Testing...
-_________________________________________________________________________
-"Roundup issue tracker" <issue_tracker@your.tracker.email.domain.example>
-http://your.tracker.url.example/issue1
-_________________________________________________________________________
+
+_______________________________________________________________________
+Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
+<http://tracker.example/cgi-bin/roundup.cgi/bugs/issue1>
+_______________________________________________________________________
 ''')
 
-    # BUG
-    # def testMultipart(self):
-    #         '''With more than one part'''
-    #        see MultipartEnc tests: but if there is more than one part
-    #        we return a multipart/mixed and the boundary contains
-    #        the ip address of the test machine. 
+    multipart_msg = '''From: mary <mary@test.test>
+To: issue_tracker@your.tracker.email.domain.example
+Message-Id: <followup_dummy_id>
+In-Reply-To: <dummy_test_message_id>
+Subject: [issue1] Testing...
+Content-Type: multipart/mixed; boundary="bxyzzy"
+Content-Disposition: inline
+
+
+--bxyzzy
+Content-Type: multipart/alternative; boundary="bCsyhTFzCvuiizWE"
+Content-Disposition: inline
+
+--bCsyhTFzCvuiizWE
+Content-Type: text/plain; charset=us-ascii
+Content-Disposition: inline
+
+test attachment first text/plain
+
+--bCsyhTFzCvuiizWE
+Content-Type: application/octet-stream
+Content-Disposition: attachment; filename="first.dvi"
+Content-Transfer-Encoding: base64
+
+SnVzdCBhIHRlc3QgAQo=
+
+--bCsyhTFzCvuiizWE
+Content-Type: text/plain; charset=us-ascii
+Content-Disposition: inline
+
+test attachment second text/plain
 
-    # BUG should test some binary attamchent too.
+--bCsyhTFzCvuiizWE
+Content-Type: text/html
+Content-Disposition: inline
+
+<html>
+to be ignored.
+</html>
+
+--bCsyhTFzCvuiizWE--
+
+--bxyzzy
+Content-Type: multipart/alternative; boundary="bCsyhTFzCvuiizWF"
+Content-Disposition: inline
+
+--bCsyhTFzCvuiizWF
+Content-Type: text/plain; charset=us-ascii
+Content-Disposition: inline
+
+test attachment third text/plain
+
+--bCsyhTFzCvuiizWF
+Content-Type: application/octet-stream
+Content-Disposition: attachment; filename="second.dvi"
+Content-Transfer-Encoding: base64
+
+SnVzdCBhIHRlc3QK
+
+--bCsyhTFzCvuiizWF--
+
+--bxyzzy--
+'''
+
+    def testMultipartKeepAlternatives(self):
+        self.doNewIssue()
+        self._handle_mail(self.multipart_msg)
+        messages = self.db.issue.get('1', 'messages')
+        messages.sort()
+        msg = self.db.msg.getnode (messages[-1])
+        assert(len(msg.files) == 5)
+        names = {0 : 'first.dvi', 4 : 'second.dvi'}
+        content = {3 : 'test attachment third text/plain\n',
+                   4 : 'Just a test\n'}
+        for n, id in enumerate (msg.files):
+            f = self.db.file.getnode (id)
+            self.assertEqual(f.name, names.get (n, 'unnamed'))
+            if n in content :
+                self.assertEqual(f.content, content [n])
+        self.assertEqual(msg.content, 'test attachment second text/plain')
+
+    def testMultipartDropAlternatives(self):
+        self.doNewIssue()
+        self.db.config.MAILGW_IGNORE_ALTERNATIVES = True
+        self._handle_mail(self.multipart_msg)
+        messages = self.db.issue.get('1', 'messages')
+        messages.sort()
+        msg = self.db.msg.getnode (messages[-1])
+        assert(len(msg.files) == 2)
+        names = {1 : 'second.dvi'}
+        content = {0 : 'test attachment third text/plain\n',
+                   1 : 'Just a test\n'}
+        for n, id in enumerate (msg.files):
+            f = self.db.file.getnode (id)
+            self.assertEqual(f.name, names.get (n, 'unnamed'))
+            if n in content :
+                self.assertEqual(f.content, content [n])
+        self.assertEqual(msg.content, 'test attachment second text/plain')
 
     def testSimpleFollowup(self):
         self.doNewIssue()
-        message = cStringIO.StringIO('''Content-Type: text/plain;
+        self._handle_mail('''Content-Type: text/plain;
   charset="iso-8859-1"
-From: mary <mary@test>
+From: mary <mary@test.test>
 To: issue_tracker@your.tracker.email.domain.example
 Message-Id: <followup_dummy_id>
 In-Reply-To: <dummy_test_message_id>
@@ -246,43 +478,42 @@ Subject: [issue1] Testing...
 
 This is a second followup
 ''')
-        handler = self.instance.MailGW(self.instance, self.db)
-        handler.trapExceptions = 0
-        handler.main(message)
-        self.compareStrings(open(os.environ['SENDMAILDEBUG']).read(),
+        self.compareMessages(self._get_mail(),
 '''FROM: roundup-admin@your.tracker.email.domain.example
-TO: chef@bork.bork.bork, richard@test
-Content-Type: text/plain
+TO: chef@bork.bork.bork, richard@test.test
+Content-Type: text/plain; charset="utf-8"
 Subject: [issue1] Testing...
-To: chef@bork.bork.bork, richard@test
-From: "mary" <issue_tracker@your.tracker.email.domain.example>
-Reply-To: "Roundup issue tracker" <issue_tracker@your.tracker.email.domain.example>
+To: chef@bork.bork.bork, richard@test.test
+From: "Contrary, Mary" <issue_tracker@your.tracker.email.domain.example>
+Reply-To: Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
 MIME-Version: 1.0
 Message-Id: <followup_dummy_id>
 In-Reply-To: <dummy_test_message_id>
 X-Roundup-Name: Roundup issue tracker
+X-Roundup-Loop: hello
+X-Roundup-Issue-Status: chatting
 Content-Transfer-Encoding: quoted-printable
 
 
-mary <mary@test> added the comment:
+Contrary, Mary <mary@test.test> added the comment:
 
 This is a second followup
 
-
 ----------
 status: unread -> chatting
-_________________________________________________________________________
-"Roundup issue tracker" <issue_tracker@your.tracker.email.domain.example>
-http://your.tracker.url.example/issue1
-_________________________________________________________________________
+
+_______________________________________________________________________
+Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
+<http://tracker.example/cgi-bin/roundup.cgi/bugs/issue1>
+_______________________________________________________________________
 ''')
 
     def testFollowup(self):
         self.doNewIssue()
 
-        message = cStringIO.StringIO('''Content-Type: text/plain;
+        self._handle_mail('''Content-Type: text/plain;
   charset="iso-8859-1"
-From: richard <richard@test>
+From: richard <richard@test.test>
 To: issue_tracker@your.tracker.email.domain.example
 Message-Id: <followup_dummy_id>
 In-Reply-To: <dummy_test_message_id>
@@ -290,95 +521,212 @@ Subject: [issue1] Testing... [assignedto=mary; nosy=+john]
 
 This is a followup
 ''')
-        handler = self.instance.MailGW(self.instance, self.db)
-        handler.trapExceptions = 0
-        handler.main(message)
         l = self.db.issue.get('1', 'nosy')
         l.sort()
-        self.assertEqual(l, ['3', '4', '5', '6'])
+        self.assertEqual(l, [self.chef_id, self.richard_id, self.mary_id,
+            self.john_id])
 
-        self.compareStrings(open(os.environ['SENDMAILDEBUG']).read(),
+        self.compareMessages(self._get_mail(),
 '''FROM: roundup-admin@your.tracker.email.domain.example
-TO: chef@bork.bork.bork, john@test, mary@test
-Content-Type: text/plain
+TO: chef@bork.bork.bork, john@test.test, mary@test.test
+Content-Type: text/plain; charset="utf-8"
 Subject: [issue1] Testing...
-To: chef@bork.bork.bork, john@test, mary@test
-From: "richard" <issue_tracker@your.tracker.email.domain.example>
-Reply-To: "Roundup issue tracker" <issue_tracker@your.tracker.email.domain.example>
+To: chef@bork.bork.bork, john@test.test, mary@test.test
+From: richard <issue_tracker@your.tracker.email.domain.example>
+Reply-To: Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
 MIME-Version: 1.0
 Message-Id: <followup_dummy_id>
 In-Reply-To: <dummy_test_message_id>
 X-Roundup-Name: Roundup issue tracker
+X-Roundup-Loop: hello
+X-Roundup-Issue-Status: chatting
 Content-Transfer-Encoding: quoted-printable
 
 
-richard <richard@test> added the comment:
+richard <richard@test.test> added the comment:
 
 This is a followup
 
-
 ----------
 assignedto:  -> mary
-nosy: +mary, john
+nosy: +john, mary
 status: unread -> chatting
-_________________________________________________________________________
-"Roundup issue tracker" <issue_tracker@your.tracker.email.domain.example>
-http://your.tracker.url.example/issue1
-_________________________________________________________________________
+
+_______________________________________________________________________
+Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
+<http://tracker.example/cgi-bin/roundup.cgi/bugs/issue1>
+_______________________________________________________________________
 ''')
 
+    def testPropertyChangeOnly(self):
+        self.doNewIssue()
+        oldvalues = self.db.getnode('issue', '1').copy()
+        oldvalues['assignedto'] = None
+        # reconstruct old behaviour: This would reuse the
+        # database-handle from the doNewIssue above which has committed
+        # as user "Chef". So we close and reopen the db as that user.
+        self.db.close()
+        self.db = self.instance.open('Chef')
+        self.db.issue.set('1', assignedto=self.chef_id)
+        self.db.commit()
+        self.db.issue.nosymessage('1', None, oldvalues)
+
+        new_mail = ""
+        for line in self._get_mail().split("\n"):
+            if "Message-Id: " in line:
+                continue
+            if "Date: " in line:
+                continue
+            new_mail += line+"\n"
+
+        self.compareMessages(new_mail, """
+FROM: roundup-admin@your.tracker.email.domain.example
+TO: chef@bork.bork.bork, richard@test.test
+Content-Type: text/plain; charset="utf-8"
+Subject: [issue1] Testing...
+To: chef@bork.bork.bork, richard@test.test
+From: "Bork, Chef" <issue_tracker@your.tracker.email.domain.example>
+X-Roundup-Name: Roundup issue tracker
+X-Roundup-Loop: hello
+X-Roundup-Issue-Status: unread
+X-Roundup-Version: 1.3.3
+MIME-Version: 1.0
+Reply-To: Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
+Content-Transfer-Encoding: quoted-printable
+
+
+Change by Bork, Chef <chef@bork.bork.bork>:
+
+
+----------
+assignedto:  -> Chef
+
+_______________________________________________________________________
+Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
+<http://tracker.example/cgi-bin/roundup.cgi/bugs/issue1>
+_______________________________________________________________________
+""")
+
+
+    #
+    # FOLLOWUP TITLE MATCH
+    #
     def testFollowupTitleMatch(self):
         self.doNewIssue()
-        message = cStringIO.StringIO('''Content-Type: text/plain;
+        self._handle_mail('''Content-Type: text/plain;
   charset="iso-8859-1"
-From: richard <richard@test>
+From: richard <richard@test.test>
 To: issue_tracker@your.tracker.email.domain.example
 Message-Id: <followup_dummy_id>
-In-Reply-To: <dummy_test_message_id>
 Subject: Re: Testing... [assignedto=mary; nosy=+john]
 
 This is a followup
 ''')
-        handler = self.instance.MailGW(self.instance, self.db)
-        handler.trapExceptions = 0
-        handler.main(message)
-
-        self.compareStrings(open(os.environ['SENDMAILDEBUG']).read(),
+        self.compareMessages(self._get_mail(),
 '''FROM: roundup-admin@your.tracker.email.domain.example
-TO: chef@bork.bork.bork, john@test, mary@test
-Content-Type: text/plain
+TO: chef@bork.bork.bork, john@test.test, mary@test.test
+Content-Type: text/plain; charset="utf-8"
 Subject: [issue1] Testing...
-To: chef@bork.bork.bork, john@test, mary@test
-From: "richard" <issue_tracker@your.tracker.email.domain.example>
-Reply-To: "Roundup issue tracker" <issue_tracker@your.tracker.email.domain.example>
+To: chef@bork.bork.bork, john@test.test, mary@test.test
+From: richard <issue_tracker@your.tracker.email.domain.example>
+Reply-To: Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
 MIME-Version: 1.0
 Message-Id: <followup_dummy_id>
 In-Reply-To: <dummy_test_message_id>
 X-Roundup-Name: Roundup issue tracker
+X-Roundup-Loop: hello
+X-Roundup-Issue-Status: chatting
 Content-Transfer-Encoding: quoted-printable
 
 
-richard <richard@test> added the comment:
+richard <richard@test.test> added the comment:
 
 This is a followup
 
-
 ----------
 assignedto:  -> mary
-nosy: +mary, john
+nosy: +john, mary
 status: unread -> chatting
-_________________________________________________________________________
-"Roundup issue tracker" <issue_tracker@your.tracker.email.domain.example>
-http://your.tracker.url.example/issue1
-_________________________________________________________________________
+
+_______________________________________________________________________
+Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
+<http://tracker.example/cgi-bin/roundup.cgi/bugs/issue1>
+_______________________________________________________________________
+''')
+
+    def testFollowupTitleMatchMultiRe(self):
+        nodeid1 = self.doNewIssue()
+        nodeid2 = self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: richard <richard@test.test>
+To: issue_tracker@your.tracker.email.domain.example
+Message-Id: <followup_dummy_id>
+Subject: Re: Testing... [assignedto=mary; nosy=+john]
+
+This is a followup
+''')
+
+        nodeid3 = self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: richard <richard@test.test>
+To: issue_tracker@your.tracker.email.domain.example
+Message-Id: <followup2_dummy_id>
+Subject: Ang: Re: Testing...
+
+This is a followup
 ''')
+        self.assertEqual(nodeid1, nodeid2)
+        self.assertEqual(nodeid1, nodeid3)
+
+    def testFollowupTitleMatchNever(self):
+        nodeid = self.doNewIssue()
+        self.db.config.MAILGW_SUBJECT_CONTENT_MATCH = 'never'
+        self.assertNotEqual(self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: richard <richard@test.test>
+To: issue_tracker@your.tracker.email.domain.example
+Message-Id: <followup_dummy_id>
+Subject: Re: Testing...
+
+This is a followup
+'''), nodeid)
+
+    def testFollowupTitleMatchNeverInterval(self):
+        nodeid = self.doNewIssue()
+        # force failure of the interval
+        time.sleep(2)
+        self.db.config.MAILGW_SUBJECT_CONTENT_MATCH = 'creation 00:00:01'
+        self.assertNotEqual(self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: richard <richard@test.test>
+To: issue_tracker@your.tracker.email.domain.example
+Message-Id: <followup_dummy_id>
+Subject: Re: Testing...
+
+This is a followup
+'''), nodeid)
+
+
+    def testFollowupTitleMatchInterval(self):
+        nodeid = self.doNewIssue()
+        self.db.config.MAILGW_SUBJECT_CONTENT_MATCH = 'creation +1d'
+        self.assertEqual(self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: richard <richard@test.test>
+To: issue_tracker@your.tracker.email.domain.example
+Message-Id: <followup_dummy_id>
+Subject: Re: Testing...
+
+This is a followup
+'''), nodeid)
+
 
     def testFollowupNosyAuthor(self):
         self.doNewIssue()
-        self.db.config.ADD_AUTHOR_TO_NOSY = self.instance.ADD_AUTHOR_TO_NOSY = 'yes'
-        message = cStringIO.StringIO('''Content-Type: text/plain;
+        self.db.config.ADD_AUTHOR_TO_NOSY = 'yes'
+        self._handle_mail('''Content-Type: text/plain;
   charset="iso-8859-1"
-From: john@test
+From: john@test.test
 To: issue_tracker@your.tracker.email.domain.example
 Message-Id: <followup_dummy_id>
 In-Reply-To: <dummy_test_message_id>
@@ -386,95 +734,92 @@ Subject: [issue1] Testing...
 
 This is a followup
 ''')
-        handler = self.instance.MailGW(self.instance, self.db)
-        handler.trapExceptions = 0
-        handler.main(message)
 
-        self.compareStrings(open(os.environ['SENDMAILDEBUG']).read(),
+        self.compareMessages(self._get_mail(),
 '''FROM: roundup-admin@your.tracker.email.domain.example
-TO: chef@bork.bork.bork, richard@test
-Content-Type: text/plain
+TO: chef@bork.bork.bork, richard@test.test
+Content-Type: text/plain; charset="utf-8"
 Subject: [issue1] Testing...
-To: chef@bork.bork.bork, richard@test
-From: "john" <issue_tracker@your.tracker.email.domain.example>
-Reply-To: "Roundup issue tracker" <issue_tracker@your.tracker.email.domain.example>
+To: chef@bork.bork.bork, richard@test.test
+From: John Doe <issue_tracker@your.tracker.email.domain.example>
+Reply-To: Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
 MIME-Version: 1.0
 Message-Id: <followup_dummy_id>
 In-Reply-To: <dummy_test_message_id>
 X-Roundup-Name: Roundup issue tracker
+X-Roundup-Loop: hello
+X-Roundup-Issue-Status: chatting
 Content-Transfer-Encoding: quoted-printable
 
 
-john <john@test> added the comment:
+John Doe <john@test.test> added the comment:
 
 This is a followup
 
-
 ----------
 nosy: +john
 status: unread -> chatting
-_________________________________________________________________________
-"Roundup issue tracker" <issue_tracker@your.tracker.email.domain.example>
-http://your.tracker.url.example/issue1
-_________________________________________________________________________
+
+_______________________________________________________________________
+Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
+<http://tracker.example/cgi-bin/roundup.cgi/bugs/issue1>
+_______________________________________________________________________
 
 ''')
 
     def testFollowupNosyRecipients(self):
         self.doNewIssue()
-        self.db.config.ADD_RECIPIENTS_TO_NOSY = self.instance.ADD_RECIPIENTS_TO_NOSY = 'yes'
-        message = cStringIO.StringIO('''Content-Type: text/plain;
+        self.db.config.ADD_RECIPIENTS_TO_NOSY = 'yes'
+        self._handle_mail('''Content-Type: text/plain;
   charset="iso-8859-1"
-From: richard@test
+From: richard@test.test
 To: issue_tracker@your.tracker.email.domain.example
-Cc: john@test
+Cc: john@test.test
 Message-Id: <followup_dummy_id>
 In-Reply-To: <dummy_test_message_id>
 Subject: [issue1] Testing...
 
 This is a followup
 ''')
-        handler = self.instance.MailGW(self.instance, self.db)
-        handler.trapExceptions = 0
-        handler.main(message)
-
-        self.compareStrings(open(os.environ['SENDMAILDEBUG']).read(),
+        self.compareMessages(self._get_mail(),
 '''FROM: roundup-admin@your.tracker.email.domain.example
 TO: chef@bork.bork.bork
-Content-Type: text/plain
+Content-Type: text/plain; charset="utf-8"
 Subject: [issue1] Testing...
 To: chef@bork.bork.bork
-From: "richard" <issue_tracker@your.tracker.email.domain.example>
-Reply-To: "Roundup issue tracker" <issue_tracker@your.tracker.email.domain.example>
+From: richard <issue_tracker@your.tracker.email.domain.example>
+Reply-To: Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
 MIME-Version: 1.0
 Message-Id: <followup_dummy_id>
 In-Reply-To: <dummy_test_message_id>
 X-Roundup-Name: Roundup issue tracker
+X-Roundup-Loop: hello
+X-Roundup-Issue-Status: chatting
 Content-Transfer-Encoding: quoted-printable
 
 
-richard <richard@test> added the comment:
+richard <richard@test.test> added the comment:
 
 This is a followup
 
-
 ----------
 nosy: +john
 status: unread -> chatting
-_________________________________________________________________________
-"Roundup issue tracker" <issue_tracker@your.tracker.email.domain.example>
-http://your.tracker.url.example/issue1
-_________________________________________________________________________
+
+_______________________________________________________________________
+Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
+<http://tracker.example/cgi-bin/roundup.cgi/bugs/issue1>
+_______________________________________________________________________
 
 ''')
 
     def testFollowupNosyAuthorAndCopy(self):
         self.doNewIssue()
-        self.db.config.ADD_AUTHOR_TO_NOSY = self.instance.ADD_AUTHOR_TO_NOSY = 'yes'
+        self.db.config.ADD_AUTHOR_TO_NOSY = 'yes'
         self.db.config.MESSAGES_TO_AUTHOR = 'yes'
-        message = cStringIO.StringIO('''Content-Type: text/plain;
+        self._handle_mail('''Content-Type: text/plain;
   charset="iso-8859-1"
-From: john@test
+From: john@test.test
 To: issue_tracker@your.tracker.email.domain.example
 Message-Id: <followup_dummy_id>
 In-Reply-To: <dummy_test_message_id>
@@ -482,46 +827,44 @@ Subject: [issue1] Testing...
 
 This is a followup
 ''')
-        handler = self.instance.MailGW(self.instance, self.db)
-        handler.trapExceptions = 0
-        handler.main(message)
-
-        self.compareStrings(open(os.environ['SENDMAILDEBUG']).read(),
+        self.compareMessages(self._get_mail(),
 '''FROM: roundup-admin@your.tracker.email.domain.example
-TO: chef@bork.bork.bork, john@test, richard@test
-Content-Type: text/plain
+TO: chef@bork.bork.bork, john@test.test, richard@test.test
+Content-Type: text/plain; charset="utf-8"
 Subject: [issue1] Testing...
-To: chef@bork.bork.bork, john@test, richard@test
-From: "john" <issue_tracker@your.tracker.email.domain.example>
-Reply-To: "Roundup issue tracker" <issue_tracker@your.tracker.email.domain.example>
+To: chef@bork.bork.bork, john@test.test, richard@test.test
+From: John Doe <issue_tracker@your.tracker.email.domain.example>
+Reply-To: Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
 MIME-Version: 1.0
 Message-Id: <followup_dummy_id>
 In-Reply-To: <dummy_test_message_id>
 X-Roundup-Name: Roundup issue tracker
+X-Roundup-Loop: hello
+X-Roundup-Issue-Status: chatting
 Content-Transfer-Encoding: quoted-printable
 
 
-john <john@test> added the comment:
+John Doe <john@test.test> added the comment:
 
 This is a followup
 
-
 ----------
 nosy: +john
 status: unread -> chatting
-_________________________________________________________________________
-"Roundup issue tracker" <issue_tracker@your.tracker.email.domain.example>
-http://your.tracker.url.example/issue1
-_________________________________________________________________________
+
+_______________________________________________________________________
+Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
+<http://tracker.example/cgi-bin/roundup.cgi/bugs/issue1>
+_______________________________________________________________________
 
 ''')
 
     def testFollowupNoNosyAuthor(self):
         self.doNewIssue()
-        self.instance.ADD_AUTHOR_TO_NOSY = 'no'
-        message = cStringIO.StringIO('''Content-Type: text/plain;
+        self.instance.config.ADD_AUTHOR_TO_NOSY = 'no'
+        self._handle_mail('''Content-Type: text/plain;
   charset="iso-8859-1"
-From: john@test
+From: john@test.test
 To: issue_tracker@your.tracker.email.domain.example
 Message-Id: <followup_dummy_id>
 In-Reply-To: <dummy_test_message_id>
@@ -529,119 +872,146 @@ Subject: [issue1] Testing...
 
 This is a followup
 ''')
-        handler = self.instance.MailGW(self.instance, self.db)
-        handler.trapExceptions = 0
-        handler.main(message)
-
-        self.compareStrings(open(os.environ['SENDMAILDEBUG']).read(),
+        self.compareMessages(self._get_mail(),
 '''FROM: roundup-admin@your.tracker.email.domain.example
-TO: chef@bork.bork.bork, richard@test
-Content-Type: text/plain
+TO: chef@bork.bork.bork, richard@test.test
+Content-Type: text/plain; charset="utf-8"
 Subject: [issue1] Testing...
-To: chef@bork.bork.bork, richard@test
-From: "john" <issue_tracker@your.tracker.email.domain.example>
-Reply-To: "Roundup issue tracker" <issue_tracker@your.tracker.email.domain.example>
+To: chef@bork.bork.bork, richard@test.test
+From: John Doe <issue_tracker@your.tracker.email.domain.example>
+Reply-To: Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
 MIME-Version: 1.0
 Message-Id: <followup_dummy_id>
 In-Reply-To: <dummy_test_message_id>
 X-Roundup-Name: Roundup issue tracker
+X-Roundup-Loop: hello
+X-Roundup-Issue-Status: chatting
 Content-Transfer-Encoding: quoted-printable
 
 
-john <john@test> added the comment:
+John Doe <john@test.test> added the comment:
 
 This is a followup
 
-
 ----------
 status: unread -> chatting
-_________________________________________________________________________
-"Roundup issue tracker" <issue_tracker@your.tracker.email.domain.example>
-http://your.tracker.url.example/issue1
-_________________________________________________________________________
+
+_______________________________________________________________________
+Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
+<http://tracker.example/cgi-bin/roundup.cgi/bugs/issue1>
+_______________________________________________________________________
 
 ''')
 
     def testFollowupNoNosyRecipients(self):
         self.doNewIssue()
-        self.instance.ADD_RECIPIENTS_TO_NOSY = 'no'
-        message = cStringIO.StringIO('''Content-Type: text/plain;
+        self.instance.config.ADD_RECIPIENTS_TO_NOSY = 'no'
+        self._handle_mail('''Content-Type: text/plain;
   charset="iso-8859-1"
-From: richard@test
+From: richard@test.test
 To: issue_tracker@your.tracker.email.domain.example
-Cc: john@test
+Cc: john@test.test
 Message-Id: <followup_dummy_id>
 In-Reply-To: <dummy_test_message_id>
 Subject: [issue1] Testing...
 
 This is a followup
 ''')
-        handler = self.instance.MailGW(self.instance, self.db)
-        handler.trapExceptions = 0
-        handler.main(message)
-
-        self.compareStrings(open(os.environ['SENDMAILDEBUG']).read(),
+        self.compareMessages(self._get_mail(),
 '''FROM: roundup-admin@your.tracker.email.domain.example
 TO: chef@bork.bork.bork
-Content-Type: text/plain
+Content-Type: text/plain; charset="utf-8"
 Subject: [issue1] Testing...
 To: chef@bork.bork.bork
-From: "richard" <issue_tracker@your.tracker.email.domain.example>
-Reply-To: "Roundup issue tracker" <issue_tracker@your.tracker.email.domain.example>
+From: richard <issue_tracker@your.tracker.email.domain.example>
+Reply-To: Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
 MIME-Version: 1.0
 Message-Id: <followup_dummy_id>
 In-Reply-To: <dummy_test_message_id>
 X-Roundup-Name: Roundup issue tracker
+X-Roundup-Loop: hello
+X-Roundup-Issue-Status: chatting
 Content-Transfer-Encoding: quoted-printable
 
 
-richard <richard@test> added the comment:
+richard <richard@test.test> added the comment:
 
 This is a followup
 
-
 ----------
 status: unread -> chatting
-_________________________________________________________________________
-"Roundup issue tracker" <issue_tracker@your.tracker.email.domain.example>
-http://your.tracker.url.example/issue1
-_________________________________________________________________________
+
+_______________________________________________________________________
+Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
+<http://tracker.example/cgi-bin/roundup.cgi/bugs/issue1>
+_______________________________________________________________________
+
+''')
+
+    def testFollowupEmptyMessage(self):
+        self.doNewIssue()
+
+        self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: richard <richard@test.test>
+To: issue_tracker@your.tracker.email.domain.example
+Message-Id: <followup_dummy_id>
+In-Reply-To: <dummy_test_message_id>
+Subject: [issue1] Testing... [assignedto=mary; nosy=+john]
+
+''')
+        l = self.db.issue.get('1', 'nosy')
+        l.sort()
+        self.assertEqual(l, [self.chef_id, self.richard_id, self.mary_id,
+            self.john_id])
+
+        # should be no file created (ie. no message)
+        assert not os.path.exists(SENDMAILDEBUG)
+
+    def testFollowupEmptyMessageNoSubject(self):
+        self.doNewIssue()
+
+        self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: richard <richard@test.test>
+To: issue_tracker@your.tracker.email.domain.example
+Message-Id: <followup_dummy_id>
+In-Reply-To: <dummy_test_message_id>
+Subject: [issue1] [assignedto=mary; nosy=+john]
 
 ''')
+        l = self.db.issue.get('1', 'nosy')
+        l.sort()
+        self.assertEqual(l, [self.chef_id, self.richard_id, self.mary_id,
+            self.john_id])
+
+        # should be no file created (ie. no message)
+        assert not os.path.exists(SENDMAILDEBUG)
 
     def testNosyRemove(self):
         self.doNewIssue()
 
-        message = cStringIO.StringIO('''Content-Type: text/plain;
+        self._handle_mail('''Content-Type: text/plain;
   charset="iso-8859-1"
-From: richard <richard@test>
+From: richard <richard@test.test>
 To: issue_tracker@your.tracker.email.domain.example
 Message-Id: <followup_dummy_id>
 In-Reply-To: <dummy_test_message_id>
 Subject: [issue1] Testing... [nosy=-richard]
 
 ''')
-        handler = self.instance.MailGW(self.instance, self.db)
-        handler.trapExceptions = 0
-        handler.main(message)
         l = self.db.issue.get('1', 'nosy')
         l.sort()
-        self.assertEqual(l, ['3'])
+        self.assertEqual(l, [self.chef_id])
 
         # NO NOSY MESSAGE SHOULD BE SENT!
-        self.assert_(not os.path.exists(os.environ['SENDMAILDEBUG']))
+        assert not os.path.exists(SENDMAILDEBUG)
 
     def testNewUserAuthor(self):
-        # first without the permission
-        # heh... just ignore the API for a second ;)
-        self.db.security.role['Anonymous'].permissions=[]
-        anonid = self.db.user.lookup('anonymous')
-        self.db.user.set(anonid, roles='Anonymous')
 
-        self.db.security.hasPermission('Email Registration', anonid)
         l = self.db.user.list()
         l.sort()
-        s = '''Content-Type: text/plain;
+        message = '''Content-Type: text/plain;
   charset="iso-8859-1"
 From: fubar <fubar@bork.bork.bork>
 To: issue_tracker@your.tracker.email.domain.example
@@ -650,30 +1020,165 @@ Subject: [issue] Testing...
 
 This is a test submission of a new issue.
 '''
-        message = cStringIO.StringIO(s)
-        handler = self.instance.MailGW(self.instance, self.db)
-        handler.trapExceptions = 0
-        self.assertRaises(Unauthorized, handler.main, message)
+        def hook (db, **kw):
+            ''' set up callback for db open '''
+            db.security.role['anonymous'].permissions=[]
+            anonid = db.user.lookup('anonymous')
+            db.user.set(anonid, roles='Anonymous')
+        self.instance.schema_hook = hook
+        try:
+            self._handle_mail(message)
+        except Unauthorized, value:
+            body_diff = self.compareMessages(str(value), """
+You are not a registered user.
+
+Unknown address: fubar@bork.bork.bork
+""")
+
+            assert not body_diff, body_diff
+
+        else:
+            raise AssertionError, "Unathorized not raised when handling mail"
+
+
+        def hook (db, **kw):
+            ''' set up callback for db open '''
+            # Add Web Access role to anonymous, and try again to make sure
+            # we get a "please register at:" message this time.
+            p = [
+                db.security.getPermission('Create', 'user'),
+                db.security.getPermission('Web Access', None),
+            ]
+            db.security.role['anonymous'].permissions=p
+        self.instance.schema_hook = hook
+        try:
+            self._handle_mail(message)
+        except Unauthorized, value:
+            body_diff = self.compareMessages(str(value), """
+You are not a registered user. Please register at:
+
+http://tracker.example/cgi-bin/roundup.cgi/bugs/user?template=register
+
+...before sending mail to the tracker.
+
+Unknown address: fubar@bork.bork.bork
+""")
+
+            assert not body_diff, body_diff
+
+        else:
+            raise AssertionError, "Unathorized not raised when handling mail"
+
+        # Make sure list of users is the same as before.
         m = self.db.user.list()
         m.sort()
         self.assertEqual(l, m)
 
-        # now with the permission
-        p = self.db.security.getPermission('Email Registration')
-        self.db.security.role['Anonymous'].permissions=[p]
-        handler = self.instance.MailGW(self.instance, self.db)
-        handler.trapExceptions = 0
-        message = cStringIO.StringIO(s)
-        handler.main(message)
+        def hook (db, **kw):
+            ''' set up callback for db open '''
+            # now with the permission
+            p = [
+                db.security.getPermission('Create', 'user'),
+                db.security.getPermission('Email Access', None),
+            ]
+            db.security.role['anonymous'].permissions=p
+        self.instance.schema_hook = hook
+        self._handle_mail(message)
         m = self.db.user.list()
         m.sort()
         self.assertNotEqual(l, m)
 
+    def testNewUserAuthorHighBit(self):
+        l = set(self.db.user.list())
+        # From: name has Euro symbol in it
+        message = '''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: =?utf8?b?SOKCrGxsbw==?= <fubar@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Message-Id: <dummy_test_message_id>
+Subject: [issue] Testing...
+
+This is a test submission of a new issue.
+'''
+        def hook (db, **kw):
+            ''' set up callback for db open '''
+            p = [
+                db.security.getPermission('Create', 'user'),
+                db.security.getPermission('Email Access', None),
+            ]
+            db.security.role['anonymous'].permissions=p
+        self.instance.schema_hook = hook
+        self._handle_mail(message)
+        m = set(self.db.user.list())
+        new = list(m - l)[0]
+        name = self.db.user.get(new, 'realname')
+        self.assertEquals(name, 'H€llo')
+
+    def testUnknownUser(self):
+        l = set(self.db.user.list())
+        message = '''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Nonexisting User <nonexisting@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Message-Id: <dummy_test_message_id>
+Subject: [issue] Testing nonexisting user...
+
+This is a test submission of a new issue.
+'''
+        self.db.close()
+        handler = self.instance.MailGW(self.instance)
+        # we want a bounce message:
+        handler.trapExceptions = 1
+        ret = handler.main(StringIO(message))
+        self.compareMessages(self._get_mail(),
+'''FROM: Roundup issue tracker <roundup-admin@your.tracker.email.domain.example>
+TO: nonexisting@bork.bork.bork
+From nobody Tue Jul 14 12:04:11 2009
+Content-Type: multipart/mixed; boundary="===============0639262320=="
+MIME-Version: 1.0
+Subject: Failed issue tracker submission
+To: nonexisting@bork.bork.bork
+From: Roundup issue tracker <roundup-admin@your.tracker.email.domain.example>
+Date: Tue, 14 Jul 2009 12:04:11 +0000
+Precedence: bulk
+X-Roundup-Name: Roundup issue tracker
+X-Roundup-Loop: hello
+X-Roundup-Version: 1.4.8
+MIME-Version: 1.0
+
+--===============0639262320==
+Content-Type: text/plain; charset="us-ascii"
+MIME-Version: 1.0
+Content-Transfer-Encoding: 7bit
+
+
+
+You are not a registered user.
+
+Unknown address: nonexisting@bork.bork.bork
+
+--===============0639262320==
+Content-Type: text/plain; charset="us-ascii"
+MIME-Version: 1.0
+Content-Transfer-Encoding: 7bit
+
+Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Nonexisting User <nonexisting@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Message-Id: <dummy_test_message_id>
+Subject: [issue] Testing nonexisting user...
+
+This is a test submission of a new issue.
+
+--===============0639262320==--
+''')
+
     def testEnc01(self):
         self.doNewIssue()
-        message = cStringIO.StringIO('''Content-Type: text/plain;
+        self._handle_mail('''Content-Type: text/plain;
   charset="iso-8859-1"
-From: mary <mary@test>
+From: mary <mary@test.test>
 To: issue_tracker@your.tracker.email.domain.example
 Message-Id: <followup_dummy_id>
 In-Reply-To: <dummy_test_message_id>
@@ -685,42 +1190,89 @@ Content-Transfer-Encoding: quoted-printable
 A message with encoding (encoded oe =F6)
 
 ''')
-        handler = self.instance.MailGW(self.instance, self.db)
-        handler.trapExceptions = 0
-        handler.main(message)
-        self.compareStrings(open(os.environ['SENDMAILDEBUG']).read(),
+        self.compareMessages(self._get_mail(),
 '''FROM: roundup-admin@your.tracker.email.domain.example
-TO: chef@bork.bork.bork, richard@test
-Content-Type: text/plain
+TO: chef@bork.bork.bork, richard@test.test
+Content-Type: text/plain; charset="utf-8"
 Subject: [issue1] Testing...
-To: chef@bork.bork.bork, richard@test
-From: "mary" <issue_tracker@your.tracker.email.domain.example>
-Reply-To: "Roundup issue tracker" <issue_tracker@your.tracker.email.domain.example>
+To: chef@bork.bork.bork, richard@test.test
+From: "Contrary, Mary" <issue_tracker@your.tracker.email.domain.example>
+Reply-To: Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
 MIME-Version: 1.0
 Message-Id: <followup_dummy_id>
 In-Reply-To: <dummy_test_message_id>
 X-Roundup-Name: Roundup issue tracker
+X-Roundup-Loop: hello
+X-Roundup-Issue-Status: chatting
 Content-Transfer-Encoding: quoted-printable
 
 
-mary <mary@test> added the comment:
+Contrary, Mary <mary@test.test> added the comment:
+
+A message with encoding (encoded oe =C3=B6)
+
+----------
+status: unread -> chatting
+
+_______________________________________________________________________
+Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
+<http://tracker.example/cgi-bin/roundup.cgi/bugs/issue1>
+_______________________________________________________________________
+''')
+
+    def testEncNonUTF8(self):
+        self.doNewIssue()
+        self.instance.config.EMAIL_CHARSET = 'iso-8859-1'
+        self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: mary <mary@test.test>
+To: issue_tracker@your.tracker.email.domain.example
+Message-Id: <followup_dummy_id>
+In-Reply-To: <dummy_test_message_id>
+Subject: [issue1] Testing...
+Content-Type: text/plain;
+        charset="iso-8859-1"
+Content-Transfer-Encoding: quoted-printable
+
+A message with encoding (encoded oe =F6)
+
+''')
+        self.compareMessages(self._get_mail(),
+'''FROM: roundup-admin@your.tracker.email.domain.example
+TO: chef@bork.bork.bork, richard@test.test
+Content-Type: text/plain; charset="iso-8859-1"
+Subject: [issue1] Testing...
+To: chef@bork.bork.bork, richard@test.test
+From: "Contrary, Mary" <issue_tracker@your.tracker.email.domain.example>
+Reply-To: Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
+MIME-Version: 1.0
+Message-Id: <followup_dummy_id>
+In-Reply-To: <dummy_test_message_id>
+X-Roundup-Name: Roundup issue tracker
+X-Roundup-Loop: hello
+X-Roundup-Issue-Status: chatting
+Content-Transfer-Encoding: quoted-printable
+
+
+Contrary, Mary <mary@test.test> added the comment:
 
 A message with encoding (encoded oe =F6)
 
 ----------
 status: unread -> chatting
-_________________________________________________________________________
-"Roundup issue tracker" <issue_tracker@your.tracker.email.domain.example>
-http://your.tracker.url.example/issue1
-_________________________________________________________________________
+
+_______________________________________________________________________
+Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
+<http://tracker.example/cgi-bin/roundup.cgi/bugs/issue1>
+_______________________________________________________________________
 ''')
 
 
     def testMultipartEnc01(self):
         self.doNewIssue()
-        message = cStringIO.StringIO('''Content-Type: text/plain;
+        self._handle_mail('''Content-Type: text/plain;
   charset="iso-8859-1"
-From: mary <mary@test>
+From: mary <mary@test.test>
 To: issue_tracker@your.tracker.email.domain.example
 Message-Id: <followup_dummy_id>
 In-Reply-To: <dummy_test_message_id>
@@ -739,184 +1291,665 @@ Content-Transfer-Encoding: quoted-printable
 A message with first part encoded (encoded oe =F6)
 
 ''')
-        handler = self.instance.MailGW(self.instance, self.db)
-        handler.trapExceptions = 0
-        handler.main(message)
-        self.compareStrings(open(os.environ['SENDMAILDEBUG']).read(),
+        self.compareMessages(self._get_mail(),
 '''FROM: roundup-admin@your.tracker.email.domain.example
-TO: chef@bork.bork.bork, richard@test
-Content-Type: text/plain
+TO: chef@bork.bork.bork, richard@test.test
+Content-Type: text/plain; charset="utf-8"
 Subject: [issue1] Testing...
-To: chef@bork.bork.bork, richard@test
-From: "mary" <issue_tracker@your.tracker.email.domain.example>
-Reply-To: "Roundup issue tracker" <issue_tracker@your.tracker.email.domain.example>
+To: chef@bork.bork.bork, richard@test.test
+From: "Contrary, Mary" <issue_tracker@your.tracker.email.domain.example>
+Reply-To: Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
 MIME-Version: 1.0
 Message-Id: <followup_dummy_id>
 In-Reply-To: <dummy_test_message_id>
 X-Roundup-Name: Roundup issue tracker
+X-Roundup-Loop: hello
+X-Roundup-Issue-Status: chatting
 Content-Transfer-Encoding: quoted-printable
 
 
-mary <mary@test> added the comment:
+Contrary, Mary <mary@test.test> added the comment:
 
-A message with first part encoded (encoded oe =F6)
+A message with first part encoded (encoded oe =C3=B6)
 
 ----------
 status: unread -> chatting
-_________________________________________________________________________
-"Roundup issue tracker" <issue_tracker@your.tracker.email.domain.example>
-http://your.tracker.url.example/issue1
-_________________________________________________________________________
+
+_______________________________________________________________________
+Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
+<http://tracker.example/cgi-bin/roundup.cgi/bugs/issue1>
+_______________________________________________________________________
 ''')
 
-class ExtMailgwTestCase(MailgwTestCase):
-    schema = 'extended'
+    def testContentDisposition(self):
+        self.doNewIssue()
+        self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: mary <mary@test.test>
+To: issue_tracker@your.tracker.email.domain.example
+Message-Id: <followup_dummy_id>
+In-Reply-To: <dummy_test_message_id>
+Subject: [issue1] Testing...
+Content-Type: multipart/mixed; boundary="bCsyhTFzCvuiizWE"
+Content-Disposition: inline
 
-def suite():
-    l = [unittest.makeSuite(MailgwTestCase),
-         unittest.makeSuite(ExtMailgwTestCase, 'test')
-    ]
-    return unittest.TestSuite(l)
 
+--bCsyhTFzCvuiizWE
+Content-Type: text/plain; charset=us-ascii
+Content-Disposition: inline
 
-#
-# $Log: not supported by cvs2svn $
-# Revision 1.24  2002/07/26 08:27:00  richard
-# Very close now. The cgi and mailgw now use the new security API. The two
-# templates have been migrated to that setup. Lots of unit tests. Still some
-# issue in the web form for editing Roles assigned to users.
-#
-# Revision 1.23  2002/07/14 02:02:43  richard
-# Fixed the unit tests for the new multilist controls in the mailgw
-#
-# Revision 1.22  2002/07/09 01:21:24  richard
-# Added ability for unit tests to turn off exception handling in mailgw so
-# that exceptions are reported earlier (and hence make sense).
-#
-# Revision 1.21  2002/06/18 03:59:59  dman13
-# Updated message strings to match the RFC822 address quoting performed
-# by the 'email' and 'rfc822' modules.  The verification really should
-# use a RFC2822 message parser rather than literal string comparisions
-# to allow for legal variations in messages.
-#
-# Revision 1.20  2002/05/29 01:16:17  richard
-# Sorry about this huge checkin! It's fixing a lot of related stuff in one go
-# though.
-#
-# . #541941 ] changing multilink properties by mail
-# . #526730 ] search for messages capability
-# . #505180 ] split MailGW.handle_Message
-#   - also changed cgi client since it was duplicating the functionality
-# . build htmlbase if tests are run using CVS checkout (removed note from
-#   installation.txt)
-# . don't create an empty message on email issue creation if the email is empty
-#
-# Revision 1.19  2002/05/23 04:26:05  richard
-# 'I must run unit tests before committing\n' * 100
-#
-# Revision 1.18  2002/05/15 03:27:16  richard
-#  . fixed SCRIPT_NAME in ZRoundup for instances not at top level of Zope
-#    (thanks dman)
-#  . fixed some sorting issues that were breaking some unit tests under py2.2
-#  . mailgw test output dir was confusing the init test (but only on 2.2 *shrug*)
-#
-# fixed bug in the init unit test that meant only the bsddb test ran if it
-# could (it clobbered the anydbm test)
-#
-# Revision 1.17  2002/05/02 07:56:34  richard
-# . added option to automatically add the authors and recipients of messages
-#   to the nosy lists with the options ADD_AUTHOR_TO_NOSY (default 'new') and
-#   ADD_RECIPIENTS_TO_NOSY (default 'new'). These settings emulate the current
-#   behaviour. Setting them to 'yes' will add the author/recipients to the nosy
-#   on messages that create issues and followup messages.
-# . added missing documentation for a few of the config option values
-#
-# Revision 1.16  2002/03/19 21:58:11  grubert
-#  . for python2.1 test_mailgw compareString allows an extra trailing empty line (for quopri.
-#
-# Revision 1.15  2002/03/19 06:37:00  richard
-# Made the email checking spit out a diff - much easier to spot the problem!
-#
-# Revision 1.14  2002/03/18 18:32:00  rochecompaan
-# All messages sent to the nosy list are now encoded as quoted-printable.
-#
-# Revision 1.13  2002/02/15 07:08:45  richard
-#  . Alternate email addresses are now available for users. See the MIGRATION
-#    file for info on how to activate the feature.
-#
-# Revision 1.12  2002/02/15 00:13:38  richard
-#  . #503204 ] mailgw needs a default class
-#     - partially done - the setting of additional properties can wait for a
-#       better configuration system.
-#
-# Revision 1.11  2002/02/14 23:38:12  richard
-# Fixed the unit tests for the mailgw re: the x-roundup-name header.
-# Also made the test runner more user-friendly:
-#   ./run_tests            - detect all tests in test/test_<name>.py and run them
-#   ./run_tests <name>     - run only test/test_<name>.py
-# eg ./run_tests mailgw    - run the mailgw test from test/test_mailgw.py
-#
-# Revision 1.10  2002/02/12 08:08:55  grubert
-#  . Clean up mail handling, multipart handling.
-#
-# Revision 1.9  2002/02/05 14:15:29  grubert
-#  . respect encodings in non multipart messages.
-#
-# Revision 1.8  2002/02/04 09:40:21  grubert
-#  . add test for multipart messages with first part being encoded.
-#
-# Revision 1.7  2002/01/22 11:54:45  rochecompaan
-# Fixed status change in mail gateway.
-#
-# Revision 1.6  2002/01/21 10:05:48  rochecompaan
-# Feature:
-#  . the mail gateway now responds with an error message when invalid
-#    values for arguments are specified for link or multilink properties
-#  . modified unit test to check nosy and assignedto when specified as
-#    arguments
-#
-# Fixed:
-#  . fixed setting nosy as argument in subject line
-#
-# Revision 1.5  2002/01/15 00:12:40  richard
-# #503340 ] creating issue with [asignedto=p.ohly]
-#
-# Revision 1.4  2002/01/14 07:12:15  richard
-# removed file writing from tests...
-#
-# Revision 1.3  2002/01/14 02:20:15  richard
-#  . changed all config accesses so they access either the instance or the
-#    config attriubute on the db. This means that all config is obtained from
-#    instance_config instead of the mish-mash of classes. This will make
-#    switching to a ConfigParser setup easier too, I hope.
-#
-# At a minimum, this makes migration a _little_ easier (a lot easier in the
-# 0.5.0 switch, I hope!)
-#
-# Revision 1.2  2002/01/11 23:22:29  richard
-#  . #502437 ] rogue reactor and unittest
-#    in short, the nosy reactor was modifying the nosy list. That code had
-#    been there for a long time, and I suspsect it was there because we
-#    weren't generating the nosy list correctly in other places of the code.
-#    We're now doing that, so the nosy-modifying code can go away from the
-#    nosy reactor.
-#
-# Revision 1.1  2002/01/02 02:31:38  richard
-# Sorry for the huge checkin message - I was only intending to implement #496356
-# but I found a number of places where things had been broken by transactions:
-#  . modified ROUNDUPDBSENDMAILDEBUG to be SENDMAILDEBUG and hold a filename
-#    for _all_ roundup-generated smtp messages to be sent to.
-#  . the transaction cache had broken the roundupdb.Class set() reactors
-#  . newly-created author users in the mailgw weren't being committed to the db
-#
-# Stuff that made it into CHANGES.txt (ie. the stuff I was actually working
-# on when I found that stuff :):
-#  . #496356 ] Use threading in messages
-#  . detectors were being registered multiple times
-#  . added tests for mailgw
-#  . much better attaching of erroneous messages in the mail gateway
-#
-#
-#
-#
-# vim: set filetype=python ts=4 sw=4 et si
+test attachment binary
+
+--bCsyhTFzCvuiizWE
+Content-Type: application/octet-stream
+Content-Disposition: attachment; filename="main.dvi"
+Content-Transfer-Encoding: base64
+
+SnVzdCBhIHRlc3QgAQo=
+
+--bCsyhTFzCvuiizWE--
+''')
+        messages = self.db.issue.get('1', 'messages')
+        messages.sort()
+        file = self.db.file.getnode (self.db.msg.get(messages[-1], 'files')[0])
+        self.assertEqual(file.name, 'main.dvi')
+        self.assertEqual(file.content, 'Just a test \001\n')
+
+    def testFollowupStupidQuoting(self):
+        self.doNewIssue()
+
+        self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: richard <richard@test.test>
+To: issue_tracker@your.tracker.email.domain.example
+Message-Id: <followup_dummy_id>
+In-Reply-To: <dummy_test_message_id>
+Subject: Re: "[issue1] Testing... "
+
+This is a followup
+''')
+        self.compareMessages(self._get_mail(),
+'''FROM: roundup-admin@your.tracker.email.domain.example
+TO: chef@bork.bork.bork
+Content-Type: text/plain; charset="utf-8"
+Subject: [issue1] Testing...
+To: chef@bork.bork.bork
+From: richard <issue_tracker@your.tracker.email.domain.example>
+Reply-To: Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
+MIME-Version: 1.0
+Message-Id: <followup_dummy_id>
+In-Reply-To: <dummy_test_message_id>
+X-Roundup-Name: Roundup issue tracker
+X-Roundup-Loop: hello
+X-Roundup-Issue-Status: chatting
+Content-Transfer-Encoding: quoted-printable
+
+
+richard <richard@test.test> added the comment:
+
+This is a followup
+
+----------
+status: unread -> chatting
+
+_______________________________________________________________________
+Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
+<http://tracker.example/cgi-bin/roundup.cgi/bugs/issue1>
+_______________________________________________________________________
+''')
+
+    def testEmailQuoting(self):
+        self.instance.config.EMAIL_KEEP_QUOTED_TEXT = 'no'
+        self.innerTestQuoting('''This is a followup
+''')
+
+    def testEmailQuotingRemove(self):
+        self.instance.config.EMAIL_KEEP_QUOTED_TEXT = 'yes'
+        self.innerTestQuoting('''Blah blah wrote:
+> Blah bklaskdfj sdf asdf jlaskdf skj sdkfjl asdf
+>  skdjlkjsdfalsdkfjasdlfkj dlfksdfalksd fj
+>
+
+This is a followup
+''')
+
+    def innerTestQuoting(self, expect):
+        nodeid = self.doNewIssue()
+
+        messages = self.db.issue.get(nodeid, 'messages')
+
+        self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: richard <richard@test.test>
+To: issue_tracker@your.tracker.email.domain.example
+Message-Id: <followup_dummy_id>
+In-Reply-To: <dummy_test_message_id>
+Subject: Re: [issue1] Testing...
+
+Blah blah wrote:
+> Blah bklaskdfj sdf asdf jlaskdf skj sdkfjl asdf
+>  skdjlkjsdfalsdkfjasdlfkj dlfksdfalksd fj
+>
+
+This is a followup
+''')
+        # figure the new message id
+        newmessages = self.db.issue.get(nodeid, 'messages')
+        for msg in messages:
+            newmessages.remove(msg)
+        messageid = newmessages[0]
+
+        self.compareMessages(self.db.msg.get(messageid, 'content'), expect)
+
+    def testUserLookup(self):
+        i = self.db.user.create(username='user1', address='user1@foo.com')
+        self.assertEqual(uidFromAddress(self.db, ('', 'user1@foo.com'), 0), i)
+        self.assertEqual(uidFromAddress(self.db, ('', 'USER1@foo.com'), 0), i)
+        i = self.db.user.create(username='user2', address='USER2@foo.com')
+        self.assertEqual(uidFromAddress(self.db, ('', 'USER2@foo.com'), 0), i)
+        self.assertEqual(uidFromAddress(self.db, ('', 'user2@foo.com'), 0), i)
+
+    def testUserAlternateLookup(self):
+        i = self.db.user.create(username='user1', address='user1@foo.com',
+                                alternate_addresses='user1@bar.com')
+        self.assertEqual(uidFromAddress(self.db, ('', 'user1@bar.com'), 0), i)
+        self.assertEqual(uidFromAddress(self.db, ('', 'USER1@bar.com'), 0), i)
+
+    def testUserCreate(self):
+        i = uidFromAddress(self.db, ('', 'user@foo.com'), 1)
+        self.assertNotEqual(uidFromAddress(self.db, ('', 'user@bar.com'), 1), i)
+
+    def testRFC2822(self):
+        ascii_header = "[issue243] This is a \"test\" - with 'quotation' marks"
+        unicode_header = '[issue244] \xd0\xb0\xd0\xbd\xd0\xb4\xd1\x80\xd0\xb5\xd0\xb9'
+        unicode_encoded = '=?utf-8?q?[issue244]_=D0=B0=D0=BD=D0=B4=D1=80=D0=B5=D0=B9?='
+        self.assertEqual(rfc2822.encode_header(ascii_header), ascii_header)
+        self.assertEqual(rfc2822.encode_header(unicode_header), unicode_encoded)
+
+    def testRegistrationConfirmation(self):
+        otk = "Aj4euk4LZSAdwePohj90SME5SpopLETL"
+        self.db.getOTKManager().set(otk, username='johannes')
+        self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Cc: richard@test.test
+Message-Id: <dummy_test_message_id>
+Subject: Re: Complete your registration to Roundup issue tracker
+ -- key %s
+
+This is a test confirmation of registration.
+''' % otk)
+        self.db.user.lookup('johannes')
+
+    def testFollowupOnNonIssue(self):
+        self.db.keyword.create(name='Foo')
+        self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: richard <richard@test.test>
+To: issue_tracker@your.tracker.email.domain.example
+Message-Id: <followup_dummy_id>
+In-Reply-To: <dummy_test_message_id>
+Subject: [keyword1] Testing... [name=Bar]
+
+''')
+        self.assertEqual(self.db.keyword.get('1', 'name'), 'Bar')
+
+    def testResentFrom(self):
+        nodeid = self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+Resent-From: mary <mary@test.test>
+To: issue_tracker@your.tracker.email.domain.example
+Cc: richard@test.test
+Message-Id: <dummy_test_message_id>
+Subject: [issue] Testing...
+
+This is a test submission of a new issue.
+''')
+        assert not os.path.exists(SENDMAILDEBUG)
+        l = self.db.issue.get(nodeid, 'nosy')
+        l.sort()
+        self.assertEqual(l, [self.richard_id, self.mary_id])
+        return nodeid
+
+    def testDejaVu(self):
+        self.assertRaises(IgnoreLoop, self._handle_mail,
+            '''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+X-Roundup-Loop: hello
+To: issue_tracker@your.tracker.email.domain.example
+Cc: richard@test.test
+Message-Id: <dummy_test_message_id>
+Subject: Re: [issue] Testing...
+
+Hi, I've been mis-configured to loop messages back to myself.
+''')
+
+    def testItsBulkStupid(self):
+        self.assertRaises(IgnoreBulk, self._handle_mail,
+            '''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+Precedence: bulk
+To: issue_tracker@your.tracker.email.domain.example
+Cc: richard@test.test
+Message-Id: <dummy_test_message_id>
+Subject: Re: [issue] Testing...
+
+Hi, I'm on holidays, and this is a dumb auto-responder.
+''')
+
+    def testAutoReplyEmailsAreIgnored(self):
+        self.assertRaises(IgnoreBulk, self._handle_mail,
+            '''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Cc: richard@test.test
+Message-Id: <dummy_test_message_id>
+Subject: Re: [issue] Out of office AutoReply: Back next week
+
+Hi, I am back in the office next week
+''')
+
+    def testNoSubject(self):
+        self.assertRaises(MailUsageError, self._handle_mail,
+            '''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Cc: richard@test.test
+Reply-To: chef@bork.bork.bork
+Message-Id: <dummy_test_message_id>
+
+''')
+
+    #
+    # TEST FOR INVALID DESIGNATOR HANDLING
+    #
+    def testInvalidDesignator(self):
+        self.assertRaises(MailUsageError, self._handle_mail,
+            '''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Subject: [frobulated] testing
+Cc: richard@test.test
+Reply-To: chef@bork.bork.bork
+Message-Id: <dummy_test_message_id>
+
+''')
+        self.assertRaises(MailUsageError, self._handle_mail,
+            '''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Subject: [issue12345] testing
+Cc: richard@test.test
+Reply-To: chef@bork.bork.bork
+Message-Id: <dummy_test_message_id>
+
+''')
+
+    def testInvalidClassLoose(self):
+        self.instance.config.MAILGW_SUBJECT_PREFIX_PARSING = 'loose'
+        nodeid = self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Subject: [frobulated] testing
+Cc: richard@test.test
+Reply-To: chef@bork.bork.bork
+Message-Id: <dummy_test_message_id>
+
+''')
+        assert not os.path.exists(SENDMAILDEBUG)
+        self.assertEqual(self.db.issue.get(nodeid, 'title'),
+            '[frobulated] testing')
+
+    def testInvalidClassLooseReply(self):
+        self.instance.config.MAILGW_SUBJECT_PREFIX_PARSING = 'loose'
+        nodeid = self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Subject: Re: [frobulated] testing
+Cc: richard@test.test
+Reply-To: chef@bork.bork.bork
+Message-Id: <dummy_test_message_id>
+
+''')
+        assert not os.path.exists(SENDMAILDEBUG)
+        self.assertEqual(self.db.issue.get(nodeid, 'title'),
+            '[frobulated] testing')
+
+    def testInvalidClassLoose(self):
+        self.instance.config.MAILGW_SUBJECT_PREFIX_PARSING = 'loose'
+        nodeid = self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Subject: [issue1234] testing
+Cc: richard@test.test
+Reply-To: chef@bork.bork.bork
+Message-Id: <dummy_test_message_id>
+
+''')
+        assert not os.path.exists(SENDMAILDEBUG)
+        self.assertEqual(self.db.issue.get(nodeid, 'title'),
+            '[issue1234] testing')
+
+    def testClassLooseOK(self):
+        self.instance.config.MAILGW_SUBJECT_PREFIX_PARSING = 'loose'
+        self.db.keyword.create(name='Foo')
+        nodeid = self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Subject: [keyword1] Testing... [name=Bar]
+Cc: richard@test.test
+Reply-To: chef@bork.bork.bork
+Message-Id: <dummy_test_message_id>
+
+''')
+        assert not os.path.exists(SENDMAILDEBUG)
+        self.assertEqual(self.db.keyword.get('1', 'name'), 'Bar')
+
+    def testClassStrictInvalid(self):
+        self.instance.config.MAILGW_SUBJECT_PREFIX_PARSING = 'strict'
+        self.instance.config.MAILGW_DEFAULT_CLASS = ''
+
+        message = '''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Subject: Testing...
+Cc: richard@test.test
+Reply-To: chef@bork.bork.bork
+Message-Id: <dummy_test_message_id>
+
+'''
+        self.assertRaises(MailUsageError, self._handle_mail, message)
+
+    def testClassStrictValid(self):
+        self.instance.config.MAILGW_SUBJECT_PREFIX_PARSING = 'strict'
+        self.instance.config.MAILGW_DEFAULT_CLASS = ''
+
+        nodeid = self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Subject: [issue] Testing...
+Cc: richard@test.test
+Reply-To: chef@bork.bork.bork
+Message-Id: <dummy_test_message_id>
+
+''')
+
+        assert not os.path.exists(SENDMAILDEBUG)
+        self.assertEqual(self.db.issue.get(nodeid, 'title'), 'Testing...')
+
+    #
+    # TEST FOR INVALID COMMANDS HANDLING
+    #
+    def testInvalidCommands(self):
+        self.assertRaises(MailUsageError, self._handle_mail,
+            '''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Subject: testing [frobulated]
+Cc: richard@test.test
+Reply-To: chef@bork.bork.bork
+Message-Id: <dummy_test_message_id>
+
+''')
+
+    def testInvalidCommandPassthrough(self):
+        self.instance.config.MAILGW_SUBJECT_SUFFIX_PARSING = 'none'
+        nodeid = self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Subject: testing [frobulated]
+Cc: richard@test.test
+Reply-To: chef@bork.bork.bork
+Message-Id: <dummy_test_message_id>
+
+''')
+        assert not os.path.exists(SENDMAILDEBUG)
+        self.assertEqual(self.db.issue.get(nodeid, 'title'),
+            'testing [frobulated]')
+
+    def testInvalidCommandPassthroughLoose(self):
+        self.instance.config.MAILGW_SUBJECT_SUFFIX_PARSING = 'loose'
+        nodeid = self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Subject: testing [frobulated]
+Cc: richard@test.test
+Reply-To: chef@bork.bork.bork
+Message-Id: <dummy_test_message_id>
+
+''')
+        assert not os.path.exists(SENDMAILDEBUG)
+        self.assertEqual(self.db.issue.get(nodeid, 'title'),
+            'testing [frobulated]')
+
+    def testInvalidCommandPassthroughLooseOK(self):
+        self.instance.config.MAILGW_SUBJECT_SUFFIX_PARSING = 'loose'
+        nodeid = self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Subject: testing [assignedto=mary]
+Cc: richard@test.test
+Reply-To: chef@bork.bork.bork
+Message-Id: <dummy_test_message_id>
+
+''')
+        assert not os.path.exists(SENDMAILDEBUG)
+        self.assertEqual(self.db.issue.get(nodeid, 'title'), 'testing')
+        self.assertEqual(self.db.issue.get(nodeid, 'assignedto'), self.mary_id)
+
+    def testCommandDelimiters(self):
+        self.instance.config.MAILGW_SUBJECT_SUFFIX_DELIMITERS = '{}'
+        nodeid = self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Subject: testing {assignedto=mary}
+Cc: richard@test.test
+Reply-To: chef@bork.bork.bork
+Message-Id: <dummy_test_message_id>
+
+''')
+        assert not os.path.exists(SENDMAILDEBUG)
+        self.assertEqual(self.db.issue.get(nodeid, 'title'), 'testing')
+        self.assertEqual(self.db.issue.get(nodeid, 'assignedto'), self.mary_id)
+
+    def testPrefixDelimiters(self):
+        self.instance.config.MAILGW_SUBJECT_SUFFIX_DELIMITERS = '{}'
+        self.db.keyword.create(name='Foo')
+        self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: richard <richard@test.test>
+To: issue_tracker@your.tracker.email.domain.example
+Message-Id: <followup_dummy_id>
+In-Reply-To: <dummy_test_message_id>
+Subject: {keyword1} Testing... {name=Bar}
+
+''')
+        assert not os.path.exists(SENDMAILDEBUG)
+        self.assertEqual(self.db.keyword.get('1', 'name'), 'Bar')
+
+    def testCommandDelimitersIgnore(self):
+        self.instance.config.MAILGW_SUBJECT_SUFFIX_DELIMITERS = '{}'
+        nodeid = self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Subject: testing [assignedto=mary]
+Cc: richard@test.test
+Reply-To: chef@bork.bork.bork
+Message-Id: <dummy_test_message_id>
+
+''')
+        assert not os.path.exists(SENDMAILDEBUG)
+        self.assertEqual(self.db.issue.get(nodeid, 'title'),
+            'testing [assignedto=mary]')
+        self.assertEqual(self.db.issue.get(nodeid, 'assignedto'), None)
+
+    def testReplytoMatch(self):
+        self.instance.config.MAILGW_SUBJECT_PREFIX_PARSING = 'loose'
+        nodeid = self.doNewIssue()
+        nodeid2 = self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Message-Id: <dummy_test_message_id2>
+In-Reply-To: <dummy_test_message_id>
+Subject: Testing...
+
+Followup message.
+''')
+
+        nodeid3 = self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Message-Id: <dummy_test_message_id3>
+In-Reply-To: <dummy_test_message_id2>
+Subject: Testing...
+
+Yet another message in the same thread/issue.
+''')
+
+        self.assertEqual(nodeid, nodeid2)
+        self.assertEqual(nodeid, nodeid3)
+
+    def testHelpSubject(self):
+        message = '''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Message-Id: <dummy_test_message_id2>
+In-Reply-To: <dummy_test_message_id>
+Subject: hElp
+
+
+'''
+        self.assertRaises(MailUsageHelp, self._handle_mail, message)
+
+    def testMaillistSubject(self):
+        self.instance.config.MAILGW_SUBJECT_SUFFIX_DELIMITERS = '[]'
+        self.db.keyword.create(name='Foo')
+        self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Subject: [mailinglist-name] [keyword1] Testing.. [name=Bar]
+Cc: richard@test.test
+Reply-To: chef@bork.bork.bork
+Message-Id: <dummy_test_message_id>
+
+''')
+
+        assert not os.path.exists(SENDMAILDEBUG)
+        self.assertEqual(self.db.keyword.get('1', 'name'), 'Bar')
+
+    def testUnknownPrefixSubject(self):
+        self.db.keyword.create(name='Foo')
+        self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Subject: VeryStrangeRe: [keyword1] Testing.. [name=Bar]
+Cc: richard@test.test
+Reply-To: chef@bork.bork.bork
+Message-Id: <dummy_test_message_id>
+
+''')
+
+        assert not os.path.exists(SENDMAILDEBUG)
+        self.assertEqual(self.db.keyword.get('1', 'name'), 'Bar')
+
+    def testIssueidLast(self):
+        nodeid1 = self.doNewIssue()
+        nodeid2 = self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: mary <mary@test.test>
+To: issue_tracker@your.tracker.email.domain.example
+Message-Id: <followup_dummy_id>
+In-Reply-To: <dummy_test_message_id>
+Subject: New title [issue1]
+
+This is a second followup
+''')
+
+        assert nodeid1 == nodeid2
+        self.assertEqual(self.db.issue.get(nodeid2, 'title'), "Testing...")
+
+    def testSecurityMessagePermissionContent(self):
+        id = self.doNewIssue()
+        issue = self.db.issue.getnode (id)
+        self.db.security.addRole(name='Nomsg')
+        self.db.security.addPermissionToRole('Nomsg', 'Email Access')
+        for cl in 'issue', 'file', 'keyword':
+            for p in 'View', 'Edit', 'Create':
+                self.db.security.addPermissionToRole('Nomsg', p, cl)
+        self.db.user.set(self.mary_id, roles='Nomsg')
+        nodeid = self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <chef@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Message-Id: <dummy_test_message_id>
+Subject: [issue%(id)s] Testing... [nosy=+mary]
+
+Just a test reply
+'''%locals())
+        assert os.path.exists(SENDMAILDEBUG)
+        self.compareMessages(self._get_mail(),
+'''FROM: roundup-admin@your.tracker.email.domain.example
+TO: chef@bork.bork.bork, richard@test.test
+Content-Type: text/plain; charset="utf-8"
+Subject: [issue1] Testing...
+To: richard@test.test
+From: "Bork, Chef" <issue_tracker@your.tracker.email.domain.example>
+Reply-To: Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
+MIME-Version: 1.0
+Message-Id: <dummy_test_message_id>
+X-Roundup-Name: Roundup issue tracker
+X-Roundup-Loop: hello
+X-Roundup-Issue-Status: chatting
+Content-Transfer-Encoding: quoted-printable
+
+
+Bork, Chef <chef@bork.bork.bork> added the comment:
+
+Just a test reply
+
+----------
+nosy: +mary
+status: unread -> chatting
+
+_______________________________________________________________________
+Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
+<http://tracker.example/cgi-bin/roundup.cgi/bugs/issue1>
+_______________________________________________________________________
+''')
+
+
+def test_suite():
+    suite = unittest.TestSuite()
+    suite.addTest(unittest.makeSuite(MailgwTestCase))
+    return suite
+
+if __name__ == '__main__':
+    runner = unittest.TextTestRunner()
+    unittest.main(testRunner=runner)
+
+# vim: set filetype=python sts=4 sw=4 et si :