Add some new encoding tests to mailgw:

- confirm that non-utf8 encodings work
- confirm that non-ASCII headers work (and they didn't but do now)

(roundup.rfc2822 is almost entirely gone from Roundup use now)

git-svn-id: http://svn.roundup-tracker.org/svnroot/roundup/roundup/trunk@4185 57a73879-2fb5-44c3-a270-3262357dd7e2

diff --git a/roundup/mailgw.py b/roundup/mailgw.py
index 5e8df2bace10af873fc66e03b82667b620bbb555..b675f8eeefabe481a8fc579a9b6fb58bfd054122 100644 (file)
--- a/roundup/mailgw.py
+++ b/roundup/mailgw.py
@@ -81,6 +81,8 @@ import string, re, os, mimetools, cStringIO, smtplib, socket, binascii, quopri
 import time, random, sys, logging
 import traceback, MimeWriter, rfc822
 
+from email.Header import decode_header
+
 from roundup import configuration, hyperdb, date, password, rfc2822, exceptions
 from roundup.mailer import Mailer, MessageSendError
 from roundup.i18n import _
@@ -261,9 +263,30 @@ class Message(mimetools.Message):
 
     def getheader(self, name, default=None):
         hdr = mimetools.Message.getheader(self, name, default)
+        if not hdr:
+            return ''
         if hdr:
             hdr = hdr.replace('\n','') # Inserted by rfc822.readheaders
-        return rfc2822.decode_header(hdr)
+        # historically this method has returned utf-8 encoded string
+        l = []
+        for part, encoding in decode_header(hdr):
+            if encoding:
+                part = part.decode(encoding)
+            l.append(part)
+        return ''.join([s.encode('utf-8') for s in l])
+
+    def getaddrlist(self, name):
+        # overload to decode the name part of the address
+        l = []
+        for (name, addr) in mimetools.Message.getaddrlist(self, name):
+            p = []
+            for part, encoding in decode_header(name):
+                if encoding:
+                    part = part.decode(encoding)
+                p.append(part)
+            name = ''.join([s.encode('utf-8') for s in p])
+            l.append((name, addr))
+        return l
 
     def getname(self):
         """Find an appropriate name for this message."""

diff --git a/test/test_mailgw.py b/test/test_mailgw.py
index d03f70d3c13cacc3ab246f8991e77df249e0c65e..7d7fba64c039b9c51d6202b552845c6cf71ac7a7 100644 (file)
--- a/test/test_mailgw.py
+++ b/test/test_mailgw.py
@@ -1,3 +1,4 @@
+# -*- encoding: utf-8 -*-
 #
 # Copyright (c) 2001 Richard Jones, richard@bofh.asn.au.
 # This module is free software, and you may redistribute it and/or modify
@@ -23,6 +24,7 @@ SENDMAILDEBUG = os.environ['SENDMAILDEBUG']
 from roundup.mailgw import MailGW, Unauthorized, uidFromAddress, \
     parseContent, IgnoreLoop, IgnoreBulk, MailUsageError, MailUsageHelp
 from roundup import init, instance, password, rfc2822, __version__
+from roundup.anypy.sets_ import set
 
 import db_test_base
 
@@ -1062,6 +1064,29 @@ Unknown address: fubar@bork.bork.bork
         m.sort()
         self.assertNotEqual(l, m)
 
+    def testNewUserAuthorHighBit(self):
+        l = set(self.db.user.list())
+        # From: name has Euro symbol in it
+        message = '''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: =?utf8?b?SOKCrGxsbw==?= <fubar@bork.bork.bork>
+To: issue_tracker@your.tracker.email.domain.example
+Message-Id: <dummy_test_message_id>
+Subject: [issue] Testing...
+
+This is a test submission of a new issue.
+'''
+        p = [
+            self.db.security.getPermission('Create', 'user'),
+            self.db.security.getPermission('Email Access', None),
+        ]
+        self.db.security.role['anonymous'].permissions=p
+        self._handle_mail(message)
+        m = set(self.db.user.list())
+        new = list(m - l)[0]
+        name = self.db.user.get(new, 'realname')
+        self.assertEquals(name, 'H€llo')
+
     def testEnc01(self):
         self.doNewIssue()
         self._handle_mail('''Content-Type: text/plain;
@@ -1102,6 +1127,53 @@ A message with encoding (encoded oe =C3=B6)
 ----------
 status: unread -> chatting
 
+_______________________________________________________________________
+Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
+<http://tracker.example/cgi-bin/roundup.cgi/bugs/issue1>
+_______________________________________________________________________
+''')
+
+    def testEncNonUTF8(self):
+        self.doNewIssue()
+        self.instance.config.EMAIL_CHARSET = 'iso-8859-1'
+        self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: mary <mary@test.test>
+To: issue_tracker@your.tracker.email.domain.example
+Message-Id: <followup_dummy_id>
+In-Reply-To: <dummy_test_message_id>
+Subject: [issue1] Testing...
+Content-Type: text/plain;
+        charset="iso-8859-1"
+Content-Transfer-Encoding: quoted-printable
+
+A message with encoding (encoded oe =F6)
+
+''')
+        self.compareMessages(self._get_mail(),
+'''FROM: roundup-admin@your.tracker.email.domain.example
+TO: chef@bork.bork.bork, richard@test.test
+Content-Type: text/plain; charset="iso-8859-1"
+Subject: [issue1] Testing...
+To: chef@bork.bork.bork, richard@test.test
+From: "Contrary, Mary" <issue_tracker@your.tracker.email.domain.example>
+Reply-To: Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
+MIME-Version: 1.0
+Message-Id: <followup_dummy_id>
+In-Reply-To: <dummy_test_message_id>
+X-Roundup-Name: Roundup issue tracker
+X-Roundup-Loop: hello
+X-Roundup-Issue-Status: chatting
+Content-Transfer-Encoding: quoted-printable
+
+
+Contrary, Mary <mary@test.test> added the comment:
+
+A message with encoding (encoded oe =F6)
+
+----------
+status: unread -> chatting
+
 _______________________________________________________________________
 Roundup issue tracker <issue_tracker@your.tracker.email.domain.example>
 <http://tracker.example/cgi-bin/roundup.cgi/bugs/issue1>