From e950389204d8d83b6dec11e69849d10d66d6d71a Mon Sep 17 00:00:00 2001 From: schlatterbeck Date: Tue, 5 Oct 2010 14:24:25 +0000 Subject: [PATCH] =?utf8?q?-=20new=20mailgw=20config=20item=20unpack=5Frfc8?= =?utf8?q?22=20that=20unpacks=20message=20attachments=20=20=20of=20type=20?= =?utf8?q?message/rfc822=20and=20attaches=20the=20individual=20parts=20ins?= =?utf8?q?tead=20of=20=20=20attaching=20the=20whole=20message/rfc822=20att?= =?utf8?q?achment=20to=20the=20roundup=20issue.=20-=20Fix=20handling=20of?= =?utf8?q?=20incoming=20message/rfc822=20attachments.=20These=20resulted?= =?utf8?q?=20in=20=20=20a=20weird=20mail=20usage=20error=20because=20the?= =?utf8?q?=20email=20module=20threw=20a=20TypeError=20=20=20which=20roundu?= =?utf8?q?p=20interprets=20as=20a=20Reject=20exception.=20Fixes=20issue255?= =?utf8?q?0667.=20=20=20Added=20regression=20tests=20for=20message/rfc822?= =?utf8?q?=20attachments=20with=20and=20without=20=20=20configured=20unpac?= =?utf8?q?king=20(mailgw=20unpack=5Frfc822,=20see=20Features=20above)=20?= =?utf8?q?=20=20Thanks=20to=20Benni=20B=C3=A4rmann=20for=20reporting.?= MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit git-svn-id: http://svn.roundup-tracker.org/svnroot/roundup/roundup/trunk@4530 57a73879-2fb5-44c3-a270-3262357dd7e2 --- CHANGES.txt | 9 +++ roundup/configuration.py | 4 ++ roundup/mailgw.py | 29 ++++++-- roundup/roundupdb.py | 10 ++- test/test_mailgw.py | 147 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 193 insertions(+), 6 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 55bcab3..2347b09 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -17,6 +17,9 @@ Features: timeout of 30 seconds configurable. This is the time a client waits for the locked database to become free before giving up. Used only for SQLite backend. +- new mailgw config item unpack_rfc822 that unpacks message attachments + of type message/rfc822 and attaches the individual parts instead of + attaching the whole message/rfc822 attachment to the roundup issue. Fixed: @@ -33,6 +36,12 @@ Fixed: - Fix charset of first text-part of outgoing multipart messages, thanks Dirk Geschke for reporting, see http://thread.gmane.org/gmane.comp.bug-tracking.roundup.user/10223 +- Fix handling of incoming message/rfc822 attachments. These resulted in + a weird mail usage error because the email module threw a TypeError + which roundup interprets as a Reject exception. Fixes issue2550667. + Added regression tests for message/rfc822 attachments with and without + configured unpacking (mailgw unpack_rfc822, see Features above) + Thanks to Benni Bärmann for reporting. 2010-07-12 1.4.15 diff --git a/roundup/configuration.py b/roundup/configuration.py index 60b4190..8e42cef 100644 --- a/roundup/configuration.py +++ b/roundup/configuration.py @@ -748,6 +748,10 @@ SETTINGS = ( "Regular expression matching end of line."), (RegExpOption, "blankline_re", r"[\r\n]+\s*[\r\n]+", "Regular expression matching a blank line."), + (BooleanOption, "unpack_rfc822", "no", + "Unpack attached messages (encoded as message/rfc822 in MIME)\n" + "as multiple parts attached as files to the issue, if not\n" + "set we handle message/rfc822 attachments as a single file."), (BooleanOption, "ignore_alternatives", "no", "When parsing incoming mails, roundup uses the first\n" "text/plain part it finds. If this part is inside a\n" diff --git a/roundup/mailgw.py b/roundup/mailgw.py index d96d5ed..67cf6a3 100644 --- a/roundup/mailgw.py +++ b/roundup/mailgw.py @@ -27,6 +27,9 @@ Incoming messages are examined for multiple parts: and given "file" class nodes that are linked to the "msg" node. . In a multipart/alternative message or part, we look for a text/plain subpart and ignore the other parts. + . A message/rfc822 is treated similar tomultipart/mixed (except for + special handling of the first text part) if unpack_rfc822 is set in + the mailgw config section. Summary ------- @@ -277,12 +280,17 @@ class Message(mimetools.Message): def getname(self): """Find an appropriate name for this message.""" + name = None if self.gettype() == 'message/rfc822': # handle message/rfc822 specially - the name should be # the subject of the actual e-mail embedded here + # we add a '.eml' extension like other email software does it self.fp.seek(0) - name = Message(self.fp).getheader('subject') - else: + s = cStringIO.StringIO(self.getbody()) + name = Message(s).getheader('subject') + if name: + name = name + '.eml' + if not name: # try name on Content-Type name = self.getparam('name') if not name: @@ -355,8 +363,11 @@ class Message(mimetools.Message): # flagging. # multipart/form-data: # For web forms only. + # message/rfc822: + # Only if configured in [mailgw] unpack_rfc822 - def extract_content(self, parent_type=None, ignore_alternatives = False): + def extract_content(self, parent_type=None, ignore_alternatives=False, + unpack_rfc822=False): """Extract the body and the attachments recursively. If the content is hidden inside a multipart/alternative part, @@ -374,7 +385,7 @@ class Message(mimetools.Message): ig = ignore_alternatives and not content_found for part in self.getparts(): new_content, new_attach = part.extract_content(content_type, - not content and ig) + not content and ig, unpack_rfc822) # If we haven't found a text/plain part yet, take this one, # otherwise make it an attachment. @@ -399,6 +410,13 @@ class Message(mimetools.Message): attachments.extend(new_attach) if ig and content_type == 'multipart/alternative' and content: attachments = [] + elif unpack_rfc822 and content_type == 'message/rfc822': + s = cStringIO.StringIO(self.getbody()) + m = Message(s) + ig = ignore_alternatives and not content + new_content, attachments = m.extract_content(m.gettype(), ig, + unpack_rfc822) + attachments.insert(0, m.text_as_attachment()) elif (parent_type == 'multipart/signed' and content_type == 'application/pgp-signature'): # ignore it so it won't be saved as an attachment @@ -1276,7 +1294,8 @@ This tracker has been configured to require all email be PGP signed or encrypted.""") # now handle the body - find the message ig = self.instance.config.MAILGW_IGNORE_ALTERNATIVES - content, attachments = message.extract_content(ignore_alternatives = ig) + content, attachments = message.extract_content(ignore_alternatives=ig, + unpack_rfc822 = self.instance.config.MAILGW_UNPACK_RFC822) if content is None: raise MailUsageError, _(""" Roundup requires the submission to be plain text. The message parser could diff --git a/roundup/roundupdb.py b/roundup/roundupdb.py index d3576b1..eb71602 100644 --- a/roundup/roundupdb.py +++ b/roundup/roundupdb.py @@ -30,6 +30,7 @@ from email.Utils import formataddr from email.Header import Header from email.MIMEText import MIMEText from email.MIMEBase import MIMEBase +from email.parser import FeedParser from roundup import password, date, hyperdb from roundup.i18n import _ @@ -492,6 +493,12 @@ class IssueClass: else: part = MIMEText(content) part['Content-Transfer-Encoding'] = '7bit' + elif mime_type == 'message/rfc822': + main, sub = mime_type.split('/') + p = FeedParser() + p.feed(content) + part = MIMEBase(main, sub) + part.set_payload([p.close()]) else: # some other type, so encode it if not mime_type: @@ -503,7 +510,8 @@ class IssueClass: part = MIMEBase(main, sub) part.set_payload(content) Encoders.encode_base64(part) - part['Content-Disposition'] = 'attachment;\n filename="%s"'%name + cd = 'Content-Disposition' + part[cd] = 'attachment;\n filename="%s"'%name message.attach(part) else: diff --git a/test/test_mailgw.py b/test/test_mailgw.py index 47b8eeb..29d063a 100644 --- a/test/test_mailgw.py +++ b/test/test_mailgw.py @@ -480,6 +480,54 @@ Content-Transfer-Encoding: quoted-printable umlaut =E4=F6=FC=C4=D6=DC=DF +--001485f339f8f361fb049188dbba-- +''' + + multipart_msg_rfc822 = '''From: mary +To: issue_tracker@your.tracker.email.domain.example +Message-Id: +In-Reply-To: +Subject: [issue1] Testing... +Content-Type: multipart/mixed; boundary=001485f339f8f361fb049188dbba + +This is a multi-part message in MIME format. +--001485f339f8f361fb049188dbba +Content-Type: text/plain; charset=ISO-8859-15 +Content-Transfer-Encoding: 7bit + +First part: Text + +--001485f339f8f361fb049188dbba +Content-Type: message/rfc822; name="Fwd: Original email subject.eml" +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment; filename="Fwd: Original email subject.eml" + +Message-Id: +In-Reply-To: +MIME-Version: 1.0 +Subject: Fwd: Original email subject +Date: Mon, 23 Aug 2010 08:23:33 +0200 +Content-Type: multipart/alternative; boundary="090500050101020406060002" + +This is a multi-part message in MIME format. +--090500050101020406060002 +Content-Type: text/plain; charset=ISO-8859-15; format=flowed +Content-Transfer-Encoding: 7bit + +some text in inner email +======================== + +--090500050101020406060002 +Content-Type: text/html; charset=ISO-8859-15 +Content-Transfer-Encoding: 7bit + + +some text in inner email +======================== + + +--090500050101020406060002-- + --001485f339f8f361fb049188dbba-- ''' @@ -746,6 +794,105 @@ PGh0bWw+dW1sYXV0IMOkw7bDvMOEw5bDnMOfPC9odG1sPgo= --utf-8-- ''') + def testMultipartRFC822(self): + self.doNewIssue() + self._handle_mail(self.multipart_msg_rfc822) + messages = self.db.issue.get('1', 'messages') + messages.sort() + msg = self.db.msg.getnode (messages[-1]) + assert(len(msg.files) == 1) + name = "Fwd: Original email subject.eml" + for n, id in enumerate (msg.files): + f = self.db.file.getnode (id) + self.assertEqual(f.name, name) + self.assertEqual(msg.content, 'First part: Text') + self.compareMessages(self._get_mail(), +'''TO: chef@bork.bork.bork, richard@test.test +Content-Type: text/plain; charset="utf-8" +Subject: [issue1] Testing... +To: chef@bork.bork.bork, richard@test.test +From: "Contrary, Mary" +Reply-To: Roundup issue tracker + +MIME-Version: 1.0 +Message-Id: +In-Reply-To: +X-Roundup-Name: Roundup issue tracker +X-Roundup-Loop: hello +X-Roundup-Issue-Status: chatting +X-Roundup-Issue-Files: Fwd: Original email subject.eml +Content-Transfer-Encoding: quoted-printable + + +--utf-8 +MIME-Version: 1.0 +Content-Type: text/plain; charset="utf-8" +Content-Transfer-Encoding: quoted-printable + + +Contrary, Mary added the comment: + +First part: Text + +---------- +status: unread -> chatting + +_______________________________________________________________________ +Roundup issue tracker + +_______________________________________________________________________ +--utf-8 +Content-Type: message/rfc822 +MIME-Version: 1.0 +Content-Disposition: attachment; + filename="Fwd: Original email subject.eml" + +Message-Id: +In-Reply-To: +MIME-Version: 1.0 +Subject: Fwd: Original email subject +Date: Mon, 23 Aug 2010 08:23:33 +0200 +Content-Type: multipart/alternative; boundary="090500050101020406060002" + +This is a multi-part message in MIME format. +--090500050101020406060002 +Content-Type: text/plain; charset=ISO-8859-15; format=flowed +Content-Transfer-Encoding: 7bit + +some text in inner email +======================== + +--090500050101020406060002 +Content-Type: text/html; charset=ISO-8859-15 +Content-Transfer-Encoding: 7bit + + +some text in inner email +======================== + + +--090500050101020406060002-- + +--utf-8-- +''') + + def testMultipartRFC822Unpack(self): + self.doNewIssue() + self.db.config.MAILGW_UNPACK_RFC822 = True + self._handle_mail(self.multipart_msg_rfc822) + messages = self.db.issue.get('1', 'messages') + messages.sort() + msg = self.db.msg.getnode (messages[-1]) + self.assertEqual(len(msg.files), 2) + t = 'some text in inner email\n========================\n' + content = {0 : t, 1 : '\n' + t + '\n'} + for n, id in enumerate (msg.files): + f = self.db.file.getnode (id) + self.assertEqual(f.name, 'unnamed') + if n in content : + self.assertEqual(f.content, content [n]) + self.assertEqual(msg.content, 'First part: Text') + def testSimpleFollowup(self): self.doNewIssue() self._handle_mail('''Content-Type: text/plain; -- 2.30.2