From: richard Date: Thu, 12 Mar 2009 05:55:16 +0000 (+0000) Subject: migrate from MimeWriter to email X-Git-Url: https://git.tokkee.org/?a=commitdiff_plain;h=349dcdaf535cfdc53b4d00f3a0d85bf3a5f97571;p=roundup.git migrate from MimeWriter to email git-svn-id: http://svn.roundup-tracker.org/svnroot/roundup/roundup/trunk@4184 57a73879-2fb5-44c3-a270-3262357dd7e2 --- diff --git a/COPYING.txt b/COPYING.txt index 481c862..b396397 100644 --- a/COPYING.txt +++ b/COPYING.txt @@ -1,7 +1,7 @@ Roundup Licensing ----------------- -Copyright (c) 2003 Richard Jones (richard@mechanicalcat.net) +Copyright (c) 2003-2009 Richard Jones (richard@mechanicalcat.net) Copyright (c) 2002 eKit.com Inc (http://www.ekit.com/) Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/) diff --git a/README.txt b/README.txt index f0b6689..b6cf8c6 100644 --- a/README.txt +++ b/README.txt @@ -2,7 +2,7 @@ Roundup: an Issue-Tracking System for Knowledge Workers ======================================================= -Copyright (c) 2003 Richard Jones (richard@mechanicalcat.net) +Copyright (c) 2003-2009 Richard Jones (richard@mechanicalcat.net) Copyright (c) 2002 eKit.com Inc (http://www.ekit.com/) Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/) @@ -31,7 +31,7 @@ directory. Upgrading ========= For upgrading instructions, please see upgrading.txt in the "doc" directory. - + Usage and Other Information =========================== diff --git a/roundup/anypy/TODO.txt b/roundup/anypy/TODO.txt index 028058e..71590ce 100644 --- a/roundup/anypy/TODO.txt +++ b/roundup/anypy/TODO.txt @@ -5,21 +5,4 @@ Python compatiblity TODO the subprocess module is available since Python 2.4, thus a roundup.anypy.subprocess_ module is needed -- the MimeWriter module is deprecated as of Python 2.6. The email package is - available since Python 2.2, thus we should manage without a ...email_ - module; however, it has suffered some API changes over the time - (http://docs.python.org/library/email.html#package-history), - so this is not sure. - - Here's an incomplete replacement table: - - MimeWriter usage checked for - -> email usage Python ... - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~ - MimeWriter.MimeWriter - -> email.Message.Message (2.3) - - MimeWriter.MimeWrite.addheader - -> email.Message.Message.add_header (2.3) - # vim: si diff --git a/roundup/mailer.py b/roundup/mailer.py index 09f808c..9c70852 100644 --- a/roundup/mailer.py +++ b/roundup/mailer.py @@ -3,24 +3,28 @@ __docformat__ = 'restructuredtext' # $Id: mailer.py,v 1.22 2008-07-21 01:44:58 richard Exp $ -import time, quopri, os, socket, smtplib, re, sys, traceback +import time, quopri, os, socket, smtplib, re, sys, traceback, email from cStringIO import StringIO -from MimeWriter import MimeWriter -from roundup.rfc2822 import encode_header from roundup import __version__ from roundup.date import get_timezone -try: - from email.Utils import formatdate -except ImportError: - def formatdate(): - return time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()) +from email.Utils import formatdate, formataddr +from email.Message import Message +from email.Header import Header +from email.MIMEText import MIMEText +from email.MIMEMultipart import MIMEMultipart class MessageSendError(RuntimeError): pass +def encode_quopri(msg): + orig = msg.get_payload() + encdata = quopri.encodestring(orig) + msg.set_payload(encdata) + msg['Content-Transfer-Encoding'] = 'quoted-printable' + class Mailer: """Roundup-specific mail sending.""" def __init__(self, config): @@ -41,7 +45,7 @@ class Mailer: os.environ['TZ'] = get_timezone(self.config.TIMEZONE).tzname(None) time.tzset() - def get_standard_message(self, to, subject, author=None): + def get_standard_message(self, to, subject, author=None, multipart=False): '''Form a standard email message from Roundup. "to" - recipients list @@ -55,38 +59,48 @@ class Mailer: ''' # encode header values if they need to be charset = getattr(self.config, 'EMAIL_CHARSET', 'utf-8') - tracker_name = self.config.TRACKER_NAME - if charset != 'utf-8': - tracker = unicode(tracker_name, 'utf-8').encode(charset) + tracker_name = unicode(self.config.TRACKER_NAME, 'utf-8') if not author: - author = straddr((tracker_name, self.config.ADMIN_EMAIL)) + author = formataddr((tracker_name, self.config.ADMIN_EMAIL)) + else: + name = unicode(author[0], 'utf-8') + author = formataddr((name, author[1])) + + if multipart: + message = MIMEMultipart() else: - name = author[0] - if charset != 'utf-8': - name = unicode(name, 'utf-8').encode(charset) - author = straddr((encode_header(name, charset), author[1])) - - message = StringIO() - writer = MimeWriter(message) - writer.addheader('Subject', encode_header(subject, charset)) - writer.addheader('To', ', '.join(to)) - writer.addheader('From', author) - writer.addheader('Date', formatdate(localtime=True)) + message = Message() + message.set_charset(charset) + message['Content-Type'] = 'text/plain; charset="%s"'%charset + + try: + message['Subject'] = subject.encode('ascii') + except UnicodeError: + message['Subject'] = Header(subject, charset) + message['To'] = ', '.join(to) + try: + message['From'] = author.encode('ascii') + except UnicodeError: + message['From'] = Header(author, charset) + message['Date'] = formatdate(localtime=True) # add a Precedence header so autoresponders ignore us - writer.addheader('Precedence', 'bulk') + message['Precedence'] = 'bulk' # Add a unique Roundup header to help filtering - writer.addheader('X-Roundup-Name', encode_header(tracker_name, - charset)) + try: + message['X-Roundup-Name'] = tracker_name.encode('ascii') + except UnicodeError: + message['X-Roundup-Name'] = Header(tracker_name, charset) + # and another one to avoid loops - writer.addheader('X-Roundup-Loop', 'hello') + message['X-Roundup-Loop'] = 'hello' # finally, an aid to debugging problems - writer.addheader('X-Roundup-Version', __version__) + message['X-Roundup-Version'] = __version__ - writer.addheader('MIME-Version', '1.0') + message['MIME-Version'] = '1.0' - return message, writer + return message def standard_message(self, to, subject, content, author=None): """Send a standard message. @@ -96,15 +110,12 @@ class Mailer: - subject: the subject as a string. - content: the body of the message as a string. - author: the sender as a (name, address) tuple - """ - message, writer = self.get_standard_message(to, subject, author) - - writer.addheader('Content-Transfer-Encoding', 'quoted-printable') - body = writer.startbody('text/plain; charset=utf-8') - content = StringIO(content) - quopri.encode(content, body, 0) - self.smtp_send(to, message) + All strings are assumed to be UTF-8 encoded. + """ + message = self.get_standard_message(to, subject, author) + message.set_payload(content) + self.smtp_send(to, str(message)) def bounce_message(self, bounced_message, to, error, subject='Failed issue tracker submission'): @@ -128,23 +139,13 @@ class Mailer: elif error_messages_to == "both": to.append(dispatcher_email) - message, writer = self.get_standard_message(to, subject) + message = self.get_standard_message(to, subject) - part = writer.startmultipartbody('mixed') - part = writer.nextpart() - part.addheader('Content-Transfer-Encoding', 'quoted-printable') - body = part.startbody('text/plain; charset=utf-8') - body.write(quopri.encodestring ('\n'.join(error))) + # add the error text + part = MIMEText(error) + message.attach(part) # attach the original message to the returned message - part = writer.nextpart() - part.addheader('Content-Disposition', 'attachment') - part.addheader('Content-Description', 'Message you sent') - body = part.startbody('text/plain') - - for header in bounced_message.headers: - body.write(header) - body.write('\n') try: bounced_message.rewindbody() except IOError, message: @@ -152,11 +153,15 @@ class Mailer: % bounced_message) else: body.write(bounced_message.fp.read()) + part = MIMEText(bounced_message.fp.read()) + part['Content-Disposition'] = 'attachment' + for header in bounced_message.headers: + part.write(header) + message.attach(part) - writer.lastpart() - + # send try: - self.smtp_send(to, message) + self.smtp_send(to, str(message)) except MessageSendError: # squash mail sending errors when bouncing mail # TODO this *could* be better, as we could notify admin of the @@ -184,16 +189,14 @@ class Mailer: # don't send - just write to a file open(self.debug, 'a').write('FROM: %s\nTO: %s\n%s\n' % (self.config.ADMIN_EMAIL, - ', '.join(to), - message.getvalue())) + ', '.join(to), message)) else: # now try to send the message try: # send the message as admin so bounces are sent there # instead of to roundup smtp = SMTPConnection(self.config) - smtp.sendmail(self.config.ADMIN_EMAIL, to, - message.getvalue()) + smtp.sendmail(self.config.ADMIN_EMAIL, to, message) except socket.error, value: raise MessageSendError("Error: couldn't send email: " "mailhost %s"%value) @@ -217,20 +220,4 @@ class SMTPConnection(smtplib.SMTP): if mailuser: self.login(mailuser, config["MAIL_PASSWORD"]) -# use the 'email' module, either imported, or our copied version -try : - from email.Utils import formataddr as straddr -except ImportError : - # code taken from the email package 2.4.3 - def straddr(pair, specialsre = re.compile(r'[][\()<>@,:;".]'), - escapesre = re.compile(r'[][\()"]')): - name, address = pair - if name: - quotes = '' - if specialsre.search(name): - quotes = '"' - name = escapesre.sub(r'\\\g<0>', name) - return '%s%s%s <%s>' % (quotes, name, quotes, address) - return address - # vim: set et sts=4 sw=4 : diff --git a/roundup/mailgw.py b/roundup/mailgw.py index b3659b9..5e8df2b 100644 --- a/roundup/mailgw.py +++ b/roundup/mailgw.py @@ -1347,6 +1347,7 @@ Mail message was rejected by a detector. else: nodeid = cl.create(**props) except (TypeError, IndexError, ValueError, exceptions.Reject), message: + raise raise MailUsageError, _(""" There was a problem with the message you sent: %(message)s diff --git a/roundup/roundupdb.py b/roundup/roundupdb.py index a129480..7c3106c 100644 --- a/roundup/roundupdb.py +++ b/roundup/roundupdb.py @@ -23,17 +23,20 @@ from __future__ import nested_scopes __docformat__ = 'restructuredtext' import re, os, smtplib, socket, time, random -import cStringIO, base64, quopri, mimetypes +import cStringIO, base64, mimetypes import os.path import logging - -from rfc2822 import encode_header +from email import Encoders +from email.Utils import formataddr +from email.Header import Header +from email.MIMEText import MIMEText +from email.MIMEBase import MIMEBase from roundup import password, date, hyperdb from roundup.i18n import _ # MessageSendError is imported for backwards compatibility -from roundup.mailer import Mailer, straddr, MessageSendError +from roundup.mailer import Mailer, MessageSendError, encode_quopri class Database: @@ -118,24 +121,24 @@ class Database: def log_debug(self, msg, *args, **kwargs): """Log a message with level DEBUG.""" - + logger = self.get_logger() logger.debug(msg, *args, **kwargs) - + def log_info(self, msg, *args, **kwargs): """Log a message with level INFO.""" - + logger = self.get_logger() logger.info(msg, *args, **kwargs) - + def get_logger(self): """Return the logger for this database.""" - + # Because getting a logger requires acquiring a lock, we want # to do it only once. if not hasattr(self, '__logger'): self.__logger = logging.getLogger('hyperdb') - + return self.__logger @@ -315,7 +318,7 @@ class IssueClass: authaddr = users.get(authid, 'address', '') if authaddr and self.db.config.MAIL_ADD_AUTHOREMAIL: - authaddr = " <%s>" % straddr( ('',authaddr) ) + authaddr = " <%s>" % formataddr( ('',authaddr) ) elif authaddr: authaddr = "" @@ -366,15 +369,11 @@ class IssueClass: if self.db.config.EMAIL_SIGNATURE_POSITION == 'bottom': m.append(self.email_signature(nodeid, msgid)) - # encode the content as quoted-printable + # figure the encoding charset = getattr(self.db.config, 'EMAIL_CHARSET', 'utf-8') - m = '\n'.join(m) - if charset != 'utf-8': - m = unicode(m, 'utf-8').encode(charset) - content = cStringIO.StringIO(m) - content_encoded = cStringIO.StringIO() - quopri.encode(content, content_encoded, 0) - content_encoded = content_encoded.getvalue() + + # construct the content and convert to unicode object + content = unicode('\n'.join(m), 'utf-8').encode(charset) # make sure the To line is always the same (for testing mostly) sendto.sort() @@ -397,6 +396,10 @@ class IssueClass: else: sendto = [sendto] + tracker_name = unicode(self.db.config.TRACKER_NAME, 'utf-8') + tracker_name = formataddr((tracker_name, from_address)) + tracker_name = Header(tracker_name, charset) + # now send one or more messages # TODO: I believe we have to create a new message each time as we # can't fiddle the recipients in the message ... worth testing @@ -405,21 +408,18 @@ class IssueClass: for sendto in sendto: # create the message mailer = Mailer(self.db.config) - message, writer = mailer.get_standard_message(sendto, subject, - author) + + message = mailer.get_standard_message(sendto, subject, author, + multipart=message_files) # set reply-to to the tracker - tracker_name = self.db.config.TRACKER_NAME - if charset != 'utf-8': - tracker = unicode(tracker_name, 'utf-8').encode(charset) - tracker_name = encode_header(tracker_name, charset) - writer.addheader('Reply-To', straddr((tracker_name, from_address))) + message['Reply-To'] = tracker_name # message ids if messageid: - writer.addheader('Message-Id', messageid) + message['Message-Id'] = messageid if inreplyto: - writer.addheader('In-Reply-To', inreplyto) + message['In-Reply-To'] = inreplyto # Generate a header for each link or multilink to # a class that has a name attribute @@ -440,8 +440,12 @@ class IssueClass: continue values = [cl.get(v, 'name') for v in values] values = ', '.join(values) - writer.addheader("X-Roundup-%s-%s" % (self.classname, propname), - values) + header = "X-Roundup-%s-%s"%(self.classname, propname) + try: + message[header] = values.encode('ascii') + except UnicodeError: + message[header] = Header(values, charset) + if not inreplyto: # Default the reply to the first message msgs = self.get(nodeid, 'messages') @@ -451,36 +455,31 @@ class IssueClass: if msgs and msgs[0] != nodeid: inreplyto = messages.get(msgs[0], 'messageid') if inreplyto: - writer.addheader('In-Reply-To', inreplyto) + message['In-Reply-To'] = inreplyto # attach files if message_files: - part = writer.startmultipartbody('mixed') - part = writer.nextpart() - part.addheader('Content-Transfer-Encoding', 'quoted-printable') - body = part.startbody('text/plain; charset=%s'%charset) - body.write(content_encoded) + # first up the text as a part + part = MIMEText(content) + encode_quopri(part) + message.attach(part) + for fileid in message_files: name = files.get(fileid, 'name') mime_type = files.get(fileid, 'type') content = files.get(fileid, 'content') - part = writer.nextpart() if mime_type == 'text/plain': - part.addheader('Content-Disposition', - 'attachment;\n filename="%s"'%name) try: content.decode('ascii') except UnicodeError: # the content cannot be 7bit-encoded. # use quoted printable - part.addheader('Content-Transfer-Encoding', - 'quoted-printable') - body = part.startbody('text/plain') - body.write(quopri.encodestring(content)) + # XXX stuffed if we know the charset though :( + part = MIMEText(content) + encode_quopri(part) else: - part.addheader('Content-Transfer-Encoding', '7bit') - body = part.startbody('text/plain') - body.write(content) + part = MIMEText(content) + part['Content-Transfer-Encoding'] = '7bit' else: # some other type, so encode it if not mime_type: @@ -488,17 +487,16 @@ class IssueClass: mime_type = mimetypes.guess_type(name)[0] if mime_type is None: mime_type = 'application/octet-stream' - part.addheader('Content-Disposition', - 'attachment;\n filename="%s"'%name) - part.addheader('Content-Transfer-Encoding', 'base64') - body = part.startbody(mime_type) - body.write(base64.encodestring(content)) - writer.lastpart() + main, sub = mime_type.split('/') + part = MIMEBase(main, sub) + part.set_payload(content) + Encoders.encode_base64(part) + part['Content-Disposition'] = 'attachment;\n filename="%s"'%name + message.attach(part) + else: - writer.addheader('Content-Transfer-Encoding', - 'quoted-printable') - body = writer.startbody('text/plain; charset=%s'%charset) - body.write(content_encoded) + message.set_payload(content) + encode_quopri(message) if first: mailer.smtp_send(sendto + bcc_sendto, message) @@ -522,7 +520,7 @@ class IssueClass: web = base + self.classname + nodeid # ensure the email address is properly quoted - email = straddr((self.db.config.TRACKER_NAME, + email = formataddr((self.db.config.TRACKER_NAME, self.db.config.TRACKER_EMAIL)) line = '_' * max(len(web)+2, len(email)) diff --git a/test/test_mailgw.py b/test/test_mailgw.py index 0a69ad6..d03f70d 100644 --- a/test/test_mailgw.py +++ b/test/test_mailgw.py @@ -50,6 +50,9 @@ class DiffHelper: res = [] for key in new.keys(): + if key.startswith('from '): + # skip the unix from line + continue if key.lower() == 'x-roundup-version': # version changes constantly, so handle it specially if new[key] != __version__: @@ -235,7 +238,7 @@ This is a test submission of a new issue. self.compareMessages(self._get_mail(), '''FROM: roundup-admin@your.tracker.email.domain.example TO: chef@bork.bork.bork, mary@test.test, richard@test.test -Content-Type: text/plain; charset=utf-8 +Content-Type: text/plain; charset="utf-8" Subject: [issue1] Testing... To: chef@bork.bork.bork, mary@test.test, richard@test.test From: "Bork, Chef" @@ -279,7 +282,7 @@ This is a test submission of a new issue. self.compareMessages(self._get_mail(), '''FROM: roundup-admin@your.tracker.email.domain.example TO: chef@bork.bork.bork, mary@test.test, richard@test.test -Content-Type: text/plain; charset=utf-8 +Content-Type: text/plain; charset="utf-8" Subject: [issue1] Testing... To: mary@test.test, richard@test.test From: "Bork, Chef" @@ -320,7 +323,7 @@ This is a test submission of a new issue. self.compareMessages(self._get_mail(), '''FROM: roundup-admin@your.tracker.email.domain.example TO: chef@bork.bork.bork, mary@test.test, richard@test.test -Content-Type: text/plain; charset=utf-8 +Content-Type: text/plain; charset="utf-8" Subject: [issue1] Testing... To: mary@test.test, richard@test.test From: "Bork, Chef" @@ -463,7 +466,7 @@ This is a second followup self.compareMessages(self._get_mail(), '''FROM: roundup-admin@your.tracker.email.domain.example TO: chef@bork.bork.bork, richard@test.test -Content-Type: text/plain; charset=utf-8 +Content-Type: text/plain; charset="utf-8" Subject: [issue1] Testing... To: chef@bork.bork.bork, richard@test.test From: "Contrary, Mary" @@ -511,7 +514,7 @@ This is a followup self.compareMessages(self._get_mail(), '''FROM: roundup-admin@your.tracker.email.domain.example TO: chef@bork.bork.bork, john@test.test, mary@test.test -Content-Type: text/plain; charset=utf-8 +Content-Type: text/plain; charset="utf-8" Subject: [issue1] Testing... To: chef@bork.bork.bork, john@test.test, mary@test.test From: richard @@ -559,7 +562,7 @@ _______________________________________________________________________ self.compareMessages(new_mail, """ FROM: roundup-admin@your.tracker.email.domain.example TO: chef@bork.bork.bork, richard@test.test -Content-Type: text/plain; charset=utf-8 +Content-Type: text/plain; charset="utf-8" Subject: [issue1] Testing... To: chef@bork.bork.bork, richard@test.test From: "Bork, Chef" @@ -602,7 +605,7 @@ This is a followup self.compareMessages(self._get_mail(), '''FROM: roundup-admin@your.tracker.email.domain.example TO: chef@bork.bork.bork, john@test.test, mary@test.test -Content-Type: text/plain; charset=utf-8 +Content-Type: text/plain; charset="utf-8" Subject: [issue1] Testing... To: chef@bork.bork.bork, john@test.test, mary@test.test From: richard @@ -715,7 +718,7 @@ This is a followup self.compareMessages(self._get_mail(), '''FROM: roundup-admin@your.tracker.email.domain.example TO: chef@bork.bork.bork, richard@test.test -Content-Type: text/plain; charset=utf-8 +Content-Type: text/plain; charset="utf-8" Subject: [issue1] Testing... To: chef@bork.bork.bork, richard@test.test From: John Doe @@ -761,7 +764,7 @@ This is a followup self.compareMessages(self._get_mail(), '''FROM: roundup-admin@your.tracker.email.domain.example TO: chef@bork.bork.bork -Content-Type: text/plain; charset=utf-8 +Content-Type: text/plain; charset="utf-8" Subject: [issue1] Testing... To: chef@bork.bork.bork From: richard @@ -807,7 +810,7 @@ This is a followup self.compareMessages(self._get_mail(), '''FROM: roundup-admin@your.tracker.email.domain.example TO: chef@bork.bork.bork, john@test.test, richard@test.test -Content-Type: text/plain; charset=utf-8 +Content-Type: text/plain; charset="utf-8" Subject: [issue1] Testing... To: chef@bork.bork.bork, john@test.test, richard@test.test From: John Doe @@ -852,7 +855,7 @@ This is a followup self.compareMessages(self._get_mail(), '''FROM: roundup-admin@your.tracker.email.domain.example TO: chef@bork.bork.bork, richard@test.test -Content-Type: text/plain; charset=utf-8 +Content-Type: text/plain; charset="utf-8" Subject: [issue1] Testing... To: chef@bork.bork.bork, richard@test.test From: John Doe @@ -897,7 +900,7 @@ This is a followup self.compareMessages(self._get_mail(), '''FROM: roundup-admin@your.tracker.email.domain.example TO: chef@bork.bork.bork -Content-Type: text/plain; charset=utf-8 +Content-Type: text/plain; charset="utf-8" Subject: [issue1] Testing... To: chef@bork.bork.bork From: richard @@ -1078,7 +1081,7 @@ A message with encoding (encoded oe =F6) self.compareMessages(self._get_mail(), '''FROM: roundup-admin@your.tracker.email.domain.example TO: chef@bork.bork.bork, richard@test.test -Content-Type: text/plain; charset=utf-8 +Content-Type: text/plain; charset="utf-8" Subject: [issue1] Testing... To: chef@bork.bork.bork, richard@test.test From: "Contrary, Mary" @@ -1132,7 +1135,7 @@ A message with first part encoded (encoded oe =F6) self.compareMessages(self._get_mail(), '''FROM: roundup-admin@your.tracker.email.domain.example TO: chef@bork.bork.bork, richard@test.test -Content-Type: text/plain; charset=utf-8 +Content-Type: text/plain; charset="utf-8" Subject: [issue1] Testing... To: chef@bork.bork.bork, richard@test.test From: "Contrary, Mary" @@ -1209,7 +1212,7 @@ This is a followup self.compareMessages(self._get_mail(), '''FROM: roundup-admin@your.tracker.email.domain.example TO: chef@bork.bork.bork -Content-Type: text/plain; charset=utf-8 +Content-Type: text/plain; charset="utf-8" Subject: [issue1] Testing... To: chef@bork.bork.bork From: richard