X-Git-Url: https://git.tokkee.org/?a=blobdiff_plain;f=roundup%2Fmailgw.py;h=3898051e4aa5b30229f4f77611126b3d190e9aa5;hb=0bf3521deec7a0467f67293527f3cdaf38e7f5f7;hp=54de2499c4e4854bdf422f1595068caa2c5ab3ee;hpb=6017df9b80fa58974ffc5f2abf67316cea57be77;p=roundup.git diff --git a/roundup/mailgw.py b/roundup/mailgw.py index 54de249..3898051 100644 --- a/roundup/mailgw.py +++ b/roundup/mailgw.py @@ -73,20 +73,29 @@ are calling the create() method to create a new node). If an auditor raises an exception, the original message is bounced back to the sender with the explanatory message given in the exception. -$Id: mailgw.py,v 1.40 2001-12-05 14:26:44 rochecompaan Exp $ +$Id: mailgw.py,v 1.65 2002-02-15 00:13:38 richard Exp $ ''' import string, re, os, mimetools, cStringIO, smtplib, socket, binascii, quopri +import time, random import traceback, MimeWriter import hyperdb, date, password +SENDMAILDEBUG = os.environ.get('SENDMAILDEBUG', '') + class MailGWError(ValueError): pass class MailUsageError(ValueError): pass +class MailUsageHelp(Exception): + pass + +class UnAuthorized(Exception): + """ Access denied """ + class Message(mimetools.Message): ''' subclass mimetools.Message so we can retrieve the parts of the message... @@ -110,8 +119,8 @@ class Message(mimetools.Message): s.seek(0) return Message(s) -subject_re = re.compile(r'(?P\s*\W?\s*(fwd|re)\s*\W?\s*)*' - r'\s*(\[(?P[^\d\s]+)(?P\d+)?\])' +subject_re = re.compile(r'(?P\s*\W?\s*(fwd|re|aw)\s*\W?\s*)*' + r'\s*(\[(?P[^\d\s]+)(?P\d+)?\])?' r'\s*(?P[^[]+)?(\[(?P<args>.+?)\])?', re.I) class MailGW: @@ -139,6 +148,15 @@ class MailGW: if sendto: try: return self.handle_message(message) + except MailUsageHelp: + # bounce the message back to the sender with the usage message + fulldoc = '\n'.join(string.split(__doc__, '\n')[2:]) + sendto = [sendto[0][1]] + m = [''] + m.append('\n\nMail Gateway Help\n=================') + m.append(fulldoc) + m = self.bounce_message(message, sendto, m, + subject="Mail Gateway Help") except MailUsageError, value: # bounce the message back to the sender with the usage message fulldoc = '\n'.join(string.split(__doc__, '\n')[2:]) @@ -148,10 +166,19 @@ class MailGW: m.append('\n\nMail Gateway Help\n=================') m.append(fulldoc) m = self.bounce_message(message, sendto, m) + except UnAuthorized, value: + # just inform the user that he is not authorized + sendto = [sendto[0][1]] + m = [''] + m.append(str(value)) + m = self.bounce_message(message, sendto, m) except: # bounce the message back to the sender with the error message - sendto = [sendto[0][1]] + sendto = [sendto[0][1], self.instance.ADMIN_EMAIL] m = [''] + m.append('An unexpected error occurred during the processing') + m.append('of your message. The tracker administrator is being') + m.append('notified.\n') m.append('---- traceback of failure ----') s = cStringIO.StringIO() import traceback @@ -160,7 +187,7 @@ class MailGW: m = self.bounce_message(message, sendto, m) else: # very bad-looking message - we don't even know who sent it - sendto = [self.ADMIN_EMAIL] + sendto = [self.instance.ADMIN_EMAIL] m = ['Subject: badly formed message from mail gateway'] m.append('') m.append('The mail gateway retrieved a message which has no From:') @@ -171,14 +198,18 @@ class MailGW: subject='Badly formed message from mail gateway') # now send the message - try: - smtp = smtplib.SMTP(self.MAILHOST) - smtp.sendmail(self.ADMIN_EMAIL, sendto, m.getvalue()) - except socket.error, value: - raise MailGWError, "Couldn't send confirmation email: "\ - "mailhost %s"%value - except smtplib.SMTPException, value: - raise MailGWError, "Couldn't send confirmation email: %s"%value + if SENDMAILDEBUG: + open(SENDMAILDEBUG, 'w').write('From: %s\nTo: %s\n%s\n'%( + self.instance.ADMIN_EMAIL, ', '.join(sendto), m.getvalue())) + else: + try: + smtp = smtplib.SMTP(self.instance.MAILHOST) + smtp.sendmail(self.instance.ADMIN_EMAIL, sendto, m.getvalue()) + except socket.error, value: + raise MailGWError, "Couldn't send error email: "\ + "mailhost %s"%value + except smtplib.SMTPException, value: + raise MailGWError, "Couldn't send error email: %s"%value def bounce_message(self, message, sendto, error, subject='Failed issue tracker submission'): @@ -190,7 +221,7 @@ class MailGW: writer = MimeWriter.MimeWriter(msg) writer.addheader('Subject', subject) writer.addheader('From', '%s <%s>'% (self.instance.INSTANCE_NAME, - self.ISSUE_TRACKER_EMAIL)) + self.instance.ISSUE_TRACKER_EMAIL)) writer.addheader('To', ','.join(sendto)) writer.addheader('MIME-Version', '1.0') part = writer.startmultipartbody('mixed') @@ -201,20 +232,28 @@ class MailGW: # reconstruct the original message m = cStringIO.StringIO() w = MimeWriter.MimeWriter(m) + # default the content_type, just in case... + content_type = 'text/plain' + # add the headers except the content-type for header in message.headers: header_name = header.split(':')[0] - if message.getheader(header_name): - w.addheader(header_name,message.getheader(header_name)) - body = w.startbody('text/plain') + if header_name.lower() == 'content-type': + content_type = message.getheader(header_name) + elif message.getheader(header_name): + w.addheader(header_name, message.getheader(header_name)) + # now attach the message body + body = w.startbody(content_type) try: - message.fp.seek(0) - except: - pass - body.write(message.fp.read()) + message.rewindbody() + except IOError: + body.write("*** couldn't include message body: read from pipe ***") + else: + body.write(message.fp.read()) # attach the original message to the returned message part = writer.nextpart() part.addheader('Content-Disposition','attachment') + part.addheader('Content-Description','Message you sent') part.addheader('Content-Transfer-Encoding', '7bit') body = part.startbody('message/rfc822') body.write(m.getvalue()) @@ -222,6 +261,25 @@ class MailGW: writer.lastpart() return msg + def get_part_data_decoded(self,part): + encoding = part.getencoding() + data = None + if encoding == 'base64': + # BUG: is base64 really used for text encoding or + # are we inserting zip files here. + data = binascii.a2b_base64(part.fp.read()) + elif encoding == 'quoted-printable': + # the quopri module wants to work with files + decoded = cStringIO.StringIO() + quopri.decode(part.fp, decoded) + data = decoded.getvalue() + elif encoding == 'uuencoded': + data = binascii.a2b_uu(part.fp.read()) + else: + # take it as text + data = part.fp.read() + return data + def handle_message(self, message): ''' message - a Message instance @@ -229,7 +287,25 @@ class MailGW: ''' # handle the subject line subject = message.getheader('subject', '') + + if subject.strip() == 'help': + raise MailUsageHelp + m = subject_re.match(subject) + + # check for well-formed subject line + if m: + # get the classname + classname = m.group('classname') + if classname is None: + # no classname, fallback on the default + if hasattr(self.instance, 'MAIL_DEFAULT_CLASS') and \ + self.instance.MAIL_DEFAULT_CLASS: + classname = self.instance.MAIL_DEFAULT_CLASS + else: + # fail + m = None + if not m: raise MailUsageError, ''' The message you sent to roundup did not contain a properly formed subject @@ -245,8 +321,7 @@ line. The subject must contain a class name or designator to indicate the Subject was: "%s" '''%subject - # get the classname - classname = m.group('classname') + # get the class try: cl = self.db.getclass(classname) except KeyError: @@ -276,7 +351,7 @@ supply a full node identifier (with number, eg "[issue123]" or keep the previous subject title intact so I can match that. Subject was: "%s" -'''%(classname, subject) +'''%subject # extract the args subject_args = m.group('args') @@ -299,6 +374,7 @@ Subject was: "%s" args = m.group('args') if args: for prop in string.split(args, ';'): + # extract the property name and value try: key, value = prop.split('=') except ValueError, message: @@ -308,6 +384,8 @@ Subject argument list not of form [arg=value,value,...;arg=value,value...] Subject was: "%s" '''%(message, subject) + + # ensure it's a valid property name key = key.strip() try: proptype = properties[key] @@ -317,6 +395,8 @@ Subject argument list refers to an invalid property: "%s" Subject was: "%s" '''%(key, subject) + + # convert the string value to a real property value if isinstance(proptype, hyperdb.String): props[key] = value.strip() if isinstance(proptype, hyperdb.Password): @@ -342,31 +422,63 @@ Error was: %s Subject was: "%s" '''%(key, message, subject) elif isinstance(proptype, hyperdb.Link): - link = self.db.classes[proptype.classname] - propkey = link.labelprop(default_to_id=1) + linkcl = self.db.classes[proptype.classname] + propkey = linkcl.labelprop(default_to_id=1) try: - props[key] = link.get(value.strip(), propkey) - except: - props[key] = link.lookup(value.strip()) + props[key] = linkcl.lookup(value) + except KeyError, message: + raise MailUsageError, ''' +Subject argument list contains an invalid value for %s. + +Error was: %s +Subject was: "%s" +'''%(key, message, subject) elif isinstance(proptype, hyperdb.Multilink): - link = self.db.classes[proptype.classname] - propkey = link.labelprop(default_to_id=1) - l = [x.strip() for x in value.split(',')] - for item in l: + # get the linked class + linkcl = self.db.classes[proptype.classname] + propkey = linkcl.labelprop(default_to_id=1) + for item in value.split(','): + item = item.strip() try: - v = link.get(item, propkey) - except: - v = link.lookup(item) + item = linkcl.lookup(item) + except KeyError, message: + raise MailUsageError, ''' +Subject argument list contains an invalid value for %s. + +Error was: %s +Subject was: "%s" +'''%(key, message, subject) if props.has_key(key): - props[key].append(v) + props[key].append(item) else: - props[key] = [v] - + props[key] = [item] # # handle the users # - author = self.db.uidFromAddress(message.getaddrlist('from')[0]) + + # Don't create users if ANONYMOUS_REGISTER_MAIL is denied + # ... fall back on ANONYMOUS_REGISTER if the other doesn't exist + create = 1 + if hasattr(self.instance, 'ANONYMOUS_REGISTER_MAIL'): + if self.instance.ANONYMOUS_REGISTER_MAIL == 'deny': + create = 0 + elif self.instance.ANONYMOUS_REGISTER == 'deny': + create = 0 + + author = self.db.uidFromAddress(message.getaddrlist('from')[0], + create=create) + if not author: + raise UnAuthorized, ''' +You are not a registered user. + +Unknown address: %s +'''%message.getaddrlist('from')[0][1] + + # the author may have been created - make sure the change is + # committed before we reopen the database + self.db.commit() + # reopen the database as the author username = self.db.user.get(author, 'username') self.db = self.instance.open(username) @@ -376,15 +488,54 @@ Subject was: "%s" # now update the recipients list recipients = [] - tracker_email = self.ISSUE_TRACKER_EMAIL.lower() + tracker_email = self.instance.ISSUE_TRACKER_EMAIL.lower() for recipient in message.getaddrlist('to') + message.getaddrlist('cc'): - if recipient[1].strip().lower() == tracker_email: + r = recipient[1].strip().lower() + if r == tracker_email or not r: continue recipients.append(self.db.uidFromAddress(recipient)) + # + # handle message-id and in-reply-to + # + messageid = message.getheader('message-id') + inreplyto = message.getheader('in-reply-to') or '' + # generate a messageid if there isn't one + if not messageid: + messageid = "<%s.%s.%s%s@%s>"%(time.time(), random.random(), + classname, nodeid, self.instance.MAIL_DOMAIN) + + # # now handle the body - find the message + # content_type = message.gettype() attachments = [] + # General multipart handling: + # Take the first text/plain part, anything else is considered an + # attachment. + # multipart/mixed: multiple "unrelated" parts. + # multipart/signed (rfc 1847): + # The control information is carried in the second of the two + # required body parts. + # ACTION: Default, so if content is text/plain we get it. + # multipart/encrypted (rfc 1847): + # The control information is carried in the first of the two + # required body parts. + # ACTION: Not handleable as the content is encrypted. + # multipart/related (rfc 1872, 2112, 2387): + # The Multipart/Related content-type addresses the MIME representation + # of compound objects. + # ACTION: Default. If we are lucky there is a text/plain. + # TODO: One should use the start part and look for an Alternative + # that is text/plain. + # multipart/Alternative (rfc 1872, 1892): + # only in "related" ? + # multipart/report (rfc 1892): + # e.g. mail system delivery status reports. + # ACTION: Default. Could be ignored or used for Delivery Notification + # flagging. + # multipart/form-data: + # For web forms only. if content_type == 'multipart/mixed': # skip over the intro to the first boundary part = message.getPart() @@ -397,12 +548,8 @@ Subject was: "%s" # parse it subtype = part.gettype() if subtype == 'text/plain' and not content: - # add all text/plain parts to the message content - if content is None: - content = part.fp.read() - else: - content = content + part.fp.read() - + # The first text/plain part is the message content. + content = self.get_part_data_decoded(part) elif subtype == 'message/rfc822': # handle message/rfc822 specially - the name should be # the subject of the actual e-mail embedded here @@ -411,23 +558,12 @@ Subject was: "%s" name = mailmess.getheader('subject') part.fp.seek(i) attachments.append((name, 'message/rfc822', part.fp.read())) - else: # try name on Content-Type name = part.getparam('name') # this is just an attachment - encoding = part.getencoding() - if encoding == 'base64': - data = binascii.a2b_base64(part.fp.read()) - elif encoding == 'quoted-printable': - # the quopri module wants to work with files - decoded = cStringIO.StringIO() - quopri.decode(part.fp, decoded) - data = decoded.getvalue() - elif encoding == 'uuencoded': - data = binascii.a2b_uu(part.fp.read()) + data = self.get_part_data_decoded(part) attachments.append((name, part.gettype(), data)) - if content is None: raise MailUsageError, ''' Roundup requires the submission to be plain text. The message parser could @@ -445,8 +581,7 @@ not find a text/plain part to use. break # parse it if part.gettype() == 'text/plain' and not content: - # this one's our content - content = part.fp.read() + content = self.get_part_data_decoded(part) if content is None: raise MailUsageError, ''' Roundup requires the submission to be plain text. The message parser could @@ -460,17 +595,23 @@ not find a text/plain part to use. ''' else: - content = message.fp.read() - + content = self.get_part_data_decoded(message) + summary, content = parseContent(content) - # handle the files + # + # handle the attachments + # files = [] for (name, mime_type, data) in attachments: + if not name: + name = "unnamed" files.append(self.db.file.create(type=mime_type, name=name, content=data)) + # # now handle the db stuff + # if nodeid: # If an item designator (class name and id number) is found there, # the newly created "msg" node is added to the "messages" property @@ -488,9 +629,10 @@ not find a text/plain part to use. except KeyError: pass else: - if (not props.has_key('status') or - props['status'] == unread_id or - props['status'] == resolved_id): + current_status = cl.get(nodeid, 'status') + if (not props.has_key('status') and + current_status == unread_id or + current_status == resolved_id): props['status'] = chatting_id # add nosy in arguments to issue's nosy list @@ -506,19 +648,16 @@ not find a text/plain part to use. if n.has_key(nid): continue n[nid] = 1 props['nosy'] = n.keys() - try: - assignedto = self.db.user.lookup(props['assignedto']) + # add assignedto to the nosy list + if props.has_key('assignedto'): + assignedto = props['assignedto'] if assignedto not in props['nosy']: props['nosy'].append(assignedto) - except: - pass - - change_note = cl.generateChangeNote(nodeid, props) - content += change_note - + message_id = self.db.msg.create(author=author, recipients=recipients, date=date.Date('.'), summary=summary, - content=content, files=files) + content=content, files=files, messageid=messageid, + inreplyto=inreplyto) try: messages = cl.get(nodeid, 'messages') except IndexError: @@ -548,7 +687,8 @@ There was a problem with the message you sent: # contain any new "file" nodes. message_id = self.db.msg.create(author=author, recipients=recipients, date=date.Date('.'), summary=summary, - content=content, files=files) + content=content, files=files, messageid=messageid, + inreplyto=inreplyto) # pre-set the issue to unread if properties.has_key('status') and not props.has_key('status'): @@ -564,25 +704,35 @@ There was a problem with the message you sent: if properties.has_key('title') and not props.has_key('title'): props['title'] = title - # pre-load the messages list and nosy list + # pre-load the messages list props['messages'] = [message_id] + + # set up (clean) the nosy list nosy = props.get('nosy', []) n = {} for value in nosy: - if self.db.hasnode('user', value): - nid = value - else: - try: - nid = self.db.user.lookup(value) - except: - continue + nid = value if n.has_key(nid): continue n[nid] = 1 - props['nosy'] = n.keys() + recipients + props['nosy'] = n.keys() + # add on the recipients of the message + for recipient in recipients: + if not n.has_key(recipient): + props['nosy'].append(recipient) + n[recipient] = 1 + + # add the author to the nosy list if not n.has_key(author): props['nosy'].append(author) n[author] = 1 + # add assignedto to the nosy list + if properties.has_key('assignedto') and props.has_key('assignedto'): + assignedto = props['assignedto'] + if not n.has_key(assignedto): + props['nosy'].append(assignedto) + n[assignedto] = 1 + # and attempt to create the new node try: nodeid = cl.create(**props) @@ -620,21 +770,162 @@ def parseContent(content, blank_line=re.compile(r'[\r\n]+\s*[\r\n]+'), if not section: continue lines = eol.split(section) - if lines[0] and lines[0][0] in '>|': - continue - if len(lines) > 1 and lines[1] and lines[1][0] in '>|': - continue + if (lines[0] and lines[0][0] in '>|') or (len(lines) > 1 and + lines[1] and lines[1][0] in '>|'): + # see if there's a response somewhere inside this section (ie. + # no blank line between quoted message and response) + for line in lines[1:]: + if line[0] not in '>|': + break + else: + # TODO: people who want to keep quoted bits will want the + # next line... + # l.append(section) + continue + # keep this section - it has reponse stuff in it + if not summary: + # and while we're at it, use the first non-quoted bit as + # our summary + summary = line + lines = lines[lines.index(line):] + section = '\n'.join(lines) + if not summary: + # if we don't have our summary yet use the first line of this + # section summary = lines[0] - l.append(section) - continue - if signature.match(lines[0]): + elif signature.match(lines[0]): break + + # and add the section to the output l.append(section) return summary, '\n\n'.join(l) # # $Log: not supported by cvs2svn $ +# Revision 1.64 2002/02/14 23:46:02 richard +# . #516883 ] mail interface + ANONYMOUS_REGISTER +# +# Revision 1.63 2002/02/12 08:08:55 grubert +# . Clean up mail handling, multipart handling. +# +# Revision 1.62 2002/02/05 14:15:29 grubert +# . respect encodings in non multipart messages. +# +# Revision 1.61 2002/02/04 09:40:21 grubert +# . add test for multipart messages with first part being encoded. +# +# Revision 1.60 2002/02/01 07:43:12 grubert +# . mailgw checks encoding on first part too. +# +# Revision 1.59 2002/01/23 21:43:23 richard +# tabnuke +# +# Revision 1.58 2002/01/23 21:41:56 richard +# . mailgw failures (unexpected ones) are forwarded to the roundup admin +# +# Revision 1.57 2002/01/22 22:27:43 richard +# . handle stripping of "AW:" from subject line +# +# Revision 1.56 2002/01/22 11:54:45 rochecompaan +# Fixed status change in mail gateway. +# +# Revision 1.55 2002/01/21 10:05:47 rochecompaan +# Feature: +# . the mail gateway now responds with an error message when invalid +# values for arguments are specified for link or multilink properties +# . modified unit test to check nosy and assignedto when specified as +# arguments +# +# Fixed: +# . fixed setting nosy as argument in subject line +# +# Revision 1.54 2002/01/16 09:14:45 grubert +# . if the attachment has no name, name it unnamed, happens with tnefs. +# +# Revision 1.53 2002/01/16 07:20:54 richard +# simple help command for mailgw +# +# Revision 1.52 2002/01/15 00:12:40 richard +# #503340 ] creating issue with [asignedto=p.ohly] +# +# Revision 1.51 2002/01/14 02:20:15 richard +# . changed all config accesses so they access either the instance or the +# config attriubute on the db. This means that all config is obtained from +# instance_config instead of the mish-mash of classes. This will make +# switching to a ConfigParser setup easier too, I hope. +# +# At a minimum, this makes migration a _little_ easier (a lot easier in the +# 0.5.0 switch, I hope!) +# +# Revision 1.50 2002/01/11 22:59:01 richard +# . #502342 ] pipe interface +# +# Revision 1.49 2002/01/10 06:19:18 richard +# followup lines directly after a quoted section were being eaten. +# +# Revision 1.48 2002/01/08 04:12:05 richard +# Changed message-id format to "<%s.%s.%s%s@%s>" so it complies with RFC822 +# +# Revision 1.47 2002/01/02 02:32:38 richard +# ANONYMOUS_ACCESS -> ANONYMOUS_REGISTER +# +# Revision 1.46 2002/01/02 02:31:38 richard +# Sorry for the huge checkin message - I was only intending to implement #496356 +# but I found a number of places where things had been broken by transactions: +# . modified ROUNDUPDBSENDMAILDEBUG to be SENDMAILDEBUG and hold a filename +# for _all_ roundup-generated smtp messages to be sent to. +# . the transaction cache had broken the roundupdb.Class set() reactors +# . newly-created author users in the mailgw weren't being committed to the db +# +# Stuff that made it into CHANGES.txt (ie. the stuff I was actually working +# on when I found that stuff :): +# . #496356 ] Use threading in messages +# . detectors were being registered multiple times +# . added tests for mailgw +# . much better attaching of erroneous messages in the mail gateway +# +# Revision 1.45 2001/12/20 15:43:01 rochecompaan +# Features added: +# . Multilink properties are now displayed as comma separated values in +# a textbox +# . The add user link is now only visible to the admin user +# . Modified the mail gateway to reject submissions from unknown +# addresses if ANONYMOUS_ACCESS is denied +# +# Revision 1.44 2001/12/18 15:30:34 rochecompaan +# Fixed bugs: +# . Fixed file creation and retrieval in same transaction in anydbm +# backend +# . Cgi interface now renders new issue after issue creation +# . Could not set issue status to resolved through cgi interface +# . Mail gateway was changing status back to 'chatting' if status was +# omitted as an argument +# +# Revision 1.43 2001/12/15 19:39:01 rochecompaan +# Oops. +# +# Revision 1.42 2001/12/15 19:24:39 rochecompaan +# . Modified cgi interface to change properties only once all changes are +# collected, files created and messages generated. +# . Moved generation of change note to nosyreactors. +# . We now check for changes to "assignedto" to ensure it's added to the +# nosy list. +# +# Revision 1.41 2001/12/10 00:57:38 richard +# From CHANGES: +# . Added the "display" command to the admin tool - displays a node's values +# . #489760 ] [issue] only subject +# . fixed the doc/index.html to include the quoting in the mail alias. +# +# Also: +# . fixed roundup-admin so it works with transactions +# . disabled the back_anydbm module if anydbm tries to use dumbdbm +# +# Revision 1.40 2001/12/05 14:26:44 rochecompaan +# Removed generation of change note from "sendmessage" in roundupdb.py. +# The change note is now generated when the message is created. +# # Revision 1.39 2001/12/02 05:06:16 richard # . We now use weakrefs in the Classes to keep the database reference, so # the close() method on the database is no longer needed.