diff --git a/roundup/mailgw.py b/roundup/mailgw.py
index e0b3ce5540d20eaeec5714e259f4a48a15cf261e..66f91cad054abfd8649b2e21b1e51abcb0907008 100644 (file)
--- a/roundup/mailgw.py
+++ b/roundup/mailgw.py
# FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
# BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
# SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
-#
-'''
+#
+
+__doc__ = '''
An e-mail gateway for Roundup.
Incoming messages are examined for multiple parts:
an exception, the original message is bounced back to the sender with the
explanatory message given in the exception.
-$Id: mailgw.py,v 1.34 2001-11-15 10:24:27 richard Exp $
+$Id: mailgw.py,v 1.52 2002-01-15 00:12:40 richard Exp $
'''
import string, re, os, mimetools, cStringIO, smtplib, socket, binascii, quopri
-import traceback
+import time, random
+import traceback, MimeWriter
import hyperdb, date, password
+SENDMAILDEBUG = os.environ.get('SENDMAILDEBUG', '')
+
class MailGWError(ValueError):
pass
class MailUsageError(ValueError):
pass
+class UnAuthorized(Exception):
+ """ Access denied """
+
class Message(mimetools.Message):
''' subclass mimetools.Message so we can retrieve the parts of the
message...
return Message(s)
subject_re = re.compile(r'(?P<refwd>\s*\W?\s*(fwd|re)\s*\W?\s*)*'
- r'\s*(\[(?P<classname>[^\d]+)(?P<nodeid>\d+)?\])'
- r'\s*(?P<title>[^\[]+)(\[(?P<args>.+?)\])?', re.I)
+ r'\s*(\[(?P<classname>[^\d\s]+)(?P<nodeid>\d+)?\])'
+ r'\s*(?P<title>[^[]+)?(\[(?P<args>.+?)\])?', re.I)
class MailGW:
def __init__(self, instance, db):
errors in a sane manner. It should be replaced if you wish to
handle errors in a different manner.
'''
- m = []
# in some rare cases, a particularly stuffed-up e-mail will make
# its way into here... try to handle it gracefully
sendto = message.getaddrlist('from')
if sendto:
try:
- self.handle_message(message)
- return
+ return self.handle_message(message)
except MailUsageError, value:
# bounce the message back to the sender with the usage message
fulldoc = '\n'.join(string.split(__doc__, '\n')[2:])
sendto = [sendto[0][1]]
- m = ['Subject: Failed issue tracker submission', '']
+ m = ['']
m.append(str(value))
m.append('\n\nMail Gateway Help\n=================')
m.append(fulldoc)
+ m = self.bounce_message(message, sendto, m)
+ except UnAuthorized, value:
+ # just inform the user that he is not authorized
+ sendto = [sendto[0][1]]
+ m = ['']
+ m.append(str(value))
+ m = self.bounce_message(message, sendto, m)
except:
# bounce the message back to the sender with the error message
sendto = [sendto[0][1]]
- m = ['Subject: failed issue tracker submission', '']
- # TODO as attachments?
+ m = ['']
m.append('---- traceback of failure ----')
s = cStringIO.StringIO()
import traceback
traceback.print_exc(None, s)
m.append(s.getvalue())
- m.append('---- failed message follows ----')
- try:
- message.fp.seek(0)
- except:
- pass
- m.append(message.fp.read())
+ m = self.bounce_message(message, sendto, m)
else:
# very bad-looking message - we don't even know who sent it
- sendto = [self.ADMIN_EMAIL]
+ sendto = [self.instance.ADMIN_EMAIL]
m = ['Subject: badly formed message from mail gateway']
m.append('')
m.append('The mail gateway retrieved a message which has no From:')
m.append('line, indicating that it is corrupt. Please check your')
- m.append('mail gateway source.')
+ m.append('mail gateway source. Failed message is attached.')
m.append('')
- m.append('---- failed message follows ----')
- try:
- message.fp.seek(0)
- except:
- pass
- m.append(message.fp.read())
+ m = self.bounce_message(message, sendto, m,
+ subject='Badly formed message from mail gateway')
# now send the message
+ if SENDMAILDEBUG:
+ open(SENDMAILDEBUG, 'w').write('From: %s\nTo: %s\n%s\n'%(
+ self.instance.ADMIN_EMAIL, ', '.join(sendto), m.getvalue()))
+ else:
+ try:
+ smtp = smtplib.SMTP(self.instance.MAILHOST)
+ smtp.sendmail(self.instance.ADMIN_EMAIL, sendto, m.getvalue())
+ except socket.error, value:
+ raise MailGWError, "Couldn't send error email: "\
+ "mailhost %s"%value
+ except smtplib.SMTPException, value:
+ raise MailGWError, "Couldn't send error email: %s"%value
+
+ def bounce_message(self, message, sendto, error,
+ subject='Failed issue tracker submission'):
+ ''' create a message that explains the reason for the failed
+ issue submission to the author and attach the original
+ message.
+ '''
+ msg = cStringIO.StringIO()
+ writer = MimeWriter.MimeWriter(msg)
+ writer.addheader('Subject', subject)
+ writer.addheader('From', '%s <%s>'% (self.instance.INSTANCE_NAME,
+ self.instance.ISSUE_TRACKER_EMAIL))
+ writer.addheader('To', ','.join(sendto))
+ writer.addheader('MIME-Version', '1.0')
+ part = writer.startmultipartbody('mixed')
+ part = writer.nextpart()
+ body = part.startbody('text/plain')
+ body.write('\n'.join(error))
+
+ # reconstruct the original message
+ m = cStringIO.StringIO()
+ w = MimeWriter.MimeWriter(m)
+ # default the content_type, just in case...
+ content_type = 'text/plain'
+ # add the headers except the content-type
+ for header in message.headers:
+ header_name = header.split(':')[0]
+ if header_name.lower() == 'content-type':
+ content_type = message.getheader(header_name)
+ elif message.getheader(header_name):
+ w.addheader(header_name, message.getheader(header_name))
+ # now attach the message body
+ body = w.startbody(content_type)
try:
- smtp = smtplib.SMTP(self.MAILHOST)
- smtp.sendmail(self.ADMIN_EMAIL, sendto, '\n'.join(m))
- except socket.error, value:
- raise MailGWError, "Couldn't send confirmation email: "\
- "mailhost %s"%value
- except smtplib.SMTPException, value:
- raise MailGWError, "Couldn't send confirmation email: %s"%value
+ message.rewindbody()
+ except IOError:
+ body.write("*** couldn't include message body: read from pipe ***")
+ else:
+ body.write(message.fp.read())
+
+ # attach the original message to the returned message
+ part = writer.nextpart()
+ part.addheader('Content-Disposition','attachment')
+ part.addheader('Content-Description','Message that caused the error')
+ part.addheader('Content-Transfer-Encoding', '7bit')
+ body = part.startbody('message/rfc822')
+ body.write(m.getvalue())
+
+ writer.lastpart()
+ return msg
def handle_message(self, message):
''' message - a Message instance
Subject was: "%s"
'''%subject
+
+ # get the classname
classname = m.group('classname')
- nodeid = m.group('nodeid')
- title = m.group('title').strip()
- subject_args = m.group('args')
try:
cl = self.db.getclass(classname)
except KeyError:
Subject was: "%s"
'''%(classname, ', '.join(self.db.getclasses()), subject)
+ # get the optional nodeid
+ nodeid = m.group('nodeid')
+
+ # title is optional too
+ title = m.group('title')
+ if title:
+ title = title.strip()
+ else:
+ title = ''
+
+ # but we do need either a title or a nodeid...
+ if not nodeid and not title:
+ raise MailUsageError, '''
+I cannot match your message to a node in the database - you need to either
+supply a full node identifier (with number, eg "[issue123]" or keep the
+previous subject title intact so I can match that.
+
+Subject was: "%s"
+'''%subject
+
+ # extract the args
+ subject_args = m.group('args')
+
# If there's no nodeid, check to see if this is a followup and
# maybe someone's responded to the initial mail that created an
# entry. Try to find the matching nodes with the same title, and
args = m.group('args')
if args:
for prop in string.split(args, ';'):
+ # extract the property name and value
try:
key, value = prop.split('=')
except ValueError, message:
Subject was: "%s"
'''%(message, subject)
+
+ # ensure it's a valid property name
+ key = key.strip()
try:
- type = properties[key]
+ proptype = properties[key]
except KeyError:
raise MailUsageError, '''
Subject argument list refers to an invalid property: "%s"
Subject was: "%s"
'''%(key, subject)
- if isinstance(type, hyperdb.String):
- props[key] = value
- if isinstance(type, hyperdb.Password):
- props[key] = password.Password(value)
- elif isinstance(type, hyperdb.Date):
+
+ # convert the string value to a real property value
+ if isinstance(proptype, hyperdb.String):
+ props[key] = value.strip()
+ if isinstance(proptype, hyperdb.Password):
+ props[key] = password.Password(value.strip())
+ elif isinstance(proptype, hyperdb.Date):
try:
- props[key] = date.Date(value)
+ props[key] = date.Date(value.strip())
except ValueError, message:
raise UsageError, '''
Subject argument list contains an invalid date for %s.
Error was: %s
Subject was: "%s"
'''%(key, message, subject)
- elif isinstance(type, hyperdb.Interval):
+ elif isinstance(proptype, hyperdb.Interval):
try:
- props[key] = date.Interval(value)
+ props[key] = date.Interval(value) # no strip needed
except ValueError, message:
raise UsageError, '''
Subject argument list contains an invalid date interval for %s.
Error was: %s
Subject was: "%s"
'''%(key, message, subject)
- elif isinstance(type, hyperdb.Link):
+ elif isinstance(proptype, hyperdb.Link):
+ link = self.db.classes[proptype.classname]
+ propkey = link.labelprop(default_to_id=1)
props[key] = value
- elif isinstance(type, hyperdb.Multilink):
- props[key] = value.split(',')
+ elif isinstance(proptype, hyperdb.Multilink):
+ # get the linked class
+ linkcl = self.db.classes[proptype.classname]
+ propkey = linkcl.labelprop(default_to_id=1)
+ for item in value.split(','):
+ item = item.split()
+ if props.has_key(key):
+ props[key].append(item)
+ else:
+ props[key] = [item]
#
# handle the users
#
- author = self.db.uidFromAddress(message.getaddrlist('from')[0])
+
+ # Don't create users if ANONYMOUS_REGISTER is denied
+ if self.instance.ANONYMOUS_REGISTER == 'deny':
+ create = 0
+ else:
+ create = 1
+ author = self.db.uidFromAddress(message.getaddrlist('from')[0],
+ create=create)
+ if not author:
+ raise UnAuthorized, '''
+You are not a registered user.
+
+Unknown address: %s
+'''%message.getaddrlist('from')[0][1]
+
+ # the author may have been created - make sure the change is
+ # committed before we reopen the database
+ self.db.commit()
+
# reopen the database as the author
username = self.db.user.get(author, 'username')
- self.db.close()
self.db = self.instance.open(username)
+
+ # re-get the class with the new database connection
+ cl = self.db.getclass(classname)
+
# now update the recipients list
recipients = []
- tracker_email = self.ISSUE_TRACKER_EMAIL.lower()
+ tracker_email = self.instance.ISSUE_TRACKER_EMAIL.lower()
for recipient in message.getaddrlist('to') + message.getaddrlist('cc'):
- if recipient[1].strip().lower() == tracker_email:
+ r = recipient[1].strip().lower()
+ if r == tracker_email or not r:
continue
recipients.append(self.db.uidFromAddress(recipient))
+ #
+ # handle message-id and in-reply-to
+ #
+ messageid = message.getheader('message-id')
+ inreplyto = message.getheader('in-reply-to') or ''
+ # generate a messageid if there isn't one
+ if not messageid:
+ messageid = "<%s.%s.%s%s@%s>"%(time.time(), random.random(),
+ classname, nodeid, self.instance.MAIL_DOMAIN)
+
+ #
# now handle the body - find the message
+ #
content_type = message.gettype()
attachments = []
if content_type == 'multipart/mixed':
summary, content = parseContent(content)
- # handle the files
+ #
+ # handle the attachments
+ #
files = []
- for (name, type, data) in attachments:
- files.append(self.db.file.create(type=type, name=name,
+ for (name, mime_type, data) in attachments:
+ files.append(self.db.file.create(type=mime_type, name=name,
content=data))
+ #
# now handle the db stuff
+ #
if nodeid:
# If an item designator (class name and id number) is found there,
# the newly created "msg" node is added to the "messages" property
# for that item, and any new "file" nodes are added to the "files"
# property for the item.
- message_id = self.db.msg.create(author=author,
- recipients=recipients, date=date.Date('.'), summary=summary,
- content=content, files=files)
- try:
- messages = cl.get(nodeid, 'messages')
- except IndexError:
- raise MailUsageError, '''
-The node specified by the designator in the subject of your message ("%s")
-does not exist.
-
-Subject was: "%s"
-'''%(nodeid, subject)
- messages.append(message_id)
- props['messages'] = messages
# if the message is currently 'unread' or 'resolved', then set
# it to 'chatting'
except KeyError:
pass
else:
- if (not props.has_key('status') or
- props['status'] == unread_id or
- props['status'] == resolved_id):
+ if (not props.has_key('status') and
+ properties['status'] == unread_id or
+ properties['status'] == resolved_id):
props['status'] = chatting_id
+ # add nosy in arguments to issue's nosy list
+ if not props.has_key('nosy'): props['nosy'] = []
+ n = {}
+ for nid in cl.get(nodeid, 'nosy'):
+ n[nid] = 1
+ for value in props['nosy']:
+ if self.db.hasnode('user', value):
+ nid = value
+ else:
+ continue
+ if n.has_key(nid): continue
+ n[nid] = 1
+ props['nosy'] = n.keys()
+ # add assignedto to the nosy list
+ try:
+ assignedto = self.db.user.lookup(props['assignedto'])
+ if assignedto not in props['nosy']:
+ props['nosy'].append(assignedto)
+ except:
+ pass
+
+ message_id = self.db.msg.create(author=author,
+ recipients=recipients, date=date.Date('.'), summary=summary,
+ content=content, files=files, messageid=messageid,
+ inreplyto=inreplyto)
+ try:
+ messages = cl.get(nodeid, 'messages')
+ except IndexError:
+ raise MailUsageError, '''
+The node specified by the designator in the subject of your message ("%s")
+does not exist.
+
+Subject was: "%s"
+'''%(nodeid, subject)
+ messages.append(message_id)
+ props['messages'] = messages
+
+ # now apply the changes
try:
cl.set(nodeid, **props)
except (TypeError, IndexError, ValueError), message:
There was a problem with the message you sent:
%s
'''%message
+ # commit the changes to the DB
+ self.db.commit()
else:
# If just an item class name is found there, we attempt to create a
# new item of that class with its "messages" property initialized to
# contain any new "file" nodes.
message_id = self.db.msg.create(author=author,
recipients=recipients, date=date.Date('.'), summary=summary,
- content=content, files=files)
- # fill out the properties with defaults where required
- if properties.has_key('assignedto') and \
- not props.has_key('assignedto'):
- props['assignedto'] = '1' # "admin"
+ content=content, files=files, messageid=messageid,
+ inreplyto=inreplyto)
# pre-set the issue to unread
if properties.has_key('status') and not props.has_key('status'):
if properties.has_key('title') and not props.has_key('title'):
props['title'] = title
- # pre-load the messages list and nosy list
+ # pre-load the messages list
props['messages'] = [message_id]
- props['nosy'] = props.get('nosy', []) + recipients
- props['nosy'].append(author)
- props['nosy'].sort()
+
+ # set up (clean) the nosy list
+ nosy = props.get('nosy', [])
+ n = {}
+ for value in nosy:
+ if self.db.hasnode('user', value):
+ nid = value
+ else:
+ continue
+ if n.has_key(nid): continue
+ n[nid] = 1
+ props['nosy'] = n.keys()
+ # add on the recipients of the message
+ for recipient in recipients:
+ if not n.has_key(recipient):
+ props['nosy'].append(recipient)
+ n[recipient] = 1
+
+ # add the author to the nosy list
+ if not n.has_key(author):
+ props['nosy'].append(author)
+ n[author] = 1
+
+ # add assignedto to the nosy list
+ if properties.has_key('assignedto') and props.has_key('assignedto'):
+ assignedto = props['assignedto']
+ if not re.match('^\d+$', assignedto):
+ try:
+ assignedto = self.db.user.lookup(assignedto)
+ except KeyError:
+ raise MailUsageError, '''
+There was a problem with the message you sent:
+ Assignedto user '%s' doesn't exist
+'''%props['assignedto']
+ if not n.has_key(assignedto):
+ props['nosy'].append(assignedto)
+ n[assignedto] = 1
# and attempt to create the new node
try:
%s
'''%message
+ # commit the new node(s) to the DB
+ self.db.commit()
+
def parseContent(content, blank_line=re.compile(r'[\r\n]+\s*[\r\n]+'),
eol=re.compile(r'[\r\n]+'), signature=re.compile(r'^[>|\s]*[-_]+\s*$')):
''' The message body is divided into sections by blank lines.
if not section:
continue
lines = eol.split(section)
- if lines[0] and lines[0][0] in '>|':
- continue
- if len(lines) > 1 and lines[1] and lines[1][0] in '>|':
- continue
+ if (lines[0] and lines[0][0] in '>|') or (len(lines) > 1 and
+ lines[1] and lines[1][0] in '>|'):
+ # see if there's a response somewhere inside this section (ie.
+ # no blank line between quoted message and response)
+ for line in lines[1:]:
+ if line[0] not in '>|':
+ break
+ else:
+ # TODO: people who want to keep quoted bits will want the
+ # next line...
+ # l.append(section)
+ continue
+ # keep this section - it has reponse stuff in it
+ if not summary:
+ # and while we're at it, use the first non-quoted bit as
+ # our summary
+ summary = line
+ lines = lines[lines.index(line):]
+ section = '\n'.join(lines)
+
if not summary:
+ # if we don't have our summary yet use the first line of this
+ # section
summary = lines[0]
- l.append(section)
- continue
- if signature.match(lines[0]):
+ elif signature.match(lines[0]):
break
+
+ # and add the section to the output
l.append(section)
return summary, '\n\n'.join(l)
#
# $Log: not supported by cvs2svn $
+# Revision 1.51 2002/01/14 02:20:15 richard
+# . changed all config accesses so they access either the instance or the
+# config attriubute on the db. This means that all config is obtained from
+# instance_config instead of the mish-mash of classes. This will make
+# switching to a ConfigParser setup easier too, I hope.
+#
+# At a minimum, this makes migration a _little_ easier (a lot easier in the
+# 0.5.0 switch, I hope!)
+#
+# Revision 1.50 2002/01/11 22:59:01 richard
+# . #502342 ] pipe interface
+#
+# Revision 1.49 2002/01/10 06:19:18 richard
+# followup lines directly after a quoted section were being eaten.
+#
+# Revision 1.48 2002/01/08 04:12:05 richard
+# Changed message-id format to "<%s.%s.%s%s@%s>" so it complies with RFC822
+#
+# Revision 1.47 2002/01/02 02:32:38 richard
+# ANONYMOUS_ACCESS -> ANONYMOUS_REGISTER
+#
+# Revision 1.46 2002/01/02 02:31:38 richard
+# Sorry for the huge checkin message - I was only intending to implement #496356
+# but I found a number of places where things had been broken by transactions:
+# . modified ROUNDUPDBSENDMAILDEBUG to be SENDMAILDEBUG and hold a filename
+# for _all_ roundup-generated smtp messages to be sent to.
+# . the transaction cache had broken the roundupdb.Class set() reactors
+# . newly-created author users in the mailgw weren't being committed to the db
+#
+# Stuff that made it into CHANGES.txt (ie. the stuff I was actually working
+# on when I found that stuff :):
+# . #496356 ] Use threading in messages
+# . detectors were being registered multiple times
+# . added tests for mailgw
+# . much better attaching of erroneous messages in the mail gateway
+#
+# Revision 1.45 2001/12/20 15:43:01 rochecompaan
+# Features added:
+# . Multilink properties are now displayed as comma separated values in
+# a textbox
+# . The add user link is now only visible to the admin user
+# . Modified the mail gateway to reject submissions from unknown
+# addresses if ANONYMOUS_ACCESS is denied
+#
+# Revision 1.44 2001/12/18 15:30:34 rochecompaan
+# Fixed bugs:
+# . Fixed file creation and retrieval in same transaction in anydbm
+# backend
+# . Cgi interface now renders new issue after issue creation
+# . Could not set issue status to resolved through cgi interface
+# . Mail gateway was changing status back to 'chatting' if status was
+# omitted as an argument
+#
+# Revision 1.43 2001/12/15 19:39:01 rochecompaan
+# Oops.
+#
+# Revision 1.42 2001/12/15 19:24:39 rochecompaan
+# . Modified cgi interface to change properties only once all changes are
+# collected, files created and messages generated.
+# . Moved generation of change note to nosyreactors.
+# . We now check for changes to "assignedto" to ensure it's added to the
+# nosy list.
+#
+# Revision 1.41 2001/12/10 00:57:38 richard
+# From CHANGES:
+# . Added the "display" command to the admin tool - displays a node's values
+# . #489760 ] [issue] only subject
+# . fixed the doc/index.html to include the quoting in the mail alias.
+#
+# Also:
+# . fixed roundup-admin so it works with transactions
+# . disabled the back_anydbm module if anydbm tries to use dumbdbm
+#
+# Revision 1.40 2001/12/05 14:26:44 rochecompaan
+# Removed generation of change note from "sendmessage" in roundupdb.py.
+# The change note is now generated when the message is created.
+#
+# Revision 1.39 2001/12/02 05:06:16 richard
+# . We now use weakrefs in the Classes to keep the database reference, so
+# the close() method on the database is no longer needed.
+# I bumped the minimum python requirement up to 2.1 accordingly.
+# . #487480 ] roundup-server
+# . #487476 ] INSTALL.txt
+#
+# I also cleaned up the change message / post-edit stuff in the cgi client.
+# There's now a clearly marked "TODO: append the change note" where I believe
+# the change note should be added there. The "changes" list will obviously
+# have to be modified to be a dict of the changes, or somesuch.
+#
+# More testing needed.
+#
+# Revision 1.38 2001/12/01 07:17:50 richard
+# . We now have basic transaction support! Information is only written to
+# the database when the commit() method is called. Only the anydbm
+# backend is modified in this way - neither of the bsddb backends have been.
+# The mail, admin and cgi interfaces all use commit (except the admin tool
+# doesn't have a commit command, so interactive users can't commit...)
+# . Fixed login/registration forwarding the user to the right page (or not,
+# on a failure)
+#
+# Revision 1.37 2001/11/28 21:55:35 richard
+# . login_action and newuser_action return values were being ignored
+# . Woohoo! Found that bloody re-login bug that was killing the mail
+# gateway.
+# (also a minor cleanup in hyperdb)
+#
+# Revision 1.36 2001/11/26 22:55:56 richard
+# Feature:
+# . Added INSTANCE_NAME to configuration - used in web and email to identify
+# the instance.
+# . Added EMAIL_SIGNATURE_POSITION to indicate where to place the roundup
+# signature info in e-mails.
+# . Some more flexibility in the mail gateway and more error handling.
+# . Login now takes you to the page you back to the were denied access to.
+#
+# Fixed:
+# . Lots of bugs, thanks Roché and others on the devel mailing list!
+#
+# Revision 1.35 2001/11/22 15:46:42 jhermann
+# Added module docstrings to all modules.
+#
+# Revision 1.34 2001/11/15 10:24:27 richard
+# handle the case where there is no file attached
+#
# Revision 1.33 2001/11/13 21:44:44 richard
# . re-open the database as the author in mail handling
#