X-Git-Url: https://git.tokkee.org/?a=blobdiff_plain;f=roundup%2Fmailgw.py;h=50a3c548884685eb2a0e7a2c0d4ac3a85abe87b6;hb=4f9b23d1dd4d91d0a3e5f3dfddb91700837c9a9f;hp=b9dd886b85dd9e770e59e8ec3ffacefbdf194e9e;hpb=bad0de7174fee417d57e15c9e77034d5f77dfd81;p=roundup.git diff --git a/roundup/mailgw.py b/roundup/mailgw.py index b9dd886..50a3c54 100644 --- a/roundup/mailgw.py +++ b/roundup/mailgw.py @@ -16,7 +16,7 @@ # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -''' +""" An e-mail gateway for Roundup. Incoming messages are examined for multiple parts: @@ -73,13 +73,15 @@ are calling the create() method to create a new node). If an auditor raises an exception, the original message is bounced back to the sender with the explanatory message given in the exception. -$Id: mailgw.py,v 1.106 2003-01-12 00:03:10 richard Exp $ -''' +$Id: mailgw.py,v 1.138 2003-11-13 03:41:38 richard Exp $ +""" import string, re, os, mimetools, cStringIO, smtplib, socket, binascii, quopri import time, random, sys -import traceback, MimeWriter -import hyperdb, date, password +import traceback, MimeWriter, rfc822 + +from roundup import hyperdb, date, password, rfc2822 +from roundup.mailer import Mailer SENDMAILDEBUG = os.environ.get('SENDMAILDEBUG', '') @@ -93,7 +95,7 @@ class MailUsageHelp(Exception): pass class MailLoop(Exception): - ''' We've seen this message before... ''' + """ We've seen this message before... """ pass class Unauthorized(Exception): @@ -111,6 +113,26 @@ def initialiseSecurity(security): description="User may use the email interface") security.addPermissionToRole('Admin', p) +def getparam(str, param): + ''' From the rfc822 "header" string, extract "param" if it appears. + ''' + if ';' not in str: + return None + str = str[str.index(';'):] + while str[:1] == ';': + str = str[1:] + if ';' in str: + # XXX Should parse quotes! + end = str.index(';') + else: + end = len(str) + f = str[:end] + if '=' in f: + i = f.index('=') + if f[:i].strip().lower() == param: + return rfc822.unquote(f[i+1:].strip()) + return None + class Message(mimetools.Message): ''' subclass mimetools.Message so we can retrieve the parts of the message... @@ -134,28 +156,45 @@ class Message(mimetools.Message): s.seek(0) return Message(s) -subject_re = re.compile(r'(?P\s*\W?\s*(fw|fwd|re|aw)\W\s*)*' - r'\s*(?P")?(\[(?P[^\d\s]+)(?P\d+)?\])?' - r'\s*(?P[^[]+)?"?(\[(?P<args>.+?)\])?', re.I) - + def getheader(self, name, default=None): + hdr = mimetools.Message.getheader(self, name, default) + if hdr: + hdr = hdr.replace('\n','') # Inserted by rfc822.readheaders + return rfc2822.decode_header(hdr) + class MailGW: + + # Matches subjects like: + # Re: "[issue1234] title of issue [status=resolved]" + subject_re = re.compile(r''' + (?P<refwd>\s*\W?\s*(fw|fwd|re|aw)\W\s*)*\s* # Re: + (?P<quote>")? # Leading " + (\[(?P<classname>[^\d\s]+) # [issue.. + (?P<nodeid>\d+)? # ..1234] + \])?\s* + (?P<title>[^[]+)? # issue title + "? # Trailing " + (\[(?P<args>.+?)\])? # [prop=value] + ''', re.IGNORECASE|re.VERBOSE) + def __init__(self, instance, db, arguments={}): self.instance = instance self.db = db - self.arguments = {} + self.arguments = arguments + self.mailer = Mailer(instance.config) # should we trap exceptions (normal usage) or pass them through # (for testing) self.trapExceptions = 1 def do_pipe(self): - ''' Read a message from standard input and pass it to the mail handler. + """ Read a message from standard input and pass it to the mail handler. Read into an internal structure that we can seek on (in case there's an error). XXX: we may want to read this into a temporary file instead... - ''' + """ s = cStringIO.StringIO() s.write(sys.stdin.read()) s.seek(0) @@ -163,11 +202,16 @@ class MailGW: return 0 def do_mailbox(self, filename): - ''' Read a series of messages from the specified unix mailbox file and + """ Read a series of messages from the specified unix mailbox file and pass each to the mail handler. - ''' + """ # open the spool file and lock it - import fcntl, FCNTL + import fcntl + # FCNTL is deprecated in py2.3 and fcntl takes over all the symbols + if hasattr(fcntl, 'LOCK_EX'): + FCNTL = fcntl + else: + import FCNTL f = open(filename, 'r+') fcntl.flock(f.fileno(), FCNTL.LOCK_EX) @@ -190,7 +234,12 @@ class MailGW: fcntl.flock(f.fileno(), FCNTL.LOCK_UN) return 0 - def do_pop(self, server, user='', password=''): + def do_apop(self, server, user='', password=''): + ''' Do authentication POP + ''' + self.do_pop(server, user, password, apop=1) + + def do_pop(self, server, user='', password='', apop=0): '''Read a series of messages from the specified POP server. ''' import getpass, poplib, socket @@ -210,8 +259,11 @@ class MailGW: except socket.error, message: print "POP server error:", message return 1 - server.user(user) - server.pass_(password) + if apop: + server.apop(user, password) + else: + server.user(user) + server.pass_(password) numMessages = len(server.list()[1]) for i in range(1, numMessages+1): # retr: returns @@ -235,17 +287,19 @@ class MailGW: return self.handle_Message(Message(fp)) def handle_Message(self, message): - '''Handle an RFC822 Message + """Handle an RFC822 Message Handle the Message object by calling handle_message() and then cope with any errors raised by handle_message. This method's job is to make that call and handle any errors in a sane manner. It should be replaced if you wish to handle errors in a different manner. - ''' + """ # in some rare cases, a particularly stuffed-up e-mail will make # its way into here... try to handle it gracefully - sendto = message.getaddrlist('from') + sendto = message.getaddrlist('resent-from') + if not sendto: + sendto = message.getaddrlist('from') if sendto: if not self.trapExceptions: return self.handle_message(message) @@ -258,7 +312,7 @@ class MailGW: m = [''] m.append('\n\nMail Gateway Help\n=================') m.append(fulldoc) - m = self.bounce_message(message, sendto, m, + self.mailer.bounce_message(message, sendto, m, subject="Mail Gateway Help") except MailUsageError, value: # bounce the message back to the sender with the usage message @@ -268,13 +322,13 @@ class MailGW: m.append(str(value)) m.append('\n\nMail Gateway Help\n=================') m.append(fulldoc) - m = self.bounce_message(message, sendto, m) + self.mailer.bounce_message(message, sendto, m) except Unauthorized, value: # just inform the user that he is not authorized sendto = [sendto[0][1]] m = [''] m.append(str(value)) - m = self.bounce_message(message, sendto, m) + self.mailer.bounce_message(message, sendto, m) except MailLoop: # XXX we should use a log file here... return @@ -291,7 +345,7 @@ class MailGW: import traceback traceback.print_exc(None, s) m.append(s.getvalue()) - m = self.bounce_message(message, sendto, m) + self.mailer.bounce_message(message, sendto, m) else: # very bad-looking message - we don't even know who sent it # XXX we should use a log file here... @@ -302,64 +356,9 @@ class MailGW: m.append('line, indicating that it is corrupt. Please check your') m.append('mail gateway source. Failed message is attached.') m.append('') - m = self.bounce_message(message, sendto, m, + self.mailer.bounce_message(message, sendto, m, subject='Badly formed message from mail gateway') - # now send the message - if SENDMAILDEBUG: - open(SENDMAILDEBUG, 'a').write('From: %s\nTo: %s\n%s\n'%( - self.instance.config.ADMIN_EMAIL, ', '.join(sendto), - m.getvalue())) - else: - try: - smtp = smtplib.SMTP(self.instance.config.MAILHOST) - smtp.sendmail(self.instance.config.ADMIN_EMAIL, sendto, - m.getvalue()) - except socket.error, value: - raise MailGWError, "Couldn't send error email: "\ - "mailhost %s"%value - except smtplib.SMTPException, value: - raise MailGWError, "Couldn't send error email: %s"%value - - def bounce_message(self, message, sendto, error, - subject='Failed issue tracker submission'): - ''' create a message that explains the reason for the failed - issue submission to the author and attach the original - message. - ''' - msg = cStringIO.StringIO() - writer = MimeWriter.MimeWriter(msg) - writer.addheader('X-Roundup-Loop', 'hello') - writer.addheader('Subject', subject) - writer.addheader('From', '%s <%s>'% (self.instance.config.TRACKER_NAME, - self.instance.config.TRACKER_EMAIL)) - writer.addheader('To', ','.join(sendto)) - writer.addheader('Date', time.strftime("%a, %d %b %Y %H:%M:%S +0000", - time.gmtime())) - writer.addheader('MIME-Version', '1.0') - part = writer.startmultipartbody('mixed') - part = writer.nextpart() - body = part.startbody('text/plain') - body.write('\n'.join(error)) - - # attach the original message to the returned message - part = writer.nextpart() - part.addheader('Content-Disposition','attachment') - part.addheader('Content-Description','Message you sent') - body = part.startbody('text/plain') - for header in message.headers: - body.write(header) - body.write('\n') - try: - message.rewindbody() - except IOError, message: - body.write("*** couldn't include message body: %s ***"%message) - else: - body.write(message.fp.read()) - - writer.lastpart() - return msg - def get_part_data_decoded(self,part): encoding = part.getencoding() data = None @@ -377,7 +376,19 @@ class MailGW: else: # take it as text data = part.fp.read() - return data + + # Encode message to unicode + charset = rfc2822.unaliasCharset(part.getparam("charset")) + if charset: + # Do conversion only if charset specified + edata = unicode(data, charset).encode('utf-8') + # Convert from dos eol to unix + edata = edata.replace('\r\n', '\n') + else: + # Leave message content as is + edata = data + + return edata def handle_message(self, message): ''' message - a Message instance @@ -405,41 +416,61 @@ class MailGW: # nodeid = issue.group('nodeid') # break + # determine the sender's address + from_list = message.getaddrlist('resent-from') + if not from_list: + from_list = message.getaddrlist('from') + # handle the subject line subject = message.getheader('subject', '') + if not subject: + raise MailUsageError, ''' +Emails to Roundup trackers must include a Subject: line! +''' + if subject.strip().lower() == 'help': raise MailUsageHelp - m = subject_re.match(subject) + m = self.subject_re.match(subject) # check for well-formed subject line if m: # get the classname classname = m.group('classname') if classname is None: - # no classname, fallback on the default - if hasattr(self.instance.config, 'MAIL_DEFAULT_CLASS') and \ - self.instance.config.MAIL_DEFAULT_CLASS: + # no classname, check if this a registration confirmation email + # or fallback on the default class + otk_re = re.compile('-- key (?P<otk>[a-zA-Z0-9]{32})') + otk = otk_re.search(m.group('title')) + if otk: + self.db.confirm_registration(otk.group('otk')) + subject = 'Your registration to %s is complete' % \ + self.instance.config.TRACKER_NAME + sendto = [from_list[0][1]] + self.mailer.standard_message(sendto, subject, '') + return + elif hasattr(self.instance.config, 'MAIL_DEFAULT_CLASS') and \ + self.instance.config.MAIL_DEFAULT_CLASS: classname = self.instance.config.MAIL_DEFAULT_CLASS else: # fail m = None if not m: - raise MailUsageError, ''' + raise MailUsageError, """ The message you sent to roundup did not contain a properly formed subject line. The subject must contain a class name or designator to indicate the -"topic" of the message. For example: +'topic' of the message. For example: Subject: [issue] This is a new issue - - this will create a new issue in the tracker with the title "This is - a new issue". + - this will create a new issue in the tracker with the title 'This is + a new issue'. Subject: [issue1234] This is a followup to issue 1234 - this will append the message's contents to the existing issue 1234 in the tracker. -Subject was: "%s" -'''%subject +Subject was: '%s' +"""%subject # get the class try: @@ -497,7 +528,6 @@ does not exist. Subject was: "%s" '''%(nodeid, subject) - # Handle the arguments specified by the email gateway command line. # We do this by looping over the list of self.arguments looking for # a -C to tell us what class then the -S setting string. @@ -551,8 +581,7 @@ The mail gateway is not properly set up. Please contact # ok, now figure out who the author is - create a new user if the # "create" flag is true - author = uidFromAddress(self.db, message.getaddrlist('from')[0], - create=create) + author = uidFromAddress(self.db, from_list[0], create=create) # if we're not recognised, and we don't get added as a user, then we # must be anonymous @@ -567,7 +596,7 @@ The mail gateway is not properly set up. Please contact You are not a registered user. Unknown address: %s -'''%message.getaddrlist('from')[0][1] +'''%from_list[0][1] else: # we're registered and we're _still_ not allowed access raise Unauthorized, 'You are not permitted to access '\ @@ -605,11 +634,6 @@ Unknown address: %s if recipient: recipients.append(recipient) - # - # XXX extract the args NOT USED WHY -- rouilj - # - subject_args = m.group('args') - # # handle the subject argument list # @@ -629,6 +653,11 @@ There were problems handling your subject line argument list: Subject was: "%s" '''%(errors, subject) + + # set the issue title to the subject + if properties.has_key('title') and not issue_props.has_key('title'): + issue_props['title'] = title.strip() + # # handle message-id and in-reply-to # @@ -695,6 +724,7 @@ Subject was: "%s" elif subtype == 'multipart/alternative': # Search for text/plain in message with attachment and # alternative text representation + # skip over intro to first boundary part.getPart() while 1: # get the next part @@ -712,7 +742,7 @@ Subject was: "%s" if not name: disp = part.getheader('content-disposition', None) if disp: - name = disp.getparam('filename') + name = getparam(disp, 'filename') if name: name = name.strip() # this is just an attachment @@ -760,21 +790,32 @@ not find a text/plain part to use. # parse the body of the message, stripping out bits as appropriate summary, content = parseContent(content, keep_citations, keep_body) + content = content.strip() # # handle the attachments # - files = [] - for (name, mime_type, data) in attachments: - if not name: - name = "unnamed" - files.append(self.db.file.create(type=mime_type, name=name, - content=data, **file_props)) + if properties.has_key('files'): + files = [] + for (name, mime_type, data) in attachments: + if not name: + name = "unnamed" + files.append(self.db.file.create(type=mime_type, name=name, + content=data, **file_props)) + # attach the files to the issue + if nodeid: + # extend the existing files list + fileprop = cl.get(nodeid, 'files') + fileprop.extend(files) + props['files'] = fileprop + else: + # pre-load the files list + props['files'] = files # # create the message if there's a message body (content) # - if content: + if (content and properties.has_key('messages')): message_id = self.db.msg.create(author=author, recipients=recipients, date=date.Date('.'), summary=summary, content=content, files=files, messageid=messageid, @@ -790,10 +831,6 @@ not find a text/plain part to use. # pre-load the messages list props['messages'] = [message_id] - # set the title to the subject - if properties.has_key('title') and not props.has_key('title'): - props['title'] = title - # # perform the node change / create # @@ -820,11 +857,10 @@ There was a problem with the message you sent: return nodeid -def setPropArrayFromString(self, cl, propString, nodeid = None): +def setPropArrayFromString(self, cl, propString, nodeid=None): ''' takes string of form prop=value,value;prop2=value and returns (error, prop[..]) ''' - properties = cl.getprops() props = {} errors = [] for prop in string.split(propString, ';'): @@ -835,100 +871,13 @@ def setPropArrayFromString(self, cl, propString, nodeid = None): errors.append('not of form [arg=value,value,...;' 'arg=value,value,...]') return (errors, props) - - # ensure it's a valid property name + # convert the value to a hyperdb-usable value propname = propname.strip() try: - proptype = properties[propname] - except KeyError: - errors.append('refers to an invalid property: "%s"'%propname) - continue - - # convert the string value to a real property value - if isinstance(proptype, hyperdb.String): - props[propname] = value.strip() - if isinstance(proptype, hyperdb.Password): - props[propname] = password.Password(value.strip()) - elif isinstance(proptype, hyperdb.Date): - try: - props[propname] = date.Date(value.strip()) - except ValueError, message: - errors.append('contains an invalid date for %s.'%propname) - elif isinstance(proptype, hyperdb.Interval): - try: - props[propname] = date.Interval(value) - except ValueError, message: - errors.append('contains an invalid date interval for %s.'% - propname) - elif isinstance(proptype, hyperdb.Link): - linkcl = self.db.classes[proptype.classname] - propkey = linkcl.labelprop(default_to_id=1) - try: - props[propname] = linkcl.lookup(value) - except KeyError, message: - errors.append('"%s" is not a value for %s.'%(value, propname)) - elif isinstance(proptype, hyperdb.Multilink): - # get the linked class - linkcl = self.db.classes[proptype.classname] - propkey = linkcl.labelprop(default_to_id=1) - if nodeid: - curvalue = cl.get(nodeid, propname) - else: - curvalue = [] - - # handle each add/remove in turn - # keep an extra list for all items that are - # definitely in the new list (in case of e.g. - # <propname>=A,+B, which should replace the old - # list with A,B) - set = 0 - newvalue = [] - for item in value.split(','): - item = item.strip() - - # handle +/- - remove = 0 - if item.startswith('-'): - remove = 1 - item = item[1:] - elif item.startswith('+'): - item = item[1:] - else: - set = 1 - - # look up the value - try: - item = linkcl.lookup(item) - except KeyError, message: - errors.append('"%s" is not a value for %s.'%(item, - propname)) - continue - - # perform the add/remove - if remove: - try: - curvalue.remove(item) - except ValueError: - errors.append('"%s" is not currently in for %s.'%(item, - propname)) - continue - else: - newvalue.append(item) - if item not in curvalue: - curvalue.append(item) - - # that's it, set the new Multilink property value, - # or overwrite it completely - if set: - props[propname] = newvalue - else: - props[propname] = curvalue - elif isinstance(proptype, hyperdb.Boolean): - value = value.strip() - props[propname] = value.lower() in ('yes', 'true', 'on', '1') - elif isinstance(proptype, hyperdb.Number): - value = value.strip() - props[propname] = int(value) + props[propname] = hyperdb.rawToHyperdb(self.db, cl, nodeid, + propname, value) + except hyperdb.HyperdbValueError, message: + errors.append(message) return errors, props @@ -960,14 +909,16 @@ def uidFromAddress(db, address, create=1, **user_props): # try a straight match of the address user = extractUserFromList(db.user, db.user.stringFind(address=address)) - if user is not None: return user + if user is not None: + return user # try the user alternate addresses if possible props = db.user.getprops() if props.has_key('alternate_addresses'): users = db.user.filter(None, {'alternate_addresses': address}) user = extractUserFromList(db.user, users) - if user is not None: return user + if user is not None: + return user # try to match the username to the address (for local # submissions where the address is empty) @@ -975,8 +926,26 @@ def uidFromAddress(db, address, create=1, **user_props): # couldn't match address or username, so create a new user if create: - return db.user.create(username=address, address=address, + # generate a username + if '@' in address: + username = address.split('@')[0] + else: + username = address + trying = username + n = 0 + while 1: + try: + # does this username exist already? + db.user.lookup(trying) + except KeyError: + break + n += 1 + trying = username + str(n) + + # create! + return db.user.create(username=trying, address=address, realname=realname, roles=db.config.NEW_EMAIL_USER_ROLES, + password=password.Password(password.generatePassword()), **user_props) else: return 0 @@ -985,7 +954,7 @@ def uidFromAddress(db, address, create=1, **user_props): def parseContent(content, keep_citations, keep_body, blank_line=re.compile(r'[\r\n]+\s*[\r\n]+'), eol=re.compile(r'[\r\n]+'), - signature=re.compile(r'^[>|\s]*[-_]+\s*$'), + signature=re.compile(r'^[>|\s]*-- ?$'), original_msg=re.compile(r'^[>|\s]*-----\s?Original Message\s?-----$')): ''' The message body is divided into sections by blank lines. Sections where the second and all subsequent lines begin with a ">"