From d5d1fe3d5323daa95acab81799a4a32f79b5e5f5 Mon Sep 17 00:00:00 2001 From: schlatterbeck Date: Thu, 23 Dec 2010 15:42:30 +0000 Subject: [PATCH] - Factor MailGW message parsing into a separate class, thanks to John Kristensen who did the major work in issue2550576 -- I wouldn't have attempted it without this. Fixes issue2550576. (Ralf) - Now if the -C option to roundup-mailgw specifies "issue" this refers to an issue-like class. The real class is determined from the configured default class, or the -c option to the mailgw, or the class resulting from mail subject parsing. We also accept multiple -S options for the same class now. (Ralf) - Add regression test for message-id generation if message id is missing in a message - Add regression tests for Option parsing (-S and -C options) git-svn-id: http://svn.roundup-tracker.org/svnroot/roundup/roundup/trunk@4577 57a73879-2fb5-44c3-a270-3262357dd7e2 --- CHANGES.txt | 8 + roundup/mailgw.py | 1721 ++++++++++++++++++++++++------------------- test/test_mailgw.py | 70 +- 3 files changed, 1025 insertions(+), 774 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index b8a8325..ce36fc4 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -11,6 +11,14 @@ Features: - Multilinks can be filtered by combining elements with AND, OR and NOT operators now. A javascript gui was added for "keywords", see issue2550648. Developed by Sascha Teichmann; funded by Intevation. (Bernhard Reiter) +- Factor MailGW message parsing into a separate class, thanks to John + Kristensen who did the major work in issue2550576 -- I wouldn't + have attempted it without this. Fixes issue2550576. (Ralf) +- Now if the -C option to roundup-mailgw specifies "issue" this refers + to an issue-like class. The real class is determined from the + configured default class, or the -c option to the mailgw, or the class + resulting from mail subject parsing. We also accept multiple -S + options for the same class now. (Ralf) Fixed: diff --git a/roundup/mailgw.py b/roundup/mailgw.py index 8346b05..b67c5c9 100644 --- a/roundup/mailgw.py +++ b/roundup/mailgw.py @@ -527,890 +527,1071 @@ class Message(mimetools.Message): result = context.op_verify_result() check_pgp_sigs(result.signatures, context, author) -class MailGW: +class parsedMessage: + + def __init__(self, mailgw, message): + self.mailgw = mailgw + self.config = mailgw.instance.config + self.db = mailgw.db + self.message = message + self.subject = message.getheader('subject', '') + self.has_prefix = False + self.matches = dict.fromkeys(['refwd', 'quote', 'classname', + 'nodeid', 'title', 'args', 'argswhole']) + self.from_list = message.getaddrlist('resent-from') \ + or message.getaddrlist('from') + self.pfxmode = self.config['MAILGW_SUBJECT_PREFIX_PARSING'] + self.sfxmode = self.config['MAILGW_SUBJECT_SUFFIX_PARSING'] + # these are filled in by subsequent parsing steps + self.classname = None + self.properties = None + self.cl = None + self.nodeid = None + self.author = None + self.recipients = None + self.props = None + self.content = None + self.attachments = None + + def handle_ignore(self): + ''' Check to see if message can be safely ignored: + detect loops and + Precedence: Bulk, or Microsoft Outlook autoreplies + ''' + if self.message.getheader('x-roundup-loop', ''): + raise IgnoreLoop + if (self.message.getheader('precedence', '') == 'bulk' + or self.subject.lower().find("autoreply") > 0): + raise IgnoreBulk - def __init__(self, instance, arguments=()): - self.instance = instance - self.arguments = arguments - self.default_class = None - for option, value in self.arguments: - if option == '-c': - self.default_class = value.strip() + def handle_help(self): + ''' Check to see if the message contains a usage/help request + ''' + if self.subject.strip().lower() == 'help': + raise MailUsageHelp - self.mailer = Mailer(instance.config) - self.logger = logging.getLogger('roundup.mailgw') + def check_subject(self): + ''' Check to see if the message contains a valid subject line + ''' + if not self.subject: + raise MailUsageError, _(""" +Emails to Roundup trackers must include a Subject: line! +""") - # should we trap exceptions (normal usage) or pass them through - # (for testing) - self.trapExceptions = 1 + def parse_subject(self): + ''' Matches subjects like: + Re: "[issue1234] title of issue [status=resolved]" + + Each part of the subject is matched, stored, then removed from the + start of the subject string as needed. The stored values are then + returned + ''' - def do_pipe(self): - """ Read a message from standard input and pass it to the mail handler. + tmpsubject = self.subject - Read into an internal structure that we can seek on (in case - there's an error). + sd_open, sd_close = self.config['MAILGW_SUBJECT_SUFFIX_DELIMITERS'] + delim_open = re.escape(sd_open) + if delim_open in '[(': delim_open = '\\' + delim_open + delim_close = re.escape(sd_close) + if delim_close in '[(': delim_close = '\\' + delim_close - XXX: we may want to read this into a temporary file instead... - """ - s = cStringIO.StringIO() - s.write(sys.stdin.read()) - s.seek(0) - self.main(s) - return 0 + # Look for Re: et. al. Used later on for MAILGW_SUBJECT_CONTENT_MATCH + re_re = r"(?P%s)\s*" % self.config["MAILGW_REFWD_RE"].pattern + m = re.match(re_re, tmpsubject, re.IGNORECASE|re.VERBOSE|re.UNICODE) + if m: + m = m.groupdict() + if m['refwd']: + self.matches.update(m) + tmpsubject = tmpsubject[len(m['refwd']):] # Consume Re: - def do_mailbox(self, filename): - """ Read a series of messages from the specified unix mailbox file and - pass each to the mail handler. - """ - # open the spool file and lock it - import fcntl - # FCNTL is deprecated in py2.3 and fcntl takes over all the symbols - if hasattr(fcntl, 'LOCK_EX'): - FCNTL = fcntl + # Look for Leading " + m = re.match(r'(?P\s*")', tmpsubject, + re.IGNORECASE) + if m: + self.matches.update(m.groupdict()) + tmpsubject = tmpsubject[len(self.matches['quote']):] # Consume quote + + # Check if the subject includes a prefix + self.has_prefix = re.search(r'^%s(\w+)%s'%(delim_open, + delim_close), tmpsubject.strip()) + + # Match the classname if specified + class_re = r'%s(?P(%s))(?P\d+)?%s'%(delim_open, + "|".join(self.db.getclasses()), delim_close) + # Note: re.search, not re.match as there might be garbage + # (mailing list prefix, etc.) before the class identifier + m = re.search(class_re, tmpsubject, re.IGNORECASE) + if m: + self.matches.update(m.groupdict()) + # Skip to the end of the class identifier, including any + # garbage before it. + + tmpsubject = tmpsubject[m.end():] + + # Match the title of the subject + # if we've not found a valid classname prefix then force the + # scanning to handle there being a leading delimiter + title_re = r'(?P%s[^%s]*)'%( + not self.matches['classname'] and '.' or '', delim_open) + m = re.match(title_re, tmpsubject.strip(), re.IGNORECASE) + if m: + self.matches.update(m.groupdict()) + tmpsubject = tmpsubject[len(self.matches['title']):] # Consume title + + if self.matches['title']: + self.matches['title'] = self.matches['title'].strip() else: - import FCNTL - f = open(filename, 'r+') - fcntl.flock(f.fileno(), FCNTL.LOCK_EX) + self.matches['title'] = '' - # handle and clear the mailbox - try: - from mailbox import UnixMailbox - mailbox = UnixMailbox(f, factory=Message) - # grab one message - message = mailbox.next() - while message: - # handle this message - self.handle_Message(message) - message = mailbox.next() - # nuke the file contents - os.ftruncate(f.fileno(), 0) - except: - import traceback - traceback.print_exc() - return 1 - fcntl.flock(f.fileno(), FCNTL.LOCK_UN) + # strip off the quotes that dumb emailers put around the subject, like + # Re: "[issue1] bla blah" + if self.matches['quote'] and self.matches['title'].endswith('"'): + self.matches['title'] = self.matches['title'][:-1] + + # Match any arguments specified + args_re = r'(?P<argswhole>%s(?P<args>.+?)%s)?'%(delim_open, + delim_close) + m = re.search(args_re, tmpsubject.strip(), re.IGNORECASE|re.VERBOSE) + if m: + self.matches.update(m.groupdict()) + + def rego_confirm(self): + ''' Check for registration OTK and confirm the registration if found + ''' + + if self.config['EMAIL_REGISTRATION_CONFIRMATION']: + otk_re = re.compile('-- key (?P<otk>[a-zA-Z0-9]{32})') + otk = otk_re.search(self.matches['title'] or '') + if otk: + self.db.confirm_registration(otk.group('otk')) + subject = 'Your registration to %s is complete' % \ + self.config['TRACKER_NAME'] + sendto = [self.from_list[0][1]] + self.mailgw.mailer.standard_message(sendto, subject, '') + return 1 return 0 - def do_imap(self, server, user='', password='', mailbox='', ssl=0, - cram=0): - ''' Do an IMAP connection + def get_classname(self): + ''' Determine the classname of the node being created/edited ''' - import getpass, imaplib, socket - try: - if not user: - user = raw_input('User: ') - if not password: - password = getpass.getpass() - except (KeyboardInterrupt, EOFError): - # Ctrl C or D maybe also Ctrl Z under Windows. - print "\nAborted by user." - return 1 - # open a connection to the server and retrieve all messages - try: - if ssl: - self.logger.debug('Trying server %r with ssl'%server) - server = imaplib.IMAP4_SSL(server) - else: - self.logger.debug('Trying server %r without ssl'%server) - server = imaplib.IMAP4(server) - except (imaplib.IMAP4.error, socket.error, socket.sslerror): - self.logger.exception('IMAP server error') - return 1 + subject = self.subject - try: - if cram: - server.login_cram_md5(user, password) - else: - server.login(user, password) - except imaplib.IMAP4.error, e: - self.logger.exception('IMAP login failure') - return 1 + # get the classname + if self.pfxmode == 'none': + classname = None + else: + classname = self.matches['classname'] - try: - if not mailbox: - (typ, data) = server.select() - else: - (typ, data) = server.select(mailbox=mailbox) - if typ != 'OK': - self.logger.error('Failed to get mailbox %r: %s'%(mailbox, - data)) - return 1 - try: - numMessages = int(data[0]) - except ValueError, value: - self.logger.error('Invalid message count from mailbox %r'% - data[0]) - return 1 - for i in range(1, numMessages+1): - (typ, data) = server.fetch(str(i), '(RFC822)') + if not classname and self.has_prefix and self.pfxmode == 'strict': + raise MailUsageError, _(""" +The message you sent to roundup did not contain a properly formed subject +line. The subject must contain a class name or designator to indicate the +'topic' of the message. For example: + Subject: [issue] This is a new issue + - this will create a new issue in the tracker with the title 'This is + a new issue'. + Subject: [issue1234] This is a followup to issue 1234 + - this will append the message's contents to the existing issue 1234 + in the tracker. - # mark the message as deleted. - server.store(str(i), '+FLAGS', r'(\Deleted)') +Subject was: '%(subject)s' +""") % locals() - # process the message - s = cStringIO.StringIO(data[0][1]) - s.seek(0) - self.handle_Message(Message(s)) - server.close() - finally: + # try to get the class specified - if "loose" or "none" then fall + # back on the default + attempts = [] + if classname: + attempts.append(classname) + + if self.mailgw.default_class: + attempts.append(self.default_class) + else: + attempts.append(self.config['MAILGW_DEFAULT_CLASS']) + + # first valid class name wins + self.cl = None + for trycl in attempts: try: - server.expunge() - except: + self.cl = self.db.getclass(trycl) + classname = self.classname = trycl + break + except KeyError: pass - server.logout() - return 0 + if not self.cl: + validname = ', '.join(self.db.getclasses()) + if classname: + raise MailUsageError, _(""" +The class name you identified in the subject line ("%(classname)s") does +not exist in the database. +Valid class names are: %(validname)s +Subject was: "%(subject)s" +""") % locals() + else: + raise MailUsageError, _(""" +You did not identify a class name in the subject line and there is no +default set for this tracker. The subject must contain a class name or +designator to indicate the 'topic' of the message. For example: + Subject: [issue] This is a new issue + - this will create a new issue in the tracker with the title 'This is + a new issue'. + Subject: [issue1234] This is a followup to issue 1234 + - this will append the message's contents to the existing issue 1234 + in the tracker. - def do_apop(self, server, user='', password='', ssl=False): - ''' Do authentication POP - ''' - self._do_pop(server, user, password, True, ssl) +Subject was: '%(subject)s' +""") % locals() + # get the class properties + self.properties = self.cl.getprops() + - def do_pop(self, server, user='', password='', ssl=False): - ''' Do plain POP + def get_nodeid(self): + ''' Determine the nodeid from the message and return it if found ''' - self._do_pop(server, user, password, False, ssl) + title = self.matches['title'] + subject = self.subject + + if self.pfxmode == 'none': + nodeid = None + else: + nodeid = self.matches['nodeid'] - def _do_pop(self, server, user, password, apop, ssl): - '''Read a series of messages from the specified POP server. - ''' - import getpass, poplib, socket - try: - if not user: - user = raw_input('User: ') - if not password: - password = getpass.getpass() - except (KeyboardInterrupt, EOFError): - # Ctrl C or D maybe also Ctrl Z under Windows. - print "\nAborted by user." - return 1 + # try in-reply-to to match the message if there's no nodeid + inreplyto = self.message.getheader('in-reply-to') or '' + if nodeid is None and inreplyto: + l = self.db.getclass('msg').stringFind(messageid=inreplyto) + if l: + nodeid = self.cl.filter(None, {'messages':l})[0] - # open a connection to the server and retrieve all messages - try: - if ssl: - klass = poplib.POP3_SSL - else: - klass = poplib.POP3 - server = klass(server) - except socket.error: - self.logger.exception('POP server error') - return 1 - if apop: - server.apop(user, password) - else: - server.user(user) - server.pass_(password) - numMessages = len(server.list()[1]) - for i in range(1, numMessages+1): - # retr: returns - # [ pop response e.g. '+OK 459 octets', - # [ array of message lines ], - # number of octets ] - lines = server.retr(i)[1] - s = cStringIO.StringIO('\n'.join(lines)) - s.seek(0) - self.handle_Message(Message(s)) - # delete the message - server.dele(i) - - # quit the server to commit changes. - server.quit() - return 0 - def main(self, fp): - ''' fp - the file from which to read the Message. - ''' - return self.handle_Message(Message(fp)) + # but we do need either a title or a nodeid... + if nodeid is None and not title: + raise MailUsageError, _(""" +I cannot match your message to a node in the database - you need to either +supply a full designator (with number, eg "[issue123]") or keep the +previous subject title intact so I can match that. - def handle_Message(self, message): - """Handle an RFC822 Message +Subject was: "%(subject)s" +""") % locals() - Handle the Message object by calling handle_message() and then cope - with any errors raised by handle_message. - This method's job is to make that call and handle any - errors in a sane manner. It should be replaced if you wish to - handle errors in a different manner. - """ - # in some rare cases, a particularly stuffed-up e-mail will make - # its way into here... try to handle it gracefully + # If there's no nodeid, check to see if this is a followup and + # maybe someone's responded to the initial mail that created an + # entry. Try to find the matching nodes with the same title, and + # use the _last_ one matched (since that'll _usually_ be the most + # recent...). The subject_content_match config may specify an + # additional restriction based on the matched node's creation or + # activity. + tmatch_mode = self.config['MAILGW_SUBJECT_CONTENT_MATCH'] + if tmatch_mode != 'never' and nodeid is None and self.matches['refwd']: + l = self.cl.stringFind(title=title) + limit = None + if (tmatch_mode.startswith('creation') or + tmatch_mode.startswith('activity')): + limit, interval = tmatch_mode.split(' ', 1) + threshold = date.Date('.') - date.Interval(interval) + for id in l: + if limit: + if threshold < self.cl.get(id, limit): + nodeid = id + else: + nodeid = id - sendto = message.getaddrlist('resent-from') - if not sendto: - sendto = message.getaddrlist('from') - if not sendto: - # very bad-looking message - we don't even know who sent it - msg = ['Badly formed message from mail gateway. Headers:'] - msg.extend(message.headers) - msg = '\n'.join(map(str, msg)) - self.logger.error(msg) - return + # if a nodeid was specified, make sure it's valid + if nodeid is not None and not self.cl.hasnode(nodeid): + if self.pfxmode == 'strict': + raise MailUsageError, _(""" +The node specified by the designator in the subject of your message +("%(nodeid)s") does not exist. - msg = 'Handling message' - if message.getheader('message-id'): - msg += ' (Message-id=%r)'%message.getheader('message-id') - self.logger.info(msg) +Subject was: "%(subject)s" +""") % locals() + else: + nodeid = None + self.nodeid = nodeid - # try normal message-handling - if not self.trapExceptions: - return self.handle_message(message) + def get_author_id(self): + ''' Attempt to get the author id from the existing registered users, + otherwise attempt to register a new user and return their id + ''' + # Don't create users if anonymous isn't allowed to register + create = 1 + anonid = self.db.user.lookup('anonymous') + if not (self.db.security.hasPermission('Register', anonid, 'user') + and self.db.security.hasPermission('Email Access', anonid)): + create = 0 - # no, we want to trap exceptions - try: - return self.handle_message(message) - except MailUsageHelp: - # bounce the message back to the sender with the usage message - fulldoc = '\n'.join(string.split(__doc__, '\n')[2:]) - m = [''] - m.append('\n\nMail Gateway Help\n=================') - m.append(fulldoc) - self.mailer.bounce_message(message, [sendto[0][1]], m, - subject="Mail Gateway Help") - except MailUsageError, value: - # bounce the message back to the sender with the usage message - fulldoc = '\n'.join(string.split(__doc__, '\n')[2:]) - m = [''] - m.append(str(value)) - m.append('\n\nMail Gateway Help\n=================') - m.append(fulldoc) - self.mailer.bounce_message(message, [sendto[0][1]], m) - except Unauthorized, value: - # just inform the user that he is not authorized - m = [''] - m.append(str(value)) - self.mailer.bounce_message(message, [sendto[0][1]], m) - except IgnoreMessage: - # do not take any action - # this exception is thrown when email should be ignored - msg = 'IgnoreMessage raised' - if message.getheader('message-id'): - msg += ' (Message-id=%r)'%message.getheader('message-id') - self.logger.info(msg) - return - except: - msg = 'Exception handling message' - if message.getheader('message-id'): - msg += ' (Message-id=%r)'%message.getheader('message-id') - self.logger.exception(msg) + # ok, now figure out who the author is - create a new user if the + # "create" flag is true + author = uidFromAddress(self.db, self.from_list[0], create=create) - # bounce the message back to the sender with the error message - # let the admin know that something very bad is happening - m = [''] - m.append('An unexpected error occurred during the processing') - m.append('of your message. The tracker administrator is being') - m.append('notified.\n') - self.mailer.bounce_message(message, [sendto[0][1]], m) + # if we're not recognised, and we don't get added as a user, then we + # must be anonymous + if not author: + author = anonid - m.append('----------------') - m.append(traceback.format_exc()) - self.mailer.bounce_message(message, [self.instance.config.ADMIN_EMAIL], m) + # make sure the author has permission to use the email interface + if not self.db.security.hasPermission('Email Access', author): + if author == anonid: + # we're anonymous and we need to be a registered user + from_address = self.from_list[0][1] + registration_info = "" + if self.db.security.hasPermission('Web Access', author) and \ + self.db.security.hasPermission('Register', anonid, 'user'): + tracker_web = self.config.TRACKER_WEB + registration_info = """ Please register at: - def handle_message(self, message): - ''' message - a Message instance +%(tracker_web)suser?template=register - Parse the message as per the module docstring. - ''' - # get database handle for handling one email - self.db = self.instance.open ('admin') - try: - return self._handle_message (message) - finally: - self.db.close() +...before sending mail to the tracker.""" % locals() - def _handle_message(self, message): - ''' message - a Message instance + raise Unauthorized, _(""" +You are not a registered user.%(registration_info)s - Parse the message as per the module docstring. +Unknown address: %(from_address)s +""") % locals() + else: + # we're registered and we're _still_ not allowed access + raise Unauthorized, _( + 'You are not permitted to access this tracker.') + self.author = author - The implementation expects an opened database and a try/finally - that closes the database. + def check_node_permissions(self): + ''' Check if the author has permission to edit or create this + class of node ''' - # detect loops - if message.getheader('x-roundup-loop', ''): - raise IgnoreLoop + if self.nodeid: + if not self.db.security.hasPermission('Edit', self.author, + self.classname, itemid=self.nodeid): + raise Unauthorized, _( + 'You are not permitted to edit %(classname)s.' + ) % self.__dict__ + else: + if not self.db.security.hasPermission('Create', self.author, + self.classname): + raise Unauthorized, _( + 'You are not permitted to create %(classname)s.' + ) % self.__dict__ - # handle the subject line - subject = message.getheader('subject', '') - if not subject: - raise MailUsageError, _(""" -Emails to Roundup trackers must include a Subject: line! -""") + def commit_and_reopen_as_author(self): + ''' the author may have been created - make sure the change is + committed before we reopen the database + then re-open the database as the author + ''' + self.db.commit() - # detect Precedence: Bulk, or Microsoft Outlook autoreplies - if (message.getheader('precedence', '') == 'bulk' - or subject.lower().find("autoreply") > 0): - raise IgnoreBulk + # set the database user as the author + username = self.db.user.get(self.author, 'username') + self.db.setCurrentUser(username) - if subject.strip().lower() == 'help': - raise MailUsageHelp + # re-get the class with the new database connection + self.cl = self.db.getclass(self.classname) - # config is used many times in this method. - # make local variable for easier access - config = self.instance.config + def get_recipients(self): + ''' Get the list of recipients who were included in message and + register them as users if possible + ''' + # Don't create users if anonymous isn't allowed to register + create = 1 + anonid = self.db.user.lookup('anonymous') + if not (self.db.security.hasPermission('Register', anonid, 'user') + and self.db.security.hasPermission('Email Access', anonid)): + create = 0 - # determine the sender's address - from_list = message.getaddrlist('resent-from') - if not from_list: - from_list = message.getaddrlist('from') + # get the user class arguments from the commandline + user_props = self.mailgw.get_class_arguments('user') - # XXX Don't enable. This doesn't work yet. -# "[^A-z.]tracker\+(?P<classname>[^\d\s]+)(?P<nodeid>\d+)\@some.dom.ain[^A-z.]" - # handle delivery to addresses like:tracker+issue25@some.dom.ain - # use the embedded issue number as our issue -# issue_re = config['MAILGW_ISSUE_ADDRESS_RE'] -# if issue_re: -# for header in ['to', 'cc', 'bcc']: -# addresses = message.getheader(header, '') -# if addresses: -# # FIXME, this only finds the first match in the addresses. -# issue = re.search(issue_re, addresses, 'i') -# if issue: -# classname = issue.group('classname') -# nodeid = issue.group('nodeid') -# break - - # Matches subjects like: - # Re: "[issue1234] title of issue [status=resolved]" - - # Alias since we need a reference to the original subject for - # later use in error messages - tmpsubject = subject - - sd_open, sd_close = config['MAILGW_SUBJECT_SUFFIX_DELIMITERS'] - delim_open = re.escape(sd_open) - if delim_open in '[(': delim_open = '\\' + delim_open - delim_close = re.escape(sd_close) - if delim_close in '[(': delim_close = '\\' + delim_close + # now update the recipients list + recipients = [] + tracker_email = self.config['TRACKER_EMAIL'].lower() + msg_to = self.message.getaddrlist('to') + msg_cc = self.message.getaddrlist('cc') + for recipient in msg_to + msg_cc: + r = recipient[1].strip().lower() + if r == tracker_email or not r: + continue - matches = dict.fromkeys(['refwd', 'quote', 'classname', - 'nodeid', 'title', 'args', - 'argswhole']) + # look up the recipient - create if necessary (and we're + # allowed to) + recipient = uidFromAddress(self.db, recipient, create, **user_props) - # Look for Re: et. al. Used later on for MAILGW_SUBJECT_CONTENT_MATCH - re_re = r"(?P<refwd>%s)\s*" % config["MAILGW_REFWD_RE"].pattern - m = re.match(re_re, tmpsubject, re.IGNORECASE|re.VERBOSE|re.UNICODE) - if m: - m = m.groupdict() - if m['refwd']: - matches.update(m) - tmpsubject = tmpsubject[len(m['refwd']):] # Consume Re: + # if all's well, add the recipient to the list + if recipient: + recipients.append(recipient) + self.recipients = recipients - # Look for Leading " - m = re.match(r'(?P<quote>\s*")', tmpsubject, - re.IGNORECASE) - if m: - matches.update(m.groupdict()) - tmpsubject = tmpsubject[len(matches['quote']):] # Consume quote + def get_props(self): + ''' Generate all the props for the new/updated node and return them + ''' + subject = self.subject + + # get the commandline arguments for issues + issue_props = self.mailgw.get_class_arguments('issue', self.classname) + + # + # handle the subject argument list + # + # figure what the properties of this Class are + props = {} + args = self.matches['args'] + argswhole = self.matches['argswhole'] + title = self.matches['title'] + + # Reform the title + if self.matches['nodeid'] and self.nodeid is None: + title = subject + + if args: + if self.sfxmode == 'none': + title += ' ' + argswhole + else: + errors, props = setPropArrayFromString(self, self.cl, args, + self.nodeid) + # handle any errors parsing the argument list + if errors: + if self.sfxmode == 'strict': + errors = '\n- '.join(map(str, errors)) + raise MailUsageError, _(""" +There were problems handling your subject line argument list: +- %(errors)s - has_prefix = re.search(r'^%s(\w+)%s'%(delim_open, - delim_close), tmpsubject.strip()) +Subject was: "%(subject)s" +""") % locals() + else: + title += ' ' + argswhole - class_re = r'%s(?P<classname>(%s))(?P<nodeid>\d+)?%s'%(delim_open, - "|".join(self.db.getclasses()), delim_close) - # Note: re.search, not re.match as there might be garbage - # (mailing list prefix, etc.) before the class identifier - m = re.search(class_re, tmpsubject, re.IGNORECASE) - if m: - matches.update(m.groupdict()) - # Skip to the end of the class identifier, including any - # garbage before it. - tmpsubject = tmpsubject[m.end():] - - # if we've not found a valid classname prefix then force the - # scanning to handle there being a leading delimiter - title_re = r'(?P<title>%s[^%s]*)'%( - not matches['classname'] and '.' or '', delim_open) - m = re.match(title_re, tmpsubject.strip(), re.IGNORECASE) - if m: - matches.update(m.groupdict()) - tmpsubject = tmpsubject[len(matches['title']):] # Consume title + # set the issue title to the subject + title = title.strip() + if (title and self.properties.has_key('title') and not + issue_props.has_key('title')): + issue_props['title'] = title + if (self.nodeid and self.properties.has_key('title') and not + self.config['MAILGW_SUBJECT_UPDATES_TITLE']): + issue_props['title'] = self.cl.get(self.nodeid,'title') + + # merge the command line props defined in issue_props into + # the props dictionary because function(**props, **issue_props) + # is a syntax error. + for prop in issue_props.keys() : + if not props.has_key(prop) : + props[prop] = issue_props[prop] + + self.props = props + + def get_pgp_message(self): + ''' If they've enabled PGP processing then verify the signature + or decrypt the message + ''' + def pgp_role(): + """ if PGP_ROLES is specified the user must have a Role in the list + or we will skip PGP processing + """ + if self.config.PGP_ROLES: + return self.db.user.has_role(self.author, + iter_roles(self.config.PGP_ROLES)) + else: + return True - args_re = r'(?P<argswhole>%s(?P<args>.+?)%s)?'%(delim_open, - delim_close) - m = re.search(args_re, tmpsubject.strip(), re.IGNORECASE|re.VERBOSE) - if m: - matches.update(m.groupdict()) + if self.config.PGP_ENABLE and pgp_role(): + assert pyme, 'pyme is not installed' + # signed/encrypted mail must come from the primary address + author_address = self.db.user.get(self.author, 'address') + if self.config.PGP_HOMEDIR: + os.environ['GNUPGHOME'] = self.config.PGP_HOMEDIR + if self.message.pgp_signed(): + self.message.verify_signature(author_address) + elif self.message.pgp_encrypted(): + # replace message with the contents of the decrypted + # message for content extraction + # TODO: encrypted message handling is far from perfect + # bounces probably include the decrypted message, for + # instance :( + self.message = self.message.decrypt(author_address) + else: + raise MailUsageError, _(""" +This tracker has been configured to require all email be PGP signed or +encrypted.""") - # figure subject line parsing modes - pfxmode = config['MAILGW_SUBJECT_PREFIX_PARSING'] - sfxmode = config['MAILGW_SUBJECT_SUFFIX_PARSING'] + def get_content_and_attachments(self): + ''' get the attachments and first text part from the message + ''' + ig = self.config.MAILGW_IGNORE_ALTERNATIVES + self.content, self.attachments = self.message.extract_content( + ignore_alternatives=ig, + unpack_rfc822=self.config.MAILGW_UNPACK_RFC822) + + + def create_files(self): + ''' Create a file for each attachment in the message + ''' + if not self.properties.has_key('files'): + return + files = [] + file_props = self.mailgw.get_class_arguments('file') + + if self.attachments: + for (name, mime_type, data) in self.attachments: + if not self.db.security.hasPermission('Create', self.author, + 'file'): + raise Unauthorized, _( + 'You are not permitted to create files.') + if not name: + name = "unnamed" + try: + fileid = self.db.file.create(type=mime_type, name=name, + content=data, **file_props) + except exceptions.Reject: + pass + else: + files.append(fileid) + # allowed to attach the files to an existing node? + if self.nodeid and not self.db.security.hasPermission('Edit', + self.author, self.classname, 'files'): + raise Unauthorized, _( + 'You are not permitted to add files to %(classname)s.' + ) % self.__dict__ - # check for registration OTK - # or fallback on the default class - if self.db.config['EMAIL_REGISTRATION_CONFIRMATION']: - otk_re = re.compile('-- key (?P<otk>[a-zA-Z0-9]{32})') - otk = otk_re.search(matches['title'] or '') - if otk: - self.db.confirm_registration(otk.group('otk')) - subject = 'Your registration to %s is complete' % \ - config['TRACKER_NAME'] - sendto = [from_list[0][1]] - self.mailer.standard_message(sendto, subject, '') - return + if self.nodeid: + # extend the existing files list + fileprop = self.cl.get(self.nodeid, 'files') + fileprop.extend(files) + files = fileprop - # get the classname - if pfxmode == 'none': - classname = None - else: - classname = matches['classname'] + self.props['files'] = files - if not classname and has_prefix and pfxmode == 'strict': + def create_msg(self): + ''' Create msg containing all the relevant information from the message + ''' + if not self.properties.has_key('messages'): + return + msg_props = self.mailgw.get_class_arguments('msg') + + # Get the message ids + inreplyto = self.message.getheader('in-reply-to') or '' + messageid = self.message.getheader('message-id') + # generate a messageid if there isn't one + if not messageid: + messageid = "<%s.%s.%s%s@%s>"%(time.time(), random.random(), + self.classname, self.nodeid, self.config['MAIL_DOMAIN']) + + if self.content is None: raise MailUsageError, _(""" -The message you sent to roundup did not contain a properly formed subject -line. The subject must contain a class name or designator to indicate the -'topic' of the message. For example: - Subject: [issue] This is a new issue - - this will create a new issue in the tracker with the title 'This is - a new issue'. - Subject: [issue1234] This is a followup to issue 1234 - - this will append the message's contents to the existing issue 1234 - in the tracker. - -Subject was: '%(subject)s' -""") % locals() +Roundup requires the submission to be plain text. The message parser could +not find a text/plain part to use. +""") - # try to get the class specified - if "loose" or "none" then fall - # back on the default - attempts = [] - if classname: - attempts.append(classname) + # parse the body of the message, stripping out bits as appropriate + summary, content = parseContent(self.content, config=self.config) + content = content.strip() - if self.default_class: - attempts.append(self.default_class) - else: - attempts.append(config['MAILGW_DEFAULT_CLASS']) + if content: + if not self.db.security.hasPermission('Create', self.author, 'msg'): + raise Unauthorized, _( + 'You are not permitted to create messages.') - # first valid class name wins - cl = None - for trycl in attempts: try: - cl = self.db.getclass(trycl) - classname = trycl - break - except KeyError: - pass - - if not cl: - validname = ', '.join(self.db.getclasses()) - if classname: + message_id = self.db.msg.create(author=self.author, + recipients=self.recipients, date=date.Date('.'), + summary=summary, content=content, files=self.props['files'], + messageid=messageid, inreplyto=inreplyto, **msg_props) + except exceptions.Reject, error: raise MailUsageError, _(""" -The class name you identified in the subject line ("%(classname)s") does -not exist in the database. - -Valid class names are: %(validname)s -Subject was: "%(subject)s" +Mail message was rejected by a detector. +%(error)s """) % locals() + # allowed to attach the message to the existing node? + if self.nodeid and not self.db.security.hasPermission('Edit', + self.author, self.classname, 'messages'): + raise Unauthorized, _( + 'You are not permitted to add messages to %(classname)s.' + ) % self.__dict__ + + if self.nodeid: + # add the message to the node's list + messages = self.cl.get(self.nodeid, 'messages') + messages.append(message_id) + self.props['messages'] = messages else: - raise MailUsageError, _(""" -You did not identify a class name in the subject line and there is no -default set for this tracker. The subject must contain a class name or -designator to indicate the 'topic' of the message. For example: - Subject: [issue] This is a new issue - - this will create a new issue in the tracker with the title 'This is - a new issue'. - Subject: [issue1234] This is a followup to issue 1234 - - this will append the message's contents to the existing issue 1234 - in the tracker. + # pre-load the messages list + self.props['messages'] = [message_id] -Subject was: '%(subject)s' + def create_node(self): + ''' Create/update a node using self.props + ''' + classname = self.classname + try: + if self.nodeid: + # Check permissions for each property + for prop in self.props.keys(): + if not self.db.security.hasPermission('Edit', self.author, + classname, prop): + raise Unauthorized, _('You are not permitted to edit ' + 'property %(prop)s of class %(classname)s.' + ) % locals() + self.cl.set(self.nodeid, **self.props) + else: + # Check permissions for each property + for prop in self.props.keys(): + if not self.db.security.hasPermission('Create', self.author, + classname, prop): + raise Unauthorized, _('You are not permitted to set ' + 'property %(prop)s of class %(classname)s.' + ) % locals() + self.nodeid = self.cl.create(**self.props) + except (TypeError, IndexError, ValueError, exceptions.Reject), message: + raise MailUsageError, _(""" +There was a problem with the message you sent: + %(message)s """) % locals() - # get the optional nodeid - if pfxmode == 'none': - nodeid = None - else: - nodeid = matches['nodeid'] + return self.nodeid - # try in-reply-to to match the message if there's no nodeid - inreplyto = message.getheader('in-reply-to') or '' - if nodeid is None and inreplyto: - l = self.db.getclass('msg').stringFind(messageid=inreplyto) - if l: - nodeid = cl.filter(None, {'messages':l})[0] - # title is optional too - title = matches['title'] - if title: - title = title.strip() - else: - title = '' - # strip off the quotes that dumb emailers put around the subject, like - # Re: "[issue1] bla blah" - if matches['quote'] and title.endswith('"'): - title = title[:-1] +class MailGW: - # but we do need either a title or a nodeid... - if nodeid is None and not title: - raise MailUsageError, _(""" -I cannot match your message to a node in the database - you need to either -supply a full designator (with number, eg "[issue123]") or keep the -previous subject title intact so I can match that. + def __init__(self, instance, arguments=()): + self.instance = instance + self.arguments = arguments + self.default_class = None + for option, value in self.arguments: + if option == '-c': + self.default_class = value.strip() -Subject was: "%(subject)s" -""") % locals() + self.mailer = Mailer(instance.config) + self.logger = logging.getLogger('roundup.mailgw') - # If there's no nodeid, check to see if this is a followup and - # maybe someone's responded to the initial mail that created an - # entry. Try to find the matching nodes with the same title, and - # use the _last_ one matched (since that'll _usually_ be the most - # recent...). The subject_content_match config may specify an - # additional restriction based on the matched node's creation or - # activity. - tmatch_mode = config['MAILGW_SUBJECT_CONTENT_MATCH'] - if tmatch_mode != 'never' and nodeid is None and matches['refwd']: - l = cl.stringFind(title=title) - limit = None - if (tmatch_mode.startswith('creation') or - tmatch_mode.startswith('activity')): - limit, interval = tmatch_mode.split(' ', 1) - threshold = date.Date('.') - date.Interval(interval) - for id in l: - if limit: - if threshold < cl.get(id, limit): - nodeid = id - else: - nodeid = id + # should we trap exceptions (normal usage) or pass them through + # (for testing) + self.trapExceptions = 1 - # if a nodeid was specified, make sure it's valid - if nodeid is not None and not cl.hasnode(nodeid): - if pfxmode == 'strict': - raise MailUsageError, _(""" -The node specified by the designator in the subject of your message -("%(nodeid)s") does not exist. + def do_pipe(self): + """ Read a message from standard input and pass it to the mail handler. -Subject was: "%(subject)s" -""") % locals() + Read into an internal structure that we can seek on (in case + there's an error). + + XXX: we may want to read this into a temporary file instead... + """ + s = cStringIO.StringIO() + s.write(sys.stdin.read()) + s.seek(0) + self.main(s) + return 0 + + def do_mailbox(self, filename): + """ Read a series of messages from the specified unix mailbox file and + pass each to the mail handler. + """ + # open the spool file and lock it + import fcntl + # FCNTL is deprecated in py2.3 and fcntl takes over all the symbols + if hasattr(fcntl, 'LOCK_EX'): + FCNTL = fcntl + else: + import FCNTL + f = open(filename, 'r+') + fcntl.flock(f.fileno(), FCNTL.LOCK_EX) + + # handle and clear the mailbox + try: + from mailbox import UnixMailbox + mailbox = UnixMailbox(f, factory=Message) + # grab one message + message = mailbox.next() + while message: + # handle this message + self.handle_Message(message) + message = mailbox.next() + # nuke the file contents + os.ftruncate(f.fileno(), 0) + except: + import traceback + traceback.print_exc() + return 1 + fcntl.flock(f.fileno(), FCNTL.LOCK_UN) + return 0 + + def do_imap(self, server, user='', password='', mailbox='', ssl=0, + cram=0): + ''' Do an IMAP connection + ''' + import getpass, imaplib, socket + try: + if not user: + user = raw_input('User: ') + if not password: + password = getpass.getpass() + except (KeyboardInterrupt, EOFError): + # Ctrl C or D maybe also Ctrl Z under Windows. + print "\nAborted by user." + return 1 + # open a connection to the server and retrieve all messages + try: + if ssl: + self.logger.debug('Trying server %r with ssl'%server) + server = imaplib.IMAP4_SSL(server) else: - title = subject - nodeid = None + self.logger.debug('Trying server %r without ssl'%server) + server = imaplib.IMAP4(server) + except (imaplib.IMAP4.error, socket.error, socket.sslerror): + self.logger.exception('IMAP server error') + return 1 - # Handle the arguments specified by the email gateway command line. - # We do this by looping over the list of self.arguments looking for - # a -C to tell us what class then the -S setting string. - msg_props = {} - user_props = {} - file_props = {} - issue_props = {} - # so, if we have any arguments, use them - if self.arguments: - current_class = 'msg' - for option, propstring in self.arguments: - if option in ( '-C', '--class'): - current_class = propstring.strip() - # XXX this is not flexible enough. - # we should chect for subclasses of these classes, - # not for the class name... - if current_class not in ('msg', 'file', 'user', 'issue'): - mailadmin = config['ADMIN_EMAIL'] - raise MailUsageError, _(""" -The mail gateway is not properly set up. Please contact -%(mailadmin)s and have them fix the incorrect class specified as: - %(current_class)s -""") % locals() - if option in ('-S', '--set'): - if current_class == 'issue' : - errors, issue_props = setPropArrayFromString(self, - cl, propstring.strip(), nodeid) - elif current_class == 'file' : - temp_cl = self.db.getclass('file') - errors, file_props = setPropArrayFromString(self, - temp_cl, propstring.strip()) - elif current_class == 'msg' : - temp_cl = self.db.getclass('msg') - errors, msg_props = setPropArrayFromString(self, - temp_cl, propstring.strip()) - elif current_class == 'user' : - temp_cl = self.db.getclass('user') - errors, user_props = setPropArrayFromString(self, - temp_cl, propstring.strip()) - if errors: - mailadmin = config['ADMIN_EMAIL'] - raise MailUsageError, _(""" -The mail gateway is not properly set up. Please contact -%(mailadmin)s and have them fix the incorrect properties: - %(errors)s -""") % locals() + try: + if cram: + server.login_cram_md5(user, password) + else: + server.login(user, password) + except imaplib.IMAP4.error, e: + self.logger.exception('IMAP login failure') + return 1 - # - # handle the users - # - # Don't create users if anonymous isn't allowed to register - create = 1 - anonid = self.db.user.lookup('anonymous') - if not (self.db.security.hasPermission('Register', anonid, 'user') - and self.db.security.hasPermission('Email Access', anonid)): - create = 0 + try: + if not mailbox: + (typ, data) = server.select() + else: + (typ, data) = server.select(mailbox=mailbox) + if typ != 'OK': + self.logger.error('Failed to get mailbox %r: %s'%(mailbox, + data)) + return 1 + try: + numMessages = int(data[0]) + except ValueError, value: + self.logger.error('Invalid message count from mailbox %r'% + data[0]) + return 1 + for i in range(1, numMessages+1): + (typ, data) = server.fetch(str(i), '(RFC822)') - # ok, now figure out who the author is - create a new user if the - # "create" flag is true - author = uidFromAddress(self.db, from_list[0], create=create) + # mark the message as deleted. + server.store(str(i), '+FLAGS', r'(\Deleted)') - # if we're not recognised, and we don't get added as a user, then we - # must be anonymous - if not author: - author = anonid + # process the message + s = cStringIO.StringIO(data[0][1]) + s.seek(0) + self.handle_Message(Message(s)) + server.close() + finally: + try: + server.expunge() + except: + pass + server.logout() + + return 0 + + + def do_apop(self, server, user='', password='', ssl=False): + ''' Do authentication POP + ''' + self._do_pop(server, user, password, True, ssl) + + def do_pop(self, server, user='', password='', ssl=False): + ''' Do plain POP + ''' + self._do_pop(server, user, password, False, ssl) + + def _do_pop(self, server, user, password, apop, ssl): + '''Read a series of messages from the specified POP server. + ''' + import getpass, poplib, socket + try: + if not user: + user = raw_input('User: ') + if not password: + password = getpass.getpass() + except (KeyboardInterrupt, EOFError): + # Ctrl C or D maybe also Ctrl Z under Windows. + print "\nAborted by user." + return 1 + + # open a connection to the server and retrieve all messages + try: + if ssl: + klass = poplib.POP3_SSL + else: + klass = poplib.POP3 + server = klass(server) + except socket.error: + self.logger.exception('POP server error') + return 1 + if apop: + server.apop(user, password) + else: + server.user(user) + server.pass_(password) + numMessages = len(server.list()[1]) + for i in range(1, numMessages+1): + # retr: returns + # [ pop response e.g. '+OK 459 octets', + # [ array of message lines ], + # number of octets ] + lines = server.retr(i)[1] + s = cStringIO.StringIO('\n'.join(lines)) + s.seek(0) + self.handle_Message(Message(s)) + # delete the message + server.dele(i) + + # quit the server to commit changes. + server.quit() + return 0 + + def main(self, fp): + ''' fp - the file from which to read the Message. + ''' + return self.handle_Message(Message(fp)) - # make sure the author has permission to use the email interface - if not self.db.security.hasPermission('Email Access', author): - if author == anonid: - # we're anonymous and we need to be a registered user - from_address = from_list[0][1] - registration_info = "" - if self.db.security.hasPermission('Web Access', author) and \ - self.db.security.hasPermission('Register', anonid, 'user'): - tracker_web = self.instance.config.TRACKER_WEB - registration_info = """ Please register at: + def handle_Message(self, message): + """Handle an RFC822 Message -%(tracker_web)suser?template=register + Handle the Message object by calling handle_message() and then cope + with any errors raised by handle_message. + This method's job is to make that call and handle any + errors in a sane manner. It should be replaced if you wish to + handle errors in a different manner. + """ + # in some rare cases, a particularly stuffed-up e-mail will make + # its way into here... try to handle it gracefully -...before sending mail to the tracker.""" % locals() + sendto = message.getaddrlist('resent-from') + if not sendto: + sendto = message.getaddrlist('from') + if not sendto: + # very bad-looking message - we don't even know who sent it + msg = ['Badly formed message from mail gateway. Headers:'] + msg.extend(message.headers) + msg = '\n'.join(map(str, msg)) + self.logger.error(msg) + return - raise Unauthorized, _(""" -You are not a registered user.%(registration_info)s + msg = 'Handling message' + if message.getheader('message-id'): + msg += ' (Message-id=%r)'%message.getheader('message-id') + self.logger.info(msg) -Unknown address: %(from_address)s -""") % locals() - else: - # we're registered and we're _still_ not allowed access - raise Unauthorized, _( - 'You are not permitted to access this tracker.') + # try normal message-handling + if not self.trapExceptions: + return self.handle_message(message) - # make sure they're allowed to edit or create this class of information - if nodeid: - if not self.db.security.hasPermission('Edit', author, classname, - itemid=nodeid): - raise Unauthorized, _( - 'You are not permitted to edit %(classname)s.') % locals() - else: - if not self.db.security.hasPermission('Create', author, classname): - raise Unauthorized, _( - 'You are not permitted to create %(classname)s.' - ) % locals() + # no, we want to trap exceptions + try: + return self.handle_message(message) + except MailUsageHelp: + # bounce the message back to the sender with the usage message + fulldoc = '\n'.join(string.split(__doc__, '\n')[2:]) + m = [''] + m.append('\n\nMail Gateway Help\n=================') + m.append(fulldoc) + self.mailer.bounce_message(message, [sendto[0][1]], m, + subject="Mail Gateway Help") + except MailUsageError, value: + # bounce the message back to the sender with the usage message + fulldoc = '\n'.join(string.split(__doc__, '\n')[2:]) + m = [''] + m.append(str(value)) + m.append('\n\nMail Gateway Help\n=================') + m.append(fulldoc) + self.mailer.bounce_message(message, [sendto[0][1]], m) + except Unauthorized, value: + # just inform the user that he is not authorized + m = [''] + m.append(str(value)) + self.mailer.bounce_message(message, [sendto[0][1]], m) + except IgnoreMessage: + # do not take any action + # this exception is thrown when email should be ignored + msg = 'IgnoreMessage raised' + if message.getheader('message-id'): + msg += ' (Message-id=%r)'%message.getheader('message-id') + self.logger.info(msg) + return + except: + msg = 'Exception handling message' + if message.getheader('message-id'): + msg += ' (Message-id=%r)'%message.getheader('message-id') + self.logger.exception(msg) - # the author may have been created - make sure the change is - # committed before we reopen the database - self.db.commit() + # bounce the message back to the sender with the error message + # let the admin know that something very bad is happening + m = [''] + m.append('An unexpected error occurred during the processing') + m.append('of your message. The tracker administrator is being') + m.append('notified.\n') + self.mailer.bounce_message(message, [sendto[0][1]], m) - # set the database user as the author - username = self.db.user.get(author, 'username') - self.db.setCurrentUser(username) + m.append('----------------') + m.append(traceback.format_exc()) + self.mailer.bounce_message(message, [self.instance.config.ADMIN_EMAIL], m) - # re-get the class with the new database connection - cl = self.db.getclass(classname) + def handle_message(self, message): + ''' message - a Message instance - # now update the recipients list - recipients = [] - tracker_email = config['TRACKER_EMAIL'].lower() - for recipient in message.getaddrlist('to') + message.getaddrlist('cc'): - r = recipient[1].strip().lower() - if r == tracker_email or not r: - continue + Parse the message as per the module docstring. + ''' + # get database handle for handling one email + self.db = self.instance.open ('admin') + try: + return self._handle_message(message) + finally: + self.db.close() - # look up the recipient - create if necessary (and we're - # allowed to) - recipient = uidFromAddress(self.db, recipient, create, **user_props) + def _handle_message(self, message): + ''' message - a Message instance - # if all's well, add the recipient to the list - if recipient: - recipients.append(recipient) + Parse the message as per the module docstring. + The following code expects an opened database and a try/finally + that closes the database. + ''' + parsed_message = parsedMessage(self, message) - # - # handle the subject argument list - # - # figure what the properties of this Class are - properties = cl.getprops() - props = {} - args = matches['args'] - argswhole = matches['argswhole'] - if args: - if sfxmode == 'none': - title += ' ' + argswhole - else: - errors, props = setPropArrayFromString(self, cl, args, nodeid) - # handle any errors parsing the argument list - if errors: - if sfxmode == 'strict': - errors = '\n- '.join(map(str, errors)) - raise MailUsageError, _(""" -There were problems handling your subject line argument list: -- %(errors)s + # Filter out messages to ignore + parsed_message.handle_ignore() + + # Check for usage/help requests + parsed_message.handle_help() + + # Check if the subject line is valid + parsed_message.check_subject() -Subject was: "%(subject)s" -""") % locals() - else: - title += ' ' + argswhole + # XXX Don't enable. This doesn't work yet. + # XXX once this works it should be moved to parsedMessage class +# "[^A-z.]tracker\+(?P<classname>[^\d\s]+)(?P<nodeid>\d+)\@some.dom.ain[^A-z.]" + # handle delivery to addresses like:tracker+issue25@some.dom.ain + # use the embedded issue number as our issue +# issue_re = config['MAILGW_ISSUE_ADDRESS_RE'] +# if issue_re: +# for header in ['to', 'cc', 'bcc']: +# addresses = message.getheader(header, '') +# if addresses: +# # FIXME, this only finds the first match in the addresses. +# issue = re.search(issue_re, addresses, 'i') +# if issue: +# classname = issue.group('classname') +# nodeid = issue.group('nodeid') +# break + + # Parse the subject line to get the importants parts + parsed_message.parse_subject() + # check for registration OTK + if parsed_message.rego_confirm(): + return - # set the issue title to the subject - title = title.strip() - if (title and properties.has_key('title') and not - issue_props.has_key('title')): - issue_props['title'] = title - if (nodeid and properties.has_key('title') and not - config['MAILGW_SUBJECT_UPDATES_TITLE']): - issue_props['title'] = cl.get(nodeid,'title') + # get the classname + parsed_message.get_classname() - # - # handle message-id and in-reply-to - # - messageid = message.getheader('message-id') - # generate a messageid if there isn't one - if not messageid: - messageid = "<%s.%s.%s%s@%s>"%(time.time(), random.random(), - classname, nodeid, config['MAIL_DOMAIN']) + # get the optional nodeid + parsed_message.get_nodeid() - # if they've enabled PGP processing then verify the signature - # or decrypt the message + # Determine who the author is + parsed_message.get_author_id() + + # make sure they're allowed to edit or create this class + parsed_message.check_node_permissions() - # if PGP_ROLES is specified the user must have a Role in the list - # or we will skip PGP processing - def pgp_role(): - if self.instance.config.PGP_ROLES: - return self.db.user.has_role(author, - iter_roles(self.instance.config.PGP_ROLES)) - else: - return True + # author may have been created: + # commit author to database and re-open as author + parsed_message.commit_and_reopen_as_author() - if self.instance.config.PGP_ENABLE and pgp_role(): - assert pyme, 'pyme is not installed' - # signed/encrypted mail must come from the primary address - author_address = self.db.user.get(author, 'address') - if self.instance.config.PGP_HOMEDIR: - os.environ['GNUPGHOME'] = self.instance.config.PGP_HOMEDIR - if message.pgp_signed(): - message.verify_signature(author_address) - elif message.pgp_encrypted(): - # replace message with the contents of the decrypted - # message for content extraction - # TODO: encrypted message handling is far from perfect - # bounces probably include the decrypted message, for - # instance :( - message = message.decrypt(author_address) - else: - raise MailUsageError, _(""" -This tracker has been configured to require all email be PGP signed or -encrypted.""") - # now handle the body - find the message - ig = self.instance.config.MAILGW_IGNORE_ALTERNATIVES - content, attachments = message.extract_content(ignore_alternatives=ig, - unpack_rfc822=self.instance.config.MAILGW_UNPACK_RFC822) - if content is None: - raise MailUsageError, _(""" -Roundup requires the submission to be plain text. The message parser could -not find a text/plain part to use. -""") + # Get the recipients list + parsed_message.get_recipients() - # parse the body of the message, stripping out bits as appropriate - summary, content = parseContent(content, config=config) - content = content.strip() + # get the new/updated node props + parsed_message.get_props() - # - # handle the attachments - # - files = [] - if attachments and properties.has_key('files'): - for (name, mime_type, data) in attachments: - if not self.db.security.hasPermission('Create', author, 'file'): - raise Unauthorized, _( - 'You are not permitted to create files.') - if not name: - name = "unnamed" - try: - fileid = self.db.file.create(type=mime_type, name=name, - content=data, **file_props) - except exceptions.Reject: - pass - else: - files.append(fileid) - # allowed to attach the files to an existing node? - if nodeid and not self.db.security.hasPermission('Edit', author, - classname, 'files'): - raise Unauthorized, _( - 'You are not permitted to add files to %(classname)s.' - ) % locals() + # Handle PGP signed or encrypted messages + parsed_message.get_pgp_message() - if nodeid: - # extend the existing files list - fileprop = cl.get(nodeid, 'files') - fileprop.extend(files) - props['files'] = fileprop - else: - # pre-load the files list - props['files'] = files + # extract content and attachments from message body + parsed_message.get_content_and_attachments() - # + # put attachments into files linked to the issue + parsed_message.create_files() + # create the message if there's a message body (content) - # - if (content and properties.has_key('messages')): - if not self.db.security.hasPermission('Create', author, 'msg'): - raise Unauthorized, _( - 'You are not permitted to create messages.') + parsed_message.create_msg() + + # perform the node change / create + nodeid = parsed_message.create_node() - try: - message_id = self.db.msg.create(author=author, - recipients=recipients, date=date.Date('.'), - summary=summary, content=content, files=files, - messageid=messageid, inreplyto=inreplyto, **msg_props) - except exceptions.Reject, error: - raise MailUsageError, _(""" -Mail message was rejected by a detector. -%(error)s -""") % locals() - # allowed to attach the message to the existing node? - if nodeid and not self.db.security.hasPermission('Edit', author, - classname, 'messages'): - raise Unauthorized, _( - 'You are not permitted to add messages to %(classname)s.' - ) % locals() + # commit the changes to the DB + self.db.commit() - if nodeid: - # add the message to the node's list - messages = cl.get(nodeid, 'messages') - messages.append(message_id) - props['messages'] = messages - else: - # pre-load the messages list - props['messages'] = [message_id] + return nodeid - # - # perform the node change / create - # + def get_class_arguments(self, class_type, classname=None): + ''' class_type - a valid node class type: + - 'user' refers to the author of a message + - 'issue' refers to an issue-type class (to which the + message is appended) specified in parameter classname + Note that this need not be the real classname, we get + the real classname used as a parameter (from previous + message-parsing steps) + - 'file' specifies a file-type class + - 'msg' is the message-class + classname - the name of the current issue-type class + + Parse the commandline arguments and retrieve the properties that + are relevant to the class_type. We now allow multiple -S options + per class_type (-C option). + ''' + allprops = {} + + classname = classname or class_type + cls_lookup = { 'issue' : classname } + + # Allow other issue-type classes -- take the real classname from + # previous parsing-steps of the message: + clsname = cls_lookup.get (class_type, class_type) + + # check if the clsname is valid try: - # merge the command line props defined in issue_props into - # the props dictionary because function(**props, **issue_props) - # is a syntax error. - for prop in issue_props.keys() : - if not props.has_key(prop) : - props[prop] = issue_props[prop] - - if nodeid: - # Check permissions for each property - for prop in props.keys(): - if not self.db.security.hasPermission('Edit', author, - classname, prop): - raise Unauthorized, _('You are not permitted to edit ' - 'property %(prop)s of class %(classname)s.') % locals() - cl.set(nodeid, **props) - else: - # Check permissions for each property - for prop in props.keys(): - if not self.db.security.hasPermission('Create', author, - classname, prop): - raise Unauthorized, _('You are not permitted to set ' - 'property %(prop)s of class %(classname)s.') % locals() - nodeid = cl.create(**props) - except (TypeError, IndexError, ValueError, exceptions.Reject), message: + self.db.getclass(clsname) + except KeyError: + mailadmin = self.instance.config['ADMIN_EMAIL'] raise MailUsageError, _(""" -There was a problem with the message you sent: - %(message)s +The mail gateway is not properly set up. Please contact +%(mailadmin)s and have them fix the incorrect class specified as: + %(clsname)s """) % locals() + + if self.arguments: + # The default type on the commandline is msg + if class_type == 'msg': + current_type = class_type + else: + current_type = None + + # Handle the arguments specified by the email gateway command line. + # We do this by looping over the list of self.arguments looking for + # a -C to match the class we want, then use the -S setting string. + for option, propstring in self.arguments: + if option in ( '-C', '--class'): + current_type = propstring.strip() + + if current_type != class_type: + current_type = None - # commit the changes to the DB - self.db.commit() + elif current_type and option in ('-S', '--set'): + cls = cls_lookup.get (current_type, current_type) + temp_cl = self.db.getclass(cls) + errors, props = setPropArrayFromString(self, + temp_cl, propstring.strip()) - return nodeid + if errors: + mailadmin = self.instance.config['ADMIN_EMAIL'] + raise MailUsageError, _(""" +The mail gateway is not properly set up. Please contact +%(mailadmin)s and have them fix the incorrect properties: + %(errors)s +""") % locals() + allprops.update(props) + + return allprops def setPropArrayFromString(self, cl, propString, nodeid=None): diff --git a/test/test_mailgw.py b/test/test_mailgw.py index 29d063a..7c11bef 100644 --- a/test/test_mailgw.py +++ b/test/test_mailgw.py @@ -149,16 +149,16 @@ class MailgwTestCase(unittest.TestCase, DiffHelper): os.remove(SENDMAILDEBUG) self.db.close() - def _create_mailgw(self, message): + def _create_mailgw(self, message, args=()): class MailGW(self.instance.MailGW): def handle_message(self, message): return self._handle_message(message) - handler = MailGW(self.instance) + handler = MailGW(self.instance, args) handler.db = self.db return handler - def _handle_mail(self, message): - handler = self._create_mailgw(message) + def _handle_mail(self, message, args=()): + handler = self._create_mailgw(message, args) handler.trapExceptions = 0 return handler.main(StringIO(message)) @@ -199,6 +199,68 @@ From here to there! msgid = self.db.issue.get(nodeid, 'messages')[0] self.assertEqual(self.db.msg.get(msgid, 'content'), 'From here to there!') + def testNoMessageId(self): + self.instance.config['MAIL_DOMAIN'] = 'example.com' + nodeid = self._handle_mail('''Content-Type: text/plain; + charset="iso-8859-1" +From: Chef <chef@bork.bork.bork> +To: issue_tracker@your.tracker.email.domain.example +Cc: richard@test.test +Reply-To: chef@bork.bork.bork +Subject: [issue] Testing... + +Hi there! +''') + assert not os.path.exists(SENDMAILDEBUG) + msgid = self.db.issue.get(nodeid, 'messages')[0] + messageid = self.db.msg.get(msgid, 'messageid') + x1, x2 = messageid.split('@') + self.assertEqual(x2, 'example.com>') + x = x1.split('.')[-1] + self.assertEqual(x, 'issueNone') + nodeid = self._handle_mail('''Content-Type: text/plain; + charset="iso-8859-1" +From: Chef <chef@bork.bork.bork> +To: issue_tracker@your.tracker.email.domain.example +Subject: [issue%(nodeid)s] Testing... + +Just a test reply +'''%locals()) + msgid = self.db.issue.get(nodeid, 'messages')[-1] + messageid = self.db.msg.get(msgid, 'messageid') + x1, x2 = messageid.split('@') + self.assertEqual(x2, 'example.com>') + x = x1.split('.')[-1] + self.assertEqual(x, "issue%s"%nodeid) + + def testOptions(self): + nodeid = self._handle_mail('''Content-Type: text/plain; + charset="iso-8859-1" +From: Chef <chef@bork.bork.bork> +To: issue_tracker@your.tracker.email.domain.example +Message-Id: <dummy_test_message_id> +Reply-To: chef@bork.bork.bork +Subject: [issue] Testing... + +Hi there! +''', (('-C', 'issue'), ('-S', 'status=chatting;priority=critical'))) + self.assertEqual(self.db.issue.get(nodeid, 'status'), '3') + self.assertEqual(self.db.issue.get(nodeid, 'priority'), '1') + + def testOptionsMulti(self): + nodeid = self._handle_mail('''Content-Type: text/plain; + charset="iso-8859-1" +From: Chef <chef@bork.bork.bork> +To: issue_tracker@your.tracker.email.domain.example +Message-Id: <dummy_test_message_id> +Reply-To: chef@bork.bork.bork +Subject: [issue] Testing... + +Hi there! +''', (('-C', 'issue'), ('-S', 'status=chatting'), ('-S', 'priority=critical'))) + self.assertEqual(self.db.issue.get(nodeid, 'status'), '3') + self.assertEqual(self.db.issue.get(nodeid, 'priority'), '1') + def doNewIssue(self): nodeid = self._handle_mail('''Content-Type: text/plain; charset="iso-8859-1" -- 2.30.2