X-Git-Url: https://git.tokkee.org/?a=blobdiff_plain;f=roundup%2Fmailgw.py;h=1558c0f421718505d2d8258bdd3754a779d59aaf;hb=6ba44a7c3806ba5f40562a0e390605a25145a01b;hp=38b3e370e217d66706a6416973b65269649fbc2c;hpb=8a6d9c22c7e4f768704c070e831de28434c7fb31;p=roundup.git diff --git a/roundup/mailgw.py b/roundup/mailgw.py index 38b3e37..1558c0f 100644 --- a/roundup/mailgw.py +++ b/roundup/mailgw.py @@ -27,6 +27,9 @@ Incoming messages are examined for multiple parts: and given "file" class nodes that are linked to the "msg" node. . In a multipart/alternative message or part, we look for a text/plain subpart and ignore the other parts. + . A message/rfc822 is treated similar tomultipart/mixed (except for + special handling of the first text part) if unpack_rfc822 is set in + the mailgw config section. Summary ------- @@ -89,7 +92,7 @@ from roundup.i18n import _ from roundup.hyperdb import iter_roles try: - import pyme, pyme.core, pyme.gpgme + import pyme, pyme.core, pyme.constants, pyme.constants.sigsum except ImportError: pyme = None @@ -153,39 +156,33 @@ def gpgh_key_getall(key, attr): ''' return list of given attribute for all uids in a key ''' - u = key.uids - while u: + for u in key.uids: yield getattr(u, attr) - u = u.next -def gpgh_sigs(sig): - ''' more pythonic iteration over GPG signatures ''' - while sig: - yield sig - sig = sig.next - -def check_pgp_sigs(sig, gpgctx, author): +def check_pgp_sigs(sigs, gpgctx, author, may_be_unsigned=False): ''' Theoretically a PGP message can have several signatures. GPGME - returns status on all signatures in a linked list. Walk that - linked list looking for the author's signature + returns status on all signatures in a list. Walk that list + looking for the author's signature. Note that even if incoming + signatures are not required, the processing fails if there is an + invalid signature. ''' - for sig in gpgh_sigs(sig): + for sig in sigs: key = gpgctx.get_key(sig.fpr, False) # we really only care about the signature of the user who # submitted the email if key and (author in gpgh_key_getall(key, 'email')): - if sig.summary & pyme.gpgme.GPGME_SIGSUM_VALID: + if sig.summary & pyme.constants.sigsum.VALID: return True else: # try to narrow down the actual problem to give a more useful # message in our bounce - if sig.summary & pyme.gpgme.GPGME_SIGSUM_KEY_MISSING: + if sig.summary & pyme.constants.sigsum.KEY_MISSING: raise MailUsageError, \ _("Message signed with unknown key: %s") % sig.fpr - elif sig.summary & pyme.gpgme.GPGME_SIGSUM_KEY_EXPIRED: + elif sig.summary & pyme.constants.sigsum.KEY_EXPIRED: raise MailUsageError, \ _("Message signed with an expired key: %s") % sig.fpr - elif sig.summary & pyme.gpgme.GPGME_SIGSUM_KEY_REVOKED: + elif sig.summary & pyme.constants.sigsum.KEY_REVOKED: raise MailUsageError, \ _("Message signed with a revoked key: %s") % sig.fpr else: @@ -193,7 +190,10 @@ def check_pgp_sigs(sig, gpgctx, author): _("Invalid PGP signature detected.") # we couldn't find a key belonging to the author of the email - raise MailUsageError, _("Message signed with unknown key: %s") % sig.fpr + if sigs: + raise MailUsageError, _("Message signed with unknown key: %s") % sig.fpr + elif not may_be_unsigned: + raise MailUsageError, _("Unsigned Message") class Message(mimetools.Message): ''' subclass mimetools.Message so we can retrieve the parts of the @@ -244,6 +244,22 @@ class Message(mimetools.Message): parts.append(part) return parts + def _decode_header_to_utf8(self, hdr): + l = [] + prev_encoded = False + for part, encoding in decode_header(hdr): + if encoding: + part = part.decode(encoding) + # RFC 2047 specifies that between encoded parts spaces are + # swallowed while at the borders from encoded to non-encoded + # or vice-versa we must preserve a space. Multiple adjacent + # non-encoded parts should not occur. + if l and prev_encoded != bool(encoding): + l.append(' ') + prev_encoded = bool(encoding) + l.append(part) + return ''.join([s.encode('utf-8') for s in l]) + def getheader(self, name, default=None): hdr = mimetools.Message.getheader(self, name, default) # TODO are there any other False values possible? @@ -254,35 +270,29 @@ class Message(mimetools.Message): return '' if hdr: hdr = hdr.replace('\n','') # Inserted by rfc822.readheaders - # historically this method has returned utf-8 encoded string - l = [] - for part, encoding in decode_header(hdr): - if encoding: - part = part.decode(encoding) - l.append(part) - return ''.join([s.encode('utf-8') for s in l]) + return self._decode_header_to_utf8(hdr) def getaddrlist(self, name): # overload to decode the name part of the address l = [] for (name, addr) in mimetools.Message.getaddrlist(self, name): - p = [] - for part, encoding in decode_header(name): - if encoding: - part = part.decode(encoding) - p.append(part) - name = ''.join([s.encode('utf-8') for s in p]) + name = self._decode_header_to_utf8(name) l.append((name, addr)) return l def getname(self): """Find an appropriate name for this message.""" + name = None if self.gettype() == 'message/rfc822': # handle message/rfc822 specially - the name should be # the subject of the actual e-mail embedded here + # we add a '.eml' extension like other email software does it self.fp.seek(0) - name = Message(self.fp).getheader('subject') - else: + s = cStringIO.StringIO(self.getbody()) + name = Message(s).getheader('subject') + if name: + name = name + '.eml' + if not name: # try name on Content-Type name = self.getparam('name') if not name: @@ -355,8 +365,11 @@ class Message(mimetools.Message): # flagging. # multipart/form-data: # For web forms only. + # message/rfc822: + # Only if configured in [mailgw] unpack_rfc822 - def extract_content(self, parent_type=None, ignore_alternatives = False): + def extract_content(self, parent_type=None, ignore_alternatives=False, + unpack_rfc822=False): """Extract the body and the attachments recursively. If the content is hidden inside a multipart/alternative part, @@ -374,7 +387,7 @@ class Message(mimetools.Message): ig = ignore_alternatives and not content_found for part in self.getparts(): new_content, new_attach = part.extract_content(content_type, - not content and ig) + not content and ig, unpack_rfc822) # If we haven't found a text/plain part yet, take this one, # otherwise make it an attachment. @@ -399,6 +412,13 @@ class Message(mimetools.Message): attachments.extend(new_attach) if ig and content_type == 'multipart/alternative' and content: attachments = [] + elif unpack_rfc822 and content_type == 'message/rfc822': + s = cStringIO.StringIO(self.getbody()) + m = Message(s) + ig = ignore_alternatives and not content + new_content, attachments = m.extract_content(m.gettype(), ig, + unpack_rfc822) + attachments.insert(0, m.text_as_attachment()) elif (parent_type == 'multipart/signed' and content_type == 'application/pgp-signature'): # ignore it so it won't be saved as an attachment @@ -437,16 +457,18 @@ class Message(mimetools.Message): return self.gettype() == 'multipart/encrypted' \ and self.typeheader.find('protocol="application/pgp-encrypted"') != -1 - def decrypt(self, author): + def decrypt(self, author, may_be_unsigned=False): ''' decrypt an OpenPGP MIME message - This message must be signed as well as encrypted using the "combined" - method. The decrypted contents are returned as a new message. + This message must be signed as well as encrypted using the + "combined" method if incoming signatures are configured. + The decrypted contents are returned as a new message. ''' (hdr, msg) = self.getparts() # According to the RFC 3156 encrypted mail must have exactly two parts. # The first part contains the control information. Let's verify that # the message meets the RFC before we try to decrypt it. - if hdr.getbody() != 'Version: 1' or hdr.gettype() != 'application/pgp-encrypted': + if hdr.getbody().strip() != 'Version: 1' \ + or hdr.gettype() != 'application/pgp-encrypted': raise MailUsageError, \ _("Unknown multipart/encrypted version.") @@ -463,7 +485,8 @@ class Message(mimetools.Message): # key to send it to us. now check the signatures to see if it # was signed by someone we trust result = context.op_verify_result() - check_pgp_sigs(result.signatures, context, author) + check_pgp_sigs(result.signatures, context, author, + may_be_unsigned = may_be_unsigned) plaintext.seek(0,0) # pyme.core.Data implements a seek method with a different signature @@ -488,7 +511,6 @@ class Message(mimetools.Message): raise MailUsageError, \ _("No PGP signature found in message.") - context = pyme.core.Context() # msg.getbody() is skipping over some headers that are # required to be present for verification to succeed so # we'll do this by hand @@ -503,895 +525,1111 @@ class Message(mimetools.Message): msg_data = pyme.core.Data(canonical_msg) sig_data = pyme.core.Data(sig.getbody()) + context = pyme.core.Context() context.op_verify(sig_data, msg_data, None) # check all signatures for validity result = context.op_verify_result() check_pgp_sigs(result.signatures, context, author) -class MailGW: +class parsedMessage: + + def __init__(self, mailgw, message): + self.mailgw = mailgw + self.config = mailgw.instance.config + self.db = mailgw.db + self.message = message + self.subject = message.getheader('subject', '') + self.has_prefix = False + self.matches = dict.fromkeys(['refwd', 'quote', 'classname', + 'nodeid', 'title', 'args', 'argswhole']) + self.from_list = message.getaddrlist('resent-from') \ + or message.getaddrlist('from') + self.pfxmode = self.config['MAILGW_SUBJECT_PREFIX_PARSING'] + self.sfxmode = self.config['MAILGW_SUBJECT_SUFFIX_PARSING'] + # these are filled in by subsequent parsing steps + self.classname = None + self.properties = None + self.cl = None + self.nodeid = None + self.author = None + self.recipients = None + self.msg_props = {} + self.props = None + self.content = None + self.attachments = None + self.crypt = False + + def handle_ignore(self): + ''' Check to see if message can be safely ignored: + detect loops and + Precedence: Bulk, or Microsoft Outlook autoreplies + ''' + if self.message.getheader('x-roundup-loop', ''): + raise IgnoreLoop + if (self.message.getheader('precedence', '') == 'bulk' + or self.subject.lower().find("autoreply") > 0): + raise IgnoreBulk - def __init__(self, instance, arguments=()): - self.instance = instance - self.arguments = arguments - self.default_class = None - for option, value in self.arguments: - if option == '-c': - self.default_class = value.strip() + def handle_help(self): + ''' Check to see if the message contains a usage/help request + ''' + if self.subject.strip().lower() == 'help': + raise MailUsageHelp - self.mailer = Mailer(instance.config) - self.logger = logging.getLogger('mailgw') + def check_subject(self): + ''' Check to see if the message contains a valid subject line + ''' + if not self.subject: + raise MailUsageError, _(""" +Emails to Roundup trackers must include a Subject: line! +""") - # should we trap exceptions (normal usage) or pass them through - # (for testing) - self.trapExceptions = 1 + def parse_subject(self): + ''' Matches subjects like: + Re: "[issue1234] title of issue [status=resolved]" + + Each part of the subject is matched, stored, then removed from the + start of the subject string as needed. The stored values are then + returned + ''' - def do_pipe(self): - """ Read a message from standard input and pass it to the mail handler. + tmpsubject = self.subject - Read into an internal structure that we can seek on (in case - there's an error). + sd_open, sd_close = self.config['MAILGW_SUBJECT_SUFFIX_DELIMITERS'] + delim_open = re.escape(sd_open) + if delim_open in '[(': delim_open = '\\' + delim_open + delim_close = re.escape(sd_close) + if delim_close in '[(': delim_close = '\\' + delim_close - XXX: we may want to read this into a temporary file instead... - """ - s = cStringIO.StringIO() - s.write(sys.stdin.read()) - s.seek(0) - self.main(s) - return 0 + # Look for Re: et. al. Used later on for MAILGW_SUBJECT_CONTENT_MATCH + re_re = r"(?P%s)\s*" % self.config["MAILGW_REFWD_RE"].pattern + m = re.match(re_re, tmpsubject, re.IGNORECASE|re.VERBOSE|re.UNICODE) + if m: + m = m.groupdict() + if m['refwd']: + self.matches.update(m) + tmpsubject = tmpsubject[len(m['refwd']):] # Consume Re: - def do_mailbox(self, filename): - """ Read a series of messages from the specified unix mailbox file and - pass each to the mail handler. - """ - # open the spool file and lock it - import fcntl - # FCNTL is deprecated in py2.3 and fcntl takes over all the symbols - if hasattr(fcntl, 'LOCK_EX'): - FCNTL = fcntl + # Look for Leading " + m = re.match(r'(?P\s*")', tmpsubject, + re.IGNORECASE) + if m: + self.matches.update(m.groupdict()) + tmpsubject = tmpsubject[len(self.matches['quote']):] # Consume quote + + # Check if the subject includes a prefix + self.has_prefix = re.search(r'^%s(\w+)%s'%(delim_open, + delim_close), tmpsubject.strip()) + + # Match the classname if specified + class_re = r'%s(?P(%s))(?P\d+)?%s'%(delim_open, + "|".join(self.db.getclasses()), delim_close) + # Note: re.search, not re.match as there might be garbage + # (mailing list prefix, etc.) before the class identifier + m = re.search(class_re, tmpsubject, re.IGNORECASE) + if m: + self.matches.update(m.groupdict()) + # Skip to the end of the class identifier, including any + # garbage before it. + + tmpsubject = tmpsubject[m.end():] + + # Match the title of the subject + # if we've not found a valid classname prefix then force the + # scanning to handle there being a leading delimiter + title_re = r'(?P%s[^%s]*)'%( + not self.matches['classname'] and '.' or '', delim_open) + m = re.match(title_re, tmpsubject.strip(), re.IGNORECASE) + if m: + self.matches.update(m.groupdict()) + tmpsubject = tmpsubject[len(self.matches['title']):] # Consume title + + if self.matches['title']: + self.matches['title'] = self.matches['title'].strip() else: - import FCNTL - f = open(filename, 'r+') - fcntl.flock(f.fileno(), FCNTL.LOCK_EX) + self.matches['title'] = '' - # handle and clear the mailbox - try: - from mailbox import UnixMailbox - mailbox = UnixMailbox(f, factory=Message) - # grab one message - message = mailbox.next() - while message: - # handle this message - self.handle_Message(message) - message = mailbox.next() - # nuke the file contents - os.ftruncate(f.fileno(), 0) - except: - import traceback - traceback.print_exc() - return 1 - fcntl.flock(f.fileno(), FCNTL.LOCK_UN) + # strip off the quotes that dumb emailers put around the subject, like + # Re: "[issue1] bla blah" + if self.matches['quote'] and self.matches['title'].endswith('"'): + self.matches['title'] = self.matches['title'][:-1] + + # Match any arguments specified + args_re = r'(?P<argswhole>%s(?P<args>.+?)%s)?'%(delim_open, + delim_close) + m = re.search(args_re, tmpsubject.strip(), re.IGNORECASE|re.VERBOSE) + if m: + self.matches.update(m.groupdict()) + + def rego_confirm(self): + ''' Check for registration OTK and confirm the registration if found + ''' + + if self.config['EMAIL_REGISTRATION_CONFIRMATION']: + otk_re = re.compile('-- key (?P<otk>[a-zA-Z0-9]{32})') + otk = otk_re.search(self.matches['title'] or '') + if otk: + self.db.confirm_registration(otk.group('otk')) + subject = 'Your registration to %s is complete' % \ + self.config['TRACKER_NAME'] + sendto = [self.from_list[0][1]] + self.mailgw.mailer.standard_message(sendto, subject, '') + return 1 return 0 - def do_imap(self, server, user='', password='', mailbox='', ssl=0, - cram=0): - ''' Do an IMAP connection + def get_classname(self): + ''' Determine the classname of the node being created/edited ''' - import getpass, imaplib, socket - try: - if not user: - user = raw_input('User: ') - if not password: - password = getpass.getpass() - except (KeyboardInterrupt, EOFError): - # Ctrl C or D maybe also Ctrl Z under Windows. - print "\nAborted by user." - return 1 - # open a connection to the server and retrieve all messages - try: - if ssl: - self.logger.debug('Trying server %r with ssl'%server) - server = imaplib.IMAP4_SSL(server) - else: - self.logger.debug('Trying server %r without ssl'%server) - server = imaplib.IMAP4(server) - except (imaplib.IMAP4.error, socket.error, socket.sslerror): - self.logger.exception('IMAP server error') - return 1 + subject = self.subject - try: - if cram: - server.login_cram_md5(user, password) - else: - server.login(user, password) - except imaplib.IMAP4.error, e: - self.logger.exception('IMAP login failure') - return 1 + # get the classname + if self.pfxmode == 'none': + classname = None + else: + classname = self.matches['classname'] - try: - if not mailbox: - (typ, data) = server.select() - else: - (typ, data) = server.select(mailbox=mailbox) - if typ != 'OK': - self.logger.error('Failed to get mailbox %r: %s'%(mailbox, - data)) - return 1 - try: - numMessages = int(data[0]) - except ValueError, value: - self.logger.error('Invalid message count from mailbox %r'% - data[0]) - return 1 - for i in range(1, numMessages+1): - (typ, data) = server.fetch(str(i), '(RFC822)') + if not classname and self.has_prefix and self.pfxmode == 'strict': + raise MailUsageError, _(""" +The message you sent to roundup did not contain a properly formed subject +line. The subject must contain a class name or designator to indicate the +'topic' of the message. For example: + Subject: [issue] This is a new issue + - this will create a new issue in the tracker with the title 'This is + a new issue'. + Subject: [issue1234] This is a followup to issue 1234 + - this will append the message's contents to the existing issue 1234 + in the tracker. - # mark the message as deleted. - server.store(str(i), '+FLAGS', r'(\Deleted)') +Subject was: '%(subject)s' +""") % locals() - # process the message - s = cStringIO.StringIO(data[0][1]) - s.seek(0) - self.handle_Message(Message(s)) - server.close() - finally: + # try to get the class specified - if "loose" or "none" then fall + # back on the default + attempts = [] + if classname: + attempts.append(classname) + + if self.mailgw.default_class: + attempts.append(self.mailgw.default_class) + else: + attempts.append(self.config['MAILGW_DEFAULT_CLASS']) + + # first valid class name wins + self.cl = None + for trycl in attempts: try: - server.expunge() - except: + self.cl = self.db.getclass(trycl) + classname = self.classname = trycl + break + except KeyError: pass - server.logout() - return 0 + if not self.cl: + validname = ', '.join(self.db.getclasses()) + if classname: + raise MailUsageError, _(""" +The class name you identified in the subject line ("%(classname)s") does +not exist in the database. +Valid class names are: %(validname)s +Subject was: "%(subject)s" +""") % locals() + else: + raise MailUsageError, _(""" +You did not identify a class name in the subject line and there is no +default set for this tracker. The subject must contain a class name or +designator to indicate the 'topic' of the message. For example: + Subject: [issue] This is a new issue + - this will create a new issue in the tracker with the title 'This is + a new issue'. + Subject: [issue1234] This is a followup to issue 1234 + - this will append the message's contents to the existing issue 1234 + in the tracker. - def do_apop(self, server, user='', password='', ssl=False): - ''' Do authentication POP - ''' - self._do_pop(server, user, password, True, ssl) +Subject was: '%(subject)s' +""") % locals() + # get the class properties + self.properties = self.cl.getprops() + - def do_pop(self, server, user='', password='', ssl=False): - ''' Do plain POP + def get_nodeid(self): + ''' Determine the nodeid from the message and return it if found ''' - self._do_pop(server, user, password, False, ssl) + title = self.matches['title'] + subject = self.subject + + if self.pfxmode == 'none': + nodeid = None + else: + nodeid = self.matches['nodeid'] - def _do_pop(self, server, user, password, apop, ssl): - '''Read a series of messages from the specified POP server. - ''' - import getpass, poplib, socket - try: - if not user: - user = raw_input('User: ') - if not password: - password = getpass.getpass() - except (KeyboardInterrupt, EOFError): - # Ctrl C or D maybe also Ctrl Z under Windows. - print "\nAborted by user." - return 1 - - # open a connection to the server and retrieve all messages - try: - if ssl: - klass = poplib.POP3_SSL - else: - klass = poplib.POP3 - server = klass(server) - except socket.error: - self.logger.exception('POP server error') - return 1 - if apop: - server.apop(user, password) - else: - server.user(user) - server.pass_(password) - numMessages = len(server.list()[1]) - for i in range(1, numMessages+1): - # retr: returns - # [ pop response e.g. '+OK 459 octets', - # [ array of message lines ], - # number of octets ] - lines = server.retr(i)[1] - s = cStringIO.StringIO('\n'.join(lines)) - s.seek(0) - self.handle_Message(Message(s)) - # delete the message - server.dele(i) + # try in-reply-to to match the message if there's no nodeid + inreplyto = self.message.getheader('in-reply-to') or '' + if nodeid is None and inreplyto: + l = self.db.getclass('msg').stringFind(messageid=inreplyto) + if l: + nodeid = self.cl.filter(None, {'messages':l})[0] - # quit the server to commit changes. - server.quit() - return 0 - def main(self, fp): - ''' fp - the file from which to read the Message. - ''' - return self.handle_Message(Message(fp)) + # but we do need either a title or a nodeid... + if nodeid is None and not title: + raise MailUsageError, _(""" +I cannot match your message to a node in the database - you need to either +supply a full designator (with number, eg "[issue123]") or keep the +previous subject title intact so I can match that. - def handle_Message(self, message): - """Handle an RFC822 Message +Subject was: "%(subject)s" +""") % locals() - Handle the Message object by calling handle_message() and then cope - with any errors raised by handle_message. - This method's job is to make that call and handle any - errors in a sane manner. It should be replaced if you wish to - handle errors in a different manner. - """ - # in some rare cases, a particularly stuffed-up e-mail will make - # its way into here... try to handle it gracefully + # If there's no nodeid, check to see if this is a followup and + # maybe someone's responded to the initial mail that created an + # entry. Try to find the matching nodes with the same title, and + # use the _last_ one matched (since that'll _usually_ be the most + # recent...). The subject_content_match config may specify an + # additional restriction based on the matched node's creation or + # activity. + tmatch_mode = self.config['MAILGW_SUBJECT_CONTENT_MATCH'] + if tmatch_mode != 'never' and nodeid is None and self.matches['refwd']: + l = self.cl.stringFind(title=title) + limit = None + if (tmatch_mode.startswith('creation') or + tmatch_mode.startswith('activity')): + limit, interval = tmatch_mode.split(' ', 1) + threshold = date.Date('.') - date.Interval(interval) + for id in l: + if limit: + if threshold < self.cl.get(id, limit): + nodeid = id + else: + nodeid = id - sendto = message.getaddrlist('resent-from') - if not sendto: - sendto = message.getaddrlist('from') - if not sendto: - # very bad-looking message - we don't even know who sent it - msg = ['Badly formed message from mail gateway. Headers:'] - msg.extend(message.headers) - msg = '\n'.join(map(str, msg)) - self.logger.error(msg) - return + # if a nodeid was specified, make sure it's valid + if nodeid is not None and not self.cl.hasnode(nodeid): + if self.pfxmode == 'strict': + raise MailUsageError, _(""" +The node specified by the designator in the subject of your message +("%(nodeid)s") does not exist. - msg = 'Handling message' - if message.getheader('message-id'): - msg += ' (Message-id=%r)'%message.getheader('message-id') - self.logger.info(msg) +Subject was: "%(subject)s" +""") % locals() + else: + nodeid = None + self.nodeid = nodeid - # try normal message-handling - if not self.trapExceptions: - return self.handle_message(message) + def get_author_id(self): + ''' Attempt to get the author id from the existing registered users, + otherwise attempt to register a new user and return their id + ''' + # Don't create users if anonymous isn't allowed to register + create = 1 + anonid = self.db.user.lookup('anonymous') + if not (self.db.security.hasPermission('Register', anonid, 'user') + and self.db.security.hasPermission('Email Access', anonid)): + create = 0 - # no, we want to trap exceptions - try: - return self.handle_message(message) - except MailUsageHelp: - # bounce the message back to the sender with the usage message - fulldoc = '\n'.join(string.split(__doc__, '\n')[2:]) - m = [''] - m.append('\n\nMail Gateway Help\n=================') - m.append(fulldoc) - self.mailer.bounce_message(message, [sendto[0][1]], m, - subject="Mail Gateway Help") - except MailUsageError, value: - # bounce the message back to the sender with the usage message - fulldoc = '\n'.join(string.split(__doc__, '\n')[2:]) - m = [''] - m.append(str(value)) - m.append('\n\nMail Gateway Help\n=================') - m.append(fulldoc) - self.mailer.bounce_message(message, [sendto[0][1]], m) - except Unauthorized, value: - # just inform the user that he is not authorized - m = [''] - m.append(str(value)) - self.mailer.bounce_message(message, [sendto[0][1]], m) - except IgnoreMessage: - # do not take any action - # this exception is thrown when email should be ignored - msg = 'IgnoreMessage raised' - if message.getheader('message-id'): - msg += ' (Message-id=%r)'%message.getheader('message-id') - self.logger.info(msg) - return - except: - msg = 'Exception handling message' - if message.getheader('message-id'): - msg += ' (Message-id=%r)'%message.getheader('message-id') - self.logger.exception(msg) + # ok, now figure out who the author is - create a new user if the + # "create" flag is true + author = uidFromAddress(self.db, self.from_list[0], create=create) - # bounce the message back to the sender with the error message - # let the admin know that something very bad is happening - m = [''] - m.append('An unexpected error occurred during the processing') - m.append('of your message. The tracker administrator is being') - m.append('notified.\n') - self.mailer.bounce_message(message, [sendto[0][1]], m) + # if we're not recognised, and we don't get added as a user, then we + # must be anonymous + if not author: + author = anonid - m.append('----------------') - m.append(traceback.format_exc()) - self.mailer.bounce_message(message, [self.instance.config.ADMIN_EMAIL], m) + # make sure the author has permission to use the email interface + if not self.db.security.hasPermission('Email Access', author): + if author == anonid: + # we're anonymous and we need to be a registered user + from_address = self.from_list[0][1] + registration_info = "" + if self.db.security.hasPermission('Web Access', author) and \ + self.db.security.hasPermission('Register', anonid, 'user'): + tracker_web = self.config.TRACKER_WEB + registration_info = """ Please register at: - def handle_message(self, message): - ''' message - a Message instance +%(tracker_web)suser?template=register - Parse the message as per the module docstring. - ''' - # get database handle for handling one email - self.db = self.instance.open ('admin') - try: - return self._handle_message (message) - finally: - self.db.close() +...before sending mail to the tracker.""" % locals() - def _handle_message(self, message): - ''' message - a Message instance + raise Unauthorized, _(""" +You are not a registered user.%(registration_info)s - Parse the message as per the module docstring. +Unknown address: %(from_address)s +""") % locals() + else: + # we're registered and we're _still_ not allowed access + raise Unauthorized, _( + 'You are not permitted to access this tracker.') + self.author = author - The implementation expects an opened database and a try/finally - that closes the database. + def check_permissions(self): + ''' Check if the author has permission to edit or create this + class of node ''' - # detect loops - if message.getheader('x-roundup-loop', ''): - raise IgnoreLoop + if self.nodeid: + if not self.db.security.hasPermission('Edit', self.author, + self.classname, itemid=self.nodeid): + raise Unauthorized, _( + 'You are not permitted to edit %(classname)s.' + ) % self.__dict__ + else: + if not self.db.security.hasPermission('Create', self.author, + self.classname): + raise Unauthorized, _( + 'You are not permitted to create %(classname)s.' + ) % self.__dict__ - # handle the subject line - subject = message.getheader('subject', '') - if not subject: - raise MailUsageError, _(""" -Emails to Roundup trackers must include a Subject: line! -""") + def commit_and_reopen_as_author(self): + ''' the author may have been created - make sure the change is + committed before we reopen the database + then re-open the database as the author + ''' + self.db.commit() - # detect Precedence: Bulk, or Microsoft Outlook autoreplies - if (message.getheader('precedence', '') == 'bulk' - or subject.lower().find("autoreply") > 0): - raise IgnoreBulk + # set the database user as the author + username = self.db.user.get(self.author, 'username') + self.db.setCurrentUser(username) - if subject.strip().lower() == 'help': - raise MailUsageHelp + # re-get the class with the new database connection + self.cl = self.db.getclass(self.classname) - # config is used many times in this method. - # make local variable for easier access - config = self.instance.config + def get_recipients(self): + ''' Get the list of recipients who were included in message and + register them as users if possible + ''' + # Don't create users if anonymous isn't allowed to register + create = 1 + anonid = self.db.user.lookup('anonymous') + if not (self.db.security.hasPermission('Register', anonid, 'user') + and self.db.security.hasPermission('Email Access', anonid)): + create = 0 - # determine the sender's address - from_list = message.getaddrlist('resent-from') - if not from_list: - from_list = message.getaddrlist('from') + # get the user class arguments from the commandline + user_props = self.mailgw.get_class_arguments('user') - # XXX Don't enable. This doesn't work yet. -# "[^A-z.]tracker\+(?P<classname>[^\d\s]+)(?P<nodeid>\d+)\@some.dom.ain[^A-z.]" - # handle delivery to addresses like:tracker+issue25@some.dom.ain - # use the embedded issue number as our issue -# issue_re = config['MAILGW_ISSUE_ADDRESS_RE'] -# if issue_re: -# for header in ['to', 'cc', 'bcc']: -# addresses = message.getheader(header, '') -# if addresses: -# # FIXME, this only finds the first match in the addresses. -# issue = re.search(issue_re, addresses, 'i') -# if issue: -# classname = issue.group('classname') -# nodeid = issue.group('nodeid') -# break - - # Matches subjects like: - # Re: "[issue1234] title of issue [status=resolved]" - - # Alias since we need a reference to the original subject for - # later use in error messages - tmpsubject = subject - - sd_open, sd_close = config['MAILGW_SUBJECT_SUFFIX_DELIMITERS'] - delim_open = re.escape(sd_open) - if delim_open in '[(': delim_open = '\\' + delim_open - delim_close = re.escape(sd_close) - if delim_close in '[(': delim_close = '\\' + delim_close + # now update the recipients list + recipients = [] + tracker_email = self.config['TRACKER_EMAIL'].lower() + msg_to = self.message.getaddrlist('to') + msg_cc = self.message.getaddrlist('cc') + for recipient in msg_to + msg_cc: + r = recipient[1].strip().lower() + if r == tracker_email or not r: + continue - matches = dict.fromkeys(['refwd', 'quote', 'classname', - 'nodeid', 'title', 'args', - 'argswhole']) + # look up the recipient - create if necessary (and we're + # allowed to) + recipient = uidFromAddress(self.db, recipient, create, **user_props) - # Look for Re: et. al. Used later on for MAILGW_SUBJECT_CONTENT_MATCH - re_re = r"(?P<refwd>%s)\s*" % config["MAILGW_REFWD_RE"].pattern - m = re.match(re_re, tmpsubject, re.IGNORECASE|re.VERBOSE|re.UNICODE) - if m: - m = m.groupdict() - if m['refwd']: - matches.update(m) - tmpsubject = tmpsubject[len(m['refwd']):] # Consume Re: + # if all's well, add the recipient to the list + if recipient: + recipients.append(recipient) + self.recipients = recipients - # Look for Leading " - m = re.match(r'(?P<quote>\s*")', tmpsubject, - re.IGNORECASE) - if m: - matches.update(m.groupdict()) - tmpsubject = tmpsubject[len(matches['quote']):] # Consume quote + def get_props(self): + ''' Generate all the props for the new/updated node and return them + ''' + subject = self.subject + + # get the commandline arguments for issues + issue_props = self.mailgw.get_class_arguments('issue', self.classname) + + # + # handle the subject argument list + # + # figure what the properties of this Class are + props = {} + args = self.matches['args'] + argswhole = self.matches['argswhole'] + title = self.matches['title'] + + # Reform the title + if self.matches['nodeid'] and self.nodeid is None: + title = subject + + if args: + if self.sfxmode == 'none': + title += ' ' + argswhole + else: + errors, props = setPropArrayFromString(self, self.cl, args, + self.nodeid) + # handle any errors parsing the argument list + if errors: + if self.sfxmode == 'strict': + errors = '\n- '.join(map(str, errors)) + raise MailUsageError, _(""" +There were problems handling your subject line argument list: +- %(errors)s - has_prefix = re.search(r'^%s(\w+)%s'%(delim_open, - delim_close), tmpsubject.strip()) - - class_re = r'%s(?P<classname>(%s))(?P<nodeid>\d+)?%s'%(delim_open, - "|".join(self.db.getclasses()), delim_close) - # Note: re.search, not re.match as there might be garbage - # (mailing list prefix, etc.) before the class identifier - m = re.search(class_re, tmpsubject, re.IGNORECASE) - if m: - matches.update(m.groupdict()) - # Skip to the end of the class identifier, including any - # garbage before it. +Subject was: "%(subject)s" +""") % locals() + else: + title += ' ' + argswhole - tmpsubject = tmpsubject[m.end():] - # if we've not found a valid classname prefix then force the - # scanning to handle there being a leading delimiter - title_re = r'(?P<title>%s[^%s]*)'%( - not matches['classname'] and '.' or '', delim_open) - m = re.match(title_re, tmpsubject.strip(), re.IGNORECASE) - if m: - matches.update(m.groupdict()) - tmpsubject = tmpsubject[len(matches['title']):] # Consume title + # set the issue title to the subject + title = title.strip() + if (title and self.properties.has_key('title') and not + issue_props.has_key('title')): + issue_props['title'] = title + if (self.nodeid and self.properties.has_key('title') and not + self.config['MAILGW_SUBJECT_UPDATES_TITLE']): + issue_props['title'] = self.cl.get(self.nodeid,'title') + + # merge the command line props defined in issue_props into + # the props dictionary because function(**props, **issue_props) + # is a syntax error. + for prop in issue_props.keys() : + if not props.has_key(prop) : + props[prop] = issue_props[prop] + + self.props = props + + def get_pgp_message(self): + ''' If they've enabled PGP processing then verify the signature + or decrypt the message + ''' + def pgp_role(): + """ if PGP_ROLES is specified the user must have a Role in the list + or we will skip PGP processing + """ + if self.config.PGP_ROLES: + return self.db.user.has_role(self.author, + *iter_roles(self.config.PGP_ROLES)) + else: + return True - args_re = r'(?P<argswhole>%s(?P<args>.+?)%s)?'%(delim_open, - delim_close) - m = re.search(args_re, tmpsubject.strip(), re.IGNORECASE|re.VERBOSE) - if m: - matches.update(m.groupdict()) + if self.config.PGP_ENABLE: + if pgp_role() and self.config.PGP_ENCRYPT: + self.crypt = True + assert pyme, 'pyme is not installed' + # signed/encrypted mail must come from the primary address + author_address = self.db.user.get(self.author, 'address') + if self.config.PGP_HOMEDIR: + os.environ['GNUPGHOME'] = self.config.PGP_HOMEDIR + if self.config.PGP_REQUIRE_INCOMING in ('encrypted', 'both') \ + and pgp_role() and not self.message.pgp_encrypted(): + raise MailUsageError, _( + "This tracker has been configured to require all email " + "be PGP encrypted.") + if self.message.pgp_signed(): + self.message.verify_signature(author_address) + elif self.message.pgp_encrypted(): + # Replace message with the contents of the decrypted + # message for content extraction + # Note: the bounce-handling code now makes sure that + # either the encrypted mail received is sent back or + # that the error message is encrypted if needed. + encr_only = self.config.PGP_REQUIRE_INCOMING == 'encrypted' + encr_only = encr_only or not pgp_role() + self.crypt = True + self.message = self.message.decrypt(author_address, + may_be_unsigned = encr_only) + elif pgp_role(): + raise MailUsageError, _(""" +This tracker has been configured to require all email be PGP signed or +encrypted.""") - # figure subject line parsing modes - pfxmode = config['MAILGW_SUBJECT_PREFIX_PARSING'] - sfxmode = config['MAILGW_SUBJECT_SUFFIX_PARSING'] + def get_content_and_attachments(self): + ''' get the attachments and first text part from the message + ''' + ig = self.config.MAILGW_IGNORE_ALTERNATIVES + self.content, self.attachments = self.message.extract_content( + ignore_alternatives=ig, + unpack_rfc822=self.config.MAILGW_UNPACK_RFC822) + + + def create_files(self): + ''' Create a file for each attachment in the message + ''' + if not self.properties.has_key('files'): + return + files = [] + file_props = self.mailgw.get_class_arguments('file') + + if self.attachments: + for (name, mime_type, data) in self.attachments: + if not self.db.security.hasPermission('Create', self.author, + 'file'): + raise Unauthorized, _( + 'You are not permitted to create files.') + if not name: + name = "unnamed" + try: + fileid = self.db.file.create(type=mime_type, name=name, + content=data, **file_props) + except exceptions.Reject: + pass + else: + files.append(fileid) + # allowed to attach the files to an existing node? + if self.nodeid and not self.db.security.hasPermission('Edit', + self.author, self.classname, 'files'): + raise Unauthorized, _( + 'You are not permitted to add files to %(classname)s.' + ) % self.__dict__ - # check for registration OTK - # or fallback on the default class - if self.db.config['EMAIL_REGISTRATION_CONFIRMATION']: - otk_re = re.compile('-- key (?P<otk>[a-zA-Z0-9]{32})') - otk = otk_re.search(matches['title'] or '') - if otk: - self.db.confirm_registration(otk.group('otk')) - subject = 'Your registration to %s is complete' % \ - config['TRACKER_NAME'] - sendto = [from_list[0][1]] - self.mailer.standard_message(sendto, subject, '') - return + self.msg_props['files'] = files + if self.nodeid: + # extend the existing files list + fileprop = self.cl.get(self.nodeid, 'files') + fileprop.extend(files) + files = fileprop - # get the classname - if pfxmode == 'none': - classname = None - else: - classname = matches['classname'] + self.props['files'] = files - if not classname and has_prefix and pfxmode == 'strict': + def create_msg(self): + ''' Create msg containing all the relevant information from the message + ''' + if not self.properties.has_key('messages'): + return + msg_props = self.mailgw.get_class_arguments('msg') + self.msg_props.update (msg_props) + + # Get the message ids + inreplyto = self.message.getheader('in-reply-to') or '' + messageid = self.message.getheader('message-id') + # generate a messageid if there isn't one + if not messageid: + messageid = "<%s.%s.%s%s@%s>"%(time.time(), random.random(), + self.classname, self.nodeid, self.config['MAIL_DOMAIN']) + + if self.content is None: raise MailUsageError, _(""" -The message you sent to roundup did not contain a properly formed subject -line. The subject must contain a class name or designator to indicate the -'topic' of the message. For example: - Subject: [issue] This is a new issue - - this will create a new issue in the tracker with the title 'This is - a new issue'. - Subject: [issue1234] This is a followup to issue 1234 - - this will append the message's contents to the existing issue 1234 - in the tracker. - -Subject was: '%(subject)s' -""") % locals() +Roundup requires the submission to be plain text. The message parser could +not find a text/plain part to use. +""") - # try to get the class specified - if "loose" or "none" then fall - # back on the default - attempts = [] - if classname: - attempts.append(classname) + # parse the body of the message, stripping out bits as appropriate + summary, content = parseContent(self.content, config=self.config) + content = content.strip() - if self.default_class: - attempts.append(self.default_class) - else: - attempts.append(config['MAILGW_DEFAULT_CLASS']) + if content: + if not self.db.security.hasPermission('Create', self.author, 'msg'): + raise Unauthorized, _( + 'You are not permitted to create messages.') - # first valid class name wins - cl = None - for trycl in attempts: try: - cl = self.db.getclass(trycl) - classname = trycl - break - except KeyError: - pass - - if not cl: - validname = ', '.join(self.db.getclasses()) - if classname: + message_id = self.db.msg.create(author=self.author, + recipients=self.recipients, date=date.Date('.'), + summary=summary, content=content, + messageid=messageid, inreplyto=inreplyto, **self.msg_props) + except exceptions.Reject, error: raise MailUsageError, _(""" -The class name you identified in the subject line ("%(classname)s") does -not exist in the database. - -Valid class names are: %(validname)s -Subject was: "%(subject)s" +Mail message was rejected by a detector. +%(error)s """) % locals() + # allowed to attach the message to the existing node? + if self.nodeid and not self.db.security.hasPermission('Edit', + self.author, self.classname, 'messages'): + raise Unauthorized, _( + 'You are not permitted to add messages to %(classname)s.' + ) % self.__dict__ + + if self.nodeid: + # add the message to the node's list + messages = self.cl.get(self.nodeid, 'messages') + messages.append(message_id) + self.props['messages'] = messages else: - raise MailUsageError, _(""" -You did not identify a class name in the subject line and there is no -default set for this tracker. The subject must contain a class name or -designator to indicate the 'topic' of the message. For example: - Subject: [issue] This is a new issue - - this will create a new issue in the tracker with the title 'This is - a new issue'. - Subject: [issue1234] This is a followup to issue 1234 - - this will append the message's contents to the existing issue 1234 - in the tracker. + # pre-load the messages list + self.props['messages'] = [message_id] -Subject was: '%(subject)s' + def create_node(self): + ''' Create/update a node using self.props + ''' + classname = self.classname + try: + if self.nodeid: + # Check permissions for each property + for prop in self.props.keys(): + if not self.db.security.hasPermission('Edit', self.author, + classname, prop): + raise Unauthorized, _('You are not permitted to edit ' + 'property %(prop)s of class %(classname)s.' + ) % locals() + self.cl.set(self.nodeid, **self.props) + else: + # Check permissions for each property + for prop in self.props.keys(): + if not self.db.security.hasPermission('Create', self.author, + classname, prop): + raise Unauthorized, _('You are not permitted to set ' + 'property %(prop)s of class %(classname)s.' + ) % locals() + self.nodeid = self.cl.create(**self.props) + except (TypeError, IndexError, ValueError, exceptions.Reject), message: + raise MailUsageError, _(""" +There was a problem with the message you sent: + %(message)s """) % locals() - # get the optional nodeid - if pfxmode == 'none': - nodeid = None - else: - nodeid = matches['nodeid'] + return self.nodeid - # try in-reply-to to match the message if there's no nodeid - inreplyto = message.getheader('in-reply-to') or '' - if nodeid is None and inreplyto: - l = self.db.getclass('msg').stringFind(messageid=inreplyto) - if l: - nodeid = cl.filter(None, {'messages':l})[0] + # XXX Don't enable. This doesn't work yet. +# "[^A-z.]tracker\+(?P<classname>[^\d\s]+)(?P<nodeid>\d+)\@some.dom.ain[^A-z.]" + # handle delivery to addresses like:tracker+issue25@some.dom.ain + # use the embedded issue number as our issue +# issue_re = config['MAILGW_ISSUE_ADDRESS_RE'] +# if issue_re: +# for header in ['to', 'cc', 'bcc']: +# addresses = message.getheader(header, '') +# if addresses: +# # FIXME, this only finds the first match in the addresses. +# issue = re.search(issue_re, addresses, 'i') +# if issue: +# classname = issue.group('classname') +# nodeid = issue.group('nodeid') +# break + + # Default sequence of methods to be called on message. Use this for + # easier override of the default message processing + # list consists of tuples (method, return_if_true), the parsing + # returns if the return_if_true flag is set for a method *and* the + # method returns something that evaluates to True. + method_list = [ + # Filter out messages to ignore + (handle_ignore, False), + # Check for usage/help requests + (handle_help, False), + # Check if the subject line is valid + (check_subject, False), + # get importants parts from subject + (parse_subject, False), + # check for registration OTK + (rego_confirm, True), + # get the classname + (get_classname, False), + # get the optional nodeid: + (get_nodeid, False), + # Determine who the author is: + (get_author_id, False), + # allowed to edit or create this class? + (check_permissions, False), + # author may have been created: + # commit author to database and re-open as author + (commit_and_reopen_as_author, False), + # Get the recipients list + (get_recipients, False), + # get the new/updated node props + (get_props, False), + # Handle PGP signed or encrypted messages + (get_pgp_message, False), + # extract content and attachments from message body: + (get_content_and_attachments, False), + # put attachments into files linked to the issue: + (create_files, False), + # create the message if there's a message body (content): + (create_msg, False), + ] + + + def parse (self): + for method, flag in self.method_list: + ret = method(self) + if flag and ret: + return + # perform the node change / create: + return self.create_node() - # title is optional too - title = matches['title'] - if title: - title = title.strip() - else: - title = '' - # strip off the quotes that dumb emailers put around the subject, like - # Re: "[issue1] bla blah" - if matches['quote'] and title.endswith('"'): - title = title[:-1] +class MailGW: - # but we do need either a title or a nodeid... - if nodeid is None and not title: - raise MailUsageError, _(""" -I cannot match your message to a node in the database - you need to either -supply a full designator (with number, eg "[issue123]") or keep the -previous subject title intact so I can match that. + # To override the message parsing, derive your own class from + # parsedMessage and assign to parsed_message_class in a derived + # class of MailGW + parsed_message_class = parsedMessage -Subject was: "%(subject)s" -""") % locals() + def __init__(self, instance, arguments=()): + self.instance = instance + self.arguments = arguments + self.default_class = None + for option, value in self.arguments: + if option == '-c': + self.default_class = value.strip() - # If there's no nodeid, check to see if this is a followup and - # maybe someone's responded to the initial mail that created an - # entry. Try to find the matching nodes with the same title, and - # use the _last_ one matched (since that'll _usually_ be the most - # recent...). The subject_content_match config may specify an - # additional restriction based on the matched node's creation or - # activity. - tmatch_mode = config['MAILGW_SUBJECT_CONTENT_MATCH'] - if tmatch_mode != 'never' and nodeid is None and matches['refwd']: - l = cl.stringFind(title=title) - limit = None - if (tmatch_mode.startswith('creation') or - tmatch_mode.startswith('activity')): - limit, interval = tmatch_mode.split(' ', 1) - threshold = date.Date('.') - date.Interval(interval) - for id in l: - if limit: - if threshold < cl.get(id, limit): - nodeid = id - else: - nodeid = id + self.mailer = Mailer(instance.config) + self.logger = logging.getLogger('roundup.mailgw') - # if a nodeid was specified, make sure it's valid - if nodeid is not None and not cl.hasnode(nodeid): - if pfxmode == 'strict': - raise MailUsageError, _(""" -The node specified by the designator in the subject of your message -("%(nodeid)s") does not exist. + # should we trap exceptions (normal usage) or pass them through + # (for testing) + self.trapExceptions = 1 -Subject was: "%(subject)s" -""") % locals() + def do_pipe(self): + """ Read a message from standard input and pass it to the mail handler. + + Read into an internal structure that we can seek on (in case + there's an error). + + XXX: we may want to read this into a temporary file instead... + """ + s = cStringIO.StringIO() + s.write(sys.stdin.read()) + s.seek(0) + self.main(s) + return 0 + + def do_mailbox(self, filename): + """ Read a series of messages from the specified unix mailbox file and + pass each to the mail handler. + """ + # open the spool file and lock it + import fcntl + # FCNTL is deprecated in py2.3 and fcntl takes over all the symbols + if hasattr(fcntl, 'LOCK_EX'): + FCNTL = fcntl + else: + import FCNTL + f = open(filename, 'r+') + fcntl.flock(f.fileno(), FCNTL.LOCK_EX) + + # handle and clear the mailbox + try: + from mailbox import UnixMailbox + mailbox = UnixMailbox(f, factory=Message) + # grab one message + message = mailbox.next() + while message: + # handle this message + self.handle_Message(message) + message = mailbox.next() + # nuke the file contents + os.ftruncate(f.fileno(), 0) + except: + import traceback + traceback.print_exc() + return 1 + fcntl.flock(f.fileno(), FCNTL.LOCK_UN) + return 0 + + def do_imap(self, server, user='', password='', mailbox='', ssl=0, + cram=0): + ''' Do an IMAP connection + ''' + import getpass, imaplib, socket + try: + if not user: + user = raw_input('User: ') + if not password: + password = getpass.getpass() + except (KeyboardInterrupt, EOFError): + # Ctrl C or D maybe also Ctrl Z under Windows. + print "\nAborted by user." + return 1 + # open a connection to the server and retrieve all messages + try: + if ssl: + self.logger.debug('Trying server %r with ssl'%server) + server = imaplib.IMAP4_SSL(server) else: - title = subject - nodeid = None + self.logger.debug('Trying server %r without ssl'%server) + server = imaplib.IMAP4(server) + except (imaplib.IMAP4.error, socket.error, socket.sslerror): + self.logger.exception('IMAP server error') + return 1 - # Handle the arguments specified by the email gateway command line. - # We do this by looping over the list of self.arguments looking for - # a -C to tell us what class then the -S setting string. - msg_props = {} - user_props = {} - file_props = {} - issue_props = {} - # so, if we have any arguments, use them - if self.arguments: - current_class = 'msg' - for option, propstring in self.arguments: - if option in ( '-C', '--class'): - current_class = propstring.strip() - # XXX this is not flexible enough. - # we should chect for subclasses of these classes, - # not for the class name... - if current_class not in ('msg', 'file', 'user', 'issue'): - mailadmin = config['ADMIN_EMAIL'] - raise MailUsageError, _(""" -The mail gateway is not properly set up. Please contact -%(mailadmin)s and have them fix the incorrect class specified as: - %(current_class)s -""") % locals() - if option in ('-S', '--set'): - if current_class == 'issue' : - errors, issue_props = setPropArrayFromString(self, - cl, propstring.strip(), nodeid) - elif current_class == 'file' : - temp_cl = self.db.getclass('file') - errors, file_props = setPropArrayFromString(self, - temp_cl, propstring.strip()) - elif current_class == 'msg' : - temp_cl = self.db.getclass('msg') - errors, msg_props = setPropArrayFromString(self, - temp_cl, propstring.strip()) - elif current_class == 'user' : - temp_cl = self.db.getclass('user') - errors, user_props = setPropArrayFromString(self, - temp_cl, propstring.strip()) - if errors: - mailadmin = config['ADMIN_EMAIL'] - raise MailUsageError, _(""" -The mail gateway is not properly set up. Please contact -%(mailadmin)s and have them fix the incorrect properties: - %(errors)s -""") % locals() + try: + if cram: + server.login_cram_md5(user, password) + else: + server.login(user, password) + except imaplib.IMAP4.error, e: + self.logger.exception('IMAP login failure') + return 1 - # - # handle the users - # - # Don't create users if anonymous isn't allowed to register - create = 1 - anonid = self.db.user.lookup('anonymous') - if not (self.db.security.hasPermission('Register', anonid, 'user') - and self.db.security.hasPermission('Email Access', anonid)): - create = 0 + try: + if not mailbox: + (typ, data) = server.select() + else: + (typ, data) = server.select(mailbox=mailbox) + if typ != 'OK': + self.logger.error('Failed to get mailbox %r: %s'%(mailbox, + data)) + return 1 + try: + numMessages = int(data[0]) + except ValueError, value: + self.logger.error('Invalid message count from mailbox %r'% + data[0]) + return 1 + for i in range(1, numMessages+1): + (typ, data) = server.fetch(str(i), '(RFC822)') - # ok, now figure out who the author is - create a new user if the - # "create" flag is true - author = uidFromAddress(self.db, from_list[0], create=create) + # mark the message as deleted. + server.store(str(i), '+FLAGS', r'(\Deleted)') - # if we're not recognised, and we don't get added as a user, then we - # must be anonymous - if not author: - author = anonid + # process the message + s = cStringIO.StringIO(data[0][1]) + s.seek(0) + self.handle_Message(Message(s)) + server.close() + finally: + try: + server.expunge() + except: + pass + server.logout() - # make sure the author has permission to use the email interface - if not self.db.security.hasPermission('Email Access', author): - if author == anonid: - # we're anonymous and we need to be a registered user - from_address = from_list[0][1] - registration_info = "" - if self.db.security.hasPermission('Web Access', author) and \ - self.db.security.hasPermission('Register', anonid, 'user'): - tracker_web = self.instance.config.TRACKER_WEB - registration_info = """ Please register at: + return 0 -%(tracker_web)suser?template=register -...before sending mail to the tracker.""" % locals() + def do_apop(self, server, user='', password='', ssl=False): + ''' Do authentication POP + ''' + self._do_pop(server, user, password, True, ssl) - raise Unauthorized, _(""" -You are not a registered user.%(registration_info)s + def do_pop(self, server, user='', password='', ssl=False): + ''' Do plain POP + ''' + self._do_pop(server, user, password, False, ssl) + + def _do_pop(self, server, user, password, apop, ssl): + '''Read a series of messages from the specified POP server. + ''' + import getpass, poplib, socket + try: + if not user: + user = raw_input('User: ') + if not password: + password = getpass.getpass() + except (KeyboardInterrupt, EOFError): + # Ctrl C or D maybe also Ctrl Z under Windows. + print "\nAborted by user." + return 1 -Unknown address: %(from_address)s -""") % locals() + # open a connection to the server and retrieve all messages + try: + if ssl: + klass = poplib.POP3_SSL else: - # we're registered and we're _still_ not allowed access - raise Unauthorized, _( - 'You are not permitted to access this tracker.') - - # make sure they're allowed to edit or create this class of information - if nodeid: - if not self.db.security.hasPermission('Edit', author, classname, - itemid=nodeid): - raise Unauthorized, _( - 'You are not permitted to edit %(classname)s.') % locals() + klass = poplib.POP3 + server = klass(server) + except socket.error: + self.logger.exception('POP server error') + return 1 + if apop: + server.apop(user, password) else: - if not self.db.security.hasPermission('Create', author, classname): - raise Unauthorized, _( - 'You are not permitted to create %(classname)s.' - ) % locals() - - # the author may have been created - make sure the change is - # committed before we reopen the database - self.db.commit() - - # set the database user as the author - username = self.db.user.get(author, 'username') - self.db.setCurrentUser(username) + server.user(user) + server.pass_(password) + numMessages = len(server.list()[1]) + for i in range(1, numMessages+1): + # retr: returns + # [ pop response e.g. '+OK 459 octets', + # [ array of message lines ], + # number of octets ] + lines = server.retr(i)[1] + s = cStringIO.StringIO('\n'.join(lines)) + s.seek(0) + self.handle_Message(Message(s)) + # delete the message + server.dele(i) - # re-get the class with the new database connection - cl = self.db.getclass(classname) + # quit the server to commit changes. + server.quit() + return 0 - # now update the recipients list - recipients = [] - tracker_email = config['TRACKER_EMAIL'].lower() - for recipient in message.getaddrlist('to') + message.getaddrlist('cc'): - r = recipient[1].strip().lower() - if r == tracker_email or not r: - continue + def main(self, fp): + ''' fp - the file from which to read the Message. + ''' + return self.handle_Message(Message(fp)) - # look up the recipient - create if necessary (and we're - # allowed to) - recipient = uidFromAddress(self.db, recipient, create, **user_props) + def handle_Message(self, message): + """Handle an RFC822 Message - # if all's well, add the recipient to the list - if recipient: - recipients.append(recipient) + Handle the Message object by calling handle_message() and then cope + with any errors raised by handle_message. + This method's job is to make that call and handle any + errors in a sane manner. It should be replaced if you wish to + handle errors in a different manner. + """ + # in some rare cases, a particularly stuffed-up e-mail will make + # its way into here... try to handle it gracefully - # - # handle the subject argument list - # - # figure what the properties of this Class are - properties = cl.getprops() - props = {} - args = matches['args'] - argswhole = matches['argswhole'] - if args: - if sfxmode == 'none': - title += ' ' + argswhole - else: - errors, props = setPropArrayFromString(self, cl, args, nodeid) - # handle any errors parsing the argument list - if errors: - if sfxmode == 'strict': - errors = '\n- '.join(map(str, errors)) - raise MailUsageError, _(""" -There were problems handling your subject line argument list: -- %(errors)s + self.parsed_message = None + sendto = message.getaddrlist('resent-from') + if not sendto: + sendto = message.getaddrlist('from') + if not sendto: + # very bad-looking message - we don't even know who sent it + msg = ['Badly formed message from mail gateway. Headers:'] + msg.extend(message.headers) + msg = '\n'.join(map(str, msg)) + self.logger.error(msg) + return -Subject was: "%(subject)s" -""") % locals() - else: - title += ' ' + argswhole + msg = 'Handling message' + if message.getheader('message-id'): + msg += ' (Message-id=%r)'%message.getheader('message-id') + self.logger.info(msg) + # try normal message-handling + if not self.trapExceptions: + return self.handle_message(message) - # set the issue title to the subject - title = title.strip() - if (title and properties.has_key('title') and not - issue_props.has_key('title')): - issue_props['title'] = title - if (nodeid and properties.has_key('title') and not - config['MAILGW_SUBJECT_UPDATES_TITLE']): - issue_props['title'] = cl.get(nodeid,'title') + # no, we want to trap exceptions + # Note: by default we return the message received not the + # internal state of the parsedMessage -- except for + # MailUsageError, Unauthorized and for unknown exceptions. For + # the latter cases we make sure the error message is encrypted + # if needed (if it either was received encrypted or pgp + # processing is turned on for the user). + try: + return self.handle_message(message) + except MailUsageHelp: + # bounce the message back to the sender with the usage message + fulldoc = '\n'.join(string.split(__doc__, '\n')[2:]) + m = [''] + m.append('\n\nMail Gateway Help\n=================') + m.append(fulldoc) + self.mailer.bounce_message(message, [sendto[0][1]], m, + subject="Mail Gateway Help") + except MailUsageError, value: + # bounce the message back to the sender with the usage message + fulldoc = '\n'.join(string.split(__doc__, '\n')[2:]) + m = [''] + m.append(str(value)) + m.append('\n\nMail Gateway Help\n=================') + m.append(fulldoc) + if self.parsed_message: + message = self.parsed_message.message + crypt = self.parsed_message.crypt + self.mailer.bounce_message(message, [sendto[0][1]], m, crypt=crypt) + except Unauthorized, value: + # just inform the user that he is not authorized + m = [''] + m.append(str(value)) + if self.parsed_message: + message = self.parsed_message.message + crypt = self.parsed_message.crypt + self.mailer.bounce_message(message, [sendto[0][1]], m, crypt=crypt) + except IgnoreMessage: + # do not take any action + # this exception is thrown when email should be ignored + msg = 'IgnoreMessage raised' + if message.getheader('message-id'): + msg += ' (Message-id=%r)'%message.getheader('message-id') + self.logger.info(msg) + return + except: + msg = 'Exception handling message' + if message.getheader('message-id'): + msg += ' (Message-id=%r)'%message.getheader('message-id') + self.logger.exception(msg) - # - # handle message-id and in-reply-to - # - messageid = message.getheader('message-id') - # generate a messageid if there isn't one - if not messageid: - messageid = "<%s.%s.%s%s@%s>"%(time.time(), random.random(), - classname, nodeid, config['MAIL_DOMAIN']) + # bounce the message back to the sender with the error message + # let the admin know that something very bad is happening + m = [''] + m.append('An unexpected error occurred during the processing') + m.append('of your message. The tracker administrator is being') + m.append('notified.\n') + if self.parsed_message: + message = self.parsed_message.message + crypt = self.parsed_message.crypt + self.mailer.bounce_message(message, [sendto[0][1]], m, crypt=crypt) - # if they've enabled PGP processing then verify the signature - # or decrypt the message + m.append('----------------') + m.append(traceback.format_exc()) + self.mailer.bounce_message(message, [self.instance.config.ADMIN_EMAIL], m) - # if PGP_ROLES is specified the user must have a Role in the list - # or we will skip PGP processing - def pgp_role(): - if self.instance.config.PGP_ROLES: - return self.db.user.has_role(author, - iter_roles(self.instance.config.PGP_ROLES)) - else: - return True + def handle_message(self, message): + ''' message - a Message instance - if self.instance.config.PGP_ENABLE and pgp_role(): - assert pyme, 'pyme is not installed' - # signed/encrypted mail must come from the primary address - author_address = self.db.user.get(author, 'address') - if self.instance.config.PGP_HOMEDIR: - os.environ['GNUPGHOME'] = self.instance.config.PGP_HOMEDIR - if message.pgp_signed(): - message.verify_signature(author_address) - elif message.pgp_encrypted(): - # replace message with the contents of the decrypted - # message for content extraction - # TODO: encrypted message handling is far from perfect - # bounces probably include the decrypted message, for - # instance :( - message = message.decrypt(author_address) - else: - raise MailUsageError, _(""" -This tracker has been configured to require all email be PGP signed or -encrypted.""") - # now handle the body - find the message - ig = self.instance.config.MAILGW_IGNORE_ALTERNATIVES - content, attachments = message.extract_content(ignore_alternatives = ig) - if content is None: - raise MailUsageError, _(""" -Roundup requires the submission to be plain text. The message parser could -not find a text/plain part to use. -""") + Parse the message as per the module docstring. + ''' + # get database handle for handling one email + self.db = self.instance.open ('admin') + try: + return self._handle_message(message) + finally: + self.db.close() - # parse the body of the message, stripping out bits as appropriate - summary, content = parseContent(content, config=config) - content = content.strip() + def _handle_message(self, message): + ''' message - a Message instance - # - # handle the attachments - # - files = [] - if attachments and properties.has_key('files'): - for (name, mime_type, data) in attachments: - if not self.db.security.hasPermission('Create', author, 'file'): - raise Unauthorized, _( - 'You are not permitted to create files.') - if not name: - name = "unnamed" - try: - fileid = self.db.file.create(type=mime_type, name=name, - content=data, **file_props) - except exceptions.Reject: - pass - else: - files.append(fileid) - # allowed to attach the files to an existing node? - if nodeid and not self.db.security.hasPermission('Edit', author, - classname, 'files'): - raise Unauthorized, _( - 'You are not permitted to add files to %(classname)s.' - ) % locals() + Parse the message as per the module docstring. + The following code expects an opened database and a try/finally + that closes the database. + ''' + self.parsed_message = self.parsed_message_class(self, message) + nodeid = self.parsed_message.parse () - if nodeid: - # extend the existing files list - fileprop = cl.get(nodeid, 'files') - fileprop.extend(files) - props['files'] = fileprop - else: - # pre-load the files list - props['files'] = files + # commit the changes to the DB + self.db.commit() - # - # create the message if there's a message body (content) - # - if (content and properties.has_key('messages')): - if not self.db.security.hasPermission('Create', author, 'msg'): - raise Unauthorized, _( - 'You are not permitted to create messages.') + self.parsed_message = None + return nodeid - try: - message_id = self.db.msg.create(author=author, - recipients=recipients, date=date.Date('.'), - summary=summary, content=content, files=files, - messageid=messageid, inreplyto=inreplyto, **msg_props) - except exceptions.Reject, error: - raise MailUsageError, _(""" -Mail message was rejected by a detector. -%(error)s -""") % locals() - # allowed to attach the message to the existing node? - if nodeid and not self.db.security.hasPermission('Edit', author, - classname, 'messages'): - raise Unauthorized, _( - 'You are not permitted to add messages to %(classname)s.' - ) % locals() + def get_class_arguments(self, class_type, classname=None): + ''' class_type - a valid node class type: + - 'user' refers to the author of a message + - 'issue' refers to an issue-type class (to which the + message is appended) specified in parameter classname + Note that this need not be the real classname, we get + the real classname used as a parameter (from previous + message-parsing steps) + - 'file' specifies a file-type class + - 'msg' is the message-class + classname - the name of the current issue-type class + + Parse the commandline arguments and retrieve the properties that + are relevant to the class_type. We now allow multiple -S options + per class_type (-C option). + ''' + allprops = {} - if nodeid: - # add the message to the node's list - messages = cl.get(nodeid, 'messages') - messages.append(message_id) - props['messages'] = messages - else: - # pre-load the messages list - props['messages'] = [message_id] + classname = classname or class_type + cls_lookup = { 'issue' : classname } + + # Allow other issue-type classes -- take the real classname from + # previous parsing-steps of the message: + clsname = cls_lookup.get (class_type, class_type) - # - # perform the node change / create - # + # check if the clsname is valid try: - # merge the command line props defined in issue_props into - # the props dictionary because function(**props, **issue_props) - # is a syntax error. - for prop in issue_props.keys() : - if not props.has_key(prop) : - props[prop] = issue_props[prop] - - if nodeid: - # Check permissions for each property - for prop in props.keys(): - if not self.db.security.hasPermission('Edit', author, - classname, prop): - raise Unauthorized, _('You are not permitted to edit ' - 'property %(prop)s of class %(classname)s.') % locals() - cl.set(nodeid, **props) - else: - # Check permissions for each property - for prop in props.keys(): - if not self.db.security.hasPermission('Create', author, - classname, prop): - raise Unauthorized, _('You are not permitted to set ' - 'property %(prop)s of class %(classname)s.') % locals() - nodeid = cl.create(**props) - except (TypeError, IndexError, ValueError, exceptions.Reject), message: + self.db.getclass(clsname) + except KeyError: + mailadmin = self.instance.config['ADMIN_EMAIL'] raise MailUsageError, _(""" -There was a problem with the message you sent: - %(message)s +The mail gateway is not properly set up. Please contact +%(mailadmin)s and have them fix the incorrect class specified as: + %(clsname)s """) % locals() + + if self.arguments: + # The default type on the commandline is msg + if class_type == 'msg': + current_type = class_type + else: + current_type = None + + # Handle the arguments specified by the email gateway command line. + # We do this by looping over the list of self.arguments looking for + # a -C to match the class we want, then use the -S setting string. + for option, propstring in self.arguments: + if option in ( '-C', '--class'): + current_type = propstring.strip() + + if current_type != class_type: + current_type = None - # commit the changes to the DB - self.db.commit() + elif current_type and option in ('-S', '--set'): + cls = cls_lookup.get (current_type, current_type) + temp_cl = self.db.getclass(cls) + errors, props = setPropArrayFromString(self, + temp_cl, propstring.strip()) - return nodeid + if errors: + mailadmin = self.instance.config['ADMIN_EMAIL'] + raise MailUsageError, _(""" +The mail gateway is not properly set up. Please contact +%(mailadmin)s and have them fix the incorrect properties: + %(errors)s +""") % locals() + allprops.update(props) + + return allprops def setPropArrayFromString(self, cl, propString, nodeid=None): @@ -1453,7 +1691,17 @@ def uidFromAddress(db, address, create=1, **user_props): props = db.user.getprops() if props.has_key('alternate_addresses'): users = db.user.filter(None, {'alternate_addresses': address}) - user = extractUserFromList(db.user, users) + # We want an exact match of the email, not just a substring + # match. Otherwise e.g. support@example.com would match + # discuss-support@example.com which is not what we want. + found_users = [] + for u in users: + alt = db.user.get(u, 'alternate_addresses').split('\n') + for a in alt: + if a.strip().lower() == address.lower(): + found_users.append(u) + break + user = extractUserFromList(db.user, found_users) if user is not None: return user @@ -1483,7 +1731,7 @@ def uidFromAddress(db, address, create=1, **user_props): try: return db.user.create(username=trying, address=address, realname=realname, roles=db.config.NEW_EMAIL_USER_ROLES, - password=password.Password(password.generatePassword()), + password=password.Password(password.generatePassword(), config=db.config), **user_props) except exceptions.Reject: return 0