X-Git-Url: https://git.tokkee.org/?a=blobdiff_plain;f=roundup%2Fmailgw.py;h=53dc0417fbd433ffececfa1765b58f530c28ab07;hb=4d313534b6cb918767a12d4a5e0189004c877e04;hp=f1edd3d33e0a684e0d77bdba5b2b0b99a700afe1;hpb=a4110db368267edb3b5c82222b7ceb1ccd5b9f7b;p=roundup.git diff --git a/roundup/mailgw.py b/roundup/mailgw.py index f1edd3d..53dc041 100644 --- a/roundup/mailgw.py +++ b/roundup/mailgw.py @@ -16,7 +16,7 @@ # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -__doc__ = ''' +''' An e-mail gateway for Roundup. Incoming messages are examined for multiple parts: @@ -73,15 +73,16 @@ are calling the create() method to create a new node). If an auditor raises an exception, the original message is bounced back to the sender with the explanatory message given in the exception. -$Id: mailgw.py,v 1.48 2002-01-08 04:12:05 richard Exp $ +$Id: mailgw.py,v 1.111 2003-02-27 05:43:01 richard Exp $ ''' - import string, re, os, mimetools, cStringIO, smtplib, socket, binascii, quopri -import time, random -import traceback, MimeWriter +import time, random, sys +import traceback, MimeWriter, rfc822 import hyperdb, date, password +import rfc2822 + SENDMAILDEBUG = os.environ.get('SENDMAILDEBUG', '') class MailGWError(ValueError): @@ -90,9 +91,48 @@ class MailGWError(ValueError): class MailUsageError(ValueError): pass -class UnAuthorized(Exception): +class MailUsageHelp(Exception): + pass + +class MailLoop(Exception): + ''' We've seen this message before... ''' + pass + +class Unauthorized(Exception): """ Access denied """ +def initialiseSecurity(security): + ''' Create some Permissions and Roles on the security object + + This function is directly invoked by security.Security.__init__() + as a part of the Security object instantiation. + ''' + security.addPermission(name="Email Registration", + description="Anonymous may register through e-mail") + p = security.addPermission(name="Email Access", + description="User may use the email interface") + security.addPermissionToRole('Admin', p) + +def getparam(str, param): + ''' From the rfc822 "header" string, extract "param" if it appears. + ''' + if ';' not in str: + return None + str = str[str.index(';'):] + while str[:1] == ';': + str = str[1:] + if ';' in str: + # XXX Should parse quotes! + end = str.index(';') + else: + end = len(str) + f = str[:end] + if '=' in f: + i = f.index('=') + if f[:i].strip().lower() == param: + return rfc822.unquote(f[i+1:].strip()) + return None + class Message(mimetools.Message): ''' subclass mimetools.Message so we can retrieve the parts of the message... @@ -116,19 +156,109 @@ class Message(mimetools.Message): s.seek(0) return Message(s) -subject_re = re.compile(r'(?P\s*\W?\s*(fwd|re)\s*\W?\s*)*' - r'\s*(\[(?P[^\d\s]+)(?P\d+)?\])' - r'\s*(?P[^[]+)?(\[(?P<args>.+?)\])?', re.I) + def getheader(self, name, default=None): + hdr = mimetools.Message.getheader(self, name, default) + return rfc2822.decode_header(hdr) + +subject_re = re.compile(r'(?P<refwd>\s*\W?\s*(fw|fwd|re|aw)\W\s*)*' + r'\s*(?P<quote>")?(\[(?P<classname>[^\d\s]+)(?P<nodeid>\d+)?\])?' + r'\s*(?P<title>[^[]+)?"?(\[(?P<args>.+?)\])?', re.I) class MailGW: - def __init__(self, instance, db): + def __init__(self, instance, db, arguments={}): self.instance = instance self.db = db + self.arguments = arguments + + # should we trap exceptions (normal usage) or pass them through + # (for testing) + self.trapExceptions = 1 + + def do_pipe(self): + ''' Read a message from standard input and pass it to the mail handler. + + Read into an internal structure that we can seek on (in case + there's an error). + + XXX: we may want to read this into a temporary file instead... + ''' + s = cStringIO.StringIO() + s.write(sys.stdin.read()) + s.seek(0) + self.main(s) + return 0 + + def do_mailbox(self, filename): + ''' Read a series of messages from the specified unix mailbox file and + pass each to the mail handler. + ''' + # open the spool file and lock it + import fcntl, FCNTL + f = open(filename, 'r+') + fcntl.flock(f.fileno(), FCNTL.LOCK_EX) + + # handle and clear the mailbox + try: + from mailbox import UnixMailbox + mailbox = UnixMailbox(f, factory=Message) + # grab one message + message = mailbox.next() + while message: + # handle this message + self.handle_Message(message) + message = mailbox.next() + # nuke the file contents + os.ftruncate(f.fileno(), 0) + except: + import traceback + traceback.print_exc() + return 1 + fcntl.flock(f.fileno(), FCNTL.LOCK_UN) + return 0 + + def do_pop(self, server, user='', password=''): + '''Read a series of messages from the specified POP server. + ''' + import getpass, poplib, socket + try: + if not user: + user = raw_input(_('User: ')) + if not password: + password = getpass.getpass() + except (KeyboardInterrupt, EOFError): + # Ctrl C or D maybe also Ctrl Z under Windows. + print "\nAborted by user." + return 1 + + # open a connection to the server and retrieve all messages + try: + server = poplib.POP3(server) + except socket.error, message: + print "POP server error:", message + return 1 + server.user(user) + server.pass_(password) + numMessages = len(server.list()[1]) + for i in range(1, numMessages+1): + # retr: returns + # [ pop response e.g. '+OK 459 octets', + # [ array of message lines ], + # number of octets ] + lines = server.retr(i)[1] + s = cStringIO.StringIO('\n'.join(lines)) + s.seek(0) + self.handle_Message(Message(s)) + # delete the message + server.dele(i) + + # quit the server to commit changes. + server.quit() + return 0 def main(self, fp): ''' fp - the file from which to read the Message. ''' - self.handle_Message(Message(fp)) + return self.handle_Message(Message(fp)) def handle_Message(self, message): '''Handle an RFC822 Message @@ -143,8 +273,19 @@ class MailGW: # its way into here... try to handle it gracefully sendto = message.getaddrlist('from') if sendto: + if not self.trapExceptions: + return self.handle_message(message) try: return self.handle_message(message) + except MailUsageHelp: + # bounce the message back to the sender with the usage message + fulldoc = '\n'.join(string.split(__doc__, '\n')[2:]) + sendto = [sendto[0][1]] + m = [''] + m.append('\n\nMail Gateway Help\n=================') + m.append(fulldoc) + m = self.bounce_message(message, sendto, m, + subject="Mail Gateway Help") except MailUsageError, value: # bounce the message back to the sender with the usage message fulldoc = '\n'.join(string.split(__doc__, '\n')[2:]) @@ -154,16 +295,23 @@ class MailGW: m.append('\n\nMail Gateway Help\n=================') m.append(fulldoc) m = self.bounce_message(message, sendto, m) - except UnAuthorized, value: + except Unauthorized, value: # just inform the user that he is not authorized sendto = [sendto[0][1]] m = [''] m.append(str(value)) m = self.bounce_message(message, sendto, m) + except MailLoop: + # XXX we should use a log file here... + return except: # bounce the message back to the sender with the error message - sendto = [sendto[0][1]] + # XXX we should use a log file here... + sendto = [sendto[0][1], self.instance.config.ADMIN_EMAIL] m = [''] + m.append('An unexpected error occurred during the processing') + m.append('of your message. The tracker administrator is being') + m.append('notified.\n') m.append('---- traceback of failure ----') s = cStringIO.StringIO() import traceback @@ -172,7 +320,8 @@ class MailGW: m = self.bounce_message(message, sendto, m) else: # very bad-looking message - we don't even know who sent it - sendto = [self.ADMIN_EMAIL] + # XXX we should use a log file here... + sendto = [self.instance.config.ADMIN_EMAIL] m = ['Subject: badly formed message from mail gateway'] m.append('') m.append('The mail gateway retrieved a message which has no From:') @@ -184,12 +333,14 @@ class MailGW: # now send the message if SENDMAILDEBUG: - open(SENDMAILDEBUG, 'w').write('From: %s\nTo: %s\n%s\n'%( - self.ADMIN_EMAIL, ', '.join(sendto), m.getvalue())) + open(SENDMAILDEBUG, 'a').write('From: %s\nTo: %s\n%s\n'%( + self.instance.config.ADMIN_EMAIL, ', '.join(sendto), + m.getvalue())) else: try: - smtp = smtplib.SMTP(self.MAILHOST) - smtp.sendmail(self.ADMIN_EMAIL, sendto, m.getvalue()) + smtp = smtplib.SMTP(self.instance.config.MAILHOST) + smtp.sendmail(self.instance.config.ADMIN_EMAIL, sendto, + m.getvalue()) except socket.error, value: raise MailGWError, "Couldn't send error email: "\ "mailhost %s"%value @@ -204,52 +355,115 @@ class MailGW: ''' msg = cStringIO.StringIO() writer = MimeWriter.MimeWriter(msg) + writer.addheader('X-Roundup-Loop', 'hello') writer.addheader('Subject', subject) - writer.addheader('From', '%s <%s>'% (self.instance.INSTANCE_NAME, - self.ISSUE_TRACKER_EMAIL)) + writer.addheader('From', '%s <%s>'% (self.instance.config.TRACKER_NAME, + self.instance.config.TRACKER_EMAIL)) writer.addheader('To', ','.join(sendto)) + writer.addheader('Date', time.strftime("%a, %d %b %Y %H:%M:%S +0000", + time.gmtime())) writer.addheader('MIME-Version', '1.0') part = writer.startmultipartbody('mixed') part = writer.nextpart() - body = part.startbody('text/plain') + body = part.startbody('text/plain; charset=utf-8') body.write('\n'.join(error)) - # reconstruct the original message - m = cStringIO.StringIO() - w = MimeWriter.MimeWriter(m) - # default the content_type, just in case... - content_type = 'text/plain' - # add the headers except the content-type - for header in message.headers: - header_name = header.split(':')[0] - if header_name.lower() == 'content-type': - content_type = message.getheader(header_name) - elif message.getheader(header_name): - w.addheader(header_name, message.getheader(header_name)) - # now attach the message body - body = w.startbody(content_type) - message.rewindbody() - body.write(message.fp.read()) - # attach the original message to the returned message part = writer.nextpart() part.addheader('Content-Disposition','attachment') - part.addheader('Content-Description','Message that caused the error') - part.addheader('Content-Transfer-Encoding', '7bit') - body = part.startbody('message/rfc822') - body.write(m.getvalue()) + part.addheader('Content-Description','Message you sent') + body = part.startbody('text/plain') + for header in message.headers: + body.write(header) + body.write('\n') + try: + message.rewindbody() + except IOError, message: + body.write("*** couldn't include message body: %s ***"%message) + else: + body.write(message.fp.read()) writer.lastpart() return msg + def get_part_data_decoded(self,part): + encoding = part.getencoding() + data = None + if encoding == 'base64': + # BUG: is base64 really used for text encoding or + # are we inserting zip files here. + data = binascii.a2b_base64(part.fp.read()) + elif encoding == 'quoted-printable': + # the quopri module wants to work with files + decoded = cStringIO.StringIO() + quopri.decode(part.fp, decoded) + data = decoded.getvalue() + elif encoding == 'uuencoded': + data = binascii.a2b_uu(part.fp.read()) + else: + # take it as text + data = part.fp.read() + + # Encode message to unicode + charset = rfc2822.unaliasCharset(part.getparam("charset")) + if charset: + # Do conversion only if charset specified + edata = unicode(data, charset).encode('utf-8') + # Convert from dos eol to unix + edata = edata.replace('\r\n', '\n') + else: + # Leave message content as is + edata = data + + return edata + def handle_message(self, message): ''' message - a Message instance Parse the message as per the module docstring. ''' + # detect loops + if message.getheader('x-roundup-loop', ''): + raise MailLoop + + # XXX Don't enable. This doesn't work yet. +# "[^A-z.]tracker\+(?P<classname>[^\d\s]+)(?P<nodeid>\d+)\@some.dom.ain[^A-z.]" + # handle delivery to addresses like:tracker+issue25@some.dom.ain + # use the embedded issue number as our issue +# if hasattr(self.instance.config, 'EMAIL_ISSUE_ADDRESS_RE') and \ +# self.instance.config.EMAIL_ISSUE_ADDRESS_RE: +# issue_re = self.instance.config.EMAIL_ISSUE_ADDRESS_RE +# for header in ['to', 'cc', 'bcc']: +# addresses = message.getheader(header, '') +# if addresses: +# # FIXME, this only finds the first match in the addresses. +# issue = re.search(issue_re, addresses, 'i') +# if issue: +# classname = issue.group('classname') +# nodeid = issue.group('nodeid') +# break + # handle the subject line subject = message.getheader('subject', '') + + if subject.strip().lower() == 'help': + raise MailUsageHelp + m = subject_re.match(subject) + + # check for well-formed subject line + if m: + # get the classname + classname = m.group('classname') + if classname is None: + # no classname, fallback on the default + if hasattr(self.instance.config, 'MAIL_DEFAULT_CLASS') and \ + self.instance.config.MAIL_DEFAULT_CLASS: + classname = self.instance.config.MAIL_DEFAULT_CLASS + else: + # fail + m = None + if not m: raise MailUsageError, ''' The message you sent to roundup did not contain a properly formed subject @@ -265,8 +479,7 @@ line. The subject must contain a class name or designator to indicate the Subject was: "%s" '''%subject - # get the classname - classname = m.group('classname') + # get the class try: cl = self.db.getclass(classname) except KeyError: @@ -288,8 +501,13 @@ Subject was: "%s" else: title = '' + # strip off the quotes that dumb emailers put around the subject, like + # Re: "[issue1] bla blah" + if m.group('quote') and title.endswith('"'): + title = title[:-1] + # but we do need either a title or a nodeid... - if not nodeid and not title: + if nodeid is None and not title: raise MailUsageError, ''' I cannot match your message to a node in the database - you need to either supply a full node identifier (with number, eg "[issue123]" or keep the @@ -298,114 +516,112 @@ previous subject title intact so I can match that. Subject was: "%s" '''%subject - # extract the args - subject_args = m.group('args') - # If there's no nodeid, check to see if this is a followup and # maybe someone's responded to the initial mail that created an # entry. Try to find the matching nodes with the same title, and # use the _last_ one matched (since that'll _usually_ be the most # recent...) - if not nodeid and m.group('refwd'): + if nodeid is None and m.group('refwd'): l = cl.stringFind(title=title) if l: nodeid = l[-1] - # start of the props - properties = cl.getprops() - props = {} - - # handle the args - args = m.group('args') - if args: - for prop in string.split(args, ';'): - try: - key, value = prop.split('=') - except ValueError, message: - raise MailUsageError, ''' -Subject argument list not of form [arg=value,value,...;arg=value,value...] - (specific exception message was "%s") - -Subject was: "%s" -'''%(message, subject) - key = key.strip() - try: - proptype = properties[key] - except KeyError: - raise MailUsageError, ''' -Subject argument list refers to an invalid property: "%s" + # if a nodeid was specified, make sure it's valid + if nodeid is not None and not cl.hasnode(nodeid): + raise MailUsageError, ''' +The node specified by the designator in the subject of your message ("%s") +does not exist. Subject was: "%s" -'''%(key, subject) - if isinstance(proptype, hyperdb.String): - props[key] = value.strip() - if isinstance(proptype, hyperdb.Password): - props[key] = password.Password(value.strip()) - elif isinstance(proptype, hyperdb.Date): - try: - props[key] = date.Date(value.strip()) - except ValueError, message: - raise UsageError, ''' -Subject argument list contains an invalid date for %s. +'''%(nodeid, subject) -Error was: %s -Subject was: "%s" -'''%(key, message, subject) - elif isinstance(proptype, hyperdb.Interval): - try: - props[key] = date.Interval(value) # no strip needed - except ValueError, message: - raise UsageError, ''' -Subject argument list contains an invalid date interval for %s. -Error was: %s -Subject was: "%s" -'''%(key, message, subject) - elif isinstance(proptype, hyperdb.Link): - link = self.db.classes[proptype.classname] - propkey = link.labelprop(default_to_id=1) - try: - props[key] = link.get(value.strip(), propkey) - except: - props[key] = link.lookup(value.strip()) - elif isinstance(proptype, hyperdb.Multilink): - link = self.db.classes[proptype.classname] - propkey = link.labelprop(default_to_id=1) - l = [x.strip() for x in value.split(',')] - for item in l: - try: - v = link.get(item, propkey) - except: - v = link.lookup(item) - if props.has_key(key): - props[key].append(v) - else: - props[key] = [v] + # Handle the arguments specified by the email gateway command line. + # We do this by looping over the list of self.arguments looking for + # a -C to tell us what class then the -S setting string. + msg_props = {} + user_props = {} + file_props = {} + issue_props = {} + # so, if we have any arguments, use them + if self.arguments: + current_class = 'msg' + for option, propstring in self.arguments: + if option in ( '-C', '--class'): + current_class = propstring.strip() + if current_class not in ('msg', 'file', 'user', 'issue'): + raise MailUsageError, ''' +The mail gateway is not properly set up. Please contact +%s and have them fix the incorrect class specified as: + %s +'''%(self.instance.config.ADMIN_EMAIL, current_class) + if option in ('-S', '--set'): + if current_class == 'issue' : + errors, issue_props = setPropArrayFromString(self, + cl, propstring.strip(), nodeid) + elif current_class == 'file' : + temp_cl = self.db.getclass('file') + errors, file_props = setPropArrayFromString(self, + temp_cl, propstring.strip()) + elif current_class == 'msg' : + temp_cl = self.db.getclass('msg') + errors, msg_props = setPropArrayFromString(self, + temp_cl, propstring.strip()) + elif current_class == 'user' : + temp_cl = self.db.getclass('user') + errors, user_props = setPropArrayFromString(self, + temp_cl, propstring.strip()) + if errors: + raise MailUsageError, ''' +The mail gateway is not properly set up. Please contact +%s and have them fix the incorrect properties: + %s +'''%(self.instance.config.ADMIN_EMAIL, errors) # # handle the users # - - # Don't create users if ANONYMOUS_REGISTER is denied - if self.ANONYMOUS_REGISTER == 'deny': + # Don't create users if anonymous isn't allowed to register + create = 1 + anonid = self.db.user.lookup('anonymous') + if not self.db.security.hasPermission('Email Registration', anonid): create = 0 - else: - create = 1 - author = self.db.uidFromAddress(message.getaddrlist('from')[0], + + # ok, now figure out who the author is - create a new user if the + # "create" flag is true + author = uidFromAddress(self.db, message.getaddrlist('from')[0], create=create) + + # if we're not recognised, and we don't get added as a user, then we + # must be anonymous if not author: - raise UnAuthorized, ''' + author = anonid + + # make sure the author has permission to use the email interface + if not self.db.security.hasPermission('Email Access', author): + if author == anonid: + # we're anonymous and we need to be a registered user + raise Unauthorized, ''' You are not a registered user. Unknown address: %s '''%message.getaddrlist('from')[0][1] + else: + # we're registered and we're _still_ not allowed access + raise Unauthorized, 'You are not permitted to access '\ + 'this tracker.' + + # make sure they're allowed to edit this class of information + if not self.db.security.hasPermission('Edit', author, classname): + raise Unauthorized, 'You are not permitted to edit %s.'%classname # the author may have been created - make sure the change is # committed before we reopen the database self.db.commit() - + # reopen the database as the author username = self.db.user.get(author, 'username') + self.db.close() self.db = self.instance.open(username) # re-get the class with the new database connection @@ -413,12 +629,43 @@ Unknown address: %s # now update the recipients list recipients = [] - tracker_email = self.ISSUE_TRACKER_EMAIL.lower() + tracker_email = self.instance.config.TRACKER_EMAIL.lower() for recipient in message.getaddrlist('to') + message.getaddrlist('cc'): r = recipient[1].strip().lower() if r == tracker_email or not r: continue - recipients.append(self.db.uidFromAddress(recipient)) + + # look up the recipient - create if necessary (and we're + # allowed to) + recipient = uidFromAddress(self.db, recipient, create, **user_props) + + # if all's well, add the recipient to the list + if recipient: + recipients.append(recipient) + + # + # XXX extract the args NOT USED WHY -- rouilj + # + subject_args = m.group('args') + + # + # handle the subject argument list + # + # figure what the properties of this Class are + properties = cl.getprops() + props = {} + args = m.group('args') + if args: + errors, props = setPropArrayFromString(self, cl, args, nodeid) + # handle any errors parsing the argument list + if errors: + errors = '\n- '.join(errors) + raise MailUsageError, ''' +There were problems handling your subject line argument list: +- %s + +Subject was: "%s" +'''%(errors, subject) # # handle message-id and in-reply-to @@ -428,13 +675,39 @@ Unknown address: %s # generate a messageid if there isn't one if not messageid: messageid = "<%s.%s.%s%s@%s>"%(time.time(), random.random(), - classname, nodeid, self.MAIL_DOMAIN) + classname, nodeid, self.instance.config.MAIL_DOMAIN) # # now handle the body - find the message # content_type = message.gettype() attachments = [] + # General multipart handling: + # Take the first text/plain part, anything else is considered an + # attachment. + # multipart/mixed: multiple "unrelated" parts. + # multipart/signed (rfc 1847): + # The control information is carried in the second of the two + # required body parts. + # ACTION: Default, so if content is text/plain we get it. + # multipart/encrypted (rfc 1847): + # The control information is carried in the first of the two + # required body parts. + # ACTION: Not handleable as the content is encrypted. + # multipart/related (rfc 1872, 2112, 2387): + # The Multipart/Related content-type addresses the MIME + # representation of compound objects. + # ACTION: Default. If we are lucky there is a text/plain. + # TODO: One should use the start part and look for an Alternative + # that is text/plain. + # multipart/Alternative (rfc 1872, 1892): + # only in "related" ? + # multipart/report (rfc 1892): + # e.g. mail system delivery status reports. + # ACTION: Default. Could be ignored or used for Delivery Notification + # flagging. + # multipart/form-data: + # For web forms only. if content_type == 'multipart/mixed': # skip over the intro to the first boundary part = message.getPart() @@ -447,12 +720,8 @@ Unknown address: %s # parse it subtype = part.gettype() if subtype == 'text/plain' and not content: - # add all text/plain parts to the message content - if content is None: - content = part.fp.read() - else: - content = content + part.fp.read() - + # The first text/plain part is the message content. + content = self.get_part_data_decoded(part) elif subtype == 'message/rfc822': # handle message/rfc822 specially - the name should be # the subject of the actual e-mail embedded here @@ -461,23 +730,33 @@ Unknown address: %s name = mailmess.getheader('subject') part.fp.seek(i) attachments.append((name, 'message/rfc822', part.fp.read())) - + elif subtype == 'multipart/alternative': + # Search for text/plain in message with attachment and + # alternative text representation + # skip over intro to first boundary + part.getPart() + while 1: + # get the next part + subpart = part.getPart() + if subpart is None: + break + # parse it + if subpart.gettype() == 'text/plain' and not content: + content = self.get_part_data_decoded(subpart) else: # try name on Content-Type name = part.getparam('name') + if name: + name = name.strip() + if not name: + disp = part.getheader('content-disposition', None) + if disp: + name = getparam(disp, 'filename') + if name: + name = name.strip() # this is just an attachment - encoding = part.getencoding() - if encoding == 'base64': - data = binascii.a2b_base64(part.fp.read()) - elif encoding == 'quoted-printable': - # the quopri module wants to work with files - decoded = cStringIO.StringIO() - quopri.decode(part.fp, decoded) - data = decoded.getvalue() - elif encoding == 'uuencoded': - data = binascii.a2b_uu(part.fp.read()) + data = self.get_part_data_decoded(part) attachments.append((name, part.gettype(), data)) - if content is None: raise MailUsageError, ''' Roundup requires the submission to be plain text. The message parser could @@ -495,8 +774,7 @@ not find a text/plain part to use. break # parse it if part.gettype() == 'text/plain' and not content: - # this one's our content - content = part.fp.read() + content = self.get_part_data_decoded(part) if content is None: raise MailUsageError, ''' Roundup requires the submission to be plain text. The message parser could @@ -510,170 +788,254 @@ not find a text/plain part to use. ''' else: - content = message.fp.read() - - summary, content = parseContent(content) + content = self.get_part_data_decoded(message) + + # figure how much we should muck around with the email body + keep_citations = getattr(self.instance.config, 'EMAIL_KEEP_QUOTED_TEXT', + 'no') == 'yes' + keep_body = getattr(self.instance.config, 'EMAIL_LEAVE_BODY_UNCHANGED', + 'no') == 'yes' + + # parse the body of the message, stripping out bits as appropriate + summary, content = parseContent(content, keep_citations, + keep_body) # # handle the attachments # files = [] for (name, mime_type, data) in attachments: + if not name: + name = "unnamed" files.append(self.db.file.create(type=mime_type, name=name, - content=data)) + content=data, **file_props)) + # + # create the message if there's a message body (content) # - # now handle the db stuff - # - if nodeid: - # If an item designator (class name and id number) is found there, - # the newly created "msg" node is added to the "messages" property - # for that item, and any new "file" nodes are added to the "files" - # property for the item. - - # if the message is currently 'unread' or 'resolved', then set - # it to 'chatting' - if properties.has_key('status'): - try: - # determine the id of 'unread', 'resolved' and 'chatting' - unread_id = self.db.status.lookup('unread') - resolved_id = self.db.status.lookup('resolved') - chatting_id = self.db.status.lookup('chatting') - except KeyError: - pass - else: - if (not props.has_key('status') and - properties['status'] == unread_id or - properties['status'] == resolved_id): - props['status'] = chatting_id - - # add nosy in arguments to issue's nosy list - if not props.has_key('nosy'): props['nosy'] = [] - n = {} - for nid in cl.get(nodeid, 'nosy'): - n[nid] = 1 - for value in props['nosy']: - if self.db.hasnode('user', value): - nid = value - else: - continue - if n.has_key(nid): continue - n[nid] = 1 - props['nosy'] = n.keys() - # add assignedto to the nosy list - try: - assignedto = self.db.user.lookup(props['assignedto']) - if assignedto not in props['nosy']: - props['nosy'].append(assignedto) - except: - pass - + if content: message_id = self.db.msg.create(author=author, recipients=recipients, date=date.Date('.'), summary=summary, content=content, files=files, messageid=messageid, - inreplyto=inreplyto) - try: + inreplyto=inreplyto, **msg_props) + + # attach the message to the node + if nodeid: + # add the message to the node's list messages = cl.get(nodeid, 'messages') - except IndexError: - raise MailUsageError, ''' -The node specified by the designator in the subject of your message ("%s") -does not exist. + messages.append(message_id) + props['messages'] = messages + else: + # pre-load the messages list + props['messages'] = [message_id] -Subject was: "%s" -'''%(nodeid, subject) - messages.append(message_id) - props['messages'] = messages + # set the title to the subject + if properties.has_key('title') and not props.has_key('title'): + props['title'] = title - # now apply the changes - try: + # + # perform the node change / create + # + try: + # merge the command line props defined in issue_props into + # the props dictionary because function(**props, **issue_props) + # is a syntax error. + for prop in issue_props.keys() : + if not props.has_key(prop) : + props[prop] = issue_props[prop] + if nodeid: cl.set(nodeid, **props) - except (TypeError, IndexError, ValueError), message: - raise MailUsageError, ''' + else: + nodeid = cl.create(**props) + except (TypeError, IndexError, ValueError), message: + raise MailUsageError, ''' There was a problem with the message you sent: %s '''%message - # commit the changes to the DB - self.db.commit() - else: - # If just an item class name is found there, we attempt to create a - # new item of that class with its "messages" property initialized to - # contain the new "msg" node and its "files" property initialized to - # contain any new "file" nodes. - message_id = self.db.msg.create(author=author, - recipients=recipients, date=date.Date('.'), summary=summary, - content=content, files=files, messageid=messageid, - inreplyto=inreplyto) - # pre-set the issue to unread - if properties.has_key('status') and not props.has_key('status'): - try: - # determine the id of 'unread' - unread_id = self.db.status.lookup('unread') - except KeyError: - pass - else: - props['status'] = '1' + # commit the changes to the DB + self.db.commit() - # set the title to the subject - if properties.has_key('title') and not props.has_key('title'): - props['title'] = title + return nodeid - # pre-load the messages list - props['messages'] = [message_id] + +def setPropArrayFromString(self, cl, propString, nodeid = None): + ''' takes string of form prop=value,value;prop2=value + and returns (error, prop[..]) + ''' + properties = cl.getprops() + props = {} + errors = [] + for prop in string.split(propString, ';'): + # extract the property name and value + try: + propname, value = prop.split('=') + except ValueError, message: + errors.append('not of form [arg=value,value,...;' + 'arg=value,value,...]') + return (errors, props) + + # ensure it's a valid property name + propname = propname.strip() + try: + proptype = properties[propname] + except KeyError: + errors.append('refers to an invalid property: "%s"'%propname) + continue - # set up (clean) the nosy list - nosy = props.get('nosy', []) - n = {} - for value in nosy: - if self.db.hasnode('user', value): - nid = value + # convert the string value to a real property value + if isinstance(proptype, hyperdb.String): + props[propname] = value.strip() + if isinstance(proptype, hyperdb.Password): + props[propname] = password.Password(value.strip()) + elif isinstance(proptype, hyperdb.Date): + try: + props[propname] = date.Date(value.strip()).local(self.db.getUserTimezone()) + except ValueError, message: + errors.append('contains an invalid date for %s.'%propname) + elif isinstance(proptype, hyperdb.Interval): + try: + props[propname] = date.Interval(value) + except ValueError, message: + errors.append('contains an invalid date interval for %s.'% + propname) + elif isinstance(proptype, hyperdb.Link): + linkcl = self.db.classes[proptype.classname] + propkey = linkcl.labelprop(default_to_id=1) + try: + props[propname] = linkcl.lookup(value) + except KeyError, message: + errors.append('"%s" is not a value for %s.'%(value, propname)) + elif isinstance(proptype, hyperdb.Multilink): + # get the linked class + linkcl = self.db.classes[proptype.classname] + propkey = linkcl.labelprop(default_to_id=1) + if nodeid: + curvalue = cl.get(nodeid, propname) + else: + curvalue = [] + + # handle each add/remove in turn + # keep an extra list for all items that are + # definitely in the new list (in case of e.g. + # <propname>=A,+B, which should replace the old + # list with A,B) + set = 0 + newvalue = [] + for item in value.split(','): + item = item.strip() + + # handle +/- + remove = 0 + if item.startswith('-'): + remove = 1 + item = item[1:] + elif item.startswith('+'): + item = item[1:] else: - continue - if n.has_key(nid): continue - n[nid] = 1 - props['nosy'] = n.keys() - # add on the recipients of the message - for recipient in recipients: - if not n.has_key(recipient): - props['nosy'].append(recipient) - n[recipient] = 1 - - # add the author to the nosy list - if not n.has_key(author): - props['nosy'].append(author) - n[author] = 1 - - # add assignedto to the nosy list - if properties.has_key('assignedto') and props.has_key('assignedto'): + set = 1 + + # look up the value try: - assignedto = self.db.user.lookup(props['assignedto']) - except KeyError: - raise MailUsageError, ''' -There was a problem with the message you sent: - Assignedto user '%s' doesn't exist -'''%props['assignedto'] - if not n.has_key(assignedto): - props['nosy'].append(assignedto) - n[assignedto] = 1 + item = linkcl.lookup(item) + except KeyError, message: + errors.append('"%s" is not a value for %s.'%(item, + propname)) + continue - # and attempt to create the new node - try: - nodeid = cl.create(**props) - except (TypeError, IndexError, ValueError), message: - raise MailUsageError, ''' -There was a problem with the message you sent: - %s -'''%message + # perform the add/remove + if remove: + try: + curvalue.remove(item) + except ValueError: + errors.append('"%s" is not currently in for %s.'%(item, + propname)) + continue + else: + newvalue.append(item) + if item not in curvalue: + curvalue.append(item) + + # that's it, set the new Multilink property value, + # or overwrite it completely + if set: + props[propname] = newvalue + else: + props[propname] = curvalue + elif isinstance(proptype, hyperdb.Boolean): + value = value.strip() + props[propname] = value.lower() in ('yes', 'true', 'on', '1') + elif isinstance(proptype, hyperdb.Number): + value = value.strip() + props[propname] = float(value) + return errors, props + + +def extractUserFromList(userClass, users): + '''Given a list of users, try to extract the first non-anonymous user + and return that user, otherwise return None + ''' + if len(users) > 1: + for user in users: + # make sure we don't match the anonymous or admin user + if userClass.get(user, 'username') in ('admin', 'anonymous'): + continue + # first valid match will do + return user + # well, I guess we have no choice + return user[0] + elif users: + return users[0] + return None - # commit the new node(s) to the DB - self.db.commit() -def parseContent(content, blank_line=re.compile(r'[\r\n]+\s*[\r\n]+'), - eol=re.compile(r'[\r\n]+'), signature=re.compile(r'^[>|\s]*[-_]+\s*$')): +def uidFromAddress(db, address, create=1, **user_props): + ''' address is from the rfc822 module, and therefore is (name, addr) + + user is created if they don't exist in the db already + user_props may supply additional user information + ''' + (realname, address) = address + + # try a straight match of the address + user = extractUserFromList(db.user, db.user.stringFind(address=address)) + if user is not None: + return user + + # try the user alternate addresses if possible + props = db.user.getprops() + if props.has_key('alternate_addresses'): + users = db.user.filter(None, {'alternate_addresses': address}) + user = extractUserFromList(db.user, users) + if user is not None: + return user + + # try to match the username to the address (for local + # submissions where the address is empty) + user = extractUserFromList(db.user, db.user.stringFind(username=address)) + + # couldn't match address or username, so create a new user + if create: + return db.user.create(username=address, address=address, + realname=realname, roles=db.config.NEW_EMAIL_USER_ROLES, + **user_props) + else: + return 0 + + +def parseContent(content, keep_citations, keep_body, + blank_line=re.compile(r'[\r\n]+\s*[\r\n]+'), + eol=re.compile(r'[\r\n]+'), + signature=re.compile(r'^[>|\s]*[-_]+\s*$'), + original_msg=re.compile(r'^[>|\s]*-----\s?Original Message\s?-----$')): ''' The message body is divided into sections by blank lines. - Sections where the second and all subsequent lines begin with a ">" or "|" - character are considered "quoting sections". The first line of the first - non-quoting section becomes the summary of the message. + Sections where the second and all subsequent lines begin with a ">" + or "|" character are considered "quoting sections". The first line of + the first non-quoting section becomes the summary of the message. + + If keep_citations is true, then we keep the "quoting sections" in the + content. + If keep_body is true, we even keep the signature sections. ''' # strip off leading carriage-returns / newlines i = 0 @@ -693,256 +1055,54 @@ def parseContent(content, blank_line=re.compile(r'[\r\n]+\s*[\r\n]+'), if not section: continue lines = eol.split(section) - if lines[0] and lines[0][0] in '>|': - continue - if len(lines) > 1 and lines[1] and lines[1][0] in '>|': - continue + if (lines[0] and lines[0][0] in '>|') or (len(lines) > 1 and + lines[1] and lines[1][0] in '>|'): + # see if there's a response somewhere inside this section (ie. + # no blank line between quoted message and response) + for line in lines[1:]: + if line and line[0] not in '>|': + break + else: + # we keep quoted bits if specified in the config + if keep_citations: + l.append(section) + continue + # keep this section - it has reponse stuff in it + lines = lines[lines.index(line):] + section = '\n'.join(lines) + # and while we're at it, use the first non-quoted bit as + # our summary + summary = section + if not summary: - summary = lines[0] - l.append(section) - continue - if signature.match(lines[0]): + # if we don't have our summary yet use the first line of this + # section + summary = section + elif signature.match(lines[0]) and 2 <= len(lines) <= 10: + # lose any signature + break + elif original_msg.match(lines[0]): + # ditch the stupid Outlook quoting of the entire original message break + + # and add the section to the output l.append(section) - return summary, '\n\n'.join(l) -# -# $Log: not supported by cvs2svn $ -# Revision 1.47 2002/01/02 02:32:38 richard -# ANONYMOUS_ACCESS -> ANONYMOUS_REGISTER -# -# Revision 1.46 2002/01/02 02:31:38 richard -# Sorry for the huge checkin message - I was only intending to implement #496356 -# but I found a number of places where things had been broken by transactions: -# . modified ROUNDUPDBSENDMAILDEBUG to be SENDMAILDEBUG and hold a filename -# for _all_ roundup-generated smtp messages to be sent to. -# . the transaction cache had broken the roundupdb.Class set() reactors -# . newly-created author users in the mailgw weren't being committed to the db -# -# Stuff that made it into CHANGES.txt (ie. the stuff I was actually working -# on when I found that stuff :): -# . #496356 ] Use threading in messages -# . detectors were being registered multiple times -# . added tests for mailgw -# . much better attaching of erroneous messages in the mail gateway -# -# Revision 1.45 2001/12/20 15:43:01 rochecompaan -# Features added: -# . Multilink properties are now displayed as comma separated values in -# a textbox -# . The add user link is now only visible to the admin user -# . Modified the mail gateway to reject submissions from unknown -# addresses if ANONYMOUS_ACCESS is denied -# -# Revision 1.44 2001/12/18 15:30:34 rochecompaan -# Fixed bugs: -# . Fixed file creation and retrieval in same transaction in anydbm -# backend -# . Cgi interface now renders new issue after issue creation -# . Could not set issue status to resolved through cgi interface -# . Mail gateway was changing status back to 'chatting' if status was -# omitted as an argument -# -# Revision 1.43 2001/12/15 19:39:01 rochecompaan -# Oops. -# -# Revision 1.42 2001/12/15 19:24:39 rochecompaan -# . Modified cgi interface to change properties only once all changes are -# collected, files created and messages generated. -# . Moved generation of change note to nosyreactors. -# . We now check for changes to "assignedto" to ensure it's added to the -# nosy list. -# -# Revision 1.41 2001/12/10 00:57:38 richard -# From CHANGES: -# . Added the "display" command to the admin tool - displays a node's values -# . #489760 ] [issue] only subject -# . fixed the doc/index.html to include the quoting in the mail alias. -# -# Also: -# . fixed roundup-admin so it works with transactions -# . disabled the back_anydbm module if anydbm tries to use dumbdbm -# -# Revision 1.40 2001/12/05 14:26:44 rochecompaan -# Removed generation of change note from "sendmessage" in roundupdb.py. -# The change note is now generated when the message is created. -# -# Revision 1.39 2001/12/02 05:06:16 richard -# . We now use weakrefs in the Classes to keep the database reference, so -# the close() method on the database is no longer needed. -# I bumped the minimum python requirement up to 2.1 accordingly. -# . #487480 ] roundup-server -# . #487476 ] INSTALL.txt -# -# I also cleaned up the change message / post-edit stuff in the cgi client. -# There's now a clearly marked "TODO: append the change note" where I believe -# the change note should be added there. The "changes" list will obviously -# have to be modified to be a dict of the changes, or somesuch. -# -# More testing needed. -# -# Revision 1.38 2001/12/01 07:17:50 richard -# . We now have basic transaction support! Information is only written to -# the database when the commit() method is called. Only the anydbm -# backend is modified in this way - neither of the bsddb backends have been. -# The mail, admin and cgi interfaces all use commit (except the admin tool -# doesn't have a commit command, so interactive users can't commit...) -# . Fixed login/registration forwarding the user to the right page (or not, -# on a failure) -# -# Revision 1.37 2001/11/28 21:55:35 richard -# . login_action and newuser_action return values were being ignored -# . Woohoo! Found that bloody re-login bug that was killing the mail -# gateway. -# (also a minor cleanup in hyperdb) -# -# Revision 1.36 2001/11/26 22:55:56 richard -# Feature: -# . Added INSTANCE_NAME to configuration - used in web and email to identify -# the instance. -# . Added EMAIL_SIGNATURE_POSITION to indicate where to place the roundup -# signature info in e-mails. -# . Some more flexibility in the mail gateway and more error handling. -# . Login now takes you to the page you back to the were denied access to. -# -# Fixed: -# . Lots of bugs, thanks Roché and others on the devel mailing list! -# -# Revision 1.35 2001/11/22 15:46:42 jhermann -# Added module docstrings to all modules. -# -# Revision 1.34 2001/11/15 10:24:27 richard -# handle the case where there is no file attached -# -# Revision 1.33 2001/11/13 21:44:44 richard -# . re-open the database as the author in mail handling -# -# Revision 1.32 2001/11/12 22:04:29 richard -# oops, left debug in there -# -# Revision 1.31 2001/11/12 22:01:06 richard -# Fixed issues with nosy reaction and author copies. -# -# Revision 1.30 2001/11/09 22:33:28 richard -# More error handling fixes. -# -# Revision 1.29 2001/11/07 05:29:26 richard -# Modified roundup-mailgw so it can read e-mails from a local mail spool -# file. Truncates the spool file after parsing. -# Fixed a couple of small bugs introduced in roundup.mailgw when I started -# the popgw. -# -# Revision 1.28 2001/11/01 22:04:37 richard -# Started work on supporting a pop3-fetching server -# Fixed bugs: -# . bug #477104 ] HTML tag error in roundup-server -# . bug #477107 ] HTTP header problem -# -# Revision 1.27 2001/10/30 11:26:10 richard -# Case-insensitive match for ISSUE_TRACKER_EMAIL in address in e-mail. -# -# Revision 1.26 2001/10/30 00:54:45 richard -# Features: -# . #467129 ] Lossage when username=e-mail-address -# . #473123 ] Change message generation for author -# . MailGW now moves 'resolved' to 'chatting' on receiving e-mail for an issue. -# -# Revision 1.25 2001/10/28 23:22:28 richard -# fixed bug #474749 ] Indentations lost -# -# Revision 1.24 2001/10/23 22:57:52 richard -# Fix unread->chatting auto transition, thanks Roch'e -# -# Revision 1.23 2001/10/21 04:00:20 richard -# MailGW now moves 'unread' to 'chatting' on receiving e-mail for an issue. -# -# Revision 1.22 2001/10/21 03:35:13 richard -# bug #473125: Paragraph in e-mails -# -# Revision 1.21 2001/10/21 00:53:42 richard -# bug #473130: Nosy list not set correctly -# -# Revision 1.20 2001/10/17 23:13:19 richard -# Did a fair bit of work on the admin tool. Now has an extra command "table" -# which displays node information in a tabular format. Also fixed import and -# export so they work. Removed freshen. -# Fixed quopri usage in mailgw from bug reports. -# -# Revision 1.19 2001/10/11 23:43:04 richard -# Implemented the comma-separated printing option in the admin tool. -# Fixed a typo (more of a vim-o actually :) in mailgw. -# -# Revision 1.18 2001/10/11 06:38:57 richard -# Initial cut at trying to handle people responding to CC'ed messages that -# create an issue. -# -# Revision 1.17 2001/10/09 07:25:59 richard -# Added the Password property type. See "pydoc roundup.password" for -# implementation details. Have updated some of the documentation too. -# -# Revision 1.16 2001/10/05 02:23:24 richard -# . roundup-admin create now prompts for property info if none is supplied -# on the command-line. -# . hyperdb Class getprops() method may now return only the mutable -# properties. -# . Login now uses cookies, which makes it a whole lot more flexible. We can -# now support anonymous user access (read-only, unless there's an -# "anonymous" user, in which case write access is permitted). Login -# handling has been moved into cgi_client.Client.main() -# . The "extended" schema is now the default in roundup init. -# . The schemas have had their page headings modified to cope with the new -# login handling. Existing installations should copy the interfaces.py -# file from the roundup lib directory to their instance home. -# . Incorrectly had a Bizar Software copyright on the cgitb.py module from -# Ping - has been removed. -# . Fixed a whole bunch of places in the CGI interface where we should have -# been returning Not Found instead of throwing an exception. -# . Fixed a deviation from the spec: trying to modify the 'id' property of -# an item now throws an exception. -# -# Revision 1.15 2001/08/30 06:01:17 richard -# Fixed missing import in mailgw :( -# -# Revision 1.14 2001/08/13 23:02:54 richard -# Make the mail parser a little more robust. -# -# Revision 1.13 2001/08/12 06:32:36 richard -# using isinstance(blah, Foo) now instead of isFooType -# -# Revision 1.12 2001/08/08 01:27:00 richard -# Added better error handling to mailgw. -# -# Revision 1.11 2001/08/08 00:08:03 richard -# oops ;) -# -# Revision 1.10 2001/08/07 00:24:42 richard -# stupid typo -# -# Revision 1.9 2001/08/07 00:15:51 richard -# Added the copyright/license notice to (nearly) all files at request of -# Bizar Software. -# -# Revision 1.8 2001/08/05 07:06:07 richard -# removed some print statements -# -# Revision 1.7 2001/08/03 07:18:22 richard -# Implemented correct mail splitting (was taking a shortcut). Added unit -# tests. Also snips signatures now too. -# -# Revision 1.6 2001/08/01 04:24:21 richard -# mailgw was assuming certain properties existed on the issues being created. -# -# Revision 1.5 2001/07/29 07:01:39 richard -# Added vim command to all source so that we don't get no steenkin' tabs :) -# -# Revision 1.4 2001/07/28 06:43:02 richard -# Multipart message class has the getPart method now. Added some tests for it. -# -# Revision 1.3 2001/07/28 00:34:34 richard -# Fixed some non-string node ids. -# -# Revision 1.2 2001/07/22 12:09:32 richard -# Final commit of Grande Splite -# -# + # figure the summary - find the first sentence-ending punctuation or the + # first whole line, whichever is longest + sentence = re.search(r'^([^!?\.]+[!?\.])', summary) + if sentence: + sentence = sentence.group(1) + else: + sentence = '' + first = eol.split(summary)[0] + summary = max(sentence, first) + + # Now reconstitute the message content minus the bits we don't care + # about. + if not keep_body: + content = '\n\n'.join(l) + + return summary, content + # vim: set filetype=python ts=4 sw=4 et si