X-Git-Url: https://git.tokkee.org/?a=blobdiff_plain;ds=sidebyside;f=roundup%2Fmailgw.py;h=53dc0417fbd433ffececfa1765b58f530c28ab07;hb=4d313534b6cb918767a12d4a5e0189004c877e04;hp=76e2638381109fa105b2528b7064566aaa5eb844;hpb=029a458ab427817cdb7d4e36b71768d48fc01f3e;p=roundup.git diff --git a/roundup/mailgw.py b/roundup/mailgw.py index 76e2638..53dc041 100644 --- a/roundup/mailgw.py +++ b/roundup/mailgw.py @@ -14,7 +14,8 @@ # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. -# +# + ''' An e-mail gateway for Roundup. @@ -72,17 +73,66 @@ are calling the create() method to create a new node). If an auditor raises an exception, the original message is bounced back to the sender with the explanatory message given in the exception. -$Id: mailgw.py,v 1.25 2001-10-28 23:22:28 richard Exp $ +$Id: mailgw.py,v 1.111 2003-02-27 05:43:01 richard Exp $ ''' - import string, re, os, mimetools, cStringIO, smtplib, socket, binascii, quopri -import traceback +import time, random, sys +import traceback, MimeWriter, rfc822 import hyperdb, date, password +import rfc2822 + +SENDMAILDEBUG = os.environ.get('SENDMAILDEBUG', '') + +class MailGWError(ValueError): + pass + class MailUsageError(ValueError): pass +class MailUsageHelp(Exception): + pass + +class MailLoop(Exception): + ''' We've seen this message before... ''' + pass + +class Unauthorized(Exception): + """ Access denied """ + +def initialiseSecurity(security): + ''' Create some Permissions and Roles on the security object + + This function is directly invoked by security.Security.__init__() + as a part of the Security object instantiation. + ''' + security.addPermission(name="Email Registration", + description="Anonymous may register through e-mail") + p = security.addPermission(name="Email Access", + description="User may use the email interface") + security.addPermissionToRole('Admin', p) + +def getparam(str, param): + ''' From the rfc822 "header" string, extract "param" if it appears. + ''' + if ';' not in str: + return None + str = str[str.index(';'):] + while str[:1] == ';': + str = str[1:] + if ';' in str: + # XXX Should parse quotes! + end = str.index(';') + else: + end = len(str) + f = str[:end] + if '=' in f: + i = f.index('=') + if f[:i].strip().lower() == param: + return rfc822.unquote(f[i+1:].strip()) + return None + class Message(mimetools.Message): ''' subclass mimetools.Message so we can retrieve the parts of the message... @@ -106,69 +156,314 @@ class Message(mimetools.Message): s.seek(0) return Message(s) -subject_re = re.compile(r'(?P\s*\W?\s*(fwd|re)\s*\W?\s*)*' - r'\s*(\[(?P[^\d]+)(?P\d+)?\])' - r'\s*(?P[^\[]+)(\[(?P<args>.+?)\])?', re.I) + def getheader(self, name, default=None): + hdr = mimetools.Message.getheader(self, name, default) + return rfc2822.decode_header(hdr) + +subject_re = re.compile(r'(?P<refwd>\s*\W?\s*(fw|fwd|re|aw)\W\s*)*' + r'\s*(?P<quote>")?(\[(?P<classname>[^\d\s]+)(?P<nodeid>\d+)?\])?' + r'\s*(?P<title>[^[]+)?"?(\[(?P<args>.+?)\])?', re.I) class MailGW: - def __init__(self, db): + def __init__(self, instance, db, arguments={}): + self.instance = instance self.db = db + self.arguments = arguments + + # should we trap exceptions (normal usage) or pass them through + # (for testing) + self.trapExceptions = 1 + + def do_pipe(self): + ''' Read a message from standard input and pass it to the mail handler. + + Read into an internal structure that we can seek on (in case + there's an error). + + XXX: we may want to read this into a temporary file instead... + ''' + s = cStringIO.StringIO() + s.write(sys.stdin.read()) + s.seek(0) + self.main(s) + return 0 + + def do_mailbox(self, filename): + ''' Read a series of messages from the specified unix mailbox file and + pass each to the mail handler. + ''' + # open the spool file and lock it + import fcntl, FCNTL + f = open(filename, 'r+') + fcntl.flock(f.fileno(), FCNTL.LOCK_EX) + + # handle and clear the mailbox + try: + from mailbox import UnixMailbox + mailbox = UnixMailbox(f, factory=Message) + # grab one message + message = mailbox.next() + while message: + # handle this message + self.handle_Message(message) + message = mailbox.next() + # nuke the file contents + os.ftruncate(f.fileno(), 0) + except: + import traceback + traceback.print_exc() + return 1 + fcntl.flock(f.fileno(), FCNTL.LOCK_UN) + return 0 + + def do_pop(self, server, user='', password=''): + '''Read a series of messages from the specified POP server. + ''' + import getpass, poplib, socket + try: + if not user: + user = raw_input(_('User: ')) + if not password: + password = getpass.getpass() + except (KeyboardInterrupt, EOFError): + # Ctrl C or D maybe also Ctrl Z under Windows. + print "\nAborted by user." + return 1 + + # open a connection to the server and retrieve all messages + try: + server = poplib.POP3(server) + except socket.error, message: + print "POP server error:", message + return 1 + server.user(user) + server.pass_(password) + numMessages = len(server.list()[1]) + for i in range(1, numMessages+1): + # retr: returns + # [ pop response e.g. '+OK 459 octets', + # [ array of message lines ], + # number of octets ] + lines = server.retr(i)[1] + s = cStringIO.StringIO('\n'.join(lines)) + s.seek(0) + self.handle_Message(Message(s)) + # delete the message + server.dele(i) + + # quit the server to commit changes. + server.quit() + return 0 def main(self, fp): ''' fp - the file from which to read the Message. + ''' + return self.handle_Message(Message(fp)) - Read a message from fp and then call handle_message() with the - result. This method's job is to make that call and handle any + def handle_Message(self, message): + '''Handle an RFC822 Message + + Handle the Message object by calling handle_message() and then cope + with any errors raised by handle_message. + This method's job is to make that call and handle any errors in a sane manner. It should be replaced if you wish to handle errors in a different manner. ''' - # ok, figure the subject, author, recipients and content-type - message = Message(fp) - m = [] - try: - self.handle_message(message) - except MailUsageError, value: - # bounce the message back to the sender with the usage message - fulldoc = '\n'.join(string.split(__doc__, '\n')[2:]) - sendto = [message.getaddrlist('from')[0][1]] - m = ['Subject: Failed issue tracker submission', ''] - m.append(str(value)) - m.append('\n\nMail Gateway Help\n=================') - m.append(fulldoc) - except: - # bounce the message back to the sender with the error message - sendto = [message.getaddrlist('from')[0][1]] - m = ['Subject: failed issue tracker submission'] - m.append('') - # TODO as attachments? - m.append('---- traceback of failure ----') - s = cStringIO.StringIO() - import traceback - traceback.print_exc(None, s) - m.append(s.getvalue()) - m.append('---- failed message follows ----') + # in some rare cases, a particularly stuffed-up e-mail will make + # its way into here... try to handle it gracefully + sendto = message.getaddrlist('from') + if sendto: + if not self.trapExceptions: + return self.handle_message(message) try: - fp.seek(0) + return self.handle_message(message) + except MailUsageHelp: + # bounce the message back to the sender with the usage message + fulldoc = '\n'.join(string.split(__doc__, '\n')[2:]) + sendto = [sendto[0][1]] + m = [''] + m.append('\n\nMail Gateway Help\n=================') + m.append(fulldoc) + m = self.bounce_message(message, sendto, m, + subject="Mail Gateway Help") + except MailUsageError, value: + # bounce the message back to the sender with the usage message + fulldoc = '\n'.join(string.split(__doc__, '\n')[2:]) + sendto = [sendto[0][1]] + m = [''] + m.append(str(value)) + m.append('\n\nMail Gateway Help\n=================') + m.append(fulldoc) + m = self.bounce_message(message, sendto, m) + except Unauthorized, value: + # just inform the user that he is not authorized + sendto = [sendto[0][1]] + m = [''] + m.append(str(value)) + m = self.bounce_message(message, sendto, m) + except MailLoop: + # XXX we should use a log file here... + return except: - pass - m.append(fp.read()) - if m: + # bounce the message back to the sender with the error message + # XXX we should use a log file here... + sendto = [sendto[0][1], self.instance.config.ADMIN_EMAIL] + m = [''] + m.append('An unexpected error occurred during the processing') + m.append('of your message. The tracker administrator is being') + m.append('notified.\n') + m.append('---- traceback of failure ----') + s = cStringIO.StringIO() + import traceback + traceback.print_exc(None, s) + m.append(s.getvalue()) + m = self.bounce_message(message, sendto, m) + else: + # very bad-looking message - we don't even know who sent it + # XXX we should use a log file here... + sendto = [self.instance.config.ADMIN_EMAIL] + m = ['Subject: badly formed message from mail gateway'] + m.append('') + m.append('The mail gateway retrieved a message which has no From:') + m.append('line, indicating that it is corrupt. Please check your') + m.append('mail gateway source. Failed message is attached.') + m.append('') + m = self.bounce_message(message, sendto, m, + subject='Badly formed message from mail gateway') + + # now send the message + if SENDMAILDEBUG: + open(SENDMAILDEBUG, 'a').write('From: %s\nTo: %s\n%s\n'%( + self.instance.config.ADMIN_EMAIL, ', '.join(sendto), + m.getvalue())) + else: try: - smtp = smtplib.SMTP(self.MAILHOST) - smtp.sendmail(self.ADMIN_EMAIL, sendto, '\n'.join(m)) + smtp = smtplib.SMTP(self.instance.config.MAILHOST) + smtp.sendmail(self.instance.config.ADMIN_EMAIL, sendto, + m.getvalue()) except socket.error, value: - return "Couldn't send confirmation email: mailhost %s"%value + raise MailGWError, "Couldn't send error email: "\ + "mailhost %s"%value except smtplib.SMTPException, value: - return "Couldn't send confirmation email: %s"%value + raise MailGWError, "Couldn't send error email: %s"%value + + def bounce_message(self, message, sendto, error, + subject='Failed issue tracker submission'): + ''' create a message that explains the reason for the failed + issue submission to the author and attach the original + message. + ''' + msg = cStringIO.StringIO() + writer = MimeWriter.MimeWriter(msg) + writer.addheader('X-Roundup-Loop', 'hello') + writer.addheader('Subject', subject) + writer.addheader('From', '%s <%s>'% (self.instance.config.TRACKER_NAME, + self.instance.config.TRACKER_EMAIL)) + writer.addheader('To', ','.join(sendto)) + writer.addheader('Date', time.strftime("%a, %d %b %Y %H:%M:%S +0000", + time.gmtime())) + writer.addheader('MIME-Version', '1.0') + part = writer.startmultipartbody('mixed') + part = writer.nextpart() + body = part.startbody('text/plain; charset=utf-8') + body.write('\n'.join(error)) + + # attach the original message to the returned message + part = writer.nextpart() + part.addheader('Content-Disposition','attachment') + part.addheader('Content-Description','Message you sent') + body = part.startbody('text/plain') + for header in message.headers: + body.write(header) + body.write('\n') + try: + message.rewindbody() + except IOError, message: + body.write("*** couldn't include message body: %s ***"%message) + else: + body.write(message.fp.read()) + + writer.lastpart() + return msg + + def get_part_data_decoded(self,part): + encoding = part.getencoding() + data = None + if encoding == 'base64': + # BUG: is base64 really used for text encoding or + # are we inserting zip files here. + data = binascii.a2b_base64(part.fp.read()) + elif encoding == 'quoted-printable': + # the quopri module wants to work with files + decoded = cStringIO.StringIO() + quopri.decode(part.fp, decoded) + data = decoded.getvalue() + elif encoding == 'uuencoded': + data = binascii.a2b_uu(part.fp.read()) + else: + # take it as text + data = part.fp.read() + + # Encode message to unicode + charset = rfc2822.unaliasCharset(part.getparam("charset")) + if charset: + # Do conversion only if charset specified + edata = unicode(data, charset).encode('utf-8') + # Convert from dos eol to unix + edata = edata.replace('\r\n', '\n') + else: + # Leave message content as is + edata = data + + return edata def handle_message(self, message): ''' message - a Message instance Parse the message as per the module docstring. ''' + # detect loops + if message.getheader('x-roundup-loop', ''): + raise MailLoop + + # XXX Don't enable. This doesn't work yet. +# "[^A-z.]tracker\+(?P<classname>[^\d\s]+)(?P<nodeid>\d+)\@some.dom.ain[^A-z.]" + # handle delivery to addresses like:tracker+issue25@some.dom.ain + # use the embedded issue number as our issue +# if hasattr(self.instance.config, 'EMAIL_ISSUE_ADDRESS_RE') and \ +# self.instance.config.EMAIL_ISSUE_ADDRESS_RE: +# issue_re = self.instance.config.EMAIL_ISSUE_ADDRESS_RE +# for header in ['to', 'cc', 'bcc']: +# addresses = message.getheader(header, '') +# if addresses: +# # FIXME, this only finds the first match in the addresses. +# issue = re.search(issue_re, addresses, 'i') +# if issue: +# classname = issue.group('classname') +# nodeid = issue.group('nodeid') +# break + # handle the subject line subject = message.getheader('subject', '') + + if subject.strip().lower() == 'help': + raise MailUsageHelp + m = subject_re.match(subject) + + # check for well-formed subject line + if m: + # get the classname + classname = m.group('classname') + if classname is None: + # no classname, fallback on the default + if hasattr(self.instance.config, 'MAIL_DEFAULT_CLASS') and \ + self.instance.config.MAIL_DEFAULT_CLASS: + classname = self.instance.config.MAIL_DEFAULT_CLASS + else: + # fail + m = None + if not m: raise MailUsageError, ''' The message you sent to roundup did not contain a properly formed subject @@ -183,10 +478,8 @@ line. The subject must contain a class name or designator to indicate the Subject was: "%s" '''%subject - classname = m.group('classname') - nodeid = m.group('nodeid') - title = m.group('title').strip() - subject_args = m.group('args') + + # get the class try: cl = self.db.getclass(classname) except KeyError: @@ -198,67 +491,223 @@ Valid class names are: %s Subject was: "%s" '''%(classname, ', '.join(self.db.getclasses()), subject) + # get the optional nodeid + nodeid = m.group('nodeid') + + # title is optional too + title = m.group('title') + if title: + title = title.strip() + else: + title = '' + + # strip off the quotes that dumb emailers put around the subject, like + # Re: "[issue1] bla blah" + if m.group('quote') and title.endswith('"'): + title = title[:-1] + + # but we do need either a title or a nodeid... + if nodeid is None and not title: + raise MailUsageError, ''' +I cannot match your message to a node in the database - you need to either +supply a full node identifier (with number, eg "[issue123]" or keep the +previous subject title intact so I can match that. + +Subject was: "%s" +'''%subject + # If there's no nodeid, check to see if this is a followup and # maybe someone's responded to the initial mail that created an # entry. Try to find the matching nodes with the same title, and # use the _last_ one matched (since that'll _usually_ be the most # recent...) - if not nodeid and m.group('refwd'): + if nodeid is None and m.group('refwd'): l = cl.stringFind(title=title) if l: nodeid = l[-1] - # start of the props - properties = cl.getprops() - props = {} - - # handle the args - args = m.group('args') - if args: - for prop in string.split(args, ';'): - try: - key, value = prop.split('=') - except ValueError, message: - raise MailUsageError, ''' -Subject argument list not of form [arg=value,value,...;arg=value,value...] - (specific exception message was "%s") + # if a nodeid was specified, make sure it's valid + if nodeid is not None and not cl.hasnode(nodeid): + raise MailUsageError, ''' +The node specified by the designator in the subject of your message ("%s") +does not exist. Subject was: "%s" -'''%(message, subject) - try: - type = properties[key] - except KeyError: - raise MailUsageError, ''' -Subject argument list refers to an invalid property: "%s" +'''%(nodeid, subject) -Subject was: "%s" -'''%(key, subject) - if isinstance(type, hyperdb.String): - props[key] = value - if isinstance(type, hyperdb.Password): - props[key] = password.Password(value) - elif isinstance(type, hyperdb.Date): - props[key] = date.Date(value) - elif isinstance(type, hyperdb.Interval): - props[key] = date.Interval(value) - elif isinstance(type, hyperdb.Link): - props[key] = value - elif isinstance(type, hyperdb.Multilink): - props[key] = value.split(',') + + # Handle the arguments specified by the email gateway command line. + # We do this by looping over the list of self.arguments looking for + # a -C to tell us what class then the -S setting string. + msg_props = {} + user_props = {} + file_props = {} + issue_props = {} + # so, if we have any arguments, use them + if self.arguments: + current_class = 'msg' + for option, propstring in self.arguments: + if option in ( '-C', '--class'): + current_class = propstring.strip() + if current_class not in ('msg', 'file', 'user', 'issue'): + raise MailUsageError, ''' +The mail gateway is not properly set up. Please contact +%s and have them fix the incorrect class specified as: + %s +'''%(self.instance.config.ADMIN_EMAIL, current_class) + if option in ('-S', '--set'): + if current_class == 'issue' : + errors, issue_props = setPropArrayFromString(self, + cl, propstring.strip(), nodeid) + elif current_class == 'file' : + temp_cl = self.db.getclass('file') + errors, file_props = setPropArrayFromString(self, + temp_cl, propstring.strip()) + elif current_class == 'msg' : + temp_cl = self.db.getclass('msg') + errors, msg_props = setPropArrayFromString(self, + temp_cl, propstring.strip()) + elif current_class == 'user' : + temp_cl = self.db.getclass('user') + errors, user_props = setPropArrayFromString(self, + temp_cl, propstring.strip()) + if errors: + raise MailUsageError, ''' +The mail gateway is not properly set up. Please contact +%s and have them fix the incorrect properties: + %s +'''%(self.instance.config.ADMIN_EMAIL, errors) # # handle the users # - author = self.db.uidFromAddress(message.getaddrlist('from')[0]) + # Don't create users if anonymous isn't allowed to register + create = 1 + anonid = self.db.user.lookup('anonymous') + if not self.db.security.hasPermission('Email Registration', anonid): + create = 0 + + # ok, now figure out who the author is - create a new user if the + # "create" flag is true + author = uidFromAddress(self.db, message.getaddrlist('from')[0], + create=create) + + # if we're not recognised, and we don't get added as a user, then we + # must be anonymous + if not author: + author = anonid + + # make sure the author has permission to use the email interface + if not self.db.security.hasPermission('Email Access', author): + if author == anonid: + # we're anonymous and we need to be a registered user + raise Unauthorized, ''' +You are not a registered user. + +Unknown address: %s +'''%message.getaddrlist('from')[0][1] + else: + # we're registered and we're _still_ not allowed access + raise Unauthorized, 'You are not permitted to access '\ + 'this tracker.' + + # make sure they're allowed to edit this class of information + if not self.db.security.hasPermission('Edit', author, classname): + raise Unauthorized, 'You are not permitted to edit %s.'%classname + + # the author may have been created - make sure the change is + # committed before we reopen the database + self.db.commit() + + # reopen the database as the author + username = self.db.user.get(author, 'username') + self.db.close() + self.db = self.instance.open(username) + + # re-get the class with the new database connection + cl = self.db.getclass(classname) + + # now update the recipients list recipients = [] + tracker_email = self.instance.config.TRACKER_EMAIL.lower() for recipient in message.getaddrlist('to') + message.getaddrlist('cc'): - if recipient[1].strip().lower() == self.ISSUE_TRACKER_EMAIL: + r = recipient[1].strip().lower() + if r == tracker_email or not r: continue - recipients.append(self.db.uidFromAddress(recipient)) + # look up the recipient - create if necessary (and we're + # allowed to) + recipient = uidFromAddress(self.db, recipient, create, **user_props) + + # if all's well, add the recipient to the list + if recipient: + recipients.append(recipient) + + # + # XXX extract the args NOT USED WHY -- rouilj + # + subject_args = m.group('args') + + # + # handle the subject argument list + # + # figure what the properties of this Class are + properties = cl.getprops() + props = {} + args = m.group('args') + if args: + errors, props = setPropArrayFromString(self, cl, args, nodeid) + # handle any errors parsing the argument list + if errors: + errors = '\n- '.join(errors) + raise MailUsageError, ''' +There were problems handling your subject line argument list: +- %s + +Subject was: "%s" +'''%(errors, subject) + + # + # handle message-id and in-reply-to + # + messageid = message.getheader('message-id') + inreplyto = message.getheader('in-reply-to') or '' + # generate a messageid if there isn't one + if not messageid: + messageid = "<%s.%s.%s%s@%s>"%(time.time(), random.random(), + classname, nodeid, self.instance.config.MAIL_DOMAIN) + + # # now handle the body - find the message + # content_type = message.gettype() attachments = [] + # General multipart handling: + # Take the first text/plain part, anything else is considered an + # attachment. + # multipart/mixed: multiple "unrelated" parts. + # multipart/signed (rfc 1847): + # The control information is carried in the second of the two + # required body parts. + # ACTION: Default, so if content is text/plain we get it. + # multipart/encrypted (rfc 1847): + # The control information is carried in the first of the two + # required body parts. + # ACTION: Not handleable as the content is encrypted. + # multipart/related (rfc 1872, 2112, 2387): + # The Multipart/Related content-type addresses the MIME + # representation of compound objects. + # ACTION: Default. If we are lucky there is a text/plain. + # TODO: One should use the start part and look for an Alternative + # that is text/plain. + # multipart/Alternative (rfc 1872, 1892): + # only in "related" ? + # multipart/report (rfc 1892): + # e.g. mail system delivery status reports. + # ACTION: Default. Could be ignored or used for Delivery Notification + # flagging. + # multipart/form-data: + # For web forms only. if content_type == 'multipart/mixed': # skip over the intro to the first boundary part = message.getPart() @@ -271,12 +720,8 @@ Subject was: "%s" # parse it subtype = part.gettype() if subtype == 'text/plain' and not content: - # add all text/plain parts to the message content - if content is None: - content = part.fp.read() - else: - content = content + part.fp.read() - + # The first text/plain part is the message content. + content = self.get_part_data_decoded(part) elif subtype == 'message/rfc822': # handle message/rfc822 specially - the name should be # the subject of the actual e-mail embedded here @@ -285,23 +730,33 @@ Subject was: "%s" name = mailmess.getheader('subject') part.fp.seek(i) attachments.append((name, 'message/rfc822', part.fp.read())) - + elif subtype == 'multipart/alternative': + # Search for text/plain in message with attachment and + # alternative text representation + # skip over intro to first boundary + part.getPart() + while 1: + # get the next part + subpart = part.getPart() + if subpart is None: + break + # parse it + if subpart.gettype() == 'text/plain' and not content: + content = self.get_part_data_decoded(subpart) else: # try name on Content-Type name = part.getparam('name') + if name: + name = name.strip() + if not name: + disp = part.getheader('content-disposition', None) + if disp: + name = getparam(disp, 'filename') + if name: + name = name.strip() # this is just an attachment - encoding = part.getencoding() - if encoding == 'base64': - data = binascii.a2b_base64(part.fp.read()) - elif encoding == 'quoted-printable': - # the quopri module wants to work with files - decoded = cStringIO.StringIO() - quopri.decode(part.fp, decoded) - data = decoded.getvalue() - elif encoding == 'uuencoded': - data = binascii.a2b_uu(part.fp.read()) + data = self.get_part_data_decoded(part) attachments.append((name, part.gettype(), data)) - if content is None: raise MailUsageError, ''' Roundup requires the submission to be plain text. The message parser could @@ -319,8 +774,7 @@ not find a text/plain part to use. break # parse it if part.gettype() == 'text/plain' and not content: - # this one's our content - content = part.fp.read() + content = self.get_part_data_decoded(part) if content is None: raise MailUsageError, ''' Roundup requires the submission to be plain text. The message parser could @@ -334,79 +788,254 @@ not find a text/plain part to use. ''' else: - content = message.fp.read() + content = self.get_part_data_decoded(message) + + # figure how much we should muck around with the email body + keep_citations = getattr(self.instance.config, 'EMAIL_KEEP_QUOTED_TEXT', + 'no') == 'yes' + keep_body = getattr(self.instance.config, 'EMAIL_LEAVE_BODY_UNCHANGED', + 'no') == 'yes' - summary, content = parseContent(content) + # parse the body of the message, stripping out bits as appropriate + summary, content = parseContent(content, keep_citations, + keep_body) - # handle the files + # + # handle the attachments + # files = [] - for (name, type, data) in attachments: - files.append(self.db.file.create(type=type, name=name, - content=data)) - - # now handle the db stuff - if nodeid: - # If an item designator (class name and id number) is found there, - # the newly created "msg" node is added to the "messages" property - # for that item, and any new "file" nodes are added to the "files" - # property for the item. + for (name, mime_type, data) in attachments: + if not name: + name = "unnamed" + files.append(self.db.file.create(type=mime_type, name=name, + content=data, **file_props)) + + # + # create the message if there's a message body (content) + # + if content: message_id = self.db.msg.create(author=author, recipients=recipients, date=date.Date('.'), summary=summary, - content=content, files=files) - try: + content=content, files=files, messageid=messageid, + inreplyto=inreplyto, **msg_props) + + # attach the message to the node + if nodeid: + # add the message to the node's list messages = cl.get(nodeid, 'messages') - except IndexError: - raise MailUsageError, ''' -The node specified by the designator in the subject of your message ("%s") -does not exist. + messages.append(message_id) + props['messages'] = messages + else: + # pre-load the messages list + props['messages'] = [message_id] -Subject was: "%s" -'''%(nodeid, subject) - messages.append(message_id) - props['messages'] = messages + # set the title to the subject + if properties.has_key('title') and not props.has_key('title'): + props['title'] = title + + # + # perform the node change / create + # + try: + # merge the command line props defined in issue_props into + # the props dictionary because function(**props, **issue_props) + # is a syntax error. + for prop in issue_props.keys() : + if not props.has_key(prop) : + props[prop] = issue_props[prop] + if nodeid: + cl.set(nodeid, **props) + else: + nodeid = cl.create(**props) + except (TypeError, IndexError, ValueError), message: + raise MailUsageError, ''' +There was a problem with the message you sent: + %s +'''%message + + # commit the changes to the DB + self.db.commit() + + return nodeid + + +def setPropArrayFromString(self, cl, propString, nodeid = None): + ''' takes string of form prop=value,value;prop2=value + and returns (error, prop[..]) + ''' + properties = cl.getprops() + props = {} + errors = [] + for prop in string.split(propString, ';'): + # extract the property name and value + try: + propname, value = prop.split('=') + except ValueError, message: + errors.append('not of form [arg=value,value,...;' + 'arg=value,value,...]') + return (errors, props) + + # ensure it's a valid property name + propname = propname.strip() + try: + proptype = properties[propname] + except KeyError: + errors.append('refers to an invalid property: "%s"'%propname) + continue + + # convert the string value to a real property value + if isinstance(proptype, hyperdb.String): + props[propname] = value.strip() + if isinstance(proptype, hyperdb.Password): + props[propname] = password.Password(value.strip()) + elif isinstance(proptype, hyperdb.Date): + try: + props[propname] = date.Date(value.strip()).local(self.db.getUserTimezone()) + except ValueError, message: + errors.append('contains an invalid date for %s.'%propname) + elif isinstance(proptype, hyperdb.Interval): + try: + props[propname] = date.Interval(value) + except ValueError, message: + errors.append('contains an invalid date interval for %s.'% + propname) + elif isinstance(proptype, hyperdb.Link): + linkcl = self.db.classes[proptype.classname] + propkey = linkcl.labelprop(default_to_id=1) + try: + props[propname] = linkcl.lookup(value) + except KeyError, message: + errors.append('"%s" is not a value for %s.'%(value, propname)) + elif isinstance(proptype, hyperdb.Multilink): + # get the linked class + linkcl = self.db.classes[proptype.classname] + propkey = linkcl.labelprop(default_to_id=1) + if nodeid: + curvalue = cl.get(nodeid, propname) + else: + curvalue = [] + + # handle each add/remove in turn + # keep an extra list for all items that are + # definitely in the new list (in case of e.g. + # <propname>=A,+B, which should replace the old + # list with A,B) + set = 0 + newvalue = [] + for item in value.split(','): + item = item.strip() - # if the message is currently 'unread', then set it to 'chatting' - if properties.has_key('status'): + # handle +/- + remove = 0 + if item.startswith('-'): + remove = 1 + item = item[1:] + elif item.startswith('+'): + item = item[1:] + else: + set = 1 + + # look up the value try: - # determine the id of 'unread' and 'chatting' - unread_id = self.db.status.lookup('unread') - chatting_id = self.db.status.lookup('chatting') - except KeyError: - pass + item = linkcl.lookup(item) + except KeyError, message: + errors.append('"%s" is not a value for %s.'%(item, + propname)) + continue + + # perform the add/remove + if remove: + try: + curvalue.remove(item) + except ValueError: + errors.append('"%s" is not currently in for %s.'%(item, + propname)) + continue else: - if (not props.has_key('status') or - props['status'] == unread_id): - props['status'] = chatting_id + newvalue.append(item) + if item not in curvalue: + curvalue.append(item) - cl.set(nodeid, **props) - else: - # If just an item class name is found there, we attempt to create a - # new item of that class with its "messages" property initialized to - # contain the new "msg" node and its "files" property initialized to - # contain any new "file" nodes. - message_id = self.db.msg.create(author=author, - recipients=recipients, date=date.Date('.'), summary=summary, - content=content, files=files) - # fill out the properties with defaults where required - if properties.has_key('assignedto') and \ - not props.has_key('assignedto'): - props['assignedto'] = '1' # "admin" - if properties.has_key('status') and not props.has_key('status'): - props['status'] = '1' # "unread" - if properties.has_key('title') and not props.has_key('title'): - props['title'] = title - props['messages'] = [message_id] - props['nosy'] = props.get('nosy', []) + recipients - props['nosy'].append(author) - props['nosy'].sort() - nodeid = cl.create(**props) - -def parseContent(content, blank_line=re.compile(r'[\r\n]+\s*[\r\n]+'), - eol=re.compile(r'[\r\n]+'), signature=re.compile(r'^[>|\s]*[-_]+\s*$')): + # that's it, set the new Multilink property value, + # or overwrite it completely + if set: + props[propname] = newvalue + else: + props[propname] = curvalue + elif isinstance(proptype, hyperdb.Boolean): + value = value.strip() + props[propname] = value.lower() in ('yes', 'true', 'on', '1') + elif isinstance(proptype, hyperdb.Number): + value = value.strip() + props[propname] = float(value) + return errors, props + + +def extractUserFromList(userClass, users): + '''Given a list of users, try to extract the first non-anonymous user + and return that user, otherwise return None + ''' + if len(users) > 1: + for user in users: + # make sure we don't match the anonymous or admin user + if userClass.get(user, 'username') in ('admin', 'anonymous'): + continue + # first valid match will do + return user + # well, I guess we have no choice + return user[0] + elif users: + return users[0] + return None + + +def uidFromAddress(db, address, create=1, **user_props): + ''' address is from the rfc822 module, and therefore is (name, addr) + + user is created if they don't exist in the db already + user_props may supply additional user information + ''' + (realname, address) = address + + # try a straight match of the address + user = extractUserFromList(db.user, db.user.stringFind(address=address)) + if user is not None: + return user + + # try the user alternate addresses if possible + props = db.user.getprops() + if props.has_key('alternate_addresses'): + users = db.user.filter(None, {'alternate_addresses': address}) + user = extractUserFromList(db.user, users) + if user is not None: + return user + + # try to match the username to the address (for local + # submissions where the address is empty) + user = extractUserFromList(db.user, db.user.stringFind(username=address)) + + # couldn't match address or username, so create a new user + if create: + return db.user.create(username=address, address=address, + realname=realname, roles=db.config.NEW_EMAIL_USER_ROLES, + **user_props) + else: + return 0 + + +def parseContent(content, keep_citations, keep_body, + blank_line=re.compile(r'[\r\n]+\s*[\r\n]+'), + eol=re.compile(r'[\r\n]+'), + signature=re.compile(r'^[>|\s]*[-_]+\s*$'), + original_msg=re.compile(r'^[>|\s]*-----\s?Original Message\s?-----$')): ''' The message body is divided into sections by blank lines. - Sections where the second and all subsequent lines begin with a ">" or "|" - character are considered "quoting sections". The first line of the first - non-quoting section becomes the summary of the message. + Sections where the second and all subsequent lines begin with a ">" + or "|" character are considered "quoting sections". The first line of + the first non-quoting section becomes the summary of the message. + + If keep_citations is true, then we keep the "quoting sections" in the + content. + If keep_body is true, we even keep the signature sections. ''' # strip off leading carriage-returns / newlines i = 0 @@ -426,114 +1055,54 @@ def parseContent(content, blank_line=re.compile(r'[\r\n]+\s*[\r\n]+'), if not section: continue lines = eol.split(section) - if lines[0] and lines[0][0] in '>|': - continue - if len(lines) > 1 and lines[1] and lines[1][0] in '>|': - continue + if (lines[0] and lines[0][0] in '>|') or (len(lines) > 1 and + lines[1] and lines[1][0] in '>|'): + # see if there's a response somewhere inside this section (ie. + # no blank line between quoted message and response) + for line in lines[1:]: + if line and line[0] not in '>|': + break + else: + # we keep quoted bits if specified in the config + if keep_citations: + l.append(section) + continue + # keep this section - it has reponse stuff in it + lines = lines[lines.index(line):] + section = '\n'.join(lines) + # and while we're at it, use the first non-quoted bit as + # our summary + summary = section + if not summary: - summary = lines[0] - l.append(section) - continue - if signature.match(lines[0]): + # if we don't have our summary yet use the first line of this + # section + summary = section + elif signature.match(lines[0]) and 2 <= len(lines) <= 10: + # lose any signature + break + elif original_msg.match(lines[0]): + # ditch the stupid Outlook quoting of the entire original message break + + # and add the section to the output l.append(section) - return summary, '\n\n'.join(l) -# -# $Log: not supported by cvs2svn $ -# Revision 1.24 2001/10/23 22:57:52 richard -# Fix unread->chatting auto transition, thanks Roch'e -# -# Revision 1.23 2001/10/21 04:00:20 richard -# MailGW now moves 'unread' to 'chatting' on receiving e-mail for an issue. -# -# Revision 1.22 2001/10/21 03:35:13 richard -# bug #473125: Paragraph in e-mails -# -# Revision 1.21 2001/10/21 00:53:42 richard -# bug #473130: Nosy list not set correctly -# -# Revision 1.20 2001/10/17 23:13:19 richard -# Did a fair bit of work on the admin tool. Now has an extra command "table" -# which displays node information in a tabular format. Also fixed import and -# export so they work. Removed freshen. -# Fixed quopri usage in mailgw from bug reports. -# -# Revision 1.19 2001/10/11 23:43:04 richard -# Implemented the comma-separated printing option in the admin tool. -# Fixed a typo (more of a vim-o actually :) in mailgw. -# -# Revision 1.18 2001/10/11 06:38:57 richard -# Initial cut at trying to handle people responding to CC'ed messages that -# create an issue. -# -# Revision 1.17 2001/10/09 07:25:59 richard -# Added the Password property type. See "pydoc roundup.password" for -# implementation details. Have updated some of the documentation too. -# -# Revision 1.16 2001/10/05 02:23:24 richard -# . roundup-admin create now prompts for property info if none is supplied -# on the command-line. -# . hyperdb Class getprops() method may now return only the mutable -# properties. -# . Login now uses cookies, which makes it a whole lot more flexible. We can -# now support anonymous user access (read-only, unless there's an -# "anonymous" user, in which case write access is permitted). Login -# handling has been moved into cgi_client.Client.main() -# . The "extended" schema is now the default in roundup init. -# . The schemas have had their page headings modified to cope with the new -# login handling. Existing installations should copy the interfaces.py -# file from the roundup lib directory to their instance home. -# . Incorrectly had a Bizar Software copyright on the cgitb.py module from -# Ping - has been removed. -# . Fixed a whole bunch of places in the CGI interface where we should have -# been returning Not Found instead of throwing an exception. -# . Fixed a deviation from the spec: trying to modify the 'id' property of -# an item now throws an exception. -# -# Revision 1.15 2001/08/30 06:01:17 richard -# Fixed missing import in mailgw :( -# -# Revision 1.14 2001/08/13 23:02:54 richard -# Make the mail parser a little more robust. -# -# Revision 1.13 2001/08/12 06:32:36 richard -# using isinstance(blah, Foo) now instead of isFooType -# -# Revision 1.12 2001/08/08 01:27:00 richard -# Added better error handling to mailgw. -# -# Revision 1.11 2001/08/08 00:08:03 richard -# oops ;) -# -# Revision 1.10 2001/08/07 00:24:42 richard -# stupid typo -# -# Revision 1.9 2001/08/07 00:15:51 richard -# Added the copyright/license notice to (nearly) all files at request of -# Bizar Software. -# -# Revision 1.8 2001/08/05 07:06:07 richard -# removed some print statements -# -# Revision 1.7 2001/08/03 07:18:22 richard -# Implemented correct mail splitting (was taking a shortcut). Added unit -# tests. Also snips signatures now too. -# -# Revision 1.6 2001/08/01 04:24:21 richard -# mailgw was assuming certain properties existed on the issues being created. -# -# Revision 1.5 2001/07/29 07:01:39 richard -# Added vim command to all source so that we don't get no steenkin' tabs :) -# -# Revision 1.4 2001/07/28 06:43:02 richard -# Multipart message class has the getPart method now. Added some tests for it. -# -# Revision 1.3 2001/07/28 00:34:34 richard -# Fixed some non-string node ids. -# -# Revision 1.2 2001/07/22 12:09:32 richard -# Final commit of Grande Splite -# -# + # figure the summary - find the first sentence-ending punctuation or the + # first whole line, whichever is longest + sentence = re.search(r'^([^!?\.]+[!?\.])', summary) + if sentence: + sentence = sentence.group(1) + else: + sentence = '' + first = eol.split(summary)[0] + summary = max(sentence, first) + + # Now reconstitute the message content minus the bits we don't care + # about. + if not keep_body: + content = '\n\n'.join(l) + + return summary, content + # vim: set filetype=python ts=4 sw=4 et si