b78d2948463bad7a66b9c10f469aa9db2258abcf
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
19 '''
20 An e-mail gateway for Roundup.
22 Incoming messages are examined for multiple parts:
23 . In a multipart/mixed message or part, each subpart is extracted and
24 examined. The text/plain subparts are assembled to form the textual
25 body of the message, to be stored in the file associated with a "msg"
26 class node. Any parts of other types are each stored in separate files
27 and given "file" class nodes that are linked to the "msg" node.
28 . In a multipart/alternative message or part, we look for a text/plain
29 subpart and ignore the other parts.
31 Summary
32 -------
33 The "summary" property on message nodes is taken from the first non-quoting
34 section in the message body. The message body is divided into sections by
35 blank lines. Sections where the second and all subsequent lines begin with
36 a ">" or "|" character are considered "quoting sections". The first line of
37 the first non-quoting section becomes the summary of the message.
39 Addresses
40 ---------
41 All of the addresses in the To: and Cc: headers of the incoming message are
42 looked up among the user nodes, and the corresponding users are placed in
43 the "recipients" property on the new "msg" node. The address in the From:
44 header similarly determines the "author" property of the new "msg"
45 node. The default handling for addresses that don't have corresponding
46 users is to create new users with no passwords and a username equal to the
47 address. (The web interface does not permit logins for users with no
48 passwords.) If we prefer to reject mail from outside sources, we can simply
49 register an auditor on the "user" class that prevents the creation of user
50 nodes with no passwords.
52 Actions
53 -------
54 The subject line of the incoming message is examined to determine whether
55 the message is an attempt to create a new item or to discuss an existing
56 item. A designator enclosed in square brackets is sought as the first thing
57 on the subject line (after skipping any "Fwd:" or "Re:" prefixes).
59 If an item designator (class name and id number) is found there, the newly
60 created "msg" node is added to the "messages" property for that item, and
61 any new "file" nodes are added to the "files" property for the item.
63 If just an item class name is found there, we attempt to create a new item
64 of that class with its "messages" property initialized to contain the new
65 "msg" node and its "files" property initialized to contain any new "file"
66 nodes.
68 Triggers
69 --------
70 Both cases may trigger detectors (in the first case we are calling the
71 set() method to add the message to the item's spool; in the second case we
72 are calling the create() method to create a new node). If an auditor raises
73 an exception, the original message is bounced back to the sender with the
74 explanatory message given in the exception.
76 $Id: mailgw.py,v 1.119 2003-04-24 07:22:08 richard Exp $
77 '''
79 import string, re, os, mimetools, cStringIO, smtplib, socket, binascii, quopri
80 import time, random, sys
81 import traceback, MimeWriter, rfc822
83 from roundup import hyperdb, date, password, rfc2822
85 SENDMAILDEBUG = os.environ.get('SENDMAILDEBUG', '')
87 class MailGWError(ValueError):
88 pass
90 class MailUsageError(ValueError):
91 pass
93 class MailUsageHelp(Exception):
94 pass
96 class MailLoop(Exception):
97 ''' We've seen this message before... '''
98 pass
100 class Unauthorized(Exception):
101 """ Access denied """
103 def initialiseSecurity(security):
104 ''' Create some Permissions and Roles on the security object
106 This function is directly invoked by security.Security.__init__()
107 as a part of the Security object instantiation.
108 '''
109 security.addPermission(name="Email Registration",
110 description="Anonymous may register through e-mail")
111 p = security.addPermission(name="Email Access",
112 description="User may use the email interface")
113 security.addPermissionToRole('Admin', p)
115 def getparam(str, param):
116 ''' From the rfc822 "header" string, extract "param" if it appears.
117 '''
118 if ';' not in str:
119 return None
120 str = str[str.index(';'):]
121 while str[:1] == ';':
122 str = str[1:]
123 if ';' in str:
124 # XXX Should parse quotes!
125 end = str.index(';')
126 else:
127 end = len(str)
128 f = str[:end]
129 if '=' in f:
130 i = f.index('=')
131 if f[:i].strip().lower() == param:
132 return rfc822.unquote(f[i+1:].strip())
133 return None
135 def openSMTPConnection(config):
136 ''' Open an SMTP connection to the mailhost specified in the config
137 '''
138 smtp = smtplib.SMTP(config.MAILHOST)
140 # use TLS?
141 use_tls = getattr(config, 'MAILHOST_TLS', 'no')
142 if use_tls == 'yes'
143 # do we have key files too?
144 keyfile = getattr(config, 'MAILHOST_TLS_KEYFILE', '')
145 if keyfile:
146 certfile = getattr(config, 'MAILHOST_TLS_CERTFILE', '')
147 if certfile:
148 args = (keyfile, certfile)
149 else:
150 args = (keyfile, )
151 else:
152 args = ()
153 # start the TLS
154 smtp.starttls(*args)
156 # ok, now do we also need to log in?
157 mailuser = getattr(config, 'MAILUSER', None)
158 if mailuser:
159 smtp.login(*config.MAILUSER)
161 # that's it, a fully-configured SMTP connection ready to go
162 return smtp
164 class Message(mimetools.Message):
165 ''' subclass mimetools.Message so we can retrieve the parts of the
166 message...
167 '''
168 def getPart(self):
169 ''' Get a single part of a multipart message and return it as a new
170 Message instance.
171 '''
172 boundary = self.getparam('boundary')
173 mid, end = '--'+boundary, '--'+boundary+'--'
174 s = cStringIO.StringIO()
175 while 1:
176 line = self.fp.readline()
177 if not line:
178 break
179 if line.strip() in (mid, end):
180 break
181 s.write(line)
182 if not s.getvalue().strip():
183 return None
184 s.seek(0)
185 return Message(s)
187 def getheader(self, name, default=None):
188 hdr = mimetools.Message.getheader(self, name, default)
189 return rfc2822.decode_header(hdr)
191 subject_re = re.compile(r'(?P<refwd>\s*\W?\s*(fw|fwd|re|aw)\W\s*)*'
192 r'\s*(?P<quote>")?(\[(?P<classname>[^\d\s]+)(?P<nodeid>\d+)?\])?'
193 r'\s*(?P<title>[^[]+)?"?(\[(?P<args>.+?)\])?', re.I)
195 class MailGW:
196 def __init__(self, instance, db, arguments={}):
197 self.instance = instance
198 self.db = db
199 self.arguments = arguments
201 # should we trap exceptions (normal usage) or pass them through
202 # (for testing)
203 self.trapExceptions = 1
205 def do_pipe(self):
206 ''' Read a message from standard input and pass it to the mail handler.
208 Read into an internal structure that we can seek on (in case
209 there's an error).
211 XXX: we may want to read this into a temporary file instead...
212 '''
213 s = cStringIO.StringIO()
214 s.write(sys.stdin.read())
215 s.seek(0)
216 self.main(s)
217 return 0
219 def do_mailbox(self, filename):
220 ''' Read a series of messages from the specified unix mailbox file and
221 pass each to the mail handler.
222 '''
223 # open the spool file and lock it
224 import fcntl, FCNTL
225 f = open(filename, 'r+')
226 fcntl.flock(f.fileno(), FCNTL.LOCK_EX)
228 # handle and clear the mailbox
229 try:
230 from mailbox import UnixMailbox
231 mailbox = UnixMailbox(f, factory=Message)
232 # grab one message
233 message = mailbox.next()
234 while message:
235 # handle this message
236 self.handle_Message(message)
237 message = mailbox.next()
238 # nuke the file contents
239 os.ftruncate(f.fileno(), 0)
240 except:
241 import traceback
242 traceback.print_exc()
243 return 1
244 fcntl.flock(f.fileno(), FCNTL.LOCK_UN)
245 return 0
247 def do_apop(self, server, user='', password=''):
248 ''' Do authentication POP
249 '''
250 self.do_pop(server, user, password, apop=1)
252 def do_pop(self, server, user='', password='', apop=0):
253 '''Read a series of messages from the specified POP server.
254 '''
255 import getpass, poplib, socket
256 try:
257 if not user:
258 user = raw_input(_('User: '))
259 if not password:
260 password = getpass.getpass()
261 except (KeyboardInterrupt, EOFError):
262 # Ctrl C or D maybe also Ctrl Z under Windows.
263 print "\nAborted by user."
264 return 1
266 # open a connection to the server and retrieve all messages
267 try:
268 server = poplib.POP3(server)
269 except socket.error, message:
270 print "POP server error:", message
271 return 1
272 if apop:
273 server.apop(user, password)
274 else:
275 server.user(user)
276 server.pass_(password)
277 numMessages = len(server.list()[1])
278 for i in range(1, numMessages+1):
279 # retr: returns
280 # [ pop response e.g. '+OK 459 octets',
281 # [ array of message lines ],
282 # number of octets ]
283 lines = server.retr(i)[1]
284 s = cStringIO.StringIO('\n'.join(lines))
285 s.seek(0)
286 self.handle_Message(Message(s))
287 # delete the message
288 server.dele(i)
290 # quit the server to commit changes.
291 server.quit()
292 return 0
294 def main(self, fp):
295 ''' fp - the file from which to read the Message.
296 '''
297 return self.handle_Message(Message(fp))
299 def handle_Message(self, message):
300 '''Handle an RFC822 Message
302 Handle the Message object by calling handle_message() and then cope
303 with any errors raised by handle_message.
304 This method's job is to make that call and handle any
305 errors in a sane manner. It should be replaced if you wish to
306 handle errors in a different manner.
307 '''
308 # in some rare cases, a particularly stuffed-up e-mail will make
309 # its way into here... try to handle it gracefully
310 sendto = message.getaddrlist('from')
311 if sendto:
312 if not self.trapExceptions:
313 return self.handle_message(message)
314 try:
315 return self.handle_message(message)
316 except MailUsageHelp:
317 # bounce the message back to the sender with the usage message
318 fulldoc = '\n'.join(string.split(__doc__, '\n')[2:])
319 sendto = [sendto[0][1]]
320 m = ['']
321 m.append('\n\nMail Gateway Help\n=================')
322 m.append(fulldoc)
323 m = self.bounce_message(message, sendto, m,
324 subject="Mail Gateway Help")
325 except MailUsageError, value:
326 # bounce the message back to the sender with the usage message
327 fulldoc = '\n'.join(string.split(__doc__, '\n')[2:])
328 sendto = [sendto[0][1]]
329 m = ['']
330 m.append(str(value))
331 m.append('\n\nMail Gateway Help\n=================')
332 m.append(fulldoc)
333 m = self.bounce_message(message, sendto, m)
334 except Unauthorized, value:
335 # just inform the user that he is not authorized
336 sendto = [sendto[0][1]]
337 m = ['']
338 m.append(str(value))
339 m = self.bounce_message(message, sendto, m)
340 except MailLoop:
341 # XXX we should use a log file here...
342 return
343 except:
344 # bounce the message back to the sender with the error message
345 # XXX we should use a log file here...
346 sendto = [sendto[0][1], self.instance.config.ADMIN_EMAIL]
347 m = ['']
348 m.append('An unexpected error occurred during the processing')
349 m.append('of your message. The tracker administrator is being')
350 m.append('notified.\n')
351 m.append('---- traceback of failure ----')
352 s = cStringIO.StringIO()
353 import traceback
354 traceback.print_exc(None, s)
355 m.append(s.getvalue())
356 m = self.bounce_message(message, sendto, m)
357 else:
358 # very bad-looking message - we don't even know who sent it
359 # XXX we should use a log file here...
360 sendto = [self.instance.config.ADMIN_EMAIL]
361 m = ['Subject: badly formed message from mail gateway']
362 m.append('')
363 m.append('The mail gateway retrieved a message which has no From:')
364 m.append('line, indicating that it is corrupt. Please check your')
365 m.append('mail gateway source. Failed message is attached.')
366 m.append('')
367 m = self.bounce_message(message, sendto, m,
368 subject='Badly formed message from mail gateway')
370 # now send the message
371 if SENDMAILDEBUG:
372 open(SENDMAILDEBUG, 'a').write('From: %s\nTo: %s\n%s\n'%(
373 self.instance.config.ADMIN_EMAIL, ', '.join(sendto),
374 m.getvalue()))
375 else:
376 try:
377 smtp = openSMTPConnection(self.instance.config)
378 smtp.sendmail(self.instance.config.ADMIN_EMAIL, sendto,
379 m.getvalue())
380 except socket.error, value:
381 raise MailGWError, "Couldn't send error email: "\
382 "mailhost %s"%value
383 except smtplib.SMTPException, value:
384 raise MailGWError, "Couldn't send error email: %s"%value
386 def bounce_message(self, message, sendto, error,
387 subject='Failed issue tracker submission'):
388 ''' create a message that explains the reason for the failed
389 issue submission to the author and attach the original
390 message.
391 '''
392 msg = cStringIO.StringIO()
393 writer = MimeWriter.MimeWriter(msg)
394 writer.addheader('X-Roundup-Loop', 'hello')
395 writer.addheader('Subject', subject)
396 writer.addheader('From', '%s <%s>'% (self.instance.config.TRACKER_NAME,
397 self.instance.config.TRACKER_EMAIL))
398 writer.addheader('To', ','.join(sendto))
399 writer.addheader('Date', time.strftime("%a, %d %b %Y %H:%M:%S +0000",
400 time.gmtime()))
401 writer.addheader('MIME-Version', '1.0')
402 part = writer.startmultipartbody('mixed')
403 part = writer.nextpart()
404 body = part.startbody('text/plain; charset=utf-8')
405 body.write('\n'.join(error))
407 # attach the original message to the returned message
408 part = writer.nextpart()
409 part.addheader('Content-Disposition','attachment')
410 part.addheader('Content-Description','Message you sent')
411 body = part.startbody('text/plain')
412 for header in message.headers:
413 body.write(header)
414 body.write('\n')
415 try:
416 message.rewindbody()
417 except IOError, message:
418 body.write("*** couldn't include message body: %s ***"%message)
419 else:
420 body.write(message.fp.read())
422 writer.lastpart()
423 return msg
425 def get_part_data_decoded(self,part):
426 encoding = part.getencoding()
427 data = None
428 if encoding == 'base64':
429 # BUG: is base64 really used for text encoding or
430 # are we inserting zip files here.
431 data = binascii.a2b_base64(part.fp.read())
432 elif encoding == 'quoted-printable':
433 # the quopri module wants to work with files
434 decoded = cStringIO.StringIO()
435 quopri.decode(part.fp, decoded)
436 data = decoded.getvalue()
437 elif encoding == 'uuencoded':
438 data = binascii.a2b_uu(part.fp.read())
439 else:
440 # take it as text
441 data = part.fp.read()
443 # Encode message to unicode
444 charset = rfc2822.unaliasCharset(part.getparam("charset"))
445 if charset:
446 # Do conversion only if charset specified
447 edata = unicode(data, charset).encode('utf-8')
448 # Convert from dos eol to unix
449 edata = edata.replace('\r\n', '\n')
450 else:
451 # Leave message content as is
452 edata = data
454 return edata
456 def handle_message(self, message):
457 ''' message - a Message instance
459 Parse the message as per the module docstring.
460 '''
461 # detect loops
462 if message.getheader('x-roundup-loop', ''):
463 raise MailLoop
465 # XXX Don't enable. This doesn't work yet.
466 # "[^A-z.]tracker\+(?P<classname>[^\d\s]+)(?P<nodeid>\d+)\@some.dom.ain[^A-z.]"
467 # handle delivery to addresses like:tracker+issue25@some.dom.ain
468 # use the embedded issue number as our issue
469 # if hasattr(self.instance.config, 'EMAIL_ISSUE_ADDRESS_RE') and \
470 # self.instance.config.EMAIL_ISSUE_ADDRESS_RE:
471 # issue_re = self.instance.config.EMAIL_ISSUE_ADDRESS_RE
472 # for header in ['to', 'cc', 'bcc']:
473 # addresses = message.getheader(header, '')
474 # if addresses:
475 # # FIXME, this only finds the first match in the addresses.
476 # issue = re.search(issue_re, addresses, 'i')
477 # if issue:
478 # classname = issue.group('classname')
479 # nodeid = issue.group('nodeid')
480 # break
482 # handle the subject line
483 subject = message.getheader('subject', '')
485 if subject.strip().lower() == 'help':
486 raise MailUsageHelp
488 m = subject_re.match(subject)
490 # check for well-formed subject line
491 if m:
492 # get the classname
493 classname = m.group('classname')
494 if classname is None:
495 # no classname, fallback on the default
496 if hasattr(self.instance.config, 'MAIL_DEFAULT_CLASS') and \
497 self.instance.config.MAIL_DEFAULT_CLASS:
498 classname = self.instance.config.MAIL_DEFAULT_CLASS
499 else:
500 # fail
501 m = None
503 if not m:
504 raise MailUsageError, '''
505 The message you sent to roundup did not contain a properly formed subject
506 line. The subject must contain a class name or designator to indicate the
507 "topic" of the message. For example:
508 Subject: [issue] This is a new issue
509 - this will create a new issue in the tracker with the title "This is
510 a new issue".
511 Subject: [issue1234] This is a followup to issue 1234
512 - this will append the message's contents to the existing issue 1234
513 in the tracker.
515 Subject was: "%s"
516 '''%subject
518 # get the class
519 try:
520 cl = self.db.getclass(classname)
521 except KeyError:
522 raise MailUsageError, '''
523 The class name you identified in the subject line ("%s") does not exist in the
524 database.
526 Valid class names are: %s
527 Subject was: "%s"
528 '''%(classname, ', '.join(self.db.getclasses()), subject)
530 # get the optional nodeid
531 nodeid = m.group('nodeid')
533 # title is optional too
534 title = m.group('title')
535 if title:
536 title = title.strip()
537 else:
538 title = ''
540 # strip off the quotes that dumb emailers put around the subject, like
541 # Re: "[issue1] bla blah"
542 if m.group('quote') and title.endswith('"'):
543 title = title[:-1]
545 # but we do need either a title or a nodeid...
546 if nodeid is None and not title:
547 raise MailUsageError, '''
548 I cannot match your message to a node in the database - you need to either
549 supply a full node identifier (with number, eg "[issue123]" or keep the
550 previous subject title intact so I can match that.
552 Subject was: "%s"
553 '''%subject
555 # If there's no nodeid, check to see if this is a followup and
556 # maybe someone's responded to the initial mail that created an
557 # entry. Try to find the matching nodes with the same title, and
558 # use the _last_ one matched (since that'll _usually_ be the most
559 # recent...)
560 if nodeid is None and m.group('refwd'):
561 l = cl.stringFind(title=title)
562 if l:
563 nodeid = l[-1]
565 # if a nodeid was specified, make sure it's valid
566 if nodeid is not None and not cl.hasnode(nodeid):
567 raise MailUsageError, '''
568 The node specified by the designator in the subject of your message ("%s")
569 does not exist.
571 Subject was: "%s"
572 '''%(nodeid, subject)
575 # Handle the arguments specified by the email gateway command line.
576 # We do this by looping over the list of self.arguments looking for
577 # a -C to tell us what class then the -S setting string.
578 msg_props = {}
579 user_props = {}
580 file_props = {}
581 issue_props = {}
582 # so, if we have any arguments, use them
583 if self.arguments:
584 current_class = 'msg'
585 for option, propstring in self.arguments:
586 if option in ( '-C', '--class'):
587 current_class = propstring.strip()
588 if current_class not in ('msg', 'file', 'user', 'issue'):
589 raise MailUsageError, '''
590 The mail gateway is not properly set up. Please contact
591 %s and have them fix the incorrect class specified as:
592 %s
593 '''%(self.instance.config.ADMIN_EMAIL, current_class)
594 if option in ('-S', '--set'):
595 if current_class == 'issue' :
596 errors, issue_props = setPropArrayFromString(self,
597 cl, propstring.strip(), nodeid)
598 elif current_class == 'file' :
599 temp_cl = self.db.getclass('file')
600 errors, file_props = setPropArrayFromString(self,
601 temp_cl, propstring.strip())
602 elif current_class == 'msg' :
603 temp_cl = self.db.getclass('msg')
604 errors, msg_props = setPropArrayFromString(self,
605 temp_cl, propstring.strip())
606 elif current_class == 'user' :
607 temp_cl = self.db.getclass('user')
608 errors, user_props = setPropArrayFromString(self,
609 temp_cl, propstring.strip())
610 if errors:
611 raise MailUsageError, '''
612 The mail gateway is not properly set up. Please contact
613 %s and have them fix the incorrect properties:
614 %s
615 '''%(self.instance.config.ADMIN_EMAIL, errors)
617 #
618 # handle the users
619 #
620 # Don't create users if anonymous isn't allowed to register
621 create = 1
622 anonid = self.db.user.lookup('anonymous')
623 if not self.db.security.hasPermission('Email Registration', anonid):
624 create = 0
626 # ok, now figure out who the author is - create a new user if the
627 # "create" flag is true
628 author = uidFromAddress(self.db, message.getaddrlist('from')[0],
629 create=create)
631 # if we're not recognised, and we don't get added as a user, then we
632 # must be anonymous
633 if not author:
634 author = anonid
636 # make sure the author has permission to use the email interface
637 if not self.db.security.hasPermission('Email Access', author):
638 if author == anonid:
639 # we're anonymous and we need to be a registered user
640 raise Unauthorized, '''
641 You are not a registered user.
643 Unknown address: %s
644 '''%message.getaddrlist('from')[0][1]
645 else:
646 # we're registered and we're _still_ not allowed access
647 raise Unauthorized, 'You are not permitted to access '\
648 'this tracker.'
650 # make sure they're allowed to edit this class of information
651 if not self.db.security.hasPermission('Edit', author, classname):
652 raise Unauthorized, 'You are not permitted to edit %s.'%classname
654 # the author may have been created - make sure the change is
655 # committed before we reopen the database
656 self.db.commit()
658 # reopen the database as the author
659 username = self.db.user.get(author, 'username')
660 self.db.close()
661 self.db = self.instance.open(username)
663 # re-get the class with the new database connection
664 cl = self.db.getclass(classname)
666 # now update the recipients list
667 recipients = []
668 tracker_email = self.instance.config.TRACKER_EMAIL.lower()
669 for recipient in message.getaddrlist('to') + message.getaddrlist('cc'):
670 r = recipient[1].strip().lower()
671 if r == tracker_email or not r:
672 continue
674 # look up the recipient - create if necessary (and we're
675 # allowed to)
676 recipient = uidFromAddress(self.db, recipient, create, **user_props)
678 # if all's well, add the recipient to the list
679 if recipient:
680 recipients.append(recipient)
682 #
683 # XXX extract the args NOT USED WHY -- rouilj
684 #
685 subject_args = m.group('args')
687 #
688 # handle the subject argument list
689 #
690 # figure what the properties of this Class are
691 properties = cl.getprops()
692 props = {}
693 args = m.group('args')
694 if args:
695 errors, props = setPropArrayFromString(self, cl, args, nodeid)
696 # handle any errors parsing the argument list
697 if errors:
698 errors = '\n- '.join(errors)
699 raise MailUsageError, '''
700 There were problems handling your subject line argument list:
701 - %s
703 Subject was: "%s"
704 '''%(errors, subject)
706 #
707 # handle message-id and in-reply-to
708 #
709 messageid = message.getheader('message-id')
710 inreplyto = message.getheader('in-reply-to') or ''
711 # generate a messageid if there isn't one
712 if not messageid:
713 messageid = "<%s.%s.%s%s@%s>"%(time.time(), random.random(),
714 classname, nodeid, self.instance.config.MAIL_DOMAIN)
716 #
717 # now handle the body - find the message
718 #
719 content_type = message.gettype()
720 attachments = []
721 # General multipart handling:
722 # Take the first text/plain part, anything else is considered an
723 # attachment.
724 # multipart/mixed: multiple "unrelated" parts.
725 # multipart/signed (rfc 1847):
726 # The control information is carried in the second of the two
727 # required body parts.
728 # ACTION: Default, so if content is text/plain we get it.
729 # multipart/encrypted (rfc 1847):
730 # The control information is carried in the first of the two
731 # required body parts.
732 # ACTION: Not handleable as the content is encrypted.
733 # multipart/related (rfc 1872, 2112, 2387):
734 # The Multipart/Related content-type addresses the MIME
735 # representation of compound objects.
736 # ACTION: Default. If we are lucky there is a text/plain.
737 # TODO: One should use the start part and look for an Alternative
738 # that is text/plain.
739 # multipart/Alternative (rfc 1872, 1892):
740 # only in "related" ?
741 # multipart/report (rfc 1892):
742 # e.g. mail system delivery status reports.
743 # ACTION: Default. Could be ignored or used for Delivery Notification
744 # flagging.
745 # multipart/form-data:
746 # For web forms only.
747 if content_type == 'multipart/mixed':
748 # skip over the intro to the first boundary
749 part = message.getPart()
750 content = None
751 while 1:
752 # get the next part
753 part = message.getPart()
754 if part is None:
755 break
756 # parse it
757 subtype = part.gettype()
758 if subtype == 'text/plain' and not content:
759 # The first text/plain part is the message content.
760 content = self.get_part_data_decoded(part)
761 elif subtype == 'message/rfc822':
762 # handle message/rfc822 specially - the name should be
763 # the subject of the actual e-mail embedded here
764 i = part.fp.tell()
765 mailmess = Message(part.fp)
766 name = mailmess.getheader('subject')
767 part.fp.seek(i)
768 attachments.append((name, 'message/rfc822', part.fp.read()))
769 elif subtype == 'multipart/alternative':
770 # Search for text/plain in message with attachment and
771 # alternative text representation
772 # skip over intro to first boundary
773 part.getPart()
774 while 1:
775 # get the next part
776 subpart = part.getPart()
777 if subpart is None:
778 break
779 # parse it
780 if subpart.gettype() == 'text/plain' and not content:
781 content = self.get_part_data_decoded(subpart)
782 else:
783 # try name on Content-Type
784 name = part.getparam('name')
785 if name:
786 name = name.strip()
787 if not name:
788 disp = part.getheader('content-disposition', None)
789 if disp:
790 name = getparam(disp, 'filename')
791 if name:
792 name = name.strip()
793 # this is just an attachment
794 data = self.get_part_data_decoded(part)
795 attachments.append((name, part.gettype(), data))
796 if content is None:
797 raise MailUsageError, '''
798 Roundup requires the submission to be plain text. The message parser could
799 not find a text/plain part to use.
800 '''
802 elif content_type[:10] == 'multipart/':
803 # skip over the intro to the first boundary
804 message.getPart()
805 content = None
806 while 1:
807 # get the next part
808 part = message.getPart()
809 if part is None:
810 break
811 # parse it
812 if part.gettype() == 'text/plain' and not content:
813 content = self.get_part_data_decoded(part)
814 if content is None:
815 raise MailUsageError, '''
816 Roundup requires the submission to be plain text. The message parser could
817 not find a text/plain part to use.
818 '''
820 elif content_type != 'text/plain':
821 raise MailUsageError, '''
822 Roundup requires the submission to be plain text. The message parser could
823 not find a text/plain part to use.
824 '''
826 else:
827 content = self.get_part_data_decoded(message)
829 # figure how much we should muck around with the email body
830 keep_citations = getattr(self.instance.config, 'EMAIL_KEEP_QUOTED_TEXT',
831 'no') == 'yes'
832 keep_body = getattr(self.instance.config, 'EMAIL_LEAVE_BODY_UNCHANGED',
833 'no') == 'yes'
835 # parse the body of the message, stripping out bits as appropriate
836 summary, content = parseContent(content, keep_citations,
837 keep_body)
838 content = content.strip()
840 #
841 # handle the attachments
842 #
843 files = []
844 for (name, mime_type, data) in attachments:
845 if not name:
846 name = "unnamed"
847 files.append(self.db.file.create(type=mime_type, name=name,
848 content=data, **file_props))
849 # attach the files to the issue
850 if nodeid:
851 # extend the existing files list
852 fileprop = cl.get(nodeid, 'files')
853 fileprop.extend(files)
854 props['files'] = fileprop
855 else:
856 # pre-load the files list
857 props['files'] = files
860 #
861 # create the message if there's a message body (content)
862 #
863 if content:
864 message_id = self.db.msg.create(author=author,
865 recipients=recipients, date=date.Date('.'), summary=summary,
866 content=content, files=files, messageid=messageid,
867 inreplyto=inreplyto, **msg_props)
869 # attach the message to the node
870 if nodeid:
871 # add the message to the node's list
872 messages = cl.get(nodeid, 'messages')
873 messages.append(message_id)
874 props['messages'] = messages
875 else:
876 # pre-load the messages list
877 props['messages'] = [message_id]
879 # set the title to the subject
880 if properties.has_key('title') and not props.has_key('title'):
881 props['title'] = title
883 #
884 # perform the node change / create
885 #
886 try:
887 # merge the command line props defined in issue_props into
888 # the props dictionary because function(**props, **issue_props)
889 # is a syntax error.
890 for prop in issue_props.keys() :
891 if not props.has_key(prop) :
892 props[prop] = issue_props[prop]
893 if nodeid:
894 cl.set(nodeid, **props)
895 else:
896 nodeid = cl.create(**props)
897 except (TypeError, IndexError, ValueError), message:
898 raise MailUsageError, '''
899 There was a problem with the message you sent:
900 %s
901 '''%message
903 # commit the changes to the DB
904 self.db.commit()
906 return nodeid
909 def setPropArrayFromString(self, cl, propString, nodeid = None):
910 ''' takes string of form prop=value,value;prop2=value
911 and returns (error, prop[..])
912 '''
913 properties = cl.getprops()
914 props = {}
915 errors = []
916 for prop in string.split(propString, ';'):
917 # extract the property name and value
918 try:
919 propname, value = prop.split('=')
920 except ValueError, message:
921 errors.append('not of form [arg=value,value,...;'
922 'arg=value,value,...]')
923 return (errors, props)
925 # ensure it's a valid property name
926 propname = propname.strip()
927 try:
928 proptype = properties[propname]
929 except KeyError:
930 errors.append('refers to an invalid property: "%s"'%propname)
931 continue
933 # convert the string value to a real property value
934 if isinstance(proptype, hyperdb.String):
935 props[propname] = value.strip()
936 if isinstance(proptype, hyperdb.Password):
937 props[propname] = password.Password(value.strip())
938 elif isinstance(proptype, hyperdb.Date):
939 try:
940 props[propname] = date.Date(value.strip()).local(self.db.getUserTimezone())
941 except ValueError, message:
942 errors.append('contains an invalid date for %s.'%propname)
943 elif isinstance(proptype, hyperdb.Interval):
944 try:
945 props[propname] = date.Interval(value)
946 except ValueError, message:
947 errors.append('contains an invalid date interval for %s.'%
948 propname)
949 elif isinstance(proptype, hyperdb.Link):
950 linkcl = self.db.classes[proptype.classname]
951 propkey = linkcl.labelprop(default_to_id=1)
952 try:
953 props[propname] = linkcl.lookup(value)
954 except KeyError, message:
955 errors.append('"%s" is not a value for %s.'%(value, propname))
956 elif isinstance(proptype, hyperdb.Multilink):
957 # get the linked class
958 linkcl = self.db.classes[proptype.classname]
959 propkey = linkcl.labelprop(default_to_id=1)
960 if nodeid:
961 curvalue = cl.get(nodeid, propname)
962 else:
963 curvalue = []
965 # handle each add/remove in turn
966 # keep an extra list for all items that are
967 # definitely in the new list (in case of e.g.
968 # <propname>=A,+B, which should replace the old
969 # list with A,B)
970 set = 0
971 newvalue = []
972 for item in value.split(','):
973 item = item.strip()
975 # handle +/-
976 remove = 0
977 if item.startswith('-'):
978 remove = 1
979 item = item[1:]
980 elif item.startswith('+'):
981 item = item[1:]
982 else:
983 set = 1
985 # look up the value
986 try:
987 item = linkcl.lookup(item)
988 except KeyError, message:
989 errors.append('"%s" is not a value for %s.'%(item,
990 propname))
991 continue
993 # perform the add/remove
994 if remove:
995 try:
996 curvalue.remove(item)
997 except ValueError:
998 errors.append('"%s" is not currently in for %s.'%(item,
999 propname))
1000 continue
1001 else:
1002 newvalue.append(item)
1003 if item not in curvalue:
1004 curvalue.append(item)
1006 # that's it, set the new Multilink property value,
1007 # or overwrite it completely
1008 if set:
1009 props[propname] = newvalue
1010 else:
1011 props[propname] = curvalue
1012 elif isinstance(proptype, hyperdb.Boolean):
1013 value = value.strip()
1014 props[propname] = value.lower() in ('yes', 'true', 'on', '1')
1015 elif isinstance(proptype, hyperdb.Number):
1016 value = value.strip()
1017 props[propname] = float(value)
1018 return errors, props
1021 def extractUserFromList(userClass, users):
1022 '''Given a list of users, try to extract the first non-anonymous user
1023 and return that user, otherwise return None
1024 '''
1025 if len(users) > 1:
1026 for user in users:
1027 # make sure we don't match the anonymous or admin user
1028 if userClass.get(user, 'username') in ('admin', 'anonymous'):
1029 continue
1030 # first valid match will do
1031 return user
1032 # well, I guess we have no choice
1033 return user[0]
1034 elif users:
1035 return users[0]
1036 return None
1039 def uidFromAddress(db, address, create=1, **user_props):
1040 ''' address is from the rfc822 module, and therefore is (name, addr)
1042 user is created if they don't exist in the db already
1043 user_props may supply additional user information
1044 '''
1045 (realname, address) = address
1047 # try a straight match of the address
1048 user = extractUserFromList(db.user, db.user.stringFind(address=address))
1049 if user is not None:
1050 return user
1052 # try the user alternate addresses if possible
1053 props = db.user.getprops()
1054 if props.has_key('alternate_addresses'):
1055 users = db.user.filter(None, {'alternate_addresses': address})
1056 user = extractUserFromList(db.user, users)
1057 if user is not None:
1058 return user
1060 # try to match the username to the address (for local
1061 # submissions where the address is empty)
1062 user = extractUserFromList(db.user, db.user.stringFind(username=address))
1064 # couldn't match address or username, so create a new user
1065 if create:
1066 # generate a username
1067 if '@' in address:
1068 username = address.split('@')[0]
1069 else:
1070 username = address
1071 trying = username
1072 n = 0
1073 while 1:
1074 try:
1075 # does this username exist already?
1076 db.user.lookup(trying)
1077 except KeyError:
1078 break
1079 n += 1
1080 trying = username + str(n)
1082 # create!
1083 return db.user.create(username=trying, address=address,
1084 realname=realname, roles=db.config.NEW_EMAIL_USER_ROLES,
1085 password=password.Password(password.generatePassword()),
1086 **user_props)
1087 else:
1088 return 0
1091 def parseContent(content, keep_citations, keep_body,
1092 blank_line=re.compile(r'[\r\n]+\s*[\r\n]+'),
1093 eol=re.compile(r'[\r\n]+'),
1094 signature=re.compile(r'^[>|\s]*[-_]+\s*$'),
1095 original_msg=re.compile(r'^[>|\s]*-----\s?Original Message\s?-----$')):
1096 ''' The message body is divided into sections by blank lines.
1097 Sections where the second and all subsequent lines begin with a ">"
1098 or "|" character are considered "quoting sections". The first line of
1099 the first non-quoting section becomes the summary of the message.
1101 If keep_citations is true, then we keep the "quoting sections" in the
1102 content.
1103 If keep_body is true, we even keep the signature sections.
1104 '''
1105 # strip off leading carriage-returns / newlines
1106 i = 0
1107 for i in range(len(content)):
1108 if content[i] not in '\r\n':
1109 break
1110 if i > 0:
1111 sections = blank_line.split(content[i:])
1112 else:
1113 sections = blank_line.split(content)
1115 # extract out the summary from the message
1116 summary = ''
1117 l = []
1118 for section in sections:
1119 #section = section.strip()
1120 if not section:
1121 continue
1122 lines = eol.split(section)
1123 if (lines[0] and lines[0][0] in '>|') or (len(lines) > 1 and
1124 lines[1] and lines[1][0] in '>|'):
1125 # see if there's a response somewhere inside this section (ie.
1126 # no blank line between quoted message and response)
1127 for line in lines[1:]:
1128 if line and line[0] not in '>|':
1129 break
1130 else:
1131 # we keep quoted bits if specified in the config
1132 if keep_citations:
1133 l.append(section)
1134 continue
1135 # keep this section - it has reponse stuff in it
1136 lines = lines[lines.index(line):]
1137 section = '\n'.join(lines)
1138 # and while we're at it, use the first non-quoted bit as
1139 # our summary
1140 summary = section
1142 if not summary:
1143 # if we don't have our summary yet use the first line of this
1144 # section
1145 summary = section
1146 elif signature.match(lines[0]) and 2 <= len(lines) <= 10:
1147 # lose any signature
1148 break
1149 elif original_msg.match(lines[0]):
1150 # ditch the stupid Outlook quoting of the entire original message
1151 break
1153 # and add the section to the output
1154 l.append(section)
1156 # figure the summary - find the first sentence-ending punctuation or the
1157 # first whole line, whichever is longest
1158 sentence = re.search(r'^([^!?\.]+[!?\.])', summary)
1159 if sentence:
1160 sentence = sentence.group(1)
1161 else:
1162 sentence = ''
1163 first = eol.split(summary)[0]
1164 summary = max(sentence, first)
1166 # Now reconstitute the message content minus the bits we don't care
1167 # about.
1168 if not keep_body:
1169 content = '\n\n'.join(l)
1171 return summary, content
1173 # vim: set filetype=python ts=4 sw=4 et si