1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
19 """
20 An e-mail gateway for Roundup.
22 Incoming messages are examined for multiple parts:
23 . In a multipart/mixed message or part, each subpart is extracted and
24 examined. The text/plain subparts are assembled to form the textual
25 body of the message, to be stored in the file associated with a "msg"
26 class node. Any parts of other types are each stored in separate files
27 and given "file" class nodes that are linked to the "msg" node.
28 . In a multipart/alternative message or part, we look for a text/plain
29 subpart and ignore the other parts.
31 Summary
32 -------
33 The "summary" property on message nodes is taken from the first non-quoting
34 section in the message body. The message body is divided into sections by
35 blank lines. Sections where the second and all subsequent lines begin with
36 a ">" or "|" character are considered "quoting sections". The first line of
37 the first non-quoting section becomes the summary of the message.
39 Addresses
40 ---------
41 All of the addresses in the To: and Cc: headers of the incoming message are
42 looked up among the user nodes, and the corresponding users are placed in
43 the "recipients" property on the new "msg" node. The address in the From:
44 header similarly determines the "author" property of the new "msg"
45 node. The default handling for addresses that don't have corresponding
46 users is to create new users with no passwords and a username equal to the
47 address. (The web interface does not permit logins for users with no
48 passwords.) If we prefer to reject mail from outside sources, we can simply
49 register an auditor on the "user" class that prevents the creation of user
50 nodes with no passwords.
52 Actions
53 -------
54 The subject line of the incoming message is examined to determine whether
55 the message is an attempt to create a new item or to discuss an existing
56 item. A designator enclosed in square brackets is sought as the first thing
57 on the subject line (after skipping any "Fwd:" or "Re:" prefixes).
59 If an item designator (class name and id number) is found there, the newly
60 created "msg" node is added to the "messages" property for that item, and
61 any new "file" nodes are added to the "files" property for the item.
63 If just an item class name is found there, we attempt to create a new item
64 of that class with its "messages" property initialized to contain the new
65 "msg" node and its "files" property initialized to contain any new "file"
66 nodes.
68 Triggers
69 --------
70 Both cases may trigger detectors (in the first case we are calling the
71 set() method to add the message to the item's spool; in the second case we
72 are calling the create() method to create a new node). If an auditor raises
73 an exception, the original message is bounced back to the sender with the
74 explanatory message given in the exception.
76 $Id: mailgw.py,v 1.139 2003-12-04 23:34:25 richard Exp $
77 """
79 import string, re, os, mimetools, cStringIO, smtplib, socket, binascii, quopri
80 import time, random, sys
81 import traceback, MimeWriter, rfc822
83 from roundup import hyperdb, date, password, rfc2822
84 from roundup.mailer import Mailer
86 SENDMAILDEBUG = os.environ.get('SENDMAILDEBUG', '')
88 class MailGWError(ValueError):
89 pass
91 class MailUsageError(ValueError):
92 pass
94 class MailUsageHelp(Exception):
95 """ We need to send the help message to the user. """
96 pass
98 class Unauthorized(Exception):
99 """ Access denied """
100 pass
102 class IgnoreMessage(Exception):
103 """ A general class of message that we should ignore. """
104 pass
105 class IgnoreBulk(IgnoreMessage):
106 """ This is email from a mailing list or from a vacation program. """
107 pass
108 class IgnoreLoop(IgnoreMessage):
109 """ We've seen this message before... """
110 pass
112 def initialiseSecurity(security):
113 ''' Create some Permissions and Roles on the security object
115 This function is directly invoked by security.Security.__init__()
116 as a part of the Security object instantiation.
117 '''
118 security.addPermission(name="Email Registration",
119 description="Anonymous may register through e-mail")
120 p = security.addPermission(name="Email Access",
121 description="User may use the email interface")
122 security.addPermissionToRole('Admin', p)
124 def getparam(str, param):
125 ''' From the rfc822 "header" string, extract "param" if it appears.
126 '''
127 if ';' not in str:
128 return None
129 str = str[str.index(';'):]
130 while str[:1] == ';':
131 str = str[1:]
132 if ';' in str:
133 # XXX Should parse quotes!
134 end = str.index(';')
135 else:
136 end = len(str)
137 f = str[:end]
138 if '=' in f:
139 i = f.index('=')
140 if f[:i].strip().lower() == param:
141 return rfc822.unquote(f[i+1:].strip())
142 return None
144 class Message(mimetools.Message):
145 ''' subclass mimetools.Message so we can retrieve the parts of the
146 message...
147 '''
148 def getPart(self):
149 ''' Get a single part of a multipart message and return it as a new
150 Message instance.
151 '''
152 boundary = self.getparam('boundary')
153 mid, end = '--'+boundary, '--'+boundary+'--'
154 s = cStringIO.StringIO()
155 while 1:
156 line = self.fp.readline()
157 if not line:
158 break
159 if line.strip() in (mid, end):
160 break
161 s.write(line)
162 if not s.getvalue().strip():
163 return None
164 s.seek(0)
165 return Message(s)
167 def getheader(self, name, default=None):
168 hdr = mimetools.Message.getheader(self, name, default)
169 if hdr:
170 hdr = hdr.replace('\n','') # Inserted by rfc822.readheaders
171 return rfc2822.decode_header(hdr)
173 class MailGW:
175 # Matches subjects like:
176 # Re: "[issue1234] title of issue [status=resolved]"
177 subject_re = re.compile(r'''
178 (?P<refwd>\s*\W?\s*(fw|fwd|re|aw)\W\s*)*\s* # Re:
179 (?P<quote>")? # Leading "
180 (\[(?P<classname>[^\d\s]+) # [issue..
181 (?P<nodeid>\d+)? # ..1234]
182 \])?\s*
183 (?P<title>[^[]+)? # issue title
184 "? # Trailing "
185 (\[(?P<args>.+?)\])? # [prop=value]
186 ''', re.IGNORECASE|re.VERBOSE)
188 def __init__(self, instance, db, arguments={}):
189 self.instance = instance
190 self.db = db
191 self.arguments = arguments
192 self.mailer = Mailer(instance.config)
194 # should we trap exceptions (normal usage) or pass them through
195 # (for testing)
196 self.trapExceptions = 1
198 def do_pipe(self):
199 """ Read a message from standard input and pass it to the mail handler.
201 Read into an internal structure that we can seek on (in case
202 there's an error).
204 XXX: we may want to read this into a temporary file instead...
205 """
206 s = cStringIO.StringIO()
207 s.write(sys.stdin.read())
208 s.seek(0)
209 self.main(s)
210 return 0
212 def do_mailbox(self, filename):
213 """ Read a series of messages from the specified unix mailbox file and
214 pass each to the mail handler.
215 """
216 # open the spool file and lock it
217 import fcntl
218 # FCNTL is deprecated in py2.3 and fcntl takes over all the symbols
219 if hasattr(fcntl, 'LOCK_EX'):
220 FCNTL = fcntl
221 else:
222 import FCNTL
223 f = open(filename, 'r+')
224 fcntl.flock(f.fileno(), FCNTL.LOCK_EX)
226 # handle and clear the mailbox
227 try:
228 from mailbox import UnixMailbox
229 mailbox = UnixMailbox(f, factory=Message)
230 # grab one message
231 message = mailbox.next()
232 while message:
233 # handle this message
234 self.handle_Message(message)
235 message = mailbox.next()
236 # nuke the file contents
237 os.ftruncate(f.fileno(), 0)
238 except:
239 import traceback
240 traceback.print_exc()
241 return 1
242 fcntl.flock(f.fileno(), FCNTL.LOCK_UN)
243 return 0
245 def do_apop(self, server, user='', password=''):
246 ''' Do authentication POP
247 '''
248 self.do_pop(server, user, password, apop=1)
250 def do_pop(self, server, user='', password='', apop=0):
251 '''Read a series of messages from the specified POP server.
252 '''
253 import getpass, poplib, socket
254 try:
255 if not user:
256 user = raw_input(_('User: '))
257 if not password:
258 password = getpass.getpass()
259 except (KeyboardInterrupt, EOFError):
260 # Ctrl C or D maybe also Ctrl Z under Windows.
261 print "\nAborted by user."
262 return 1
264 # open a connection to the server and retrieve all messages
265 try:
266 server = poplib.POP3(server)
267 except socket.error, message:
268 print "POP server error:", message
269 return 1
270 if apop:
271 server.apop(user, password)
272 else:
273 server.user(user)
274 server.pass_(password)
275 numMessages = len(server.list()[1])
276 for i in range(1, numMessages+1):
277 # retr: returns
278 # [ pop response e.g. '+OK 459 octets',
279 # [ array of message lines ],
280 # number of octets ]
281 lines = server.retr(i)[1]
282 s = cStringIO.StringIO('\n'.join(lines))
283 s.seek(0)
284 self.handle_Message(Message(s))
285 # delete the message
286 server.dele(i)
288 # quit the server to commit changes.
289 server.quit()
290 return 0
292 def main(self, fp):
293 ''' fp - the file from which to read the Message.
294 '''
295 return self.handle_Message(Message(fp))
297 def handle_Message(self, message):
298 """Handle an RFC822 Message
300 Handle the Message object by calling handle_message() and then cope
301 with any errors raised by handle_message.
302 This method's job is to make that call and handle any
303 errors in a sane manner. It should be replaced if you wish to
304 handle errors in a different manner.
305 """
306 # in some rare cases, a particularly stuffed-up e-mail will make
307 # its way into here... try to handle it gracefully
308 sendto = message.getaddrlist('resent-from')
309 if not sendto:
310 sendto = message.getaddrlist('from')
311 if not sendto:
312 # very bad-looking message - we don't even know who sent it
313 # XXX we should use a log file here...
314 sendto = [self.instance.config.ADMIN_EMAIL]
315 m = ['Subject: badly formed message from mail gateway']
316 m.append('')
317 m.append('The mail gateway retrieved a message which has no From:')
318 m.append('line, indicating that it is corrupt. Please check your')
319 m.append('mail gateway source. Failed message is attached.')
320 m.append('')
321 self.mailer.bounce_message(message, sendto, m,
322 subject='Badly formed message from mail gateway')
323 return
325 # try normal message-handling
326 if not self.trapExceptions:
327 return self.handle_message(message)
328 try:
329 return self.handle_message(message)
330 except MailUsageHelp:
331 # bounce the message back to the sender with the usage message
332 fulldoc = '\n'.join(string.split(__doc__, '\n')[2:])
333 sendto = [sendto[0][1]]
334 m = ['']
335 m.append('\n\nMail Gateway Help\n=================')
336 m.append(fulldoc)
337 self.mailer.bounce_message(message, sendto, m,
338 subject="Mail Gateway Help")
339 except MailUsageError, value:
340 # bounce the message back to the sender with the usage message
341 fulldoc = '\n'.join(string.split(__doc__, '\n')[2:])
342 sendto = [sendto[0][1]]
343 m = ['']
344 m.append(str(value))
345 m.append('\n\nMail Gateway Help\n=================')
346 m.append(fulldoc)
347 self.mailer.bounce_message(message, sendto, m)
348 except Unauthorized, value:
349 # just inform the user that he is not authorized
350 sendto = [sendto[0][1]]
351 m = ['']
352 m.append(str(value))
353 self.mailer.bounce_message(message, sendto, m)
354 except IgnoreMessage:
355 # XXX we should use a log file here...
356 # do not take any action
357 # this exception is thrown when email should be ignored
358 return
359 except:
360 # bounce the message back to the sender with the error message
361 # XXX we should use a log file here...
362 sendto = [sendto[0][1], self.instance.config.ADMIN_EMAIL]
363 m = ['']
364 m.append('An unexpected error occurred during the processing')
365 m.append('of your message. The tracker administrator is being')
366 m.append('notified.\n')
367 m.append('---- traceback of failure ----')
368 s = cStringIO.StringIO()
369 import traceback
370 traceback.print_exc(None, s)
371 m.append(s.getvalue())
372 self.mailer.bounce_message(message, sendto, m)
374 def get_part_data_decoded(self,part):
375 encoding = part.getencoding()
376 data = None
377 if encoding == 'base64':
378 # BUG: is base64 really used for text encoding or
379 # are we inserting zip files here.
380 data = binascii.a2b_base64(part.fp.read())
381 elif encoding == 'quoted-printable':
382 # the quopri module wants to work with files
383 decoded = cStringIO.StringIO()
384 quopri.decode(part.fp, decoded)
385 data = decoded.getvalue()
386 elif encoding == 'uuencoded':
387 data = binascii.a2b_uu(part.fp.read())
388 else:
389 # take it as text
390 data = part.fp.read()
392 # Encode message to unicode
393 charset = rfc2822.unaliasCharset(part.getparam("charset"))
394 if charset:
395 # Do conversion only if charset specified
396 edata = unicode(data, charset).encode('utf-8')
397 # Convert from dos eol to unix
398 edata = edata.replace('\r\n', '\n')
399 else:
400 # Leave message content as is
401 edata = data
403 return edata
405 def handle_message(self, message):
406 ''' message - a Message instance
408 Parse the message as per the module docstring.
409 '''
410 # detect loops
411 if message.getheader('x-roundup-loop', ''):
412 raise IgnoreLoop
414 # detect Precedence: Bulk
415 if (message.getheader('precedence', '') == 'bulk'):
416 raise IgnoreBulk
418 # XXX Don't enable. This doesn't work yet.
419 # "[^A-z.]tracker\+(?P<classname>[^\d\s]+)(?P<nodeid>\d+)\@some.dom.ain[^A-z.]"
420 # handle delivery to addresses like:tracker+issue25@some.dom.ain
421 # use the embedded issue number as our issue
422 # if hasattr(self.instance.config, 'EMAIL_ISSUE_ADDRESS_RE') and \
423 # self.instance.config.EMAIL_ISSUE_ADDRESS_RE:
424 # issue_re = self.instance.config.EMAIL_ISSUE_ADDRESS_RE
425 # for header in ['to', 'cc', 'bcc']:
426 # addresses = message.getheader(header, '')
427 # if addresses:
428 # # FIXME, this only finds the first match in the addresses.
429 # issue = re.search(issue_re, addresses, 'i')
430 # if issue:
431 # classname = issue.group('classname')
432 # nodeid = issue.group('nodeid')
433 # break
435 # determine the sender's address
436 from_list = message.getaddrlist('resent-from')
437 if not from_list:
438 from_list = message.getaddrlist('from')
440 # handle the subject line
441 subject = message.getheader('subject', '')
443 if not subject:
444 raise MailUsageError, '''
445 Emails to Roundup trackers must include a Subject: line!
446 '''
448 if subject.strip().lower() == 'help':
449 raise MailUsageHelp
451 m = self.subject_re.match(subject)
453 # check for well-formed subject line
454 if m:
455 # get the classname
456 classname = m.group('classname')
457 if classname is None:
458 # no classname, check if this a registration confirmation email
459 # or fallback on the default class
460 otk_re = re.compile('-- key (?P<otk>[a-zA-Z0-9]{32})')
461 otk = otk_re.search(m.group('title'))
462 if otk:
463 self.db.confirm_registration(otk.group('otk'))
464 subject = 'Your registration to %s is complete' % \
465 self.instance.config.TRACKER_NAME
466 sendto = [from_list[0][1]]
467 self.mailer.standard_message(sendto, subject, '')
468 return
469 elif hasattr(self.instance.config, 'MAIL_DEFAULT_CLASS') and \
470 self.instance.config.MAIL_DEFAULT_CLASS:
471 classname = self.instance.config.MAIL_DEFAULT_CLASS
472 else:
473 # fail
474 m = None
476 if not m:
477 raise MailUsageError, """
478 The message you sent to roundup did not contain a properly formed subject
479 line. The subject must contain a class name or designator to indicate the
480 'topic' of the message. For example:
481 Subject: [issue] This is a new issue
482 - this will create a new issue in the tracker with the title 'This is
483 a new issue'.
484 Subject: [issue1234] This is a followup to issue 1234
485 - this will append the message's contents to the existing issue 1234
486 in the tracker.
488 Subject was: '%s'
489 """%subject
491 # get the class
492 try:
493 cl = self.db.getclass(classname)
494 except KeyError:
495 raise MailUsageError, '''
496 The class name you identified in the subject line ("%s") does not exist in the
497 database.
499 Valid class names are: %s
500 Subject was: "%s"
501 '''%(classname, ', '.join(self.db.getclasses()), subject)
503 # get the optional nodeid
504 nodeid = m.group('nodeid')
506 # title is optional too
507 title = m.group('title')
508 if title:
509 title = title.strip()
510 else:
511 title = ''
513 # strip off the quotes that dumb emailers put around the subject, like
514 # Re: "[issue1] bla blah"
515 if m.group('quote') and title.endswith('"'):
516 title = title[:-1]
518 # but we do need either a title or a nodeid...
519 if nodeid is None and not title:
520 raise MailUsageError, '''
521 I cannot match your message to a node in the database - you need to either
522 supply a full node identifier (with number, eg "[issue123]" or keep the
523 previous subject title intact so I can match that.
525 Subject was: "%s"
526 '''%subject
528 # If there's no nodeid, check to see if this is a followup and
529 # maybe someone's responded to the initial mail that created an
530 # entry. Try to find the matching nodes with the same title, and
531 # use the _last_ one matched (since that'll _usually_ be the most
532 # recent...)
533 if nodeid is None and m.group('refwd'):
534 l = cl.stringFind(title=title)
535 if l:
536 nodeid = l[-1]
538 # if a nodeid was specified, make sure it's valid
539 if nodeid is not None and not cl.hasnode(nodeid):
540 raise MailUsageError, '''
541 The node specified by the designator in the subject of your message ("%s")
542 does not exist.
544 Subject was: "%s"
545 '''%(nodeid, subject)
547 # Handle the arguments specified by the email gateway command line.
548 # We do this by looping over the list of self.arguments looking for
549 # a -C to tell us what class then the -S setting string.
550 msg_props = {}
551 user_props = {}
552 file_props = {}
553 issue_props = {}
554 # so, if we have any arguments, use them
555 if self.arguments:
556 current_class = 'msg'
557 for option, propstring in self.arguments:
558 if option in ( '-C', '--class'):
559 current_class = propstring.strip()
560 if current_class not in ('msg', 'file', 'user', 'issue'):
561 raise MailUsageError, '''
562 The mail gateway is not properly set up. Please contact
563 %s and have them fix the incorrect class specified as:
564 %s
565 '''%(self.instance.config.ADMIN_EMAIL, current_class)
566 if option in ('-S', '--set'):
567 if current_class == 'issue' :
568 errors, issue_props = setPropArrayFromString(self,
569 cl, propstring.strip(), nodeid)
570 elif current_class == 'file' :
571 temp_cl = self.db.getclass('file')
572 errors, file_props = setPropArrayFromString(self,
573 temp_cl, propstring.strip())
574 elif current_class == 'msg' :
575 temp_cl = self.db.getclass('msg')
576 errors, msg_props = setPropArrayFromString(self,
577 temp_cl, propstring.strip())
578 elif current_class == 'user' :
579 temp_cl = self.db.getclass('user')
580 errors, user_props = setPropArrayFromString(self,
581 temp_cl, propstring.strip())
582 if errors:
583 raise MailUsageError, '''
584 The mail gateway is not properly set up. Please contact
585 %s and have them fix the incorrect properties:
586 %s
587 '''%(self.instance.config.ADMIN_EMAIL, errors)
589 #
590 # handle the users
591 #
592 # Don't create users if anonymous isn't allowed to register
593 create = 1
594 anonid = self.db.user.lookup('anonymous')
595 if not self.db.security.hasPermission('Email Registration', anonid):
596 create = 0
598 # ok, now figure out who the author is - create a new user if the
599 # "create" flag is true
600 author = uidFromAddress(self.db, from_list[0], create=create)
602 # if we're not recognised, and we don't get added as a user, then we
603 # must be anonymous
604 if not author:
605 author = anonid
607 # make sure the author has permission to use the email interface
608 if not self.db.security.hasPermission('Email Access', author):
609 if author == anonid:
610 # we're anonymous and we need to be a registered user
611 raise Unauthorized, '''
612 You are not a registered user.
614 Unknown address: %s
615 '''%from_list[0][1]
616 else:
617 # we're registered and we're _still_ not allowed access
618 raise Unauthorized, 'You are not permitted to access '\
619 'this tracker.'
621 # make sure they're allowed to edit this class of information
622 if not self.db.security.hasPermission('Edit', author, classname):
623 raise Unauthorized, 'You are not permitted to edit %s.'%classname
625 # the author may have been created - make sure the change is
626 # committed before we reopen the database
627 self.db.commit()
629 # reopen the database as the author
630 username = self.db.user.get(author, 'username')
631 self.db.close()
632 self.db = self.instance.open(username)
634 # re-get the class with the new database connection
635 cl = self.db.getclass(classname)
637 # now update the recipients list
638 recipients = []
639 tracker_email = self.instance.config.TRACKER_EMAIL.lower()
640 for recipient in message.getaddrlist('to') + message.getaddrlist('cc'):
641 r = recipient[1].strip().lower()
642 if r == tracker_email or not r:
643 continue
645 # look up the recipient - create if necessary (and we're
646 # allowed to)
647 recipient = uidFromAddress(self.db, recipient, create, **user_props)
649 # if all's well, add the recipient to the list
650 if recipient:
651 recipients.append(recipient)
653 #
654 # handle the subject argument list
655 #
656 # figure what the properties of this Class are
657 properties = cl.getprops()
658 props = {}
659 args = m.group('args')
660 if args:
661 errors, props = setPropArrayFromString(self, cl, args, nodeid)
662 # handle any errors parsing the argument list
663 if errors:
664 errors = '\n- '.join(errors)
665 raise MailUsageError, '''
666 There were problems handling your subject line argument list:
667 - %s
669 Subject was: "%s"
670 '''%(errors, subject)
673 # set the issue title to the subject
674 if properties.has_key('title') and not issue_props.has_key('title'):
675 issue_props['title'] = title.strip()
677 #
678 # handle message-id and in-reply-to
679 #
680 messageid = message.getheader('message-id')
681 inreplyto = message.getheader('in-reply-to') or ''
682 # generate a messageid if there isn't one
683 if not messageid:
684 messageid = "<%s.%s.%s%s@%s>"%(time.time(), random.random(),
685 classname, nodeid, self.instance.config.MAIL_DOMAIN)
687 #
688 # now handle the body - find the message
689 #
690 content_type = message.gettype()
691 attachments = []
692 # General multipart handling:
693 # Take the first text/plain part, anything else is considered an
694 # attachment.
695 # multipart/mixed: multiple "unrelated" parts.
696 # multipart/signed (rfc 1847):
697 # The control information is carried in the second of the two
698 # required body parts.
699 # ACTION: Default, so if content is text/plain we get it.
700 # multipart/encrypted (rfc 1847):
701 # The control information is carried in the first of the two
702 # required body parts.
703 # ACTION: Not handleable as the content is encrypted.
704 # multipart/related (rfc 1872, 2112, 2387):
705 # The Multipart/Related content-type addresses the MIME
706 # representation of compound objects.
707 # ACTION: Default. If we are lucky there is a text/plain.
708 # TODO: One should use the start part and look for an Alternative
709 # that is text/plain.
710 # multipart/Alternative (rfc 1872, 1892):
711 # only in "related" ?
712 # multipart/report (rfc 1892):
713 # e.g. mail system delivery status reports.
714 # ACTION: Default. Could be ignored or used for Delivery Notification
715 # flagging.
716 # multipart/form-data:
717 # For web forms only.
718 if content_type == 'multipart/mixed':
719 # skip over the intro to the first boundary
720 part = message.getPart()
721 content = None
722 while 1:
723 # get the next part
724 part = message.getPart()
725 if part is None:
726 break
727 # parse it
728 subtype = part.gettype()
729 if subtype == 'text/plain' and not content:
730 # The first text/plain part is the message content.
731 content = self.get_part_data_decoded(part)
732 elif subtype == 'message/rfc822':
733 # handle message/rfc822 specially - the name should be
734 # the subject of the actual e-mail embedded here
735 i = part.fp.tell()
736 mailmess = Message(part.fp)
737 name = mailmess.getheader('subject')
738 part.fp.seek(i)
739 attachments.append((name, 'message/rfc822', part.fp.read()))
740 elif subtype == 'multipart/alternative':
741 # Search for text/plain in message with attachment and
742 # alternative text representation
743 # skip over intro to first boundary
744 part.getPart()
745 while 1:
746 # get the next part
747 subpart = part.getPart()
748 if subpart is None:
749 break
750 # parse it
751 if subpart.gettype() == 'text/plain' and not content:
752 content = self.get_part_data_decoded(subpart)
753 else:
754 # try name on Content-Type
755 name = part.getparam('name')
756 if name:
757 name = name.strip()
758 if not name:
759 disp = part.getheader('content-disposition', None)
760 if disp:
761 name = getparam(disp, 'filename')
762 if name:
763 name = name.strip()
764 # this is just an attachment
765 data = self.get_part_data_decoded(part)
766 attachments.append((name, part.gettype(), data))
767 if content is None:
768 raise MailUsageError, '''
769 Roundup requires the submission to be plain text. The message parser could
770 not find a text/plain part to use.
771 '''
773 elif content_type[:10] == 'multipart/':
774 # skip over the intro to the first boundary
775 message.getPart()
776 content = None
777 while 1:
778 # get the next part
779 part = message.getPart()
780 if part is None:
781 break
782 # parse it
783 if part.gettype() == 'text/plain' and not content:
784 content = self.get_part_data_decoded(part)
785 if content is None:
786 raise MailUsageError, '''
787 Roundup requires the submission to be plain text. The message parser could
788 not find a text/plain part to use.
789 '''
791 elif content_type != 'text/plain':
792 raise MailUsageError, '''
793 Roundup requires the submission to be plain text. The message parser could
794 not find a text/plain part to use.
795 '''
797 else:
798 content = self.get_part_data_decoded(message)
800 # figure how much we should muck around with the email body
801 keep_citations = getattr(self.instance.config, 'EMAIL_KEEP_QUOTED_TEXT',
802 'no') == 'yes'
803 keep_body = getattr(self.instance.config, 'EMAIL_LEAVE_BODY_UNCHANGED',
804 'no') == 'yes'
806 # parse the body of the message, stripping out bits as appropriate
807 summary, content = parseContent(content, keep_citations,
808 keep_body)
809 content = content.strip()
811 #
812 # handle the attachments
813 #
814 if properties.has_key('files'):
815 files = []
816 for (name, mime_type, data) in attachments:
817 if not name:
818 name = "unnamed"
819 files.append(self.db.file.create(type=mime_type, name=name,
820 content=data, **file_props))
821 # attach the files to the issue
822 if nodeid:
823 # extend the existing files list
824 fileprop = cl.get(nodeid, 'files')
825 fileprop.extend(files)
826 props['files'] = fileprop
827 else:
828 # pre-load the files list
829 props['files'] = files
831 #
832 # create the message if there's a message body (content)
833 #
834 if (content and properties.has_key('messages')):
835 message_id = self.db.msg.create(author=author,
836 recipients=recipients, date=date.Date('.'), summary=summary,
837 content=content, files=files, messageid=messageid,
838 inreplyto=inreplyto, **msg_props)
840 # attach the message to the node
841 if nodeid:
842 # add the message to the node's list
843 messages = cl.get(nodeid, 'messages')
844 messages.append(message_id)
845 props['messages'] = messages
846 else:
847 # pre-load the messages list
848 props['messages'] = [message_id]
850 #
851 # perform the node change / create
852 #
853 try:
854 # merge the command line props defined in issue_props into
855 # the props dictionary because function(**props, **issue_props)
856 # is a syntax error.
857 for prop in issue_props.keys() :
858 if not props.has_key(prop) :
859 props[prop] = issue_props[prop]
860 if nodeid:
861 cl.set(nodeid, **props)
862 else:
863 nodeid = cl.create(**props)
864 except (TypeError, IndexError, ValueError), message:
865 raise MailUsageError, '''
866 There was a problem with the message you sent:
867 %s
868 '''%message
870 # commit the changes to the DB
871 self.db.commit()
873 return nodeid
876 def setPropArrayFromString(self, cl, propString, nodeid=None):
877 ''' takes string of form prop=value,value;prop2=value
878 and returns (error, prop[..])
879 '''
880 props = {}
881 errors = []
882 for prop in string.split(propString, ';'):
883 # extract the property name and value
884 try:
885 propname, value = prop.split('=')
886 except ValueError, message:
887 errors.append('not of form [arg=value,value,...;'
888 'arg=value,value,...]')
889 return (errors, props)
890 # convert the value to a hyperdb-usable value
891 propname = propname.strip()
892 try:
893 props[propname] = hyperdb.rawToHyperdb(self.db, cl, nodeid,
894 propname, value)
895 except hyperdb.HyperdbValueError, message:
896 errors.append(message)
897 return errors, props
900 def extractUserFromList(userClass, users):
901 '''Given a list of users, try to extract the first non-anonymous user
902 and return that user, otherwise return None
903 '''
904 if len(users) > 1:
905 for user in users:
906 # make sure we don't match the anonymous or admin user
907 if userClass.get(user, 'username') in ('admin', 'anonymous'):
908 continue
909 # first valid match will do
910 return user
911 # well, I guess we have no choice
912 return user[0]
913 elif users:
914 return users[0]
915 return None
918 def uidFromAddress(db, address, create=1, **user_props):
919 ''' address is from the rfc822 module, and therefore is (name, addr)
921 user is created if they don't exist in the db already
922 user_props may supply additional user information
923 '''
924 (realname, address) = address
926 # try a straight match of the address
927 user = extractUserFromList(db.user, db.user.stringFind(address=address))
928 if user is not None:
929 return user
931 # try the user alternate addresses if possible
932 props = db.user.getprops()
933 if props.has_key('alternate_addresses'):
934 users = db.user.filter(None, {'alternate_addresses': address})
935 user = extractUserFromList(db.user, users)
936 if user is not None:
937 return user
939 # try to match the username to the address (for local
940 # submissions where the address is empty)
941 user = extractUserFromList(db.user, db.user.stringFind(username=address))
943 # couldn't match address or username, so create a new user
944 if create:
945 # generate a username
946 if '@' in address:
947 username = address.split('@')[0]
948 else:
949 username = address
950 trying = username
951 n = 0
952 while 1:
953 try:
954 # does this username exist already?
955 db.user.lookup(trying)
956 except KeyError:
957 break
958 n += 1
959 trying = username + str(n)
961 # create!
962 return db.user.create(username=trying, address=address,
963 realname=realname, roles=db.config.NEW_EMAIL_USER_ROLES,
964 password=password.Password(password.generatePassword()),
965 **user_props)
966 else:
967 return 0
970 def parseContent(content, keep_citations, keep_body,
971 blank_line=re.compile(r'[\r\n]+\s*[\r\n]+'),
972 eol=re.compile(r'[\r\n]+'),
973 signature=re.compile(r'^[>|\s]*-- ?$'),
974 original_msg=re.compile(r'^[>|\s]*-----\s?Original Message\s?-----$')):
975 ''' The message body is divided into sections by blank lines.
976 Sections where the second and all subsequent lines begin with a ">"
977 or "|" character are considered "quoting sections". The first line of
978 the first non-quoting section becomes the summary of the message.
980 If keep_citations is true, then we keep the "quoting sections" in the
981 content.
982 If keep_body is true, we even keep the signature sections.
983 '''
984 # strip off leading carriage-returns / newlines
985 i = 0
986 for i in range(len(content)):
987 if content[i] not in '\r\n':
988 break
989 if i > 0:
990 sections = blank_line.split(content[i:])
991 else:
992 sections = blank_line.split(content)
994 # extract out the summary from the message
995 summary = ''
996 l = []
997 for section in sections:
998 #section = section.strip()
999 if not section:
1000 continue
1001 lines = eol.split(section)
1002 if (lines[0] and lines[0][0] in '>|') or (len(lines) > 1 and
1003 lines[1] and lines[1][0] in '>|'):
1004 # see if there's a response somewhere inside this section (ie.
1005 # no blank line between quoted message and response)
1006 for line in lines[1:]:
1007 if line and line[0] not in '>|':
1008 break
1009 else:
1010 # we keep quoted bits if specified in the config
1011 if keep_citations:
1012 l.append(section)
1013 continue
1014 # keep this section - it has reponse stuff in it
1015 lines = lines[lines.index(line):]
1016 section = '\n'.join(lines)
1017 # and while we're at it, use the first non-quoted bit as
1018 # our summary
1019 summary = section
1021 if not summary:
1022 # if we don't have our summary yet use the first line of this
1023 # section
1024 summary = section
1025 elif signature.match(lines[0]) and 2 <= len(lines) <= 10:
1026 # lose any signature
1027 break
1028 elif original_msg.match(lines[0]):
1029 # ditch the stupid Outlook quoting of the entire original message
1030 break
1032 # and add the section to the output
1033 l.append(section)
1035 # figure the summary - find the first sentence-ending punctuation or the
1036 # first whole line, whichever is longest
1037 sentence = re.search(r'^([^!?\.]+[!?\.])', summary)
1038 if sentence:
1039 sentence = sentence.group(1)
1040 else:
1041 sentence = ''
1042 first = eol.split(summary)[0]
1043 summary = max(sentence, first)
1045 # Now reconstitute the message content minus the bits we don't care
1046 # about.
1047 if not keep_body:
1048 content = '\n\n'.join(l)
1050 return summary, content
1052 # vim: set filetype=python ts=4 sw=4 et si