53dc0417fbd433ffececfa1765b58f530c28ab07
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
19 '''
20 An e-mail gateway for Roundup.
22 Incoming messages are examined for multiple parts:
23 . In a multipart/mixed message or part, each subpart is extracted and
24 examined. The text/plain subparts are assembled to form the textual
25 body of the message, to be stored in the file associated with a "msg"
26 class node. Any parts of other types are each stored in separate files
27 and given "file" class nodes that are linked to the "msg" node.
28 . In a multipart/alternative message or part, we look for a text/plain
29 subpart and ignore the other parts.
31 Summary
32 -------
33 The "summary" property on message nodes is taken from the first non-quoting
34 section in the message body. The message body is divided into sections by
35 blank lines. Sections where the second and all subsequent lines begin with
36 a ">" or "|" character are considered "quoting sections". The first line of
37 the first non-quoting section becomes the summary of the message.
39 Addresses
40 ---------
41 All of the addresses in the To: and Cc: headers of the incoming message are
42 looked up among the user nodes, and the corresponding users are placed in
43 the "recipients" property on the new "msg" node. The address in the From:
44 header similarly determines the "author" property of the new "msg"
45 node. The default handling for addresses that don't have corresponding
46 users is to create new users with no passwords and a username equal to the
47 address. (The web interface does not permit logins for users with no
48 passwords.) If we prefer to reject mail from outside sources, we can simply
49 register an auditor on the "user" class that prevents the creation of user
50 nodes with no passwords.
52 Actions
53 -------
54 The subject line of the incoming message is examined to determine whether
55 the message is an attempt to create a new item or to discuss an existing
56 item. A designator enclosed in square brackets is sought as the first thing
57 on the subject line (after skipping any "Fwd:" or "Re:" prefixes).
59 If an item designator (class name and id number) is found there, the newly
60 created "msg" node is added to the "messages" property for that item, and
61 any new "file" nodes are added to the "files" property for the item.
63 If just an item class name is found there, we attempt to create a new item
64 of that class with its "messages" property initialized to contain the new
65 "msg" node and its "files" property initialized to contain any new "file"
66 nodes.
68 Triggers
69 --------
70 Both cases may trigger detectors (in the first case we are calling the
71 set() method to add the message to the item's spool; in the second case we
72 are calling the create() method to create a new node). If an auditor raises
73 an exception, the original message is bounced back to the sender with the
74 explanatory message given in the exception.
76 $Id: mailgw.py,v 1.111 2003-02-27 05:43:01 richard Exp $
77 '''
79 import string, re, os, mimetools, cStringIO, smtplib, socket, binascii, quopri
80 import time, random, sys
81 import traceback, MimeWriter, rfc822
82 import hyperdb, date, password
84 import rfc2822
86 SENDMAILDEBUG = os.environ.get('SENDMAILDEBUG', '')
88 class MailGWError(ValueError):
89 pass
91 class MailUsageError(ValueError):
92 pass
94 class MailUsageHelp(Exception):
95 pass
97 class MailLoop(Exception):
98 ''' We've seen this message before... '''
99 pass
101 class Unauthorized(Exception):
102 """ Access denied """
104 def initialiseSecurity(security):
105 ''' Create some Permissions and Roles on the security object
107 This function is directly invoked by security.Security.__init__()
108 as a part of the Security object instantiation.
109 '''
110 security.addPermission(name="Email Registration",
111 description="Anonymous may register through e-mail")
112 p = security.addPermission(name="Email Access",
113 description="User may use the email interface")
114 security.addPermissionToRole('Admin', p)
116 def getparam(str, param):
117 ''' From the rfc822 "header" string, extract "param" if it appears.
118 '''
119 if ';' not in str:
120 return None
121 str = str[str.index(';'):]
122 while str[:1] == ';':
123 str = str[1:]
124 if ';' in str:
125 # XXX Should parse quotes!
126 end = str.index(';')
127 else:
128 end = len(str)
129 f = str[:end]
130 if '=' in f:
131 i = f.index('=')
132 if f[:i].strip().lower() == param:
133 return rfc822.unquote(f[i+1:].strip())
134 return None
136 class Message(mimetools.Message):
137 ''' subclass mimetools.Message so we can retrieve the parts of the
138 message...
139 '''
140 def getPart(self):
141 ''' Get a single part of a multipart message and return it as a new
142 Message instance.
143 '''
144 boundary = self.getparam('boundary')
145 mid, end = '--'+boundary, '--'+boundary+'--'
146 s = cStringIO.StringIO()
147 while 1:
148 line = self.fp.readline()
149 if not line:
150 break
151 if line.strip() in (mid, end):
152 break
153 s.write(line)
154 if not s.getvalue().strip():
155 return None
156 s.seek(0)
157 return Message(s)
159 def getheader(self, name, default=None):
160 hdr = mimetools.Message.getheader(self, name, default)
161 return rfc2822.decode_header(hdr)
163 subject_re = re.compile(r'(?P<refwd>\s*\W?\s*(fw|fwd|re|aw)\W\s*)*'
164 r'\s*(?P<quote>")?(\[(?P<classname>[^\d\s]+)(?P<nodeid>\d+)?\])?'
165 r'\s*(?P<title>[^[]+)?"?(\[(?P<args>.+?)\])?', re.I)
167 class MailGW:
168 def __init__(self, instance, db, arguments={}):
169 self.instance = instance
170 self.db = db
171 self.arguments = arguments
173 # should we trap exceptions (normal usage) or pass them through
174 # (for testing)
175 self.trapExceptions = 1
177 def do_pipe(self):
178 ''' Read a message from standard input and pass it to the mail handler.
180 Read into an internal structure that we can seek on (in case
181 there's an error).
183 XXX: we may want to read this into a temporary file instead...
184 '''
185 s = cStringIO.StringIO()
186 s.write(sys.stdin.read())
187 s.seek(0)
188 self.main(s)
189 return 0
191 def do_mailbox(self, filename):
192 ''' Read a series of messages from the specified unix mailbox file and
193 pass each to the mail handler.
194 '''
195 # open the spool file and lock it
196 import fcntl, FCNTL
197 f = open(filename, 'r+')
198 fcntl.flock(f.fileno(), FCNTL.LOCK_EX)
200 # handle and clear the mailbox
201 try:
202 from mailbox import UnixMailbox
203 mailbox = UnixMailbox(f, factory=Message)
204 # grab one message
205 message = mailbox.next()
206 while message:
207 # handle this message
208 self.handle_Message(message)
209 message = mailbox.next()
210 # nuke the file contents
211 os.ftruncate(f.fileno(), 0)
212 except:
213 import traceback
214 traceback.print_exc()
215 return 1
216 fcntl.flock(f.fileno(), FCNTL.LOCK_UN)
217 return 0
219 def do_pop(self, server, user='', password=''):
220 '''Read a series of messages from the specified POP server.
221 '''
222 import getpass, poplib, socket
223 try:
224 if not user:
225 user = raw_input(_('User: '))
226 if not password:
227 password = getpass.getpass()
228 except (KeyboardInterrupt, EOFError):
229 # Ctrl C or D maybe also Ctrl Z under Windows.
230 print "\nAborted by user."
231 return 1
233 # open a connection to the server and retrieve all messages
234 try:
235 server = poplib.POP3(server)
236 except socket.error, message:
237 print "POP server error:", message
238 return 1
239 server.user(user)
240 server.pass_(password)
241 numMessages = len(server.list()[1])
242 for i in range(1, numMessages+1):
243 # retr: returns
244 # [ pop response e.g. '+OK 459 octets',
245 # [ array of message lines ],
246 # number of octets ]
247 lines = server.retr(i)[1]
248 s = cStringIO.StringIO('\n'.join(lines))
249 s.seek(0)
250 self.handle_Message(Message(s))
251 # delete the message
252 server.dele(i)
254 # quit the server to commit changes.
255 server.quit()
256 return 0
258 def main(self, fp):
259 ''' fp - the file from which to read the Message.
260 '''
261 return self.handle_Message(Message(fp))
263 def handle_Message(self, message):
264 '''Handle an RFC822 Message
266 Handle the Message object by calling handle_message() and then cope
267 with any errors raised by handle_message.
268 This method's job is to make that call and handle any
269 errors in a sane manner. It should be replaced if you wish to
270 handle errors in a different manner.
271 '''
272 # in some rare cases, a particularly stuffed-up e-mail will make
273 # its way into here... try to handle it gracefully
274 sendto = message.getaddrlist('from')
275 if sendto:
276 if not self.trapExceptions:
277 return self.handle_message(message)
278 try:
279 return self.handle_message(message)
280 except MailUsageHelp:
281 # bounce the message back to the sender with the usage message
282 fulldoc = '\n'.join(string.split(__doc__, '\n')[2:])
283 sendto = [sendto[0][1]]
284 m = ['']
285 m.append('\n\nMail Gateway Help\n=================')
286 m.append(fulldoc)
287 m = self.bounce_message(message, sendto, m,
288 subject="Mail Gateway Help")
289 except MailUsageError, value:
290 # bounce the message back to the sender with the usage message
291 fulldoc = '\n'.join(string.split(__doc__, '\n')[2:])
292 sendto = [sendto[0][1]]
293 m = ['']
294 m.append(str(value))
295 m.append('\n\nMail Gateway Help\n=================')
296 m.append(fulldoc)
297 m = self.bounce_message(message, sendto, m)
298 except Unauthorized, value:
299 # just inform the user that he is not authorized
300 sendto = [sendto[0][1]]
301 m = ['']
302 m.append(str(value))
303 m = self.bounce_message(message, sendto, m)
304 except MailLoop:
305 # XXX we should use a log file here...
306 return
307 except:
308 # bounce the message back to the sender with the error message
309 # XXX we should use a log file here...
310 sendto = [sendto[0][1], self.instance.config.ADMIN_EMAIL]
311 m = ['']
312 m.append('An unexpected error occurred during the processing')
313 m.append('of your message. The tracker administrator is being')
314 m.append('notified.\n')
315 m.append('---- traceback of failure ----')
316 s = cStringIO.StringIO()
317 import traceback
318 traceback.print_exc(None, s)
319 m.append(s.getvalue())
320 m = self.bounce_message(message, sendto, m)
321 else:
322 # very bad-looking message - we don't even know who sent it
323 # XXX we should use a log file here...
324 sendto = [self.instance.config.ADMIN_EMAIL]
325 m = ['Subject: badly formed message from mail gateway']
326 m.append('')
327 m.append('The mail gateway retrieved a message which has no From:')
328 m.append('line, indicating that it is corrupt. Please check your')
329 m.append('mail gateway source. Failed message is attached.')
330 m.append('')
331 m = self.bounce_message(message, sendto, m,
332 subject='Badly formed message from mail gateway')
334 # now send the message
335 if SENDMAILDEBUG:
336 open(SENDMAILDEBUG, 'a').write('From: %s\nTo: %s\n%s\n'%(
337 self.instance.config.ADMIN_EMAIL, ', '.join(sendto),
338 m.getvalue()))
339 else:
340 try:
341 smtp = smtplib.SMTP(self.instance.config.MAILHOST)
342 smtp.sendmail(self.instance.config.ADMIN_EMAIL, sendto,
343 m.getvalue())
344 except socket.error, value:
345 raise MailGWError, "Couldn't send error email: "\
346 "mailhost %s"%value
347 except smtplib.SMTPException, value:
348 raise MailGWError, "Couldn't send error email: %s"%value
350 def bounce_message(self, message, sendto, error,
351 subject='Failed issue tracker submission'):
352 ''' create a message that explains the reason for the failed
353 issue submission to the author and attach the original
354 message.
355 '''
356 msg = cStringIO.StringIO()
357 writer = MimeWriter.MimeWriter(msg)
358 writer.addheader('X-Roundup-Loop', 'hello')
359 writer.addheader('Subject', subject)
360 writer.addheader('From', '%s <%s>'% (self.instance.config.TRACKER_NAME,
361 self.instance.config.TRACKER_EMAIL))
362 writer.addheader('To', ','.join(sendto))
363 writer.addheader('Date', time.strftime("%a, %d %b %Y %H:%M:%S +0000",
364 time.gmtime()))
365 writer.addheader('MIME-Version', '1.0')
366 part = writer.startmultipartbody('mixed')
367 part = writer.nextpart()
368 body = part.startbody('text/plain; charset=utf-8')
369 body.write('\n'.join(error))
371 # attach the original message to the returned message
372 part = writer.nextpart()
373 part.addheader('Content-Disposition','attachment')
374 part.addheader('Content-Description','Message you sent')
375 body = part.startbody('text/plain')
376 for header in message.headers:
377 body.write(header)
378 body.write('\n')
379 try:
380 message.rewindbody()
381 except IOError, message:
382 body.write("*** couldn't include message body: %s ***"%message)
383 else:
384 body.write(message.fp.read())
386 writer.lastpart()
387 return msg
389 def get_part_data_decoded(self,part):
390 encoding = part.getencoding()
391 data = None
392 if encoding == 'base64':
393 # BUG: is base64 really used for text encoding or
394 # are we inserting zip files here.
395 data = binascii.a2b_base64(part.fp.read())
396 elif encoding == 'quoted-printable':
397 # the quopri module wants to work with files
398 decoded = cStringIO.StringIO()
399 quopri.decode(part.fp, decoded)
400 data = decoded.getvalue()
401 elif encoding == 'uuencoded':
402 data = binascii.a2b_uu(part.fp.read())
403 else:
404 # take it as text
405 data = part.fp.read()
407 # Encode message to unicode
408 charset = rfc2822.unaliasCharset(part.getparam("charset"))
409 if charset:
410 # Do conversion only if charset specified
411 edata = unicode(data, charset).encode('utf-8')
412 # Convert from dos eol to unix
413 edata = edata.replace('\r\n', '\n')
414 else:
415 # Leave message content as is
416 edata = data
418 return edata
420 def handle_message(self, message):
421 ''' message - a Message instance
423 Parse the message as per the module docstring.
424 '''
425 # detect loops
426 if message.getheader('x-roundup-loop', ''):
427 raise MailLoop
429 # XXX Don't enable. This doesn't work yet.
430 # "[^A-z.]tracker\+(?P<classname>[^\d\s]+)(?P<nodeid>\d+)\@some.dom.ain[^A-z.]"
431 # handle delivery to addresses like:tracker+issue25@some.dom.ain
432 # use the embedded issue number as our issue
433 # if hasattr(self.instance.config, 'EMAIL_ISSUE_ADDRESS_RE') and \
434 # self.instance.config.EMAIL_ISSUE_ADDRESS_RE:
435 # issue_re = self.instance.config.EMAIL_ISSUE_ADDRESS_RE
436 # for header in ['to', 'cc', 'bcc']:
437 # addresses = message.getheader(header, '')
438 # if addresses:
439 # # FIXME, this only finds the first match in the addresses.
440 # issue = re.search(issue_re, addresses, 'i')
441 # if issue:
442 # classname = issue.group('classname')
443 # nodeid = issue.group('nodeid')
444 # break
446 # handle the subject line
447 subject = message.getheader('subject', '')
449 if subject.strip().lower() == 'help':
450 raise MailUsageHelp
452 m = subject_re.match(subject)
454 # check for well-formed subject line
455 if m:
456 # get the classname
457 classname = m.group('classname')
458 if classname is None:
459 # no classname, fallback on the default
460 if hasattr(self.instance.config, 'MAIL_DEFAULT_CLASS') and \
461 self.instance.config.MAIL_DEFAULT_CLASS:
462 classname = self.instance.config.MAIL_DEFAULT_CLASS
463 else:
464 # fail
465 m = None
467 if not m:
468 raise MailUsageError, '''
469 The message you sent to roundup did not contain a properly formed subject
470 line. The subject must contain a class name or designator to indicate the
471 "topic" of the message. For example:
472 Subject: [issue] This is a new issue
473 - this will create a new issue in the tracker with the title "This is
474 a new issue".
475 Subject: [issue1234] This is a followup to issue 1234
476 - this will append the message's contents to the existing issue 1234
477 in the tracker.
479 Subject was: "%s"
480 '''%subject
482 # get the class
483 try:
484 cl = self.db.getclass(classname)
485 except KeyError:
486 raise MailUsageError, '''
487 The class name you identified in the subject line ("%s") does not exist in the
488 database.
490 Valid class names are: %s
491 Subject was: "%s"
492 '''%(classname, ', '.join(self.db.getclasses()), subject)
494 # get the optional nodeid
495 nodeid = m.group('nodeid')
497 # title is optional too
498 title = m.group('title')
499 if title:
500 title = title.strip()
501 else:
502 title = ''
504 # strip off the quotes that dumb emailers put around the subject, like
505 # Re: "[issue1] bla blah"
506 if m.group('quote') and title.endswith('"'):
507 title = title[:-1]
509 # but we do need either a title or a nodeid...
510 if nodeid is None and not title:
511 raise MailUsageError, '''
512 I cannot match your message to a node in the database - you need to either
513 supply a full node identifier (with number, eg "[issue123]" or keep the
514 previous subject title intact so I can match that.
516 Subject was: "%s"
517 '''%subject
519 # If there's no nodeid, check to see if this is a followup and
520 # maybe someone's responded to the initial mail that created an
521 # entry. Try to find the matching nodes with the same title, and
522 # use the _last_ one matched (since that'll _usually_ be the most
523 # recent...)
524 if nodeid is None and m.group('refwd'):
525 l = cl.stringFind(title=title)
526 if l:
527 nodeid = l[-1]
529 # if a nodeid was specified, make sure it's valid
530 if nodeid is not None and not cl.hasnode(nodeid):
531 raise MailUsageError, '''
532 The node specified by the designator in the subject of your message ("%s")
533 does not exist.
535 Subject was: "%s"
536 '''%(nodeid, subject)
539 # Handle the arguments specified by the email gateway command line.
540 # We do this by looping over the list of self.arguments looking for
541 # a -C to tell us what class then the -S setting string.
542 msg_props = {}
543 user_props = {}
544 file_props = {}
545 issue_props = {}
546 # so, if we have any arguments, use them
547 if self.arguments:
548 current_class = 'msg'
549 for option, propstring in self.arguments:
550 if option in ( '-C', '--class'):
551 current_class = propstring.strip()
552 if current_class not in ('msg', 'file', 'user', 'issue'):
553 raise MailUsageError, '''
554 The mail gateway is not properly set up. Please contact
555 %s and have them fix the incorrect class specified as:
556 %s
557 '''%(self.instance.config.ADMIN_EMAIL, current_class)
558 if option in ('-S', '--set'):
559 if current_class == 'issue' :
560 errors, issue_props = setPropArrayFromString(self,
561 cl, propstring.strip(), nodeid)
562 elif current_class == 'file' :
563 temp_cl = self.db.getclass('file')
564 errors, file_props = setPropArrayFromString(self,
565 temp_cl, propstring.strip())
566 elif current_class == 'msg' :
567 temp_cl = self.db.getclass('msg')
568 errors, msg_props = setPropArrayFromString(self,
569 temp_cl, propstring.strip())
570 elif current_class == 'user' :
571 temp_cl = self.db.getclass('user')
572 errors, user_props = setPropArrayFromString(self,
573 temp_cl, propstring.strip())
574 if errors:
575 raise MailUsageError, '''
576 The mail gateway is not properly set up. Please contact
577 %s and have them fix the incorrect properties:
578 %s
579 '''%(self.instance.config.ADMIN_EMAIL, errors)
581 #
582 # handle the users
583 #
584 # Don't create users if anonymous isn't allowed to register
585 create = 1
586 anonid = self.db.user.lookup('anonymous')
587 if not self.db.security.hasPermission('Email Registration', anonid):
588 create = 0
590 # ok, now figure out who the author is - create a new user if the
591 # "create" flag is true
592 author = uidFromAddress(self.db, message.getaddrlist('from')[0],
593 create=create)
595 # if we're not recognised, and we don't get added as a user, then we
596 # must be anonymous
597 if not author:
598 author = anonid
600 # make sure the author has permission to use the email interface
601 if not self.db.security.hasPermission('Email Access', author):
602 if author == anonid:
603 # we're anonymous and we need to be a registered user
604 raise Unauthorized, '''
605 You are not a registered user.
607 Unknown address: %s
608 '''%message.getaddrlist('from')[0][1]
609 else:
610 # we're registered and we're _still_ not allowed access
611 raise Unauthorized, 'You are not permitted to access '\
612 'this tracker.'
614 # make sure they're allowed to edit this class of information
615 if not self.db.security.hasPermission('Edit', author, classname):
616 raise Unauthorized, 'You are not permitted to edit %s.'%classname
618 # the author may have been created - make sure the change is
619 # committed before we reopen the database
620 self.db.commit()
622 # reopen the database as the author
623 username = self.db.user.get(author, 'username')
624 self.db.close()
625 self.db = self.instance.open(username)
627 # re-get the class with the new database connection
628 cl = self.db.getclass(classname)
630 # now update the recipients list
631 recipients = []
632 tracker_email = self.instance.config.TRACKER_EMAIL.lower()
633 for recipient in message.getaddrlist('to') + message.getaddrlist('cc'):
634 r = recipient[1].strip().lower()
635 if r == tracker_email or not r:
636 continue
638 # look up the recipient - create if necessary (and we're
639 # allowed to)
640 recipient = uidFromAddress(self.db, recipient, create, **user_props)
642 # if all's well, add the recipient to the list
643 if recipient:
644 recipients.append(recipient)
646 #
647 # XXX extract the args NOT USED WHY -- rouilj
648 #
649 subject_args = m.group('args')
651 #
652 # handle the subject argument list
653 #
654 # figure what the properties of this Class are
655 properties = cl.getprops()
656 props = {}
657 args = m.group('args')
658 if args:
659 errors, props = setPropArrayFromString(self, cl, args, nodeid)
660 # handle any errors parsing the argument list
661 if errors:
662 errors = '\n- '.join(errors)
663 raise MailUsageError, '''
664 There were problems handling your subject line argument list:
665 - %s
667 Subject was: "%s"
668 '''%(errors, subject)
670 #
671 # handle message-id and in-reply-to
672 #
673 messageid = message.getheader('message-id')
674 inreplyto = message.getheader('in-reply-to') or ''
675 # generate a messageid if there isn't one
676 if not messageid:
677 messageid = "<%s.%s.%s%s@%s>"%(time.time(), random.random(),
678 classname, nodeid, self.instance.config.MAIL_DOMAIN)
680 #
681 # now handle the body - find the message
682 #
683 content_type = message.gettype()
684 attachments = []
685 # General multipart handling:
686 # Take the first text/plain part, anything else is considered an
687 # attachment.
688 # multipart/mixed: multiple "unrelated" parts.
689 # multipart/signed (rfc 1847):
690 # The control information is carried in the second of the two
691 # required body parts.
692 # ACTION: Default, so if content is text/plain we get it.
693 # multipart/encrypted (rfc 1847):
694 # The control information is carried in the first of the two
695 # required body parts.
696 # ACTION: Not handleable as the content is encrypted.
697 # multipart/related (rfc 1872, 2112, 2387):
698 # The Multipart/Related content-type addresses the MIME
699 # representation of compound objects.
700 # ACTION: Default. If we are lucky there is a text/plain.
701 # TODO: One should use the start part and look for an Alternative
702 # that is text/plain.
703 # multipart/Alternative (rfc 1872, 1892):
704 # only in "related" ?
705 # multipart/report (rfc 1892):
706 # e.g. mail system delivery status reports.
707 # ACTION: Default. Could be ignored or used for Delivery Notification
708 # flagging.
709 # multipart/form-data:
710 # For web forms only.
711 if content_type == 'multipart/mixed':
712 # skip over the intro to the first boundary
713 part = message.getPart()
714 content = None
715 while 1:
716 # get the next part
717 part = message.getPart()
718 if part is None:
719 break
720 # parse it
721 subtype = part.gettype()
722 if subtype == 'text/plain' and not content:
723 # The first text/plain part is the message content.
724 content = self.get_part_data_decoded(part)
725 elif subtype == 'message/rfc822':
726 # handle message/rfc822 specially - the name should be
727 # the subject of the actual e-mail embedded here
728 i = part.fp.tell()
729 mailmess = Message(part.fp)
730 name = mailmess.getheader('subject')
731 part.fp.seek(i)
732 attachments.append((name, 'message/rfc822', part.fp.read()))
733 elif subtype == 'multipart/alternative':
734 # Search for text/plain in message with attachment and
735 # alternative text representation
736 # skip over intro to first boundary
737 part.getPart()
738 while 1:
739 # get the next part
740 subpart = part.getPart()
741 if subpart is None:
742 break
743 # parse it
744 if subpart.gettype() == 'text/plain' and not content:
745 content = self.get_part_data_decoded(subpart)
746 else:
747 # try name on Content-Type
748 name = part.getparam('name')
749 if name:
750 name = name.strip()
751 if not name:
752 disp = part.getheader('content-disposition', None)
753 if disp:
754 name = getparam(disp, 'filename')
755 if name:
756 name = name.strip()
757 # this is just an attachment
758 data = self.get_part_data_decoded(part)
759 attachments.append((name, part.gettype(), data))
760 if content is None:
761 raise MailUsageError, '''
762 Roundup requires the submission to be plain text. The message parser could
763 not find a text/plain part to use.
764 '''
766 elif content_type[:10] == 'multipart/':
767 # skip over the intro to the first boundary
768 message.getPart()
769 content = None
770 while 1:
771 # get the next part
772 part = message.getPart()
773 if part is None:
774 break
775 # parse it
776 if part.gettype() == 'text/plain' and not content:
777 content = self.get_part_data_decoded(part)
778 if content is None:
779 raise MailUsageError, '''
780 Roundup requires the submission to be plain text. The message parser could
781 not find a text/plain part to use.
782 '''
784 elif content_type != 'text/plain':
785 raise MailUsageError, '''
786 Roundup requires the submission to be plain text. The message parser could
787 not find a text/plain part to use.
788 '''
790 else:
791 content = self.get_part_data_decoded(message)
793 # figure how much we should muck around with the email body
794 keep_citations = getattr(self.instance.config, 'EMAIL_KEEP_QUOTED_TEXT',
795 'no') == 'yes'
796 keep_body = getattr(self.instance.config, 'EMAIL_LEAVE_BODY_UNCHANGED',
797 'no') == 'yes'
799 # parse the body of the message, stripping out bits as appropriate
800 summary, content = parseContent(content, keep_citations,
801 keep_body)
803 #
804 # handle the attachments
805 #
806 files = []
807 for (name, mime_type, data) in attachments:
808 if not name:
809 name = "unnamed"
810 files.append(self.db.file.create(type=mime_type, name=name,
811 content=data, **file_props))
813 #
814 # create the message if there's a message body (content)
815 #
816 if content:
817 message_id = self.db.msg.create(author=author,
818 recipients=recipients, date=date.Date('.'), summary=summary,
819 content=content, files=files, messageid=messageid,
820 inreplyto=inreplyto, **msg_props)
822 # attach the message to the node
823 if nodeid:
824 # add the message to the node's list
825 messages = cl.get(nodeid, 'messages')
826 messages.append(message_id)
827 props['messages'] = messages
828 else:
829 # pre-load the messages list
830 props['messages'] = [message_id]
832 # set the title to the subject
833 if properties.has_key('title') and not props.has_key('title'):
834 props['title'] = title
836 #
837 # perform the node change / create
838 #
839 try:
840 # merge the command line props defined in issue_props into
841 # the props dictionary because function(**props, **issue_props)
842 # is a syntax error.
843 for prop in issue_props.keys() :
844 if not props.has_key(prop) :
845 props[prop] = issue_props[prop]
846 if nodeid:
847 cl.set(nodeid, **props)
848 else:
849 nodeid = cl.create(**props)
850 except (TypeError, IndexError, ValueError), message:
851 raise MailUsageError, '''
852 There was a problem with the message you sent:
853 %s
854 '''%message
856 # commit the changes to the DB
857 self.db.commit()
859 return nodeid
862 def setPropArrayFromString(self, cl, propString, nodeid = None):
863 ''' takes string of form prop=value,value;prop2=value
864 and returns (error, prop[..])
865 '''
866 properties = cl.getprops()
867 props = {}
868 errors = []
869 for prop in string.split(propString, ';'):
870 # extract the property name and value
871 try:
872 propname, value = prop.split('=')
873 except ValueError, message:
874 errors.append('not of form [arg=value,value,...;'
875 'arg=value,value,...]')
876 return (errors, props)
878 # ensure it's a valid property name
879 propname = propname.strip()
880 try:
881 proptype = properties[propname]
882 except KeyError:
883 errors.append('refers to an invalid property: "%s"'%propname)
884 continue
886 # convert the string value to a real property value
887 if isinstance(proptype, hyperdb.String):
888 props[propname] = value.strip()
889 if isinstance(proptype, hyperdb.Password):
890 props[propname] = password.Password(value.strip())
891 elif isinstance(proptype, hyperdb.Date):
892 try:
893 props[propname] = date.Date(value.strip()).local(self.db.getUserTimezone())
894 except ValueError, message:
895 errors.append('contains an invalid date for %s.'%propname)
896 elif isinstance(proptype, hyperdb.Interval):
897 try:
898 props[propname] = date.Interval(value)
899 except ValueError, message:
900 errors.append('contains an invalid date interval for %s.'%
901 propname)
902 elif isinstance(proptype, hyperdb.Link):
903 linkcl = self.db.classes[proptype.classname]
904 propkey = linkcl.labelprop(default_to_id=1)
905 try:
906 props[propname] = linkcl.lookup(value)
907 except KeyError, message:
908 errors.append('"%s" is not a value for %s.'%(value, propname))
909 elif isinstance(proptype, hyperdb.Multilink):
910 # get the linked class
911 linkcl = self.db.classes[proptype.classname]
912 propkey = linkcl.labelprop(default_to_id=1)
913 if nodeid:
914 curvalue = cl.get(nodeid, propname)
915 else:
916 curvalue = []
918 # handle each add/remove in turn
919 # keep an extra list for all items that are
920 # definitely in the new list (in case of e.g.
921 # <propname>=A,+B, which should replace the old
922 # list with A,B)
923 set = 0
924 newvalue = []
925 for item in value.split(','):
926 item = item.strip()
928 # handle +/-
929 remove = 0
930 if item.startswith('-'):
931 remove = 1
932 item = item[1:]
933 elif item.startswith('+'):
934 item = item[1:]
935 else:
936 set = 1
938 # look up the value
939 try:
940 item = linkcl.lookup(item)
941 except KeyError, message:
942 errors.append('"%s" is not a value for %s.'%(item,
943 propname))
944 continue
946 # perform the add/remove
947 if remove:
948 try:
949 curvalue.remove(item)
950 except ValueError:
951 errors.append('"%s" is not currently in for %s.'%(item,
952 propname))
953 continue
954 else:
955 newvalue.append(item)
956 if item not in curvalue:
957 curvalue.append(item)
959 # that's it, set the new Multilink property value,
960 # or overwrite it completely
961 if set:
962 props[propname] = newvalue
963 else:
964 props[propname] = curvalue
965 elif isinstance(proptype, hyperdb.Boolean):
966 value = value.strip()
967 props[propname] = value.lower() in ('yes', 'true', 'on', '1')
968 elif isinstance(proptype, hyperdb.Number):
969 value = value.strip()
970 props[propname] = float(value)
971 return errors, props
974 def extractUserFromList(userClass, users):
975 '''Given a list of users, try to extract the first non-anonymous user
976 and return that user, otherwise return None
977 '''
978 if len(users) > 1:
979 for user in users:
980 # make sure we don't match the anonymous or admin user
981 if userClass.get(user, 'username') in ('admin', 'anonymous'):
982 continue
983 # first valid match will do
984 return user
985 # well, I guess we have no choice
986 return user[0]
987 elif users:
988 return users[0]
989 return None
992 def uidFromAddress(db, address, create=1, **user_props):
993 ''' address is from the rfc822 module, and therefore is (name, addr)
995 user is created if they don't exist in the db already
996 user_props may supply additional user information
997 '''
998 (realname, address) = address
1000 # try a straight match of the address
1001 user = extractUserFromList(db.user, db.user.stringFind(address=address))
1002 if user is not None:
1003 return user
1005 # try the user alternate addresses if possible
1006 props = db.user.getprops()
1007 if props.has_key('alternate_addresses'):
1008 users = db.user.filter(None, {'alternate_addresses': address})
1009 user = extractUserFromList(db.user, users)
1010 if user is not None:
1011 return user
1013 # try to match the username to the address (for local
1014 # submissions where the address is empty)
1015 user = extractUserFromList(db.user, db.user.stringFind(username=address))
1017 # couldn't match address or username, so create a new user
1018 if create:
1019 return db.user.create(username=address, address=address,
1020 realname=realname, roles=db.config.NEW_EMAIL_USER_ROLES,
1021 **user_props)
1022 else:
1023 return 0
1026 def parseContent(content, keep_citations, keep_body,
1027 blank_line=re.compile(r'[\r\n]+\s*[\r\n]+'),
1028 eol=re.compile(r'[\r\n]+'),
1029 signature=re.compile(r'^[>|\s]*[-_]+\s*$'),
1030 original_msg=re.compile(r'^[>|\s]*-----\s?Original Message\s?-----$')):
1031 ''' The message body is divided into sections by blank lines.
1032 Sections where the second and all subsequent lines begin with a ">"
1033 or "|" character are considered "quoting sections". The first line of
1034 the first non-quoting section becomes the summary of the message.
1036 If keep_citations is true, then we keep the "quoting sections" in the
1037 content.
1038 If keep_body is true, we even keep the signature sections.
1039 '''
1040 # strip off leading carriage-returns / newlines
1041 i = 0
1042 for i in range(len(content)):
1043 if content[i] not in '\r\n':
1044 break
1045 if i > 0:
1046 sections = blank_line.split(content[i:])
1047 else:
1048 sections = blank_line.split(content)
1050 # extract out the summary from the message
1051 summary = ''
1052 l = []
1053 for section in sections:
1054 #section = section.strip()
1055 if not section:
1056 continue
1057 lines = eol.split(section)
1058 if (lines[0] and lines[0][0] in '>|') or (len(lines) > 1 and
1059 lines[1] and lines[1][0] in '>|'):
1060 # see if there's a response somewhere inside this section (ie.
1061 # no blank line between quoted message and response)
1062 for line in lines[1:]:
1063 if line and line[0] not in '>|':
1064 break
1065 else:
1066 # we keep quoted bits if specified in the config
1067 if keep_citations:
1068 l.append(section)
1069 continue
1070 # keep this section - it has reponse stuff in it
1071 lines = lines[lines.index(line):]
1072 section = '\n'.join(lines)
1073 # and while we're at it, use the first non-quoted bit as
1074 # our summary
1075 summary = section
1077 if not summary:
1078 # if we don't have our summary yet use the first line of this
1079 # section
1080 summary = section
1081 elif signature.match(lines[0]) and 2 <= len(lines) <= 10:
1082 # lose any signature
1083 break
1084 elif original_msg.match(lines[0]):
1085 # ditch the stupid Outlook quoting of the entire original message
1086 break
1088 # and add the section to the output
1089 l.append(section)
1091 # figure the summary - find the first sentence-ending punctuation or the
1092 # first whole line, whichever is longest
1093 sentence = re.search(r'^([^!?\.]+[!?\.])', summary)
1094 if sentence:
1095 sentence = sentence.group(1)
1096 else:
1097 sentence = ''
1098 first = eol.split(summary)[0]
1099 summary = max(sentence, first)
1101 # Now reconstitute the message content minus the bits we don't care
1102 # about.
1103 if not keep_body:
1104 content = '\n\n'.join(l)
1106 return summary, content
1108 # vim: set filetype=python ts=4 sw=4 et si