c7a8b68f75913961e1619c1391b437b4faa38492
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
19 '''
20 An e-mail gateway for Roundup.
22 Incoming messages are examined for multiple parts:
23 . In a multipart/mixed message or part, each subpart is extracted and
24 examined. The text/plain subparts are assembled to form the textual
25 body of the message, to be stored in the file associated with a "msg"
26 class node. Any parts of other types are each stored in separate files
27 and given "file" class nodes that are linked to the "msg" node.
28 . In a multipart/alternative message or part, we look for a text/plain
29 subpart and ignore the other parts.
31 Summary
32 -------
33 The "summary" property on message nodes is taken from the first non-quoting
34 section in the message body. The message body is divided into sections by
35 blank lines. Sections where the second and all subsequent lines begin with
36 a ">" or "|" character are considered "quoting sections". The first line of
37 the first non-quoting section becomes the summary of the message.
39 Addresses
40 ---------
41 All of the addresses in the To: and Cc: headers of the incoming message are
42 looked up among the user nodes, and the corresponding users are placed in
43 the "recipients" property on the new "msg" node. The address in the From:
44 header similarly determines the "author" property of the new "msg"
45 node. The default handling for addresses that don't have corresponding
46 users is to create new users with no passwords and a username equal to the
47 address. (The web interface does not permit logins for users with no
48 passwords.) If we prefer to reject mail from outside sources, we can simply
49 register an auditor on the "user" class that prevents the creation of user
50 nodes with no passwords.
52 Actions
53 -------
54 The subject line of the incoming message is examined to determine whether
55 the message is an attempt to create a new item or to discuss an existing
56 item. A designator enclosed in square brackets is sought as the first thing
57 on the subject line (after skipping any "Fwd:" or "Re:" prefixes).
59 If an item designator (class name and id number) is found there, the newly
60 created "msg" node is added to the "messages" property for that item, and
61 any new "file" nodes are added to the "files" property for the item.
63 If just an item class name is found there, we attempt to create a new item
64 of that class with its "messages" property initialized to contain the new
65 "msg" node and its "files" property initialized to contain any new "file"
66 nodes.
68 Triggers
69 --------
70 Both cases may trigger detectors (in the first case we are calling the
71 set() method to add the message to the item's spool; in the second case we
72 are calling the create() method to create a new node). If an auditor raises
73 an exception, the original message is bounced back to the sender with the
74 explanatory message given in the exception.
76 $Id: mailgw.py,v 1.112 2003-03-24 02:51:21 richard Exp $
77 '''
79 import string, re, os, mimetools, cStringIO, smtplib, socket, binascii, quopri
80 import time, random, sys
81 import traceback, MimeWriter, rfc822
82 import hyperdb, date, password
84 import rfc2822
86 SENDMAILDEBUG = os.environ.get('SENDMAILDEBUG', '')
88 class MailGWError(ValueError):
89 pass
91 class MailUsageError(ValueError):
92 pass
94 class MailUsageHelp(Exception):
95 pass
97 class MailLoop(Exception):
98 ''' We've seen this message before... '''
99 pass
101 class Unauthorized(Exception):
102 """ Access denied """
104 def initialiseSecurity(security):
105 ''' Create some Permissions and Roles on the security object
107 This function is directly invoked by security.Security.__init__()
108 as a part of the Security object instantiation.
109 '''
110 security.addPermission(name="Email Registration",
111 description="Anonymous may register through e-mail")
112 p = security.addPermission(name="Email Access",
113 description="User may use the email interface")
114 security.addPermissionToRole('Admin', p)
116 def getparam(str, param):
117 ''' From the rfc822 "header" string, extract "param" if it appears.
118 '''
119 if ';' not in str:
120 return None
121 str = str[str.index(';'):]
122 while str[:1] == ';':
123 str = str[1:]
124 if ';' in str:
125 # XXX Should parse quotes!
126 end = str.index(';')
127 else:
128 end = len(str)
129 f = str[:end]
130 if '=' in f:
131 i = f.index('=')
132 if f[:i].strip().lower() == param:
133 return rfc822.unquote(f[i+1:].strip())
134 return None
136 class Message(mimetools.Message):
137 ''' subclass mimetools.Message so we can retrieve the parts of the
138 message...
139 '''
140 def getPart(self):
141 ''' Get a single part of a multipart message and return it as a new
142 Message instance.
143 '''
144 boundary = self.getparam('boundary')
145 mid, end = '--'+boundary, '--'+boundary+'--'
146 s = cStringIO.StringIO()
147 while 1:
148 line = self.fp.readline()
149 if not line:
150 break
151 if line.strip() in (mid, end):
152 break
153 s.write(line)
154 if not s.getvalue().strip():
155 return None
156 s.seek(0)
157 return Message(s)
159 def getheader(self, name, default=None):
160 hdr = mimetools.Message.getheader(self, name, default)
161 return rfc2822.decode_header(hdr)
163 subject_re = re.compile(r'(?P<refwd>\s*\W?\s*(fw|fwd|re|aw)\W\s*)*'
164 r'\s*(?P<quote>")?(\[(?P<classname>[^\d\s]+)(?P<nodeid>\d+)?\])?'
165 r'\s*(?P<title>[^[]+)?"?(\[(?P<args>.+?)\])?', re.I)
167 class MailGW:
168 def __init__(self, instance, db, arguments={}):
169 self.instance = instance
170 self.db = db
171 self.arguments = arguments
173 # should we trap exceptions (normal usage) or pass them through
174 # (for testing)
175 self.trapExceptions = 1
177 def do_pipe(self):
178 ''' Read a message from standard input and pass it to the mail handler.
180 Read into an internal structure that we can seek on (in case
181 there's an error).
183 XXX: we may want to read this into a temporary file instead...
184 '''
185 s = cStringIO.StringIO()
186 s.write(sys.stdin.read())
187 s.seek(0)
188 self.main(s)
189 return 0
191 def do_mailbox(self, filename):
192 ''' Read a series of messages from the specified unix mailbox file and
193 pass each to the mail handler.
194 '''
195 # open the spool file and lock it
196 import fcntl, FCNTL
197 f = open(filename, 'r+')
198 fcntl.flock(f.fileno(), FCNTL.LOCK_EX)
200 # handle and clear the mailbox
201 try:
202 from mailbox import UnixMailbox
203 mailbox = UnixMailbox(f, factory=Message)
204 # grab one message
205 message = mailbox.next()
206 while message:
207 # handle this message
208 self.handle_Message(message)
209 message = mailbox.next()
210 # nuke the file contents
211 os.ftruncate(f.fileno(), 0)
212 except:
213 import traceback
214 traceback.print_exc()
215 return 1
216 fcntl.flock(f.fileno(), FCNTL.LOCK_UN)
217 return 0
219 def do_apop(self, server, user='', password=''):
220 ''' Do authentication POP
221 '''
222 self.do_pop(server, user, password, apop=1):
224 def do_pop(self, server, user='', password='', apop=0):
225 '''Read a series of messages from the specified POP server.
226 '''
227 import getpass, poplib, socket
228 try:
229 if not user:
230 user = raw_input(_('User: '))
231 if not password:
232 password = getpass.getpass()
233 except (KeyboardInterrupt, EOFError):
234 # Ctrl C or D maybe also Ctrl Z under Windows.
235 print "\nAborted by user."
236 return 1
238 # open a connection to the server and retrieve all messages
239 try:
240 server = poplib.POP3(server)
241 except socket.error, message:
242 print "POP server error:", message
243 return 1
244 if apop:
245 server.apop(user, password)
246 else:
247 server.user(user)
248 server.pass_(password)
249 numMessages = len(server.list()[1])
250 for i in range(1, numMessages+1):
251 # retr: returns
252 # [ pop response e.g. '+OK 459 octets',
253 # [ array of message lines ],
254 # number of octets ]
255 lines = server.retr(i)[1]
256 s = cStringIO.StringIO('\n'.join(lines))
257 s.seek(0)
258 self.handle_Message(Message(s))
259 # delete the message
260 server.dele(i)
262 # quit the server to commit changes.
263 server.quit()
264 return 0
266 def main(self, fp):
267 ''' fp - the file from which to read the Message.
268 '''
269 return self.handle_Message(Message(fp))
271 def handle_Message(self, message):
272 '''Handle an RFC822 Message
274 Handle the Message object by calling handle_message() and then cope
275 with any errors raised by handle_message.
276 This method's job is to make that call and handle any
277 errors in a sane manner. It should be replaced if you wish to
278 handle errors in a different manner.
279 '''
280 # in some rare cases, a particularly stuffed-up e-mail will make
281 # its way into here... try to handle it gracefully
282 sendto = message.getaddrlist('from')
283 if sendto:
284 if not self.trapExceptions:
285 return self.handle_message(message)
286 try:
287 return self.handle_message(message)
288 except MailUsageHelp:
289 # bounce the message back to the sender with the usage message
290 fulldoc = '\n'.join(string.split(__doc__, '\n')[2:])
291 sendto = [sendto[0][1]]
292 m = ['']
293 m.append('\n\nMail Gateway Help\n=================')
294 m.append(fulldoc)
295 m = self.bounce_message(message, sendto, m,
296 subject="Mail Gateway Help")
297 except MailUsageError, value:
298 # bounce the message back to the sender with the usage message
299 fulldoc = '\n'.join(string.split(__doc__, '\n')[2:])
300 sendto = [sendto[0][1]]
301 m = ['']
302 m.append(str(value))
303 m.append('\n\nMail Gateway Help\n=================')
304 m.append(fulldoc)
305 m = self.bounce_message(message, sendto, m)
306 except Unauthorized, value:
307 # just inform the user that he is not authorized
308 sendto = [sendto[0][1]]
309 m = ['']
310 m.append(str(value))
311 m = self.bounce_message(message, sendto, m)
312 except MailLoop:
313 # XXX we should use a log file here...
314 return
315 except:
316 # bounce the message back to the sender with the error message
317 # XXX we should use a log file here...
318 sendto = [sendto[0][1], self.instance.config.ADMIN_EMAIL]
319 m = ['']
320 m.append('An unexpected error occurred during the processing')
321 m.append('of your message. The tracker administrator is being')
322 m.append('notified.\n')
323 m.append('---- traceback of failure ----')
324 s = cStringIO.StringIO()
325 import traceback
326 traceback.print_exc(None, s)
327 m.append(s.getvalue())
328 m = self.bounce_message(message, sendto, m)
329 else:
330 # very bad-looking message - we don't even know who sent it
331 # XXX we should use a log file here...
332 sendto = [self.instance.config.ADMIN_EMAIL]
333 m = ['Subject: badly formed message from mail gateway']
334 m.append('')
335 m.append('The mail gateway retrieved a message which has no From:')
336 m.append('line, indicating that it is corrupt. Please check your')
337 m.append('mail gateway source. Failed message is attached.')
338 m.append('')
339 m = self.bounce_message(message, sendto, m,
340 subject='Badly formed message from mail gateway')
342 # now send the message
343 if SENDMAILDEBUG:
344 open(SENDMAILDEBUG, 'a').write('From: %s\nTo: %s\n%s\n'%(
345 self.instance.config.ADMIN_EMAIL, ', '.join(sendto),
346 m.getvalue()))
347 else:
348 try:
349 smtp = smtplib.SMTP(self.instance.config.MAILHOST)
350 smtp.sendmail(self.instance.config.ADMIN_EMAIL, sendto,
351 m.getvalue())
352 except socket.error, value:
353 raise MailGWError, "Couldn't send error email: "\
354 "mailhost %s"%value
355 except smtplib.SMTPException, value:
356 raise MailGWError, "Couldn't send error email: %s"%value
358 def bounce_message(self, message, sendto, error,
359 subject='Failed issue tracker submission'):
360 ''' create a message that explains the reason for the failed
361 issue submission to the author and attach the original
362 message.
363 '''
364 msg = cStringIO.StringIO()
365 writer = MimeWriter.MimeWriter(msg)
366 writer.addheader('X-Roundup-Loop', 'hello')
367 writer.addheader('Subject', subject)
368 writer.addheader('From', '%s <%s>'% (self.instance.config.TRACKER_NAME,
369 self.instance.config.TRACKER_EMAIL))
370 writer.addheader('To', ','.join(sendto))
371 writer.addheader('Date', time.strftime("%a, %d %b %Y %H:%M:%S +0000",
372 time.gmtime()))
373 writer.addheader('MIME-Version', '1.0')
374 part = writer.startmultipartbody('mixed')
375 part = writer.nextpart()
376 body = part.startbody('text/plain; charset=utf-8')
377 body.write('\n'.join(error))
379 # attach the original message to the returned message
380 part = writer.nextpart()
381 part.addheader('Content-Disposition','attachment')
382 part.addheader('Content-Description','Message you sent')
383 body = part.startbody('text/plain')
384 for header in message.headers:
385 body.write(header)
386 body.write('\n')
387 try:
388 message.rewindbody()
389 except IOError, message:
390 body.write("*** couldn't include message body: %s ***"%message)
391 else:
392 body.write(message.fp.read())
394 writer.lastpart()
395 return msg
397 def get_part_data_decoded(self,part):
398 encoding = part.getencoding()
399 data = None
400 if encoding == 'base64':
401 # BUG: is base64 really used for text encoding or
402 # are we inserting zip files here.
403 data = binascii.a2b_base64(part.fp.read())
404 elif encoding == 'quoted-printable':
405 # the quopri module wants to work with files
406 decoded = cStringIO.StringIO()
407 quopri.decode(part.fp, decoded)
408 data = decoded.getvalue()
409 elif encoding == 'uuencoded':
410 data = binascii.a2b_uu(part.fp.read())
411 else:
412 # take it as text
413 data = part.fp.read()
415 # Encode message to unicode
416 charset = rfc2822.unaliasCharset(part.getparam("charset"))
417 if charset:
418 # Do conversion only if charset specified
419 edata = unicode(data, charset).encode('utf-8')
420 # Convert from dos eol to unix
421 edata = edata.replace('\r\n', '\n')
422 else:
423 # Leave message content as is
424 edata = data
426 return edata
428 def handle_message(self, message):
429 ''' message - a Message instance
431 Parse the message as per the module docstring.
432 '''
433 # detect loops
434 if message.getheader('x-roundup-loop', ''):
435 raise MailLoop
437 # XXX Don't enable. This doesn't work yet.
438 # "[^A-z.]tracker\+(?P<classname>[^\d\s]+)(?P<nodeid>\d+)\@some.dom.ain[^A-z.]"
439 # handle delivery to addresses like:tracker+issue25@some.dom.ain
440 # use the embedded issue number as our issue
441 # if hasattr(self.instance.config, 'EMAIL_ISSUE_ADDRESS_RE') and \
442 # self.instance.config.EMAIL_ISSUE_ADDRESS_RE:
443 # issue_re = self.instance.config.EMAIL_ISSUE_ADDRESS_RE
444 # for header in ['to', 'cc', 'bcc']:
445 # addresses = message.getheader(header, '')
446 # if addresses:
447 # # FIXME, this only finds the first match in the addresses.
448 # issue = re.search(issue_re, addresses, 'i')
449 # if issue:
450 # classname = issue.group('classname')
451 # nodeid = issue.group('nodeid')
452 # break
454 # handle the subject line
455 subject = message.getheader('subject', '')
457 if subject.strip().lower() == 'help':
458 raise MailUsageHelp
460 m = subject_re.match(subject)
462 # check for well-formed subject line
463 if m:
464 # get the classname
465 classname = m.group('classname')
466 if classname is None:
467 # no classname, fallback on the default
468 if hasattr(self.instance.config, 'MAIL_DEFAULT_CLASS') and \
469 self.instance.config.MAIL_DEFAULT_CLASS:
470 classname = self.instance.config.MAIL_DEFAULT_CLASS
471 else:
472 # fail
473 m = None
475 if not m:
476 raise MailUsageError, '''
477 The message you sent to roundup did not contain a properly formed subject
478 line. The subject must contain a class name or designator to indicate the
479 "topic" of the message. For example:
480 Subject: [issue] This is a new issue
481 - this will create a new issue in the tracker with the title "This is
482 a new issue".
483 Subject: [issue1234] This is a followup to issue 1234
484 - this will append the message's contents to the existing issue 1234
485 in the tracker.
487 Subject was: "%s"
488 '''%subject
490 # get the class
491 try:
492 cl = self.db.getclass(classname)
493 except KeyError:
494 raise MailUsageError, '''
495 The class name you identified in the subject line ("%s") does not exist in the
496 database.
498 Valid class names are: %s
499 Subject was: "%s"
500 '''%(classname, ', '.join(self.db.getclasses()), subject)
502 # get the optional nodeid
503 nodeid = m.group('nodeid')
505 # title is optional too
506 title = m.group('title')
507 if title:
508 title = title.strip()
509 else:
510 title = ''
512 # strip off the quotes that dumb emailers put around the subject, like
513 # Re: "[issue1] bla blah"
514 if m.group('quote') and title.endswith('"'):
515 title = title[:-1]
517 # but we do need either a title or a nodeid...
518 if nodeid is None and not title:
519 raise MailUsageError, '''
520 I cannot match your message to a node in the database - you need to either
521 supply a full node identifier (with number, eg "[issue123]" or keep the
522 previous subject title intact so I can match that.
524 Subject was: "%s"
525 '''%subject
527 # If there's no nodeid, check to see if this is a followup and
528 # maybe someone's responded to the initial mail that created an
529 # entry. Try to find the matching nodes with the same title, and
530 # use the _last_ one matched (since that'll _usually_ be the most
531 # recent...)
532 if nodeid is None and m.group('refwd'):
533 l = cl.stringFind(title=title)
534 if l:
535 nodeid = l[-1]
537 # if a nodeid was specified, make sure it's valid
538 if nodeid is not None and not cl.hasnode(nodeid):
539 raise MailUsageError, '''
540 The node specified by the designator in the subject of your message ("%s")
541 does not exist.
543 Subject was: "%s"
544 '''%(nodeid, subject)
547 # Handle the arguments specified by the email gateway command line.
548 # We do this by looping over the list of self.arguments looking for
549 # a -C to tell us what class then the -S setting string.
550 msg_props = {}
551 user_props = {}
552 file_props = {}
553 issue_props = {}
554 # so, if we have any arguments, use them
555 if self.arguments:
556 current_class = 'msg'
557 for option, propstring in self.arguments:
558 if option in ( '-C', '--class'):
559 current_class = propstring.strip()
560 if current_class not in ('msg', 'file', 'user', 'issue'):
561 raise MailUsageError, '''
562 The mail gateway is not properly set up. Please contact
563 %s and have them fix the incorrect class specified as:
564 %s
565 '''%(self.instance.config.ADMIN_EMAIL, current_class)
566 if option in ('-S', '--set'):
567 if current_class == 'issue' :
568 errors, issue_props = setPropArrayFromString(self,
569 cl, propstring.strip(), nodeid)
570 elif current_class == 'file' :
571 temp_cl = self.db.getclass('file')
572 errors, file_props = setPropArrayFromString(self,
573 temp_cl, propstring.strip())
574 elif current_class == 'msg' :
575 temp_cl = self.db.getclass('msg')
576 errors, msg_props = setPropArrayFromString(self,
577 temp_cl, propstring.strip())
578 elif current_class == 'user' :
579 temp_cl = self.db.getclass('user')
580 errors, user_props = setPropArrayFromString(self,
581 temp_cl, propstring.strip())
582 if errors:
583 raise MailUsageError, '''
584 The mail gateway is not properly set up. Please contact
585 %s and have them fix the incorrect properties:
586 %s
587 '''%(self.instance.config.ADMIN_EMAIL, errors)
589 #
590 # handle the users
591 #
592 # Don't create users if anonymous isn't allowed to register
593 create = 1
594 anonid = self.db.user.lookup('anonymous')
595 if not self.db.security.hasPermission('Email Registration', anonid):
596 create = 0
598 # ok, now figure out who the author is - create a new user if the
599 # "create" flag is true
600 author = uidFromAddress(self.db, message.getaddrlist('from')[0],
601 create=create)
603 # if we're not recognised, and we don't get added as a user, then we
604 # must be anonymous
605 if not author:
606 author = anonid
608 # make sure the author has permission to use the email interface
609 if not self.db.security.hasPermission('Email Access', author):
610 if author == anonid:
611 # we're anonymous and we need to be a registered user
612 raise Unauthorized, '''
613 You are not a registered user.
615 Unknown address: %s
616 '''%message.getaddrlist('from')[0][1]
617 else:
618 # we're registered and we're _still_ not allowed access
619 raise Unauthorized, 'You are not permitted to access '\
620 'this tracker.'
622 # make sure they're allowed to edit this class of information
623 if not self.db.security.hasPermission('Edit', author, classname):
624 raise Unauthorized, 'You are not permitted to edit %s.'%classname
626 # the author may have been created - make sure the change is
627 # committed before we reopen the database
628 self.db.commit()
630 # reopen the database as the author
631 username = self.db.user.get(author, 'username')
632 self.db.close()
633 self.db = self.instance.open(username)
635 # re-get the class with the new database connection
636 cl = self.db.getclass(classname)
638 # now update the recipients list
639 recipients = []
640 tracker_email = self.instance.config.TRACKER_EMAIL.lower()
641 for recipient in message.getaddrlist('to') + message.getaddrlist('cc'):
642 r = recipient[1].strip().lower()
643 if r == tracker_email or not r:
644 continue
646 # look up the recipient - create if necessary (and we're
647 # allowed to)
648 recipient = uidFromAddress(self.db, recipient, create, **user_props)
650 # if all's well, add the recipient to the list
651 if recipient:
652 recipients.append(recipient)
654 #
655 # XXX extract the args NOT USED WHY -- rouilj
656 #
657 subject_args = m.group('args')
659 #
660 # handle the subject argument list
661 #
662 # figure what the properties of this Class are
663 properties = cl.getprops()
664 props = {}
665 args = m.group('args')
666 if args:
667 errors, props = setPropArrayFromString(self, cl, args, nodeid)
668 # handle any errors parsing the argument list
669 if errors:
670 errors = '\n- '.join(errors)
671 raise MailUsageError, '''
672 There were problems handling your subject line argument list:
673 - %s
675 Subject was: "%s"
676 '''%(errors, subject)
678 #
679 # handle message-id and in-reply-to
680 #
681 messageid = message.getheader('message-id')
682 inreplyto = message.getheader('in-reply-to') or ''
683 # generate a messageid if there isn't one
684 if not messageid:
685 messageid = "<%s.%s.%s%s@%s>"%(time.time(), random.random(),
686 classname, nodeid, self.instance.config.MAIL_DOMAIN)
688 #
689 # now handle the body - find the message
690 #
691 content_type = message.gettype()
692 attachments = []
693 # General multipart handling:
694 # Take the first text/plain part, anything else is considered an
695 # attachment.
696 # multipart/mixed: multiple "unrelated" parts.
697 # multipart/signed (rfc 1847):
698 # The control information is carried in the second of the two
699 # required body parts.
700 # ACTION: Default, so if content is text/plain we get it.
701 # multipart/encrypted (rfc 1847):
702 # The control information is carried in the first of the two
703 # required body parts.
704 # ACTION: Not handleable as the content is encrypted.
705 # multipart/related (rfc 1872, 2112, 2387):
706 # The Multipart/Related content-type addresses the MIME
707 # representation of compound objects.
708 # ACTION: Default. If we are lucky there is a text/plain.
709 # TODO: One should use the start part and look for an Alternative
710 # that is text/plain.
711 # multipart/Alternative (rfc 1872, 1892):
712 # only in "related" ?
713 # multipart/report (rfc 1892):
714 # e.g. mail system delivery status reports.
715 # ACTION: Default. Could be ignored or used for Delivery Notification
716 # flagging.
717 # multipart/form-data:
718 # For web forms only.
719 if content_type == 'multipart/mixed':
720 # skip over the intro to the first boundary
721 part = message.getPart()
722 content = None
723 while 1:
724 # get the next part
725 part = message.getPart()
726 if part is None:
727 break
728 # parse it
729 subtype = part.gettype()
730 if subtype == 'text/plain' and not content:
731 # The first text/plain part is the message content.
732 content = self.get_part_data_decoded(part)
733 elif subtype == 'message/rfc822':
734 # handle message/rfc822 specially - the name should be
735 # the subject of the actual e-mail embedded here
736 i = part.fp.tell()
737 mailmess = Message(part.fp)
738 name = mailmess.getheader('subject')
739 part.fp.seek(i)
740 attachments.append((name, 'message/rfc822', part.fp.read()))
741 elif subtype == 'multipart/alternative':
742 # Search for text/plain in message with attachment and
743 # alternative text representation
744 # skip over intro to first boundary
745 part.getPart()
746 while 1:
747 # get the next part
748 subpart = part.getPart()
749 if subpart is None:
750 break
751 # parse it
752 if subpart.gettype() == 'text/plain' and not content:
753 content = self.get_part_data_decoded(subpart)
754 else:
755 # try name on Content-Type
756 name = part.getparam('name')
757 if name:
758 name = name.strip()
759 if not name:
760 disp = part.getheader('content-disposition', None)
761 if disp:
762 name = getparam(disp, 'filename')
763 if name:
764 name = name.strip()
765 # this is just an attachment
766 data = self.get_part_data_decoded(part)
767 attachments.append((name, part.gettype(), data))
768 if content is None:
769 raise MailUsageError, '''
770 Roundup requires the submission to be plain text. The message parser could
771 not find a text/plain part to use.
772 '''
774 elif content_type[:10] == 'multipart/':
775 # skip over the intro to the first boundary
776 message.getPart()
777 content = None
778 while 1:
779 # get the next part
780 part = message.getPart()
781 if part is None:
782 break
783 # parse it
784 if part.gettype() == 'text/plain' and not content:
785 content = self.get_part_data_decoded(part)
786 if content is None:
787 raise MailUsageError, '''
788 Roundup requires the submission to be plain text. The message parser could
789 not find a text/plain part to use.
790 '''
792 elif content_type != 'text/plain':
793 raise MailUsageError, '''
794 Roundup requires the submission to be plain text. The message parser could
795 not find a text/plain part to use.
796 '''
798 else:
799 content = self.get_part_data_decoded(message)
801 # figure how much we should muck around with the email body
802 keep_citations = getattr(self.instance.config, 'EMAIL_KEEP_QUOTED_TEXT',
803 'no') == 'yes'
804 keep_body = getattr(self.instance.config, 'EMAIL_LEAVE_BODY_UNCHANGED',
805 'no') == 'yes'
807 # parse the body of the message, stripping out bits as appropriate
808 summary, content = parseContent(content, keep_citations,
809 keep_body)
811 #
812 # handle the attachments
813 #
814 files = []
815 for (name, mime_type, data) in attachments:
816 if not name:
817 name = "unnamed"
818 files.append(self.db.file.create(type=mime_type, name=name,
819 content=data, **file_props))
821 #
822 # create the message if there's a message body (content)
823 #
824 if content:
825 message_id = self.db.msg.create(author=author,
826 recipients=recipients, date=date.Date('.'), summary=summary,
827 content=content, files=files, messageid=messageid,
828 inreplyto=inreplyto, **msg_props)
830 # attach the message to the node
831 if nodeid:
832 # add the message to the node's list
833 messages = cl.get(nodeid, 'messages')
834 messages.append(message_id)
835 props['messages'] = messages
836 else:
837 # pre-load the messages list
838 props['messages'] = [message_id]
840 # set the title to the subject
841 if properties.has_key('title') and not props.has_key('title'):
842 props['title'] = title
844 #
845 # perform the node change / create
846 #
847 try:
848 # merge the command line props defined in issue_props into
849 # the props dictionary because function(**props, **issue_props)
850 # is a syntax error.
851 for prop in issue_props.keys() :
852 if not props.has_key(prop) :
853 props[prop] = issue_props[prop]
854 if nodeid:
855 cl.set(nodeid, **props)
856 else:
857 nodeid = cl.create(**props)
858 except (TypeError, IndexError, ValueError), message:
859 raise MailUsageError, '''
860 There was a problem with the message you sent:
861 %s
862 '''%message
864 # commit the changes to the DB
865 self.db.commit()
867 return nodeid
870 def setPropArrayFromString(self, cl, propString, nodeid = None):
871 ''' takes string of form prop=value,value;prop2=value
872 and returns (error, prop[..])
873 '''
874 properties = cl.getprops()
875 props = {}
876 errors = []
877 for prop in string.split(propString, ';'):
878 # extract the property name and value
879 try:
880 propname, value = prop.split('=')
881 except ValueError, message:
882 errors.append('not of form [arg=value,value,...;'
883 'arg=value,value,...]')
884 return (errors, props)
886 # ensure it's a valid property name
887 propname = propname.strip()
888 try:
889 proptype = properties[propname]
890 except KeyError:
891 errors.append('refers to an invalid property: "%s"'%propname)
892 continue
894 # convert the string value to a real property value
895 if isinstance(proptype, hyperdb.String):
896 props[propname] = value.strip()
897 if isinstance(proptype, hyperdb.Password):
898 props[propname] = password.Password(value.strip())
899 elif isinstance(proptype, hyperdb.Date):
900 try:
901 props[propname] = date.Date(value.strip()).local(self.db.getUserTimezone())
902 except ValueError, message:
903 errors.append('contains an invalid date for %s.'%propname)
904 elif isinstance(proptype, hyperdb.Interval):
905 try:
906 props[propname] = date.Interval(value)
907 except ValueError, message:
908 errors.append('contains an invalid date interval for %s.'%
909 propname)
910 elif isinstance(proptype, hyperdb.Link):
911 linkcl = self.db.classes[proptype.classname]
912 propkey = linkcl.labelprop(default_to_id=1)
913 try:
914 props[propname] = linkcl.lookup(value)
915 except KeyError, message:
916 errors.append('"%s" is not a value for %s.'%(value, propname))
917 elif isinstance(proptype, hyperdb.Multilink):
918 # get the linked class
919 linkcl = self.db.classes[proptype.classname]
920 propkey = linkcl.labelprop(default_to_id=1)
921 if nodeid:
922 curvalue = cl.get(nodeid, propname)
923 else:
924 curvalue = []
926 # handle each add/remove in turn
927 # keep an extra list for all items that are
928 # definitely in the new list (in case of e.g.
929 # <propname>=A,+B, which should replace the old
930 # list with A,B)
931 set = 0
932 newvalue = []
933 for item in value.split(','):
934 item = item.strip()
936 # handle +/-
937 remove = 0
938 if item.startswith('-'):
939 remove = 1
940 item = item[1:]
941 elif item.startswith('+'):
942 item = item[1:]
943 else:
944 set = 1
946 # look up the value
947 try:
948 item = linkcl.lookup(item)
949 except KeyError, message:
950 errors.append('"%s" is not a value for %s.'%(item,
951 propname))
952 continue
954 # perform the add/remove
955 if remove:
956 try:
957 curvalue.remove(item)
958 except ValueError:
959 errors.append('"%s" is not currently in for %s.'%(item,
960 propname))
961 continue
962 else:
963 newvalue.append(item)
964 if item not in curvalue:
965 curvalue.append(item)
967 # that's it, set the new Multilink property value,
968 # or overwrite it completely
969 if set:
970 props[propname] = newvalue
971 else:
972 props[propname] = curvalue
973 elif isinstance(proptype, hyperdb.Boolean):
974 value = value.strip()
975 props[propname] = value.lower() in ('yes', 'true', 'on', '1')
976 elif isinstance(proptype, hyperdb.Number):
977 value = value.strip()
978 props[propname] = float(value)
979 return errors, props
982 def extractUserFromList(userClass, users):
983 '''Given a list of users, try to extract the first non-anonymous user
984 and return that user, otherwise return None
985 '''
986 if len(users) > 1:
987 for user in users:
988 # make sure we don't match the anonymous or admin user
989 if userClass.get(user, 'username') in ('admin', 'anonymous'):
990 continue
991 # first valid match will do
992 return user
993 # well, I guess we have no choice
994 return user[0]
995 elif users:
996 return users[0]
997 return None
1000 def uidFromAddress(db, address, create=1, **user_props):
1001 ''' address is from the rfc822 module, and therefore is (name, addr)
1003 user is created if they don't exist in the db already
1004 user_props may supply additional user information
1005 '''
1006 (realname, address) = address
1008 # try a straight match of the address
1009 user = extractUserFromList(db.user, db.user.stringFind(address=address))
1010 if user is not None:
1011 return user
1013 # try the user alternate addresses if possible
1014 props = db.user.getprops()
1015 if props.has_key('alternate_addresses'):
1016 users = db.user.filter(None, {'alternate_addresses': address})
1017 user = extractUserFromList(db.user, users)
1018 if user is not None:
1019 return user
1021 # try to match the username to the address (for local
1022 # submissions where the address is empty)
1023 user = extractUserFromList(db.user, db.user.stringFind(username=address))
1025 # couldn't match address or username, so create a new user
1026 if create:
1027 return db.user.create(username=address, address=address,
1028 realname=realname, roles=db.config.NEW_EMAIL_USER_ROLES,
1029 **user_props)
1030 else:
1031 return 0
1034 def parseContent(content, keep_citations, keep_body,
1035 blank_line=re.compile(r'[\r\n]+\s*[\r\n]+'),
1036 eol=re.compile(r'[\r\n]+'),
1037 signature=re.compile(r'^[>|\s]*[-_]+\s*$'),
1038 original_msg=re.compile(r'^[>|\s]*-----\s?Original Message\s?-----$')):
1039 ''' The message body is divided into sections by blank lines.
1040 Sections where the second and all subsequent lines begin with a ">"
1041 or "|" character are considered "quoting sections". The first line of
1042 the first non-quoting section becomes the summary of the message.
1044 If keep_citations is true, then we keep the "quoting sections" in the
1045 content.
1046 If keep_body is true, we even keep the signature sections.
1047 '''
1048 # strip off leading carriage-returns / newlines
1049 i = 0
1050 for i in range(len(content)):
1051 if content[i] not in '\r\n':
1052 break
1053 if i > 0:
1054 sections = blank_line.split(content[i:])
1055 else:
1056 sections = blank_line.split(content)
1058 # extract out the summary from the message
1059 summary = ''
1060 l = []
1061 for section in sections:
1062 #section = section.strip()
1063 if not section:
1064 continue
1065 lines = eol.split(section)
1066 if (lines[0] and lines[0][0] in '>|') or (len(lines) > 1 and
1067 lines[1] and lines[1][0] in '>|'):
1068 # see if there's a response somewhere inside this section (ie.
1069 # no blank line between quoted message and response)
1070 for line in lines[1:]:
1071 if line and line[0] not in '>|':
1072 break
1073 else:
1074 # we keep quoted bits if specified in the config
1075 if keep_citations:
1076 l.append(section)
1077 continue
1078 # keep this section - it has reponse stuff in it
1079 lines = lines[lines.index(line):]
1080 section = '\n'.join(lines)
1081 # and while we're at it, use the first non-quoted bit as
1082 # our summary
1083 summary = section
1085 if not summary:
1086 # if we don't have our summary yet use the first line of this
1087 # section
1088 summary = section
1089 elif signature.match(lines[0]) and 2 <= len(lines) <= 10:
1090 # lose any signature
1091 break
1092 elif original_msg.match(lines[0]):
1093 # ditch the stupid Outlook quoting of the entire original message
1094 break
1096 # and add the section to the output
1097 l.append(section)
1099 # figure the summary - find the first sentence-ending punctuation or the
1100 # first whole line, whichever is longest
1101 sentence = re.search(r'^([^!?\.]+[!?\.])', summary)
1102 if sentence:
1103 sentence = sentence.group(1)
1104 else:
1105 sentence = ''
1106 first = eol.split(summary)[0]
1107 summary = max(sentence, first)
1109 # Now reconstitute the message content minus the bits we don't care
1110 # about.
1111 if not keep_body:
1112 content = '\n\n'.join(l)
1114 return summary, content
1116 # vim: set filetype=python ts=4 sw=4 et si