8f91118ca8e19f350e1d73bb919258621c303c85
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
19 '''
20 An e-mail gateway for Roundup.
22 Incoming messages are examined for multiple parts:
23 . In a multipart/mixed message or part, each subpart is extracted and
24 examined. The text/plain subparts are assembled to form the textual
25 body of the message, to be stored in the file associated with a "msg"
26 class node. Any parts of other types are each stored in separate files
27 and given "file" class nodes that are linked to the "msg" node.
28 . In a multipart/alternative message or part, we look for a text/plain
29 subpart and ignore the other parts.
31 Summary
32 -------
33 The "summary" property on message nodes is taken from the first non-quoting
34 section in the message body. The message body is divided into sections by
35 blank lines. Sections where the second and all subsequent lines begin with
36 a ">" or "|" character are considered "quoting sections". The first line of
37 the first non-quoting section becomes the summary of the message.
39 Addresses
40 ---------
41 All of the addresses in the To: and Cc: headers of the incoming message are
42 looked up among the user nodes, and the corresponding users are placed in
43 the "recipients" property on the new "msg" node. The address in the From:
44 header similarly determines the "author" property of the new "msg"
45 node. The default handling for addresses that don't have corresponding
46 users is to create new users with no passwords and a username equal to the
47 address. (The web interface does not permit logins for users with no
48 passwords.) If we prefer to reject mail from outside sources, we can simply
49 register an auditor on the "user" class that prevents the creation of user
50 nodes with no passwords.
52 Actions
53 -------
54 The subject line of the incoming message is examined to determine whether
55 the message is an attempt to create a new item or to discuss an existing
56 item. A designator enclosed in square brackets is sought as the first thing
57 on the subject line (after skipping any "Fwd:" or "Re:" prefixes).
59 If an item designator (class name and id number) is found there, the newly
60 created "msg" node is added to the "messages" property for that item, and
61 any new "file" nodes are added to the "files" property for the item.
63 If just an item class name is found there, we attempt to create a new item
64 of that class with its "messages" property initialized to contain the new
65 "msg" node and its "files" property initialized to contain any new "file"
66 nodes.
68 Triggers
69 --------
70 Both cases may trigger detectors (in the first case we are calling the
71 set() method to add the message to the item's spool; in the second case we
72 are calling the create() method to create a new node). If an auditor raises
73 an exception, the original message is bounced back to the sender with the
74 explanatory message given in the exception.
76 $Id: mailgw.py,v 1.105 2003-01-11 23:52:27 richard Exp $
77 '''
79 import string, re, os, mimetools, cStringIO, smtplib, socket, binascii, quopri
80 import time, random, sys
81 import traceback, MimeWriter
82 import hyperdb, date, password
84 SENDMAILDEBUG = os.environ.get('SENDMAILDEBUG', '')
86 class MailGWError(ValueError):
87 pass
89 class MailUsageError(ValueError):
90 pass
92 class MailUsageHelp(Exception):
93 pass
95 class MailLoop(Exception):
96 ''' We've seen this message before... '''
97 pass
99 class Unauthorized(Exception):
100 """ Access denied """
102 def initialiseSecurity(security):
103 ''' Create some Permissions and Roles on the security object
105 This function is directly invoked by security.Security.__init__()
106 as a part of the Security object instantiation.
107 '''
108 security.addPermission(name="Email Registration",
109 description="Anonymous may register through e-mail")
110 p = security.addPermission(name="Email Access",
111 description="User may use the email interface")
112 security.addPermissionToRole('Admin', p)
114 class Message(mimetools.Message):
115 ''' subclass mimetools.Message so we can retrieve the parts of the
116 message...
117 '''
118 def getPart(self):
119 ''' Get a single part of a multipart message and return it as a new
120 Message instance.
121 '''
122 boundary = self.getparam('boundary')
123 mid, end = '--'+boundary, '--'+boundary+'--'
124 s = cStringIO.StringIO()
125 while 1:
126 line = self.fp.readline()
127 if not line:
128 break
129 if line.strip() in (mid, end):
130 break
131 s.write(line)
132 if not s.getvalue().strip():
133 return None
134 s.seek(0)
135 return Message(s)
137 subject_re = re.compile(r'(?P<refwd>\s*\W?\s*(fw|fwd|re|aw)\W\s*)*'
138 r'\s*(?P<quote>")?(\[(?P<classname>[^\d\s]+)(?P<nodeid>\d+)?\])?'
139 r'\s*(?P<title>[^[]+)?"?(\[(?P<args>.+?)\])?', re.I)
141 class MailGW:
142 def __init__(self, instance, db):
143 self.instance = instance
144 self.db = db
146 # should we trap exceptions (normal usage) or pass them through
147 # (for testing)
148 self.trapExceptions = 1
150 def do_pipe(self):
151 ''' Read a message from standard input and pass it to the mail handler.
153 Read into an internal structure that we can seek on (in case
154 there's an error).
156 XXX: we may want to read this into a temporary file instead...
157 '''
158 s = cStringIO.StringIO()
159 s.write(sys.stdin.read())
160 s.seek(0)
161 self.main(s)
162 return 0
164 def do_mailbox(self, filename):
165 ''' Read a series of messages from the specified unix mailbox file and
166 pass each to the mail handler.
167 '''
168 # open the spool file and lock it
169 import fcntl, FCNTL
170 f = open(filename, 'r+')
171 fcntl.flock(f.fileno(), FCNTL.LOCK_EX)
173 # handle and clear the mailbox
174 try:
175 from mailbox import UnixMailbox
176 mailbox = UnixMailbox(f, factory=Message)
177 # grab one message
178 message = mailbox.next()
179 while message:
180 # handle this message
181 self.handle_Message(message)
182 message = mailbox.next()
183 # nuke the file contents
184 os.ftruncate(f.fileno(), 0)
185 except:
186 import traceback
187 traceback.print_exc()
188 return 1
189 fcntl.flock(f.fileno(), FCNTL.LOCK_UN)
190 return 0
192 def do_pop(self, server, user='', password=''):
193 '''Read a series of messages from the specified POP server.
194 '''
195 import getpass, poplib, socket
196 try:
197 if not user:
198 user = raw_input(_('User: '))
199 if not password:
200 password = getpass.getpass()
201 except (KeyboardInterrupt, EOFError):
202 # Ctrl C or D maybe also Ctrl Z under Windows.
203 print "\nAborted by user."
204 return 1
206 # open a connection to the server and retrieve all messages
207 try:
208 server = poplib.POP3(server)
209 except socket.error, message:
210 print "POP server error:", message
211 return 1
212 server.user(user)
213 server.pass_(password)
214 numMessages = len(server.list()[1])
215 for i in range(1, numMessages+1):
216 # retr: returns
217 # [ pop response e.g. '+OK 459 octets',
218 # [ array of message lines ],
219 # number of octets ]
220 lines = server.retr(i)[1]
221 s = cStringIO.StringIO('\n'.join(lines))
222 s.seek(0)
223 self.handle_Message(Message(s))
224 # delete the message
225 server.dele(i)
227 # quit the server to commit changes.
228 server.quit()
229 return 0
231 def main(self, fp):
232 ''' fp - the file from which to read the Message.
233 '''
234 return self.handle_Message(Message(fp))
236 def handle_Message(self, message):
237 '''Handle an RFC822 Message
239 Handle the Message object by calling handle_message() and then cope
240 with any errors raised by handle_message.
241 This method's job is to make that call and handle any
242 errors in a sane manner. It should be replaced if you wish to
243 handle errors in a different manner.
244 '''
245 # in some rare cases, a particularly stuffed-up e-mail will make
246 # its way into here... try to handle it gracefully
247 sendto = message.getaddrlist('from')
248 if sendto:
249 if not self.trapExceptions:
250 return self.handle_message(message)
251 try:
252 return self.handle_message(message)
253 except MailUsageHelp:
254 # bounce the message back to the sender with the usage message
255 fulldoc = '\n'.join(string.split(__doc__, '\n')[2:])
256 sendto = [sendto[0][1]]
257 m = ['']
258 m.append('\n\nMail Gateway Help\n=================')
259 m.append(fulldoc)
260 m = self.bounce_message(message, sendto, m,
261 subject="Mail Gateway Help")
262 except MailUsageError, value:
263 # bounce the message back to the sender with the usage message
264 fulldoc = '\n'.join(string.split(__doc__, '\n')[2:])
265 sendto = [sendto[0][1]]
266 m = ['']
267 m.append(str(value))
268 m.append('\n\nMail Gateway Help\n=================')
269 m.append(fulldoc)
270 m = self.bounce_message(message, sendto, m)
271 except Unauthorized, value:
272 # just inform the user that he is not authorized
273 sendto = [sendto[0][1]]
274 m = ['']
275 m.append(str(value))
276 m = self.bounce_message(message, sendto, m)
277 except MailLoop:
278 # XXX we should use a log file here...
279 return
280 except:
281 # bounce the message back to the sender with the error message
282 # XXX we should use a log file here...
283 sendto = [sendto[0][1], self.instance.config.ADMIN_EMAIL]
284 m = ['']
285 m.append('An unexpected error occurred during the processing')
286 m.append('of your message. The tracker administrator is being')
287 m.append('notified.\n')
288 m.append('---- traceback of failure ----')
289 s = cStringIO.StringIO()
290 import traceback
291 traceback.print_exc(None, s)
292 m.append(s.getvalue())
293 m = self.bounce_message(message, sendto, m)
294 else:
295 # very bad-looking message - we don't even know who sent it
296 # XXX we should use a log file here...
297 sendto = [self.instance.config.ADMIN_EMAIL]
298 m = ['Subject: badly formed message from mail gateway']
299 m.append('')
300 m.append('The mail gateway retrieved a message which has no From:')
301 m.append('line, indicating that it is corrupt. Please check your')
302 m.append('mail gateway source. Failed message is attached.')
303 m.append('')
304 m = self.bounce_message(message, sendto, m,
305 subject='Badly formed message from mail gateway')
307 # now send the message
308 if SENDMAILDEBUG:
309 open(SENDMAILDEBUG, 'a').write('From: %s\nTo: %s\n%s\n'%(
310 self.instance.config.ADMIN_EMAIL, ', '.join(sendto),
311 m.getvalue()))
312 else:
313 try:
314 smtp = smtplib.SMTP(self.instance.config.MAILHOST)
315 smtp.sendmail(self.instance.config.ADMIN_EMAIL, sendto,
316 m.getvalue())
317 except socket.error, value:
318 raise MailGWError, "Couldn't send error email: "\
319 "mailhost %s"%value
320 except smtplib.SMTPException, value:
321 raise MailGWError, "Couldn't send error email: %s"%value
323 def bounce_message(self, message, sendto, error,
324 subject='Failed issue tracker submission'):
325 ''' create a message that explains the reason for the failed
326 issue submission to the author and attach the original
327 message.
328 '''
329 msg = cStringIO.StringIO()
330 writer = MimeWriter.MimeWriter(msg)
331 writer.addheader('X-Roundup-Loop', 'hello')
332 writer.addheader('Subject', subject)
333 writer.addheader('From', '%s <%s>'% (self.instance.config.TRACKER_NAME,
334 self.instance.config.TRACKER_EMAIL))
335 writer.addheader('To', ','.join(sendto))
336 writer.addheader('Date', time.strftime("%a, %d %b %Y %H:%M:%S +0000",
337 time.gmtime()))
338 writer.addheader('MIME-Version', '1.0')
339 part = writer.startmultipartbody('mixed')
340 part = writer.nextpart()
341 body = part.startbody('text/plain')
342 body.write('\n'.join(error))
344 # attach the original message to the returned message
345 part = writer.nextpart()
346 part.addheader('Content-Disposition','attachment')
347 part.addheader('Content-Description','Message you sent')
348 body = part.startbody('text/plain')
349 for header in message.headers:
350 body.write(header)
351 body.write('\n')
352 try:
353 message.rewindbody()
354 except IOError, message:
355 body.write("*** couldn't include message body: %s ***"%message)
356 else:
357 body.write(message.fp.read())
359 writer.lastpart()
360 return msg
362 def get_part_data_decoded(self,part):
363 encoding = part.getencoding()
364 data = None
365 if encoding == 'base64':
366 # BUG: is base64 really used for text encoding or
367 # are we inserting zip files here.
368 data = binascii.a2b_base64(part.fp.read())
369 elif encoding == 'quoted-printable':
370 # the quopri module wants to work with files
371 decoded = cStringIO.StringIO()
372 quopri.decode(part.fp, decoded)
373 data = decoded.getvalue()
374 elif encoding == 'uuencoded':
375 data = binascii.a2b_uu(part.fp.read())
376 else:
377 # take it as text
378 data = part.fp.read()
379 return data
381 def handle_message(self, message):
382 ''' message - a Message instance
384 Parse the message as per the module docstring.
385 '''
386 # detect loops
387 if message.getheader('x-roundup-loop', ''):
388 raise MailLoop
390 # XXX Don't enable. This doesn't work yet.
391 # "[^A-z.]tracker\+(?P<classname>[^\d\s]+)(?P<nodeid>\d+)\@some.dom.ain[^A-z.]"
392 # handle delivery to addresses like:tracker+issue25@some.dom.ain
393 # use the embedded issue number as our issue
394 # if hasattr(self.instance.config, 'EMAIL_ISSUE_ADDRESS_RE') and \
395 # self.instance.config.EMAIL_ISSUE_ADDRESS_RE:
396 # issue_re = self.instance.config.EMAIL_ISSUE_ADDRESS_RE
397 # for header in ['to', 'cc', 'bcc']:
398 # addresses = message.getheader(header, '')
399 # if addresses:
400 # # FIXME, this only finds the first match in the addresses.
401 # issue = re.search(issue_re, addresses, 'i')
402 # if issue:
403 # classname = issue.group('classname')
404 # nodeid = issue.group('nodeid')
405 # break
407 # handle the subject line
408 subject = message.getheader('subject', '')
410 if subject.strip().lower() == 'help':
411 raise MailUsageHelp
413 m = subject_re.match(subject)
415 # check for well-formed subject line
416 if m:
417 # get the classname
418 classname = m.group('classname')
419 if classname is None:
420 # no classname, fallback on the default
421 if hasattr(self.instance.config, 'MAIL_DEFAULT_CLASS') and \
422 self.instance.config.MAIL_DEFAULT_CLASS:
423 classname = self.instance.config.MAIL_DEFAULT_CLASS
424 else:
425 # fail
426 m = None
428 if not m:
429 raise MailUsageError, '''
430 The message you sent to roundup did not contain a properly formed subject
431 line. The subject must contain a class name or designator to indicate the
432 "topic" of the message. For example:
433 Subject: [issue] This is a new issue
434 - this will create a new issue in the tracker with the title "This is
435 a new issue".
436 Subject: [issue1234] This is a followup to issue 1234
437 - this will append the message's contents to the existing issue 1234
438 in the tracker.
440 Subject was: "%s"
441 '''%subject
443 # get the class
444 try:
445 cl = self.db.getclass(classname)
446 except KeyError:
447 raise MailUsageError, '''
448 The class name you identified in the subject line ("%s") does not exist in the
449 database.
451 Valid class names are: %s
452 Subject was: "%s"
453 '''%(classname, ', '.join(self.db.getclasses()), subject)
455 # get the optional nodeid
456 nodeid = m.group('nodeid')
458 # title is optional too
459 title = m.group('title')
460 if title:
461 title = title.strip()
462 else:
463 title = ''
465 # strip off the quotes that dumb emailers put around the subject, like
466 # Re: "[issue1] bla blah"
467 if m.group('quote') and title.endswith('"'):
468 title = title[:-1]
470 # but we do need either a title or a nodeid...
471 if nodeid is None and not title:
472 raise MailUsageError, '''
473 I cannot match your message to a node in the database - you need to either
474 supply a full node identifier (with number, eg "[issue123]" or keep the
475 previous subject title intact so I can match that.
477 Subject was: "%s"
478 '''%subject
480 # If there's no nodeid, check to see if this is a followup and
481 # maybe someone's responded to the initial mail that created an
482 # entry. Try to find the matching nodes with the same title, and
483 # use the _last_ one matched (since that'll _usually_ be the most
484 # recent...)
485 if nodeid is None and m.group('refwd'):
486 l = cl.stringFind(title=title)
487 if l:
488 nodeid = l[-1]
490 # if a nodeid was specified, make sure it's valid
491 if nodeid is not None and not cl.hasnode(nodeid):
492 raise MailUsageError, '''
493 The node specified by the designator in the subject of your message ("%s")
494 does not exist.
496 Subject was: "%s"
497 '''%(nodeid, subject)
499 #
500 # Handle the options specified by the email gateway
501 # command line. I do this by looping over the list of
502 # self.options looking for a -C to tell me what class
503 # I add the -S setting string to.
504 #
505 msg_props = {}
506 user_props = {}
507 file_props = {}
508 issue_props = {}
509 # this should be true if options are set on command
510 # line
511 if hasattr(self, 'options'):
512 current_class = 'msg'
513 for option, propstring in self.options:
514 if option in ( '-C', '--class'):
515 current_class = propstring.strip()
516 if current_class not in ('msg', 'file', 'user', 'issue'):
517 raise MailUsageError, '''
518 The mail gateway is not properly set up. Please contact
519 %s and have them fix the incorrect class specified as:
520 %s
521 '''%(self.instance.config.ADMIN_EMAIL, current_class)
522 if option in ('-S', '--set'):
523 if current_class == 'issue' :
524 errors, issue_props = setPropArrayFromString(self,
525 cl, propstring.strip(), nodeid)
526 elif current_class == 'file' :
527 temp_cl = self.db.getclass('file')
528 errors, file_props = setPropArrayFromString(self,
529 temp_cl, propstring.strip())
530 elif current_class == 'msg' :
531 temp_cl = self.db.getclass('msg')
532 errors, msg_props = setPropArrayFromString(self,
533 temp_cl, propstring.strip())
534 elif current_class == 'user' :
535 temp_cl = self.db.getclass('user')
536 errors, user_props = setPropArrayFromString(self,
537 temp_cl, propstring.strip())
538 if errors:
539 raise MailUsageError, '''
540 The mail gateway is not properly set up. Please contact
541 %s and have them fix the incorrect properties:
542 %s
543 '''%(self.instance.config.ADMIN_EMAIL, errors)
545 #
546 # handle the users
547 #
548 # Don't create users if anonymous isn't allowed to register
549 create = 1
550 anonid = self.db.user.lookup('anonymous')
551 if not self.db.security.hasPermission('Email Registration', anonid):
552 create = 0
554 # ok, now figure out who the author is - create a new user if the
555 # "create" flag is true
556 author = uidFromAddress(self.db, message.getaddrlist('from')[0],
557 create=create)
559 # if we're not recognised, and we don't get added as a user, then we
560 # must be anonymous
561 if not author:
562 author = anonid
564 # make sure the author has permission to use the email interface
565 if not self.db.security.hasPermission('Email Access', author):
566 if author == anonid:
567 # we're anonymous and we need to be a registered user
568 raise Unauthorized, '''
569 You are not a registered user.
571 Unknown address: %s
572 '''%message.getaddrlist('from')[0][1]
573 else:
574 # we're registered and we're _still_ not allowed access
575 raise Unauthorized, 'You are not permitted to access '\
576 'this tracker.'
578 # make sure they're allowed to edit this class of information
579 if not self.db.security.hasPermission('Edit', author, classname):
580 raise Unauthorized, 'You are not permitted to edit %s.'%classname
582 # the author may have been created - make sure the change is
583 # committed before we reopen the database
584 self.db.commit()
586 # reopen the database as the author
587 username = self.db.user.get(author, 'username')
588 self.db.close()
589 self.db = self.instance.open(username)
591 # re-get the class with the new database connection
592 cl = self.db.getclass(classname)
594 # now update the recipients list
595 recipients = []
596 tracker_email = self.instance.config.TRACKER_EMAIL.lower()
597 for recipient in message.getaddrlist('to') + message.getaddrlist('cc'):
598 r = recipient[1].strip().lower()
599 if r == tracker_email or not r:
600 continue
602 # look up the recipient - create if necessary (and we're
603 # allowed to)
604 recipient = uidFromAddress(self.db, recipient, create, **user_props)
606 # if all's well, add the recipient to the list
607 if recipient:
608 recipients.append(recipient)
610 #
611 # XXX extract the args NOT USED WHY -- rouilj
612 #
613 subject_args = m.group('args')
615 #
616 # handle the subject argument list
617 #
618 # figure what the properties of this Class are
619 properties = cl.getprops()
620 props = {}
621 args = m.group('args')
622 if args:
623 errors, props = setPropArrayFromString(self, cl, args, nodeid)
624 # handle any errors parsing the argument list
625 if errors:
626 errors = '\n- '.join(errors)
627 raise MailUsageError, '''
628 There were problems handling your subject line argument list:
629 - %s
631 Subject was: "%s"
632 '''%(errors, subject)
634 #
635 # handle message-id and in-reply-to
636 #
637 messageid = message.getheader('message-id')
638 inreplyto = message.getheader('in-reply-to') or ''
639 # generate a messageid if there isn't one
640 if not messageid:
641 messageid = "<%s.%s.%s%s@%s>"%(time.time(), random.random(),
642 classname, nodeid, self.instance.config.MAIL_DOMAIN)
644 #
645 # now handle the body - find the message
646 #
647 content_type = message.gettype()
648 attachments = []
649 # General multipart handling:
650 # Take the first text/plain part, anything else is considered an
651 # attachment.
652 # multipart/mixed: multiple "unrelated" parts.
653 # multipart/signed (rfc 1847):
654 # The control information is carried in the second of the two
655 # required body parts.
656 # ACTION: Default, so if content is text/plain we get it.
657 # multipart/encrypted (rfc 1847):
658 # The control information is carried in the first of the two
659 # required body parts.
660 # ACTION: Not handleable as the content is encrypted.
661 # multipart/related (rfc 1872, 2112, 2387):
662 # The Multipart/Related content-type addresses the MIME
663 # representation of compound objects.
664 # ACTION: Default. If we are lucky there is a text/plain.
665 # TODO: One should use the start part and look for an Alternative
666 # that is text/plain.
667 # multipart/Alternative (rfc 1872, 1892):
668 # only in "related" ?
669 # multipart/report (rfc 1892):
670 # e.g. mail system delivery status reports.
671 # ACTION: Default. Could be ignored or used for Delivery Notification
672 # flagging.
673 # multipart/form-data:
674 # For web forms only.
675 if content_type == 'multipart/mixed':
676 # skip over the intro to the first boundary
677 part = message.getPart()
678 content = None
679 while 1:
680 # get the next part
681 part = message.getPart()
682 if part is None:
683 break
684 # parse it
685 subtype = part.gettype()
686 if subtype == 'text/plain' and not content:
687 # The first text/plain part is the message content.
688 content = self.get_part_data_decoded(part)
689 elif subtype == 'message/rfc822':
690 # handle message/rfc822 specially - the name should be
691 # the subject of the actual e-mail embedded here
692 i = part.fp.tell()
693 mailmess = Message(part.fp)
694 name = mailmess.getheader('subject')
695 part.fp.seek(i)
696 attachments.append((name, 'message/rfc822', part.fp.read()))
697 elif subtype == 'multipart/alternative':
698 # Search for text/plain in message with attachment and
699 # alternative text representation
700 part.getPart()
701 while 1:
702 # get the next part
703 subpart = part.getPart()
704 if subpart is None:
705 break
706 # parse it
707 if subpart.gettype() == 'text/plain' and not content:
708 content = self.get_part_data_decoded(subpart)
709 else:
710 # try name on Content-Type
711 name = part.getparam('name')
712 if name:
713 name = name.strip()
714 if not name:
715 disp = part.getheader('content-disposition', None)
716 if disp:
717 name = disp.getparam('filename')
718 if name:
719 name = name.strip()
720 # this is just an attachment
721 data = self.get_part_data_decoded(part)
722 attachments.append((name, part.gettype(), data))
723 if content is None:
724 raise MailUsageError, '''
725 Roundup requires the submission to be plain text. The message parser could
726 not find a text/plain part to use.
727 '''
729 elif content_type[:10] == 'multipart/':
730 # skip over the intro to the first boundary
731 message.getPart()
732 content = None
733 while 1:
734 # get the next part
735 part = message.getPart()
736 if part is None:
737 break
738 # parse it
739 if part.gettype() == 'text/plain' and not content:
740 content = self.get_part_data_decoded(part)
741 if content is None:
742 raise MailUsageError, '''
743 Roundup requires the submission to be plain text. The message parser could
744 not find a text/plain part to use.
745 '''
747 elif content_type != 'text/plain':
748 raise MailUsageError, '''
749 Roundup requires the submission to be plain text. The message parser could
750 not find a text/plain part to use.
751 '''
753 else:
754 content = self.get_part_data_decoded(message)
756 # figure how much we should muck around with the email body
757 keep_citations = getattr(self.instance.config, 'EMAIL_KEEP_QUOTED_TEXT',
758 'no') == 'yes'
759 keep_body = getattr(self.instance.config, 'EMAIL_LEAVE_BODY_UNCHANGED',
760 'no') == 'yes'
762 # parse the body of the message, stripping out bits as appropriate
763 summary, content = parseContent(content, keep_citations,
764 keep_body)
766 #
767 # handle the attachments
768 #
769 files = []
770 for (name, mime_type, data) in attachments:
771 if not name:
772 name = "unnamed"
773 files.append(self.db.file.create(type=mime_type, name=name,
774 content=data, **file_props))
776 #
777 # create the message if there's a message body (content)
778 #
779 if content:
780 message_id = self.db.msg.create(author=author,
781 recipients=recipients, date=date.Date('.'), summary=summary,
782 content=content, files=files, messageid=messageid,
783 inreplyto=inreplyto, **msg_props)
785 # attach the message to the node
786 if nodeid:
787 # add the message to the node's list
788 messages = cl.get(nodeid, 'messages')
789 messages.append(message_id)
790 props['messages'] = messages
791 else:
792 # pre-load the messages list
793 props['messages'] = [message_id]
795 # set the title to the subject
796 if properties.has_key('title') and not props.has_key('title'):
797 props['title'] = title
799 #
800 # perform the node change / create
801 #
802 try:
803 # merge the command line props defined in issue_props into
804 # the props dictionary because function(**props, **issue_props)
805 # is a syntax error.
806 for prop in issue_props.keys() :
807 if not props.has_key(prop) :
808 props[prop] = issue_props[prop]
809 if nodeid:
810 cl.set(nodeid, **props)
811 else:
812 nodeid = cl.create(**props)
813 except (TypeError, IndexError, ValueError), message:
814 raise MailUsageError, '''
815 There was a problem with the message you sent:
816 %s
817 '''%message
819 # commit the changes to the DB
820 self.db.commit()
822 return nodeid
825 def setPropArrayFromString(self, cl, propString, nodeid = None):
826 ''' takes string of form prop=value,value;prop2=value
827 and returns (error, prop[..])
828 '''
829 properties = cl.getprops()
830 props = {}
831 errors = []
832 for prop in string.split(propString, ';'):
833 # extract the property name and value
834 try:
835 propname, value = prop.split('=')
836 except ValueError, message:
837 errors.append('not of form [arg=value,value,...;'
838 'arg=value,value,...]')
839 return (errors, props)
841 # ensure it's a valid property name
842 propname = propname.strip()
843 try:
844 proptype = properties[propname]
845 except KeyError:
846 errors.append('refers to an invalid property: "%s"'%propname)
847 continue
849 # convert the string value to a real property value
850 if isinstance(proptype, hyperdb.String):
851 props[propname] = value.strip()
852 if isinstance(proptype, hyperdb.Password):
853 props[propname] = password.Password(value.strip())
854 elif isinstance(proptype, hyperdb.Date):
855 try:
856 props[propname] = date.Date(value.strip())
857 except ValueError, message:
858 errors.append('contains an invalid date for %s.'%propname)
859 elif isinstance(proptype, hyperdb.Interval):
860 try:
861 props[propname] = date.Interval(value)
862 except ValueError, message:
863 errors.append('contains an invalid date interval for %s.'%
864 propname)
865 elif isinstance(proptype, hyperdb.Link):
866 linkcl = self.db.classes[proptype.classname]
867 propkey = linkcl.labelprop(default_to_id=1)
868 try:
869 props[propname] = linkcl.lookup(value)
870 except KeyError, message:
871 errors.append('"%s" is not a value for %s.'%(value, propname))
872 elif isinstance(proptype, hyperdb.Multilink):
873 # get the linked class
874 linkcl = self.db.classes[proptype.classname]
875 propkey = linkcl.labelprop(default_to_id=1)
876 if nodeid:
877 curvalue = cl.get(nodeid, propname)
878 else:
879 curvalue = []
881 # handle each add/remove in turn
882 # keep an extra list for all items that are
883 # definitely in the new list (in case of e.g.
884 # <propname>=A,+B, which should replace the old
885 # list with A,B)
886 set = 0
887 newvalue = []
888 for item in value.split(','):
889 item = item.strip()
891 # handle +/-
892 remove = 0
893 if item.startswith('-'):
894 remove = 1
895 item = item[1:]
896 elif item.startswith('+'):
897 item = item[1:]
898 else:
899 set = 1
901 # look up the value
902 try:
903 item = linkcl.lookup(item)
904 except KeyError, message:
905 errors.append('"%s" is not a value for %s.'%(item,
906 propname))
907 continue
909 # perform the add/remove
910 if remove:
911 try:
912 curvalue.remove(item)
913 except ValueError:
914 errors.append('"%s" is not currently in for %s.'%(item,
915 propname))
916 continue
917 else:
918 newvalue.append(item)
919 if item not in curvalue:
920 curvalue.append(item)
922 # that's it, set the new Multilink property value,
923 # or overwrite it completely
924 if set:
925 props[propname] = newvalue
926 else:
927 props[propname] = curvalue
928 elif isinstance(proptype, hyperdb.Boolean):
929 value = value.strip()
930 props[propname] = value.lower() in ('yes', 'true', 'on', '1')
931 elif isinstance(proptype, hyperdb.Number):
932 value = value.strip()
933 props[propname] = int(value)
934 return errors, props
937 def extractUserFromList(userClass, users):
938 '''Given a list of users, try to extract the first non-anonymous user
939 and return that user, otherwise return None
940 '''
941 if len(users) > 1:
942 for user in users:
943 # make sure we don't match the anonymous or admin user
944 if userClass.get(user, 'username') in ('admin', 'anonymous'):
945 continue
946 # first valid match will do
947 return user
948 # well, I guess we have no choice
949 return user[0]
950 elif users:
951 return users[0]
952 return None
955 def uidFromAddress(db, address, create=1, **user_props):
956 ''' address is from the rfc822 module, and therefore is (name, addr)
958 user is created if they don't exist in the db already
959 user_props may supply additional user information
960 '''
961 (realname, address) = address
963 # try a straight match of the address
964 user = extractUserFromList(db.user, db.user.stringFind(address=address))
965 if user is not None: return user
967 # try the user alternate addresses if possible
968 props = db.user.getprops()
969 if props.has_key('alternate_addresses'):
970 users = db.user.filter(None, {'alternate_addresses': address})
971 user = extractUserFromList(db.user, users)
972 if user is not None: return user
974 # try to match the username to the address (for local
975 # submissions where the address is empty)
976 user = extractUserFromList(db.user, db.user.stringFind(username=address))
978 # couldn't match address or username, so create a new user
979 if create:
980 return db.user.create(username=address, address=address,
981 realname=realname, roles=db.config.NEW_EMAIL_USER_ROLES,
982 **user_props)
983 else:
984 return 0
987 def parseContent(content, keep_citations, keep_body,
988 blank_line=re.compile(r'[\r\n]+\s*[\r\n]+'),
989 eol=re.compile(r'[\r\n]+'),
990 signature=re.compile(r'^[>|\s]*[-_]+\s*$'),
991 original_msg=re.compile(r'^[>|\s]*-----\s?Original Message\s?-----$')):
992 ''' The message body is divided into sections by blank lines.
993 Sections where the second and all subsequent lines begin with a ">"
994 or "|" character are considered "quoting sections". The first line of
995 the first non-quoting section becomes the summary of the message.
997 If keep_citations is true, then we keep the "quoting sections" in the
998 content.
999 If keep_body is true, we even keep the signature sections.
1000 '''
1001 # strip off leading carriage-returns / newlines
1002 i = 0
1003 for i in range(len(content)):
1004 if content[i] not in '\r\n':
1005 break
1006 if i > 0:
1007 sections = blank_line.split(content[i:])
1008 else:
1009 sections = blank_line.split(content)
1011 # extract out the summary from the message
1012 summary = ''
1013 l = []
1014 for section in sections:
1015 #section = section.strip()
1016 if not section:
1017 continue
1018 lines = eol.split(section)
1019 if (lines[0] and lines[0][0] in '>|') or (len(lines) > 1 and
1020 lines[1] and lines[1][0] in '>|'):
1021 # see if there's a response somewhere inside this section (ie.
1022 # no blank line between quoted message and response)
1023 for line in lines[1:]:
1024 if line and line[0] not in '>|':
1025 break
1026 else:
1027 # we keep quoted bits if specified in the config
1028 if keep_citations:
1029 l.append(section)
1030 continue
1031 # keep this section - it has reponse stuff in it
1032 lines = lines[lines.index(line):]
1033 section = '\n'.join(lines)
1034 # and while we're at it, use the first non-quoted bit as
1035 # our summary
1036 summary = section
1038 if not summary:
1039 # if we don't have our summary yet use the first line of this
1040 # section
1041 summary = section
1042 elif signature.match(lines[0]) and 2 <= len(lines) <= 10:
1043 # lose any signature
1044 break
1045 elif original_msg.match(lines[0]):
1046 # ditch the stupid Outlook quoting of the entire original message
1047 break
1049 # and add the section to the output
1050 l.append(section)
1052 # figure the summary - find the first sentence-ending punctuation or the
1053 # first whole line, whichever is longest
1054 sentence = re.search(r'^([^!?\.]+[!?\.])', summary)
1055 if sentence:
1056 sentence = sentence.group(1)
1057 else:
1058 sentence = ''
1059 first = eol.split(summary)[0]
1060 summary = max(sentence, first)
1062 # Now reconstitute the message content minus the bits we don't care
1063 # about.
1064 if not keep_body:
1065 content = '\n\n'.join(l)
1067 return summary, content
1069 # vim: set filetype=python ts=4 sw=4 et si