1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
19 """An e-mail gateway for Roundup.
21 Incoming messages are examined for multiple parts:
22 . In a multipart/mixed message or part, each subpart is extracted and
23 examined. The text/plain subparts are assembled to form the textual
24 body of the message, to be stored in the file associated with a "msg"
25 class node. Any parts of other types are each stored in separate files
26 and given "file" class nodes that are linked to the "msg" node.
27 . In a multipart/alternative message or part, we look for a text/plain
28 subpart and ignore the other parts.
30 Summary
31 -------
32 The "summary" property on message nodes is taken from the first non-quoting
33 section in the message body. The message body is divided into sections by
34 blank lines. Sections where the second and all subsequent lines begin with
35 a ">" or "|" character are considered "quoting sections". The first line of
36 the first non-quoting section becomes the summary of the message.
38 Addresses
39 ---------
40 All of the addresses in the To: and Cc: headers of the incoming message are
41 looked up among the user nodes, and the corresponding users are placed in
42 the "recipients" property on the new "msg" node. The address in the From:
43 header similarly determines the "author" property of the new "msg"
44 node. The default handling for addresses that don't have corresponding
45 users is to create new users with no passwords and a username equal to the
46 address. (The web interface does not permit logins for users with no
47 passwords.) If we prefer to reject mail from outside sources, we can simply
48 register an auditor on the "user" class that prevents the creation of user
49 nodes with no passwords.
51 Actions
52 -------
53 The subject line of the incoming message is examined to determine whether
54 the message is an attempt to create a new item or to discuss an existing
55 item. A designator enclosed in square brackets is sought as the first thing
56 on the subject line (after skipping any "Fwd:" or "Re:" prefixes).
58 If an item designator (class name and id number) is found there, the newly
59 created "msg" node is added to the "messages" property for that item, and
60 any new "file" nodes are added to the "files" property for the item.
62 If just an item class name is found there, we attempt to create a new item
63 of that class with its "messages" property initialized to contain the new
64 "msg" node and its "files" property initialized to contain any new "file"
65 nodes.
67 Triggers
68 --------
69 Both cases may trigger detectors (in the first case we are calling the
70 set() method to add the message to the item's spool; in the second case we
71 are calling the create() method to create a new node). If an auditor raises
72 an exception, the original message is bounced back to the sender with the
73 explanatory message given in the exception.
75 $Id: mailgw.py,v 1.146 2004-03-26 00:44:11 richard Exp $
76 """
77 __docformat__ = 'restructuredtext'
79 import string, re, os, mimetools, cStringIO, smtplib, socket, binascii, quopri
80 import time, random, sys
81 import traceback, MimeWriter, rfc822
83 from roundup import hyperdb, date, password, rfc2822, exceptions
84 from roundup.mailer import Mailer
86 SENDMAILDEBUG = os.environ.get('SENDMAILDEBUG', '')
88 class MailGWError(ValueError):
89 pass
91 class MailUsageError(ValueError):
92 pass
94 class MailUsageHelp(Exception):
95 """ We need to send the help message to the user. """
96 pass
98 class Unauthorized(Exception):
99 """ Access denied """
100 pass
102 class IgnoreMessage(Exception):
103 """ A general class of message that we should ignore. """
104 pass
105 class IgnoreBulk(IgnoreMessage):
106 """ This is email from a mailing list or from a vacation program. """
107 pass
108 class IgnoreLoop(IgnoreMessage):
109 """ We've seen this message before... """
110 pass
112 def initialiseSecurity(security):
113 ''' Create some Permissions and Roles on the security object
115 This function is directly invoked by security.Security.__init__()
116 as a part of the Security object instantiation.
117 '''
118 security.addPermission(name="Email Registration",
119 description="Anonymous may register through e-mail")
120 p = security.addPermission(name="Email Access",
121 description="User may use the email interface")
122 security.addPermissionToRole('Admin', p)
124 def getparam(str, param):
125 ''' From the rfc822 "header" string, extract "param" if it appears.
126 '''
127 if ';' not in str:
128 return None
129 str = str[str.index(';'):]
130 while str[:1] == ';':
131 str = str[1:]
132 if ';' in str:
133 # XXX Should parse quotes!
134 end = str.index(';')
135 else:
136 end = len(str)
137 f = str[:end]
138 if '=' in f:
139 i = f.index('=')
140 if f[:i].strip().lower() == param:
141 return rfc822.unquote(f[i+1:].strip())
142 return None
144 class Message(mimetools.Message):
145 ''' subclass mimetools.Message so we can retrieve the parts of the
146 message...
147 '''
148 def getpart(self):
149 ''' Get a single part of a multipart message and return it as a new
150 Message instance.
151 '''
152 boundary = self.getparam('boundary')
153 mid, end = '--'+boundary, '--'+boundary+'--'
154 s = cStringIO.StringIO()
155 while 1:
156 line = self.fp.readline()
157 if not line:
158 break
159 if line.strip() in (mid, end):
160 break
161 s.write(line)
162 if not s.getvalue().strip():
163 return None
164 s.seek(0)
165 return Message(s)
167 def getparts(self):
168 """Get all parts of this multipart message."""
169 # skip over the intro to the first boundary
170 self.getpart()
172 # accumulate the other parts
173 parts = []
174 while 1:
175 part = self.getpart()
176 if part is None:
177 break
178 parts.append(part)
179 return parts
181 def getheader(self, name, default=None):
182 hdr = mimetools.Message.getheader(self, name, default)
183 if hdr:
184 hdr = hdr.replace('\n','') # Inserted by rfc822.readheaders
185 return rfc2822.decode_header(hdr)
187 def getname(self):
188 """Find an appropriate name for this message."""
189 if self.gettype() == 'message/rfc822':
190 # handle message/rfc822 specially - the name should be
191 # the subject of the actual e-mail embedded here
192 self.fp.seek(0)
193 name = Message(self.fp).getheader('subject')
194 else:
195 # try name on Content-Type
196 name = self.getparam('name')
197 if not name:
198 disp = self.getheader('content-disposition', None)
199 if disp:
200 name = getparam(disp, 'filename')
202 if name:
203 return name.strip()
205 def getbody(self):
206 """Get the decoded message body."""
207 self.rewindbody()
208 encoding = self.getencoding()
209 data = None
210 if encoding == 'base64':
211 # BUG: is base64 really used for text encoding or
212 # are we inserting zip files here.
213 data = binascii.a2b_base64(self.fp.read())
214 elif encoding == 'quoted-printable':
215 # the quopri module wants to work with files
216 decoded = cStringIO.StringIO()
217 quopri.decode(self.fp, decoded)
218 data = decoded.getvalue()
219 elif encoding == 'uuencoded':
220 data = binascii.a2b_uu(self.fp.read())
221 else:
222 # take it as text
223 data = self.fp.read()
225 # Encode message to unicode
226 charset = rfc2822.unaliasCharset(self.getparam("charset"))
227 if charset:
228 # Do conversion only if charset specified
229 edata = unicode(data, charset).encode('utf-8')
230 # Convert from dos eol to unix
231 edata = edata.replace('\r\n', '\n')
232 else:
233 # Leave message content as is
234 edata = data
236 return edata
238 # General multipart handling:
239 # Take the first text/plain part, anything else is considered an
240 # attachment.
241 # multipart/mixed: multiple "unrelated" parts.
242 # multipart/signed (rfc 1847):
243 # The control information is carried in the second of the two
244 # required body parts.
245 # ACTION: Default, so if content is text/plain we get it.
246 # multipart/encrypted (rfc 1847):
247 # The control information is carried in the first of the two
248 # required body parts.
249 # ACTION: Not handleable as the content is encrypted.
250 # multipart/related (rfc 1872, 2112, 2387):
251 # The Multipart/Related content-type addresses the MIME
252 # representation of compound objects.
253 # ACTION: Default. If we are lucky there is a text/plain.
254 # TODO: One should use the start part and look for an Alternative
255 # that is text/plain.
256 # multipart/Alternative (rfc 1872, 1892):
257 # only in "related" ?
258 # multipart/report (rfc 1892):
259 # e.g. mail system delivery status reports.
260 # ACTION: Default. Could be ignored or used for Delivery Notification
261 # flagging.
262 # multipart/form-data:
263 # For web forms only.
265 def extract_content(self, parent_type=None):
266 """Extract the body and the attachments recursively."""
267 content_type = self.gettype()
268 content = None
269 attachments = []
271 if content_type == 'text/plain':
272 content = self.getbody()
273 elif content_type[:10] == 'multipart/':
274 for part in self.getparts():
275 new_content, new_attach = part.extract_content(content_type)
277 # If we haven't found a text/plain part yet, take this one,
278 # otherwise make it an attachment.
279 if not content:
280 content = new_content
281 elif new_content:
282 attachments.append(part.as_attachment())
284 attachments.extend(new_attach)
285 elif (parent_type == 'multipart/signed' and
286 content_type == 'application/pgp-signature'):
287 # ignore it so it won't be saved as an attachment
288 pass
289 else:
290 attachments.append(self.as_attachment())
291 return content, attachments
293 def as_attachment(self):
294 """Return this message as an attachment."""
295 return (self.getname(), self.gettype(), self.getbody())
297 class MailGW:
299 # Matches subjects like:
300 # Re: "[issue1234] title of issue [status=resolved]"
301 subject_re = re.compile(r'''
302 (?P<refwd>\s*\W?\s*(fw|fwd|re|aw)\W\s*)*\s* # Re:
303 (?P<quote>")? # Leading "
304 (\[(?P<classname>[^\d\s]+) # [issue..
305 (?P<nodeid>\d+)? # ..1234]
306 \])?\s*
307 (?P<title>[^[]+)? # issue title
308 "? # Trailing "
309 (\[(?P<args>.+?)\])? # [prop=value]
310 ''', re.IGNORECASE|re.VERBOSE)
312 def __init__(self, instance, db, arguments={}):
313 self.instance = instance
314 self.db = db
315 self.arguments = arguments
316 self.mailer = Mailer(instance.config)
318 # should we trap exceptions (normal usage) or pass them through
319 # (for testing)
320 self.trapExceptions = 1
322 def do_pipe(self):
323 """ Read a message from standard input and pass it to the mail handler.
325 Read into an internal structure that we can seek on (in case
326 there's an error).
328 XXX: we may want to read this into a temporary file instead...
329 """
330 s = cStringIO.StringIO()
331 s.write(sys.stdin.read())
332 s.seek(0)
333 self.main(s)
334 return 0
336 def do_mailbox(self, filename):
337 """ Read a series of messages from the specified unix mailbox file and
338 pass each to the mail handler.
339 """
340 # open the spool file and lock it
341 import fcntl
342 # FCNTL is deprecated in py2.3 and fcntl takes over all the symbols
343 if hasattr(fcntl, 'LOCK_EX'):
344 FCNTL = fcntl
345 else:
346 import FCNTL
347 f = open(filename, 'r+')
348 fcntl.flock(f.fileno(), FCNTL.LOCK_EX)
350 # handle and clear the mailbox
351 try:
352 from mailbox import UnixMailbox
353 mailbox = UnixMailbox(f, factory=Message)
354 # grab one message
355 message = mailbox.next()
356 while message:
357 # handle this message
358 self.handle_Message(message)
359 message = mailbox.next()
360 # nuke the file contents
361 os.ftruncate(f.fileno(), 0)
362 except:
363 import traceback
364 traceback.print_exc()
365 return 1
366 fcntl.flock(f.fileno(), FCNTL.LOCK_UN)
367 return 0
369 def do_apop(self, server, user='', password=''):
370 ''' Do authentication POP
371 '''
372 self.do_pop(server, user, password, apop=1)
374 def do_pop(self, server, user='', password='', apop=0):
375 '''Read a series of messages from the specified POP server.
376 '''
377 import getpass, poplib, socket
378 try:
379 if not user:
380 user = raw_input('User: ')
381 if not password:
382 password = getpass.getpass()
383 except (KeyboardInterrupt, EOFError):
384 # Ctrl C or D maybe also Ctrl Z under Windows.
385 print "\nAborted by user."
386 return 1
388 # open a connection to the server and retrieve all messages
389 try:
390 server = poplib.POP3(server)
391 except socket.error, message:
392 print "POP server error:", message
393 return 1
394 if apop:
395 server.apop(user, password)
396 else:
397 server.user(user)
398 server.pass_(password)
399 numMessages = len(server.list()[1])
400 for i in range(1, numMessages+1):
401 # retr: returns
402 # [ pop response e.g. '+OK 459 octets',
403 # [ array of message lines ],
404 # number of octets ]
405 lines = server.retr(i)[1]
406 s = cStringIO.StringIO('\n'.join(lines))
407 s.seek(0)
408 self.handle_Message(Message(s))
409 # delete the message
410 server.dele(i)
412 # quit the server to commit changes.
413 server.quit()
414 return 0
416 def main(self, fp):
417 ''' fp - the file from which to read the Message.
418 '''
419 return self.handle_Message(Message(fp))
421 def handle_Message(self, message):
422 """Handle an RFC822 Message
424 Handle the Message object by calling handle_message() and then cope
425 with any errors raised by handle_message.
426 This method's job is to make that call and handle any
427 errors in a sane manner. It should be replaced if you wish to
428 handle errors in a different manner.
429 """
430 # in some rare cases, a particularly stuffed-up e-mail will make
431 # its way into here... try to handle it gracefully
433 sendto = message.getaddrlist('resent-from')
434 if not sendto:
435 sendto = message.getaddrlist('from')
436 if not sendto:
437 # very bad-looking message - we don't even know who sent it
438 # XXX we should use a log file here...
440 sendto = [self.instance.config.ADMIN_EMAIL]
442 m = ['Subject: badly formed message from mail gateway']
443 m.append('')
444 m.append('The mail gateway retrieved a message which has no From:')
445 m.append('line, indicating that it is corrupt. Please check your')
446 m.append('mail gateway source. Failed message is attached.')
447 m.append('')
448 self.mailer.bounce_message(message, sendto, m,
449 subject='Badly formed message from mail gateway')
450 return
452 # try normal message-handling
453 if not self.trapExceptions:
454 return self.handle_message(message)
455 try:
456 return self.handle_message(message)
457 except MailUsageHelp:
458 # bounce the message back to the sender with the usage message
459 fulldoc = '\n'.join(string.split(__doc__, '\n')[2:])
460 m = ['']
461 m.append('\n\nMail Gateway Help\n=================')
462 m.append(fulldoc)
463 self.mailer.bounce_message(message, [sendto[0][1]], m,
464 subject="Mail Gateway Help")
465 except MailUsageError, value:
466 # bounce the message back to the sender with the usage message
467 fulldoc = '\n'.join(string.split(__doc__, '\n')[2:])
468 m = ['']
469 m.append(str(value))
470 m.append('\n\nMail Gateway Help\n=================')
471 m.append(fulldoc)
472 self.mailer.bounce_message(message, [sendto[0][1]], m)
473 except Unauthorized, value:
474 # just inform the user that he is not authorized
475 m = ['']
476 m.append(str(value))
477 self.mailer.bounce_message(message, [sendto[0][1]], m)
478 except IgnoreMessage:
479 # XXX we should use a log file here...
480 # do not take any action
481 # this exception is thrown when email should be ignored
482 return
483 except:
484 # bounce the message back to the sender with the error message
485 # XXX we should use a log file here...
486 # let the admin know that something very bad is happening
487 sendto = [sendto[0][1], self.instance.config.ADMIN_EMAIL]
488 m = ['']
489 m.append('An unexpected error occurred during the processing')
490 m.append('of your message. The tracker administrator is being')
491 m.append('notified.\n')
492 m.append('---- traceback of failure ----')
493 s = cStringIO.StringIO()
494 import traceback
495 traceback.print_exc(None, s)
496 m.append(s.getvalue())
497 self.mailer.bounce_message(message, sendto, m)
499 def handle_message(self, message):
500 ''' message - a Message instance
502 Parse the message as per the module docstring.
503 '''
504 # detect loops
505 if message.getheader('x-roundup-loop', ''):
506 raise IgnoreLoop
508 # detect Precedence: Bulk
509 if (message.getheader('precedence', '') == 'bulk'):
510 raise IgnoreBulk
512 # XXX Don't enable. This doesn't work yet.
513 # "[^A-z.]tracker\+(?P<classname>[^\d\s]+)(?P<nodeid>\d+)\@some.dom.ain[^A-z.]"
514 # handle delivery to addresses like:tracker+issue25@some.dom.ain
515 # use the embedded issue number as our issue
516 # if hasattr(self.instance.config, 'EMAIL_ISSUE_ADDRESS_RE') and \
517 # self.instance.config.EMAIL_ISSUE_ADDRESS_RE:
518 # issue_re = self.instance.config.EMAIL_ISSUE_ADDRESS_RE
519 # for header in ['to', 'cc', 'bcc']:
520 # addresses = message.getheader(header, '')
521 # if addresses:
522 # # FIXME, this only finds the first match in the addresses.
523 # issue = re.search(issue_re, addresses, 'i')
524 # if issue:
525 # classname = issue.group('classname')
526 # nodeid = issue.group('nodeid')
527 # break
529 # determine the sender's address
530 from_list = message.getaddrlist('resent-from')
531 if not from_list:
532 from_list = message.getaddrlist('from')
534 # handle the subject line
535 subject = message.getheader('subject', '')
537 if not subject:
538 raise MailUsageError, '''
539 Emails to Roundup trackers must include a Subject: line!
540 '''
542 if subject.strip().lower() == 'help':
543 raise MailUsageHelp
545 m = self.subject_re.match(subject)
547 # check for well-formed subject line
548 if m:
549 # get the classname
550 classname = m.group('classname')
551 if classname is None:
552 # no classname, check if this a registration confirmation email
553 # or fallback on the default class
554 otk_re = re.compile('-- key (?P<otk>[a-zA-Z0-9]{32})')
555 otk = otk_re.search(m.group('title'))
556 if otk:
557 self.db.confirm_registration(otk.group('otk'))
558 subject = 'Your registration to %s is complete' % \
559 self.instance.config.TRACKER_NAME
560 sendto = [from_list[0][1]]
561 self.mailer.standard_message(sendto, subject, '')
562 return
563 elif hasattr(self.instance.config, 'MAIL_DEFAULT_CLASS') and \
564 self.instance.config.MAIL_DEFAULT_CLASS:
565 classname = self.instance.config.MAIL_DEFAULT_CLASS
566 else:
567 # fail
568 m = None
570 if not m:
571 raise MailUsageError, """
572 The message you sent to roundup did not contain a properly formed subject
573 line. The subject must contain a class name or designator to indicate the
574 'topic' of the message. For example:
575 Subject: [issue] This is a new issue
576 - this will create a new issue in the tracker with the title 'This is
577 a new issue'.
578 Subject: [issue1234] This is a followup to issue 1234
579 - this will append the message's contents to the existing issue 1234
580 in the tracker.
582 Subject was: '%s'
583 """%subject
585 # get the class
586 try:
587 cl = self.db.getclass(classname)
588 except KeyError:
589 raise MailUsageError, '''
590 The class name you identified in the subject line ("%s") does not exist in the
591 database.
593 Valid class names are: %s
594 Subject was: "%s"
595 '''%(classname, ', '.join(self.db.getclasses()), subject)
597 # get the optional nodeid
598 nodeid = m.group('nodeid')
600 # title is optional too
601 title = m.group('title')
602 if title:
603 title = title.strip()
604 else:
605 title = ''
607 # strip off the quotes that dumb emailers put around the subject, like
608 # Re: "[issue1] bla blah"
609 if m.group('quote') and title.endswith('"'):
610 title = title[:-1]
612 # but we do need either a title or a nodeid...
613 if nodeid is None and not title:
614 raise MailUsageError, '''
615 I cannot match your message to a node in the database - you need to either
616 supply a full node identifier (with number, eg "[issue123]" or keep the
617 previous subject title intact so I can match that.
619 Subject was: "%s"
620 '''%subject
622 # If there's no nodeid, check to see if this is a followup and
623 # maybe someone's responded to the initial mail that created an
624 # entry. Try to find the matching nodes with the same title, and
625 # use the _last_ one matched (since that'll _usually_ be the most
626 # recent...)
627 if nodeid is None and m.group('refwd'):
628 l = cl.stringFind(title=title)
629 if l:
630 nodeid = l[-1]
632 # if a nodeid was specified, make sure it's valid
633 if nodeid is not None and not cl.hasnode(nodeid):
634 raise MailUsageError, '''
635 The node specified by the designator in the subject of your message ("%s")
636 does not exist.
638 Subject was: "%s"
639 '''%(nodeid, subject)
641 # Handle the arguments specified by the email gateway command line.
642 # We do this by looping over the list of self.arguments looking for
643 # a -C to tell us what class then the -S setting string.
644 msg_props = {}
645 user_props = {}
646 file_props = {}
647 issue_props = {}
648 # so, if we have any arguments, use them
649 if self.arguments:
650 current_class = 'msg'
651 for option, propstring in self.arguments:
652 if option in ( '-C', '--class'):
653 current_class = propstring.strip()
654 if current_class not in ('msg', 'file', 'user', 'issue'):
655 raise MailUsageError, '''
656 The mail gateway is not properly set up. Please contact
657 %s and have them fix the incorrect class specified as:
658 %s
659 '''%(self.instance.config.ADMIN_EMAIL, current_class)
660 if option in ('-S', '--set'):
661 if current_class == 'issue' :
662 errors, issue_props = setPropArrayFromString(self,
663 cl, propstring.strip(), nodeid)
664 elif current_class == 'file' :
665 temp_cl = self.db.getclass('file')
666 errors, file_props = setPropArrayFromString(self,
667 temp_cl, propstring.strip())
668 elif current_class == 'msg' :
669 temp_cl = self.db.getclass('msg')
670 errors, msg_props = setPropArrayFromString(self,
671 temp_cl, propstring.strip())
672 elif current_class == 'user' :
673 temp_cl = self.db.getclass('user')
674 errors, user_props = setPropArrayFromString(self,
675 temp_cl, propstring.strip())
676 if errors:
677 raise MailUsageError, '''
678 The mail gateway is not properly set up. Please contact
679 %s and have them fix the incorrect properties:
680 %s
681 '''%(self.instance.config.ADMIN_EMAIL, errors)
683 #
684 # handle the users
685 #
686 # Don't create users if anonymous isn't allowed to register
687 create = 1
688 anonid = self.db.user.lookup('anonymous')
689 if not self.db.security.hasPermission('Email Registration', anonid):
690 create = 0
692 # ok, now figure out who the author is - create a new user if the
693 # "create" flag is true
694 author = uidFromAddress(self.db, from_list[0], create=create)
696 # if we're not recognised, and we don't get added as a user, then we
697 # must be anonymous
698 if not author:
699 author = anonid
701 # make sure the author has permission to use the email interface
702 if not self.db.security.hasPermission('Email Access', author):
703 if author == anonid:
704 # we're anonymous and we need to be a registered user
705 raise Unauthorized, '''
706 You are not a registered user.
708 Unknown address: %s
709 '''%from_list[0][1]
710 else:
711 # we're registered and we're _still_ not allowed access
712 raise Unauthorized, 'You are not permitted to access '\
713 'this tracker.'
715 # make sure they're allowed to edit this class of information
716 if not self.db.security.hasPermission('Edit', author, classname):
717 raise Unauthorized, 'You are not permitted to edit %s.'%classname
719 # the author may have been created - make sure the change is
720 # committed before we reopen the database
721 self.db.commit()
723 # reopen the database as the author
724 username = self.db.user.get(author, 'username')
725 self.db.close()
726 self.db = self.instance.open(username)
728 # re-get the class with the new database connection
729 cl = self.db.getclass(classname)
731 # now update the recipients list
732 recipients = []
733 tracker_email = self.instance.config.TRACKER_EMAIL.lower()
734 for recipient in message.getaddrlist('to') + message.getaddrlist('cc'):
735 r = recipient[1].strip().lower()
736 if r == tracker_email or not r:
737 continue
739 # look up the recipient - create if necessary (and we're
740 # allowed to)
741 recipient = uidFromAddress(self.db, recipient, create, **user_props)
743 # if all's well, add the recipient to the list
744 if recipient:
745 recipients.append(recipient)
747 #
748 # handle the subject argument list
749 #
750 # figure what the properties of this Class are
751 properties = cl.getprops()
752 props = {}
753 args = m.group('args')
754 if args:
755 errors, props = setPropArrayFromString(self, cl, args, nodeid)
756 # handle any errors parsing the argument list
757 if errors:
758 errors = '\n- '.join(map(str, errors))
759 raise MailUsageError, '''
760 There were problems handling your subject line argument list:
761 - %s
763 Subject was: "%s"
764 '''%(errors, subject)
767 # set the issue title to the subject
768 if properties.has_key('title') and not issue_props.has_key('title'):
769 issue_props['title'] = title.strip()
771 #
772 # handle message-id and in-reply-to
773 #
774 messageid = message.getheader('message-id')
775 inreplyto = message.getheader('in-reply-to') or ''
776 # generate a messageid if there isn't one
777 if not messageid:
778 messageid = "<%s.%s.%s%s@%s>"%(time.time(), random.random(),
779 classname, nodeid, self.instance.config.MAIL_DOMAIN)
781 # now handle the body - find the message
782 content, attachments = message.extract_content()
783 if content is None:
784 raise MailUsageError, '''
785 Roundup requires the submission to be plain text. The message parser could
786 not find a text/plain part to use.
787 '''
789 # figure how much we should muck around with the email body
790 keep_citations = getattr(self.instance.config, 'EMAIL_KEEP_QUOTED_TEXT',
791 'no') == 'yes'
792 keep_body = getattr(self.instance.config, 'EMAIL_LEAVE_BODY_UNCHANGED',
793 'no') == 'yes'
795 # parse the body of the message, stripping out bits as appropriate
796 summary, content = parseContent(content, keep_citations,
797 keep_body)
798 content = content.strip()
800 #
801 # handle the attachments
802 #
803 if properties.has_key('files'):
804 files = []
805 for (name, mime_type, data) in attachments:
806 if not name:
807 name = "unnamed"
808 try:
809 fileid = self.db.file.create(type=mime_type, name=name,
810 content=data, **file_props)
811 except exceptions.Reject:
812 pass
813 else:
814 files.append(fileid)
815 # attach the files to the issue
816 if nodeid:
817 # extend the existing files list
818 fileprop = cl.get(nodeid, 'files')
819 fileprop.extend(files)
820 props['files'] = fileprop
821 else:
822 # pre-load the files list
823 props['files'] = files
825 #
826 # create the message if there's a message body (content)
827 #
828 if (content and properties.has_key('messages')):
829 try:
830 message_id = self.db.msg.create(author=author,
831 recipients=recipients, date=date.Date('.'),
832 summary=summary, content=content, files=files,
833 messageid=messageid, inreplyto=inreplyto, **msg_props)
834 except exceptions.Reject:
835 pass
836 else:
837 # attach the message to the node
838 if nodeid:
839 # add the message to the node's list
840 messages = cl.get(nodeid, 'messages')
841 messages.append(message_id)
842 props['messages'] = messages
843 else:
844 # pre-load the messages list
845 props['messages'] = [message_id]
847 #
848 # perform the node change / create
849 #
850 try:
851 # merge the command line props defined in issue_props into
852 # the props dictionary because function(**props, **issue_props)
853 # is a syntax error.
854 for prop in issue_props.keys() :
855 if not props.has_key(prop) :
856 props[prop] = issue_props[prop]
857 if nodeid:
858 cl.set(nodeid, **props)
859 else:
860 nodeid = cl.create(**props)
861 except (TypeError, IndexError, ValueError), message:
862 raise MailUsageError, '''
863 There was a problem with the message you sent:
864 %s
865 '''%message
867 # commit the changes to the DB
868 self.db.commit()
870 return nodeid
873 def setPropArrayFromString(self, cl, propString, nodeid=None):
874 ''' takes string of form prop=value,value;prop2=value
875 and returns (error, prop[..])
876 '''
877 props = {}
878 errors = []
879 for prop in string.split(propString, ';'):
880 # extract the property name and value
881 try:
882 propname, value = prop.split('=')
883 except ValueError, message:
884 errors.append('not of form [arg=value,value,...;'
885 'arg=value,value,...]')
886 return (errors, props)
887 # convert the value to a hyperdb-usable value
888 propname = propname.strip()
889 try:
890 props[propname] = hyperdb.rawToHyperdb(self.db, cl, nodeid,
891 propname, value)
892 except hyperdb.HyperdbValueError, message:
893 errors.append(message)
894 return errors, props
897 def extractUserFromList(userClass, users):
898 '''Given a list of users, try to extract the first non-anonymous user
899 and return that user, otherwise return None
900 '''
901 if len(users) > 1:
902 for user in users:
903 # make sure we don't match the anonymous or admin user
904 if userClass.get(user, 'username') in ('admin', 'anonymous'):
905 continue
906 # first valid match will do
907 return user
908 # well, I guess we have no choice
909 return user[0]
910 elif users:
911 return users[0]
912 return None
915 def uidFromAddress(db, address, create=1, **user_props):
916 ''' address is from the rfc822 module, and therefore is (name, addr)
918 user is created if they don't exist in the db already
919 user_props may supply additional user information
920 '''
921 (realname, address) = address
923 # try a straight match of the address
924 user = extractUserFromList(db.user, db.user.stringFind(address=address))
925 if user is not None:
926 return user
928 # try the user alternate addresses if possible
929 props = db.user.getprops()
930 if props.has_key('alternate_addresses'):
931 users = db.user.filter(None, {'alternate_addresses': address})
932 user = extractUserFromList(db.user, users)
933 if user is not None:
934 return user
936 # try to match the username to the address (for local
937 # submissions where the address is empty)
938 user = extractUserFromList(db.user, db.user.stringFind(username=address))
940 # couldn't match address or username, so create a new user
941 if create:
942 # generate a username
943 if '@' in address:
944 username = address.split('@')[0]
945 else:
946 username = address
947 trying = username
948 n = 0
949 while 1:
950 try:
951 # does this username exist already?
952 db.user.lookup(trying)
953 except KeyError:
954 break
955 n += 1
956 trying = username + str(n)
958 # create!
959 try:
960 return db.user.create(username=trying, address=address,
961 realname=realname, roles=db.config.NEW_EMAIL_USER_ROLES,
962 password=password.Password(password.generatePassword()),
963 **user_props)
964 except exceptions.Reject:
965 return 0
966 else:
967 return 0
970 def parseContent(content, keep_citations, keep_body,
971 blank_line=re.compile(r'[\r\n]+\s*[\r\n]+'),
972 eol=re.compile(r'[\r\n]+'),
973 signature=re.compile(r'^[>|\s]*-- ?$'),
974 original_msg=re.compile(r'^[>|\s]*-----\s?Original Message\s?-----$')):
975 ''' The message body is divided into sections by blank lines.
976 Sections where the second and all subsequent lines begin with a ">"
977 or "|" character are considered "quoting sections". The first line of
978 the first non-quoting section becomes the summary of the message.
980 If keep_citations is true, then we keep the "quoting sections" in the
981 content.
982 If keep_body is true, we even keep the signature sections.
983 '''
984 # strip off leading carriage-returns / newlines
985 i = 0
986 for i in range(len(content)):
987 if content[i] not in '\r\n':
988 break
989 if i > 0:
990 sections = blank_line.split(content[i:])
991 else:
992 sections = blank_line.split(content)
994 # extract out the summary from the message
995 summary = ''
996 l = []
997 for section in sections:
998 #section = section.strip()
999 if not section:
1000 continue
1001 lines = eol.split(section)
1002 if (lines[0] and lines[0][0] in '>|') or (len(lines) > 1 and
1003 lines[1] and lines[1][0] in '>|'):
1004 # see if there's a response somewhere inside this section (ie.
1005 # no blank line between quoted message and response)
1006 for line in lines[1:]:
1007 if line and line[0] not in '>|':
1008 break
1009 else:
1010 # we keep quoted bits if specified in the config
1011 if keep_citations:
1012 l.append(section)
1013 continue
1014 # keep this section - it has reponse stuff in it
1015 lines = lines[lines.index(line):]
1016 section = '\n'.join(lines)
1017 # and while we're at it, use the first non-quoted bit as
1018 # our summary
1019 summary = section
1021 if not summary:
1022 # if we don't have our summary yet use the first line of this
1023 # section
1024 summary = section
1025 elif signature.match(lines[0]) and 2 <= len(lines) <= 10:
1026 # lose any signature
1027 break
1028 elif original_msg.match(lines[0]):
1029 # ditch the stupid Outlook quoting of the entire original message
1030 break
1032 # and add the section to the output
1033 l.append(section)
1035 # figure the summary - find the first sentence-ending punctuation or the
1036 # first whole line, whichever is longest
1037 sentence = re.search(r'^([^!?\.]+[!?\.])', summary)
1038 if sentence:
1039 sentence = sentence.group(1)
1040 else:
1041 sentence = ''
1042 first = eol.split(summary)[0]
1043 summary = max(sentence, first)
1045 # Now reconstitute the message content minus the bits we don't care
1046 # about.
1047 if not keep_body:
1048 content = '\n\n'.join(l)
1050 return summary, content
1052 # vim: set filetype=python ts=4 sw=4 et si