1 # -*- coding: utf-8 -*-
2 #
3 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
4 # This module is free software, and you may redistribute it and/or modify
5 # under the same terms as Python, so long as this copyright message and
6 # disclaimer are retained in their original form.
7 #
8 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
9 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
10 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
11 # POSSIBILITY OF SUCH DAMAGE.
12 #
13 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
14 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
15 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
16 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
17 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
18 #
20 """An e-mail gateway for Roundup.
22 Incoming messages are examined for multiple parts:
23 . In a multipart/mixed message or part, each subpart is extracted and
24 examined. The text/plain subparts are assembled to form the textual
25 body of the message, to be stored in the file associated with a "msg"
26 class node. Any parts of other types are each stored in separate files
27 and given "file" class nodes that are linked to the "msg" node.
28 . In a multipart/alternative message or part, we look for a text/plain
29 subpart and ignore the other parts.
30 . A message/rfc822 is treated similar tomultipart/mixed (except for
31 special handling of the first text part) if unpack_rfc822 is set in
32 the mailgw config section.
34 Summary
35 -------
36 The "summary" property on message nodes is taken from the first non-quoting
37 section in the message body. The message body is divided into sections by
38 blank lines. Sections where the second and all subsequent lines begin with
39 a ">" or "|" character are considered "quoting sections". The first line of
40 the first non-quoting section becomes the summary of the message.
42 Addresses
43 ---------
44 All of the addresses in the To: and Cc: headers of the incoming message are
45 looked up among the user nodes, and the corresponding users are placed in
46 the "recipients" property on the new "msg" node. The address in the From:
47 header similarly determines the "author" property of the new "msg"
48 node. The default handling for addresses that don't have corresponding
49 users is to create new users with no passwords and a username equal to the
50 address. (The web interface does not permit logins for users with no
51 passwords.) If we prefer to reject mail from outside sources, we can simply
52 register an auditor on the "user" class that prevents the creation of user
53 nodes with no passwords.
55 Actions
56 -------
57 The subject line of the incoming message is examined to determine whether
58 the message is an attempt to create a new item or to discuss an existing
59 item. A designator enclosed in square brackets is sought as the first thing
60 on the subject line (after skipping any "Fwd:" or "Re:" prefixes).
62 If an item designator (class name and id number) is found there, the newly
63 created "msg" node is added to the "messages" property for that item, and
64 any new "file" nodes are added to the "files" property for the item.
66 If just an item class name is found there, we attempt to create a new item
67 of that class with its "messages" property initialized to contain the new
68 "msg" node and its "files" property initialized to contain any new "file"
69 nodes.
71 Triggers
72 --------
73 Both cases may trigger detectors (in the first case we are calling the
74 set() method to add the message to the item's spool; in the second case we
75 are calling the create() method to create a new node). If an auditor raises
76 an exception, the original message is bounced back to the sender with the
77 explanatory message given in the exception.
79 $Id: mailgw.py,v 1.196 2008-07-23 03:04:44 richard Exp $
80 """
81 __docformat__ = 'restructuredtext'
83 import string, re, os, mimetools, cStringIO, smtplib, socket, binascii, quopri
84 import time, random, sys, logging
85 import traceback, rfc822
87 from email.Header import decode_header
89 from roundup import configuration, hyperdb, date, password, rfc2822, exceptions
90 from roundup.mailer import Mailer, MessageSendError
91 from roundup.i18n import _
92 from roundup.hyperdb import iter_roles
94 try:
95 import pyme, pyme.core, pyme.constants, pyme.constants.sigsum
96 except ImportError:
97 pyme = None
99 SENDMAILDEBUG = os.environ.get('SENDMAILDEBUG', '')
101 class MailGWError(ValueError):
102 pass
104 class MailUsageError(ValueError):
105 pass
107 class MailUsageHelp(Exception):
108 """ We need to send the help message to the user. """
109 pass
111 class Unauthorized(Exception):
112 """ Access denied """
113 pass
115 class IgnoreMessage(Exception):
116 """ A general class of message that we should ignore. """
117 pass
118 class IgnoreBulk(IgnoreMessage):
119 """ This is email from a mailing list or from a vacation program. """
120 pass
121 class IgnoreLoop(IgnoreMessage):
122 """ We've seen this message before... """
123 pass
125 def initialiseSecurity(security):
126 ''' Create some Permissions and Roles on the security object
128 This function is directly invoked by security.Security.__init__()
129 as a part of the Security object instantiation.
130 '''
131 p = security.addPermission(name="Email Access",
132 description="User may use the email interface")
133 security.addPermissionToRole('Admin', p)
135 def getparam(str, param):
136 ''' From the rfc822 "header" string, extract "param" if it appears.
137 '''
138 if ';' not in str:
139 return None
140 str = str[str.index(';'):]
141 while str[:1] == ';':
142 str = str[1:]
143 if ';' in str:
144 # XXX Should parse quotes!
145 end = str.index(';')
146 else:
147 end = len(str)
148 f = str[:end]
149 if '=' in f:
150 i = f.index('=')
151 if f[:i].strip().lower() == param:
152 return rfc822.unquote(f[i+1:].strip())
153 return None
155 def gpgh_key_getall(key, attr):
156 ''' return list of given attribute for all uids in
157 a key
158 '''
159 for u in key.uids:
160 yield getattr(u, attr)
162 def check_pgp_sigs(sigs, gpgctx, author, may_be_unsigned=False):
163 ''' Theoretically a PGP message can have several signatures. GPGME
164 returns status on all signatures in a list. Walk that list
165 looking for the author's signature. Note that even if incoming
166 signatures are not required, the processing fails if there is an
167 invalid signature.
168 '''
169 for sig in sigs:
170 key = gpgctx.get_key(sig.fpr, False)
171 # we really only care about the signature of the user who
172 # submitted the email
173 if key and (author in gpgh_key_getall(key, 'email')):
174 if sig.summary & pyme.constants.sigsum.VALID:
175 return True
176 else:
177 # try to narrow down the actual problem to give a more useful
178 # message in our bounce
179 if sig.summary & pyme.constants.sigsum.KEY_MISSING:
180 raise MailUsageError, \
181 _("Message signed with unknown key: %s") % sig.fpr
182 elif sig.summary & pyme.constants.sigsum.KEY_EXPIRED:
183 raise MailUsageError, \
184 _("Message signed with an expired key: %s") % sig.fpr
185 elif sig.summary & pyme.constants.sigsum.KEY_REVOKED:
186 raise MailUsageError, \
187 _("Message signed with a revoked key: %s") % sig.fpr
188 else:
189 raise MailUsageError, \
190 _("Invalid PGP signature detected.")
192 # we couldn't find a key belonging to the author of the email
193 if sigs:
194 raise MailUsageError, _("Message signed with unknown key: %s") % sig.fpr
195 elif not may_be_unsigned:
196 raise MailUsageError, _("Unsigned Message")
198 class Message(mimetools.Message):
199 ''' subclass mimetools.Message so we can retrieve the parts of the
200 message...
201 '''
202 def getpart(self):
203 ''' Get a single part of a multipart message and return it as a new
204 Message instance.
205 '''
206 boundary = self.getparam('boundary')
207 mid, end = '--'+boundary, '--'+boundary+'--'
208 s = cStringIO.StringIO()
209 while 1:
210 line = self.fp.readline()
211 if not line:
212 break
213 if line.strip() in (mid, end):
214 # according to rfc 1431 the preceding line ending is part of
215 # the boundary so we need to strip that
216 length = s.tell()
217 s.seek(-2, 1)
218 lineending = s.read(2)
219 if lineending == '\r\n':
220 s.truncate(length - 2)
221 elif lineending[1] in ('\r', '\n'):
222 s.truncate(length - 1)
223 else:
224 raise ValueError('Unknown line ending in message.')
225 break
226 s.write(line)
227 if not s.getvalue().strip():
228 return None
229 s.seek(0)
230 return Message(s)
232 def getparts(self):
233 """Get all parts of this multipart message."""
234 # skip over the intro to the first boundary
235 self.fp.seek(0)
236 self.getpart()
238 # accumulate the other parts
239 parts = []
240 while 1:
241 part = self.getpart()
242 if part is None:
243 break
244 parts.append(part)
245 return parts
247 def _decode_header_to_utf8(self, hdr):
248 l = []
249 prev_encoded = False
250 for part, encoding in decode_header(hdr):
251 if encoding:
252 part = part.decode(encoding)
253 # RFC 2047 specifies that between encoded parts spaces are
254 # swallowed while at the borders from encoded to non-encoded
255 # or vice-versa we must preserve a space. Multiple adjacent
256 # non-encoded parts should not occur.
257 if l and prev_encoded != bool(encoding):
258 l.append(' ')
259 prev_encoded = bool(encoding)
260 l.append(part)
261 return ''.join([s.encode('utf-8') for s in l])
263 def getheader(self, name, default=None):
264 hdr = mimetools.Message.getheader(self, name, default)
265 # TODO are there any other False values possible?
266 # TODO if not hdr: return hdr
267 if hdr is None:
268 return None
269 if not hdr:
270 return ''
271 if hdr:
272 hdr = hdr.replace('\n','') # Inserted by rfc822.readheaders
273 return self._decode_header_to_utf8(hdr)
275 def getaddrlist(self, name):
276 # overload to decode the name part of the address
277 l = []
278 for (name, addr) in mimetools.Message.getaddrlist(self, name):
279 name = self._decode_header_to_utf8(name)
280 l.append((name, addr))
281 return l
283 def getname(self):
284 """Find an appropriate name for this message."""
285 name = None
286 if self.gettype() == 'message/rfc822':
287 # handle message/rfc822 specially - the name should be
288 # the subject of the actual e-mail embedded here
289 # we add a '.eml' extension like other email software does it
290 self.fp.seek(0)
291 s = cStringIO.StringIO(self.getbody())
292 name = Message(s).getheader('subject')
293 if name:
294 name = name + '.eml'
295 if not name:
296 # try name on Content-Type
297 name = self.getparam('name')
298 if not name:
299 disp = self.getheader('content-disposition', None)
300 if disp:
301 name = getparam(disp, 'filename')
303 if name:
304 return name.strip()
306 def getbody(self):
307 """Get the decoded message body."""
308 self.rewindbody()
309 encoding = self.getencoding()
310 data = None
311 if encoding == 'base64':
312 # BUG: is base64 really used for text encoding or
313 # are we inserting zip files here.
314 data = binascii.a2b_base64(self.fp.read())
315 elif encoding == 'quoted-printable':
316 # the quopri module wants to work with files
317 decoded = cStringIO.StringIO()
318 quopri.decode(self.fp, decoded)
319 data = decoded.getvalue()
320 elif encoding == 'uuencoded':
321 data = binascii.a2b_uu(self.fp.read())
322 else:
323 # take it as text
324 data = self.fp.read()
326 # Encode message to unicode
327 charset = rfc2822.unaliasCharset(self.getparam("charset"))
328 if charset:
329 # Do conversion only if charset specified - handle
330 # badly-specified charsets
331 edata = unicode(data, charset, 'replace').encode('utf-8')
332 # Convert from dos eol to unix
333 edata = edata.replace('\r\n', '\n')
334 else:
335 # Leave message content as is
336 edata = data
338 return edata
340 # General multipart handling:
341 # Take the first text/plain part, anything else is considered an
342 # attachment.
343 # multipart/mixed:
344 # Multiple "unrelated" parts.
345 # multipart/Alternative (rfc 1521):
346 # Like multipart/mixed, except that we'd only want one of the
347 # alternatives. Generally a top-level part from MUAs sending HTML
348 # mail - there will be a text/plain version.
349 # multipart/signed (rfc 1847):
350 # The control information is carried in the second of the two
351 # required body parts.
352 # ACTION: Default, so if content is text/plain we get it.
353 # multipart/encrypted (rfc 1847):
354 # The control information is carried in the first of the two
355 # required body parts.
356 # ACTION: Not handleable as the content is encrypted.
357 # multipart/related (rfc 1872, 2112, 2387):
358 # The Multipart/Related content-type addresses the MIME
359 # representation of compound objects, usually HTML mail with embedded
360 # images. Usually appears as an alternative.
361 # ACTION: Default, if we must.
362 # multipart/report (rfc 1892):
363 # e.g. mail system delivery status reports.
364 # ACTION: Default. Could be ignored or used for Delivery Notification
365 # flagging.
366 # multipart/form-data:
367 # For web forms only.
368 # message/rfc822:
369 # Only if configured in [mailgw] unpack_rfc822
371 def extract_content(self, parent_type=None, ignore_alternatives=False,
372 unpack_rfc822=False):
373 """Extract the body and the attachments recursively.
375 If the content is hidden inside a multipart/alternative part,
376 we use the *last* text/plain part of the *first*
377 multipart/alternative in the whole message.
378 """
379 content_type = self.gettype()
380 content = None
381 attachments = []
383 if content_type == 'text/plain':
384 content = self.getbody()
385 elif content_type[:10] == 'multipart/':
386 content_found = bool (content)
387 ig = ignore_alternatives and not content_found
388 for part in self.getparts():
389 new_content, new_attach = part.extract_content(content_type,
390 not content and ig, unpack_rfc822)
392 # If we haven't found a text/plain part yet, take this one,
393 # otherwise make it an attachment.
394 if not content:
395 content = new_content
396 cpart = part
397 elif new_content:
398 if content_found or content_type != 'multipart/alternative':
399 attachments.append(part.text_as_attachment())
400 else:
401 # if we have found a text/plain in the current
402 # multipart/alternative and find another one, we
403 # use the first as an attachment (if configured)
404 # and use the second one because rfc 2046, sec.
405 # 5.1.4. specifies that later parts are better
406 # (thanks to Philipp Gortan for pointing this
407 # out)
408 attachments.append(cpart.text_as_attachment())
409 content = new_content
410 cpart = part
412 attachments.extend(new_attach)
413 if ig and content_type == 'multipart/alternative' and content:
414 attachments = []
415 elif unpack_rfc822 and content_type == 'message/rfc822':
416 s = cStringIO.StringIO(self.getbody())
417 m = Message(s)
418 ig = ignore_alternatives and not content
419 new_content, attachments = m.extract_content(m.gettype(), ig,
420 unpack_rfc822)
421 attachments.insert(0, m.text_as_attachment())
422 elif (parent_type == 'multipart/signed' and
423 content_type == 'application/pgp-signature'):
424 # ignore it so it won't be saved as an attachment
425 pass
426 else:
427 attachments.append(self.as_attachment())
428 return content, attachments
430 def text_as_attachment(self):
431 """Return first text/plain part as Message"""
432 if not self.gettype().startswith ('multipart/'):
433 return self.as_attachment()
434 for part in self.getparts():
435 content_type = part.gettype()
436 if content_type == 'text/plain':
437 return part.as_attachment()
438 elif content_type.startswith ('multipart/'):
439 p = part.text_as_attachment()
440 if p:
441 return p
442 return None
444 def as_attachment(self):
445 """Return this message as an attachment."""
446 return (self.getname(), self.gettype(), self.getbody())
448 def pgp_signed(self):
449 ''' RFC 3156 requires OpenPGP MIME mail to have the protocol parameter
450 '''
451 return self.gettype() == 'multipart/signed' \
452 and self.typeheader.find('protocol="application/pgp-signature"') != -1
454 def pgp_encrypted(self):
455 ''' RFC 3156 requires OpenPGP MIME mail to have the protocol parameter
456 '''
457 return self.gettype() == 'multipart/encrypted' \
458 and self.typeheader.find('protocol="application/pgp-encrypted"') != -1
460 def decrypt(self, author, may_be_unsigned=False):
461 ''' decrypt an OpenPGP MIME message
462 This message must be signed as well as encrypted using the
463 "combined" method if incoming signatures are configured.
464 The decrypted contents are returned as a new message.
465 '''
466 (hdr, msg) = self.getparts()
467 # According to the RFC 3156 encrypted mail must have exactly two parts.
468 # The first part contains the control information. Let's verify that
469 # the message meets the RFC before we try to decrypt it.
470 if hdr.getbody().strip() != 'Version: 1' \
471 or hdr.gettype() != 'application/pgp-encrypted':
472 raise MailUsageError, \
473 _("Unknown multipart/encrypted version.")
475 context = pyme.core.Context()
476 ciphertext = pyme.core.Data(msg.getbody())
477 plaintext = pyme.core.Data()
479 result = context.op_decrypt_verify(ciphertext, plaintext)
481 if result:
482 raise MailUsageError, _("Unable to decrypt your message.")
484 # we've decrypted it but that just means they used our public
485 # key to send it to us. now check the signatures to see if it
486 # was signed by someone we trust
487 result = context.op_verify_result()
488 check_pgp_sigs(result.signatures, context, author,
489 may_be_unsigned = may_be_unsigned)
491 plaintext.seek(0,0)
492 # pyme.core.Data implements a seek method with a different signature
493 # than roundup can handle. So we'll put the data in a container that
494 # the Message class can work with.
495 c = cStringIO.StringIO()
496 c.write(plaintext.read())
497 c.seek(0)
498 return Message(c)
500 def verify_signature(self, author):
501 ''' verify the signature of an OpenPGP MIME message
502 This only handles detached signatures. Old style
503 PGP mail (i.e. '-----BEGIN PGP SIGNED MESSAGE----')
504 is archaic and not supported :)
505 '''
506 # we don't check the micalg parameter...gpgme seems to
507 # figure things out on its own
508 (msg, sig) = self.getparts()
510 if sig.gettype() != 'application/pgp-signature':
511 raise MailUsageError, \
512 _("No PGP signature found in message.")
514 # msg.getbody() is skipping over some headers that are
515 # required to be present for verification to succeed so
516 # we'll do this by hand
517 msg.fp.seek(0)
518 # according to rfc 3156 the data "MUST first be converted
519 # to its content-type specific canonical form. For
520 # text/plain this means conversion to an appropriate
521 # character set and conversion of line endings to the
522 # canonical <CR><LF> sequence."
523 # TODO: what about character set conversion?
524 canonical_msg = re.sub('(?<!\r)\n', '\r\n', msg.fp.read())
525 msg_data = pyme.core.Data(canonical_msg)
526 sig_data = pyme.core.Data(sig.getbody())
528 context = pyme.core.Context()
529 context.op_verify(sig_data, msg_data, None)
531 # check all signatures for validity
532 result = context.op_verify_result()
533 check_pgp_sigs(result.signatures, context, author)
535 class parsedMessage:
537 def __init__(self, mailgw, message):
538 self.mailgw = mailgw
539 self.config = mailgw.instance.config
540 self.db = mailgw.db
541 self.message = message
542 self.subject = message.getheader('subject', '')
543 self.has_prefix = False
544 self.matches = dict.fromkeys(['refwd', 'quote', 'classname',
545 'nodeid', 'title', 'args', 'argswhole'])
546 self.from_list = message.getaddrlist('resent-from') \
547 or message.getaddrlist('from')
548 self.pfxmode = self.config['MAILGW_SUBJECT_PREFIX_PARSING']
549 self.sfxmode = self.config['MAILGW_SUBJECT_SUFFIX_PARSING']
550 # these are filled in by subsequent parsing steps
551 self.classname = None
552 self.properties = None
553 self.cl = None
554 self.nodeid = None
555 self.author = None
556 self.recipients = None
557 self.msg_props = {}
558 self.props = None
559 self.content = None
560 self.attachments = None
561 self.crypt = False
563 def handle_ignore(self):
564 ''' Check to see if message can be safely ignored:
565 detect loops and
566 Precedence: Bulk, or Microsoft Outlook autoreplies
567 '''
568 if self.message.getheader('x-roundup-loop', ''):
569 raise IgnoreLoop
570 if (self.message.getheader('precedence', '') == 'bulk'
571 or self.subject.lower().find("autoreply") > 0):
572 raise IgnoreBulk
574 def handle_help(self):
575 ''' Check to see if the message contains a usage/help request
576 '''
577 if self.subject.strip().lower() == 'help':
578 raise MailUsageHelp
580 def check_subject(self):
581 ''' Check to see if the message contains a valid subject line
582 '''
583 if not self.subject:
584 raise MailUsageError, _("""
585 Emails to Roundup trackers must include a Subject: line!
586 """)
588 def parse_subject(self):
589 ''' Matches subjects like:
590 Re: "[issue1234] title of issue [status=resolved]"
592 Each part of the subject is matched, stored, then removed from the
593 start of the subject string as needed. The stored values are then
594 returned
595 '''
597 tmpsubject = self.subject
599 sd_open, sd_close = self.config['MAILGW_SUBJECT_SUFFIX_DELIMITERS']
600 delim_open = re.escape(sd_open)
601 if delim_open in '[(': delim_open = '\\' + delim_open
602 delim_close = re.escape(sd_close)
603 if delim_close in '[(': delim_close = '\\' + delim_close
605 # Look for Re: et. al. Used later on for MAILGW_SUBJECT_CONTENT_MATCH
606 re_re = r"(?P<refwd>%s)\s*" % self.config["MAILGW_REFWD_RE"].pattern
607 m = re.match(re_re, tmpsubject, re.IGNORECASE|re.VERBOSE|re.UNICODE)
608 if m:
609 m = m.groupdict()
610 if m['refwd']:
611 self.matches.update(m)
612 tmpsubject = tmpsubject[len(m['refwd']):] # Consume Re:
614 # Look for Leading "
615 m = re.match(r'(?P<quote>\s*")', tmpsubject,
616 re.IGNORECASE)
617 if m:
618 self.matches.update(m.groupdict())
619 tmpsubject = tmpsubject[len(self.matches['quote']):] # Consume quote
621 # Check if the subject includes a prefix
622 self.has_prefix = re.search(r'^%s(\w+)%s'%(delim_open,
623 delim_close), tmpsubject.strip())
625 # Match the classname if specified
626 class_re = r'%s(?P<classname>(%s))(?P<nodeid>\d+)?%s'%(delim_open,
627 "|".join(self.db.getclasses()), delim_close)
628 # Note: re.search, not re.match as there might be garbage
629 # (mailing list prefix, etc.) before the class identifier
630 m = re.search(class_re, tmpsubject, re.IGNORECASE)
631 if m:
632 self.matches.update(m.groupdict())
633 # Skip to the end of the class identifier, including any
634 # garbage before it.
636 tmpsubject = tmpsubject[m.end():]
638 # Match the title of the subject
639 # if we've not found a valid classname prefix then force the
640 # scanning to handle there being a leading delimiter
641 title_re = r'(?P<title>%s[^%s]*)'%(
642 not self.matches['classname'] and '.' or '', delim_open)
643 m = re.match(title_re, tmpsubject.strip(), re.IGNORECASE)
644 if m:
645 self.matches.update(m.groupdict())
646 tmpsubject = tmpsubject[len(self.matches['title']):] # Consume title
648 if self.matches['title']:
649 self.matches['title'] = self.matches['title'].strip()
650 else:
651 self.matches['title'] = ''
653 # strip off the quotes that dumb emailers put around the subject, like
654 # Re: "[issue1] bla blah"
655 if self.matches['quote'] and self.matches['title'].endswith('"'):
656 self.matches['title'] = self.matches['title'][:-1]
658 # Match any arguments specified
659 args_re = r'(?P<argswhole>%s(?P<args>.+?)%s)?'%(delim_open,
660 delim_close)
661 m = re.search(args_re, tmpsubject.strip(), re.IGNORECASE|re.VERBOSE)
662 if m:
663 self.matches.update(m.groupdict())
665 def rego_confirm(self):
666 ''' Check for registration OTK and confirm the registration if found
667 '''
669 if self.config['EMAIL_REGISTRATION_CONFIRMATION']:
670 otk_re = re.compile('-- key (?P<otk>[a-zA-Z0-9]{32})')
671 otk = otk_re.search(self.matches['title'] or '')
672 if otk:
673 self.db.confirm_registration(otk.group('otk'))
674 subject = 'Your registration to %s is complete' % \
675 self.config['TRACKER_NAME']
676 sendto = [self.from_list[0][1]]
677 self.mailgw.mailer.standard_message(sendto, subject, '')
678 return 1
679 return 0
681 def get_classname(self):
682 ''' Determine the classname of the node being created/edited
683 '''
684 subject = self.subject
686 # get the classname
687 if self.pfxmode == 'none':
688 classname = None
689 else:
690 classname = self.matches['classname']
692 if not classname and self.has_prefix and self.pfxmode == 'strict':
693 raise MailUsageError, _("""
694 The message you sent to roundup did not contain a properly formed subject
695 line. The subject must contain a class name or designator to indicate the
696 'topic' of the message. For example:
697 Subject: [issue] This is a new issue
698 - this will create a new issue in the tracker with the title 'This is
699 a new issue'.
700 Subject: [issue1234] This is a followup to issue 1234
701 - this will append the message's contents to the existing issue 1234
702 in the tracker.
704 Subject was: '%(subject)s'
705 """) % locals()
707 # try to get the class specified - if "loose" or "none" then fall
708 # back on the default
709 attempts = []
710 if classname:
711 attempts.append(classname)
713 if self.mailgw.default_class:
714 attempts.append(self.mailgw.default_class)
715 else:
716 attempts.append(self.config['MAILGW_DEFAULT_CLASS'])
718 # first valid class name wins
719 self.cl = None
720 for trycl in attempts:
721 try:
722 self.cl = self.db.getclass(trycl)
723 classname = self.classname = trycl
724 break
725 except KeyError:
726 pass
728 if not self.cl:
729 validname = ', '.join(self.db.getclasses())
730 if classname:
731 raise MailUsageError, _("""
732 The class name you identified in the subject line ("%(classname)s") does
733 not exist in the database.
735 Valid class names are: %(validname)s
736 Subject was: "%(subject)s"
737 """) % locals()
738 else:
739 raise MailUsageError, _("""
740 You did not identify a class name in the subject line and there is no
741 default set for this tracker. The subject must contain a class name or
742 designator to indicate the 'topic' of the message. For example:
743 Subject: [issue] This is a new issue
744 - this will create a new issue in the tracker with the title 'This is
745 a new issue'.
746 Subject: [issue1234] This is a followup to issue 1234
747 - this will append the message's contents to the existing issue 1234
748 in the tracker.
750 Subject was: '%(subject)s'
751 """) % locals()
752 # get the class properties
753 self.properties = self.cl.getprops()
756 def get_nodeid(self):
757 ''' Determine the nodeid from the message and return it if found
758 '''
759 title = self.matches['title']
760 subject = self.subject
762 if self.pfxmode == 'none':
763 nodeid = None
764 else:
765 nodeid = self.matches['nodeid']
767 # try in-reply-to to match the message if there's no nodeid
768 inreplyto = self.message.getheader('in-reply-to') or ''
769 if nodeid is None and inreplyto:
770 l = self.db.getclass('msg').stringFind(messageid=inreplyto)
771 if l:
772 nodeid = self.cl.filter(None, {'messages':l})[0]
775 # but we do need either a title or a nodeid...
776 if nodeid is None and not title:
777 raise MailUsageError, _("""
778 I cannot match your message to a node in the database - you need to either
779 supply a full designator (with number, eg "[issue123]") or keep the
780 previous subject title intact so I can match that.
782 Subject was: "%(subject)s"
783 """) % locals()
785 # If there's no nodeid, check to see if this is a followup and
786 # maybe someone's responded to the initial mail that created an
787 # entry. Try to find the matching nodes with the same title, and
788 # use the _last_ one matched (since that'll _usually_ be the most
789 # recent...). The subject_content_match config may specify an
790 # additional restriction based on the matched node's creation or
791 # activity.
792 tmatch_mode = self.config['MAILGW_SUBJECT_CONTENT_MATCH']
793 if tmatch_mode != 'never' and nodeid is None and self.matches['refwd']:
794 l = self.cl.stringFind(title=title)
795 limit = None
796 if (tmatch_mode.startswith('creation') or
797 tmatch_mode.startswith('activity')):
798 limit, interval = tmatch_mode.split(' ', 1)
799 threshold = date.Date('.') - date.Interval(interval)
800 for id in l:
801 if limit:
802 if threshold < self.cl.get(id, limit):
803 nodeid = id
804 else:
805 nodeid = id
807 # if a nodeid was specified, make sure it's valid
808 if nodeid is not None and not self.cl.hasnode(nodeid):
809 if self.pfxmode == 'strict':
810 raise MailUsageError, _("""
811 The node specified by the designator in the subject of your message
812 ("%(nodeid)s") does not exist.
814 Subject was: "%(subject)s"
815 """) % locals()
816 else:
817 nodeid = None
818 self.nodeid = nodeid
820 def get_author_id(self):
821 ''' Attempt to get the author id from the existing registered users,
822 otherwise attempt to register a new user and return their id
823 '''
824 # Don't create users if anonymous isn't allowed to register
825 create = 1
826 anonid = self.db.user.lookup('anonymous')
827 if not (self.db.security.hasPermission('Register', anonid, 'user')
828 and self.db.security.hasPermission('Email Access', anonid)):
829 create = 0
831 # ok, now figure out who the author is - create a new user if the
832 # "create" flag is true
833 author = uidFromAddress(self.db, self.from_list[0], create=create)
835 # if we're not recognised, and we don't get added as a user, then we
836 # must be anonymous
837 if not author:
838 author = anonid
840 # make sure the author has permission to use the email interface
841 if not self.db.security.hasPermission('Email Access', author):
842 if author == anonid:
843 # we're anonymous and we need to be a registered user
844 from_address = self.from_list[0][1]
845 registration_info = ""
846 if self.db.security.hasPermission('Web Access', author) and \
847 self.db.security.hasPermission('Register', anonid, 'user'):
848 tracker_web = self.config.TRACKER_WEB
849 registration_info = """ Please register at:
851 %(tracker_web)suser?template=register
853 ...before sending mail to the tracker.""" % locals()
855 raise Unauthorized, _("""
856 You are not a registered user.%(registration_info)s
858 Unknown address: %(from_address)s
859 """) % locals()
860 else:
861 # we're registered and we're _still_ not allowed access
862 raise Unauthorized, _(
863 'You are not permitted to access this tracker.')
864 self.author = author
866 def check_permissions(self):
867 ''' Check if the author has permission to edit or create this
868 class of node
869 '''
870 if self.nodeid:
871 if not self.db.security.hasPermission('Edit', self.author,
872 self.classname, itemid=self.nodeid):
873 raise Unauthorized, _(
874 'You are not permitted to edit %(classname)s.'
875 ) % self.__dict__
876 else:
877 if not self.db.security.hasPermission('Create', self.author,
878 self.classname):
879 raise Unauthorized, _(
880 'You are not permitted to create %(classname)s.'
881 ) % self.__dict__
883 def commit_and_reopen_as_author(self):
884 ''' the author may have been created - make sure the change is
885 committed before we reopen the database
886 then re-open the database as the author
887 '''
888 self.db.commit()
890 # set the database user as the author
891 username = self.db.user.get(self.author, 'username')
892 self.db.setCurrentUser(username)
894 # re-get the class with the new database connection
895 self.cl = self.db.getclass(self.classname)
897 def get_recipients(self):
898 ''' Get the list of recipients who were included in message and
899 register them as users if possible
900 '''
901 # Don't create users if anonymous isn't allowed to register
902 create = 1
903 anonid = self.db.user.lookup('anonymous')
904 if not (self.db.security.hasPermission('Register', anonid, 'user')
905 and self.db.security.hasPermission('Email Access', anonid)):
906 create = 0
908 # get the user class arguments from the commandline
909 user_props = self.mailgw.get_class_arguments('user')
911 # now update the recipients list
912 recipients = []
913 tracker_email = self.config['TRACKER_EMAIL'].lower()
914 msg_to = self.message.getaddrlist('to')
915 msg_cc = self.message.getaddrlist('cc')
916 for recipient in msg_to + msg_cc:
917 r = recipient[1].strip().lower()
918 if r == tracker_email or not r:
919 continue
921 # look up the recipient - create if necessary (and we're
922 # allowed to)
923 recipient = uidFromAddress(self.db, recipient, create, **user_props)
925 # if all's well, add the recipient to the list
926 if recipient:
927 recipients.append(recipient)
928 self.recipients = recipients
930 def get_props(self):
931 ''' Generate all the props for the new/updated node and return them
932 '''
933 subject = self.subject
935 # get the commandline arguments for issues
936 issue_props = self.mailgw.get_class_arguments('issue', self.classname)
938 #
939 # handle the subject argument list
940 #
941 # figure what the properties of this Class are
942 props = {}
943 args = self.matches['args']
944 argswhole = self.matches['argswhole']
945 title = self.matches['title']
947 # Reform the title
948 if self.matches['nodeid'] and self.nodeid is None:
949 title = subject
951 if args:
952 if self.sfxmode == 'none':
953 title += ' ' + argswhole
954 else:
955 errors, props = setPropArrayFromString(self, self.cl, args,
956 self.nodeid)
957 # handle any errors parsing the argument list
958 if errors:
959 if self.sfxmode == 'strict':
960 errors = '\n- '.join(map(str, errors))
961 raise MailUsageError, _("""
962 There were problems handling your subject line argument list:
963 - %(errors)s
965 Subject was: "%(subject)s"
966 """) % locals()
967 else:
968 title += ' ' + argswhole
971 # set the issue title to the subject
972 title = title.strip()
973 if (title and self.properties.has_key('title') and not
974 issue_props.has_key('title')):
975 issue_props['title'] = title
976 if (self.nodeid and self.properties.has_key('title') and not
977 self.config['MAILGW_SUBJECT_UPDATES_TITLE']):
978 issue_props['title'] = self.cl.get(self.nodeid,'title')
980 # merge the command line props defined in issue_props into
981 # the props dictionary because function(**props, **issue_props)
982 # is a syntax error.
983 for prop in issue_props.keys() :
984 if not props.has_key(prop) :
985 props[prop] = issue_props[prop]
987 self.props = props
989 def get_pgp_message(self):
990 ''' If they've enabled PGP processing then verify the signature
991 or decrypt the message
992 '''
993 def pgp_role():
994 """ if PGP_ROLES is specified the user must have a Role in the list
995 or we will skip PGP processing
996 """
997 if self.config.PGP_ROLES:
998 return self.db.user.has_role(self.author,
999 *iter_roles(self.config.PGP_ROLES))
1000 else:
1001 return True
1003 if self.config.PGP_ENABLE:
1004 if pgp_role() and self.config.PGP_ENCRYPT:
1005 self.crypt = True
1006 assert pyme, 'pyme is not installed'
1007 # signed/encrypted mail must come from the primary address
1008 author_address = self.db.user.get(self.author, 'address')
1009 if self.config.PGP_HOMEDIR:
1010 os.environ['GNUPGHOME'] = self.config.PGP_HOMEDIR
1011 if self.config.PGP_REQUIRE_INCOMING in ('encrypted', 'both') \
1012 and pgp_role() and not self.message.pgp_encrypted():
1013 raise MailUsageError, _(
1014 "This tracker has been configured to require all email "
1015 "be PGP encrypted.")
1016 if self.message.pgp_signed():
1017 self.message.verify_signature(author_address)
1018 elif self.message.pgp_encrypted():
1019 # Replace message with the contents of the decrypted
1020 # message for content extraction
1021 # Note: the bounce-handling code now makes sure that
1022 # either the encrypted mail received is sent back or
1023 # that the error message is encrypted if needed.
1024 encr_only = self.config.PGP_REQUIRE_INCOMING == 'encrypted'
1025 encr_only = encr_only or not pgp_role()
1026 self.crypt = True
1027 self.message = self.message.decrypt(author_address,
1028 may_be_unsigned = encr_only)
1029 elif pgp_role():
1030 raise MailUsageError, _("""
1031 This tracker has been configured to require all email be PGP signed or
1032 encrypted.""")
1034 def get_content_and_attachments(self):
1035 ''' get the attachments and first text part from the message
1036 '''
1037 ig = self.config.MAILGW_IGNORE_ALTERNATIVES
1038 self.content, self.attachments = self.message.extract_content(
1039 ignore_alternatives=ig,
1040 unpack_rfc822=self.config.MAILGW_UNPACK_RFC822)
1043 def create_files(self):
1044 ''' Create a file for each attachment in the message
1045 '''
1046 if not self.properties.has_key('files'):
1047 return
1048 files = []
1049 file_props = self.mailgw.get_class_arguments('file')
1051 if self.attachments:
1052 for (name, mime_type, data) in self.attachments:
1053 if not self.db.security.hasPermission('Create', self.author,
1054 'file'):
1055 raise Unauthorized, _(
1056 'You are not permitted to create files.')
1057 if not name:
1058 name = "unnamed"
1059 try:
1060 fileid = self.db.file.create(type=mime_type, name=name,
1061 content=data, **file_props)
1062 except exceptions.Reject:
1063 pass
1064 else:
1065 files.append(fileid)
1066 # allowed to attach the files to an existing node?
1067 if self.nodeid and not self.db.security.hasPermission('Edit',
1068 self.author, self.classname, 'files'):
1069 raise Unauthorized, _(
1070 'You are not permitted to add files to %(classname)s.'
1071 ) % self.__dict__
1073 self.msg_props['files'] = files
1074 if self.nodeid:
1075 # extend the existing files list
1076 fileprop = self.cl.get(self.nodeid, 'files')
1077 fileprop.extend(files)
1078 files = fileprop
1080 self.props['files'] = files
1082 def create_msg(self):
1083 ''' Create msg containing all the relevant information from the message
1084 '''
1085 if not self.properties.has_key('messages'):
1086 return
1087 msg_props = self.mailgw.get_class_arguments('msg')
1088 self.msg_props.update (msg_props)
1090 # Get the message ids
1091 inreplyto = self.message.getheader('in-reply-to') or ''
1092 messageid = self.message.getheader('message-id')
1093 # generate a messageid if there isn't one
1094 if not messageid:
1095 messageid = "<%s.%s.%s%s@%s>"%(time.time(), random.random(),
1096 self.classname, self.nodeid, self.config['MAIL_DOMAIN'])
1098 if self.content is None:
1099 raise MailUsageError, _("""
1100 Roundup requires the submission to be plain text. The message parser could
1101 not find a text/plain part to use.
1102 """)
1104 # parse the body of the message, stripping out bits as appropriate
1105 summary, content = parseContent(self.content, config=self.config)
1106 content = content.strip()
1108 if content:
1109 if not self.db.security.hasPermission('Create', self.author, 'msg'):
1110 raise Unauthorized, _(
1111 'You are not permitted to create messages.')
1113 try:
1114 message_id = self.db.msg.create(author=self.author,
1115 recipients=self.recipients, date=date.Date('.'),
1116 summary=summary, content=content,
1117 messageid=messageid, inreplyto=inreplyto, **self.msg_props)
1118 except exceptions.Reject, error:
1119 raise MailUsageError, _("""
1120 Mail message was rejected by a detector.
1121 %(error)s
1122 """) % locals()
1123 # allowed to attach the message to the existing node?
1124 if self.nodeid and not self.db.security.hasPermission('Edit',
1125 self.author, self.classname, 'messages'):
1126 raise Unauthorized, _(
1127 'You are not permitted to add messages to %(classname)s.'
1128 ) % self.__dict__
1130 if self.nodeid:
1131 # add the message to the node's list
1132 messages = self.cl.get(self.nodeid, 'messages')
1133 messages.append(message_id)
1134 self.props['messages'] = messages
1135 else:
1136 # pre-load the messages list
1137 self.props['messages'] = [message_id]
1139 def create_node(self):
1140 ''' Create/update a node using self.props
1141 '''
1142 classname = self.classname
1143 try:
1144 if self.nodeid:
1145 # Check permissions for each property
1146 for prop in self.props.keys():
1147 if not self.db.security.hasPermission('Edit', self.author,
1148 classname, prop):
1149 raise Unauthorized, _('You are not permitted to edit '
1150 'property %(prop)s of class %(classname)s.'
1151 ) % locals()
1152 self.cl.set(self.nodeid, **self.props)
1153 else:
1154 # Check permissions for each property
1155 for prop in self.props.keys():
1156 if not self.db.security.hasPermission('Create', self.author,
1157 classname, prop):
1158 raise Unauthorized, _('You are not permitted to set '
1159 'property %(prop)s of class %(classname)s.'
1160 ) % locals()
1161 self.nodeid = self.cl.create(**self.props)
1162 except (TypeError, IndexError, ValueError, exceptions.Reject), message:
1163 raise MailUsageError, _("""
1164 There was a problem with the message you sent:
1165 %(message)s
1166 """) % locals()
1168 return self.nodeid
1170 # XXX Don't enable. This doesn't work yet.
1171 # "[^A-z.]tracker\+(?P<classname>[^\d\s]+)(?P<nodeid>\d+)\@some.dom.ain[^A-z.]"
1172 # handle delivery to addresses like:tracker+issue25@some.dom.ain
1173 # use the embedded issue number as our issue
1174 # issue_re = config['MAILGW_ISSUE_ADDRESS_RE']
1175 # if issue_re:
1176 # for header in ['to', 'cc', 'bcc']:
1177 # addresses = message.getheader(header, '')
1178 # if addresses:
1179 # # FIXME, this only finds the first match in the addresses.
1180 # issue = re.search(issue_re, addresses, 'i')
1181 # if issue:
1182 # classname = issue.group('classname')
1183 # nodeid = issue.group('nodeid')
1184 # break
1186 # Default sequence of methods to be called on message. Use this for
1187 # easier override of the default message processing
1188 # list consists of tuples (method, return_if_true), the parsing
1189 # returns if the return_if_true flag is set for a method *and* the
1190 # method returns something that evaluates to True.
1191 method_list = [
1192 # Filter out messages to ignore
1193 ("handle_ignore", False),
1194 # Check for usage/help requests
1195 ("handle_help", False),
1196 # Check if the subject line is valid
1197 ("check_subject", False),
1198 # get importants parts from subject
1199 ("parse_subject", False),
1200 # check for registration OTK
1201 ("rego_confirm", True),
1202 # get the classname
1203 ("get_classname", False),
1204 # get the optional nodeid:
1205 ("get_nodeid", False),
1206 # Determine who the author is:
1207 ("get_author_id", False),
1208 # allowed to edit or create this class?
1209 ("check_permissions", False),
1210 # author may have been created:
1211 # commit author to database and re-open as author
1212 ("commit_and_reopen_as_author", False),
1213 # Get the recipients list
1214 ("get_recipients", False),
1215 # get the new/updated node props
1216 ("get_props", False),
1217 # Handle PGP signed or encrypted messages
1218 ("get_pgp_message", False),
1219 # extract content and attachments from message body:
1220 ("get_content_and_attachments", False),
1221 # put attachments into files linked to the issue:
1222 ("create_files", False),
1223 # create the message if there's a message body (content):
1224 ("create_msg", False),
1225 ]
1228 def parse (self):
1229 for methodname, flag in self.method_list:
1230 method = getattr (self, methodname)
1231 ret = method()
1232 if flag and ret:
1233 return
1234 # perform the node change / create:
1235 return self.create_node()
1238 class MailGW:
1240 # To override the message parsing, derive your own class from
1241 # parsedMessage and assign to parsed_message_class in a derived
1242 # class of MailGW
1243 parsed_message_class = parsedMessage
1245 def __init__(self, instance, arguments=()):
1246 self.instance = instance
1247 self.arguments = arguments
1248 self.default_class = None
1249 for option, value in self.arguments:
1250 if option == '-c':
1251 self.default_class = value.strip()
1253 self.mailer = Mailer(instance.config)
1254 self.logger = logging.getLogger('roundup.mailgw')
1256 # should we trap exceptions (normal usage) or pass them through
1257 # (for testing)
1258 self.trapExceptions = 1
1260 def do_pipe(self):
1261 """ Read a message from standard input and pass it to the mail handler.
1263 Read into an internal structure that we can seek on (in case
1264 there's an error).
1266 XXX: we may want to read this into a temporary file instead...
1267 """
1268 s = cStringIO.StringIO()
1269 s.write(sys.stdin.read())
1270 s.seek(0)
1271 self.main(s)
1272 return 0
1274 def do_mailbox(self, filename):
1275 """ Read a series of messages from the specified unix mailbox file and
1276 pass each to the mail handler.
1277 """
1278 # open the spool file and lock it
1279 import fcntl
1280 # FCNTL is deprecated in py2.3 and fcntl takes over all the symbols
1281 if hasattr(fcntl, 'LOCK_EX'):
1282 FCNTL = fcntl
1283 else:
1284 import FCNTL
1285 f = open(filename, 'r+')
1286 fcntl.flock(f.fileno(), FCNTL.LOCK_EX)
1288 # handle and clear the mailbox
1289 try:
1290 from mailbox import UnixMailbox
1291 mailbox = UnixMailbox(f, factory=Message)
1292 # grab one message
1293 message = mailbox.next()
1294 while message:
1295 # handle this message
1296 self.handle_Message(message)
1297 message = mailbox.next()
1298 # nuke the file contents
1299 os.ftruncate(f.fileno(), 0)
1300 except:
1301 import traceback
1302 traceback.print_exc()
1303 return 1
1304 fcntl.flock(f.fileno(), FCNTL.LOCK_UN)
1305 return 0
1307 def do_imap(self, server, user='', password='', mailbox='', ssl=0,
1308 cram=0):
1309 ''' Do an IMAP connection
1310 '''
1311 import getpass, imaplib, socket
1312 try:
1313 if not user:
1314 user = raw_input('User: ')
1315 if not password:
1316 password = getpass.getpass()
1317 except (KeyboardInterrupt, EOFError):
1318 # Ctrl C or D maybe also Ctrl Z under Windows.
1319 print "\nAborted by user."
1320 return 1
1321 # open a connection to the server and retrieve all messages
1322 try:
1323 if ssl:
1324 self.logger.debug('Trying server %r with ssl'%server)
1325 server = imaplib.IMAP4_SSL(server)
1326 else:
1327 self.logger.debug('Trying server %r without ssl'%server)
1328 server = imaplib.IMAP4(server)
1329 except (imaplib.IMAP4.error, socket.error, socket.sslerror):
1330 self.logger.exception('IMAP server error')
1331 return 1
1333 try:
1334 if cram:
1335 server.login_cram_md5(user, password)
1336 else:
1337 server.login(user, password)
1338 except imaplib.IMAP4.error, e:
1339 self.logger.exception('IMAP login failure')
1340 return 1
1342 try:
1343 if not mailbox:
1344 (typ, data) = server.select()
1345 else:
1346 (typ, data) = server.select(mailbox=mailbox)
1347 if typ != 'OK':
1348 self.logger.error('Failed to get mailbox %r: %s'%(mailbox,
1349 data))
1350 return 1
1351 try:
1352 numMessages = int(data[0])
1353 except ValueError, value:
1354 self.logger.error('Invalid message count from mailbox %r'%
1355 data[0])
1356 return 1
1357 for i in range(1, numMessages+1):
1358 (typ, data) = server.fetch(str(i), '(RFC822)')
1360 # mark the message as deleted.
1361 server.store(str(i), '+FLAGS', r'(\Deleted)')
1363 # process the message
1364 s = cStringIO.StringIO(data[0][1])
1365 s.seek(0)
1366 self.handle_Message(Message(s))
1367 server.close()
1368 finally:
1369 try:
1370 server.expunge()
1371 except:
1372 pass
1373 server.logout()
1375 return 0
1378 def do_apop(self, server, user='', password='', ssl=False):
1379 ''' Do authentication POP
1380 '''
1381 self._do_pop(server, user, password, True, ssl)
1383 def do_pop(self, server, user='', password='', ssl=False):
1384 ''' Do plain POP
1385 '''
1386 self._do_pop(server, user, password, False, ssl)
1388 def _do_pop(self, server, user, password, apop, ssl):
1389 '''Read a series of messages from the specified POP server.
1390 '''
1391 import getpass, poplib, socket
1392 try:
1393 if not user:
1394 user = raw_input('User: ')
1395 if not password:
1396 password = getpass.getpass()
1397 except (KeyboardInterrupt, EOFError):
1398 # Ctrl C or D maybe also Ctrl Z under Windows.
1399 print "\nAborted by user."
1400 return 1
1402 # open a connection to the server and retrieve all messages
1403 try:
1404 if ssl:
1405 klass = poplib.POP3_SSL
1406 else:
1407 klass = poplib.POP3
1408 server = klass(server)
1409 except socket.error:
1410 self.logger.exception('POP server error')
1411 return 1
1412 if apop:
1413 server.apop(user, password)
1414 else:
1415 server.user(user)
1416 server.pass_(password)
1417 numMessages = len(server.list()[1])
1418 for i in range(1, numMessages+1):
1419 # retr: returns
1420 # [ pop response e.g. '+OK 459 octets',
1421 # [ array of message lines ],
1422 # number of octets ]
1423 lines = server.retr(i)[1]
1424 s = cStringIO.StringIO('\n'.join(lines))
1425 s.seek(0)
1426 self.handle_Message(Message(s))
1427 # delete the message
1428 server.dele(i)
1430 # quit the server to commit changes.
1431 server.quit()
1432 return 0
1434 def main(self, fp):
1435 ''' fp - the file from which to read the Message.
1436 '''
1437 return self.handle_Message(Message(fp))
1439 def handle_Message(self, message):
1440 """Handle an RFC822 Message
1442 Handle the Message object by calling handle_message() and then cope
1443 with any errors raised by handle_message.
1444 This method's job is to make that call and handle any
1445 errors in a sane manner. It should be replaced if you wish to
1446 handle errors in a different manner.
1447 """
1448 # in some rare cases, a particularly stuffed-up e-mail will make
1449 # its way into here... try to handle it gracefully
1451 self.parsed_message = None
1452 sendto = message.getaddrlist('resent-from')
1453 if not sendto:
1454 sendto = message.getaddrlist('from')
1455 if not sendto:
1456 # very bad-looking message - we don't even know who sent it
1457 msg = ['Badly formed message from mail gateway. Headers:']
1458 msg.extend(message.headers)
1459 msg = '\n'.join(map(str, msg))
1460 self.logger.error(msg)
1461 return
1463 msg = 'Handling message'
1464 if message.getheader('message-id'):
1465 msg += ' (Message-id=%r)'%message.getheader('message-id')
1466 self.logger.info(msg)
1468 # try normal message-handling
1469 if not self.trapExceptions:
1470 return self.handle_message(message)
1472 # no, we want to trap exceptions
1473 # Note: by default we return the message received not the
1474 # internal state of the parsedMessage -- except for
1475 # MailUsageError, Unauthorized and for unknown exceptions. For
1476 # the latter cases we make sure the error message is encrypted
1477 # if needed (if it either was received encrypted or pgp
1478 # processing is turned on for the user).
1479 try:
1480 return self.handle_message(message)
1481 except MailUsageHelp:
1482 # bounce the message back to the sender with the usage message
1483 fulldoc = '\n'.join(string.split(__doc__, '\n')[2:])
1484 m = ['']
1485 m.append('\n\nMail Gateway Help\n=================')
1486 m.append(fulldoc)
1487 self.mailer.bounce_message(message, [sendto[0][1]], m,
1488 subject="Mail Gateway Help")
1489 except MailUsageError, value:
1490 # bounce the message back to the sender with the usage message
1491 fulldoc = '\n'.join(string.split(__doc__, '\n')[2:])
1492 m = ['']
1493 m.append(str(value))
1494 m.append('\n\nMail Gateway Help\n=================')
1495 m.append(fulldoc)
1496 if self.parsed_message:
1497 message = self.parsed_message.message
1498 crypt = self.parsed_message.crypt
1499 self.mailer.bounce_message(message, [sendto[0][1]], m, crypt=crypt)
1500 except Unauthorized, value:
1501 # just inform the user that he is not authorized
1502 m = ['']
1503 m.append(str(value))
1504 if self.parsed_message:
1505 message = self.parsed_message.message
1506 crypt = self.parsed_message.crypt
1507 self.mailer.bounce_message(message, [sendto[0][1]], m, crypt=crypt)
1508 except IgnoreMessage:
1509 # do not take any action
1510 # this exception is thrown when email should be ignored
1511 msg = 'IgnoreMessage raised'
1512 if message.getheader('message-id'):
1513 msg += ' (Message-id=%r)'%message.getheader('message-id')
1514 self.logger.info(msg)
1515 return
1516 except:
1517 msg = 'Exception handling message'
1518 if message.getheader('message-id'):
1519 msg += ' (Message-id=%r)'%message.getheader('message-id')
1520 self.logger.exception(msg)
1522 # bounce the message back to the sender with the error message
1523 # let the admin know that something very bad is happening
1524 m = ['']
1525 m.append('An unexpected error occurred during the processing')
1526 m.append('of your message. The tracker administrator is being')
1527 m.append('notified.\n')
1528 if self.parsed_message:
1529 message = self.parsed_message.message
1530 crypt = self.parsed_message.crypt
1531 self.mailer.bounce_message(message, [sendto[0][1]], m, crypt=crypt)
1533 m.append('----------------')
1534 m.append(traceback.format_exc())
1535 self.mailer.bounce_message(message, [self.instance.config.ADMIN_EMAIL], m)
1537 def handle_message(self, message):
1538 ''' message - a Message instance
1540 Parse the message as per the module docstring.
1541 '''
1542 # get database handle for handling one email
1543 self.db = self.instance.open ('admin')
1544 try:
1545 return self._handle_message(message)
1546 finally:
1547 self.db.close()
1549 def _handle_message(self, message):
1550 ''' message - a Message instance
1552 Parse the message as per the module docstring.
1553 The following code expects an opened database and a try/finally
1554 that closes the database.
1555 '''
1556 self.parsed_message = self.parsed_message_class(self, message)
1557 nodeid = self.parsed_message.parse ()
1559 # commit the changes to the DB
1560 self.db.commit()
1562 self.parsed_message = None
1563 return nodeid
1565 def get_class_arguments(self, class_type, classname=None):
1566 ''' class_type - a valid node class type:
1567 - 'user' refers to the author of a message
1568 - 'issue' refers to an issue-type class (to which the
1569 message is appended) specified in parameter classname
1570 Note that this need not be the real classname, we get
1571 the real classname used as a parameter (from previous
1572 message-parsing steps)
1573 - 'file' specifies a file-type class
1574 - 'msg' is the message-class
1575 classname - the name of the current issue-type class
1577 Parse the commandline arguments and retrieve the properties that
1578 are relevant to the class_type. We now allow multiple -S options
1579 per class_type (-C option).
1580 '''
1581 allprops = {}
1583 classname = classname or class_type
1584 cls_lookup = { 'issue' : classname }
1586 # Allow other issue-type classes -- take the real classname from
1587 # previous parsing-steps of the message:
1588 clsname = cls_lookup.get (class_type, class_type)
1590 # check if the clsname is valid
1591 try:
1592 self.db.getclass(clsname)
1593 except KeyError:
1594 mailadmin = self.instance.config['ADMIN_EMAIL']
1595 raise MailUsageError, _("""
1596 The mail gateway is not properly set up. Please contact
1597 %(mailadmin)s and have them fix the incorrect class specified as:
1598 %(clsname)s
1599 """) % locals()
1601 if self.arguments:
1602 # The default type on the commandline is msg
1603 if class_type == 'msg':
1604 current_type = class_type
1605 else:
1606 current_type = None
1608 # Handle the arguments specified by the email gateway command line.
1609 # We do this by looping over the list of self.arguments looking for
1610 # a -C to match the class we want, then use the -S setting string.
1611 for option, propstring in self.arguments:
1612 if option in ( '-C', '--class'):
1613 current_type = propstring.strip()
1615 if current_type != class_type:
1616 current_type = None
1618 elif current_type and option in ('-S', '--set'):
1619 cls = cls_lookup.get (current_type, current_type)
1620 temp_cl = self.db.getclass(cls)
1621 errors, props = setPropArrayFromString(self,
1622 temp_cl, propstring.strip())
1624 if errors:
1625 mailadmin = self.instance.config['ADMIN_EMAIL']
1626 raise MailUsageError, _("""
1627 The mail gateway is not properly set up. Please contact
1628 %(mailadmin)s and have them fix the incorrect properties:
1629 %(errors)s
1630 """) % locals()
1631 allprops.update(props)
1633 return allprops
1636 def setPropArrayFromString(self, cl, propString, nodeid=None):
1637 ''' takes string of form prop=value,value;prop2=value
1638 and returns (error, prop[..])
1639 '''
1640 props = {}
1641 errors = []
1642 for prop in string.split(propString, ';'):
1643 # extract the property name and value
1644 try:
1645 propname, value = prop.split('=')
1646 except ValueError, message:
1647 errors.append(_('not of form [arg=value,value,...;'
1648 'arg=value,value,...]'))
1649 return (errors, props)
1650 # convert the value to a hyperdb-usable value
1651 propname = propname.strip()
1652 try:
1653 props[propname] = hyperdb.rawToHyperdb(self.db, cl, nodeid,
1654 propname, value)
1655 except hyperdb.HyperdbValueError, message:
1656 errors.append(str(message))
1657 return errors, props
1660 def extractUserFromList(userClass, users):
1661 '''Given a list of users, try to extract the first non-anonymous user
1662 and return that user, otherwise return None
1663 '''
1664 if len(users) > 1:
1665 for user in users:
1666 # make sure we don't match the anonymous or admin user
1667 if userClass.get(user, 'username') in ('admin', 'anonymous'):
1668 continue
1669 # first valid match will do
1670 return user
1671 # well, I guess we have no choice
1672 return user[0]
1673 elif users:
1674 return users[0]
1675 return None
1678 def uidFromAddress(db, address, create=1, **user_props):
1679 ''' address is from the rfc822 module, and therefore is (name, addr)
1681 user is created if they don't exist in the db already
1682 user_props may supply additional user information
1683 '''
1684 (realname, address) = address
1686 # try a straight match of the address
1687 user = extractUserFromList(db.user, db.user.stringFind(address=address))
1688 if user is not None:
1689 return user
1691 # try the user alternate addresses if possible
1692 props = db.user.getprops()
1693 if props.has_key('alternate_addresses'):
1694 users = db.user.filter(None, {'alternate_addresses': address})
1695 # We want an exact match of the email, not just a substring
1696 # match. Otherwise e.g. support@example.com would match
1697 # discuss-support@example.com which is not what we want.
1698 found_users = []
1699 for u in users:
1700 alt = db.user.get(u, 'alternate_addresses').split('\n')
1701 for a in alt:
1702 if a.strip().lower() == address.lower():
1703 found_users.append(u)
1704 break
1705 user = extractUserFromList(db.user, found_users)
1706 if user is not None:
1707 return user
1709 # try to match the username to the address (for local
1710 # submissions where the address is empty)
1711 user = extractUserFromList(db.user, db.user.stringFind(username=address))
1713 # couldn't match address or username, so create a new user
1714 if create:
1715 # generate a username
1716 if '@' in address:
1717 username = address.split('@')[0]
1718 else:
1719 username = address
1720 trying = username
1721 n = 0
1722 while 1:
1723 try:
1724 # does this username exist already?
1725 db.user.lookup(trying)
1726 except KeyError:
1727 break
1728 n += 1
1729 trying = username + str(n)
1731 # create!
1732 try:
1733 return db.user.create(username=trying, address=address,
1734 realname=realname, roles=db.config.NEW_EMAIL_USER_ROLES,
1735 password=password.Password(password.generatePassword(), config=db.config),
1736 **user_props)
1737 except exceptions.Reject:
1738 return 0
1739 else:
1740 return 0
1742 def parseContent(content, keep_citations=None, keep_body=None, config=None):
1743 """Parse mail message; return message summary and stripped content
1745 The message body is divided into sections by blank lines.
1746 Sections where the second and all subsequent lines begin with a ">"
1747 or "|" character are considered "quoting sections". The first line of
1748 the first non-quoting section becomes the summary of the message.
1750 Arguments:
1752 keep_citations: declared for backward compatibility.
1753 If omitted or None, use config["MAILGW_KEEP_QUOTED_TEXT"]
1755 keep_body: declared for backward compatibility.
1756 If omitted or None, use config["MAILGW_LEAVE_BODY_UNCHANGED"]
1758 config: tracker configuration object.
1759 If omitted or None, use default configuration.
1761 """
1762 if config is None:
1763 config = configuration.CoreConfig()
1764 if keep_citations is None:
1765 keep_citations = config["MAILGW_KEEP_QUOTED_TEXT"]
1766 if keep_body is None:
1767 keep_body = config["MAILGW_LEAVE_BODY_UNCHANGED"]
1768 eol = config["MAILGW_EOL_RE"]
1769 signature = config["MAILGW_SIGN_RE"]
1770 original_msg = config["MAILGW_ORIGMSG_RE"]
1772 # strip off leading carriage-returns / newlines
1773 i = 0
1774 for i in range(len(content)):
1775 if content[i] not in '\r\n':
1776 break
1777 if i > 0:
1778 sections = config["MAILGW_BLANKLINE_RE"].split(content[i:])
1779 else:
1780 sections = config["MAILGW_BLANKLINE_RE"].split(content)
1782 # extract out the summary from the message
1783 summary = ''
1784 l = []
1785 for section in sections:
1786 #section = section.strip()
1787 if not section:
1788 continue
1789 lines = eol.split(section)
1790 if (lines[0] and lines[0][0] in '>|') or (len(lines) > 1 and
1791 lines[1] and lines[1][0] in '>|'):
1792 # see if there's a response somewhere inside this section (ie.
1793 # no blank line between quoted message and response)
1794 for line in lines[1:]:
1795 if line and line[0] not in '>|':
1796 break
1797 else:
1798 # we keep quoted bits if specified in the config
1799 if keep_citations:
1800 l.append(section)
1801 continue
1802 # keep this section - it has reponse stuff in it
1803 lines = lines[lines.index(line):]
1804 section = '\n'.join(lines)
1805 # and while we're at it, use the first non-quoted bit as
1806 # our summary
1807 summary = section
1809 if not summary:
1810 # if we don't have our summary yet use the first line of this
1811 # section
1812 summary = section
1813 elif signature.match(lines[0]) and 2 <= len(lines) <= 10:
1814 # lose any signature
1815 break
1816 elif original_msg.match(lines[0]):
1817 # ditch the stupid Outlook quoting of the entire original message
1818 break
1820 # and add the section to the output
1821 l.append(section)
1823 # figure the summary - find the first sentence-ending punctuation or the
1824 # first whole line, whichever is longest
1825 sentence = re.search(r'^([^!?\.]+[!?\.])', summary)
1826 if sentence:
1827 sentence = sentence.group(1)
1828 else:
1829 sentence = ''
1830 first = eol.split(summary)[0]
1831 summary = max(sentence, first)
1833 # Now reconstitute the message content minus the bits we don't care
1834 # about.
1835 if not keep_body:
1836 content = '\n\n'.join(l)
1838 return summary, content
1840 # vim: set filetype=python sts=4 sw=4 et si :