d269554f8783216a05f51de73ed062c15fa28ca3
1 # -*- coding: utf-8 -*-
2 #
3 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
4 # This module is free software, and you may redistribute it and/or modify
5 # under the same terms as Python, so long as this copyright message and
6 # disclaimer are retained in their original form.
7 #
8 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
9 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
10 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
11 # POSSIBILITY OF SUCH DAMAGE.
12 #
13 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
14 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
15 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
16 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
17 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
18 #
20 """An e-mail gateway for Roundup.
22 Incoming messages are examined for multiple parts:
23 . In a multipart/mixed message or part, each subpart is extracted and
24 examined. The text/plain subparts are assembled to form the textual
25 body of the message, to be stored in the file associated with a "msg"
26 class node. Any parts of other types are each stored in separate files
27 and given "file" class nodes that are linked to the "msg" node.
28 . In a multipart/alternative message or part, we look for a text/plain
29 subpart and ignore the other parts.
30 . A message/rfc822 is treated similar tomultipart/mixed (except for
31 special handling of the first text part) if unpack_rfc822 is set in
32 the mailgw config section.
34 Summary
35 -------
36 The "summary" property on message nodes is taken from the first non-quoting
37 section in the message body. The message body is divided into sections by
38 blank lines. Sections where the second and all subsequent lines begin with
39 a ">" or "|" character are considered "quoting sections". The first line of
40 the first non-quoting section becomes the summary of the message.
42 Addresses
43 ---------
44 All of the addresses in the To: and Cc: headers of the incoming message are
45 looked up among the user nodes, and the corresponding users are placed in
46 the "recipients" property on the new "msg" node. The address in the From:
47 header similarly determines the "author" property of the new "msg"
48 node. The default handling for addresses that don't have corresponding
49 users is to create new users with no passwords and a username equal to the
50 address. (The web interface does not permit logins for users with no
51 passwords.) If we prefer to reject mail from outside sources, we can simply
52 register an auditor on the "user" class that prevents the creation of user
53 nodes with no passwords.
55 Actions
56 -------
57 The subject line of the incoming message is examined to determine whether
58 the message is an attempt to create a new item or to discuss an existing
59 item. A designator enclosed in square brackets is sought as the first thing
60 on the subject line (after skipping any "Fwd:" or "Re:" prefixes).
62 If an item designator (class name and id number) is found there, the newly
63 created "msg" node is added to the "messages" property for that item, and
64 any new "file" nodes are added to the "files" property for the item.
66 If just an item class name is found there, we attempt to create a new item
67 of that class with its "messages" property initialized to contain the new
68 "msg" node and its "files" property initialized to contain any new "file"
69 nodes.
71 Triggers
72 --------
73 Both cases may trigger detectors (in the first case we are calling the
74 set() method to add the message to the item's spool; in the second case we
75 are calling the create() method to create a new node). If an auditor raises
76 an exception, the original message is bounced back to the sender with the
77 explanatory message given in the exception.
79 $Id: mailgw.py,v 1.196 2008-07-23 03:04:44 richard Exp $
80 """
81 __docformat__ = 'restructuredtext'
83 import string, re, os, mimetools, cStringIO, smtplib, socket, binascii, quopri
84 import time, random, sys, logging
85 import traceback, rfc822
87 from email.Header import decode_header
89 from roundup import configuration, hyperdb, date, password, rfc2822, exceptions
90 from roundup.mailer import Mailer, MessageSendError
91 from roundup.i18n import _
92 from roundup.hyperdb import iter_roles
94 try:
95 import pyme, pyme.core, pyme.constants, pyme.constants.sigsum
96 except ImportError:
97 pyme = None
99 SENDMAILDEBUG = os.environ.get('SENDMAILDEBUG', '')
101 class MailGWError(ValueError):
102 pass
104 class MailUsageError(ValueError):
105 pass
107 class MailUsageHelp(Exception):
108 """ We need to send the help message to the user. """
109 pass
111 class Unauthorized(Exception):
112 """ Access denied """
113 pass
115 class IgnoreMessage(Exception):
116 """ A general class of message that we should ignore. """
117 pass
118 class IgnoreBulk(IgnoreMessage):
119 """ This is email from a mailing list or from a vacation program. """
120 pass
121 class IgnoreLoop(IgnoreMessage):
122 """ We've seen this message before... """
123 pass
125 def initialiseSecurity(security):
126 ''' Create some Permissions and Roles on the security object
128 This function is directly invoked by security.Security.__init__()
129 as a part of the Security object instantiation.
130 '''
131 p = security.addPermission(name="Email Access",
132 description="User may use the email interface")
133 security.addPermissionToRole('Admin', p)
135 def getparam(str, param):
136 ''' From the rfc822 "header" string, extract "param" if it appears.
137 '''
138 if ';' not in str:
139 return None
140 str = str[str.index(';'):]
141 while str[:1] == ';':
142 str = str[1:]
143 if ';' in str:
144 # XXX Should parse quotes!
145 end = str.index(';')
146 else:
147 end = len(str)
148 f = str[:end]
149 if '=' in f:
150 i = f.index('=')
151 if f[:i].strip().lower() == param:
152 return rfc822.unquote(f[i+1:].strip())
153 return None
155 def gpgh_key_getall(key, attr):
156 ''' return list of given attribute for all uids in
157 a key
158 '''
159 for u in key.uids:
160 yield getattr(u, attr)
162 def check_pgp_sigs(sigs, gpgctx, author):
163 ''' Theoretically a PGP message can have several signatures. GPGME
164 returns status on all signatures in a list. Walk that list
165 looking for the author's signature
166 '''
167 for sig in sigs:
168 key = gpgctx.get_key(sig.fpr, False)
169 # we really only care about the signature of the user who
170 # submitted the email
171 if key and (author in gpgh_key_getall(key, 'email')):
172 if sig.summary & pyme.constants.sigsum.VALID:
173 return True
174 else:
175 # try to narrow down the actual problem to give a more useful
176 # message in our bounce
177 if sig.summary & pyme.constants.sigsum.KEY_MISSING:
178 raise MailUsageError, \
179 _("Message signed with unknown key: %s") % sig.fpr
180 elif sig.summary & pyme.constants.sigsum.KEY_EXPIRED:
181 raise MailUsageError, \
182 _("Message signed with an expired key: %s") % sig.fpr
183 elif sig.summary & pyme.constants.sigsum.KEY_REVOKED:
184 raise MailUsageError, \
185 _("Message signed with a revoked key: %s") % sig.fpr
186 else:
187 raise MailUsageError, \
188 _("Invalid PGP signature detected.")
190 # we couldn't find a key belonging to the author of the email
191 raise MailUsageError, _("Message signed with unknown key: %s") % sig.fpr
193 class Message(mimetools.Message):
194 ''' subclass mimetools.Message so we can retrieve the parts of the
195 message...
196 '''
197 def getpart(self):
198 ''' Get a single part of a multipart message and return it as a new
199 Message instance.
200 '''
201 boundary = self.getparam('boundary')
202 mid, end = '--'+boundary, '--'+boundary+'--'
203 s = cStringIO.StringIO()
204 while 1:
205 line = self.fp.readline()
206 if not line:
207 break
208 if line.strip() in (mid, end):
209 # according to rfc 1431 the preceding line ending is part of
210 # the boundary so we need to strip that
211 length = s.tell()
212 s.seek(-2, 1)
213 lineending = s.read(2)
214 if lineending == '\r\n':
215 s.truncate(length - 2)
216 elif lineending[1] in ('\r', '\n'):
217 s.truncate(length - 1)
218 else:
219 raise ValueError('Unknown line ending in message.')
220 break
221 s.write(line)
222 if not s.getvalue().strip():
223 return None
224 s.seek(0)
225 return Message(s)
227 def getparts(self):
228 """Get all parts of this multipart message."""
229 # skip over the intro to the first boundary
230 self.fp.seek(0)
231 self.getpart()
233 # accumulate the other parts
234 parts = []
235 while 1:
236 part = self.getpart()
237 if part is None:
238 break
239 parts.append(part)
240 return parts
242 def _decode_header_to_utf8(self, hdr):
243 l = []
244 prev_encoded = False
245 for part, encoding in decode_header(hdr):
246 if encoding:
247 part = part.decode(encoding)
248 # RFC 2047 specifies that between encoded parts spaces are
249 # swallowed while at the borders from encoded to non-encoded
250 # or vice-versa we must preserve a space. Multiple adjacent
251 # non-encoded parts should not occur.
252 if l and prev_encoded != bool(encoding):
253 l.append(' ')
254 prev_encoded = bool(encoding)
255 l.append(part)
256 return ''.join([s.encode('utf-8') for s in l])
258 def getheader(self, name, default=None):
259 hdr = mimetools.Message.getheader(self, name, default)
260 # TODO are there any other False values possible?
261 # TODO if not hdr: return hdr
262 if hdr is None:
263 return None
264 if not hdr:
265 return ''
266 if hdr:
267 hdr = hdr.replace('\n','') # Inserted by rfc822.readheaders
268 return self._decode_header_to_utf8(hdr)
270 def getaddrlist(self, name):
271 # overload to decode the name part of the address
272 l = []
273 for (name, addr) in mimetools.Message.getaddrlist(self, name):
274 name = self._decode_header_to_utf8(name)
275 l.append((name, addr))
276 return l
278 def getname(self):
279 """Find an appropriate name for this message."""
280 name = None
281 if self.gettype() == 'message/rfc822':
282 # handle message/rfc822 specially - the name should be
283 # the subject of the actual e-mail embedded here
284 # we add a '.eml' extension like other email software does it
285 self.fp.seek(0)
286 s = cStringIO.StringIO(self.getbody())
287 name = Message(s).getheader('subject')
288 if name:
289 name = name + '.eml'
290 if not name:
291 # try name on Content-Type
292 name = self.getparam('name')
293 if not name:
294 disp = self.getheader('content-disposition', None)
295 if disp:
296 name = getparam(disp, 'filename')
298 if name:
299 return name.strip()
301 def getbody(self):
302 """Get the decoded message body."""
303 self.rewindbody()
304 encoding = self.getencoding()
305 data = None
306 if encoding == 'base64':
307 # BUG: is base64 really used for text encoding or
308 # are we inserting zip files here.
309 data = binascii.a2b_base64(self.fp.read())
310 elif encoding == 'quoted-printable':
311 # the quopri module wants to work with files
312 decoded = cStringIO.StringIO()
313 quopri.decode(self.fp, decoded)
314 data = decoded.getvalue()
315 elif encoding == 'uuencoded':
316 data = binascii.a2b_uu(self.fp.read())
317 else:
318 # take it as text
319 data = self.fp.read()
321 # Encode message to unicode
322 charset = rfc2822.unaliasCharset(self.getparam("charset"))
323 if charset:
324 # Do conversion only if charset specified - handle
325 # badly-specified charsets
326 edata = unicode(data, charset, 'replace').encode('utf-8')
327 # Convert from dos eol to unix
328 edata = edata.replace('\r\n', '\n')
329 else:
330 # Leave message content as is
331 edata = data
333 return edata
335 # General multipart handling:
336 # Take the first text/plain part, anything else is considered an
337 # attachment.
338 # multipart/mixed:
339 # Multiple "unrelated" parts.
340 # multipart/Alternative (rfc 1521):
341 # Like multipart/mixed, except that we'd only want one of the
342 # alternatives. Generally a top-level part from MUAs sending HTML
343 # mail - there will be a text/plain version.
344 # multipart/signed (rfc 1847):
345 # The control information is carried in the second of the two
346 # required body parts.
347 # ACTION: Default, so if content is text/plain we get it.
348 # multipart/encrypted (rfc 1847):
349 # The control information is carried in the first of the two
350 # required body parts.
351 # ACTION: Not handleable as the content is encrypted.
352 # multipart/related (rfc 1872, 2112, 2387):
353 # The Multipart/Related content-type addresses the MIME
354 # representation of compound objects, usually HTML mail with embedded
355 # images. Usually appears as an alternative.
356 # ACTION: Default, if we must.
357 # multipart/report (rfc 1892):
358 # e.g. mail system delivery status reports.
359 # ACTION: Default. Could be ignored or used for Delivery Notification
360 # flagging.
361 # multipart/form-data:
362 # For web forms only.
363 # message/rfc822:
364 # Only if configured in [mailgw] unpack_rfc822
366 def extract_content(self, parent_type=None, ignore_alternatives=False,
367 unpack_rfc822=False):
368 """Extract the body and the attachments recursively.
370 If the content is hidden inside a multipart/alternative part,
371 we use the *last* text/plain part of the *first*
372 multipart/alternative in the whole message.
373 """
374 content_type = self.gettype()
375 content = None
376 attachments = []
378 if content_type == 'text/plain':
379 content = self.getbody()
380 elif content_type[:10] == 'multipart/':
381 content_found = bool (content)
382 ig = ignore_alternatives and not content_found
383 for part in self.getparts():
384 new_content, new_attach = part.extract_content(content_type,
385 not content and ig, unpack_rfc822)
387 # If we haven't found a text/plain part yet, take this one,
388 # otherwise make it an attachment.
389 if not content:
390 content = new_content
391 cpart = part
392 elif new_content:
393 if content_found or content_type != 'multipart/alternative':
394 attachments.append(part.text_as_attachment())
395 else:
396 # if we have found a text/plain in the current
397 # multipart/alternative and find another one, we
398 # use the first as an attachment (if configured)
399 # and use the second one because rfc 2046, sec.
400 # 5.1.4. specifies that later parts are better
401 # (thanks to Philipp Gortan for pointing this
402 # out)
403 attachments.append(cpart.text_as_attachment())
404 content = new_content
405 cpart = part
407 attachments.extend(new_attach)
408 if ig and content_type == 'multipart/alternative' and content:
409 attachments = []
410 elif unpack_rfc822 and content_type == 'message/rfc822':
411 s = cStringIO.StringIO(self.getbody())
412 m = Message(s)
413 ig = ignore_alternatives and not content
414 new_content, attachments = m.extract_content(m.gettype(), ig,
415 unpack_rfc822)
416 attachments.insert(0, m.text_as_attachment())
417 elif (parent_type == 'multipart/signed' and
418 content_type == 'application/pgp-signature'):
419 # ignore it so it won't be saved as an attachment
420 pass
421 else:
422 attachments.append(self.as_attachment())
423 return content, attachments
425 def text_as_attachment(self):
426 """Return first text/plain part as Message"""
427 if not self.gettype().startswith ('multipart/'):
428 return self.as_attachment()
429 for part in self.getparts():
430 content_type = part.gettype()
431 if content_type == 'text/plain':
432 return part.as_attachment()
433 elif content_type.startswith ('multipart/'):
434 p = part.text_as_attachment()
435 if p:
436 return p
437 return None
439 def as_attachment(self):
440 """Return this message as an attachment."""
441 return (self.getname(), self.gettype(), self.getbody())
443 def pgp_signed(self):
444 ''' RFC 3156 requires OpenPGP MIME mail to have the protocol parameter
445 '''
446 return self.gettype() == 'multipart/signed' \
447 and self.typeheader.find('protocol="application/pgp-signature"') != -1
449 def pgp_encrypted(self):
450 ''' RFC 3156 requires OpenPGP MIME mail to have the protocol parameter
451 '''
452 return self.gettype() == 'multipart/encrypted' \
453 and self.typeheader.find('protocol="application/pgp-encrypted"') != -1
455 def decrypt(self, author):
456 ''' decrypt an OpenPGP MIME message
457 This message must be signed as well as encrypted using the "combined"
458 method. The decrypted contents are returned as a new message.
459 '''
460 (hdr, msg) = self.getparts()
461 # According to the RFC 3156 encrypted mail must have exactly two parts.
462 # The first part contains the control information. Let's verify that
463 # the message meets the RFC before we try to decrypt it.
464 if hdr.getbody() != 'Version: 1' or hdr.gettype() != 'application/pgp-encrypted':
465 raise MailUsageError, \
466 _("Unknown multipart/encrypted version.")
468 context = pyme.core.Context()
469 ciphertext = pyme.core.Data(msg.getbody())
470 plaintext = pyme.core.Data()
472 result = context.op_decrypt_verify(ciphertext, plaintext)
474 if result:
475 raise MailUsageError, _("Unable to decrypt your message.")
477 # we've decrypted it but that just means they used our public
478 # key to send it to us. now check the signatures to see if it
479 # was signed by someone we trust
480 result = context.op_verify_result()
481 check_pgp_sigs(result.signatures, context, author)
483 plaintext.seek(0,0)
484 # pyme.core.Data implements a seek method with a different signature
485 # than roundup can handle. So we'll put the data in a container that
486 # the Message class can work with.
487 c = cStringIO.StringIO()
488 c.write(plaintext.read())
489 c.seek(0)
490 return Message(c)
492 def verify_signature(self, author):
493 ''' verify the signature of an OpenPGP MIME message
494 This only handles detached signatures. Old style
495 PGP mail (i.e. '-----BEGIN PGP SIGNED MESSAGE----')
496 is archaic and not supported :)
497 '''
498 # we don't check the micalg parameter...gpgme seems to
499 # figure things out on its own
500 (msg, sig) = self.getparts()
502 if sig.gettype() != 'application/pgp-signature':
503 raise MailUsageError, \
504 _("No PGP signature found in message.")
506 # msg.getbody() is skipping over some headers that are
507 # required to be present for verification to succeed so
508 # we'll do this by hand
509 msg.fp.seek(0)
510 # according to rfc 3156 the data "MUST first be converted
511 # to its content-type specific canonical form. For
512 # text/plain this means conversion to an appropriate
513 # character set and conversion of line endings to the
514 # canonical <CR><LF> sequence."
515 # TODO: what about character set conversion?
516 canonical_msg = re.sub('(?<!\r)\n', '\r\n', msg.fp.read())
517 msg_data = pyme.core.Data(canonical_msg)
518 sig_data = pyme.core.Data(sig.getbody())
520 context = pyme.core.Context()
521 context.op_verify(sig_data, msg_data, None)
523 # check all signatures for validity
524 result = context.op_verify_result()
525 check_pgp_sigs(result.signatures, context, author)
527 class parsedMessage:
529 def __init__(self, mailgw, message):
530 self.mailgw = mailgw
531 self.config = mailgw.instance.config
532 self.db = mailgw.db
533 self.message = message
534 self.subject = message.getheader('subject', '')
535 self.has_prefix = False
536 self.matches = dict.fromkeys(['refwd', 'quote', 'classname',
537 'nodeid', 'title', 'args', 'argswhole'])
538 self.from_list = message.getaddrlist('resent-from') \
539 or message.getaddrlist('from')
540 self.pfxmode = self.config['MAILGW_SUBJECT_PREFIX_PARSING']
541 self.sfxmode = self.config['MAILGW_SUBJECT_SUFFIX_PARSING']
542 # these are filled in by subsequent parsing steps
543 self.classname = None
544 self.properties = None
545 self.cl = None
546 self.nodeid = None
547 self.author = None
548 self.recipients = None
549 self.msg_props = {}
550 self.props = None
551 self.content = None
552 self.attachments = None
554 def handle_ignore(self):
555 ''' Check to see if message can be safely ignored:
556 detect loops and
557 Precedence: Bulk, or Microsoft Outlook autoreplies
558 '''
559 if self.message.getheader('x-roundup-loop', ''):
560 raise IgnoreLoop
561 if (self.message.getheader('precedence', '') == 'bulk'
562 or self.subject.lower().find("autoreply") > 0):
563 raise IgnoreBulk
565 def handle_help(self):
566 ''' Check to see if the message contains a usage/help request
567 '''
568 if self.subject.strip().lower() == 'help':
569 raise MailUsageHelp
571 def check_subject(self):
572 ''' Check to see if the message contains a valid subject line
573 '''
574 if not self.subject:
575 raise MailUsageError, _("""
576 Emails to Roundup trackers must include a Subject: line!
577 """)
579 def parse_subject(self):
580 ''' Matches subjects like:
581 Re: "[issue1234] title of issue [status=resolved]"
583 Each part of the subject is matched, stored, then removed from the
584 start of the subject string as needed. The stored values are then
585 returned
586 '''
588 tmpsubject = self.subject
590 sd_open, sd_close = self.config['MAILGW_SUBJECT_SUFFIX_DELIMITERS']
591 delim_open = re.escape(sd_open)
592 if delim_open in '[(': delim_open = '\\' + delim_open
593 delim_close = re.escape(sd_close)
594 if delim_close in '[(': delim_close = '\\' + delim_close
596 # Look for Re: et. al. Used later on for MAILGW_SUBJECT_CONTENT_MATCH
597 re_re = r"(?P<refwd>%s)\s*" % self.config["MAILGW_REFWD_RE"].pattern
598 m = re.match(re_re, tmpsubject, re.IGNORECASE|re.VERBOSE|re.UNICODE)
599 if m:
600 m = m.groupdict()
601 if m['refwd']:
602 self.matches.update(m)
603 tmpsubject = tmpsubject[len(m['refwd']):] # Consume Re:
605 # Look for Leading "
606 m = re.match(r'(?P<quote>\s*")', tmpsubject,
607 re.IGNORECASE)
608 if m:
609 self.matches.update(m.groupdict())
610 tmpsubject = tmpsubject[len(self.matches['quote']):] # Consume quote
612 # Check if the subject includes a prefix
613 self.has_prefix = re.search(r'^%s(\w+)%s'%(delim_open,
614 delim_close), tmpsubject.strip())
616 # Match the classname if specified
617 class_re = r'%s(?P<classname>(%s))(?P<nodeid>\d+)?%s'%(delim_open,
618 "|".join(self.db.getclasses()), delim_close)
619 # Note: re.search, not re.match as there might be garbage
620 # (mailing list prefix, etc.) before the class identifier
621 m = re.search(class_re, tmpsubject, re.IGNORECASE)
622 if m:
623 self.matches.update(m.groupdict())
624 # Skip to the end of the class identifier, including any
625 # garbage before it.
627 tmpsubject = tmpsubject[m.end():]
629 # Match the title of the subject
630 # if we've not found a valid classname prefix then force the
631 # scanning to handle there being a leading delimiter
632 title_re = r'(?P<title>%s[^%s]*)'%(
633 not self.matches['classname'] and '.' or '', delim_open)
634 m = re.match(title_re, tmpsubject.strip(), re.IGNORECASE)
635 if m:
636 self.matches.update(m.groupdict())
637 tmpsubject = tmpsubject[len(self.matches['title']):] # Consume title
639 if self.matches['title']:
640 self.matches['title'] = self.matches['title'].strip()
641 else:
642 self.matches['title'] = ''
644 # strip off the quotes that dumb emailers put around the subject, like
645 # Re: "[issue1] bla blah"
646 if self.matches['quote'] and self.matches['title'].endswith('"'):
647 self.matches['title'] = self.matches['title'][:-1]
649 # Match any arguments specified
650 args_re = r'(?P<argswhole>%s(?P<args>.+?)%s)?'%(delim_open,
651 delim_close)
652 m = re.search(args_re, tmpsubject.strip(), re.IGNORECASE|re.VERBOSE)
653 if m:
654 self.matches.update(m.groupdict())
656 def rego_confirm(self):
657 ''' Check for registration OTK and confirm the registration if found
658 '''
660 if self.config['EMAIL_REGISTRATION_CONFIRMATION']:
661 otk_re = re.compile('-- key (?P<otk>[a-zA-Z0-9]{32})')
662 otk = otk_re.search(self.matches['title'] or '')
663 if otk:
664 self.db.confirm_registration(otk.group('otk'))
665 subject = 'Your registration to %s is complete' % \
666 self.config['TRACKER_NAME']
667 sendto = [self.from_list[0][1]]
668 self.mailgw.mailer.standard_message(sendto, subject, '')
669 return 1
670 return 0
672 def get_classname(self):
673 ''' Determine the classname of the node being created/edited
674 '''
675 subject = self.subject
677 # get the classname
678 if self.pfxmode == 'none':
679 classname = None
680 else:
681 classname = self.matches['classname']
683 if not classname and self.has_prefix and self.pfxmode == 'strict':
684 raise MailUsageError, _("""
685 The message you sent to roundup did not contain a properly formed subject
686 line. The subject must contain a class name or designator to indicate the
687 'topic' of the message. For example:
688 Subject: [issue] This is a new issue
689 - this will create a new issue in the tracker with the title 'This is
690 a new issue'.
691 Subject: [issue1234] This is a followup to issue 1234
692 - this will append the message's contents to the existing issue 1234
693 in the tracker.
695 Subject was: '%(subject)s'
696 """) % locals()
698 # try to get the class specified - if "loose" or "none" then fall
699 # back on the default
700 attempts = []
701 if classname:
702 attempts.append(classname)
704 if self.mailgw.default_class:
705 attempts.append(self.mailgw.default_class)
706 else:
707 attempts.append(self.config['MAILGW_DEFAULT_CLASS'])
709 # first valid class name wins
710 self.cl = None
711 for trycl in attempts:
712 try:
713 self.cl = self.db.getclass(trycl)
714 classname = self.classname = trycl
715 break
716 except KeyError:
717 pass
719 if not self.cl:
720 validname = ', '.join(self.db.getclasses())
721 if classname:
722 raise MailUsageError, _("""
723 The class name you identified in the subject line ("%(classname)s") does
724 not exist in the database.
726 Valid class names are: %(validname)s
727 Subject was: "%(subject)s"
728 """) % locals()
729 else:
730 raise MailUsageError, _("""
731 You did not identify a class name in the subject line and there is no
732 default set for this tracker. The subject must contain a class name or
733 designator to indicate the 'topic' of the message. For example:
734 Subject: [issue] This is a new issue
735 - this will create a new issue in the tracker with the title 'This is
736 a new issue'.
737 Subject: [issue1234] This is a followup to issue 1234
738 - this will append the message's contents to the existing issue 1234
739 in the tracker.
741 Subject was: '%(subject)s'
742 """) % locals()
743 # get the class properties
744 self.properties = self.cl.getprops()
747 def get_nodeid(self):
748 ''' Determine the nodeid from the message and return it if found
749 '''
750 title = self.matches['title']
751 subject = self.subject
753 if self.pfxmode == 'none':
754 nodeid = None
755 else:
756 nodeid = self.matches['nodeid']
758 # try in-reply-to to match the message if there's no nodeid
759 inreplyto = self.message.getheader('in-reply-to') or ''
760 if nodeid is None and inreplyto:
761 l = self.db.getclass('msg').stringFind(messageid=inreplyto)
762 if l:
763 nodeid = self.cl.filter(None, {'messages':l})[0]
766 # but we do need either a title or a nodeid...
767 if nodeid is None and not title:
768 raise MailUsageError, _("""
769 I cannot match your message to a node in the database - you need to either
770 supply a full designator (with number, eg "[issue123]") or keep the
771 previous subject title intact so I can match that.
773 Subject was: "%(subject)s"
774 """) % locals()
776 # If there's no nodeid, check to see if this is a followup and
777 # maybe someone's responded to the initial mail that created an
778 # entry. Try to find the matching nodes with the same title, and
779 # use the _last_ one matched (since that'll _usually_ be the most
780 # recent...). The subject_content_match config may specify an
781 # additional restriction based on the matched node's creation or
782 # activity.
783 tmatch_mode = self.config['MAILGW_SUBJECT_CONTENT_MATCH']
784 if tmatch_mode != 'never' and nodeid is None and self.matches['refwd']:
785 l = self.cl.stringFind(title=title)
786 limit = None
787 if (tmatch_mode.startswith('creation') or
788 tmatch_mode.startswith('activity')):
789 limit, interval = tmatch_mode.split(' ', 1)
790 threshold = date.Date('.') - date.Interval(interval)
791 for id in l:
792 if limit:
793 if threshold < self.cl.get(id, limit):
794 nodeid = id
795 else:
796 nodeid = id
798 # if a nodeid was specified, make sure it's valid
799 if nodeid is not None and not self.cl.hasnode(nodeid):
800 if self.pfxmode == 'strict':
801 raise MailUsageError, _("""
802 The node specified by the designator in the subject of your message
803 ("%(nodeid)s") does not exist.
805 Subject was: "%(subject)s"
806 """) % locals()
807 else:
808 nodeid = None
809 self.nodeid = nodeid
811 def get_author_id(self):
812 ''' Attempt to get the author id from the existing registered users,
813 otherwise attempt to register a new user and return their id
814 '''
815 # Don't create users if anonymous isn't allowed to register
816 create = 1
817 anonid = self.db.user.lookup('anonymous')
818 if not (self.db.security.hasPermission('Register', anonid, 'user')
819 and self.db.security.hasPermission('Email Access', anonid)):
820 create = 0
822 # ok, now figure out who the author is - create a new user if the
823 # "create" flag is true
824 author = uidFromAddress(self.db, self.from_list[0], create=create)
826 # if we're not recognised, and we don't get added as a user, then we
827 # must be anonymous
828 if not author:
829 author = anonid
831 # make sure the author has permission to use the email interface
832 if not self.db.security.hasPermission('Email Access', author):
833 if author == anonid:
834 # we're anonymous and we need to be a registered user
835 from_address = self.from_list[0][1]
836 registration_info = ""
837 if self.db.security.hasPermission('Web Access', author) and \
838 self.db.security.hasPermission('Register', anonid, 'user'):
839 tracker_web = self.config.TRACKER_WEB
840 registration_info = """ Please register at:
842 %(tracker_web)suser?template=register
844 ...before sending mail to the tracker.""" % locals()
846 raise Unauthorized, _("""
847 You are not a registered user.%(registration_info)s
849 Unknown address: %(from_address)s
850 """) % locals()
851 else:
852 # we're registered and we're _still_ not allowed access
853 raise Unauthorized, _(
854 'You are not permitted to access this tracker.')
855 self.author = author
857 def check_permissions(self):
858 ''' Check if the author has permission to edit or create this
859 class of node
860 '''
861 if self.nodeid:
862 if not self.db.security.hasPermission('Edit', self.author,
863 self.classname, itemid=self.nodeid):
864 raise Unauthorized, _(
865 'You are not permitted to edit %(classname)s.'
866 ) % self.__dict__
867 else:
868 if not self.db.security.hasPermission('Create', self.author,
869 self.classname):
870 raise Unauthorized, _(
871 'You are not permitted to create %(classname)s.'
872 ) % self.__dict__
874 def commit_and_reopen_as_author(self):
875 ''' the author may have been created - make sure the change is
876 committed before we reopen the database
877 then re-open the database as the author
878 '''
879 self.db.commit()
881 # set the database user as the author
882 username = self.db.user.get(self.author, 'username')
883 self.db.setCurrentUser(username)
885 # re-get the class with the new database connection
886 self.cl = self.db.getclass(self.classname)
888 def get_recipients(self):
889 ''' Get the list of recipients who were included in message and
890 register them as users if possible
891 '''
892 # Don't create users if anonymous isn't allowed to register
893 create = 1
894 anonid = self.db.user.lookup('anonymous')
895 if not (self.db.security.hasPermission('Register', anonid, 'user')
896 and self.db.security.hasPermission('Email Access', anonid)):
897 create = 0
899 # get the user class arguments from the commandline
900 user_props = self.mailgw.get_class_arguments('user')
902 # now update the recipients list
903 recipients = []
904 tracker_email = self.config['TRACKER_EMAIL'].lower()
905 msg_to = self.message.getaddrlist('to')
906 msg_cc = self.message.getaddrlist('cc')
907 for recipient in msg_to + msg_cc:
908 r = recipient[1].strip().lower()
909 if r == tracker_email or not r:
910 continue
912 # look up the recipient - create if necessary (and we're
913 # allowed to)
914 recipient = uidFromAddress(self.db, recipient, create, **user_props)
916 # if all's well, add the recipient to the list
917 if recipient:
918 recipients.append(recipient)
919 self.recipients = recipients
921 def get_props(self):
922 ''' Generate all the props for the new/updated node and return them
923 '''
924 subject = self.subject
926 # get the commandline arguments for issues
927 issue_props = self.mailgw.get_class_arguments('issue', self.classname)
929 #
930 # handle the subject argument list
931 #
932 # figure what the properties of this Class are
933 props = {}
934 args = self.matches['args']
935 argswhole = self.matches['argswhole']
936 title = self.matches['title']
938 # Reform the title
939 if self.matches['nodeid'] and self.nodeid is None:
940 title = subject
942 if args:
943 if self.sfxmode == 'none':
944 title += ' ' + argswhole
945 else:
946 errors, props = setPropArrayFromString(self, self.cl, args,
947 self.nodeid)
948 # handle any errors parsing the argument list
949 if errors:
950 if self.sfxmode == 'strict':
951 errors = '\n- '.join(map(str, errors))
952 raise MailUsageError, _("""
953 There were problems handling your subject line argument list:
954 - %(errors)s
956 Subject was: "%(subject)s"
957 """) % locals()
958 else:
959 title += ' ' + argswhole
962 # set the issue title to the subject
963 title = title.strip()
964 if (title and self.properties.has_key('title') and not
965 issue_props.has_key('title')):
966 issue_props['title'] = title
967 if (self.nodeid and self.properties.has_key('title') and not
968 self.config['MAILGW_SUBJECT_UPDATES_TITLE']):
969 issue_props['title'] = self.cl.get(self.nodeid,'title')
971 # merge the command line props defined in issue_props into
972 # the props dictionary because function(**props, **issue_props)
973 # is a syntax error.
974 for prop in issue_props.keys() :
975 if not props.has_key(prop) :
976 props[prop] = issue_props[prop]
978 self.props = props
980 def get_pgp_message(self):
981 ''' If they've enabled PGP processing then verify the signature
982 or decrypt the message
983 '''
984 def pgp_role():
985 """ if PGP_ROLES is specified the user must have a Role in the list
986 or we will skip PGP processing
987 """
988 if self.config.PGP_ROLES:
989 return self.db.user.has_role(self.author,
990 *iter_roles(self.config.PGP_ROLES))
991 else:
992 return True
994 if self.config.PGP_ENABLE and pgp_role():
995 assert pyme, 'pyme is not installed'
996 # signed/encrypted mail must come from the primary address
997 author_address = self.db.user.get(self.author, 'address')
998 if self.config.PGP_HOMEDIR:
999 os.environ['GNUPGHOME'] = self.config.PGP_HOMEDIR
1000 if self.message.pgp_signed():
1001 self.message.verify_signature(author_address)
1002 elif self.message.pgp_encrypted():
1003 # replace message with the contents of the decrypted
1004 # message for content extraction
1005 # TODO: encrypted message handling is far from perfect
1006 # bounces probably include the decrypted message, for
1007 # instance :(
1008 self.message = self.message.decrypt(author_address)
1009 else:
1010 raise MailUsageError, _("""
1011 This tracker has been configured to require all email be PGP signed or
1012 encrypted.""")
1014 def get_content_and_attachments(self):
1015 ''' get the attachments and first text part from the message
1016 '''
1017 ig = self.config.MAILGW_IGNORE_ALTERNATIVES
1018 self.content, self.attachments = self.message.extract_content(
1019 ignore_alternatives=ig,
1020 unpack_rfc822=self.config.MAILGW_UNPACK_RFC822)
1023 def create_files(self):
1024 ''' Create a file for each attachment in the message
1025 '''
1026 if not self.properties.has_key('files'):
1027 return
1028 files = []
1029 file_props = self.mailgw.get_class_arguments('file')
1031 if self.attachments:
1032 for (name, mime_type, data) in self.attachments:
1033 if not self.db.security.hasPermission('Create', self.author,
1034 'file'):
1035 raise Unauthorized, _(
1036 'You are not permitted to create files.')
1037 if not name:
1038 name = "unnamed"
1039 try:
1040 fileid = self.db.file.create(type=mime_type, name=name,
1041 content=data, **file_props)
1042 except exceptions.Reject:
1043 pass
1044 else:
1045 files.append(fileid)
1046 # allowed to attach the files to an existing node?
1047 if self.nodeid and not self.db.security.hasPermission('Edit',
1048 self.author, self.classname, 'files'):
1049 raise Unauthorized, _(
1050 'You are not permitted to add files to %(classname)s.'
1051 ) % self.__dict__
1053 self.msg_props['files'] = files
1054 if self.nodeid:
1055 # extend the existing files list
1056 fileprop = self.cl.get(self.nodeid, 'files')
1057 fileprop.extend(files)
1058 files = fileprop
1060 self.props['files'] = files
1062 def create_msg(self):
1063 ''' Create msg containing all the relevant information from the message
1064 '''
1065 if not self.properties.has_key('messages'):
1066 return
1067 msg_props = self.mailgw.get_class_arguments('msg')
1068 self.msg_props.update (msg_props)
1070 # Get the message ids
1071 inreplyto = self.message.getheader('in-reply-to') or ''
1072 messageid = self.message.getheader('message-id')
1073 # generate a messageid if there isn't one
1074 if not messageid:
1075 messageid = "<%s.%s.%s%s@%s>"%(time.time(), random.random(),
1076 self.classname, self.nodeid, self.config['MAIL_DOMAIN'])
1078 if self.content is None:
1079 raise MailUsageError, _("""
1080 Roundup requires the submission to be plain text. The message parser could
1081 not find a text/plain part to use.
1082 """)
1084 # parse the body of the message, stripping out bits as appropriate
1085 summary, content = parseContent(self.content, config=self.config)
1086 content = content.strip()
1088 if content:
1089 if not self.db.security.hasPermission('Create', self.author, 'msg'):
1090 raise Unauthorized, _(
1091 'You are not permitted to create messages.')
1093 try:
1094 message_id = self.db.msg.create(author=self.author,
1095 recipients=self.recipients, date=date.Date('.'),
1096 summary=summary, content=content,
1097 messageid=messageid, inreplyto=inreplyto, **self.msg_props)
1098 except exceptions.Reject, error:
1099 raise MailUsageError, _("""
1100 Mail message was rejected by a detector.
1101 %(error)s
1102 """) % locals()
1103 # allowed to attach the message to the existing node?
1104 if self.nodeid and not self.db.security.hasPermission('Edit',
1105 self.author, self.classname, 'messages'):
1106 raise Unauthorized, _(
1107 'You are not permitted to add messages to %(classname)s.'
1108 ) % self.__dict__
1110 if self.nodeid:
1111 # add the message to the node's list
1112 messages = self.cl.get(self.nodeid, 'messages')
1113 messages.append(message_id)
1114 self.props['messages'] = messages
1115 else:
1116 # pre-load the messages list
1117 self.props['messages'] = [message_id]
1119 def create_node(self):
1120 ''' Create/update a node using self.props
1121 '''
1122 classname = self.classname
1123 try:
1124 if self.nodeid:
1125 # Check permissions for each property
1126 for prop in self.props.keys():
1127 if not self.db.security.hasPermission('Edit', self.author,
1128 classname, prop):
1129 raise Unauthorized, _('You are not permitted to edit '
1130 'property %(prop)s of class %(classname)s.'
1131 ) % locals()
1132 self.cl.set(self.nodeid, **self.props)
1133 else:
1134 # Check permissions for each property
1135 for prop in self.props.keys():
1136 if not self.db.security.hasPermission('Create', self.author,
1137 classname, prop):
1138 raise Unauthorized, _('You are not permitted to set '
1139 'property %(prop)s of class %(classname)s.'
1140 ) % locals()
1141 self.nodeid = self.cl.create(**self.props)
1142 except (TypeError, IndexError, ValueError, exceptions.Reject), message:
1143 raise MailUsageError, _("""
1144 There was a problem with the message you sent:
1145 %(message)s
1146 """) % locals()
1148 return self.nodeid
1150 # XXX Don't enable. This doesn't work yet.
1151 # "[^A-z.]tracker\+(?P<classname>[^\d\s]+)(?P<nodeid>\d+)\@some.dom.ain[^A-z.]"
1152 # handle delivery to addresses like:tracker+issue25@some.dom.ain
1153 # use the embedded issue number as our issue
1154 # issue_re = config['MAILGW_ISSUE_ADDRESS_RE']
1155 # if issue_re:
1156 # for header in ['to', 'cc', 'bcc']:
1157 # addresses = message.getheader(header, '')
1158 # if addresses:
1159 # # FIXME, this only finds the first match in the addresses.
1160 # issue = re.search(issue_re, addresses, 'i')
1161 # if issue:
1162 # classname = issue.group('classname')
1163 # nodeid = issue.group('nodeid')
1164 # break
1166 # Default sequence of methods to be called on message. Use this for
1167 # easier override of the default message processing
1168 # list consists of tuples (method, return_if_true), the parsing
1169 # returns if the return_if_true flag is set for a method *and* the
1170 # method returns something that evaluates to True.
1171 method_list = [
1172 # Filter out messages to ignore
1173 (handle_ignore, False),
1174 # Check for usage/help requests
1175 (handle_help, False),
1176 # Check if the subject line is valid
1177 (check_subject, False),
1178 # get importants parts from subject
1179 (parse_subject, False),
1180 # check for registration OTK
1181 (rego_confirm, True),
1182 # get the classname
1183 (get_classname, False),
1184 # get the optional nodeid:
1185 (get_nodeid, False),
1186 # Determine who the author is:
1187 (get_author_id, False),
1188 # allowed to edit or create this class?
1189 (check_permissions, False),
1190 # author may have been created:
1191 # commit author to database and re-open as author
1192 (commit_and_reopen_as_author, False),
1193 # Get the recipients list
1194 (get_recipients, False),
1195 # get the new/updated node props
1196 (get_props, False),
1197 # Handle PGP signed or encrypted messages
1198 (get_pgp_message, False),
1199 # extract content and attachments from message body:
1200 (get_content_and_attachments, False),
1201 # put attachments into files linked to the issue:
1202 (create_files, False),
1203 # create the message if there's a message body (content):
1204 (create_msg, False),
1205 ]
1208 def parse (self):
1209 for method, flag in self.method_list:
1210 ret = method(self)
1211 if flag and ret:
1212 return
1213 # perform the node change / create:
1214 return self.create_node()
1217 class MailGW:
1219 # To override the message parsing, derive your own class from
1220 # parsedMessage and assign to parsed_message_class in a derived
1221 # class of MailGW
1222 parsed_message_class = parsedMessage
1224 def __init__(self, instance, arguments=()):
1225 self.instance = instance
1226 self.arguments = arguments
1227 self.default_class = None
1228 for option, value in self.arguments:
1229 if option == '-c':
1230 self.default_class = value.strip()
1232 self.mailer = Mailer(instance.config)
1233 self.logger = logging.getLogger('roundup.mailgw')
1235 # should we trap exceptions (normal usage) or pass them through
1236 # (for testing)
1237 self.trapExceptions = 1
1239 def do_pipe(self):
1240 """ Read a message from standard input and pass it to the mail handler.
1242 Read into an internal structure that we can seek on (in case
1243 there's an error).
1245 XXX: we may want to read this into a temporary file instead...
1246 """
1247 s = cStringIO.StringIO()
1248 s.write(sys.stdin.read())
1249 s.seek(0)
1250 self.main(s)
1251 return 0
1253 def do_mailbox(self, filename):
1254 """ Read a series of messages from the specified unix mailbox file and
1255 pass each to the mail handler.
1256 """
1257 # open the spool file and lock it
1258 import fcntl
1259 # FCNTL is deprecated in py2.3 and fcntl takes over all the symbols
1260 if hasattr(fcntl, 'LOCK_EX'):
1261 FCNTL = fcntl
1262 else:
1263 import FCNTL
1264 f = open(filename, 'r+')
1265 fcntl.flock(f.fileno(), FCNTL.LOCK_EX)
1267 # handle and clear the mailbox
1268 try:
1269 from mailbox import UnixMailbox
1270 mailbox = UnixMailbox(f, factory=Message)
1271 # grab one message
1272 message = mailbox.next()
1273 while message:
1274 # handle this message
1275 self.handle_Message(message)
1276 message = mailbox.next()
1277 # nuke the file contents
1278 os.ftruncate(f.fileno(), 0)
1279 except:
1280 import traceback
1281 traceback.print_exc()
1282 return 1
1283 fcntl.flock(f.fileno(), FCNTL.LOCK_UN)
1284 return 0
1286 def do_imap(self, server, user='', password='', mailbox='', ssl=0,
1287 cram=0):
1288 ''' Do an IMAP connection
1289 '''
1290 import getpass, imaplib, socket
1291 try:
1292 if not user:
1293 user = raw_input('User: ')
1294 if not password:
1295 password = getpass.getpass()
1296 except (KeyboardInterrupt, EOFError):
1297 # Ctrl C or D maybe also Ctrl Z under Windows.
1298 print "\nAborted by user."
1299 return 1
1300 # open a connection to the server and retrieve all messages
1301 try:
1302 if ssl:
1303 self.logger.debug('Trying server %r with ssl'%server)
1304 server = imaplib.IMAP4_SSL(server)
1305 else:
1306 self.logger.debug('Trying server %r without ssl'%server)
1307 server = imaplib.IMAP4(server)
1308 except (imaplib.IMAP4.error, socket.error, socket.sslerror):
1309 self.logger.exception('IMAP server error')
1310 return 1
1312 try:
1313 if cram:
1314 server.login_cram_md5(user, password)
1315 else:
1316 server.login(user, password)
1317 except imaplib.IMAP4.error, e:
1318 self.logger.exception('IMAP login failure')
1319 return 1
1321 try:
1322 if not mailbox:
1323 (typ, data) = server.select()
1324 else:
1325 (typ, data) = server.select(mailbox=mailbox)
1326 if typ != 'OK':
1327 self.logger.error('Failed to get mailbox %r: %s'%(mailbox,
1328 data))
1329 return 1
1330 try:
1331 numMessages = int(data[0])
1332 except ValueError, value:
1333 self.logger.error('Invalid message count from mailbox %r'%
1334 data[0])
1335 return 1
1336 for i in range(1, numMessages+1):
1337 (typ, data) = server.fetch(str(i), '(RFC822)')
1339 # mark the message as deleted.
1340 server.store(str(i), '+FLAGS', r'(\Deleted)')
1342 # process the message
1343 s = cStringIO.StringIO(data[0][1])
1344 s.seek(0)
1345 self.handle_Message(Message(s))
1346 server.close()
1347 finally:
1348 try:
1349 server.expunge()
1350 except:
1351 pass
1352 server.logout()
1354 return 0
1357 def do_apop(self, server, user='', password='', ssl=False):
1358 ''' Do authentication POP
1359 '''
1360 self._do_pop(server, user, password, True, ssl)
1362 def do_pop(self, server, user='', password='', ssl=False):
1363 ''' Do plain POP
1364 '''
1365 self._do_pop(server, user, password, False, ssl)
1367 def _do_pop(self, server, user, password, apop, ssl):
1368 '''Read a series of messages from the specified POP server.
1369 '''
1370 import getpass, poplib, socket
1371 try:
1372 if not user:
1373 user = raw_input('User: ')
1374 if not password:
1375 password = getpass.getpass()
1376 except (KeyboardInterrupt, EOFError):
1377 # Ctrl C or D maybe also Ctrl Z under Windows.
1378 print "\nAborted by user."
1379 return 1
1381 # open a connection to the server and retrieve all messages
1382 try:
1383 if ssl:
1384 klass = poplib.POP3_SSL
1385 else:
1386 klass = poplib.POP3
1387 server = klass(server)
1388 except socket.error:
1389 self.logger.exception('POP server error')
1390 return 1
1391 if apop:
1392 server.apop(user, password)
1393 else:
1394 server.user(user)
1395 server.pass_(password)
1396 numMessages = len(server.list()[1])
1397 for i in range(1, numMessages+1):
1398 # retr: returns
1399 # [ pop response e.g. '+OK 459 octets',
1400 # [ array of message lines ],
1401 # number of octets ]
1402 lines = server.retr(i)[1]
1403 s = cStringIO.StringIO('\n'.join(lines))
1404 s.seek(0)
1405 self.handle_Message(Message(s))
1406 # delete the message
1407 server.dele(i)
1409 # quit the server to commit changes.
1410 server.quit()
1411 return 0
1413 def main(self, fp):
1414 ''' fp - the file from which to read the Message.
1415 '''
1416 return self.handle_Message(Message(fp))
1418 def handle_Message(self, message):
1419 """Handle an RFC822 Message
1421 Handle the Message object by calling handle_message() and then cope
1422 with any errors raised by handle_message.
1423 This method's job is to make that call and handle any
1424 errors in a sane manner. It should be replaced if you wish to
1425 handle errors in a different manner.
1426 """
1427 # in some rare cases, a particularly stuffed-up e-mail will make
1428 # its way into here... try to handle it gracefully
1430 self.parsed_message = None
1431 sendto = message.getaddrlist('resent-from')
1432 if not sendto:
1433 sendto = message.getaddrlist('from')
1434 if not sendto:
1435 # very bad-looking message - we don't even know who sent it
1436 msg = ['Badly formed message from mail gateway. Headers:']
1437 msg.extend(message.headers)
1438 msg = '\n'.join(map(str, msg))
1439 self.logger.error(msg)
1440 return
1442 msg = 'Handling message'
1443 if message.getheader('message-id'):
1444 msg += ' (Message-id=%r)'%message.getheader('message-id')
1445 self.logger.info(msg)
1447 # try normal message-handling
1448 if not self.trapExceptions:
1449 return self.handle_message(message)
1451 # no, we want to trap exceptions
1452 try:
1453 return self.handle_message(message)
1454 except MailUsageHelp:
1455 # bounce the message back to the sender with the usage message
1456 fulldoc = '\n'.join(string.split(__doc__, '\n')[2:])
1457 m = ['']
1458 m.append('\n\nMail Gateway Help\n=================')
1459 m.append(fulldoc)
1460 self.mailer.bounce_message(message, [sendto[0][1]], m,
1461 subject="Mail Gateway Help")
1462 except MailUsageError, value:
1463 # bounce the message back to the sender with the usage message
1464 fulldoc = '\n'.join(string.split(__doc__, '\n')[2:])
1465 m = ['']
1466 m.append(str(value))
1467 m.append('\n\nMail Gateway Help\n=================')
1468 m.append(fulldoc)
1469 self.mailer.bounce_message(message, [sendto[0][1]], m)
1470 except Unauthorized, value:
1471 # just inform the user that he is not authorized
1472 m = ['']
1473 m.append(str(value))
1474 self.mailer.bounce_message(message, [sendto[0][1]], m)
1475 except IgnoreMessage:
1476 # do not take any action
1477 # this exception is thrown when email should be ignored
1478 msg = 'IgnoreMessage raised'
1479 if message.getheader('message-id'):
1480 msg += ' (Message-id=%r)'%message.getheader('message-id')
1481 self.logger.info(msg)
1482 return
1483 except:
1484 msg = 'Exception handling message'
1485 if message.getheader('message-id'):
1486 msg += ' (Message-id=%r)'%message.getheader('message-id')
1487 self.logger.exception(msg)
1489 # bounce the message back to the sender with the error message
1490 # let the admin know that something very bad is happening
1491 m = ['']
1492 m.append('An unexpected error occurred during the processing')
1493 m.append('of your message. The tracker administrator is being')
1494 m.append('notified.\n')
1495 self.mailer.bounce_message(message, [sendto[0][1]], m)
1497 m.append('----------------')
1498 m.append(traceback.format_exc())
1499 self.mailer.bounce_message(message, [self.instance.config.ADMIN_EMAIL], m)
1501 def handle_message(self, message):
1502 ''' message - a Message instance
1504 Parse the message as per the module docstring.
1505 '''
1506 # get database handle for handling one email
1507 self.db = self.instance.open ('admin')
1508 try:
1509 return self._handle_message(message)
1510 finally:
1511 self.db.close()
1513 def _handle_message(self, message):
1514 ''' message - a Message instance
1516 Parse the message as per the module docstring.
1517 The following code expects an opened database and a try/finally
1518 that closes the database.
1519 '''
1520 self.parsed_message = self.parsed_message_class(self, message)
1521 nodeid = self.parsed_message.parse ()
1523 # commit the changes to the DB
1524 self.db.commit()
1526 return nodeid
1528 def get_class_arguments(self, class_type, classname=None):
1529 ''' class_type - a valid node class type:
1530 - 'user' refers to the author of a message
1531 - 'issue' refers to an issue-type class (to which the
1532 message is appended) specified in parameter classname
1533 Note that this need not be the real classname, we get
1534 the real classname used as a parameter (from previous
1535 message-parsing steps)
1536 - 'file' specifies a file-type class
1537 - 'msg' is the message-class
1538 classname - the name of the current issue-type class
1540 Parse the commandline arguments and retrieve the properties that
1541 are relevant to the class_type. We now allow multiple -S options
1542 per class_type (-C option).
1543 '''
1544 allprops = {}
1546 classname = classname or class_type
1547 cls_lookup = { 'issue' : classname }
1549 # Allow other issue-type classes -- take the real classname from
1550 # previous parsing-steps of the message:
1551 clsname = cls_lookup.get (class_type, class_type)
1553 # check if the clsname is valid
1554 try:
1555 self.db.getclass(clsname)
1556 except KeyError:
1557 mailadmin = self.instance.config['ADMIN_EMAIL']
1558 raise MailUsageError, _("""
1559 The mail gateway is not properly set up. Please contact
1560 %(mailadmin)s and have them fix the incorrect class specified as:
1561 %(clsname)s
1562 """) % locals()
1564 if self.arguments:
1565 # The default type on the commandline is msg
1566 if class_type == 'msg':
1567 current_type = class_type
1568 else:
1569 current_type = None
1571 # Handle the arguments specified by the email gateway command line.
1572 # We do this by looping over the list of self.arguments looking for
1573 # a -C to match the class we want, then use the -S setting string.
1574 for option, propstring in self.arguments:
1575 if option in ( '-C', '--class'):
1576 current_type = propstring.strip()
1578 if current_type != class_type:
1579 current_type = None
1581 elif current_type and option in ('-S', '--set'):
1582 cls = cls_lookup.get (current_type, current_type)
1583 temp_cl = self.db.getclass(cls)
1584 errors, props = setPropArrayFromString(self,
1585 temp_cl, propstring.strip())
1587 if errors:
1588 mailadmin = self.instance.config['ADMIN_EMAIL']
1589 raise MailUsageError, _("""
1590 The mail gateway is not properly set up. Please contact
1591 %(mailadmin)s and have them fix the incorrect properties:
1592 %(errors)s
1593 """) % locals()
1594 allprops.update(props)
1596 return allprops
1599 def setPropArrayFromString(self, cl, propString, nodeid=None):
1600 ''' takes string of form prop=value,value;prop2=value
1601 and returns (error, prop[..])
1602 '''
1603 props = {}
1604 errors = []
1605 for prop in string.split(propString, ';'):
1606 # extract the property name and value
1607 try:
1608 propname, value = prop.split('=')
1609 except ValueError, message:
1610 errors.append(_('not of form [arg=value,value,...;'
1611 'arg=value,value,...]'))
1612 return (errors, props)
1613 # convert the value to a hyperdb-usable value
1614 propname = propname.strip()
1615 try:
1616 props[propname] = hyperdb.rawToHyperdb(self.db, cl, nodeid,
1617 propname, value)
1618 except hyperdb.HyperdbValueError, message:
1619 errors.append(str(message))
1620 return errors, props
1623 def extractUserFromList(userClass, users):
1624 '''Given a list of users, try to extract the first non-anonymous user
1625 and return that user, otherwise return None
1626 '''
1627 if len(users) > 1:
1628 for user in users:
1629 # make sure we don't match the anonymous or admin user
1630 if userClass.get(user, 'username') in ('admin', 'anonymous'):
1631 continue
1632 # first valid match will do
1633 return user
1634 # well, I guess we have no choice
1635 return user[0]
1636 elif users:
1637 return users[0]
1638 return None
1641 def uidFromAddress(db, address, create=1, **user_props):
1642 ''' address is from the rfc822 module, and therefore is (name, addr)
1644 user is created if they don't exist in the db already
1645 user_props may supply additional user information
1646 '''
1647 (realname, address) = address
1649 # try a straight match of the address
1650 user = extractUserFromList(db.user, db.user.stringFind(address=address))
1651 if user is not None:
1652 return user
1654 # try the user alternate addresses if possible
1655 props = db.user.getprops()
1656 if props.has_key('alternate_addresses'):
1657 users = db.user.filter(None, {'alternate_addresses': address})
1658 # We want an exact match of the email, not just a substring
1659 # match. Otherwise e.g. support@example.com would match
1660 # discuss-support@example.com which is not what we want.
1661 found_users = []
1662 for u in users:
1663 alt = db.user.get(u, 'alternate_addresses').split('\n')
1664 for a in alt:
1665 if a.strip().lower() == address.lower():
1666 found_users.append(u)
1667 break
1668 user = extractUserFromList(db.user, found_users)
1669 if user is not None:
1670 return user
1672 # try to match the username to the address (for local
1673 # submissions where the address is empty)
1674 user = extractUserFromList(db.user, db.user.stringFind(username=address))
1676 # couldn't match address or username, so create a new user
1677 if create:
1678 # generate a username
1679 if '@' in address:
1680 username = address.split('@')[0]
1681 else:
1682 username = address
1683 trying = username
1684 n = 0
1685 while 1:
1686 try:
1687 # does this username exist already?
1688 db.user.lookup(trying)
1689 except KeyError:
1690 break
1691 n += 1
1692 trying = username + str(n)
1694 # create!
1695 try:
1696 return db.user.create(username=trying, address=address,
1697 realname=realname, roles=db.config.NEW_EMAIL_USER_ROLES,
1698 password=password.Password(password.generatePassword(), config=db.config),
1699 **user_props)
1700 except exceptions.Reject:
1701 return 0
1702 else:
1703 return 0
1705 def parseContent(content, keep_citations=None, keep_body=None, config=None):
1706 """Parse mail message; return message summary and stripped content
1708 The message body is divided into sections by blank lines.
1709 Sections where the second and all subsequent lines begin with a ">"
1710 or "|" character are considered "quoting sections". The first line of
1711 the first non-quoting section becomes the summary of the message.
1713 Arguments:
1715 keep_citations: declared for backward compatibility.
1716 If omitted or None, use config["MAILGW_KEEP_QUOTED_TEXT"]
1718 keep_body: declared for backward compatibility.
1719 If omitted or None, use config["MAILGW_LEAVE_BODY_UNCHANGED"]
1721 config: tracker configuration object.
1722 If omitted or None, use default configuration.
1724 """
1725 if config is None:
1726 config = configuration.CoreConfig()
1727 if keep_citations is None:
1728 keep_citations = config["MAILGW_KEEP_QUOTED_TEXT"]
1729 if keep_body is None:
1730 keep_body = config["MAILGW_LEAVE_BODY_UNCHANGED"]
1731 eol = config["MAILGW_EOL_RE"]
1732 signature = config["MAILGW_SIGN_RE"]
1733 original_msg = config["MAILGW_ORIGMSG_RE"]
1735 # strip off leading carriage-returns / newlines
1736 i = 0
1737 for i in range(len(content)):
1738 if content[i] not in '\r\n':
1739 break
1740 if i > 0:
1741 sections = config["MAILGW_BLANKLINE_RE"].split(content[i:])
1742 else:
1743 sections = config["MAILGW_BLANKLINE_RE"].split(content)
1745 # extract out the summary from the message
1746 summary = ''
1747 l = []
1748 for section in sections:
1749 #section = section.strip()
1750 if not section:
1751 continue
1752 lines = eol.split(section)
1753 if (lines[0] and lines[0][0] in '>|') or (len(lines) > 1 and
1754 lines[1] and lines[1][0] in '>|'):
1755 # see if there's a response somewhere inside this section (ie.
1756 # no blank line between quoted message and response)
1757 for line in lines[1:]:
1758 if line and line[0] not in '>|':
1759 break
1760 else:
1761 # we keep quoted bits if specified in the config
1762 if keep_citations:
1763 l.append(section)
1764 continue
1765 # keep this section - it has reponse stuff in it
1766 lines = lines[lines.index(line):]
1767 section = '\n'.join(lines)
1768 # and while we're at it, use the first non-quoted bit as
1769 # our summary
1770 summary = section
1772 if not summary:
1773 # if we don't have our summary yet use the first line of this
1774 # section
1775 summary = section
1776 elif signature.match(lines[0]) and 2 <= len(lines) <= 10:
1777 # lose any signature
1778 break
1779 elif original_msg.match(lines[0]):
1780 # ditch the stupid Outlook quoting of the entire original message
1781 break
1783 # and add the section to the output
1784 l.append(section)
1786 # figure the summary - find the first sentence-ending punctuation or the
1787 # first whole line, whichever is longest
1788 sentence = re.search(r'^([^!?\.]+[!?\.])', summary)
1789 if sentence:
1790 sentence = sentence.group(1)
1791 else:
1792 sentence = ''
1793 first = eol.split(summary)[0]
1794 summary = max(sentence, first)
1796 # Now reconstitute the message content minus the bits we don't care
1797 # about.
1798 if not keep_body:
1799 content = '\n\n'.join(l)
1801 return summary, content
1803 # vim: set filetype=python sts=4 sw=4 et si :