1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 '''
19 An e-mail gateway for Roundup.
21 Incoming messages are examined for multiple parts:
22 . In a multipart/mixed message or part, each subpart is extracted and
23 examined. The text/plain subparts are assembled to form the textual
24 body of the message, to be stored in the file associated with a "msg"
25 class node. Any parts of other types are each stored in separate files
26 and given "file" class nodes that are linked to the "msg" node.
27 . In a multipart/alternative message or part, we look for a text/plain
28 subpart and ignore the other parts.
30 Summary
31 -------
32 The "summary" property on message nodes is taken from the first non-quoting
33 section in the message body. The message body is divided into sections by
34 blank lines. Sections where the second and all subsequent lines begin with
35 a ">" or "|" character are considered "quoting sections". The first line of
36 the first non-quoting section becomes the summary of the message.
38 Addresses
39 ---------
40 All of the addresses in the To: and Cc: headers of the incoming message are
41 looked up among the user nodes, and the corresponding users are placed in
42 the "recipients" property on the new "msg" node. The address in the From:
43 header similarly determines the "author" property of the new "msg"
44 node. The default handling for addresses that don't have corresponding
45 users is to create new users with no passwords and a username equal to the
46 address. (The web interface does not permit logins for users with no
47 passwords.) If we prefer to reject mail from outside sources, we can simply
48 register an auditor on the "user" class that prevents the creation of user
49 nodes with no passwords.
51 Actions
52 -------
53 The subject line of the incoming message is examined to determine whether
54 the message is an attempt to create a new item or to discuss an existing
55 item. A designator enclosed in square brackets is sought as the first thing
56 on the subject line (after skipping any "Fwd:" or "Re:" prefixes).
58 If an item designator (class name and id number) is found there, the newly
59 created "msg" node is added to the "messages" property for that item, and
60 any new "file" nodes are added to the "files" property for the item.
62 If just an item class name is found there, we attempt to create a new item
63 of that class with its "messages" property initialized to contain the new
64 "msg" node and its "files" property initialized to contain any new "file"
65 nodes.
67 Triggers
68 --------
69 Both cases may trigger detectors (in the first case we are calling the
70 set() method to add the message to the item's spool; in the second case we
71 are calling the create() method to create a new node). If an auditor raises
72 an exception, the original message is bounced back to the sender with the
73 explanatory message given in the exception.
75 $Id: mailgw.py,v 1.28 2001-11-01 22:04:37 richard Exp $
76 '''
79 import string, re, os, mimetools, cStringIO, smtplib, socket, binascii, quopri
80 import traceback
81 import hyperdb, date, password
83 class MailUsageError(ValueError):
84 pass
86 class Message(mimetools.Message):
87 ''' subclass mimetools.Message so we can retrieve the parts of the
88 message...
89 '''
90 def getPart(self):
91 ''' Get a single part of a multipart message and return it as a new
92 Message instance.
93 '''
94 boundary = self.getparam('boundary')
95 mid, end = '--'+boundary, '--'+boundary+'--'
96 s = cStringIO.StringIO()
97 while 1:
98 line = self.fp.readline()
99 if not line:
100 break
101 if line.strip() in (mid, end):
102 break
103 s.write(line)
104 if not s.getvalue().strip():
105 return None
106 s.seek(0)
107 return Message(s)
109 subject_re = re.compile(r'(?P<refwd>\s*\W?\s*(fwd|re)\s*\W?\s*)*'
110 r'\s*(\[(?P<classname>[^\d]+)(?P<nodeid>\d+)?\])'
111 r'\s*(?P<title>[^\[]+)(\[(?P<args>.+?)\])?', re.I)
113 class MailGW:
114 def __init__(self, db):
115 self.db = db
117 def main(self, fp):
118 ''' fp - the file from which to read the Message.
119 '''
120 self.handle_Message(Message(fp))
122 def handle_Message(self, message):
123 '''Handle an RFC822 Message
125 Hanle the Message object by calling handle_message() and then cope
126 with any errors raised by handle_message.
127 This method's job is to make that call and handle any
128 errors in a sane manner. It should be replaced if you wish to
129 handle errors in a different manner.
130 '''
131 m = []
132 try:
133 self.handle_message(message)
134 except MailUsageError, value:
135 # bounce the message back to the sender with the usage message
136 fulldoc = '\n'.join(string.split(__doc__, '\n')[2:])
137 sendto = [message.getaddrlist('from')[0][1]]
138 m = ['Subject: Failed issue tracker submission', '']
139 m.append(str(value))
140 m.append('\n\nMail Gateway Help\n=================')
141 m.append(fulldoc)
142 except:
143 # bounce the message back to the sender with the error message
144 sendto = [message.getaddrlist('from')[0][1]]
145 m = ['Subject: failed issue tracker submission']
146 m.append('')
147 # TODO as attachments?
148 m.append('---- traceback of failure ----')
149 s = cStringIO.StringIO()
150 import traceback
151 traceback.print_exc(None, s)
152 m.append(s.getvalue())
153 m.append('---- failed message follows ----')
154 try:
155 fp.seek(0)
156 except:
157 pass
158 m.append(fp.read())
159 if m:
160 try:
161 smtp = smtplib.SMTP(self.MAILHOST)
162 smtp.sendmail(self.ADMIN_EMAIL, sendto, '\n'.join(m))
163 except socket.error, value:
164 return "Couldn't send confirmation email: mailhost %s"%value
165 except smtplib.SMTPException, value:
166 return "Couldn't send confirmation email: %s"%value
168 def handle_message(self, message):
169 ''' message - a Message instance
171 Parse the message as per the module docstring.
172 '''
173 # handle the subject line
174 subject = message.getheader('subject', '')
175 m = subject_re.match(subject)
176 if not m:
177 raise MailUsageError, '''
178 The message you sent to roundup did not contain a properly formed subject
179 line. The subject must contain a class name or designator to indicate the
180 "topic" of the message. For example:
181 Subject: [issue] This is a new issue
182 - this will create a new issue in the tracker with the title "This is
183 a new issue".
184 Subject: [issue1234] This is a followup to issue 1234
185 - this will append the message's contents to the existing issue 1234
186 in the tracker.
188 Subject was: "%s"
189 '''%subject
190 classname = m.group('classname')
191 nodeid = m.group('nodeid')
192 title = m.group('title').strip()
193 subject_args = m.group('args')
194 try:
195 cl = self.db.getclass(classname)
196 except KeyError:
197 raise MailUsageError, '''
198 The class name you identified in the subject line ("%s") does not exist in the
199 database.
201 Valid class names are: %s
202 Subject was: "%s"
203 '''%(classname, ', '.join(self.db.getclasses()), subject)
205 # If there's no nodeid, check to see if this is a followup and
206 # maybe someone's responded to the initial mail that created an
207 # entry. Try to find the matching nodes with the same title, and
208 # use the _last_ one matched (since that'll _usually_ be the most
209 # recent...)
210 if not nodeid and m.group('refwd'):
211 l = cl.stringFind(title=title)
212 if l:
213 nodeid = l[-1]
215 # start of the props
216 properties = cl.getprops()
217 props = {}
219 # handle the args
220 args = m.group('args')
221 if args:
222 for prop in string.split(args, ';'):
223 try:
224 key, value = prop.split('=')
225 except ValueError, message:
226 raise MailUsageError, '''
227 Subject argument list not of form [arg=value,value,...;arg=value,value...]
228 (specific exception message was "%s")
230 Subject was: "%s"
231 '''%(message, subject)
232 try:
233 type = properties[key]
234 except KeyError:
235 raise MailUsageError, '''
236 Subject argument list refers to an invalid property: "%s"
238 Subject was: "%s"
239 '''%(key, subject)
240 if isinstance(type, hyperdb.String):
241 props[key] = value
242 if isinstance(type, hyperdb.Password):
243 props[key] = password.Password(value)
244 elif isinstance(type, hyperdb.Date):
245 props[key] = date.Date(value)
246 elif isinstance(type, hyperdb.Interval):
247 props[key] = date.Interval(value)
248 elif isinstance(type, hyperdb.Link):
249 props[key] = value
250 elif isinstance(type, hyperdb.Multilink):
251 props[key] = value.split(',')
253 #
254 # handle the users
255 #
256 author = self.db.uidFromAddress(message.getaddrlist('from')[0])
257 recipients = []
258 tracker_email = self.ISSUE_TRACKER_EMAIL.lower()
259 for recipient in message.getaddrlist('to') + message.getaddrlist('cc'):
260 if recipient[1].strip().lower() == tracker_email:
261 continue
262 recipients.append(self.db.uidFromAddress(recipient))
264 # now handle the body - find the message
265 content_type = message.gettype()
266 attachments = []
267 if content_type == 'multipart/mixed':
268 # skip over the intro to the first boundary
269 part = message.getPart()
270 content = None
271 while 1:
272 # get the next part
273 part = message.getPart()
274 if part is None:
275 break
276 # parse it
277 subtype = part.gettype()
278 if subtype == 'text/plain' and not content:
279 # add all text/plain parts to the message content
280 if content is None:
281 content = part.fp.read()
282 else:
283 content = content + part.fp.read()
285 elif subtype == 'message/rfc822':
286 # handle message/rfc822 specially - the name should be
287 # the subject of the actual e-mail embedded here
288 i = part.fp.tell()
289 mailmess = Message(part.fp)
290 name = mailmess.getheader('subject')
291 part.fp.seek(i)
292 attachments.append((name, 'message/rfc822', part.fp.read()))
294 else:
295 # try name on Content-Type
296 name = part.getparam('name')
297 # this is just an attachment
298 encoding = part.getencoding()
299 if encoding == 'base64':
300 data = binascii.a2b_base64(part.fp.read())
301 elif encoding == 'quoted-printable':
302 # the quopri module wants to work with files
303 decoded = cStringIO.StringIO()
304 quopri.decode(part.fp, decoded)
305 data = decoded.getvalue()
306 elif encoding == 'uuencoded':
307 data = binascii.a2b_uu(part.fp.read())
308 attachments.append((name, part.gettype(), data))
310 if content is None:
311 raise MailUsageError, '''
312 Roundup requires the submission to be plain text. The message parser could
313 not find a text/plain part to use.
314 '''
316 elif content_type[:10] == 'multipart/':
317 # skip over the intro to the first boundary
318 message.getPart()
319 content = None
320 while 1:
321 # get the next part
322 part = message.getPart()
323 if part is None:
324 break
325 # parse it
326 if part.gettype() == 'text/plain' and not content:
327 # this one's our content
328 content = part.fp.read()
329 if content is None:
330 raise MailUsageError, '''
331 Roundup requires the submission to be plain text. The message parser could
332 not find a text/plain part to use.
333 '''
335 elif content_type != 'text/plain':
336 raise MailUsageError, '''
337 Roundup requires the submission to be plain text. The message parser could
338 not find a text/plain part to use.
339 '''
341 else:
342 content = message.fp.read()
344 summary, content = parseContent(content)
346 # handle the files
347 files = []
348 for (name, type, data) in attachments:
349 files.append(self.db.file.create(type=type, name=name,
350 content=data))
352 # now handle the db stuff
353 if nodeid:
354 # If an item designator (class name and id number) is found there,
355 # the newly created "msg" node is added to the "messages" property
356 # for that item, and any new "file" nodes are added to the "files"
357 # property for the item.
358 message_id = self.db.msg.create(author=author,
359 recipients=recipients, date=date.Date('.'), summary=summary,
360 content=content, files=files)
361 try:
362 messages = cl.get(nodeid, 'messages')
363 except IndexError:
364 raise MailUsageError, '''
365 The node specified by the designator in the subject of your message ("%s")
366 does not exist.
368 Subject was: "%s"
369 '''%(nodeid, subject)
370 messages.append(message_id)
371 props['messages'] = messages
373 # if the message is currently 'unread' or 'resolved', then set
374 # it to 'chatting'
375 if properties.has_key('status'):
376 try:
377 # determine the id of 'unread', 'resolved' and 'chatting'
378 unread_id = self.db.status.lookup('unread')
379 resolved_id = self.db.status.lookup('resolved')
380 chatting_id = self.db.status.lookup('chatting')
381 except KeyError:
382 pass
383 else:
384 if (not props.has_key('status') or
385 props['status'] == unread_id or
386 props['status'] == resolved_id):
387 props['status'] = chatting_id
389 cl.set(nodeid, **props)
390 else:
391 # If just an item class name is found there, we attempt to create a
392 # new item of that class with its "messages" property initialized to
393 # contain the new "msg" node and its "files" property initialized to
394 # contain any new "file" nodes.
395 message_id = self.db.msg.create(author=author,
396 recipients=recipients, date=date.Date('.'), summary=summary,
397 content=content, files=files)
398 # fill out the properties with defaults where required
399 if properties.has_key('assignedto') and \
400 not props.has_key('assignedto'):
401 props['assignedto'] = '1' # "admin"
402 if properties.has_key('status') and not props.has_key('status'):
403 props['status'] = '1' # "unread"
404 if properties.has_key('title') and not props.has_key('title'):
405 props['title'] = title
406 props['messages'] = [message_id]
407 props['nosy'] = props.get('nosy', []) + recipients
408 props['nosy'].append(author)
409 props['nosy'].sort()
410 nodeid = cl.create(**props)
412 def parseContent(content, blank_line=re.compile(r'[\r\n]+\s*[\r\n]+'),
413 eol=re.compile(r'[\r\n]+'), signature=re.compile(r'^[>|\s]*[-_]+\s*$')):
414 ''' The message body is divided into sections by blank lines.
415 Sections where the second and all subsequent lines begin with a ">" or "|"
416 character are considered "quoting sections". The first line of the first
417 non-quoting section becomes the summary of the message.
418 '''
419 # strip off leading carriage-returns / newlines
420 i = 0
421 for i in range(len(content)):
422 if content[i] not in '\r\n':
423 break
424 if i > 0:
425 sections = blank_line.split(content[i:])
426 else:
427 sections = blank_line.split(content)
429 # extract out the summary from the message
430 summary = ''
431 l = []
432 for section in sections:
433 #section = section.strip()
434 if not section:
435 continue
436 lines = eol.split(section)
437 if lines[0] and lines[0][0] in '>|':
438 continue
439 if len(lines) > 1 and lines[1] and lines[1][0] in '>|':
440 continue
441 if not summary:
442 summary = lines[0]
443 l.append(section)
444 continue
445 if signature.match(lines[0]):
446 break
447 l.append(section)
448 return summary, '\n\n'.join(l)
450 #
451 # $Log: not supported by cvs2svn $
452 # Revision 1.27 2001/10/30 11:26:10 richard
453 # Case-insensitive match for ISSUE_TRACKER_EMAIL in address in e-mail.
454 #
455 # Revision 1.26 2001/10/30 00:54:45 richard
456 # Features:
457 # . #467129 ] Lossage when username=e-mail-address
458 # . #473123 ] Change message generation for author
459 # . MailGW now moves 'resolved' to 'chatting' on receiving e-mail for an issue.
460 #
461 # Revision 1.25 2001/10/28 23:22:28 richard
462 # fixed bug #474749 ] Indentations lost
463 #
464 # Revision 1.24 2001/10/23 22:57:52 richard
465 # Fix unread->chatting auto transition, thanks Roch'e
466 #
467 # Revision 1.23 2001/10/21 04:00:20 richard
468 # MailGW now moves 'unread' to 'chatting' on receiving e-mail for an issue.
469 #
470 # Revision 1.22 2001/10/21 03:35:13 richard
471 # bug #473125: Paragraph in e-mails
472 #
473 # Revision 1.21 2001/10/21 00:53:42 richard
474 # bug #473130: Nosy list not set correctly
475 #
476 # Revision 1.20 2001/10/17 23:13:19 richard
477 # Did a fair bit of work on the admin tool. Now has an extra command "table"
478 # which displays node information in a tabular format. Also fixed import and
479 # export so they work. Removed freshen.
480 # Fixed quopri usage in mailgw from bug reports.
481 #
482 # Revision 1.19 2001/10/11 23:43:04 richard
483 # Implemented the comma-separated printing option in the admin tool.
484 # Fixed a typo (more of a vim-o actually :) in mailgw.
485 #
486 # Revision 1.18 2001/10/11 06:38:57 richard
487 # Initial cut at trying to handle people responding to CC'ed messages that
488 # create an issue.
489 #
490 # Revision 1.17 2001/10/09 07:25:59 richard
491 # Added the Password property type. See "pydoc roundup.password" for
492 # implementation details. Have updated some of the documentation too.
493 #
494 # Revision 1.16 2001/10/05 02:23:24 richard
495 # . roundup-admin create now prompts for property info if none is supplied
496 # on the command-line.
497 # . hyperdb Class getprops() method may now return only the mutable
498 # properties.
499 # . Login now uses cookies, which makes it a whole lot more flexible. We can
500 # now support anonymous user access (read-only, unless there's an
501 # "anonymous" user, in which case write access is permitted). Login
502 # handling has been moved into cgi_client.Client.main()
503 # . The "extended" schema is now the default in roundup init.
504 # . The schemas have had their page headings modified to cope with the new
505 # login handling. Existing installations should copy the interfaces.py
506 # file from the roundup lib directory to their instance home.
507 # . Incorrectly had a Bizar Software copyright on the cgitb.py module from
508 # Ping - has been removed.
509 # . Fixed a whole bunch of places in the CGI interface where we should have
510 # been returning Not Found instead of throwing an exception.
511 # . Fixed a deviation from the spec: trying to modify the 'id' property of
512 # an item now throws an exception.
513 #
514 # Revision 1.15 2001/08/30 06:01:17 richard
515 # Fixed missing import in mailgw :(
516 #
517 # Revision 1.14 2001/08/13 23:02:54 richard
518 # Make the mail parser a little more robust.
519 #
520 # Revision 1.13 2001/08/12 06:32:36 richard
521 # using isinstance(blah, Foo) now instead of isFooType
522 #
523 # Revision 1.12 2001/08/08 01:27:00 richard
524 # Added better error handling to mailgw.
525 #
526 # Revision 1.11 2001/08/08 00:08:03 richard
527 # oops ;)
528 #
529 # Revision 1.10 2001/08/07 00:24:42 richard
530 # stupid typo
531 #
532 # Revision 1.9 2001/08/07 00:15:51 richard
533 # Added the copyright/license notice to (nearly) all files at request of
534 # Bizar Software.
535 #
536 # Revision 1.8 2001/08/05 07:06:07 richard
537 # removed some print statements
538 #
539 # Revision 1.7 2001/08/03 07:18:22 richard
540 # Implemented correct mail splitting (was taking a shortcut). Added unit
541 # tests. Also snips signatures now too.
542 #
543 # Revision 1.6 2001/08/01 04:24:21 richard
544 # mailgw was assuming certain properties existed on the issues being created.
545 #
546 # Revision 1.5 2001/07/29 07:01:39 richard
547 # Added vim command to all source so that we don't get no steenkin' tabs :)
548 #
549 # Revision 1.4 2001/07/28 06:43:02 richard
550 # Multipart message class has the getPart method now. Added some tests for it.
551 #
552 # Revision 1.3 2001/07/28 00:34:34 richard
553 # Fixed some non-string node ids.
554 #
555 # Revision 1.2 2001/07/22 12:09:32 richard
556 # Final commit of Grande Splite
557 #
558 #
559 # vim: set filetype=python ts=4 sw=4 et si