1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 '''
19 An e-mail gateway for Roundup.
21 Incoming messages are examined for multiple parts:
22 . In a multipart/mixed message or part, each subpart is extracted and
23 examined. The text/plain subparts are assembled to form the textual
24 body of the message, to be stored in the file associated with a "msg"
25 class node. Any parts of other types are each stored in separate files
26 and given "file" class nodes that are linked to the "msg" node.
27 . In a multipart/alternative message or part, we look for a text/plain
28 subpart and ignore the other parts.
30 Summary
31 -------
32 The "summary" property on message nodes is taken from the first non-quoting
33 section in the message body. The message body is divided into sections by
34 blank lines. Sections where the second and all subsequent lines begin with
35 a ">" or "|" character are considered "quoting sections". The first line of
36 the first non-quoting section becomes the summary of the message.
38 Addresses
39 ---------
40 All of the addresses in the To: and Cc: headers of the incoming message are
41 looked up among the user nodes, and the corresponding users are placed in
42 the "recipients" property on the new "msg" node. The address in the From:
43 header similarly determines the "author" property of the new "msg"
44 node. The default handling for addresses that don't have corresponding
45 users is to create new users with no passwords and a username equal to the
46 address. (The web interface does not permit logins for users with no
47 passwords.) If we prefer to reject mail from outside sources, we can simply
48 register an auditor on the "user" class that prevents the creation of user
49 nodes with no passwords.
51 Actions
52 -------
53 The subject line of the incoming message is examined to determine whether
54 the message is an attempt to create a new item or to discuss an existing
55 item. A designator enclosed in square brackets is sought as the first thing
56 on the subject line (after skipping any "Fwd:" or "Re:" prefixes).
58 If an item designator (class name and id number) is found there, the newly
59 created "msg" node is added to the "messages" property for that item, and
60 any new "file" nodes are added to the "files" property for the item.
62 If just an item class name is found there, we attempt to create a new item
63 of that class with its "messages" property initialized to contain the new
64 "msg" node and its "files" property initialized to contain any new "file"
65 nodes.
67 Triggers
68 --------
69 Both cases may trigger detectors (in the first case we are calling the
70 set() method to add the message to the item's spool; in the second case we
71 are calling the create() method to create a new node). If an auditor raises
72 an exception, the original message is bounced back to the sender with the
73 explanatory message given in the exception.
75 $Id: mailgw.py,v 1.27 2001-10-30 11:26:10 richard Exp $
76 '''
79 import string, re, os, mimetools, cStringIO, smtplib, socket, binascii, quopri
80 import traceback
81 import hyperdb, date, password
83 class MailUsageError(ValueError):
84 pass
86 class Message(mimetools.Message):
87 ''' subclass mimetools.Message so we can retrieve the parts of the
88 message...
89 '''
90 def getPart(self):
91 ''' Get a single part of a multipart message and return it as a new
92 Message instance.
93 '''
94 boundary = self.getparam('boundary')
95 mid, end = '--'+boundary, '--'+boundary+'--'
96 s = cStringIO.StringIO()
97 while 1:
98 line = self.fp.readline()
99 if not line:
100 break
101 if line.strip() in (mid, end):
102 break
103 s.write(line)
104 if not s.getvalue().strip():
105 return None
106 s.seek(0)
107 return Message(s)
109 subject_re = re.compile(r'(?P<refwd>\s*\W?\s*(fwd|re)\s*\W?\s*)*'
110 r'\s*(\[(?P<classname>[^\d]+)(?P<nodeid>\d+)?\])'
111 r'\s*(?P<title>[^\[]+)(\[(?P<args>.+?)\])?', re.I)
113 class MailGW:
114 def __init__(self, db):
115 self.db = db
117 def main(self, fp):
118 ''' fp - the file from which to read the Message.
120 Read a message from fp and then call handle_message() with the
121 result. This method's job is to make that call and handle any
122 errors in a sane manner. It should be replaced if you wish to
123 handle errors in a different manner.
124 '''
125 # ok, figure the subject, author, recipients and content-type
126 message = Message(fp)
127 m = []
128 try:
129 self.handle_message(message)
130 except MailUsageError, value:
131 # bounce the message back to the sender with the usage message
132 fulldoc = '\n'.join(string.split(__doc__, '\n')[2:])
133 sendto = [message.getaddrlist('from')[0][1]]
134 m = ['Subject: Failed issue tracker submission', '']
135 m.append(str(value))
136 m.append('\n\nMail Gateway Help\n=================')
137 m.append(fulldoc)
138 except:
139 # bounce the message back to the sender with the error message
140 sendto = [message.getaddrlist('from')[0][1]]
141 m = ['Subject: failed issue tracker submission']
142 m.append('')
143 # TODO as attachments?
144 m.append('---- traceback of failure ----')
145 s = cStringIO.StringIO()
146 import traceback
147 traceback.print_exc(None, s)
148 m.append(s.getvalue())
149 m.append('---- failed message follows ----')
150 try:
151 fp.seek(0)
152 except:
153 pass
154 m.append(fp.read())
155 if m:
156 try:
157 smtp = smtplib.SMTP(self.MAILHOST)
158 smtp.sendmail(self.ADMIN_EMAIL, sendto, '\n'.join(m))
159 except socket.error, value:
160 return "Couldn't send confirmation email: mailhost %s"%value
161 except smtplib.SMTPException, value:
162 return "Couldn't send confirmation email: %s"%value
164 def handle_message(self, message):
165 ''' message - a Message instance
167 Parse the message as per the module docstring.
168 '''
169 # handle the subject line
170 subject = message.getheader('subject', '')
171 m = subject_re.match(subject)
172 if not m:
173 raise MailUsageError, '''
174 The message you sent to roundup did not contain a properly formed subject
175 line. The subject must contain a class name or designator to indicate the
176 "topic" of the message. For example:
177 Subject: [issue] This is a new issue
178 - this will create a new issue in the tracker with the title "This is
179 a new issue".
180 Subject: [issue1234] This is a followup to issue 1234
181 - this will append the message's contents to the existing issue 1234
182 in the tracker.
184 Subject was: "%s"
185 '''%subject
186 classname = m.group('classname')
187 nodeid = m.group('nodeid')
188 title = m.group('title').strip()
189 subject_args = m.group('args')
190 try:
191 cl = self.db.getclass(classname)
192 except KeyError:
193 raise MailUsageError, '''
194 The class name you identified in the subject line ("%s") does not exist in the
195 database.
197 Valid class names are: %s
198 Subject was: "%s"
199 '''%(classname, ', '.join(self.db.getclasses()), subject)
201 # If there's no nodeid, check to see if this is a followup and
202 # maybe someone's responded to the initial mail that created an
203 # entry. Try to find the matching nodes with the same title, and
204 # use the _last_ one matched (since that'll _usually_ be the most
205 # recent...)
206 if not nodeid and m.group('refwd'):
207 l = cl.stringFind(title=title)
208 if l:
209 nodeid = l[-1]
211 # start of the props
212 properties = cl.getprops()
213 props = {}
215 # handle the args
216 args = m.group('args')
217 if args:
218 for prop in string.split(args, ';'):
219 try:
220 key, value = prop.split('=')
221 except ValueError, message:
222 raise MailUsageError, '''
223 Subject argument list not of form [arg=value,value,...;arg=value,value...]
224 (specific exception message was "%s")
226 Subject was: "%s"
227 '''%(message, subject)
228 try:
229 type = properties[key]
230 except KeyError:
231 raise MailUsageError, '''
232 Subject argument list refers to an invalid property: "%s"
234 Subject was: "%s"
235 '''%(key, subject)
236 if isinstance(type, hyperdb.String):
237 props[key] = value
238 if isinstance(type, hyperdb.Password):
239 props[key] = password.Password(value)
240 elif isinstance(type, hyperdb.Date):
241 props[key] = date.Date(value)
242 elif isinstance(type, hyperdb.Interval):
243 props[key] = date.Interval(value)
244 elif isinstance(type, hyperdb.Link):
245 props[key] = value
246 elif isinstance(type, hyperdb.Multilink):
247 props[key] = value.split(',')
249 #
250 # handle the users
251 #
252 author = self.db.uidFromAddress(message.getaddrlist('from')[0])
253 recipients = []
254 tracker_email = self.ISSUE_TRACKER_EMAIL.lower()
255 for recipient in message.getaddrlist('to') + message.getaddrlist('cc'):
256 if recipient[1].strip().lower() == tracker_email:
257 continue
258 recipients.append(self.db.uidFromAddress(recipient))
260 # now handle the body - find the message
261 content_type = message.gettype()
262 attachments = []
263 if content_type == 'multipart/mixed':
264 # skip over the intro to the first boundary
265 part = message.getPart()
266 content = None
267 while 1:
268 # get the next part
269 part = message.getPart()
270 if part is None:
271 break
272 # parse it
273 subtype = part.gettype()
274 if subtype == 'text/plain' and not content:
275 # add all text/plain parts to the message content
276 if content is None:
277 content = part.fp.read()
278 else:
279 content = content + part.fp.read()
281 elif subtype == 'message/rfc822':
282 # handle message/rfc822 specially - the name should be
283 # the subject of the actual e-mail embedded here
284 i = part.fp.tell()
285 mailmess = Message(part.fp)
286 name = mailmess.getheader('subject')
287 part.fp.seek(i)
288 attachments.append((name, 'message/rfc822', part.fp.read()))
290 else:
291 # try name on Content-Type
292 name = part.getparam('name')
293 # this is just an attachment
294 encoding = part.getencoding()
295 if encoding == 'base64':
296 data = binascii.a2b_base64(part.fp.read())
297 elif encoding == 'quoted-printable':
298 # the quopri module wants to work with files
299 decoded = cStringIO.StringIO()
300 quopri.decode(part.fp, decoded)
301 data = decoded.getvalue()
302 elif encoding == 'uuencoded':
303 data = binascii.a2b_uu(part.fp.read())
304 attachments.append((name, part.gettype(), data))
306 if content is None:
307 raise MailUsageError, '''
308 Roundup requires the submission to be plain text. The message parser could
309 not find a text/plain part to use.
310 '''
312 elif content_type[:10] == 'multipart/':
313 # skip over the intro to the first boundary
314 message.getPart()
315 content = None
316 while 1:
317 # get the next part
318 part = message.getPart()
319 if part is None:
320 break
321 # parse it
322 if part.gettype() == 'text/plain' and not content:
323 # this one's our content
324 content = part.fp.read()
325 if content is None:
326 raise MailUsageError, '''
327 Roundup requires the submission to be plain text. The message parser could
328 not find a text/plain part to use.
329 '''
331 elif content_type != 'text/plain':
332 raise MailUsageError, '''
333 Roundup requires the submission to be plain text. The message parser could
334 not find a text/plain part to use.
335 '''
337 else:
338 content = message.fp.read()
340 summary, content = parseContent(content)
342 # handle the files
343 files = []
344 for (name, type, data) in attachments:
345 files.append(self.db.file.create(type=type, name=name,
346 content=data))
348 # now handle the db stuff
349 if nodeid:
350 # If an item designator (class name and id number) is found there,
351 # the newly created "msg" node is added to the "messages" property
352 # for that item, and any new "file" nodes are added to the "files"
353 # property for the item.
354 message_id = self.db.msg.create(author=author,
355 recipients=recipients, date=date.Date('.'), summary=summary,
356 content=content, files=files)
357 try:
358 messages = cl.get(nodeid, 'messages')
359 except IndexError:
360 raise MailUsageError, '''
361 The node specified by the designator in the subject of your message ("%s")
362 does not exist.
364 Subject was: "%s"
365 '''%(nodeid, subject)
366 messages.append(message_id)
367 props['messages'] = messages
369 # if the message is currently 'unread' or 'resolved', then set
370 # it to 'chatting'
371 if properties.has_key('status'):
372 try:
373 # determine the id of 'unread', 'resolved' and 'chatting'
374 unread_id = self.db.status.lookup('unread')
375 resolved_id = self.db.status.lookup('resolved')
376 chatting_id = self.db.status.lookup('chatting')
377 except KeyError:
378 pass
379 else:
380 if (not props.has_key('status') or
381 props['status'] == unread_id or
382 props['status'] == resolved_id):
383 props['status'] = chatting_id
385 cl.set(nodeid, **props)
386 else:
387 # If just an item class name is found there, we attempt to create a
388 # new item of that class with its "messages" property initialized to
389 # contain the new "msg" node and its "files" property initialized to
390 # contain any new "file" nodes.
391 message_id = self.db.msg.create(author=author,
392 recipients=recipients, date=date.Date('.'), summary=summary,
393 content=content, files=files)
394 # fill out the properties with defaults where required
395 if properties.has_key('assignedto') and \
396 not props.has_key('assignedto'):
397 props['assignedto'] = '1' # "admin"
398 if properties.has_key('status') and not props.has_key('status'):
399 props['status'] = '1' # "unread"
400 if properties.has_key('title') and not props.has_key('title'):
401 props['title'] = title
402 props['messages'] = [message_id]
403 props['nosy'] = props.get('nosy', []) + recipients
404 props['nosy'].append(author)
405 props['nosy'].sort()
406 nodeid = cl.create(**props)
408 def parseContent(content, blank_line=re.compile(r'[\r\n]+\s*[\r\n]+'),
409 eol=re.compile(r'[\r\n]+'), signature=re.compile(r'^[>|\s]*[-_]+\s*$')):
410 ''' The message body is divided into sections by blank lines.
411 Sections where the second and all subsequent lines begin with a ">" or "|"
412 character are considered "quoting sections". The first line of the first
413 non-quoting section becomes the summary of the message.
414 '''
415 # strip off leading carriage-returns / newlines
416 i = 0
417 for i in range(len(content)):
418 if content[i] not in '\r\n':
419 break
420 if i > 0:
421 sections = blank_line.split(content[i:])
422 else:
423 sections = blank_line.split(content)
425 # extract out the summary from the message
426 summary = ''
427 l = []
428 for section in sections:
429 #section = section.strip()
430 if not section:
431 continue
432 lines = eol.split(section)
433 if lines[0] and lines[0][0] in '>|':
434 continue
435 if len(lines) > 1 and lines[1] and lines[1][0] in '>|':
436 continue
437 if not summary:
438 summary = lines[0]
439 l.append(section)
440 continue
441 if signature.match(lines[0]):
442 break
443 l.append(section)
444 return summary, '\n\n'.join(l)
446 #
447 # $Log: not supported by cvs2svn $
448 # Revision 1.26 2001/10/30 00:54:45 richard
449 # Features:
450 # . #467129 ] Lossage when username=e-mail-address
451 # . #473123 ] Change message generation for author
452 # . MailGW now moves 'resolved' to 'chatting' on receiving e-mail for an issue.
453 #
454 # Revision 1.25 2001/10/28 23:22:28 richard
455 # fixed bug #474749 ] Indentations lost
456 #
457 # Revision 1.24 2001/10/23 22:57:52 richard
458 # Fix unread->chatting auto transition, thanks Roch'e
459 #
460 # Revision 1.23 2001/10/21 04:00:20 richard
461 # MailGW now moves 'unread' to 'chatting' on receiving e-mail for an issue.
462 #
463 # Revision 1.22 2001/10/21 03:35:13 richard
464 # bug #473125: Paragraph in e-mails
465 #
466 # Revision 1.21 2001/10/21 00:53:42 richard
467 # bug #473130: Nosy list not set correctly
468 #
469 # Revision 1.20 2001/10/17 23:13:19 richard
470 # Did a fair bit of work on the admin tool. Now has an extra command "table"
471 # which displays node information in a tabular format. Also fixed import and
472 # export so they work. Removed freshen.
473 # Fixed quopri usage in mailgw from bug reports.
474 #
475 # Revision 1.19 2001/10/11 23:43:04 richard
476 # Implemented the comma-separated printing option in the admin tool.
477 # Fixed a typo (more of a vim-o actually :) in mailgw.
478 #
479 # Revision 1.18 2001/10/11 06:38:57 richard
480 # Initial cut at trying to handle people responding to CC'ed messages that
481 # create an issue.
482 #
483 # Revision 1.17 2001/10/09 07:25:59 richard
484 # Added the Password property type. See "pydoc roundup.password" for
485 # implementation details. Have updated some of the documentation too.
486 #
487 # Revision 1.16 2001/10/05 02:23:24 richard
488 # . roundup-admin create now prompts for property info if none is supplied
489 # on the command-line.
490 # . hyperdb Class getprops() method may now return only the mutable
491 # properties.
492 # . Login now uses cookies, which makes it a whole lot more flexible. We can
493 # now support anonymous user access (read-only, unless there's an
494 # "anonymous" user, in which case write access is permitted). Login
495 # handling has been moved into cgi_client.Client.main()
496 # . The "extended" schema is now the default in roundup init.
497 # . The schemas have had their page headings modified to cope with the new
498 # login handling. Existing installations should copy the interfaces.py
499 # file from the roundup lib directory to their instance home.
500 # . Incorrectly had a Bizar Software copyright on the cgitb.py module from
501 # Ping - has been removed.
502 # . Fixed a whole bunch of places in the CGI interface where we should have
503 # been returning Not Found instead of throwing an exception.
504 # . Fixed a deviation from the spec: trying to modify the 'id' property of
505 # an item now throws an exception.
506 #
507 # Revision 1.15 2001/08/30 06:01:17 richard
508 # Fixed missing import in mailgw :(
509 #
510 # Revision 1.14 2001/08/13 23:02:54 richard
511 # Make the mail parser a little more robust.
512 #
513 # Revision 1.13 2001/08/12 06:32:36 richard
514 # using isinstance(blah, Foo) now instead of isFooType
515 #
516 # Revision 1.12 2001/08/08 01:27:00 richard
517 # Added better error handling to mailgw.
518 #
519 # Revision 1.11 2001/08/08 00:08:03 richard
520 # oops ;)
521 #
522 # Revision 1.10 2001/08/07 00:24:42 richard
523 # stupid typo
524 #
525 # Revision 1.9 2001/08/07 00:15:51 richard
526 # Added the copyright/license notice to (nearly) all files at request of
527 # Bizar Software.
528 #
529 # Revision 1.8 2001/08/05 07:06:07 richard
530 # removed some print statements
531 #
532 # Revision 1.7 2001/08/03 07:18:22 richard
533 # Implemented correct mail splitting (was taking a shortcut). Added unit
534 # tests. Also snips signatures now too.
535 #
536 # Revision 1.6 2001/08/01 04:24:21 richard
537 # mailgw was assuming certain properties existed on the issues being created.
538 #
539 # Revision 1.5 2001/07/29 07:01:39 richard
540 # Added vim command to all source so that we don't get no steenkin' tabs :)
541 #
542 # Revision 1.4 2001/07/28 06:43:02 richard
543 # Multipart message class has the getPart method now. Added some tests for it.
544 #
545 # Revision 1.3 2001/07/28 00:34:34 richard
546 # Fixed some non-string node ids.
547 #
548 # Revision 1.2 2001/07/22 12:09:32 richard
549 # Final commit of Grande Splite
550 #
551 #
552 # vim: set filetype=python ts=4 sw=4 et si