1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
19 __doc__ = '''
20 An e-mail gateway for Roundup.
22 Incoming messages are examined for multiple parts:
23 . In a multipart/mixed message or part, each subpart is extracted and
24 examined. The text/plain subparts are assembled to form the textual
25 body of the message, to be stored in the file associated with a "msg"
26 class node. Any parts of other types are each stored in separate files
27 and given "file" class nodes that are linked to the "msg" node.
28 . In a multipart/alternative message or part, we look for a text/plain
29 subpart and ignore the other parts.
31 Summary
32 -------
33 The "summary" property on message nodes is taken from the first non-quoting
34 section in the message body. The message body is divided into sections by
35 blank lines. Sections where the second and all subsequent lines begin with
36 a ">" or "|" character are considered "quoting sections". The first line of
37 the first non-quoting section becomes the summary of the message.
39 Addresses
40 ---------
41 All of the addresses in the To: and Cc: headers of the incoming message are
42 looked up among the user nodes, and the corresponding users are placed in
43 the "recipients" property on the new "msg" node. The address in the From:
44 header similarly determines the "author" property of the new "msg"
45 node. The default handling for addresses that don't have corresponding
46 users is to create new users with no passwords and a username equal to the
47 address. (The web interface does not permit logins for users with no
48 passwords.) If we prefer to reject mail from outside sources, we can simply
49 register an auditor on the "user" class that prevents the creation of user
50 nodes with no passwords.
52 Actions
53 -------
54 The subject line of the incoming message is examined to determine whether
55 the message is an attempt to create a new item or to discuss an existing
56 item. A designator enclosed in square brackets is sought as the first thing
57 on the subject line (after skipping any "Fwd:" or "Re:" prefixes).
59 If an item designator (class name and id number) is found there, the newly
60 created "msg" node is added to the "messages" property for that item, and
61 any new "file" nodes are added to the "files" property for the item.
63 If just an item class name is found there, we attempt to create a new item
64 of that class with its "messages" property initialized to contain the new
65 "msg" node and its "files" property initialized to contain any new "file"
66 nodes.
68 Triggers
69 --------
70 Both cases may trigger detectors (in the first case we are calling the
71 set() method to add the message to the item's spool; in the second case we
72 are calling the create() method to create a new node). If an auditor raises
73 an exception, the original message is bounced back to the sender with the
74 explanatory message given in the exception.
76 $Id: mailgw.py,v 1.80 2002-08-01 00:56:22 richard Exp $
77 '''
80 import string, re, os, mimetools, cStringIO, smtplib, socket, binascii, quopri
81 import time, random
82 import traceback, MimeWriter
83 import hyperdb, date, password
85 SENDMAILDEBUG = os.environ.get('SENDMAILDEBUG', '')
87 class MailGWError(ValueError):
88 pass
90 class MailUsageError(ValueError):
91 pass
93 class MailUsageHelp(Exception):
94 pass
96 class Unauthorized(Exception):
97 """ Access denied """
99 def initialiseSecurity(security):
100 ''' Create some Permissions and Roles on the security object
102 This function is directly invoked by security.Security.__init__()
103 as a part of the Security object instantiation.
104 '''
105 newid = security.addPermission(name="Email Registration",
106 description="Anonymous may register through e-mail")
107 security.addPermission(name="Email Access",
108 description="User may use the email interface")
110 class Message(mimetools.Message):
111 ''' subclass mimetools.Message so we can retrieve the parts of the
112 message...
113 '''
114 def getPart(self):
115 ''' Get a single part of a multipart message and return it as a new
116 Message instance.
117 '''
118 boundary = self.getparam('boundary')
119 mid, end = '--'+boundary, '--'+boundary+'--'
120 s = cStringIO.StringIO()
121 while 1:
122 line = self.fp.readline()
123 if not line:
124 break
125 if line.strip() in (mid, end):
126 break
127 s.write(line)
128 if not s.getvalue().strip():
129 return None
130 s.seek(0)
131 return Message(s)
133 subject_re = re.compile(r'(?P<refwd>\s*\W?\s*(fwd|re|aw)\s*\W?\s*)*'
134 r'\s*(\[(?P<classname>[^\d\s]+)(?P<nodeid>\d+)?\])?'
135 r'\s*(?P<title>[^[]+)?(\[(?P<args>.+?)\])?', re.I)
137 class MailGW:
138 def __init__(self, instance, db):
139 self.instance = instance
140 self.db = db
142 # should we trap exceptions (normal usage) or pass them through
143 # (for testing)
144 self.trapExceptions = 1
146 def main(self, fp):
147 ''' fp - the file from which to read the Message.
148 '''
149 return self.handle_Message(Message(fp))
151 def handle_Message(self, message):
152 '''Handle an RFC822 Message
154 Handle the Message object by calling handle_message() and then cope
155 with any errors raised by handle_message.
156 This method's job is to make that call and handle any
157 errors in a sane manner. It should be replaced if you wish to
158 handle errors in a different manner.
159 '''
160 # in some rare cases, a particularly stuffed-up e-mail will make
161 # its way into here... try to handle it gracefully
162 sendto = message.getaddrlist('from')
163 if sendto:
164 if not self.trapExceptions:
165 return self.handle_message(message)
166 try:
167 return self.handle_message(message)
168 except MailUsageHelp:
169 # bounce the message back to the sender with the usage message
170 fulldoc = '\n'.join(string.split(__doc__, '\n')[2:])
171 sendto = [sendto[0][1]]
172 m = ['']
173 m.append('\n\nMail Gateway Help\n=================')
174 m.append(fulldoc)
175 m = self.bounce_message(message, sendto, m,
176 subject="Mail Gateway Help")
177 except MailUsageError, value:
178 # bounce the message back to the sender with the usage message
179 fulldoc = '\n'.join(string.split(__doc__, '\n')[2:])
180 sendto = [sendto[0][1]]
181 m = ['']
182 m.append(str(value))
183 m.append('\n\nMail Gateway Help\n=================')
184 m.append(fulldoc)
185 m = self.bounce_message(message, sendto, m)
186 except Unauthorized, value:
187 # just inform the user that he is not authorized
188 sendto = [sendto[0][1]]
189 m = ['']
190 m.append(str(value))
191 m = self.bounce_message(message, sendto, m)
192 except:
193 # bounce the message back to the sender with the error message
194 sendto = [sendto[0][1], self.instance.ADMIN_EMAIL]
195 m = ['']
196 m.append('An unexpected error occurred during the processing')
197 m.append('of your message. The tracker administrator is being')
198 m.append('notified.\n')
199 m.append('---- traceback of failure ----')
200 s = cStringIO.StringIO()
201 import traceback
202 traceback.print_exc(None, s)
203 m.append(s.getvalue())
204 m = self.bounce_message(message, sendto, m)
205 else:
206 # very bad-looking message - we don't even know who sent it
207 sendto = [self.instance.ADMIN_EMAIL]
208 m = ['Subject: badly formed message from mail gateway']
209 m.append('')
210 m.append('The mail gateway retrieved a message which has no From:')
211 m.append('line, indicating that it is corrupt. Please check your')
212 m.append('mail gateway source. Failed message is attached.')
213 m.append('')
214 m = self.bounce_message(message, sendto, m,
215 subject='Badly formed message from mail gateway')
217 # now send the message
218 if SENDMAILDEBUG:
219 open(SENDMAILDEBUG, 'w').write('From: %s\nTo: %s\n%s\n'%(
220 self.instance.ADMIN_EMAIL, ', '.join(sendto), m.getvalue()))
221 else:
222 try:
223 smtp = smtplib.SMTP(self.instance.MAILHOST)
224 smtp.sendmail(self.instance.ADMIN_EMAIL, sendto, m.getvalue())
225 except socket.error, value:
226 raise MailGWError, "Couldn't send error email: "\
227 "mailhost %s"%value
228 except smtplib.SMTPException, value:
229 raise MailGWError, "Couldn't send error email: %s"%value
231 def bounce_message(self, message, sendto, error,
232 subject='Failed issue tracker submission'):
233 ''' create a message that explains the reason for the failed
234 issue submission to the author and attach the original
235 message.
236 '''
237 msg = cStringIO.StringIO()
238 writer = MimeWriter.MimeWriter(msg)
239 writer.addheader('Subject', subject)
240 writer.addheader('From', '%s <%s>'% (self.instance.INSTANCE_NAME,
241 self.instance.ISSUE_TRACKER_EMAIL))
242 writer.addheader('To', ','.join(sendto))
243 writer.addheader('MIME-Version', '1.0')
244 part = writer.startmultipartbody('mixed')
245 part = writer.nextpart()
246 body = part.startbody('text/plain')
247 body.write('\n'.join(error))
249 # reconstruct the original message
250 m = cStringIO.StringIO()
251 w = MimeWriter.MimeWriter(m)
252 # default the content_type, just in case...
253 content_type = 'text/plain'
254 # add the headers except the content-type
255 for header in message.headers:
256 header_name = header.split(':')[0]
257 if header_name.lower() == 'content-type':
258 content_type = message.getheader(header_name)
259 elif message.getheader(header_name):
260 w.addheader(header_name, message.getheader(header_name))
261 # now attach the message body
262 body = w.startbody(content_type)
263 try:
264 message.rewindbody()
265 except IOError:
266 body.write("*** couldn't include message body: read from pipe ***")
267 else:
268 body.write(message.fp.read())
270 # attach the original message to the returned message
271 part = writer.nextpart()
272 part.addheader('Content-Disposition','attachment')
273 part.addheader('Content-Description','Message you sent')
274 part.addheader('Content-Transfer-Encoding', '7bit')
275 body = part.startbody('message/rfc822')
276 body.write(m.getvalue())
278 writer.lastpart()
279 return msg
281 def get_part_data_decoded(self,part):
282 encoding = part.getencoding()
283 data = None
284 if encoding == 'base64':
285 # BUG: is base64 really used for text encoding or
286 # are we inserting zip files here.
287 data = binascii.a2b_base64(part.fp.read())
288 elif encoding == 'quoted-printable':
289 # the quopri module wants to work with files
290 decoded = cStringIO.StringIO()
291 quopri.decode(part.fp, decoded)
292 data = decoded.getvalue()
293 elif encoding == 'uuencoded':
294 data = binascii.a2b_uu(part.fp.read())
295 else:
296 # take it as text
297 data = part.fp.read()
298 return data
300 def handle_message(self, message):
301 ''' message - a Message instance
303 Parse the message as per the module docstring.
304 '''
305 # handle the subject line
306 subject = message.getheader('subject', '')
308 if subject.strip() == 'help':
309 raise MailUsageHelp
311 m = subject_re.match(subject)
313 # check for well-formed subject line
314 if m:
315 # get the classname
316 classname = m.group('classname')
317 if classname is None:
318 # no classname, fallback on the default
319 if hasattr(self.instance, 'MAIL_DEFAULT_CLASS') and \
320 self.instance.MAIL_DEFAULT_CLASS:
321 classname = self.instance.MAIL_DEFAULT_CLASS
322 else:
323 # fail
324 m = None
326 if not m:
327 raise MailUsageError, '''
328 The message you sent to roundup did not contain a properly formed subject
329 line. The subject must contain a class name or designator to indicate the
330 "topic" of the message. For example:
331 Subject: [issue] This is a new issue
332 - this will create a new issue in the tracker with the title "This is
333 a new issue".
334 Subject: [issue1234] This is a followup to issue 1234
335 - this will append the message's contents to the existing issue 1234
336 in the tracker.
338 Subject was: "%s"
339 '''%subject
341 # get the class
342 try:
343 cl = self.db.getclass(classname)
344 except KeyError:
345 raise MailUsageError, '''
346 The class name you identified in the subject line ("%s") does not exist in the
347 database.
349 Valid class names are: %s
350 Subject was: "%s"
351 '''%(classname, ', '.join(self.db.getclasses()), subject)
353 # get the optional nodeid
354 nodeid = m.group('nodeid')
356 # title is optional too
357 title = m.group('title')
358 if title:
359 title = title.strip()
360 else:
361 title = ''
363 # but we do need either a title or a nodeid...
364 if nodeid is None and not title:
365 raise MailUsageError, '''
366 I cannot match your message to a node in the database - you need to either
367 supply a full node identifier (with number, eg "[issue123]" or keep the
368 previous subject title intact so I can match that.
370 Subject was: "%s"
371 '''%subject
373 # If there's no nodeid, check to see if this is a followup and
374 # maybe someone's responded to the initial mail that created an
375 # entry. Try to find the matching nodes with the same title, and
376 # use the _last_ one matched (since that'll _usually_ be the most
377 # recent...)
378 if nodeid is None and m.group('refwd'):
379 l = cl.stringFind(title=title)
380 if l:
381 nodeid = l[-1]
383 # if a nodeid was specified, make sure it's valid
384 if nodeid is not None and not cl.hasnode(nodeid):
385 raise MailUsageError, '''
386 The node specified by the designator in the subject of your message ("%s")
387 does not exist.
389 Subject was: "%s"
390 '''%(nodeid, subject)
392 #
393 # extract the args
394 #
395 subject_args = m.group('args')
397 #
398 # handle the subject argument list
399 #
400 # figure what the properties of this Class are
401 properties = cl.getprops()
402 props = {}
403 args = m.group('args')
404 if args:
405 errors = []
406 for prop in string.split(args, ';'):
407 # extract the property name and value
408 try:
409 propname, value = prop.split('=')
410 except ValueError, message:
411 errors.append('not of form [arg=value,'
412 'value,...;arg=value,value...]')
413 break
415 # ensure it's a valid property name
416 propname = propname.strip()
417 try:
418 proptype = properties[propname]
419 except KeyError:
420 errors.append('refers to an invalid property: '
421 '"%s"'%propname)
422 continue
424 # convert the string value to a real property value
425 if isinstance(proptype, hyperdb.String):
426 props[propname] = value.strip()
427 if isinstance(proptype, hyperdb.Password):
428 props[propname] = password.Password(value.strip())
429 elif isinstance(proptype, hyperdb.Date):
430 try:
431 props[propname] = date.Date(value.strip())
432 except ValueError, message:
433 errors.append('contains an invalid date for '
434 '%s.'%propname)
435 elif isinstance(proptype, hyperdb.Interval):
436 try:
437 props[propname] = date.Interval(value)
438 except ValueError, message:
439 errors.append('contains an invalid date interval'
440 'for %s.'%propname)
441 elif isinstance(proptype, hyperdb.Link):
442 linkcl = self.db.classes[proptype.classname]
443 propkey = linkcl.labelprop(default_to_id=1)
444 try:
445 props[propname] = linkcl.lookup(value)
446 except KeyError, message:
447 errors.append('"%s" is not a value for %s.'%(value,
448 propname))
449 elif isinstance(proptype, hyperdb.Multilink):
450 # get the linked class
451 linkcl = self.db.classes[proptype.classname]
452 propkey = linkcl.labelprop(default_to_id=1)
453 if nodeid:
454 curvalue = cl.get(nodeid, propname)
455 else:
456 curvalue = []
458 # handle each add/remove in turn
459 # keep an extra list for all items that are
460 # definitely in the new list (in case of e.g.
461 # <propname>=A,+B, which should replace the old
462 # list with A,B)
463 set = 0
464 newvalue = []
465 for item in value.split(','):
466 item = item.strip()
468 # handle +/-
469 remove = 0
470 if item.startswith('-'):
471 remove = 1
472 item = item[1:]
473 elif item.startswith('+'):
474 item = item[1:]
475 else:
476 set = 1
478 # look up the value
479 try:
480 item = linkcl.lookup(item)
481 except KeyError, message:
482 errors.append('"%s" is not a value for %s.'%(item,
483 propname))
484 continue
486 # perform the add/remove
487 if remove:
488 try:
489 curvalue.remove(item)
490 except ValueError:
491 errors.append('"%s" is not currently in '
492 'for %s.'%(item, propname))
493 continue
494 else:
495 newvalue.append(item)
496 if item not in curvalue:
497 curvalue.append(item)
499 # that's it, set the new Multilink property value,
500 # or overwrite it completely
501 if set:
502 props[propname] = newvalue
503 else:
504 props[propname] = curvalue
505 elif isinstance(proptype, hyperdb.Boolean):
506 value = value.strip()
507 props[propname] = value.lower() in ('yes', 'true', 'on', '1')
508 elif isinstance(proptype, hyperdb.Number):
509 value = value.strip()
510 props[propname] = int(value)
512 # handle any errors parsing the argument list
513 if errors:
514 errors = '\n- '.join(errors)
515 raise MailUsageError, '''
516 There were problems handling your subject line argument list:
517 - %s
519 Subject was: "%s"
520 '''%(errors, subject)
522 #
523 # handle the users
524 #
526 # Don't create users if anonymous isn't allowed to register
527 create = 1
528 anonid = self.db.user.lookup('anonymous')
529 if not self.db.security.hasPermission('Email Registration', anonid):
530 create = 0
532 # ok, now figure out who the author is - create a new user if the
533 # "create" flag is true
534 author = uidFromAddress(self.db, message.getaddrlist('from')[0],
535 create=create)
537 # no author? means we're not author
538 if not author:
539 raise Unauthorized, '''
540 You are not a registered user.
542 Unknown address: %s
543 '''%message.getaddrlist('from')[0][1]
545 # make sure the author has permission to use the email interface
546 if not self.db.security.hasPermission('Email Access', author):
547 raise Unauthorized, 'You are not permitted to access this tracker.'
549 # the author may have been created - make sure the change is
550 # committed before we reopen the database
551 self.db.commit()
553 # reopen the database as the author
554 username = self.db.user.get(author, 'username')
555 self.db = self.instance.open(username)
557 # re-get the class with the new database connection
558 cl = self.db.getclass(classname)
560 # now update the recipients list
561 recipients = []
562 tracker_email = self.instance.ISSUE_TRACKER_EMAIL.lower()
563 for recipient in message.getaddrlist('to') + message.getaddrlist('cc'):
564 r = recipient[1].strip().lower()
565 if r == tracker_email or not r:
566 continue
568 # look up the recipient - create if necessary (and we're
569 # allowed to)
570 recipient = uidFromAddress(self.db, recipient, create)
572 # if all's well, add the recipient to the list
573 if recipient:
574 recipients.append(recipient)
576 #
577 # handle message-id and in-reply-to
578 #
579 messageid = message.getheader('message-id')
580 inreplyto = message.getheader('in-reply-to') or ''
581 # generate a messageid if there isn't one
582 if not messageid:
583 messageid = "<%s.%s.%s%s@%s>"%(time.time(), random.random(),
584 classname, nodeid, self.instance.MAIL_DOMAIN)
586 #
587 # now handle the body - find the message
588 #
589 content_type = message.gettype()
590 attachments = []
591 # General multipart handling:
592 # Take the first text/plain part, anything else is considered an
593 # attachment.
594 # multipart/mixed: multiple "unrelated" parts.
595 # multipart/signed (rfc 1847):
596 # The control information is carried in the second of the two
597 # required body parts.
598 # ACTION: Default, so if content is text/plain we get it.
599 # multipart/encrypted (rfc 1847):
600 # The control information is carried in the first of the two
601 # required body parts.
602 # ACTION: Not handleable as the content is encrypted.
603 # multipart/related (rfc 1872, 2112, 2387):
604 # The Multipart/Related content-type addresses the MIME
605 # representation of compound objects.
606 # ACTION: Default. If we are lucky there is a text/plain.
607 # TODO: One should use the start part and look for an Alternative
608 # that is text/plain.
609 # multipart/Alternative (rfc 1872, 1892):
610 # only in "related" ?
611 # multipart/report (rfc 1892):
612 # e.g. mail system delivery status reports.
613 # ACTION: Default. Could be ignored or used for Delivery Notification
614 # flagging.
615 # multipart/form-data:
616 # For web forms only.
617 if content_type == 'multipart/mixed':
618 # skip over the intro to the first boundary
619 part = message.getPart()
620 content = None
621 while 1:
622 # get the next part
623 part = message.getPart()
624 if part is None:
625 break
626 # parse it
627 subtype = part.gettype()
628 if subtype == 'text/plain' and not content:
629 # The first text/plain part is the message content.
630 content = self.get_part_data_decoded(part)
631 elif subtype == 'message/rfc822':
632 # handle message/rfc822 specially - the name should be
633 # the subject of the actual e-mail embedded here
634 i = part.fp.tell()
635 mailmess = Message(part.fp)
636 name = mailmess.getheader('subject')
637 part.fp.seek(i)
638 attachments.append((name, 'message/rfc822', part.fp.read()))
639 else:
640 # try name on Content-Type
641 name = part.getparam('name')
642 # this is just an attachment
643 data = self.get_part_data_decoded(part)
644 attachments.append((name, part.gettype(), data))
645 if content is None:
646 raise MailUsageError, '''
647 Roundup requires the submission to be plain text. The message parser could
648 not find a text/plain part to use.
649 '''
651 elif content_type[:10] == 'multipart/':
652 # skip over the intro to the first boundary
653 message.getPart()
654 content = None
655 while 1:
656 # get the next part
657 part = message.getPart()
658 if part is None:
659 break
660 # parse it
661 if part.gettype() == 'text/plain' and not content:
662 content = self.get_part_data_decoded(part)
663 if content is None:
664 raise MailUsageError, '''
665 Roundup requires the submission to be plain text. The message parser could
666 not find a text/plain part to use.
667 '''
669 elif content_type != 'text/plain':
670 raise MailUsageError, '''
671 Roundup requires the submission to be plain text. The message parser could
672 not find a text/plain part to use.
673 '''
675 else:
676 content = self.get_part_data_decoded(message)
678 # figure how much we should muck around with the email body
679 keep_citations = getattr(self.instance, 'EMAIL_KEEP_QUOTED_TEXT',
680 'no') == 'yes'
681 keep_body = getattr(self.instance, 'EMAIL_LEAVE_BODY_UNCHANGED',
682 'no') == 'yes'
684 # parse the body of the message, stripping out bits as appropriate
685 summary, content = parseContent(content, keep_citations,
686 keep_body)
688 #
689 # handle the attachments
690 #
691 files = []
692 for (name, mime_type, data) in attachments:
693 if not name:
694 name = "unnamed"
695 files.append(self.db.file.create(type=mime_type, name=name,
696 content=data))
698 #
699 # create the message if there's a message body (content)
700 #
701 if content:
702 message_id = self.db.msg.create(author=author,
703 recipients=recipients, date=date.Date('.'), summary=summary,
704 content=content, files=files, messageid=messageid,
705 inreplyto=inreplyto)
707 # attach the message to the node
708 if nodeid:
709 # add the message to the node's list
710 messages = cl.get(nodeid, 'messages')
711 messages.append(message_id)
712 props['messages'] = messages
713 else:
714 # pre-load the messages list
715 props['messages'] = [message_id]
717 # set the title to the subject
718 if properties.has_key('title') and not props.has_key('title'):
719 props['title'] = title
721 #
722 # perform the node change / create
723 #
724 try:
725 if nodeid:
726 cl.set(nodeid, **props)
727 else:
728 nodeid = cl.create(**props)
729 except (TypeError, IndexError, ValueError), message:
730 raise MailUsageError, '''
731 There was a problem with the message you sent:
732 %s
733 '''%message
735 # commit the changes to the DB
736 self.db.commit()
738 return nodeid
740 def extractUserFromList(userClass, users):
741 '''Given a list of users, try to extract the first non-anonymous user
742 and return that user, otherwise return None
743 '''
744 if len(users) > 1:
745 for user in users:
746 # make sure we don't match the anonymous or admin user
747 if userClass.get(user, 'username') in ('admin', 'anonymous'):
748 continue
749 # first valid match will do
750 return user
751 # well, I guess we have no choice
752 return user[0]
753 elif users:
754 return users[0]
755 return None
757 def uidFromAddress(db, address, create=1):
758 ''' address is from the rfc822 module, and therefore is (name, addr)
760 user is created if they don't exist in the db already
761 '''
762 (realname, address) = address
764 # try a straight match of the address
765 user = extractUserFromList(db.user, db.user.stringFind(address=address))
766 if user is not None: return user
768 # try the user alternate addresses if possible
769 props = db.user.getprops()
770 if props.has_key('alternate_addresses'):
771 users = db.user.filter(None, {'alternate_addresses': address},
772 [], [])
773 user = extractUserFromList(db.user, users)
774 if user is not None: return user
776 # try to match the username to the address (for local
777 # submissions where the address is empty)
778 user = extractUserFromList(db.user, db.user.stringFind(username=address))
780 # couldn't match address or username, so create a new user
781 if create:
782 return db.user.create(username=address, address=address,
783 realname=realname, roles=db.config.NEW_EMAIL_USER_ROLES)
784 else:
785 return 0
787 def parseContent(content, keep_citations, keep_body,
788 blank_line=re.compile(r'[\r\n]+\s*[\r\n]+'),
789 eol=re.compile(r'[\r\n]+'),
790 signature=re.compile(r'^[>|\s]*[-_]+\s*$'),
791 original_message=re.compile(r'^[>|\s]*-----Original Message-----$')):
792 ''' The message body is divided into sections by blank lines.
793 Sections where the second and all subsequent lines begin with a ">" or "|"
794 character are considered "quoting sections". The first line of the first
795 non-quoting section becomes the summary of the message.
796 '''
797 # strip off leading carriage-returns / newlines
798 i = 0
799 for i in range(len(content)):
800 if content[i] not in '\r\n':
801 break
802 if i > 0:
803 sections = blank_line.split(content[i:])
804 else:
805 sections = blank_line.split(content)
807 # extract out the summary from the message
808 summary = ''
809 l = []
810 for section in sections:
811 #section = section.strip()
812 if not section:
813 continue
814 lines = eol.split(section)
815 if (lines[0] and lines[0][0] in '>|') or (len(lines) > 1 and
816 lines[1] and lines[1][0] in '>|'):
817 # see if there's a response somewhere inside this section (ie.
818 # no blank line between quoted message and response)
819 for line in lines[1:]:
820 if line[0] not in '>|':
821 break
822 else:
823 # we keep quoted bits if specified in the config
824 if keep_citations:
825 l.append(section)
826 continue
827 # keep this section - it has reponse stuff in it
828 if not summary:
829 # and while we're at it, use the first non-quoted bit as
830 # our summary
831 summary = line
832 lines = lines[lines.index(line):]
833 section = '\n'.join(lines)
835 if not summary:
836 # if we don't have our summary yet use the first line of this
837 # section
838 summary = lines[0]
839 elif signature.match(lines[0]) and 2 <= len(lines) <= 10:
840 # lose any signature
841 break
842 elif original_message.match(lines[0]):
843 # ditch the stupid Outlook quoting of the entire original message
844 break
846 # and add the section to the output
847 l.append(section)
848 # we only set content for those who want to delete cruft from the
849 # message body, otherwise the body is left untouched.
850 if not keep_body:
851 content = '\n\n'.join(l)
852 return summary, content
854 #
855 # $Log: not supported by cvs2svn $
856 # Revision 1.79 2002/07/26 08:26:59 richard
857 # Very close now. The cgi and mailgw now use the new security API. The two
858 # templates have been migrated to that setup. Lots of unit tests. Still some
859 # issue in the web form for editing Roles assigned to users.
860 #
861 # Revision 1.78 2002/07/25 07:14:06 richard
862 # Bugger it. Here's the current shape of the new security implementation.
863 # Still to do:
864 # . call the security funcs from cgi and mailgw
865 # . change shipped templates to include correct initialisation and remove
866 # the old config vars
867 # ... that seems like a lot. The bulk of the work has been done though. Honest :)
868 #
869 # Revision 1.77 2002/07/18 11:17:31 gmcm
870 # Add Number and Boolean types to hyperdb.
871 # Add conversion cases to web, mail & admin interfaces.
872 # Add storage/serialization cases to back_anydbm & back_metakit.
873 #
874 # Revision 1.76 2002/07/10 06:39:37 richard
875 # . made mailgw handle set and modify operations on multilinks (bug #579094)
876 #
877 # Revision 1.75 2002/07/09 01:21:24 richard
878 # Added ability for unit tests to turn off exception handling in mailgw so
879 # that exceptions are reported earlier (and hence make sense).
880 #
881 # Revision 1.74 2002/05/29 01:16:17 richard
882 # Sorry about this huge checkin! It's fixing a lot of related stuff in one go
883 # though.
884 #
885 # . #541941 ] changing multilink properties by mail
886 # . #526730 ] search for messages capability
887 # . #505180 ] split MailGW.handle_Message
888 # - also changed cgi client since it was duplicating the functionality
889 # . build htmlbase if tests are run using CVS checkout (removed note from
890 # installation.txt)
891 # . don't create an empty message on email issue creation if the email is empty
892 #
893 # Revision 1.73 2002/05/22 04:12:05 richard
894 # . applied patch #558876 ] cgi client customization
895 # ... with significant additions and modifications ;)
896 # - extended handling of ML assignedto to all places it's handled
897 # - added more NotFound info
898 #
899 # Revision 1.72 2002/05/22 01:24:51 richard
900 # Added note to MIGRATION about new config vars. Also made us more resilient
901 # for upgraders. Reinstated list header style (oops)
902 #
903 # Revision 1.71 2002/05/08 02:40:55 richard
904 # grr
905 #
906 # Revision 1.70 2002/05/06 23:40:07 richard
907 # hrm
908 #
909 # Revision 1.69 2002/05/06 23:37:21 richard
910 # Tweaking the signature deletion from mail messages.
911 # Added nuking of the "-----Original Message-----" crap from Outlook.
912 #
913 # Revision 1.68 2002/05/02 07:56:34 richard
914 # . added option to automatically add the authors and recipients of messages
915 # to the nosy lists with the options ADD_AUTHOR_TO_NOSY (default 'new') and
916 # ADD_RECIPIENTS_TO_NOSY (default 'new'). These settings emulate the current
917 # behaviour. Setting them to 'yes' will add the author/recipients to the nosy
918 # on messages that create issues and followup messages.
919 # . added missing documentation for a few of the config option values
920 #
921 # Revision 1.67 2002/04/23 15:46:49 rochecompaan
922 # . stripping of the email message body can now be controlled through
923 # the config variables EMAIL_KEEP_QUOTED_TEST and
924 # EMAIL_LEAVE_BODY_UNCHANGED.
925 #
926 # Revision 1.66 2002/03/14 23:59:24 richard
927 # . #517734 ] web header customisation is obscure
928 #
929 # Revision 1.65 2002/02/15 00:13:38 richard
930 # . #503204 ] mailgw needs a default class
931 # - partially done - the setting of additional properties can wait for a
932 # better configuration system.
933 #
934 # Revision 1.64 2002/02/14 23:46:02 richard
935 # . #516883 ] mail interface + ANONYMOUS_REGISTER
936 #
937 # Revision 1.63 2002/02/12 08:08:55 grubert
938 # . Clean up mail handling, multipart handling.
939 #
940 # Revision 1.62 2002/02/05 14:15:29 grubert
941 # . respect encodings in non multipart messages.
942 #
943 # Revision 1.61 2002/02/04 09:40:21 grubert
944 # . add test for multipart messages with first part being encoded.
945 #
946 # Revision 1.60 2002/02/01 07:43:12 grubert
947 # . mailgw checks encoding on first part too.
948 #
949 # Revision 1.59 2002/01/23 21:43:23 richard
950 # tabnuke
951 #
952 # Revision 1.58 2002/01/23 21:41:56 richard
953 # . mailgw failures (unexpected ones) are forwarded to the roundup admin
954 #
955 # Revision 1.57 2002/01/22 22:27:43 richard
956 # . handle stripping of "AW:" from subject line
957 #
958 # Revision 1.56 2002/01/22 11:54:45 rochecompaan
959 # Fixed status change in mail gateway.
960 #
961 # Revision 1.55 2002/01/21 10:05:47 rochecompaan
962 # Feature:
963 # . the mail gateway now responds with an error message when invalid
964 # values for arguments are specified for link or multilink properties
965 # . modified unit test to check nosy and assignedto when specified as
966 # arguments
967 #
968 # Fixed:
969 # . fixed setting nosy as argument in subject line
970 #
971 # Revision 1.54 2002/01/16 09:14:45 grubert
972 # . if the attachment has no name, name it unnamed, happens with tnefs.
973 #
974 # Revision 1.53 2002/01/16 07:20:54 richard
975 # simple help command for mailgw
976 #
977 # Revision 1.52 2002/01/15 00:12:40 richard
978 # #503340 ] creating issue with [asignedto=p.ohly]
979 #
980 # Revision 1.51 2002/01/14 02:20:15 richard
981 # . changed all config accesses so they access either the instance or the
982 # config attriubute on the db. This means that all config is obtained from
983 # instance_config instead of the mish-mash of classes. This will make
984 # switching to a ConfigParser setup easier too, I hope.
985 #
986 # At a minimum, this makes migration a _little_ easier (a lot easier in the
987 # 0.5.0 switch, I hope!)
988 #
989 # Revision 1.50 2002/01/11 22:59:01 richard
990 # . #502342 ] pipe interface
991 #
992 # Revision 1.49 2002/01/10 06:19:18 richard
993 # followup lines directly after a quoted section were being eaten.
994 #
995 # Revision 1.48 2002/01/08 04:12:05 richard
996 # Changed message-id format to "<%s.%s.%s%s@%s>" so it complies with RFC822
997 #
998 # Revision 1.47 2002/01/02 02:32:38 richard
999 # ANONYMOUS_ACCESS -> ANONYMOUS_REGISTER
1000 #
1001 # Revision 1.46 2002/01/02 02:31:38 richard
1002 # Sorry for the huge checkin message - I was only intending to implement #496356
1003 # but I found a number of places where things had been broken by transactions:
1004 # . modified ROUNDUPDBSENDMAILDEBUG to be SENDMAILDEBUG and hold a filename
1005 # for _all_ roundup-generated smtp messages to be sent to.
1006 # . the transaction cache had broken the roundupdb.Class set() reactors
1007 # . newly-created author users in the mailgw weren't being committed to the db
1008 #
1009 # Stuff that made it into CHANGES.txt (ie. the stuff I was actually working
1010 # on when I found that stuff :):
1011 # . #496356 ] Use threading in messages
1012 # . detectors were being registered multiple times
1013 # . added tests for mailgw
1014 # . much better attaching of erroneous messages in the mail gateway
1015 #
1016 # Revision 1.45 2001/12/20 15:43:01 rochecompaan
1017 # Features added:
1018 # . Multilink properties are now displayed as comma separated values in
1019 # a textbox
1020 # . The add user link is now only visible to the admin user
1021 # . Modified the mail gateway to reject submissions from unknown
1022 # addresses if ANONYMOUS_ACCESS is denied
1023 #
1024 # Revision 1.44 2001/12/18 15:30:34 rochecompaan
1025 # Fixed bugs:
1026 # . Fixed file creation and retrieval in same transaction in anydbm
1027 # backend
1028 # . Cgi interface now renders new issue after issue creation
1029 # . Could not set issue status to resolved through cgi interface
1030 # . Mail gateway was changing status back to 'chatting' if status was
1031 # omitted as an argument
1032 #
1033 # Revision 1.43 2001/12/15 19:39:01 rochecompaan
1034 # Oops.
1035 #
1036 # Revision 1.42 2001/12/15 19:24:39 rochecompaan
1037 # . Modified cgi interface to change properties only once all changes are
1038 # collected, files created and messages generated.
1039 # . Moved generation of change note to nosyreactors.
1040 # . We now check for changes to "assignedto" to ensure it's added to the
1041 # nosy list.
1042 #
1043 # Revision 1.41 2001/12/10 00:57:38 richard
1044 # From CHANGES:
1045 # . Added the "display" command to the admin tool - displays a node's values
1046 # . #489760 ] [issue] only subject
1047 # . fixed the doc/index.html to include the quoting in the mail alias.
1048 #
1049 # Also:
1050 # . fixed roundup-admin so it works with transactions
1051 # . disabled the back_anydbm module if anydbm tries to use dumbdbm
1052 #
1053 # Revision 1.40 2001/12/05 14:26:44 rochecompaan
1054 # Removed generation of change note from "sendmessage" in roundupdb.py.
1055 # The change note is now generated when the message is created.
1056 #
1057 # Revision 1.39 2001/12/02 05:06:16 richard
1058 # . We now use weakrefs in the Classes to keep the database reference, so
1059 # the close() method on the database is no longer needed.
1060 # I bumped the minimum python requirement up to 2.1 accordingly.
1061 # . #487480 ] roundup-server
1062 # . #487476 ] INSTALL.txt
1063 #
1064 # I also cleaned up the change message / post-edit stuff in the cgi client.
1065 # There's now a clearly marked "TODO: append the change note" where I believe
1066 # the change note should be added there. The "changes" list will obviously
1067 # have to be modified to be a dict of the changes, or somesuch.
1068 #
1069 # More testing needed.
1070 #
1071 # Revision 1.38 2001/12/01 07:17:50 richard
1072 # . We now have basic transaction support! Information is only written to
1073 # the database when the commit() method is called. Only the anydbm
1074 # backend is modified in this way - neither of the bsddb backends have been.
1075 # The mail, admin and cgi interfaces all use commit (except the admin tool
1076 # doesn't have a commit command, so interactive users can't commit...)
1077 # . Fixed login/registration forwarding the user to the right page (or not,
1078 # on a failure)
1079 #
1080 # Revision 1.37 2001/11/28 21:55:35 richard
1081 # . login_action and newuser_action return values were being ignored
1082 # . Woohoo! Found that bloody re-login bug that was killing the mail
1083 # gateway.
1084 # (also a minor cleanup in hyperdb)
1085 #
1086 # Revision 1.36 2001/11/26 22:55:56 richard
1087 # Feature:
1088 # . Added INSTANCE_NAME to configuration - used in web and email to identify
1089 # the instance.
1090 # . Added EMAIL_SIGNATURE_POSITION to indicate where to place the roundup
1091 # signature info in e-mails.
1092 # . Some more flexibility in the mail gateway and more error handling.
1093 # . Login now takes you to the page you back to the were denied access to.
1094 #
1095 # Fixed:
1096 # . Lots of bugs, thanks Roché and others on the devel mailing list!
1097 #
1098 # Revision 1.35 2001/11/22 15:46:42 jhermann
1099 # Added module docstrings to all modules.
1100 #
1101 # Revision 1.34 2001/11/15 10:24:27 richard
1102 # handle the case where there is no file attached
1103 #
1104 # Revision 1.33 2001/11/13 21:44:44 richard
1105 # . re-open the database as the author in mail handling
1106 #
1107 # Revision 1.32 2001/11/12 22:04:29 richard
1108 # oops, left debug in there
1109 #
1110 # Revision 1.31 2001/11/12 22:01:06 richard
1111 # Fixed issues with nosy reaction and author copies.
1112 #
1113 # Revision 1.30 2001/11/09 22:33:28 richard
1114 # More error handling fixes.
1115 #
1116 # Revision 1.29 2001/11/07 05:29:26 richard
1117 # Modified roundup-mailgw so it can read e-mails from a local mail spool
1118 # file. Truncates the spool file after parsing.
1119 # Fixed a couple of small bugs introduced in roundup.mailgw when I started
1120 # the popgw.
1121 #
1122 # Revision 1.28 2001/11/01 22:04:37 richard
1123 # Started work on supporting a pop3-fetching server
1124 # Fixed bugs:
1125 # . bug #477104 ] HTML tag error in roundup-server
1126 # . bug #477107 ] HTTP header problem
1127 #
1128 # Revision 1.27 2001/10/30 11:26:10 richard
1129 # Case-insensitive match for ISSUE_TRACKER_EMAIL in address in e-mail.
1130 #
1131 # Revision 1.26 2001/10/30 00:54:45 richard
1132 # Features:
1133 # . #467129 ] Lossage when username=e-mail-address
1134 # . #473123 ] Change message generation for author
1135 # . MailGW now moves 'resolved' to 'chatting' on receiving e-mail for an issue.
1136 #
1137 # Revision 1.25 2001/10/28 23:22:28 richard
1138 # fixed bug #474749 ] Indentations lost
1139 #
1140 # Revision 1.24 2001/10/23 22:57:52 richard
1141 # Fix unread->chatting auto transition, thanks Roch'e
1142 #
1143 # Revision 1.23 2001/10/21 04:00:20 richard
1144 # MailGW now moves 'unread' to 'chatting' on receiving e-mail for an issue.
1145 #
1146 # Revision 1.22 2001/10/21 03:35:13 richard
1147 # bug #473125: Paragraph in e-mails
1148 #
1149 # Revision 1.21 2001/10/21 00:53:42 richard
1150 # bug #473130: Nosy list not set correctly
1151 #
1152 # Revision 1.20 2001/10/17 23:13:19 richard
1153 # Did a fair bit of work on the admin tool. Now has an extra command "table"
1154 # which displays node information in a tabular format. Also fixed import and
1155 # export so they work. Removed freshen.
1156 # Fixed quopri usage in mailgw from bug reports.
1157 #
1158 # Revision 1.19 2001/10/11 23:43:04 richard
1159 # Implemented the comma-separated printing option in the admin tool.
1160 # Fixed a typo (more of a vim-o actually :) in mailgw.
1161 #
1162 # Revision 1.18 2001/10/11 06:38:57 richard
1163 # Initial cut at trying to handle people responding to CC'ed messages that
1164 # create an issue.
1165 #
1166 # Revision 1.17 2001/10/09 07:25:59 richard
1167 # Added the Password property type. See "pydoc roundup.password" for
1168 # implementation details. Have updated some of the documentation too.
1169 #
1170 # Revision 1.16 2001/10/05 02:23:24 richard
1171 # . roundup-admin create now prompts for property info if none is supplied
1172 # on the command-line.
1173 # . hyperdb Class getprops() method may now return only the mutable
1174 # properties.
1175 # . Login now uses cookies, which makes it a whole lot more flexible. We can
1176 # now support anonymous user access (read-only, unless there's an
1177 # "anonymous" user, in which case write access is permitted). Login
1178 # handling has been moved into cgi_client.Client.main()
1179 # . The "extended" schema is now the default in roundup init.
1180 # . The schemas have had their page headings modified to cope with the new
1181 # login handling. Existing installations should copy the interfaces.py
1182 # file from the roundup lib directory to their instance home.
1183 # . Incorrectly had a Bizar Software copyright on the cgitb.py module from
1184 # Ping - has been removed.
1185 # . Fixed a whole bunch of places in the CGI interface where we should have
1186 # been returning Not Found instead of throwing an exception.
1187 # . Fixed a deviation from the spec: trying to modify the 'id' property of
1188 # an item now throws an exception.
1189 #
1190 # Revision 1.15 2001/08/30 06:01:17 richard
1191 # Fixed missing import in mailgw :(
1192 #
1193 # Revision 1.14 2001/08/13 23:02:54 richard
1194 # Make the mail parser a little more robust.
1195 #
1196 # Revision 1.13 2001/08/12 06:32:36 richard
1197 # using isinstance(blah, Foo) now instead of isFooType
1198 #
1199 # Revision 1.12 2001/08/08 01:27:00 richard
1200 # Added better error handling to mailgw.
1201 #
1202 # Revision 1.11 2001/08/08 00:08:03 richard
1203 # oops ;)
1204 #
1205 # Revision 1.10 2001/08/07 00:24:42 richard
1206 # stupid typo
1207 #
1208 # Revision 1.9 2001/08/07 00:15:51 richard
1209 # Added the copyright/license notice to (nearly) all files at request of
1210 # Bizar Software.
1211 #
1212 # Revision 1.8 2001/08/05 07:06:07 richard
1213 # removed some print statements
1214 #
1215 # Revision 1.7 2001/08/03 07:18:22 richard
1216 # Implemented correct mail splitting (was taking a shortcut). Added unit
1217 # tests. Also snips signatures now too.
1218 #
1219 # Revision 1.6 2001/08/01 04:24:21 richard
1220 # mailgw was assuming certain properties existed on the issues being created.
1221 #
1222 # Revision 1.5 2001/07/29 07:01:39 richard
1223 # Added vim command to all source so that we don't get no steenkin' tabs :)
1224 #
1225 # Revision 1.4 2001/07/28 06:43:02 richard
1226 # Multipart message class has the getPart method now. Added some tests for it.
1227 #
1228 # Revision 1.3 2001/07/28 00:34:34 richard
1229 # Fixed some non-string node ids.
1230 #
1231 # Revision 1.2 2001/07/22 12:09:32 richard
1232 # Final commit of Grande Splite
1233 #
1234 #
1235 # vim: set filetype=python ts=4 sw=4 et si