1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
19 __doc__ = '''
20 An e-mail gateway for Roundup.
22 Incoming messages are examined for multiple parts:
23 . In a multipart/mixed message or part, each subpart is extracted and
24 examined. The text/plain subparts are assembled to form the textual
25 body of the message, to be stored in the file associated with a "msg"
26 class node. Any parts of other types are each stored in separate files
27 and given "file" class nodes that are linked to the "msg" node.
28 . In a multipart/alternative message or part, we look for a text/plain
29 subpart and ignore the other parts.
31 Summary
32 -------
33 The "summary" property on message nodes is taken from the first non-quoting
34 section in the message body. The message body is divided into sections by
35 blank lines. Sections where the second and all subsequent lines begin with
36 a ">" or "|" character are considered "quoting sections". The first line of
37 the first non-quoting section becomes the summary of the message.
39 Addresses
40 ---------
41 All of the addresses in the To: and Cc: headers of the incoming message are
42 looked up among the user nodes, and the corresponding users are placed in
43 the "recipients" property on the new "msg" node. The address in the From:
44 header similarly determines the "author" property of the new "msg"
45 node. The default handling for addresses that don't have corresponding
46 users is to create new users with no passwords and a username equal to the
47 address. (The web interface does not permit logins for users with no
48 passwords.) If we prefer to reject mail from outside sources, we can simply
49 register an auditor on the "user" class that prevents the creation of user
50 nodes with no passwords.
52 Actions
53 -------
54 The subject line of the incoming message is examined to determine whether
55 the message is an attempt to create a new item or to discuss an existing
56 item. A designator enclosed in square brackets is sought as the first thing
57 on the subject line (after skipping any "Fwd:" or "Re:" prefixes).
59 If an item designator (class name and id number) is found there, the newly
60 created "msg" node is added to the "messages" property for that item, and
61 any new "file" nodes are added to the "files" property for the item.
63 If just an item class name is found there, we attempt to create a new item
64 of that class with its "messages" property initialized to contain the new
65 "msg" node and its "files" property initialized to contain any new "file"
66 nodes.
68 Triggers
69 --------
70 Both cases may trigger detectors (in the first case we are calling the
71 set() method to add the message to the item's spool; in the second case we
72 are calling the create() method to create a new node). If an auditor raises
73 an exception, the original message is bounced back to the sender with the
74 explanatory message given in the exception.
76 $Id: mailgw.py,v 1.81 2002-08-19 00:21:56 richard Exp $
77 '''
80 import string, re, os, mimetools, cStringIO, smtplib, socket, binascii, quopri
81 import time, random
82 import traceback, MimeWriter
83 import hyperdb, date, password
85 SENDMAILDEBUG = os.environ.get('SENDMAILDEBUG', '')
87 class MailGWError(ValueError):
88 pass
90 class MailUsageError(ValueError):
91 pass
93 class MailUsageHelp(Exception):
94 pass
96 class Unauthorized(Exception):
97 """ Access denied """
99 def initialiseSecurity(security):
100 ''' Create some Permissions and Roles on the security object
102 This function is directly invoked by security.Security.__init__()
103 as a part of the Security object instantiation.
104 '''
105 security.addPermission(name="Email Registration",
106 description="Anonymous may register through e-mail")
107 p = security.addPermission(name="Email Access",
108 description="User may use the email interface")
109 security.addPermissionToRole('Admin', p)
111 class Message(mimetools.Message):
112 ''' subclass mimetools.Message so we can retrieve the parts of the
113 message...
114 '''
115 def getPart(self):
116 ''' Get a single part of a multipart message and return it as a new
117 Message instance.
118 '''
119 boundary = self.getparam('boundary')
120 mid, end = '--'+boundary, '--'+boundary+'--'
121 s = cStringIO.StringIO()
122 while 1:
123 line = self.fp.readline()
124 if not line:
125 break
126 if line.strip() in (mid, end):
127 break
128 s.write(line)
129 if not s.getvalue().strip():
130 return None
131 s.seek(0)
132 return Message(s)
134 subject_re = re.compile(r'(?P<refwd>\s*\W?\s*(fwd|re|aw)\s*\W?\s*)*'
135 r'\s*(\[(?P<classname>[^\d\s]+)(?P<nodeid>\d+)?\])?'
136 r'\s*(?P<title>[^[]+)?(\[(?P<args>.+?)\])?', re.I)
138 class MailGW:
139 def __init__(self, instance, db):
140 self.instance = instance
141 self.db = db
143 # should we trap exceptions (normal usage) or pass them through
144 # (for testing)
145 self.trapExceptions = 1
147 def main(self, fp):
148 ''' fp - the file from which to read the Message.
149 '''
150 return self.handle_Message(Message(fp))
152 def handle_Message(self, message):
153 '''Handle an RFC822 Message
155 Handle the Message object by calling handle_message() and then cope
156 with any errors raised by handle_message.
157 This method's job is to make that call and handle any
158 errors in a sane manner. It should be replaced if you wish to
159 handle errors in a different manner.
160 '''
161 # in some rare cases, a particularly stuffed-up e-mail will make
162 # its way into here... try to handle it gracefully
163 sendto = message.getaddrlist('from')
164 if sendto:
165 if not self.trapExceptions:
166 return self.handle_message(message)
167 try:
168 return self.handle_message(message)
169 except MailUsageHelp:
170 # bounce the message back to the sender with the usage message
171 fulldoc = '\n'.join(string.split(__doc__, '\n')[2:])
172 sendto = [sendto[0][1]]
173 m = ['']
174 m.append('\n\nMail Gateway Help\n=================')
175 m.append(fulldoc)
176 m = self.bounce_message(message, sendto, m,
177 subject="Mail Gateway Help")
178 except MailUsageError, value:
179 # bounce the message back to the sender with the usage message
180 fulldoc = '\n'.join(string.split(__doc__, '\n')[2:])
181 sendto = [sendto[0][1]]
182 m = ['']
183 m.append(str(value))
184 m.append('\n\nMail Gateway Help\n=================')
185 m.append(fulldoc)
186 m = self.bounce_message(message, sendto, m)
187 except Unauthorized, value:
188 # just inform the user that he is not authorized
189 sendto = [sendto[0][1]]
190 m = ['']
191 m.append(str(value))
192 m = self.bounce_message(message, sendto, m)
193 except:
194 # bounce the message back to the sender with the error message
195 sendto = [sendto[0][1], self.instance.ADMIN_EMAIL]
196 m = ['']
197 m.append('An unexpected error occurred during the processing')
198 m.append('of your message. The tracker administrator is being')
199 m.append('notified.\n')
200 m.append('---- traceback of failure ----')
201 s = cStringIO.StringIO()
202 import traceback
203 traceback.print_exc(None, s)
204 m.append(s.getvalue())
205 m = self.bounce_message(message, sendto, m)
206 else:
207 # very bad-looking message - we don't even know who sent it
208 sendto = [self.instance.ADMIN_EMAIL]
209 m = ['Subject: badly formed message from mail gateway']
210 m.append('')
211 m.append('The mail gateway retrieved a message which has no From:')
212 m.append('line, indicating that it is corrupt. Please check your')
213 m.append('mail gateway source. Failed message is attached.')
214 m.append('')
215 m = self.bounce_message(message, sendto, m,
216 subject='Badly formed message from mail gateway')
218 # now send the message
219 if SENDMAILDEBUG:
220 open(SENDMAILDEBUG, 'w').write('From: %s\nTo: %s\n%s\n'%(
221 self.instance.ADMIN_EMAIL, ', '.join(sendto), m.getvalue()))
222 else:
223 try:
224 smtp = smtplib.SMTP(self.instance.MAILHOST)
225 smtp.sendmail(self.instance.ADMIN_EMAIL, sendto, m.getvalue())
226 except socket.error, value:
227 raise MailGWError, "Couldn't send error email: "\
228 "mailhost %s"%value
229 except smtplib.SMTPException, value:
230 raise MailGWError, "Couldn't send error email: %s"%value
232 def bounce_message(self, message, sendto, error,
233 subject='Failed issue tracker submission'):
234 ''' create a message that explains the reason for the failed
235 issue submission to the author and attach the original
236 message.
237 '''
238 msg = cStringIO.StringIO()
239 writer = MimeWriter.MimeWriter(msg)
240 writer.addheader('Subject', subject)
241 writer.addheader('From', '%s <%s>'% (self.instance.INSTANCE_NAME,
242 self.instance.ISSUE_TRACKER_EMAIL))
243 writer.addheader('To', ','.join(sendto))
244 writer.addheader('MIME-Version', '1.0')
245 part = writer.startmultipartbody('mixed')
246 part = writer.nextpart()
247 body = part.startbody('text/plain')
248 body.write('\n'.join(error))
250 # reconstruct the original message
251 m = cStringIO.StringIO()
252 w = MimeWriter.MimeWriter(m)
253 # default the content_type, just in case...
254 content_type = 'text/plain'
255 # add the headers except the content-type
256 for header in message.headers:
257 header_name = header.split(':')[0]
258 if header_name.lower() == 'content-type':
259 content_type = message.getheader(header_name)
260 elif message.getheader(header_name):
261 w.addheader(header_name, message.getheader(header_name))
262 # now attach the message body
263 body = w.startbody(content_type)
264 try:
265 message.rewindbody()
266 except IOError:
267 body.write("*** couldn't include message body: read from pipe ***")
268 else:
269 body.write(message.fp.read())
271 # attach the original message to the returned message
272 part = writer.nextpart()
273 part.addheader('Content-Disposition','attachment')
274 part.addheader('Content-Description','Message you sent')
275 part.addheader('Content-Transfer-Encoding', '7bit')
276 body = part.startbody('message/rfc822')
277 body.write(m.getvalue())
279 writer.lastpart()
280 return msg
282 def get_part_data_decoded(self,part):
283 encoding = part.getencoding()
284 data = None
285 if encoding == 'base64':
286 # BUG: is base64 really used for text encoding or
287 # are we inserting zip files here.
288 data = binascii.a2b_base64(part.fp.read())
289 elif encoding == 'quoted-printable':
290 # the quopri module wants to work with files
291 decoded = cStringIO.StringIO()
292 quopri.decode(part.fp, decoded)
293 data = decoded.getvalue()
294 elif encoding == 'uuencoded':
295 data = binascii.a2b_uu(part.fp.read())
296 else:
297 # take it as text
298 data = part.fp.read()
299 return data
301 def handle_message(self, message):
302 ''' message - a Message instance
304 Parse the message as per the module docstring.
305 '''
306 # handle the subject line
307 subject = message.getheader('subject', '')
309 if subject.strip() == 'help':
310 raise MailUsageHelp
312 m = subject_re.match(subject)
314 # check for well-formed subject line
315 if m:
316 # get the classname
317 classname = m.group('classname')
318 if classname is None:
319 # no classname, fallback on the default
320 if hasattr(self.instance, 'MAIL_DEFAULT_CLASS') and \
321 self.instance.MAIL_DEFAULT_CLASS:
322 classname = self.instance.MAIL_DEFAULT_CLASS
323 else:
324 # fail
325 m = None
327 if not m:
328 raise MailUsageError, '''
329 The message you sent to roundup did not contain a properly formed subject
330 line. The subject must contain a class name or designator to indicate the
331 "topic" of the message. For example:
332 Subject: [issue] This is a new issue
333 - this will create a new issue in the tracker with the title "This is
334 a new issue".
335 Subject: [issue1234] This is a followup to issue 1234
336 - this will append the message's contents to the existing issue 1234
337 in the tracker.
339 Subject was: "%s"
340 '''%subject
342 # get the class
343 try:
344 cl = self.db.getclass(classname)
345 except KeyError:
346 raise MailUsageError, '''
347 The class name you identified in the subject line ("%s") does not exist in the
348 database.
350 Valid class names are: %s
351 Subject was: "%s"
352 '''%(classname, ', '.join(self.db.getclasses()), subject)
354 # get the optional nodeid
355 nodeid = m.group('nodeid')
357 # title is optional too
358 title = m.group('title')
359 if title:
360 title = title.strip()
361 else:
362 title = ''
364 # but we do need either a title or a nodeid...
365 if nodeid is None and not title:
366 raise MailUsageError, '''
367 I cannot match your message to a node in the database - you need to either
368 supply a full node identifier (with number, eg "[issue123]" or keep the
369 previous subject title intact so I can match that.
371 Subject was: "%s"
372 '''%subject
374 # If there's no nodeid, check to see if this is a followup and
375 # maybe someone's responded to the initial mail that created an
376 # entry. Try to find the matching nodes with the same title, and
377 # use the _last_ one matched (since that'll _usually_ be the most
378 # recent...)
379 if nodeid is None and m.group('refwd'):
380 l = cl.stringFind(title=title)
381 if l:
382 nodeid = l[-1]
384 # if a nodeid was specified, make sure it's valid
385 if nodeid is not None and not cl.hasnode(nodeid):
386 raise MailUsageError, '''
387 The node specified by the designator in the subject of your message ("%s")
388 does not exist.
390 Subject was: "%s"
391 '''%(nodeid, subject)
393 #
394 # extract the args
395 #
396 subject_args = m.group('args')
398 #
399 # handle the subject argument list
400 #
401 # figure what the properties of this Class are
402 properties = cl.getprops()
403 props = {}
404 args = m.group('args')
405 if args:
406 errors = []
407 for prop in string.split(args, ';'):
408 # extract the property name and value
409 try:
410 propname, value = prop.split('=')
411 except ValueError, message:
412 errors.append('not of form [arg=value,'
413 'value,...;arg=value,value...]')
414 break
416 # ensure it's a valid property name
417 propname = propname.strip()
418 try:
419 proptype = properties[propname]
420 except KeyError:
421 errors.append('refers to an invalid property: '
422 '"%s"'%propname)
423 continue
425 # convert the string value to a real property value
426 if isinstance(proptype, hyperdb.String):
427 props[propname] = value.strip()
428 if isinstance(proptype, hyperdb.Password):
429 props[propname] = password.Password(value.strip())
430 elif isinstance(proptype, hyperdb.Date):
431 try:
432 props[propname] = date.Date(value.strip())
433 except ValueError, message:
434 errors.append('contains an invalid date for '
435 '%s.'%propname)
436 elif isinstance(proptype, hyperdb.Interval):
437 try:
438 props[propname] = date.Interval(value)
439 except ValueError, message:
440 errors.append('contains an invalid date interval'
441 'for %s.'%propname)
442 elif isinstance(proptype, hyperdb.Link):
443 linkcl = self.db.classes[proptype.classname]
444 propkey = linkcl.labelprop(default_to_id=1)
445 try:
446 props[propname] = linkcl.lookup(value)
447 except KeyError, message:
448 errors.append('"%s" is not a value for %s.'%(value,
449 propname))
450 elif isinstance(proptype, hyperdb.Multilink):
451 # get the linked class
452 linkcl = self.db.classes[proptype.classname]
453 propkey = linkcl.labelprop(default_to_id=1)
454 if nodeid:
455 curvalue = cl.get(nodeid, propname)
456 else:
457 curvalue = []
459 # handle each add/remove in turn
460 # keep an extra list for all items that are
461 # definitely in the new list (in case of e.g.
462 # <propname>=A,+B, which should replace the old
463 # list with A,B)
464 set = 0
465 newvalue = []
466 for item in value.split(','):
467 item = item.strip()
469 # handle +/-
470 remove = 0
471 if item.startswith('-'):
472 remove = 1
473 item = item[1:]
474 elif item.startswith('+'):
475 item = item[1:]
476 else:
477 set = 1
479 # look up the value
480 try:
481 item = linkcl.lookup(item)
482 except KeyError, message:
483 errors.append('"%s" is not a value for %s.'%(item,
484 propname))
485 continue
487 # perform the add/remove
488 if remove:
489 try:
490 curvalue.remove(item)
491 except ValueError:
492 errors.append('"%s" is not currently in '
493 'for %s.'%(item, propname))
494 continue
495 else:
496 newvalue.append(item)
497 if item not in curvalue:
498 curvalue.append(item)
500 # that's it, set the new Multilink property value,
501 # or overwrite it completely
502 if set:
503 props[propname] = newvalue
504 else:
505 props[propname] = curvalue
506 elif isinstance(proptype, hyperdb.Boolean):
507 value = value.strip()
508 props[propname] = value.lower() in ('yes', 'true', 'on', '1')
509 elif isinstance(proptype, hyperdb.Number):
510 value = value.strip()
511 props[propname] = int(value)
513 # handle any errors parsing the argument list
514 if errors:
515 errors = '\n- '.join(errors)
516 raise MailUsageError, '''
517 There were problems handling your subject line argument list:
518 - %s
520 Subject was: "%s"
521 '''%(errors, subject)
523 #
524 # handle the users
525 #
527 # Don't create users if anonymous isn't allowed to register
528 create = 1
529 anonid = self.db.user.lookup('anonymous')
530 if not self.db.security.hasPermission('Email Registration', anonid):
531 create = 0
533 # ok, now figure out who the author is - create a new user if the
534 # "create" flag is true
535 author = uidFromAddress(self.db, message.getaddrlist('from')[0],
536 create=create)
538 # no author? means we're not author
539 if not author:
540 raise Unauthorized, '''
541 You are not a registered user.
543 Unknown address: %s
544 '''%message.getaddrlist('from')[0][1]
546 # make sure the author has permission to use the email interface
547 if not self.db.security.hasPermission('Email Access', author):
548 raise Unauthorized, 'You are not permitted to access this tracker.'
550 # the author may have been created - make sure the change is
551 # committed before we reopen the database
552 self.db.commit()
554 # reopen the database as the author
555 username = self.db.user.get(author, 'username')
556 self.db = self.instance.open(username)
558 # re-get the class with the new database connection
559 cl = self.db.getclass(classname)
561 # now update the recipients list
562 recipients = []
563 tracker_email = self.instance.ISSUE_TRACKER_EMAIL.lower()
564 for recipient in message.getaddrlist('to') + message.getaddrlist('cc'):
565 r = recipient[1].strip().lower()
566 if r == tracker_email or not r:
567 continue
569 # look up the recipient - create if necessary (and we're
570 # allowed to)
571 recipient = uidFromAddress(self.db, recipient, create)
573 # if all's well, add the recipient to the list
574 if recipient:
575 recipients.append(recipient)
577 #
578 # handle message-id and in-reply-to
579 #
580 messageid = message.getheader('message-id')
581 inreplyto = message.getheader('in-reply-to') or ''
582 # generate a messageid if there isn't one
583 if not messageid:
584 messageid = "<%s.%s.%s%s@%s>"%(time.time(), random.random(),
585 classname, nodeid, self.instance.MAIL_DOMAIN)
587 #
588 # now handle the body - find the message
589 #
590 content_type = message.gettype()
591 attachments = []
592 # General multipart handling:
593 # Take the first text/plain part, anything else is considered an
594 # attachment.
595 # multipart/mixed: multiple "unrelated" parts.
596 # multipart/signed (rfc 1847):
597 # The control information is carried in the second of the two
598 # required body parts.
599 # ACTION: Default, so if content is text/plain we get it.
600 # multipart/encrypted (rfc 1847):
601 # The control information is carried in the first of the two
602 # required body parts.
603 # ACTION: Not handleable as the content is encrypted.
604 # multipart/related (rfc 1872, 2112, 2387):
605 # The Multipart/Related content-type addresses the MIME
606 # representation of compound objects.
607 # ACTION: Default. If we are lucky there is a text/plain.
608 # TODO: One should use the start part and look for an Alternative
609 # that is text/plain.
610 # multipart/Alternative (rfc 1872, 1892):
611 # only in "related" ?
612 # multipart/report (rfc 1892):
613 # e.g. mail system delivery status reports.
614 # ACTION: Default. Could be ignored or used for Delivery Notification
615 # flagging.
616 # multipart/form-data:
617 # For web forms only.
618 if content_type == 'multipart/mixed':
619 # skip over the intro to the first boundary
620 part = message.getPart()
621 content = None
622 while 1:
623 # get the next part
624 part = message.getPart()
625 if part is None:
626 break
627 # parse it
628 subtype = part.gettype()
629 if subtype == 'text/plain' and not content:
630 # The first text/plain part is the message content.
631 content = self.get_part_data_decoded(part)
632 elif subtype == 'message/rfc822':
633 # handle message/rfc822 specially - the name should be
634 # the subject of the actual e-mail embedded here
635 i = part.fp.tell()
636 mailmess = Message(part.fp)
637 name = mailmess.getheader('subject')
638 part.fp.seek(i)
639 attachments.append((name, 'message/rfc822', part.fp.read()))
640 else:
641 # try name on Content-Type
642 name = part.getparam('name')
643 # this is just an attachment
644 data = self.get_part_data_decoded(part)
645 attachments.append((name, part.gettype(), data))
646 if content is None:
647 raise MailUsageError, '''
648 Roundup requires the submission to be plain text. The message parser could
649 not find a text/plain part to use.
650 '''
652 elif content_type[:10] == 'multipart/':
653 # skip over the intro to the first boundary
654 message.getPart()
655 content = None
656 while 1:
657 # get the next part
658 part = message.getPart()
659 if part is None:
660 break
661 # parse it
662 if part.gettype() == 'text/plain' and not content:
663 content = self.get_part_data_decoded(part)
664 if content is None:
665 raise MailUsageError, '''
666 Roundup requires the submission to be plain text. The message parser could
667 not find a text/plain part to use.
668 '''
670 elif content_type != 'text/plain':
671 raise MailUsageError, '''
672 Roundup requires the submission to be plain text. The message parser could
673 not find a text/plain part to use.
674 '''
676 else:
677 content = self.get_part_data_decoded(message)
679 # figure how much we should muck around with the email body
680 keep_citations = getattr(self.instance, 'EMAIL_KEEP_QUOTED_TEXT',
681 'no') == 'yes'
682 keep_body = getattr(self.instance, 'EMAIL_LEAVE_BODY_UNCHANGED',
683 'no') == 'yes'
685 # parse the body of the message, stripping out bits as appropriate
686 summary, content = parseContent(content, keep_citations,
687 keep_body)
689 #
690 # handle the attachments
691 #
692 files = []
693 for (name, mime_type, data) in attachments:
694 if not name:
695 name = "unnamed"
696 files.append(self.db.file.create(type=mime_type, name=name,
697 content=data))
699 #
700 # create the message if there's a message body (content)
701 #
702 if content:
703 message_id = self.db.msg.create(author=author,
704 recipients=recipients, date=date.Date('.'), summary=summary,
705 content=content, files=files, messageid=messageid,
706 inreplyto=inreplyto)
708 # attach the message to the node
709 if nodeid:
710 # add the message to the node's list
711 messages = cl.get(nodeid, 'messages')
712 messages.append(message_id)
713 props['messages'] = messages
714 else:
715 # pre-load the messages list
716 props['messages'] = [message_id]
718 # set the title to the subject
719 if properties.has_key('title') and not props.has_key('title'):
720 props['title'] = title
722 #
723 # perform the node change / create
724 #
725 try:
726 if nodeid:
727 cl.set(nodeid, **props)
728 else:
729 nodeid = cl.create(**props)
730 except (TypeError, IndexError, ValueError), message:
731 raise MailUsageError, '''
732 There was a problem with the message you sent:
733 %s
734 '''%message
736 # commit the changes to the DB
737 self.db.commit()
739 return nodeid
741 def extractUserFromList(userClass, users):
742 '''Given a list of users, try to extract the first non-anonymous user
743 and return that user, otherwise return None
744 '''
745 if len(users) > 1:
746 for user in users:
747 # make sure we don't match the anonymous or admin user
748 if userClass.get(user, 'username') in ('admin', 'anonymous'):
749 continue
750 # first valid match will do
751 return user
752 # well, I guess we have no choice
753 return user[0]
754 elif users:
755 return users[0]
756 return None
758 def uidFromAddress(db, address, create=1):
759 ''' address is from the rfc822 module, and therefore is (name, addr)
761 user is created if they don't exist in the db already
762 '''
763 (realname, address) = address
765 # try a straight match of the address
766 user = extractUserFromList(db.user, db.user.stringFind(address=address))
767 if user is not None: return user
769 # try the user alternate addresses if possible
770 props = db.user.getprops()
771 if props.has_key('alternate_addresses'):
772 users = db.user.filter(None, {'alternate_addresses': address},
773 [], [])
774 user = extractUserFromList(db.user, users)
775 if user is not None: return user
777 # try to match the username to the address (for local
778 # submissions where the address is empty)
779 user = extractUserFromList(db.user, db.user.stringFind(username=address))
781 # couldn't match address or username, so create a new user
782 if create:
783 return db.user.create(username=address, address=address,
784 realname=realname, roles=db.config.NEW_EMAIL_USER_ROLES)
785 else:
786 return 0
788 def parseContent(content, keep_citations, keep_body,
789 blank_line=re.compile(r'[\r\n]+\s*[\r\n]+'),
790 eol=re.compile(r'[\r\n]+'),
791 signature=re.compile(r'^[>|\s]*[-_]+\s*$'),
792 original_message=re.compile(r'^[>|\s]*-----Original Message-----$')):
793 ''' The message body is divided into sections by blank lines.
794 Sections where the second and all subsequent lines begin with a ">" or "|"
795 character are considered "quoting sections". The first line of the first
796 non-quoting section becomes the summary of the message.
797 '''
798 # strip off leading carriage-returns / newlines
799 i = 0
800 for i in range(len(content)):
801 if content[i] not in '\r\n':
802 break
803 if i > 0:
804 sections = blank_line.split(content[i:])
805 else:
806 sections = blank_line.split(content)
808 # extract out the summary from the message
809 summary = ''
810 l = []
811 for section in sections:
812 #section = section.strip()
813 if not section:
814 continue
815 lines = eol.split(section)
816 if (lines[0] and lines[0][0] in '>|') or (len(lines) > 1 and
817 lines[1] and lines[1][0] in '>|'):
818 # see if there's a response somewhere inside this section (ie.
819 # no blank line between quoted message and response)
820 for line in lines[1:]:
821 if line[0] not in '>|':
822 break
823 else:
824 # we keep quoted bits if specified in the config
825 if keep_citations:
826 l.append(section)
827 continue
828 # keep this section - it has reponse stuff in it
829 if not summary:
830 # and while we're at it, use the first non-quoted bit as
831 # our summary
832 summary = line
833 lines = lines[lines.index(line):]
834 section = '\n'.join(lines)
836 if not summary:
837 # if we don't have our summary yet use the first line of this
838 # section
839 summary = lines[0]
840 elif signature.match(lines[0]) and 2 <= len(lines) <= 10:
841 # lose any signature
842 break
843 elif original_message.match(lines[0]):
844 # ditch the stupid Outlook quoting of the entire original message
845 break
847 # and add the section to the output
848 l.append(section)
849 # we only set content for those who want to delete cruft from the
850 # message body, otherwise the body is left untouched.
851 if not keep_body:
852 content = '\n\n'.join(l)
853 return summary, content
855 #
856 # $Log: not supported by cvs2svn $
857 # Revision 1.80 2002/08/01 00:56:22 richard
858 # Added the web access and email access permissions, so people can restrict
859 # access to users who register through the email interface (for example).
860 # Also added "security" command to the roundup-admin interface to display the
861 # Role/Permission config for an instance.
862 #
863 # Revision 1.79 2002/07/26 08:26:59 richard
864 # Very close now. The cgi and mailgw now use the new security API. The two
865 # templates have been migrated to that setup. Lots of unit tests. Still some
866 # issue in the web form for editing Roles assigned to users.
867 #
868 # Revision 1.78 2002/07/25 07:14:06 richard
869 # Bugger it. Here's the current shape of the new security implementation.
870 # Still to do:
871 # . call the security funcs from cgi and mailgw
872 # . change shipped templates to include correct initialisation and remove
873 # the old config vars
874 # ... that seems like a lot. The bulk of the work has been done though. Honest :)
875 #
876 # Revision 1.77 2002/07/18 11:17:31 gmcm
877 # Add Number and Boolean types to hyperdb.
878 # Add conversion cases to web, mail & admin interfaces.
879 # Add storage/serialization cases to back_anydbm & back_metakit.
880 #
881 # Revision 1.76 2002/07/10 06:39:37 richard
882 # . made mailgw handle set and modify operations on multilinks (bug #579094)
883 #
884 # Revision 1.75 2002/07/09 01:21:24 richard
885 # Added ability for unit tests to turn off exception handling in mailgw so
886 # that exceptions are reported earlier (and hence make sense).
887 #
888 # Revision 1.74 2002/05/29 01:16:17 richard
889 # Sorry about this huge checkin! It's fixing a lot of related stuff in one go
890 # though.
891 #
892 # . #541941 ] changing multilink properties by mail
893 # . #526730 ] search for messages capability
894 # . #505180 ] split MailGW.handle_Message
895 # - also changed cgi client since it was duplicating the functionality
896 # . build htmlbase if tests are run using CVS checkout (removed note from
897 # installation.txt)
898 # . don't create an empty message on email issue creation if the email is empty
899 #
900 # Revision 1.73 2002/05/22 04:12:05 richard
901 # . applied patch #558876 ] cgi client customization
902 # ... with significant additions and modifications ;)
903 # - extended handling of ML assignedto to all places it's handled
904 # - added more NotFound info
905 #
906 # Revision 1.72 2002/05/22 01:24:51 richard
907 # Added note to MIGRATION about new config vars. Also made us more resilient
908 # for upgraders. Reinstated list header style (oops)
909 #
910 # Revision 1.71 2002/05/08 02:40:55 richard
911 # grr
912 #
913 # Revision 1.70 2002/05/06 23:40:07 richard
914 # hrm
915 #
916 # Revision 1.69 2002/05/06 23:37:21 richard
917 # Tweaking the signature deletion from mail messages.
918 # Added nuking of the "-----Original Message-----" crap from Outlook.
919 #
920 # Revision 1.68 2002/05/02 07:56:34 richard
921 # . added option to automatically add the authors and recipients of messages
922 # to the nosy lists with the options ADD_AUTHOR_TO_NOSY (default 'new') and
923 # ADD_RECIPIENTS_TO_NOSY (default 'new'). These settings emulate the current
924 # behaviour. Setting them to 'yes' will add the author/recipients to the nosy
925 # on messages that create issues and followup messages.
926 # . added missing documentation for a few of the config option values
927 #
928 # Revision 1.67 2002/04/23 15:46:49 rochecompaan
929 # . stripping of the email message body can now be controlled through
930 # the config variables EMAIL_KEEP_QUOTED_TEST and
931 # EMAIL_LEAVE_BODY_UNCHANGED.
932 #
933 # Revision 1.66 2002/03/14 23:59:24 richard
934 # . #517734 ] web header customisation is obscure
935 #
936 # Revision 1.65 2002/02/15 00:13:38 richard
937 # . #503204 ] mailgw needs a default class
938 # - partially done - the setting of additional properties can wait for a
939 # better configuration system.
940 #
941 # Revision 1.64 2002/02/14 23:46:02 richard
942 # . #516883 ] mail interface + ANONYMOUS_REGISTER
943 #
944 # Revision 1.63 2002/02/12 08:08:55 grubert
945 # . Clean up mail handling, multipart handling.
946 #
947 # Revision 1.62 2002/02/05 14:15:29 grubert
948 # . respect encodings in non multipart messages.
949 #
950 # Revision 1.61 2002/02/04 09:40:21 grubert
951 # . add test for multipart messages with first part being encoded.
952 #
953 # Revision 1.60 2002/02/01 07:43:12 grubert
954 # . mailgw checks encoding on first part too.
955 #
956 # Revision 1.59 2002/01/23 21:43:23 richard
957 # tabnuke
958 #
959 # Revision 1.58 2002/01/23 21:41:56 richard
960 # . mailgw failures (unexpected ones) are forwarded to the roundup admin
961 #
962 # Revision 1.57 2002/01/22 22:27:43 richard
963 # . handle stripping of "AW:" from subject line
964 #
965 # Revision 1.56 2002/01/22 11:54:45 rochecompaan
966 # Fixed status change in mail gateway.
967 #
968 # Revision 1.55 2002/01/21 10:05:47 rochecompaan
969 # Feature:
970 # . the mail gateway now responds with an error message when invalid
971 # values for arguments are specified for link or multilink properties
972 # . modified unit test to check nosy and assignedto when specified as
973 # arguments
974 #
975 # Fixed:
976 # . fixed setting nosy as argument in subject line
977 #
978 # Revision 1.54 2002/01/16 09:14:45 grubert
979 # . if the attachment has no name, name it unnamed, happens with tnefs.
980 #
981 # Revision 1.53 2002/01/16 07:20:54 richard
982 # simple help command for mailgw
983 #
984 # Revision 1.52 2002/01/15 00:12:40 richard
985 # #503340 ] creating issue with [asignedto=p.ohly]
986 #
987 # Revision 1.51 2002/01/14 02:20:15 richard
988 # . changed all config accesses so they access either the instance or the
989 # config attriubute on the db. This means that all config is obtained from
990 # instance_config instead of the mish-mash of classes. This will make
991 # switching to a ConfigParser setup easier too, I hope.
992 #
993 # At a minimum, this makes migration a _little_ easier (a lot easier in the
994 # 0.5.0 switch, I hope!)
995 #
996 # Revision 1.50 2002/01/11 22:59:01 richard
997 # . #502342 ] pipe interface
998 #
999 # Revision 1.49 2002/01/10 06:19:18 richard
1000 # followup lines directly after a quoted section were being eaten.
1001 #
1002 # Revision 1.48 2002/01/08 04:12:05 richard
1003 # Changed message-id format to "<%s.%s.%s%s@%s>" so it complies with RFC822
1004 #
1005 # Revision 1.47 2002/01/02 02:32:38 richard
1006 # ANONYMOUS_ACCESS -> ANONYMOUS_REGISTER
1007 #
1008 # Revision 1.46 2002/01/02 02:31:38 richard
1009 # Sorry for the huge checkin message - I was only intending to implement #496356
1010 # but I found a number of places where things had been broken by transactions:
1011 # . modified ROUNDUPDBSENDMAILDEBUG to be SENDMAILDEBUG and hold a filename
1012 # for _all_ roundup-generated smtp messages to be sent to.
1013 # . the transaction cache had broken the roundupdb.Class set() reactors
1014 # . newly-created author users in the mailgw weren't being committed to the db
1015 #
1016 # Stuff that made it into CHANGES.txt (ie. the stuff I was actually working
1017 # on when I found that stuff :):
1018 # . #496356 ] Use threading in messages
1019 # . detectors were being registered multiple times
1020 # . added tests for mailgw
1021 # . much better attaching of erroneous messages in the mail gateway
1022 #
1023 # Revision 1.45 2001/12/20 15:43:01 rochecompaan
1024 # Features added:
1025 # . Multilink properties are now displayed as comma separated values in
1026 # a textbox
1027 # . The add user link is now only visible to the admin user
1028 # . Modified the mail gateway to reject submissions from unknown
1029 # addresses if ANONYMOUS_ACCESS is denied
1030 #
1031 # Revision 1.44 2001/12/18 15:30:34 rochecompaan
1032 # Fixed bugs:
1033 # . Fixed file creation and retrieval in same transaction in anydbm
1034 # backend
1035 # . Cgi interface now renders new issue after issue creation
1036 # . Could not set issue status to resolved through cgi interface
1037 # . Mail gateway was changing status back to 'chatting' if status was
1038 # omitted as an argument
1039 #
1040 # Revision 1.43 2001/12/15 19:39:01 rochecompaan
1041 # Oops.
1042 #
1043 # Revision 1.42 2001/12/15 19:24:39 rochecompaan
1044 # . Modified cgi interface to change properties only once all changes are
1045 # collected, files created and messages generated.
1046 # . Moved generation of change note to nosyreactors.
1047 # . We now check for changes to "assignedto" to ensure it's added to the
1048 # nosy list.
1049 #
1050 # Revision 1.41 2001/12/10 00:57:38 richard
1051 # From CHANGES:
1052 # . Added the "display" command to the admin tool - displays a node's values
1053 # . #489760 ] [issue] only subject
1054 # . fixed the doc/index.html to include the quoting in the mail alias.
1055 #
1056 # Also:
1057 # . fixed roundup-admin so it works with transactions
1058 # . disabled the back_anydbm module if anydbm tries to use dumbdbm
1059 #
1060 # Revision 1.40 2001/12/05 14:26:44 rochecompaan
1061 # Removed generation of change note from "sendmessage" in roundupdb.py.
1062 # The change note is now generated when the message is created.
1063 #
1064 # Revision 1.39 2001/12/02 05:06:16 richard
1065 # . We now use weakrefs in the Classes to keep the database reference, so
1066 # the close() method on the database is no longer needed.
1067 # I bumped the minimum python requirement up to 2.1 accordingly.
1068 # . #487480 ] roundup-server
1069 # . #487476 ] INSTALL.txt
1070 #
1071 # I also cleaned up the change message / post-edit stuff in the cgi client.
1072 # There's now a clearly marked "TODO: append the change note" where I believe
1073 # the change note should be added there. The "changes" list will obviously
1074 # have to be modified to be a dict of the changes, or somesuch.
1075 #
1076 # More testing needed.
1077 #
1078 # Revision 1.38 2001/12/01 07:17:50 richard
1079 # . We now have basic transaction support! Information is only written to
1080 # the database when the commit() method is called. Only the anydbm
1081 # backend is modified in this way - neither of the bsddb backends have been.
1082 # The mail, admin and cgi interfaces all use commit (except the admin tool
1083 # doesn't have a commit command, so interactive users can't commit...)
1084 # . Fixed login/registration forwarding the user to the right page (or not,
1085 # on a failure)
1086 #
1087 # Revision 1.37 2001/11/28 21:55:35 richard
1088 # . login_action and newuser_action return values were being ignored
1089 # . Woohoo! Found that bloody re-login bug that was killing the mail
1090 # gateway.
1091 # (also a minor cleanup in hyperdb)
1092 #
1093 # Revision 1.36 2001/11/26 22:55:56 richard
1094 # Feature:
1095 # . Added INSTANCE_NAME to configuration - used in web and email to identify
1096 # the instance.
1097 # . Added EMAIL_SIGNATURE_POSITION to indicate where to place the roundup
1098 # signature info in e-mails.
1099 # . Some more flexibility in the mail gateway and more error handling.
1100 # . Login now takes you to the page you back to the were denied access to.
1101 #
1102 # Fixed:
1103 # . Lots of bugs, thanks Roché and others on the devel mailing list!
1104 #
1105 # Revision 1.35 2001/11/22 15:46:42 jhermann
1106 # Added module docstrings to all modules.
1107 #
1108 # Revision 1.34 2001/11/15 10:24:27 richard
1109 # handle the case where there is no file attached
1110 #
1111 # Revision 1.33 2001/11/13 21:44:44 richard
1112 # . re-open the database as the author in mail handling
1113 #
1114 # Revision 1.32 2001/11/12 22:04:29 richard
1115 # oops, left debug in there
1116 #
1117 # Revision 1.31 2001/11/12 22:01:06 richard
1118 # Fixed issues with nosy reaction and author copies.
1119 #
1120 # Revision 1.30 2001/11/09 22:33:28 richard
1121 # More error handling fixes.
1122 #
1123 # Revision 1.29 2001/11/07 05:29:26 richard
1124 # Modified roundup-mailgw so it can read e-mails from a local mail spool
1125 # file. Truncates the spool file after parsing.
1126 # Fixed a couple of small bugs introduced in roundup.mailgw when I started
1127 # the popgw.
1128 #
1129 # Revision 1.28 2001/11/01 22:04:37 richard
1130 # Started work on supporting a pop3-fetching server
1131 # Fixed bugs:
1132 # . bug #477104 ] HTML tag error in roundup-server
1133 # . bug #477107 ] HTTP header problem
1134 #
1135 # Revision 1.27 2001/10/30 11:26:10 richard
1136 # Case-insensitive match for ISSUE_TRACKER_EMAIL in address in e-mail.
1137 #
1138 # Revision 1.26 2001/10/30 00:54:45 richard
1139 # Features:
1140 # . #467129 ] Lossage when username=e-mail-address
1141 # . #473123 ] Change message generation for author
1142 # . MailGW now moves 'resolved' to 'chatting' on receiving e-mail for an issue.
1143 #
1144 # Revision 1.25 2001/10/28 23:22:28 richard
1145 # fixed bug #474749 ] Indentations lost
1146 #
1147 # Revision 1.24 2001/10/23 22:57:52 richard
1148 # Fix unread->chatting auto transition, thanks Roch'e
1149 #
1150 # Revision 1.23 2001/10/21 04:00:20 richard
1151 # MailGW now moves 'unread' to 'chatting' on receiving e-mail for an issue.
1152 #
1153 # Revision 1.22 2001/10/21 03:35:13 richard
1154 # bug #473125: Paragraph in e-mails
1155 #
1156 # Revision 1.21 2001/10/21 00:53:42 richard
1157 # bug #473130: Nosy list not set correctly
1158 #
1159 # Revision 1.20 2001/10/17 23:13:19 richard
1160 # Did a fair bit of work on the admin tool. Now has an extra command "table"
1161 # which displays node information in a tabular format. Also fixed import and
1162 # export so they work. Removed freshen.
1163 # Fixed quopri usage in mailgw from bug reports.
1164 #
1165 # Revision 1.19 2001/10/11 23:43:04 richard
1166 # Implemented the comma-separated printing option in the admin tool.
1167 # Fixed a typo (more of a vim-o actually :) in mailgw.
1168 #
1169 # Revision 1.18 2001/10/11 06:38:57 richard
1170 # Initial cut at trying to handle people responding to CC'ed messages that
1171 # create an issue.
1172 #
1173 # Revision 1.17 2001/10/09 07:25:59 richard
1174 # Added the Password property type. See "pydoc roundup.password" for
1175 # implementation details. Have updated some of the documentation too.
1176 #
1177 # Revision 1.16 2001/10/05 02:23:24 richard
1178 # . roundup-admin create now prompts for property info if none is supplied
1179 # on the command-line.
1180 # . hyperdb Class getprops() method may now return only the mutable
1181 # properties.
1182 # . Login now uses cookies, which makes it a whole lot more flexible. We can
1183 # now support anonymous user access (read-only, unless there's an
1184 # "anonymous" user, in which case write access is permitted). Login
1185 # handling has been moved into cgi_client.Client.main()
1186 # . The "extended" schema is now the default in roundup init.
1187 # . The schemas have had their page headings modified to cope with the new
1188 # login handling. Existing installations should copy the interfaces.py
1189 # file from the roundup lib directory to their instance home.
1190 # . Incorrectly had a Bizar Software copyright on the cgitb.py module from
1191 # Ping - has been removed.
1192 # . Fixed a whole bunch of places in the CGI interface where we should have
1193 # been returning Not Found instead of throwing an exception.
1194 # . Fixed a deviation from the spec: trying to modify the 'id' property of
1195 # an item now throws an exception.
1196 #
1197 # Revision 1.15 2001/08/30 06:01:17 richard
1198 # Fixed missing import in mailgw :(
1199 #
1200 # Revision 1.14 2001/08/13 23:02:54 richard
1201 # Make the mail parser a little more robust.
1202 #
1203 # Revision 1.13 2001/08/12 06:32:36 richard
1204 # using isinstance(blah, Foo) now instead of isFooType
1205 #
1206 # Revision 1.12 2001/08/08 01:27:00 richard
1207 # Added better error handling to mailgw.
1208 #
1209 # Revision 1.11 2001/08/08 00:08:03 richard
1210 # oops ;)
1211 #
1212 # Revision 1.10 2001/08/07 00:24:42 richard
1213 # stupid typo
1214 #
1215 # Revision 1.9 2001/08/07 00:15:51 richard
1216 # Added the copyright/license notice to (nearly) all files at request of
1217 # Bizar Software.
1218 #
1219 # Revision 1.8 2001/08/05 07:06:07 richard
1220 # removed some print statements
1221 #
1222 # Revision 1.7 2001/08/03 07:18:22 richard
1223 # Implemented correct mail splitting (was taking a shortcut). Added unit
1224 # tests. Also snips signatures now too.
1225 #
1226 # Revision 1.6 2001/08/01 04:24:21 richard
1227 # mailgw was assuming certain properties existed on the issues being created.
1228 #
1229 # Revision 1.5 2001/07/29 07:01:39 richard
1230 # Added vim command to all source so that we don't get no steenkin' tabs :)
1231 #
1232 # Revision 1.4 2001/07/28 06:43:02 richard
1233 # Multipart message class has the getPart method now. Added some tests for it.
1234 #
1235 # Revision 1.3 2001/07/28 00:34:34 richard
1236 # Fixed some non-string node ids.
1237 #
1238 # Revision 1.2 2001/07/22 12:09:32 richard
1239 # Final commit of Grande Splite
1240 #
1241 #
1242 # vim: set filetype=python ts=4 sw=4 et si