ab03bda4e1fbb2db97200ae80d4e1d61b066398b
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
19 """
20 An e-mail gateway for Roundup.
22 Incoming messages are examined for multiple parts:
23 . In a multipart/mixed message or part, each subpart is extracted and
24 examined. The text/plain subparts are assembled to form the textual
25 body of the message, to be stored in the file associated with a "msg"
26 class node. Any parts of other types are each stored in separate files
27 and given "file" class nodes that are linked to the "msg" node.
28 . In a multipart/alternative message or part, we look for a text/plain
29 subpart and ignore the other parts.
31 Summary
32 -------
33 The "summary" property on message nodes is taken from the first non-quoting
34 section in the message body. The message body is divided into sections by
35 blank lines. Sections where the second and all subsequent lines begin with
36 a ">" or "|" character are considered "quoting sections". The first line of
37 the first non-quoting section becomes the summary of the message.
39 Addresses
40 ---------
41 All of the addresses in the To: and Cc: headers of the incoming message are
42 looked up among the user nodes, and the corresponding users are placed in
43 the "recipients" property on the new "msg" node. The address in the From:
44 header similarly determines the "author" property of the new "msg"
45 node. The default handling for addresses that don't have corresponding
46 users is to create new users with no passwords and a username equal to the
47 address. (The web interface does not permit logins for users with no
48 passwords.) If we prefer to reject mail from outside sources, we can simply
49 register an auditor on the "user" class that prevents the creation of user
50 nodes with no passwords.
52 Actions
53 -------
54 The subject line of the incoming message is examined to determine whether
55 the message is an attempt to create a new item or to discuss an existing
56 item. A designator enclosed in square brackets is sought as the first thing
57 on the subject line (after skipping any "Fwd:" or "Re:" prefixes).
59 If an item designator (class name and id number) is found there, the newly
60 created "msg" node is added to the "messages" property for that item, and
61 any new "file" nodes are added to the "files" property for the item.
63 If just an item class name is found there, we attempt to create a new item
64 of that class with its "messages" property initialized to contain the new
65 "msg" node and its "files" property initialized to contain any new "file"
66 nodes.
68 Triggers
69 --------
70 Both cases may trigger detectors (in the first case we are calling the
71 set() method to add the message to the item's spool; in the second case we
72 are calling the create() method to create a new node). If an auditor raises
73 an exception, the original message is bounced back to the sender with the
74 explanatory message given in the exception.
76 $Id: mailgw.py,v 1.136 2003-11-03 18:34:03 jlgijsbers Exp $
77 """
79 import string, re, os, mimetools, cStringIO, smtplib, socket, binascii, quopri
80 import time, random, sys
81 import traceback, MimeWriter, rfc822
83 from roundup import hyperdb, date, password, rfc2822
84 from roundup.mailer import Mailer
86 SENDMAILDEBUG = os.environ.get('SENDMAILDEBUG', '')
88 class MailGWError(ValueError):
89 pass
91 class MailUsageError(ValueError):
92 pass
94 class MailUsageHelp(Exception):
95 pass
97 class MailLoop(Exception):
98 """ We've seen this message before... """
99 pass
101 class Unauthorized(Exception):
102 """ Access denied """
104 def initialiseSecurity(security):
105 ''' Create some Permissions and Roles on the security object
107 This function is directly invoked by security.Security.__init__()
108 as a part of the Security object instantiation.
109 '''
110 security.addPermission(name="Email Registration",
111 description="Anonymous may register through e-mail")
112 p = security.addPermission(name="Email Access",
113 description="User may use the email interface")
114 security.addPermissionToRole('Admin', p)
116 def getparam(str, param):
117 ''' From the rfc822 "header" string, extract "param" if it appears.
118 '''
119 if ';' not in str:
120 return None
121 str = str[str.index(';'):]
122 while str[:1] == ';':
123 str = str[1:]
124 if ';' in str:
125 # XXX Should parse quotes!
126 end = str.index(';')
127 else:
128 end = len(str)
129 f = str[:end]
130 if '=' in f:
131 i = f.index('=')
132 if f[:i].strip().lower() == param:
133 return rfc822.unquote(f[i+1:].strip())
134 return None
136 class Message(mimetools.Message):
137 ''' subclass mimetools.Message so we can retrieve the parts of the
138 message...
139 '''
140 def getPart(self):
141 ''' Get a single part of a multipart message and return it as a new
142 Message instance.
143 '''
144 boundary = self.getparam('boundary')
145 mid, end = '--'+boundary, '--'+boundary+'--'
146 s = cStringIO.StringIO()
147 while 1:
148 line = self.fp.readline()
149 if not line:
150 break
151 if line.strip() in (mid, end):
152 break
153 s.write(line)
154 if not s.getvalue().strip():
155 return None
156 s.seek(0)
157 return Message(s)
159 def getheader(self, name, default=None):
160 hdr = mimetools.Message.getheader(self, name, default)
161 if hdr:
162 hdr = hdr.replace('\n','') # Inserted by rfc822.readheaders
163 return rfc2822.decode_header(hdr)
165 class MailGW:
167 # Matches subjects like:
168 # Re: "[issue1234] title of issue [status=resolved]"
169 subject_re = re.compile(r'''
170 (?P<refwd>\s*\W?\s*(fw|fwd|re|aw)\W\s*)*\s* # Re:
171 (?P<quote>")? # Leading "
172 (\[(?P<classname>[^\d\s]+) # [issue..
173 (?P<nodeid>\d+)? # ..1234]
174 \])?\s*
175 (?P<title>[^[]+)? # issue title
176 "? # Trailing "
177 (\[(?P<args>.+?)\])? # [prop=value]
178 ''', re.IGNORECASE|re.VERBOSE)
180 def __init__(self, instance, db, arguments={}):
181 self.instance = instance
182 self.db = db
183 self.arguments = arguments
184 self.mailer = Mailer(instance.config)
186 # should we trap exceptions (normal usage) or pass them through
187 # (for testing)
188 self.trapExceptions = 1
190 def do_pipe(self):
191 """ Read a message from standard input and pass it to the mail handler.
193 Read into an internal structure that we can seek on (in case
194 there's an error).
196 XXX: we may want to read this into a temporary file instead...
197 """
198 s = cStringIO.StringIO()
199 s.write(sys.stdin.read())
200 s.seek(0)
201 self.main(s)
202 return 0
204 def do_mailbox(self, filename):
205 """ Read a series of messages from the specified unix mailbox file and
206 pass each to the mail handler.
207 """
208 # open the spool file and lock it
209 import fcntl
210 # FCNTL is deprecated in py2.3 and fcntl takes over all the symbols
211 if hasattr(fcntl, 'LOCK_EX'):
212 FCNTL = fcntl
213 else:
214 import FCNTL
215 f = open(filename, 'r+')
216 fcntl.flock(f.fileno(), FCNTL.LOCK_EX)
218 # handle and clear the mailbox
219 try:
220 from mailbox import UnixMailbox
221 mailbox = UnixMailbox(f, factory=Message)
222 # grab one message
223 message = mailbox.next()
224 while message:
225 # handle this message
226 self.handle_Message(message)
227 message = mailbox.next()
228 # nuke the file contents
229 os.ftruncate(f.fileno(), 0)
230 except:
231 import traceback
232 traceback.print_exc()
233 return 1
234 fcntl.flock(f.fileno(), FCNTL.LOCK_UN)
235 return 0
237 def do_apop(self, server, user='', password=''):
238 ''' Do authentication POP
239 '''
240 self.do_pop(server, user, password, apop=1)
242 def do_pop(self, server, user='', password='', apop=0):
243 '''Read a series of messages from the specified POP server.
244 '''
245 import getpass, poplib, socket
246 try:
247 if not user:
248 user = raw_input(_('User: '))
249 if not password:
250 password = getpass.getpass()
251 except (KeyboardInterrupt, EOFError):
252 # Ctrl C or D maybe also Ctrl Z under Windows.
253 print "\nAborted by user."
254 return 1
256 # open a connection to the server and retrieve all messages
257 try:
258 server = poplib.POP3(server)
259 except socket.error, message:
260 print "POP server error:", message
261 return 1
262 if apop:
263 server.apop(user, password)
264 else:
265 server.user(user)
266 server.pass_(password)
267 numMessages = len(server.list()[1])
268 for i in range(1, numMessages+1):
269 # retr: returns
270 # [ pop response e.g. '+OK 459 octets',
271 # [ array of message lines ],
272 # number of octets ]
273 lines = server.retr(i)[1]
274 s = cStringIO.StringIO('\n'.join(lines))
275 s.seek(0)
276 self.handle_Message(Message(s))
277 # delete the message
278 server.dele(i)
280 # quit the server to commit changes.
281 server.quit()
282 return 0
284 def main(self, fp):
285 ''' fp - the file from which to read the Message.
286 '''
287 return self.handle_Message(Message(fp))
289 def handle_Message(self, message):
290 """Handle an RFC822 Message
292 Handle the Message object by calling handle_message() and then cope
293 with any errors raised by handle_message.
294 This method's job is to make that call and handle any
295 errors in a sane manner. It should be replaced if you wish to
296 handle errors in a different manner.
297 """
298 # in some rare cases, a particularly stuffed-up e-mail will make
299 # its way into here... try to handle it gracefully
300 sendto = message.getaddrlist('from')
301 if sendto:
302 if not self.trapExceptions:
303 return self.handle_message(message)
304 try:
305 return self.handle_message(message)
306 except MailUsageHelp:
307 # bounce the message back to the sender with the usage message
308 fulldoc = '\n'.join(string.split(__doc__, '\n')[2:])
309 sendto = [sendto[0][1]]
310 m = ['']
311 m.append('\n\nMail Gateway Help\n=================')
312 m.append(fulldoc)
313 self.mailer.bounce_message(message, sendto, m,
314 subject="Mail Gateway Help")
315 except MailUsageError, value:
316 # bounce the message back to the sender with the usage message
317 fulldoc = '\n'.join(string.split(__doc__, '\n')[2:])
318 sendto = [sendto[0][1]]
319 m = ['']
320 m.append(str(value))
321 m.append('\n\nMail Gateway Help\n=================')
322 m.append(fulldoc)
323 self.mailer.bounce_message(message, sendto, m)
324 except Unauthorized, value:
325 # just inform the user that he is not authorized
326 sendto = [sendto[0][1]]
327 m = ['']
328 m.append(str(value))
329 self.mailer.bounce_message(message, sendto, m)
330 except MailLoop:
331 # XXX we should use a log file here...
332 return
333 except:
334 # bounce the message back to the sender with the error message
335 # XXX we should use a log file here...
336 sendto = [sendto[0][1], self.instance.config.ADMIN_EMAIL]
337 m = ['']
338 m.append('An unexpected error occurred during the processing')
339 m.append('of your message. The tracker administrator is being')
340 m.append('notified.\n')
341 m.append('---- traceback of failure ----')
342 s = cStringIO.StringIO()
343 import traceback
344 traceback.print_exc(None, s)
345 m.append(s.getvalue())
346 self.mailer.bounce_message(message, sendto, m)
347 else:
348 # very bad-looking message - we don't even know who sent it
349 # XXX we should use a log file here...
350 sendto = [self.instance.config.ADMIN_EMAIL]
351 m = ['Subject: badly formed message from mail gateway']
352 m.append('')
353 m.append('The mail gateway retrieved a message which has no From:')
354 m.append('line, indicating that it is corrupt. Please check your')
355 m.append('mail gateway source. Failed message is attached.')
356 m.append('')
357 self.mailer.bounce_message(message, sendto, m,
358 subject='Badly formed message from mail gateway')
360 def get_part_data_decoded(self,part):
361 encoding = part.getencoding()
362 data = None
363 if encoding == 'base64':
364 # BUG: is base64 really used for text encoding or
365 # are we inserting zip files here.
366 data = binascii.a2b_base64(part.fp.read())
367 elif encoding == 'quoted-printable':
368 # the quopri module wants to work with files
369 decoded = cStringIO.StringIO()
370 quopri.decode(part.fp, decoded)
371 data = decoded.getvalue()
372 elif encoding == 'uuencoded':
373 data = binascii.a2b_uu(part.fp.read())
374 else:
375 # take it as text
376 data = part.fp.read()
378 # Encode message to unicode
379 charset = rfc2822.unaliasCharset(part.getparam("charset"))
380 if charset:
381 # Do conversion only if charset specified
382 edata = unicode(data, charset).encode('utf-8')
383 # Convert from dos eol to unix
384 edata = edata.replace('\r\n', '\n')
385 else:
386 # Leave message content as is
387 edata = data
389 return edata
391 def handle_message(self, message):
392 ''' message - a Message instance
394 Parse the message as per the module docstring.
395 '''
396 # detect loops
397 if message.getheader('x-roundup-loop', ''):
398 raise MailLoop
400 # XXX Don't enable. This doesn't work yet.
401 # "[^A-z.]tracker\+(?P<classname>[^\d\s]+)(?P<nodeid>\d+)\@some.dom.ain[^A-z.]"
402 # handle delivery to addresses like:tracker+issue25@some.dom.ain
403 # use the embedded issue number as our issue
404 # if hasattr(self.instance.config, 'EMAIL_ISSUE_ADDRESS_RE') and \
405 # self.instance.config.EMAIL_ISSUE_ADDRESS_RE:
406 # issue_re = self.instance.config.EMAIL_ISSUE_ADDRESS_RE
407 # for header in ['to', 'cc', 'bcc']:
408 # addresses = message.getheader(header, '')
409 # if addresses:
410 # # FIXME, this only finds the first match in the addresses.
411 # issue = re.search(issue_re, addresses, 'i')
412 # if issue:
413 # classname = issue.group('classname')
414 # nodeid = issue.group('nodeid')
415 # break
417 # handle the subject line
418 subject = message.getheader('subject', '')
420 if not subject:
421 raise MailUsageError, '''
422 Emails to Roundup trackers must include a Subject: line!
423 '''
425 if subject.strip().lower() == 'help':
426 raise MailUsageHelp
428 m = self.subject_re.match(subject)
430 # check for well-formed subject line
431 if m:
432 # get the classname
433 classname = m.group('classname')
434 if classname is None:
435 # no classname, check if this a registration confirmation email
436 # or fallback on the default class
437 otk_re = re.compile('-- key (?P<otk>[a-zA-Z0-9]{32})')
438 otk = otk_re.search(m.group('title'))
439 if otk:
440 self.db.confirm_registration(otk.group('otk'))
441 subject = 'Your registration to %s is complete' % \
442 self.instance.config.TRACKER_NAME
443 sendto = [message.getheader('from')]
444 self.mailer.standard_message(sendto, subject, '')
445 return
446 elif hasattr(self.instance.config, 'MAIL_DEFAULT_CLASS') and \
447 self.instance.config.MAIL_DEFAULT_CLASS:
448 classname = self.instance.config.MAIL_DEFAULT_CLASS
449 else:
450 # fail
451 m = None
453 if not m:
454 raise MailUsageError, """
455 The message you sent to roundup did not contain a properly formed subject
456 line. The subject must contain a class name or designator to indicate the
457 'topic' of the message. For example:
458 Subject: [issue] This is a new issue
459 - this will create a new issue in the tracker with the title 'This is
460 a new issue'.
461 Subject: [issue1234] This is a followup to issue 1234
462 - this will append the message's contents to the existing issue 1234
463 in the tracker.
465 Subject was: '%s'
466 """%subject
468 # get the class
469 try:
470 cl = self.db.getclass(classname)
471 except KeyError:
472 raise MailUsageError, '''
473 The class name you identified in the subject line ("%s") does not exist in the
474 database.
476 Valid class names are: %s
477 Subject was: "%s"
478 '''%(classname, ', '.join(self.db.getclasses()), subject)
480 # get the optional nodeid
481 nodeid = m.group('nodeid')
483 # title is optional too
484 title = m.group('title')
485 if title:
486 title = title.strip()
487 else:
488 title = ''
490 # strip off the quotes that dumb emailers put around the subject, like
491 # Re: "[issue1] bla blah"
492 if m.group('quote') and title.endswith('"'):
493 title = title[:-1]
495 # but we do need either a title or a nodeid...
496 if nodeid is None and not title:
497 raise MailUsageError, '''
498 I cannot match your message to a node in the database - you need to either
499 supply a full node identifier (with number, eg "[issue123]" or keep the
500 previous subject title intact so I can match that.
502 Subject was: "%s"
503 '''%subject
505 # If there's no nodeid, check to see if this is a followup and
506 # maybe someone's responded to the initial mail that created an
507 # entry. Try to find the matching nodes with the same title, and
508 # use the _last_ one matched (since that'll _usually_ be the most
509 # recent...)
510 if nodeid is None and m.group('refwd'):
511 l = cl.stringFind(title=title)
512 if l:
513 nodeid = l[-1]
515 # if a nodeid was specified, make sure it's valid
516 if nodeid is not None and not cl.hasnode(nodeid):
517 raise MailUsageError, '''
518 The node specified by the designator in the subject of your message ("%s")
519 does not exist.
521 Subject was: "%s"
522 '''%(nodeid, subject)
524 # Handle the arguments specified by the email gateway command line.
525 # We do this by looping over the list of self.arguments looking for
526 # a -C to tell us what class then the -S setting string.
527 msg_props = {}
528 user_props = {}
529 file_props = {}
530 issue_props = {}
531 # so, if we have any arguments, use them
532 if self.arguments:
533 current_class = 'msg'
534 for option, propstring in self.arguments:
535 if option in ( '-C', '--class'):
536 current_class = propstring.strip()
537 if current_class not in ('msg', 'file', 'user', 'issue'):
538 raise MailUsageError, '''
539 The mail gateway is not properly set up. Please contact
540 %s and have them fix the incorrect class specified as:
541 %s
542 '''%(self.instance.config.ADMIN_EMAIL, current_class)
543 if option in ('-S', '--set'):
544 if current_class == 'issue' :
545 errors, issue_props = setPropArrayFromString(self,
546 cl, propstring.strip(), nodeid)
547 elif current_class == 'file' :
548 temp_cl = self.db.getclass('file')
549 errors, file_props = setPropArrayFromString(self,
550 temp_cl, propstring.strip())
551 elif current_class == 'msg' :
552 temp_cl = self.db.getclass('msg')
553 errors, msg_props = setPropArrayFromString(self,
554 temp_cl, propstring.strip())
555 elif current_class == 'user' :
556 temp_cl = self.db.getclass('user')
557 errors, user_props = setPropArrayFromString(self,
558 temp_cl, propstring.strip())
559 if errors:
560 raise MailUsageError, '''
561 The mail gateway is not properly set up. Please contact
562 %s and have them fix the incorrect properties:
563 %s
564 '''%(self.instance.config.ADMIN_EMAIL, errors)
566 #
567 # handle the users
568 #
569 # Don't create users if anonymous isn't allowed to register
570 create = 1
571 anonid = self.db.user.lookup('anonymous')
572 if not self.db.security.hasPermission('Email Registration', anonid):
573 create = 0
575 # ok, now figure out who the author is - create a new user if the
576 # "create" flag is true
577 author = uidFromAddress(self.db, message.getaddrlist('from')[0],
578 create=create)
580 # if we're not recognised, and we don't get added as a user, then we
581 # must be anonymous
582 if not author:
583 author = anonid
585 # make sure the author has permission to use the email interface
586 if not self.db.security.hasPermission('Email Access', author):
587 if author == anonid:
588 # we're anonymous and we need to be a registered user
589 raise Unauthorized, '''
590 You are not a registered user.
592 Unknown address: %s
593 '''%message.getaddrlist('from')[0][1]
594 else:
595 # we're registered and we're _still_ not allowed access
596 raise Unauthorized, 'You are not permitted to access '\
597 'this tracker.'
599 # make sure they're allowed to edit this class of information
600 if not self.db.security.hasPermission('Edit', author, classname):
601 raise Unauthorized, 'You are not permitted to edit %s.'%classname
603 # the author may have been created - make sure the change is
604 # committed before we reopen the database
605 self.db.commit()
607 # reopen the database as the author
608 username = self.db.user.get(author, 'username')
609 self.db.close()
610 self.db = self.instance.open(username)
612 # re-get the class with the new database connection
613 cl = self.db.getclass(classname)
615 # now update the recipients list
616 recipients = []
617 tracker_email = self.instance.config.TRACKER_EMAIL.lower()
618 for recipient in message.getaddrlist('to') + message.getaddrlist('cc'):
619 r = recipient[1].strip().lower()
620 if r == tracker_email or not r:
621 continue
623 # look up the recipient - create if necessary (and we're
624 # allowed to)
625 recipient = uidFromAddress(self.db, recipient, create, **user_props)
627 # if all's well, add the recipient to the list
628 if recipient:
629 recipients.append(recipient)
631 #
632 # handle the subject argument list
633 #
634 # figure what the properties of this Class are
635 properties = cl.getprops()
636 props = {}
637 args = m.group('args')
638 if args:
639 errors, props = setPropArrayFromString(self, cl, args, nodeid)
640 # handle any errors parsing the argument list
641 if errors:
642 errors = '\n- '.join(errors)
643 raise MailUsageError, '''
644 There were problems handling your subject line argument list:
645 - %s
647 Subject was: "%s"
648 '''%(errors, subject)
651 # set the issue title to the subject
652 if properties.has_key('title') and not issue_props.has_key('title'):
653 issue_props['title'] = title.strip()
655 #
656 # handle message-id and in-reply-to
657 #
658 messageid = message.getheader('message-id')
659 inreplyto = message.getheader('in-reply-to') or ''
660 # generate a messageid if there isn't one
661 if not messageid:
662 messageid = "<%s.%s.%s%s@%s>"%(time.time(), random.random(),
663 classname, nodeid, self.instance.config.MAIL_DOMAIN)
665 #
666 # now handle the body - find the message
667 #
668 content_type = message.gettype()
669 attachments = []
670 # General multipart handling:
671 # Take the first text/plain part, anything else is considered an
672 # attachment.
673 # multipart/mixed: multiple "unrelated" parts.
674 # multipart/signed (rfc 1847):
675 # The control information is carried in the second of the two
676 # required body parts.
677 # ACTION: Default, so if content is text/plain we get it.
678 # multipart/encrypted (rfc 1847):
679 # The control information is carried in the first of the two
680 # required body parts.
681 # ACTION: Not handleable as the content is encrypted.
682 # multipart/related (rfc 1872, 2112, 2387):
683 # The Multipart/Related content-type addresses the MIME
684 # representation of compound objects.
685 # ACTION: Default. If we are lucky there is a text/plain.
686 # TODO: One should use the start part and look for an Alternative
687 # that is text/plain.
688 # multipart/Alternative (rfc 1872, 1892):
689 # only in "related" ?
690 # multipart/report (rfc 1892):
691 # e.g. mail system delivery status reports.
692 # ACTION: Default. Could be ignored or used for Delivery Notification
693 # flagging.
694 # multipart/form-data:
695 # For web forms only.
696 if content_type == 'multipart/mixed':
697 # skip over the intro to the first boundary
698 part = message.getPart()
699 content = None
700 while 1:
701 # get the next part
702 part = message.getPart()
703 if part is None:
704 break
705 # parse it
706 subtype = part.gettype()
707 if subtype == 'text/plain' and not content:
708 # The first text/plain part is the message content.
709 content = self.get_part_data_decoded(part)
710 elif subtype == 'message/rfc822':
711 # handle message/rfc822 specially - the name should be
712 # the subject of the actual e-mail embedded here
713 i = part.fp.tell()
714 mailmess = Message(part.fp)
715 name = mailmess.getheader('subject')
716 part.fp.seek(i)
717 attachments.append((name, 'message/rfc822', part.fp.read()))
718 elif subtype == 'multipart/alternative':
719 # Search for text/plain in message with attachment and
720 # alternative text representation
721 # skip over intro to first boundary
722 part.getPart()
723 while 1:
724 # get the next part
725 subpart = part.getPart()
726 if subpart is None:
727 break
728 # parse it
729 if subpart.gettype() == 'text/plain' and not content:
730 content = self.get_part_data_decoded(subpart)
731 else:
732 # try name on Content-Type
733 name = part.getparam('name')
734 if name:
735 name = name.strip()
736 if not name:
737 disp = part.getheader('content-disposition', None)
738 if disp:
739 name = getparam(disp, 'filename')
740 if name:
741 name = name.strip()
742 # this is just an attachment
743 data = self.get_part_data_decoded(part)
744 attachments.append((name, part.gettype(), data))
745 if content is None:
746 raise MailUsageError, '''
747 Roundup requires the submission to be plain text. The message parser could
748 not find a text/plain part to use.
749 '''
751 elif content_type[:10] == 'multipart/':
752 # skip over the intro to the first boundary
753 message.getPart()
754 content = None
755 while 1:
756 # get the next part
757 part = message.getPart()
758 if part is None:
759 break
760 # parse it
761 if part.gettype() == 'text/plain' and not content:
762 content = self.get_part_data_decoded(part)
763 if content is None:
764 raise MailUsageError, '''
765 Roundup requires the submission to be plain text. The message parser could
766 not find a text/plain part to use.
767 '''
769 elif content_type != 'text/plain':
770 raise MailUsageError, '''
771 Roundup requires the submission to be plain text. The message parser could
772 not find a text/plain part to use.
773 '''
775 else:
776 content = self.get_part_data_decoded(message)
778 # figure how much we should muck around with the email body
779 keep_citations = getattr(self.instance.config, 'EMAIL_KEEP_QUOTED_TEXT',
780 'no') == 'yes'
781 keep_body = getattr(self.instance.config, 'EMAIL_LEAVE_BODY_UNCHANGED',
782 'no') == 'yes'
784 # parse the body of the message, stripping out bits as appropriate
785 summary, content = parseContent(content, keep_citations,
786 keep_body)
787 content = content.strip()
789 #
790 # handle the attachments
791 #
792 if properties.has_key('files'):
793 files = []
794 for (name, mime_type, data) in attachments:
795 if not name:
796 name = "unnamed"
797 files.append(self.db.file.create(type=mime_type, name=name,
798 content=data, **file_props))
799 # attach the files to the issue
800 if nodeid:
801 # extend the existing files list
802 fileprop = cl.get(nodeid, 'files')
803 fileprop.extend(files)
804 props['files'] = fileprop
805 else:
806 # pre-load the files list
807 props['files'] = files
809 #
810 # create the message if there's a message body (content)
811 #
812 if (content and properties.has_key('messages')):
813 message_id = self.db.msg.create(author=author,
814 recipients=recipients, date=date.Date('.'), summary=summary,
815 content=content, files=files, messageid=messageid,
816 inreplyto=inreplyto, **msg_props)
818 # attach the message to the node
819 if nodeid:
820 # add the message to the node's list
821 messages = cl.get(nodeid, 'messages')
822 messages.append(message_id)
823 props['messages'] = messages
824 else:
825 # pre-load the messages list
826 props['messages'] = [message_id]
828 #
829 # perform the node change / create
830 #
831 try:
832 # merge the command line props defined in issue_props into
833 # the props dictionary because function(**props, **issue_props)
834 # is a syntax error.
835 for prop in issue_props.keys() :
836 if not props.has_key(prop) :
837 props[prop] = issue_props[prop]
838 if nodeid:
839 cl.set(nodeid, **props)
840 else:
841 nodeid = cl.create(**props)
842 except (TypeError, IndexError, ValueError), message:
843 raise MailUsageError, '''
844 There was a problem with the message you sent:
845 %s
846 '''%message
848 # commit the changes to the DB
849 self.db.commit()
851 return nodeid
854 def setPropArrayFromString(self, cl, propString, nodeid = None):
855 ''' takes string of form prop=value,value;prop2=value
856 and returns (error, prop[..])
857 '''
858 properties = cl.getprops()
859 props = {}
860 errors = []
861 for prop in string.split(propString, ';'):
862 # extract the property name and value
863 try:
864 propname, value = prop.split('=')
865 except ValueError, message:
866 errors.append('not of form [arg=value,value,...;'
867 'arg=value,value,...]')
868 return (errors, props)
870 # ensure it's a valid property name
871 propname = propname.strip()
872 try:
873 proptype = properties[propname]
874 except KeyError:
875 errors.append('refers to an invalid property: "%s"'%propname)
876 continue
878 # convert the string value to a real property value
879 if isinstance(proptype, hyperdb.String):
880 props[propname] = value.strip()
881 if isinstance(proptype, hyperdb.Password):
882 props[propname] = password.Password(value.strip())
883 elif isinstance(proptype, hyperdb.Date):
884 try:
885 props[propname] = date.Date(value.strip()).local(self.db.getUserTimezone())
886 except ValueError, message:
887 errors.append('contains an invalid date for %s.'%propname)
888 elif isinstance(proptype, hyperdb.Interval):
889 try:
890 props[propname] = date.Interval(value)
891 except ValueError, message:
892 errors.append('contains an invalid date interval for %s.'%
893 propname)
894 elif isinstance(proptype, hyperdb.Link):
895 linkcl = self.db.classes[proptype.classname]
896 propkey = linkcl.labelprop(default_to_id=1)
897 try:
898 props[propname] = linkcl.lookup(value)
899 except KeyError, message:
900 errors.append('"%s" is not a value for %s.'%(value, propname))
901 elif isinstance(proptype, hyperdb.Multilink):
902 # get the linked class
903 linkcl = self.db.classes[proptype.classname]
904 propkey = linkcl.labelprop(default_to_id=1)
905 if nodeid:
906 curvalue = cl.get(nodeid, propname)
907 else:
908 curvalue = []
910 # handle each add/remove in turn
911 # keep an extra list for all items that are
912 # definitely in the new list (in case of e.g.
913 # <propname>=A,+B, which should replace the old
914 # list with A,B)
915 set = 0
916 newvalue = []
917 for item in value.split(','):
918 item = item.strip()
920 # handle +/-
921 remove = 0
922 if item.startswith('-'):
923 remove = 1
924 item = item[1:]
925 elif item.startswith('+'):
926 item = item[1:]
927 else:
928 set = 1
930 # look up the value
931 try:
932 item = linkcl.lookup(item)
933 except KeyError, message:
934 errors.append('"%s" is not a value for %s.'%(item,
935 propname))
936 continue
938 # perform the add/remove
939 if remove:
940 try:
941 curvalue.remove(item)
942 except ValueError:
943 errors.append('"%s" is not currently in for %s.'%(item,
944 propname))
945 continue
946 else:
947 newvalue.append(item)
948 if item not in curvalue:
949 curvalue.append(item)
951 # that's it, set the new Multilink property value,
952 # or overwrite it completely
953 if set:
954 props[propname] = newvalue
955 else:
956 props[propname] = curvalue
957 elif isinstance(proptype, hyperdb.Boolean):
958 value = value.strip()
959 props[propname] = value.lower() in ('yes', 'true', 'on', '1')
960 elif isinstance(proptype, hyperdb.Number):
961 value = value.strip()
962 props[propname] = float(value)
963 return errors, props
966 def extractUserFromList(userClass, users):
967 '''Given a list of users, try to extract the first non-anonymous user
968 and return that user, otherwise return None
969 '''
970 if len(users) > 1:
971 for user in users:
972 # make sure we don't match the anonymous or admin user
973 if userClass.get(user, 'username') in ('admin', 'anonymous'):
974 continue
975 # first valid match will do
976 return user
977 # well, I guess we have no choice
978 return user[0]
979 elif users:
980 return users[0]
981 return None
984 def uidFromAddress(db, address, create=1, **user_props):
985 ''' address is from the rfc822 module, and therefore is (name, addr)
987 user is created if they don't exist in the db already
988 user_props may supply additional user information
989 '''
990 (realname, address) = address
992 # try a straight match of the address
993 user = extractUserFromList(db.user, db.user.stringFind(address=address))
994 if user is not None:
995 return user
997 # try the user alternate addresses if possible
998 props = db.user.getprops()
999 if props.has_key('alternate_addresses'):
1000 users = db.user.filter(None, {'alternate_addresses': address})
1001 user = extractUserFromList(db.user, users)
1002 if user is not None:
1003 return user
1005 # try to match the username to the address (for local
1006 # submissions where the address is empty)
1007 user = extractUserFromList(db.user, db.user.stringFind(username=address))
1009 # couldn't match address or username, so create a new user
1010 if create:
1011 # generate a username
1012 if '@' in address:
1013 username = address.split('@')[0]
1014 else:
1015 username = address
1016 trying = username
1017 n = 0
1018 while 1:
1019 try:
1020 # does this username exist already?
1021 db.user.lookup(trying)
1022 except KeyError:
1023 break
1024 n += 1
1025 trying = username + str(n)
1027 # create!
1028 return db.user.create(username=trying, address=address,
1029 realname=realname, roles=db.config.NEW_EMAIL_USER_ROLES,
1030 password=password.Password(password.generatePassword()),
1031 **user_props)
1032 else:
1033 return 0
1036 def parseContent(content, keep_citations, keep_body,
1037 blank_line=re.compile(r'[\r\n]+\s*[\r\n]+'),
1038 eol=re.compile(r'[\r\n]+'),
1039 signature=re.compile(r'^[>|\s]*-- ?$'),
1040 original_msg=re.compile(r'^[>|\s]*-----\s?Original Message\s?-----$')):
1041 ''' The message body is divided into sections by blank lines.
1042 Sections where the second and all subsequent lines begin with a ">"
1043 or "|" character are considered "quoting sections". The first line of
1044 the first non-quoting section becomes the summary of the message.
1046 If keep_citations is true, then we keep the "quoting sections" in the
1047 content.
1048 If keep_body is true, we even keep the signature sections.
1049 '''
1050 # strip off leading carriage-returns / newlines
1051 i = 0
1052 for i in range(len(content)):
1053 if content[i] not in '\r\n':
1054 break
1055 if i > 0:
1056 sections = blank_line.split(content[i:])
1057 else:
1058 sections = blank_line.split(content)
1060 # extract out the summary from the message
1061 summary = ''
1062 l = []
1063 for section in sections:
1064 #section = section.strip()
1065 if not section:
1066 continue
1067 lines = eol.split(section)
1068 if (lines[0] and lines[0][0] in '>|') or (len(lines) > 1 and
1069 lines[1] and lines[1][0] in '>|'):
1070 # see if there's a response somewhere inside this section (ie.
1071 # no blank line between quoted message and response)
1072 for line in lines[1:]:
1073 if line and line[0] not in '>|':
1074 break
1075 else:
1076 # we keep quoted bits if specified in the config
1077 if keep_citations:
1078 l.append(section)
1079 continue
1080 # keep this section - it has reponse stuff in it
1081 lines = lines[lines.index(line):]
1082 section = '\n'.join(lines)
1083 # and while we're at it, use the first non-quoted bit as
1084 # our summary
1085 summary = section
1087 if not summary:
1088 # if we don't have our summary yet use the first line of this
1089 # section
1090 summary = section
1091 elif signature.match(lines[0]) and 2 <= len(lines) <= 10:
1092 # lose any signature
1093 break
1094 elif original_msg.match(lines[0]):
1095 # ditch the stupid Outlook quoting of the entire original message
1096 break
1098 # and add the section to the output
1099 l.append(section)
1101 # figure the summary - find the first sentence-ending punctuation or the
1102 # first whole line, whichever is longest
1103 sentence = re.search(r'^([^!?\.]+[!?\.])', summary)
1104 if sentence:
1105 sentence = sentence.group(1)
1106 else:
1107 sentence = ''
1108 first = eol.split(summary)[0]
1109 summary = max(sentence, first)
1111 # Now reconstitute the message content minus the bits we don't care
1112 # about.
1113 if not keep_body:
1114 content = '\n\n'.join(l)
1116 return summary, content
1118 # vim: set filetype=python ts=4 sw=4 et si