Code

Fix mailer (sf bug #817470) and add docstrings to prevent this from happening again.
[roundup.git] / roundup / mailgw.py
index bca88512ca436da354579c4f19f646c1e9cc3a64..8ec27fe983d0e0689a8dc9c233bfbff7369cd855 100644 (file)
@@ -16,7 +16,7 @@
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 #
 
-'''
+"""
 An e-mail gateway for Roundup.
 
 Incoming messages are examined for multiple parts:
@@ -73,13 +73,15 @@ are calling the create() method to create a new node). If an auditor raises
 an exception, the original message is bounced back to the sender with the
 explanatory message given in the exception. 
 
-$Id: mailgw.py,v 1.104 2003-01-06 21:28:38 richard Exp $
-'''
+$Id: mailgw.py,v 1.133 2003-10-04 11:21:47 jlgijsbers Exp $
+"""
 
 import string, re, os, mimetools, cStringIO, smtplib, socket, binascii, quopri
 import time, random, sys
-import traceback, MimeWriter
-import hyperdb, date, password
+import traceback, MimeWriter, rfc822
+
+from roundup import hyperdb, date, password, rfc2822
+from roundup.mailer import Mailer
 
 SENDMAILDEBUG = os.environ.get('SENDMAILDEBUG', '')
 
@@ -93,7 +95,7 @@ class MailUsageHelp(Exception):
     pass
 
 class MailLoop(Exception):
-    ''' We've seen this message before... '''
+    """ We've seen this message before... """
     pass
 
 class Unauthorized(Exception):
@@ -111,6 +113,26 @@ def initialiseSecurity(security):
         description="User may use the email interface")
     security.addPermissionToRole('Admin', p)
 
+def getparam(str, param):
+    ''' From the rfc822 "header" string, extract "param" if it appears.
+    '''
+    if ';' not in str:
+        return None
+    str = str[str.index(';'):]
+    while str[:1] == ';':
+        str = str[1:]
+        if ';' in str:
+            # XXX Should parse quotes!
+            end = str.index(';')
+        else:
+            end = len(str)
+        f = str[:end]
+        if '=' in f:
+            i = f.index('=')
+            if f[:i].strip().lower() == param:
+                return rfc822.unquote(f[i+1:].strip())
+    return None
+
 class Message(mimetools.Message):
     ''' subclass mimetools.Message so we can retrieve the parts of the
         message...
@@ -134,27 +156,45 @@ class Message(mimetools.Message):
         s.seek(0)
         return Message(s)
 
-subject_re = re.compile(r'(?P<refwd>\s*\W?\s*(fw|fwd|re|aw)\W\s*)*'
-    r'\s*(?P<quote>")?(\[(?P<classname>[^\d\s]+)(?P<nodeid>\d+)?\])?'
-    r'\s*(?P<title>[^[]+)?"?(\[(?P<args>.+?)\])?', re.I)
-
+    def getheader(self, name, default=None):
+        hdr = mimetools.Message.getheader(self, name, default)
+        if hdr:
+            hdr = hdr.replace('\n','') # Inserted by rfc822.readheaders
+        return rfc2822.decode_header(hdr)
 class MailGW:
-    def __init__(self, instance, db):
+
+    # Matches subjects like:
+    # Re: "[issue1234] title of issue [status=resolved]"
+    subject_re = re.compile(r'''
+        (?P<refwd>\s*\W?\s*(fw|fwd|re|aw)\W\s*)*\s*   # Re:
+        (?P<quote>")?                                 # Leading "
+        (\[(?P<classname>[^\d\s]+)                    # [issue..
+           (?P<nodeid>\d+)?                           # ..1234]
+         \])?\s*
+        (?P<title>[^[]+)?                             # issue title 
+        "?                                            # Trailing "
+        (\[(?P<args>.+?)\])?                          # [prop=value]
+        ''', re.IGNORECASE|re.VERBOSE)
+
+    def __init__(self, instance, db, arguments={}):
         self.instance = instance
         self.db = db
+        self.arguments = arguments
+        self.mailer = Mailer(instance.config)
 
         # should we trap exceptions (normal usage) or pass them through
         # (for testing)
         self.trapExceptions = 1
 
     def do_pipe(self):
-        ''' Read a message from standard input and pass it to the mail handler.
+        """ Read a message from standard input and pass it to the mail handler.
 
             Read into an internal structure that we can seek on (in case
             there's an error).
 
             XXX: we may want to read this into a temporary file instead...
-        '''
+        """
         s = cStringIO.StringIO()
         s.write(sys.stdin.read())
         s.seek(0)
@@ -162,11 +202,16 @@ class MailGW:
         return 0
 
     def do_mailbox(self, filename):
-        ''' Read a series of messages from the specified unix mailbox file and
+        """ Read a series of messages from the specified unix mailbox file and
             pass each to the mail handler.
-        '''
+        """
         # open the spool file and lock it
-        import fcntl, FCNTL
+        import fcntl
+        # FCNTL is deprecated in py2.3 and fcntl takes over all the symbols
+        if hasattr(fcntl, 'LOCK_EX'):
+            FCNTL = fcntl
+        else:
+            import FCNTL
         f = open(filename, 'r+')
         fcntl.flock(f.fileno(), FCNTL.LOCK_EX)
 
@@ -189,7 +234,12 @@ class MailGW:
         fcntl.flock(f.fileno(), FCNTL.LOCK_UN)
         return 0
 
-    def do_pop(self, server, user='', password=''):
+    def do_apop(self, server, user='', password=''):
+        ''' Do authentication POP
+        '''
+        self.do_pop(server, user, password, apop=1)
+
+    def do_pop(self, server, user='', password='', apop=0):
         '''Read a series of messages from the specified POP server.
         '''
         import getpass, poplib, socket
@@ -209,8 +259,11 @@ class MailGW:
         except socket.error, message:
             print "POP server error:", message
             return 1
-        server.user(user)
-        server.pass_(password)
+        if apop:
+            server.apop(user, password)
+        else:
+            server.user(user)
+            server.pass_(password)
         numMessages = len(server.list()[1])
         for i in range(1, numMessages+1):
             # retr: returns 
@@ -234,14 +287,14 @@ class MailGW:
         return self.handle_Message(Message(fp))
 
     def handle_Message(self, message):
-        '''Handle an RFC822 Message
+        """Handle an RFC822 Message
 
         Handle the Message object by calling handle_message() and then cope
         with any errors raised by handle_message.
         This method's job is to make that call and handle any
         errors in a sane manner. It should be replaced if you wish to
         handle errors in a different manner.
-        '''
+        """
         # in some rare cases, a particularly stuffed-up e-mail will make
         # its way into here... try to handle it gracefully
         sendto = message.getaddrlist('from')
@@ -257,7 +310,7 @@ class MailGW:
                 m = ['']
                 m.append('\n\nMail Gateway Help\n=================')
                 m.append(fulldoc)
-                m = self.bounce_message(message, sendto, m,
+                self.mailer.bounce_message(message, sendto, m,
                     subject="Mail Gateway Help")
             except MailUsageError, value:
                 # bounce the message back to the sender with the usage message
@@ -267,13 +320,13 @@ class MailGW:
                 m.append(str(value))
                 m.append('\n\nMail Gateway Help\n=================')
                 m.append(fulldoc)
-                m = self.bounce_message(message, sendto, m)
+                self.mailer.bounce_message(message, sendto, m)
             except Unauthorized, value:
                 # just inform the user that he is not authorized
                 sendto = [sendto[0][1]]
                 m = ['']
                 m.append(str(value))
-                m = self.bounce_message(message, sendto, m)
+                self.mailer.bounce_message(message, sendto, m)
             except MailLoop:
                 # XXX we should use a log file here...
                 return
@@ -290,7 +343,7 @@ class MailGW:
                 import traceback
                 traceback.print_exc(None, s)
                 m.append(s.getvalue())
-                m = self.bounce_message(message, sendto, m)
+                self.mailer.bounce_message(message, sendto, m)
         else:
             # very bad-looking message - we don't even know who sent it
             # XXX we should use a log file here...
@@ -301,64 +354,9 @@ class MailGW:
             m.append('line, indicating that it is corrupt. Please check your')
             m.append('mail gateway source. Failed message is attached.')
             m.append('')
-            m = self.bounce_message(message, sendto, m,
+            self.mailer.bounce_message(message, sendto, m,
                 subject='Badly formed message from mail gateway')
 
-        # now send the message
-        if SENDMAILDEBUG:
-            open(SENDMAILDEBUG, 'w').write('From: %s\nTo: %s\n%s\n'%(
-                self.instance.config.ADMIN_EMAIL, ', '.join(sendto),
-                    m.getvalue()))
-        else:
-            try:
-                smtp = smtplib.SMTP(self.instance.config.MAILHOST)
-                smtp.sendmail(self.instance.config.ADMIN_EMAIL, sendto,
-                    m.getvalue())
-            except socket.error, value:
-                raise MailGWError, "Couldn't send error email: "\
-                    "mailhost %s"%value
-            except smtplib.SMTPException, value:
-                raise MailGWError, "Couldn't send error email: %s"%value
-
-    def bounce_message(self, message, sendto, error,
-            subject='Failed issue tracker submission'):
-        ''' create a message that explains the reason for the failed
-            issue submission to the author and attach the original
-            message.
-        '''
-        msg = cStringIO.StringIO()
-        writer = MimeWriter.MimeWriter(msg)
-        writer.addheader('X-Roundup-Loop', 'hello')
-        writer.addheader('Subject', subject)
-        writer.addheader('From', '%s <%s>'% (self.instance.config.TRACKER_NAME,
-            self.instance.config.TRACKER_EMAIL))
-        writer.addheader('To', ','.join(sendto))
-        writer.addheader('Date', time.strftime("%a, %d %b %Y %H:%M:%S +0000",
-            time.gmtime()))
-        writer.addheader('MIME-Version', '1.0')
-        part = writer.startmultipartbody('mixed')
-        part = writer.nextpart()
-        body = part.startbody('text/plain')
-        body.write('\n'.join(error))
-
-        # attach the original message to the returned message
-        part = writer.nextpart()
-        part.addheader('Content-Disposition','attachment')
-        part.addheader('Content-Description','Message you sent')
-        body = part.startbody('text/plain')
-        for header in message.headers:
-            body.write(header)
-        body.write('\n')
-        try:
-            message.rewindbody()
-        except IOError, message:
-            body.write("*** couldn't include message body: %s ***"%message)
-        else:
-            body.write(message.fp.read())
-
-        writer.lastpart()
-        return msg
-
     def get_part_data_decoded(self,part):
         encoding = part.getencoding()
         data = None
@@ -376,7 +374,19 @@ class MailGW:
         else:
             # take it as text
             data = part.fp.read()
-        return data
+        
+        # Encode message to unicode
+        charset = rfc2822.unaliasCharset(part.getparam("charset"))
+        if charset:
+            # Do conversion only if charset specified
+            edata = unicode(data, charset).encode('utf-8')
+            # Convert from dos eol to unix
+            edata = edata.replace('\r\n', '\n')
+        else:
+            # Leave message content as is
+            edata = data
+                
+        return edata
 
     def handle_message(self, message):
         ''' message - a Message instance
@@ -387,41 +397,73 @@ class MailGW:
         if message.getheader('x-roundup-loop', ''):
             raise MailLoop
 
+        # XXX Don't enable. This doesn't work yet.
+#  "[^A-z.]tracker\+(?P<classname>[^\d\s]+)(?P<nodeid>\d+)\@some.dom.ain[^A-z.]"
+        # handle delivery to addresses like:tracker+issue25@some.dom.ain
+        # use the embedded issue number as our issue
+#        if hasattr(self.instance.config, 'EMAIL_ISSUE_ADDRESS_RE') and \
+#                self.instance.config.EMAIL_ISSUE_ADDRESS_RE:
+#            issue_re = self.instance.config.EMAIL_ISSUE_ADDRESS_RE
+#            for header in ['to', 'cc', 'bcc']:
+#                addresses = message.getheader(header, '')
+#            if addresses:
+#              # FIXME, this only finds the first match in the addresses.
+#                issue = re.search(issue_re, addresses, 'i')
+#                if issue:
+#                    classname = issue.group('classname')
+#                    nodeid = issue.group('nodeid')
+#                    break
+
         # handle the subject line
         subject = message.getheader('subject', '')
 
+        if not subject:
+            raise MailUsageError, '''
+Emails to Roundup trackers must include a Subject: line!
+'''
+
         if subject.strip().lower() == 'help':
             raise MailUsageHelp
 
-        m = subject_re.match(subject)
+        m = self.subject_re.match(subject)
 
         # check for well-formed subject line
         if m:
             # get the classname
             classname = m.group('classname')
             if classname is None:
-                # no classname, fallback on the default
-                if hasattr(self.instance.config, 'MAIL_DEFAULT_CLASS') and \
-                        self.instance.config.MAIL_DEFAULT_CLASS:
+                # no classname, check if this a registration confirmation email
+                # or fallback on the default class
+                otk_re = re.compile('-- key (?P<otk>[a-zA-Z0-9]{32})')
+                otk = otk_re.search(m.group('title'))
+                if otk:
+                    self.db.confirm_registration(otk.group('otk'))
+                    subject = 'Your registration to %s is complete' % \
+                              self.instance.config.TRACKER_NAME
+                    sendto = [message.getheader('from')]
+                    self.mailer.standard_message(sendto, subject, '') 
+                    return
+                elif hasattr(self.instance.config, 'MAIL_DEFAULT_CLASS') and \
+                         self.instance.config.MAIL_DEFAULT_CLASS:
                     classname = self.instance.config.MAIL_DEFAULT_CLASS
                 else:
                     # fail
                     m = None
 
         if not m:
-            raise MailUsageError, '''
+            raise MailUsageError, """
 The message you sent to roundup did not contain a properly formed subject
 line. The subject must contain a class name or designator to indicate the
-"topic" of the message. For example:
+'topic' of the message. For example:
     Subject: [issue] This is a new issue
-      - this will create a new issue in the tracker with the title "This is
-        a new issue".
+      - this will create a new issue in the tracker with the title 'This is
+        a new issue'.
     Subject: [issue1234] This is a followup to issue 1234
       - this will append the message's contents to the existing issue 1234
         in the tracker.
 
-Subject was: "%s"
-'''%subject
+Subject was: '%s'
+"""%subject
 
         # get the class
         try:
@@ -479,6 +521,48 @@ does not exist.
 Subject was: "%s"
 '''%(nodeid, subject)
 
+        # Handle the arguments specified by the email gateway command line.
+        # We do this by looping over the list of self.arguments looking for
+        # a -C to tell us what class then the -S setting string.
+        msg_props = {}
+        user_props = {}
+        file_props = {}
+        issue_props = {}
+        # so, if we have any arguments, use them
+        if self.arguments:
+            current_class = 'msg'
+            for option, propstring in self.arguments:
+                if option in ( '-C', '--class'):
+                    current_class = propstring.strip()
+                    if current_class not in ('msg', 'file', 'user', 'issue'):
+                        raise MailUsageError, '''
+The mail gateway is not properly set up. Please contact
+%s and have them fix the incorrect class specified as:
+  %s
+'''%(self.instance.config.ADMIN_EMAIL, current_class)
+                if option in ('-S', '--set'):
+                    if current_class == 'issue' :
+                        errors, issue_props = setPropArrayFromString(self,
+                            cl, propstring.strip(), nodeid)
+                    elif current_class == 'file' :
+                        temp_cl = self.db.getclass('file')
+                        errors, file_props = setPropArrayFromString(self,
+                            temp_cl, propstring.strip())
+                    elif current_class == 'msg' :
+                        temp_cl = self.db.getclass('msg')
+                        errors, msg_props = setPropArrayFromString(self,
+                            temp_cl, propstring.strip())
+                    elif current_class == 'user' :
+                        temp_cl = self.db.getclass('user')
+                        errors, user_props = setPropArrayFromString(self,
+                            temp_cl, propstring.strip())
+                    if errors:
+                        raise MailUsageError, '''
+The mail gateway is not properly set up. Please contact
+%s and have them fix the incorrect properties:
+  %s
+'''%(self.instance.config.ADMIN_EMAIL, errors)
+
         #
         # handle the users
         #
@@ -538,17 +622,12 @@ Unknown address: %s
 
             # look up the recipient - create if necessary (and we're
             # allowed to)
-            recipient = uidFromAddress(self.db, recipient, create)
+            recipient = uidFromAddress(self.db, recipient, create, **user_props)
 
             # if all's well, add the recipient to the list
             if recipient:
                 recipients.append(recipient)
 
-        #
-        # extract the args
-        #
-        subject_args = m.group('args')
-
         #
         # handle the subject argument list
         #
@@ -557,113 +636,7 @@ Unknown address: %s
         props = {}
         args = m.group('args')
         if args:
-            errors = []
-            for prop in string.split(args, ';'):
-                # extract the property name and value
-                try:
-                    propname, value = prop.split('=')
-                except ValueError, message:
-                    errors.append('not of form [arg=value,'
-                        'value,...;arg=value,value...]')
-                    break
-
-                # ensure it's a valid property name
-                propname = propname.strip()
-                try:
-                    proptype =  properties[propname]
-                except KeyError:
-                    errors.append('refers to an invalid property: '
-                        '"%s"'%propname)
-                    continue
-
-                # convert the string value to a real property value
-                if isinstance(proptype, hyperdb.String):
-                    props[propname] = value.strip()
-                if isinstance(proptype, hyperdb.Password):
-                    props[propname] = password.Password(value.strip())
-                elif isinstance(proptype, hyperdb.Date):
-                    try:
-                        props[propname] = date.Date(value.strip())
-                    except ValueError, message:
-                        errors.append('contains an invalid date for '
-                            '%s.'%propname)
-                elif isinstance(proptype, hyperdb.Interval):
-                    try:
-                        props[propname] = date.Interval(value)
-                    except ValueError, message:
-                        errors.append('contains an invalid date interval'
-                            'for %s.'%propname)
-                elif isinstance(proptype, hyperdb.Link):
-                    linkcl = self.db.classes[proptype.classname]
-                    propkey = linkcl.labelprop(default_to_id=1)
-                    try:
-                        props[propname] = linkcl.lookup(value)
-                    except KeyError, message:
-                        errors.append('"%s" is not a value for %s.'%(value,
-                            propname))
-                elif isinstance(proptype, hyperdb.Multilink):
-                    # get the linked class
-                    linkcl = self.db.classes[proptype.classname]
-                    propkey = linkcl.labelprop(default_to_id=1)
-                    if nodeid:
-                        curvalue = cl.get(nodeid, propname)
-                    else:
-                        curvalue = []
-
-                    # handle each add/remove in turn
-                    # keep an extra list for all items that are
-                    # definitely in the new list (in case of e.g.
-                    # <propname>=A,+B, which should replace the old
-                    # list with A,B)
-                    set = 0
-                    newvalue = []
-                    for item in value.split(','):
-                        item = item.strip()
-
-                        # handle +/-
-                        remove = 0
-                        if item.startswith('-'):
-                            remove = 1
-                            item = item[1:]
-                        elif item.startswith('+'):
-                            item = item[1:]
-                        else:
-                            set = 1
-
-                        # look up the value
-                        try:
-                            item = linkcl.lookup(item)
-                        except KeyError, message:
-                            errors.append('"%s" is not a value for %s.'%(item,
-                                propname))
-                            continue
-
-                        # perform the add/remove
-                        if remove:
-                            try:
-                                curvalue.remove(item)
-                            except ValueError:
-                                errors.append('"%s" is not currently in '
-                                    'for %s.'%(item, propname))
-                                continue
-                        else:
-                            newvalue.append(item)
-                            if item not in curvalue:
-                                curvalue.append(item)
-
-                    # that's it, set the new Multilink property value,
-                    # or overwrite it completely
-                    if set:
-                        props[propname] = newvalue
-                    else:
-                        props[propname] = curvalue
-                elif isinstance(proptype, hyperdb.Boolean):
-                    value = value.strip()
-                    props[propname] = value.lower() in ('yes', 'true', 'on', '1')
-                elif isinstance(proptype, hyperdb.Number):
-                    value = value.strip()
-                    props[propname] = int(value)
-
+            errors, props = setPropArrayFromString(self, cl, args, nodeid)
             # handle any errors parsing the argument list
             if errors:
                 errors = '\n- '.join(errors)
@@ -674,6 +647,11 @@ There were problems handling your subject line argument list:
 Subject was: "%s"
 '''%(errors, subject)
 
+
+        # set the issue title to the subject
+        if properties.has_key('title') and not issue_props.has_key('title'):
+            issue_props['title'] = title.strip()
+
         #
         # handle message-id and in-reply-to
         #
@@ -740,6 +718,7 @@ Subject was: "%s"
                 elif subtype == 'multipart/alternative':
                     # Search for text/plain in message with attachment and
                     # alternative text representation
+                    # skip over intro to first boundary
                     part.getPart()
                     while 1:
                         # get the next part
@@ -757,7 +736,7 @@ Subject was: "%s"
                     if not name:
                         disp = part.getheader('content-disposition', None)
                         if disp:
-                            name = disp.getparam('filename')
+                            name = getparam(disp, 'filename')
                             if name:
                                 name = name.strip()
                     # this is just an attachment
@@ -805,6 +784,7 @@ not find a text/plain part to use.
         # parse the body of the message, stripping out bits as appropriate
         summary, content = parseContent(content, keep_citations, 
             keep_body)
+        content = content.strip()
 
         # 
         # handle the attachments
@@ -814,7 +794,17 @@ not find a text/plain part to use.
             if not name:
                 name = "unnamed"
             files.append(self.db.file.create(type=mime_type, name=name,
-                content=data))
+                content=data, **file_props))
+        # attach the files to the issue
+        if nodeid:
+            # extend the existing files list
+            fileprop = cl.get(nodeid, 'files')
+            fileprop.extend(files)
+            props['files'] = fileprop
+        else:
+            # pre-load the files list
+            props['files'] = files
+
 
         # 
         # create the message if there's a message body (content)
@@ -823,7 +813,7 @@ not find a text/plain part to use.
             message_id = self.db.msg.create(author=author,
                 recipients=recipients, date=date.Date('.'), summary=summary,
                 content=content, files=files, messageid=messageid,
-                inreplyto=inreplyto)
+                inreplyto=inreplyto, **msg_props)
 
             # attach the message to the node
             if nodeid:
@@ -835,14 +825,16 @@ not find a text/plain part to use.
                 # pre-load the messages list
                 props['messages'] = [message_id]
 
-                # set the title to the subject
-                if properties.has_key('title') and not props.has_key('title'):
-                    props['title'] = title
-
         #
         # perform the node change / create
         #
         try:
+            # merge the command line props defined in issue_props into
+            # the props dictionary because function(**props, **issue_props)
+            # is a syntax error.
+            for prop in issue_props.keys() :
+                if not props.has_key(prop) :
+                    props[prop] = issue_props[prop]
             if nodeid:
                 cl.set(nodeid, **props)
             else:
@@ -858,6 +850,119 @@ There was a problem with the message you sent:
 
         return nodeid
 
+def setPropArrayFromString(self, cl, propString, nodeid = None):
+    ''' takes string of form prop=value,value;prop2=value
+        and returns (error, prop[..])
+    '''
+    properties = cl.getprops()
+    props = {}
+    errors = []
+    for prop in string.split(propString, ';'):
+        # extract the property name and value
+        try:
+            propname, value = prop.split('=')
+        except ValueError, message:
+            errors.append('not of form [arg=value,value,...;'
+                'arg=value,value,...]')
+            return (errors, props)
+
+        # ensure it's a valid property name
+        propname = propname.strip()
+        try:
+            proptype =  properties[propname]
+        except KeyError:
+            errors.append('refers to an invalid property: "%s"'%propname)
+            continue
+
+        # convert the string value to a real property value
+        if isinstance(proptype, hyperdb.String):
+            props[propname] = value.strip()
+        if isinstance(proptype, hyperdb.Password):
+            props[propname] = password.Password(value.strip())
+        elif isinstance(proptype, hyperdb.Date):
+            try:
+                props[propname] = date.Date(value.strip()).local(self.db.getUserTimezone())
+            except ValueError, message:
+                errors.append('contains an invalid date for %s.'%propname)
+        elif isinstance(proptype, hyperdb.Interval):
+            try:
+                props[propname] = date.Interval(value)
+            except ValueError, message:
+                errors.append('contains an invalid date interval for %s.'%
+                    propname)
+        elif isinstance(proptype, hyperdb.Link):
+            linkcl = self.db.classes[proptype.classname]
+            propkey = linkcl.labelprop(default_to_id=1)
+            try:
+                props[propname] = linkcl.lookup(value)
+            except KeyError, message:
+                errors.append('"%s" is not a value for %s.'%(value, propname))
+        elif isinstance(proptype, hyperdb.Multilink):
+            # get the linked class
+            linkcl = self.db.classes[proptype.classname]
+            propkey = linkcl.labelprop(default_to_id=1)
+            if nodeid:
+                curvalue = cl.get(nodeid, propname)
+            else:
+                curvalue = []
+
+            # handle each add/remove in turn
+            # keep an extra list for all items that are
+            # definitely in the new list (in case of e.g.
+            # <propname>=A,+B, which should replace the old
+            # list with A,B)
+            set = 0
+            newvalue = []
+            for item in value.split(','):
+                item = item.strip()
+
+                # handle +/-
+                remove = 0
+                if item.startswith('-'):
+                    remove = 1
+                    item = item[1:]
+                elif item.startswith('+'):
+                    item = item[1:]
+                else:
+                    set = 1
+
+                # look up the value
+                try:
+                    item = linkcl.lookup(item)
+                except KeyError, message:
+                    errors.append('"%s" is not a value for %s.'%(item,
+                        propname))
+                    continue
+
+                # perform the add/remove
+                if remove:
+                    try:
+                        curvalue.remove(item)
+                    except ValueError:
+                        errors.append('"%s" is not currently in for %s.'%(item,
+                            propname))
+                        continue
+                else:
+                    newvalue.append(item)
+                    if item not in curvalue:
+                        curvalue.append(item)
+
+            # that's it, set the new Multilink property value,
+            # or overwrite it completely
+            if set:
+                props[propname] = newvalue
+            else:
+                props[propname] = curvalue
+        elif isinstance(proptype, hyperdb.Boolean):
+            value = value.strip()
+            props[propname] = value.lower() in ('yes', 'true', 'on', '1')
+        elif isinstance(proptype, hyperdb.Number):
+            value = value.strip()
+            props[propname] = float(value)
+    return errors, props
+
+
 def extractUserFromList(userClass, users):
     '''Given a list of users, try to extract the first non-anonymous user
        and return that user, otherwise return None
@@ -875,23 +980,27 @@ def extractUserFromList(userClass, users):
         return users[0]
     return None
 
-def uidFromAddress(db, address, create=1):
+
+def uidFromAddress(db, address, create=1, **user_props):
     ''' address is from the rfc822 module, and therefore is (name, addr)
 
         user is created if they don't exist in the db already
+        user_props may supply additional user information
     '''
     (realname, address) = address
 
     # try a straight match of the address
     user = extractUserFromList(db.user, db.user.stringFind(address=address))
-    if user is not None: return user
+    if user is not None:
+        return user
 
     # try the user alternate addresses if possible
     props = db.user.getprops()
     if props.has_key('alternate_addresses'):
         users = db.user.filter(None, {'alternate_addresses': address})
         user = extractUserFromList(db.user, users)
-        if user is not None: return user
+        if user is not None:
+            return user
 
     # try to match the username to the address (for local
     # submissions where the address is empty)
@@ -899,11 +1008,31 @@ def uidFromAddress(db, address, create=1):
 
     # couldn't match address or username, so create a new user
     if create:
-        return db.user.create(username=address, address=address,
-            realname=realname, roles=db.config.NEW_EMAIL_USER_ROLES)
+        # generate a username
+        if '@' in address:
+            username = address.split('@')[0]
+        else:
+            username = address
+        trying = username
+        n = 0
+        while 1:
+            try:
+                # does this username exist already?
+                db.user.lookup(trying)
+            except KeyError:
+                break
+            n += 1
+            trying = username + str(n)
+
+        # create!
+        return db.user.create(username=trying, address=address,
+            realname=realname, roles=db.config.NEW_EMAIL_USER_ROLES,
+            password=password.Password(password.generatePassword()),
+            **user_props)
     else:
         return 0
 
+
 def parseContent(content, keep_citations, keep_body,
         blank_line=re.compile(r'[\r\n]+\s*[\r\n]+'),
         eol=re.compile(r'[\r\n]+'),