Code

Fix mailer (sf bug #817470) and add docstrings to prevent this from happening again.
[roundup.git] / roundup / cgi / client.py
index 23edf8fdacb330c5cfbe7159b3096f5db8405fb1..d4499bb334839aef8aa2abe05e2a15eebff01ca7 100644 (file)
@@ -1,34 +1,51 @@
-# $Id: client.py,v 1.87 2003-02-17 00:39:28 richard Exp $
+# $Id: client.py,v 1.141 2003-10-04 11:21:47 jlgijsbers Exp $
 
 __doc__ = """
 WWW request handler (also used in the stand-alone server).
 """
 
 import os, os.path, cgi, StringIO, urlparse, re, traceback, mimetypes, urllib
-import binascii, Cookie, time, random
+import binascii, Cookie, time, random, MimeWriter, smtplib, socket, quopri
+import stat, rfc822
 
-from roundup import roundupdb, date, hyperdb, password
+from roundup import roundupdb, date, hyperdb, password, token, rcsv
 from roundup.i18n import _
-
 from roundup.cgi.templating import Templates, HTMLRequest, NoTemplate
 from roundup.cgi import cgitb
-
 from roundup.cgi.PageTemplates import PageTemplate
-
-class Unauthorised(ValueError):
-    pass
-
-class NotFound(ValueError):
-    pass
-
-class Redirect(Exception):
+from roundup.rfc2822 import encode_header
+from roundup.mailgw import uidFromAddress
+from roundup.mailer import Mailer, MessageSendError
+
+class HTTPException(Exception):
+      pass
+class  Unauthorised(HTTPException):
+       pass
+class  NotFound(HTTPException):
+       pass
+class  Redirect(HTTPException):
+       pass
+class  NotModified(HTTPException):
+       pass
+
+# used by a couple of routines
+chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
+
+class FormError(ValueError):
+    ''' An "expected" exception occurred during form parsing.
+        - ie. something we know can go wrong, and don't want to alarm the
+          user with
+
+        We trap this at the user interface level and feed back a nice error
+        to the user.
+    '''
     pass
 
 class SendFile(Exception):
-    ' Sent a file from the database '
+    ''' Send a file from the database '''
 
 class SendStaticFile(Exception):
-    ' Send a static file from the instance html directory '
+    ''' Send a static file from the instance html directory '''
 
 def initialiseSecurity(security):
     ''' Create some Permissions and Roles on the security object
@@ -47,6 +64,19 @@ def initialiseSecurity(security):
         description="User may manipulate user Roles through the web")
     security.addPermissionToRole('Admin', p)
 
+# used to clean messages passed through CGI variables - HTML-escape any tag
+# that isn't <a href="">, <i>, <b> and <br> (including XHTML variants) so
+# that people can't pass through nasties like <script>, <iframe>, ...
+CLEAN_MESSAGE_RE = r'(<(/?(.*?)(\s*href="[^"]")?\s*/?)>)'
+def clean_message(message, mc=re.compile(CLEAN_MESSAGE_RE, re.I)):
+    return mc.sub(clean_message_callback, message)
+def clean_message_callback(match, ok={'a':1,'i':1,'b':1,'br':1}):
+    ''' Strip all non <a>,<i>,<b> and <br> tags from a string
+    '''
+    if ok.has_key(match.group(3).lower()):
+        return match.group(1)
+    return '&lt;%s&gt;'%match.group(2)
+
 class Client:
     ''' Instantiate to handle one CGI request.
 
@@ -93,16 +123,29 @@ class Client:
     FV_OK_MESSAGE = re.compile(r'[@:]ok_message')
     FV_ERROR_MESSAGE = re.compile(r'[@:]error_message')
 
-    # specials for parsePropsFromForm
-    FV_REQUIRED = re.compile(r'[@:]required')
-    FV_ADD = re.compile(r'([@:])add\1')
-    FV_REMOVE = re.compile(r'([@:])remove\1')
-    FV_CONFIRM = re.compile(r'.+[@:]confirm')
-    FV_LINK = re.compile(r'([@:])link\1(.+)')
-
-    # deprecated
-    FV_NOTE = re.compile(r'[@:]note')
-    FV_FILE = re.compile(r'[@:]file')
+    FV_QUERYNAME = re.compile(r'[@:]queryname')
+
+    # edit form variable handling (see unit tests)
+    FV_LABELS = r'''
+       ^(
+         (?P<note>[@:]note)|
+         (?P<file>[@:]file)|
+         (
+          ((?P<classname>%s)(?P<id>[-\d]+))?  # optional leading designator
+          ((?P<required>[@:]required$)|       # :required
+           (
+            (
+             (?P<add>[@:]add[@:])|            # :add:<prop>
+             (?P<remove>[@:]remove[@:])|      # :remove:<prop>
+             (?P<confirm>[@:]confirm[@:])|    # :confirm:<prop>
+             (?P<link>[@:]link[@:])|          # :link:<prop>
+             ([@:])                           # just a separator
+            )?
+            (?P<propname>[^@:]+)             # <prop>
+           )
+          )
+         )
+        )$'''
 
     # Note: index page stuff doesn't appear here:
     # columns, sort, sortdir, filter, group, groupdir, search_text,
@@ -113,6 +156,7 @@ class Client:
         self.instance = instance
         self.request = request
         self.env = env
+        self.mailer = Mailer(instance.config)
 
         # save off the path
         self.path = env['PATH_INFO']
@@ -185,19 +229,28 @@ class Client:
         self.ok_message = []
         self.error_message = []
         try:
-            # make sure we're identified (even anonymously)
-            self.determine_user()
             # figure out the context and desired content template
+            # do this first so we don't authenticate for static files
+            # Note: this method opens the database as "admin" in order to
+            # perform context checks
             self.determine_context()
+
+            # make sure we're identified (even anonymously)
+            self.determine_user()
+
             # possibly handle a form submit action (may change self.classname
             # and self.template, and may also append error/ok_messages)
             self.handle_action()
-            # now render the page
 
+            # now render the page
             # we don't want clients caching our dynamic pages
             self.additional_headers['Cache-Control'] = 'no-cache'
-            self.additional_headers['Pragma'] = 'no-cache'
-            self.additional_headers['Expires'] = 'Thu, 1 Jan 1970 00:00:00 GMT'
+# Pragma: no-cache makes Mozilla and its ilk double-load all pages!!
+#            self.additional_headers['Pragma'] = 'no-cache'
+
+            # expire this page 5 seconds from now
+            date = rfc822.formatdate(time.time() + 5)
+            self.additional_headers['Expires'] = date
 
             # render the content
             self.write(self.renderContext())
@@ -212,22 +265,35 @@ class Client:
         except SendFile, designator:
             self.serve_file(designator)
         except SendStaticFile, file:
-            self.serve_static_file(str(file))
+            try:
+                self.serve_static_file(str(file))
+            except NotModified:
+                # send the 304 response
+                self.request.send_response(304)
+                self.request.end_headers()
         except Unauthorised, message:
-            self.classname=None
-            self.template=''
+            self.classname = None
+            self.template = ''
             self.error_message.append(message)
             self.write(self.renderContext())
         except NotFound:
             # pass through
             raise
+        except FormError, e:
+            self.error_message.append(_('Form Error: ') + str(e))
+            self.write(self.renderContext())
         except:
             # everything else
             self.write(cgitb.html())
 
     def clean_sessions(self):
-        '''age sessions, remove when they haven't been used for a week.
-        Do it only once an hour'''
+        ''' Age sessions, remove when they haven't been used for a week.
+        
+            Do it only once an hour.
+
+            Note: also cleans One Time Keys, and other "session" based
+            stuff.
+        '''
         sessions = self.db.sessions
         last_clean = sessions.get('last_clean', 'last_use') or 0
 
@@ -235,25 +301,33 @@ class Client:
         hour = 60*60
         now = time.time()
         if now - last_clean > hour:
-            # remove age sessions
+            # remove aged sessions
             for sessid in sessions.list():
                 interval = now - sessions.get(sessid, 'last_use')
                 if interval > week:
                     sessions.destroy(sessid)
+            # remove aged otks
+            otks = self.db.otks
+            for sessid in otks.list():
+                interval = now - otks.get(sessid, '__time')
+                if interval > week:
+                    otks.destroy(sessid)
             sessions.set('last_clean', last_use=time.time())
 
     def determine_user(self):
         ''' Determine who the user is
         '''
-        # determine the uid to use
+        # open the database as admin
         self.opendb('admin')
+
         # clean age sessions
         self.clean_sessions()
+
         # make sure we have the session Class
         sessions = self.db.sessions
 
         # look up the user session cookie
-        cookie = Cookie.Cookie(self.env.get('HTTP_COOKIE', ''))
+        cookie = Cookie.SimpleCookie(self.env.get('HTTP_COOKIE', ''))
         user = 'anonymous'
 
         # bump the "revision" of the cookie since the format changed
@@ -335,8 +409,10 @@ class Client:
                 template_override = self.form[key].value
             elif self.FV_OK_MESSAGE.match(key):
                 ok_message = self.form[key].value
+                ok_message = clean_message(ok_message)
             elif self.FV_ERROR_MESSAGE.match(key):
                 error_message = self.form[key].value
+                error_message = clean_message(error_message)
 
         # determine the classname and possibly nodeid
         path = self.path.split('/')
@@ -354,6 +430,9 @@ class Client:
                 # send the file identified by the designator in path[0]
                 raise SendFile, path[0]
 
+        # we need the db for further context stuff - open it as admin
+        self.opendb('admin')
+
         # see if we got a designator
         m = dre.match(self.classname)
         if m:
@@ -394,16 +473,38 @@ class Client:
             raise NotFound, designator
 
         # we just want to serve up the file named
+        self.opendb('admin')
         file = self.db.file
         self.additional_headers['Content-Type'] = file.get(nodeid, 'type')
         self.write(file.get(nodeid, 'content'))
 
     def serve_static_file(self, file):
+        ims = None
+        # see if there's an if-modified-since...
+        if hasattr(self.request, 'headers'):
+            ims = self.request.headers.getheader('if-modified-since')
+        elif self.env.has_key('HTTP_IF_MODIFIED_SINCE'):
+            # cgi will put the header in the env var
+            ims = self.env['HTTP_IF_MODIFIED_SINCE']
+        filename = os.path.join(self.instance.config.TEMPLATES, file)
+        lmt = os.stat(filename)[stat.ST_MTIME]
+        if ims:
+            ims = rfc822.parsedate(ims)[:6]
+            lmtt = time.gmtime(lmt)[:6]
+            if lmtt <= ims:
+                raise NotModified
+
         # we just want to serve up the file named
-        mt = mimetypes.guess_type(str(file))[0]
+        file = str(file)
+        mt = mimetypes.guess_type(file)[0]
+        if not mt:
+            if file.endswith('.css'):
+                mt = 'text/css'
+            else:
+                mt = 'text/plain'
         self.additional_headers['Content-Type'] = mt
-        self.write(open(os.path.join(self.instance.config.TEMPLATES,
-            file)).read())
+        self.additional_headers['Last-Modifed'] = rfc822.formatdate(lmt)
+        self.write(open(filename, 'rb').read())
 
     def renderContext(self):
         ''' Return a PageTemplate for the named page
@@ -429,9 +530,11 @@ class Client:
     # these are the actions that are available
     actions = (
         ('edit',     'editItemAction'),
-        ('editCSV',  'editCSVAction'),
+        ('editcsv',  'editCSVAction'),
         ('new',      'newItemAction'),
         ('register', 'registerAction'),
+        ('confrego', 'confRegoAction'),
+        ('passrst',  'passResetAction'),
         ('login',    'loginAction'),
         ('logout',   'logout_action'),
         ('search',   'searchAction'),
@@ -439,41 +542,31 @@ class Client:
         ('show',     'showAction'),
     )
     def handle_action(self):
-        ''' Determine whether there should be an _action called.
+        ''' Determine whether there should be an Action called.
 
             The action is defined by the form variable :action which
-            identifies the method on this object to call. The four basic
-            actions are defined in the "actions" sequence on this class:
-             "edit"      -> self.editItemAction
-             "new"       -> self.newItemAction
-             "register"  -> self.registerAction
-             "login"     -> self.loginAction
-             "logout"    -> self.logout_action
-             "search"    -> self.searchAction
-             "retire"    -> self.retireAction
+            identifies the method on this object to call. The actions
+            are defined in the "actions" sequence on this class.
         '''
-        if not self.form.has_key(':action'):
+        if self.form.has_key(':action'):
+            action = self.form[':action'].value.lower()
+        elif self.form.has_key('@action'):
+            action = self.form['@action'].value.lower()
+        else:
             return None
         try:
             # get the action, validate it
-            action = self.form[':action'].value
             for name, method in self.actions:
                 if name == action:
                     break
             else:
                 raise ValueError, 'No such action "%s"'%action
-
             # call the mapped action
             getattr(self, method)()
         except Redirect:
             raise
         except Unauthorised:
             raise
-        except:
-            self.db.rollback()
-            s = StringIO.StringIO()
-            traceback.print_exc(None, s)
-            self.error_message.append('<pre>%s</pre>'%cgi.escape(s.getvalue()))
 
     def write(self, content):
         if not self.headers_done:
@@ -635,36 +728,93 @@ class Client:
 
         return 1 on successful login
         '''
-        # create the new user
-        cl = self.db.user
-
-        # parse the props from the form
-        try:
-            props = self.parsePropsFromForm()
-        except (ValueError, KeyError), message:
-            self.error_message.append(_('Error: ') + str(message))
-            return
+        props = self.parsePropsFromForm()[0][('user', None)]
 
         # make sure we're allowed to register
         if not self.registerPermission(props):
             raise Unauthorised, _("You do not have permission to register")
 
-        # re-open the database as "admin"
-        if self.user != 'admin':
-            self.opendb('admin')
+        try:
+            self.db.user.lookup(props['username'])
+            self.error_message.append('Error: A user with the username "%s" '
+                'already exists'%props['username'])
+            return
+        except KeyError:
+            pass
+
+        # generate the one-time-key and store the props for later
+        otk = ''.join([random.choice(chars) for x in range(32)])
+        for propname, proptype in self.db.user.getprops().items():
+            value = props.get(propname, None)
+            if value is None:
+                pass
+            elif isinstance(proptype, hyperdb.Date):
+                props[propname] = str(value)
+            elif isinstance(proptype, hyperdb.Interval):
+                props[propname] = str(value)
+            elif isinstance(proptype, hyperdb.Password):
+                props[propname] = str(value)
+        props['__time'] = time.time()
+        self.db.otks.set(otk, **props)
+
+        # send the email
+        tracker_name = self.db.config.TRACKER_NAME
+        tracker_email = self.db.config.TRACKER_EMAIL
+        subject = 'Complete your registration to %s -- key %s' % (tracker_name,
+                                                                  otk)
+        body = """To complete your registration of the user "%(name)s" with
+%(tracker)s, please do one of the following:
+
+- send a reply to %(tracker_email)s and maintain the subject line as is (the
+reply's additional "Re:" is ok),
+
+- or visit the following URL:
+
+   %(url)s?@action=confrego&otk=%(otk)s
+""" % {'name': props['username'], 'tracker': tracker_name, 'url': self.base,
+       'otk': otk, 'tracker_email': tracker_email}
+        if not self.standard_message([props['address']], subject, body,
+                                     tracker_email):
+            return
+
+        # commit changes to the database
+        self.db.commit()
+
+        # redirect to the "you're almost there" page
+        raise Redirect, '%suser?@template=rego_progress'%self.base
+
+    def standard_message(self, to, subject, body, author=None):
+        try:
+            self.mailer.standard_message(to, subject, body, author)
+            return 1
+        except MessageSendError, e:
+            self.error_message.append(str(e))
             
-        # create the new user
-        cl = self.db.user
+    def registerPermission(self, props):
+        ''' Determine whether the user has permission to register
+
+            Base behaviour is to check the user has "Web Registration".
+        '''
+        # registration isn't allowed to supply roles
+        if props.has_key('roles'):
+            return 0
+        if self.db.security.hasPermission('Web Registration', self.userid):
+            return 1
+        return 0
+
+    def confRegoAction(self):
+        ''' Grab the OTK, use it to load up the new user details
+        '''
         try:
-            props['roles'] = self.instance.config.NEW_WEB_USER_ROLES
-            self.userid = cl.create(**props['user'])
-            self.db.commit()
+            # pull the rego information out of the otk database
+            self.userid = self.db.confirm_registration(self.form['otk'].value)
         except (ValueError, KeyError), message:
-            self.error_message.append(message)
+            # XXX: we need to make the "default" page be able to display errors!
+            self.error_message.append(str(message))
             return
-
+        
         # log the new user in
-        self.user = cl.get(self.userid, 'username')
+        self.user = self.db.user.get(self.userid, 'username')
         # re-open the database for real, using the user
         self.opendb(self.user)
 
@@ -679,47 +829,126 @@ class Client:
         # nice message
         message = _('You are now registered, welcome!')
 
-        # redirect to the item's edit page
-        raise Redirect, '%s%s%s?+ok_message=%s'%(
-            self.base, self.classname, self.userid,  urllib.quote(message))
+        # redirect to the user's page
+        raise Redirect, '%suser%s?@ok_message=%s'%(self.base,
+            self.userid, urllib.quote(message))
 
-    def registerPermission(self, props):
-        ''' Determine whether the user has permission to register
+    def passResetAction(self):
+        ''' Handle password reset requests.
 
-            Base behaviour is to check the user has "Web Registration".
+            Presence of either "name" or "address" generate email.
+            Presense of "otk" performs the reset.
         '''
-        # registration isn't allowed to supply roles
-        if props.has_key('roles'):
-            return 0
-        if self.db.security.hasPermission('Web Registration', self.userid):
-            return 1
-        return 0
+        if self.form.has_key('otk'):
+            # pull the rego information out of the otk database
+            otk = self.form['otk'].value
+            uid = self.db.otks.get(otk, 'uid')
+            if uid is None:
+                self.error_message.append('Invalid One Time Key!')
+                return
+
+            # re-open the database as "admin"
+            if self.user != 'admin':
+                self.opendb('admin')
+
+            # change the password
+            newpw = password.generatePassword()
+
+            cl = self.db.user
+# XXX we need to make the "default" page be able to display errors!
+            try:
+                # set the password
+                cl.set(uid, password=password.Password(newpw))
+                # clear the props from the otk database
+                self.db.otks.destroy(otk)
+                self.db.commit()
+            except (ValueError, KeyError), message:
+                self.error_message.append(str(message))
+                return
+
+            # user info
+            address = self.db.user.get(uid, 'address')
+            name = self.db.user.get(uid, 'username')
+
+            # send the email
+            tracker_name = self.db.config.TRACKER_NAME
+            subject = 'Password reset for %s'%tracker_name
+            body = '''
+The password has been reset for username "%(name)s".
+
+Your password is now: %(password)s
+'''%{'name': name, 'password': newpw}
+            if not self.standard_message([address], subject, body):
+                return
+
+            self.ok_message.append('Password reset and email sent to %s'%address)
+            return
+
+        # no OTK, so now figure the user
+        if self.form.has_key('username'):
+            name = self.form['username'].value
+            try:
+                uid = self.db.user.lookup(name)
+            except KeyError:
+                self.error_message.append('Unknown username')
+                return
+            address = self.db.user.get(uid, 'address')
+        elif self.form.has_key('address'):
+            address = self.form['address'].value
+            uid = uidFromAddress(self.db, ('', address), create=0)
+            if not uid:
+                self.error_message.append('Unknown email address')
+                return
+            name = self.db.user.get(uid, 'username')
+        else:
+            self.error_message.append('You need to specify a username '
+                'or address')
+            return
+
+        # generate the one-time-key and store the props for later
+        otk = ''.join([random.choice(chars) for x in range(32)])
+        self.db.otks.set(otk, uid=uid, __time=time.time())
+
+        # send the email
+        tracker_name = self.db.config.TRACKER_NAME
+        subject = 'Confirm reset of password for %s'%tracker_name
+        body = '''
+Someone, perhaps you, has requested that the password be changed for your
+username, "%(name)s". If you wish to proceed with the change, please follow
+the link below:
+
+  %(url)suser?@template=forgotten&@action=passrst&otk=%(otk)s
+
+You should then receive another email with the new password.
+'''%{'name': name, 'tracker': tracker_name, 'url': self.base, 'otk': otk}
+        if not self.standard_message([address], subject, body):
+            return
+
+        self.ok_message.append('Email sent to %s'%address)
 
     def editItemAction(self):
         ''' Perform an edit of an item in the database.
 
            See parsePropsFromForm and _editnodes for special variables
         '''
-        # parse the props from the form
-        try:
-            props, links = self.parsePropsFromForm()
-        except (ValueError, KeyError), message:
-            self.error_message.append(_('Error: ') + str(message))
-            return
+        props, links = self.parsePropsFromForm()
 
         # handle the props
         try:
             message = self._editnodes(props, links)
         except (ValueError, KeyError, IndexError), message:
-            self.error_message.append(_('Error: ') + str(message))
+            self.error_message.append(_('Apply Error: ') + str(message))
             return
 
         # commit now that all the tricky stuff is done
         self.db.commit()
 
         # redirect to the item's edit page
-        raise Redirect, '%s%s%s?+ok_message=%s'%(self.base, self.classname,
-            self.nodeid,  urllib.quote(message))
+        raise Redirect, '%s%s%s?@ok_message=%s&@template=%s'%(self.base,
+            self.classname, self.nodeid, urllib.quote(message),
+            urllib.quote(self.template))
+
+    newItemAction = editItemAction
 
     def editItemPermission(self, props):
         ''' Determine whether the user has permission to edit this item.
@@ -745,48 +974,6 @@ class Client:
             return 1
         return 0
 
-    def newItemAction(self):
-        ''' Add a new item to the database.
-
-            This follows the same form as the editItemAction, with the same
-            special form values.
-        '''
-        # parse the props from the form
-# XXX reinstate exception handling
-#        try:
-        if 1:
-            props, links = self.parsePropsFromForm()
-#        except (ValueError, KeyError), message:
-#            self.error_message.append(_('Error: ') + str(message))
-#            return
-
-        # handle the props - edit or create
-# XXX reinstate exception handling
-#        try:
-        if 1:
-            # create the context here
-            cn = self.classname
-            nid = self._createnode(cn, props[(cn, None)])
-            del props[(cn, None)]
-
-            extra = self._editnodes(props, links, {(cn, None): nid})
-            if extra:
-                extra = '<br>' + extra
-
-            # now do the rest
-            messages = '%s %s created'%(cn, nid) + extra
-#        except (ValueError, KeyError, IndexError), message:
-#            # these errors might just be indicative of user dumbness
-#            self.error_message.append(_('Error: ') + str(message))
-#            return
-
-        # commit now that all the tricky stuff is done
-        self.db.commit()
-
-        # redirect to the new item's page
-        raise Redirect, '%s%s%s?@ok_message=%s'%(self.base, self.classname,
-            nid, urllib.quote(messages))
-
     def newItemPermission(self, props):
         ''' Determine whether the user has permission to create (edit) this
             item.
@@ -803,6 +990,128 @@ class Client:
             return 1
         return 0
 
+
+    #
+    #  Utility methods for editing
+    #
+    def _editnodes(self, all_props, all_links, newids=None):
+        ''' Use the props in all_props to perform edit and creation, then
+            use the link specs in all_links to do linking.
+        '''
+        # figure dependencies and re-work links
+        deps = {}
+        links = {}
+        for cn, nodeid, propname, vlist in all_links:
+            if not all_props.has_key((cn, nodeid)):
+                # link item to link to doesn't (and won't) exist
+                continue
+            for value in vlist:
+                if not all_props.has_key(value):
+                    # link item to link to doesn't (and won't) exist
+                    continue
+                deps.setdefault((cn, nodeid), []).append(value)
+                links.setdefault(value, []).append((cn, nodeid, propname))
+
+        # figure chained dependencies ordering
+        order = []
+        done = {}
+        # loop detection
+        change = 0
+        while len(all_props) != len(done):
+            for needed in all_props.keys():
+                if done.has_key(needed):
+                    continue
+                tlist = deps.get(needed, [])
+                for target in tlist:
+                    if not done.has_key(target):
+                        break
+                else:
+                    done[needed] = 1
+                    order.append(needed)
+                    change = 1
+            if not change:
+                raise ValueError, 'linking must not loop!'
+
+        # now, edit / create
+        m = []
+        for needed in order:
+            props = all_props[needed]
+            if not props:
+                # nothing to do
+                continue
+            cn, nodeid = needed
+
+            if nodeid is not None and int(nodeid) > 0:
+                # make changes to the node
+                props = self._changenode(cn, nodeid, props)
+
+                # and some nice feedback for the user
+                if props:
+                    info = ', '.join(props.keys())
+                    m.append('%s %s %s edited ok'%(cn, nodeid, info))
+                else:
+                    m.append('%s %s - nothing changed'%(cn, nodeid))
+            else:
+                assert props
+
+                # make a new node
+                newid = self._createnode(cn, props)
+                if nodeid is None:
+                    self.nodeid = newid
+                nodeid = newid
+
+                # and some nice feedback for the user
+                m.append('%s %s created'%(cn, newid))
+
+            # fill in new ids in links
+            if links.has_key(needed):
+                for linkcn, linkid, linkprop in links[needed]:
+                    props = all_props[(linkcn, linkid)]
+                    cl = self.db.classes[linkcn]
+                    propdef = cl.getprops()[linkprop]
+                    if not props.has_key(linkprop):
+                        if linkid is None or linkid.startswith('-'):
+                            # linking to a new item
+                            if isinstance(propdef, hyperdb.Multilink):
+                                props[linkprop] = [newid]
+                            else:
+                                props[linkprop] = newid
+                        else:
+                            # linking to an existing item
+                            if isinstance(propdef, hyperdb.Multilink):
+                                existing = cl.get(linkid, linkprop)[:]
+                                existing.append(nodeid)
+                                props[linkprop] = existing
+                            else:
+                                props[linkprop] = newid
+
+        return '<br>'.join(m)
+
+    def _changenode(self, cn, nodeid, props):
+        ''' change the node based on the contents of the form
+        '''
+        # check for permission
+        if not self.editItemPermission(props):
+            raise Unauthorised, 'You do not have permission to edit %s'%cn
+
+        # make the changes
+        cl = self.db.classes[cn]
+        return cl.set(nodeid, **props)
+
+    def _createnode(self, cn, props):
+        ''' create a node based on the contents of the form
+        '''
+        # check for permission
+        if not self.newItemPermission(props):
+            raise Unauthorised, 'You do not have permission to create %s'%cn
+
+        # create the node and return its id
+        cl = self.db.classes[cn]
+        return cl.create(**props)
+
+    # 
+    # More actions
+    #
     def editCSVAction(self):
         ''' Performs an edit of all of a class' items in one go.
 
@@ -816,12 +1125,8 @@ class Client:
                 _('You do not have permission to edit %s' %self.classname))
 
         # get the CSV module
-        try:
-            import csv
-        except ImportError:
-            self.error_message.append(_(
-                'Sorry, you need the csv module to use this function.<br>\n'
-                'Get it from: <a href="http://www.object-craft.com.au/projects/csv/">http://www.object-craft.com.au/projects/csv/'))
+        if rcsv.error:
+            self.error_message.append(_(rcsv.error))
             return
 
         cl = self.db.classes[self.classname]
@@ -830,16 +1135,13 @@ class Client:
         props = ['id'] + idlessprops
 
         # do the edit
-        rows = self.form['rows'].value.splitlines()
-        p = csv.parser()
+        rows = StringIO.StringIO(self.form['rows'].value)
+        reader = rcsv.reader(rows, rcsv.comma_separated)
         found = {}
         line = 0
-        for row in rows[1:]:
+        for values in reader:
             line += 1
-            values = p.parse(row)
-            # not a complete row, keep going
-            if not values: continue
-
+            if line == 1: continue
             # skip property names header
             if values == props:
                 continue
@@ -848,6 +1150,12 @@ class Client:
             nodeid, values = values[0], values[1:]
             found[nodeid] = 1
 
+            # see if the node exists
+            if nodeid in ('x', 'X') or not cl.hasnode(nodeid):
+                exists = 0
+            else:
+                exists = 1
+
             # confirm correct weight
             if len(idlessprops) != len(values):
                 self.error_message.append(
@@ -857,16 +1165,33 @@ class Client:
             # extract the new values
             d = {}
             for name, value in zip(idlessprops, values):
+                prop = cl.properties[name]
                 value = value.strip()
                 # only add the property if it has a value
                 if value:
                     # if it's a multilink, split it
-                    if isinstance(cl.properties[name], hyperdb.Multilink):
+                    if isinstance(prop, hyperdb.Multilink):
                         value = value.split(':')
+                    elif isinstance(prop, hyperdb.Password):
+                        value = password.Password(value)
+                    elif isinstance(prop, hyperdb.Interval):
+                        value = date.Interval(value)
+                    elif isinstance(prop, hyperdb.Date):
+                        value = date.Date(value)
+                    elif isinstance(prop, hyperdb.Boolean):
+                        value = value.lower() in ('yes', 'true', 'on', '1')
+                    elif isinstance(prop, hyperdb.Number):
+                        value = float(value)
                     d[name] = value
+                elif exists:
+                    # nuke the existing value
+                    if isinstance(prop, hyperdb.Multilink):
+                        d[name] = []
+                    else:
+                        d[name] = None
 
             # perform the edit
-            if cl.hasnode(nodeid):
+            if exists:
                 # edit existing
                 cl.set(nodeid, **d)
             else:
@@ -893,15 +1218,17 @@ class Client:
             return 0
         return 1
 
-    def searchAction(self):
+    def searchAction(self, wcre=re.compile(r'[\s,]+')):
         ''' Mangle some of the form variables.
 
             Set the form ":filter" variable based on the values of the
             filter variables - if they're set to anything other than
             "dontcare" then add them to :filter.
 
-            Also handle the ":queryname" variable and save off the query to
+            Handle the ":queryname" variable and save off the query to
             the user's query list.
+
+            Split any String query values on whitespace and comma.
         '''
         # generic edit is per-class only
         if not self.searchPermission():
@@ -909,10 +1236,16 @@ class Client:
                 _('You do not have permission to search %s' %self.classname))
 
         # add a faked :filter form variable for each filtering prop
-# XXX migrate to new : @ + 
         props = self.db.classes[self.classname].getprops()
+        queryname = ''
         for key in self.form.keys():
-            if not props.has_key(key): continue
+            # special vars
+            if self.FV_QUERYNAME.match(key):
+                queryname = self.form[key].value.strip()
+                continue
+
+            if not props.has_key(key):
+                continue
             if isinstance(self.form[key], type([])):
                 # search for at least one entry which is not empty
                 for minifield in self.form[key]:
@@ -921,33 +1254,44 @@ class Client:
                 else:
                     continue
             else:
-                if not self.form[key].value: continue
-            self.form.value.append(cgi.MiniFieldStorage(':filter', key))
+                if not self.form[key].value:
+                    continue
+                if isinstance(props[key], hyperdb.String):
+                    v = self.form[key].value
+                    l = token.token_split(v)
+                    if len(l) > 1 or l[0] != v:
+                        self.form.value.remove(self.form[key])
+                        # replace the single value with the split list
+                        for v in l:
+                            self.form.value.append(cgi.MiniFieldStorage(key, v))
+
+            self.form.value.append(cgi.MiniFieldStorage('@filter', key))
 
         # handle saving the query params
-        if self.form.has_key(':queryname'):
-            queryname = self.form[':queryname'].value.strip()
-            if queryname:
-                # parse the environment and figure what the query _is_
-                req = HTMLRequest(self)
-                url = req.indexargs_href('', {})
-
-                # handle editing an existing query
-                try:
-                    qid = self.db.query.lookup(queryname)
-                    self.db.query.set(qid, klass=self.classname, url=url)
-                except KeyError:
-                    # create a query
-                    qid = self.db.query.create(name=queryname,
-                        klass=self.classname, url=url)
+        if queryname:
+            # parse the environment and figure what the query _is_
+            req = HTMLRequest(self)
 
-                    # and add it to the user's query multilink
-                    queries = self.db.user.get(self.userid, 'queries')
-                    queries.append(qid)
-                    self.db.user.set(self.userid, queries=queries)
+            # The [1:] strips off the '?' character, it isn't part of the
+            # query string.
+            url = req.indexargs_href('', {})[1:]
 
-                # commit the query change to the database
-                self.db.commit()
+            # handle editing an existing query
+            try:
+                qid = self.db.query.lookup(queryname)
+                self.db.query.set(qid, klass=self.classname, url=url)
+            except KeyError:
+                # create a query
+                qid = self.db.query.create(name=queryname,
+                    klass=self.classname, url=url)
+
+                # and add it to the user's query multilink
+                queries = self.db.user.get(self.userid, 'queries')
+                queries.append(qid)
+                self.db.user.set(self.userid, queries=queries)
+
+            # commit the query change to the database
+            self.db.commit()
 
     def searchPermission(self):
         ''' Determine whether the user has permission to search this class.
@@ -1001,136 +1345,154 @@ class Client:
         return 1
 
 
-    def showAction(self):
-        ''' Show a node
+    def showAction(self, typere=re.compile('[@:]type'),
+            numre=re.compile('[@:]number')):
+        ''' Show a node of a particular class/id
         '''
-# XXX allow : @ +
-        t = self.form[':type'].value
-        n = self.form[':number'].value
+        t = n = ''
+        for key in self.form.keys():
+            if typere.match(key):
+                t = self.form[key].value.strip()
+            elif numre.match(key):
+                n = self.form[key].value.strip()
+        if not t:
+            raise ValueError, 'Invalid %s number'%t
         url = '%s%s%s'%(self.db.config.TRACKER_WEB, t, n)
         raise Redirect, url
 
+    def parsePropsFromForm(self, num_re=re.compile('^\d+$')):
+        ''' Item properties and their values are edited with html FORM
+            variables and their values. You can:
+
+            - Change the value of some property of the current item.
+            - Create a new item of any class, and edit the new item's
+              properties,
+            - Attach newly created items to a multilink property of the
+              current item.
+            - Remove items from a multilink property of the current item.
+            - Specify that some properties are required for the edit
+              operation to be successful.
+
+            In the following, <bracketed> values are variable, "@" may be
+            either ":" or "@", and other text "required" is fixed.
+
+            Most properties are specified as form variables:
+
+             <propname>
+              - property on the current context item
+
+             <designator>"@"<propname>
+              - property on the indicated item (for editing related
+                information)
+
+            Designators name a specific item of a class.
+
+            <classname><N>
+
+                Name an existing item of class <classname>.
+
+            <classname>"-"<N>
+
+                Name the <N>th new item of class <classname>. If the form
+                submission is successful, a new item of <classname> is
+                created. Within the submitted form, a particular
+                designator of this form always refers to the same new
+                item.
+
+            Once we have determined the "propname", we look at it to see
+            if it's special:
+
+            @required
+                The associated form value is a comma-separated list of
+                property names that must be specified when the form is
+                submitted for the edit operation to succeed.  
+
+                When the <designator> is missing, the properties are
+                for the current context item.  When <designator> is
+                present, they are for the item specified by
+                <designator>.
+
+                The "@required" specifier must come before any of the
+                properties it refers to are assigned in the form.
+
+            @remove@<propname>=id(s) or @add@<propname>=id(s)
+                The "@add@" and "@remove@" edit actions apply only to
+                Multilink properties.  The form value must be a
+                comma-separate list of keys for the class specified by
+                the simple form variable.  The listed items are added
+                to (respectively, removed from) the specified
+                property.
+
+            @link@<propname>=<designator>
+                If the edit action is "@link@", the simple form
+                variable must specify a Link or Multilink property.
+                The form value is a comma-separated list of
+                designators.  The item corresponding to each
+                designator is linked to the property given by simple
+                form variable.  These are collected up and returned in
+                all_links.
+
+            None of the above (ie. just a simple form value)
+                The value of the form variable is converted
+                appropriately, depending on the type of the property.
+
+                For a Link('klass') property, the form value is a
+                single key for 'klass', where the key field is
+                specified in dbinit.py.  
+
+                For a Multilink('klass') property, the form value is a
+                comma-separated list of keys for 'klass', where the
+                key field is specified in dbinit.py.  
+
+                Note that for simple-form-variables specifiying Link
+                and Multilink properties, the linked-to class must
+                have a key field.
+
+                For a String() property specifying a filename, the
+                file named by the form value is uploaded. This means we
+                try to set additional properties "filename" and "type" (if
+                they are valid for the class).  Otherwise, the property
+                is set to the form value.
+
+                For Date(), Interval(), Boolean(), and Number()
+                properties, the form value is converted to the
+                appropriate
 
-    #
-    #  Utility methods for editing
-    #
-    def _editnodes(self, all_props, all_links, newids=None):
-        ''' Use the props in all_props to perform edit and creation, then
-            use the link specs in all_links to do linking.
-        '''
-        m = []
-        if newids is None:
-            newids = {}
-        for (cn, nodeid), props in all_props.items():
-            if int(nodeid) > 0:
-                # make changes to the node
-                props = self._changenode(cn, nodeid, props)
-
-                # and some nice feedback for the user
-                if props:
-                    info = ', '.join(props.keys())
-                    m.append('%s %s %s edited ok'%(cn, nodeid, info))
-                else:
-                    m.append('%s %s - nothing changed'%(cn, nodeid))
-            elif props:
-                # make a new node
-                newid = self._createnode(cn, props)
-                newids[(cn, nodeid)] = newid
-                nodeid = newid
-
-                # and some nice feedback for the user
-                m.append('%s %s created'%(cn, newid))
-
-        # handle linked nodes
-        keys = self.form.keys()
-        for cn, nodeid, propname, value in all_links:
-            cl = self.db.classes[cn]
-            property = cl.getprops()[propname]
-            if nodeid is None or nodeid.startswith('-'):
-                if not newids.has_key((cn, nodeid)):
-                    continue
-                nodeid = newids[(cn, nodeid)]
-
-            # map the desired classnames to their actual created ids
-            for link in value:
-                if not newids.has_key(link):
-                    continue
-                linkid = newids[link]
-                if isinstance(property, hyperdb.Multilink):
-                    # take a dupe of the list so we're not changing the cache
-                    existing = cl.get(nodeid, propname)[:]
-                    existing.append(linkid)
-                    cl.set(nodeid, **{propname: existing})
-                elif isinstance(property, hyperdb.Link):
-                    # make the Link set
-                    cl.set(nodeid, **{propname: linkid})
-                else:
-                    raise ValueError, '%s %s is not a link or multilink '\
-                        'property'%(cn, propname)
-                m.append('%s %s linked to <a href="%s%s">%s %s</a>'%(
-                    link[0], linkid, cn, nodeid, cn, nodeid))
-
-        return '<br>'.join(m)
+            Any of the form variables may be prefixed with a classname or
+            designator.
 
-    def _changenode(self, cn, nodeid, props):
-        ''' change the node based on the contents of the form
-        '''
-        # check for permission
-        if not self.editItemPermission(props):
-            raise PermissionError, 'You do not have permission to edit %s'%cn
+            Two special form values are supported for backwards
+            compatibility:
 
-        # make the changes
-        cl = self.db.classes[cn]
-        return cl.set(nodeid, **props)
+            @note
+                This is equivalent to::
 
-    def _createnode(self, cn, props):
-        ''' create a node based on the contents of the form
-        '''
-        # check for permission
-        if not self.newItemPermission(props):
-            raise PermissionError, 'You do not have permission to create %s'%cn
+                    @link@messages=msg-1
+                    @msg-1@content=value
 
-        # create the node and return its id
-        cl = self.db.classes[cn]
-        return cl.create(**props)
+                except that in addition, the "author" and "date"
+                properties of "msg-1" are set to the userid of the
+                submitter, and the current time, respectively.
 
-    def parsePropsFromForm(self, num_re=re.compile('^\d+$')):
-        ''' Pull properties for the given class out of the form.
+            @file
+                This is equivalent to::
 
-            In the following, <bracketed> values are variable, ":" may be
-            one of ":" or "@", and other text "required" is fixed.
+                    @link@files=file-1
+                    @file-1@content=value
 
-            Properties are specified as form variables
-             <designator>:<propname>
+                The String content value is handled as described above for
+                file uploads.
 
-            where the propery belongs to the context class or item if the
-            designator is not specified. The designator may specify a
-            negative item id value (ie. "issue-1") and a new item of the
-            specified class will be created for each negative id found.
+            If both the "@note" and "@file" form variables are
+            specified, the action::
 
-            If a "<designator>:required" parameter is supplied,
-            then the named property values must be supplied or a
-            ValueError will be raised.
+                    @link@msg-1@files=file-1
 
-            Other special form values:
-             [classname|designator]:remove:<propname>=id(s)
-              The ids will be removed from the multilink property.
-             [classname|designator]:add:<propname>=id(s)
-              The ids will be added to the multilink property.
+            is also performed.
 
-             [classname|designator]:link:<propname>=<classname>
-              Used to add a link to new items created during edit.
-              These are collected up and returned in all_links. This will
-              result in an additional linking operation (either Link set or
-              Multilink append) after the edit/create is done using
-              all_props in _editnodes. The <propname> on
-              [classname|designator] will be set/appended the id of the
-              newly created item of class <classname>.
-
-            Note: the colon may be either ":" or "@".
-
-            Any of the form variables may be prefixed with a classname or
-            designator.
+            We also check that FileClass items have a "content" property with
+            actual content, otherwise we remove them from all_props before
+            returning.
 
             The return from this method is a dict of 
                 (classname, id): properties
@@ -1139,27 +1501,17 @@ class Client:
             doesn't result in any changes would return {('issue','123'): {}})
             The id may be None, which indicates that an item should be
             created.
-
-            If a String property's form value is a file upload, then we
-            try to set additional properties "filename" and "type" (if
-            they are valid for the class).
-
-            Two special form values are supported for backwards
-            compatibility:
-             :note - create a message (with content, author and date), link
-                     to the context item
-             :file - create a file, attach to the current item and any
-                     message created by :note
         '''
         # some very useful variables
         db = self.db
         form = self.form
 
-        if not hasattr(self, 'FV_ITEMSPEC'):
-            # generate the regexp for detecting
-            # <classname|designator>[@:+]property
+        if not hasattr(self, 'FV_SPECIAL'):
+            # generate the regexp for handling special form values
             classes = '|'.join(db.classes.keys())
-            self.FV_ITEMSPEC = re.compile(r'(%s)([-\d]+)[@:](.+)$'%classes)
+            # specials for parsePropsFromForm
+            # handle the various forms (see unit tests)
+            self.FV_SPECIAL = re.compile(self.FV_LABELS%classes, re.VERBOSE)
             self.FV_DESIGNATOR = re.compile(r'(%s)([-\d]+)'%classes)
 
         # these indicate the default class / item
@@ -1168,8 +1520,9 @@ class Client:
         default_nodeid = self.nodeid
 
         # we'll store info about the individual class/item edit in these
-        all_required = {}       # one entry per class/item
-        all_props = {}          # one entry per class/item
+        all_required = {}       # required props per class/item
+        all_props = {}          # props to set per class/item
+        got_props = {}          # props received per class/item
         all_propdef = {}        # note - only one entry per class
         all_links = []          # as many as are required
 
@@ -1179,54 +1532,52 @@ class Client:
         keys = form.keys()
         timezone = db.getUserTimezone()
 
+        # sentinels for the :note and :file props
+        have_note = have_file = 0
+
+        # extract the usable form labels from the form
+        matches = []
         for key in keys:
-            # see if this value modifies a different item to the default
-            m = self.FV_ITEMSPEC.match(key)
+            m = self.FV_SPECIAL.match(key)
             if m:
+                matches.append((key, m.groupdict()))
+
+        # now handle the matches
+        for key, d in matches:
+            if d['classname']:
                 # we got a designator
-                cn = m.group(1)
+                cn = d['classname']
                 cl = self.db.classes[cn]
-                nodeid = m.group(2)
-                propname = m.group(3)
-            elif key == ':note':
-                # backwards compatibility: the special note field
+                nodeid = d['id']
+                propname = d['propname']
+            elif d['note']:
+                # the special note field
                 cn = 'msg'
                 cl = self.db.classes[cn]
                 nodeid = '-1'
                 propname = 'content'
                 all_links.append((default_cn, default_nodeid, 'messages',
                     [('msg', '-1')]))
-            elif key == ':file':
-                # backwards compatibility: the special file field
+                have_note = 1
+            elif d['file']:
+                # the special file field
                 cn = 'file'
                 cl = self.db.classes[cn]
                 nodeid = '-1'
                 propname = 'content'
                 all_links.append((default_cn, default_nodeid, 'files',
                     [('file', '-1')]))
-                if self.form.has_key(':note'):
-                    all_links.append(('msg', '-1', 'files', [('file', '-1')]))
+                have_file = 1
             else:
                 # default
                 cn = default_cn
                 cl = default_cl
                 nodeid = default_nodeid
-                propname = key
+                propname = d['propname']
 
             # the thing this value relates to is...
             this = (cn, nodeid)
 
-            # is this a link command?
-            if self.FV_LINK.match(propname):
-                value = []
-                for entry in extractFormList(form[key]):
-                    m = self.FV_DESIGNATOR.match(entry)
-                    if not m:
-                        raise ValueError, \
-                            'link "%s" value "%s" not a designator'%(key, entry)
-                    value.append((m.groups(1), m.groups(2)))
-                all_links.append((cn, nodeid, propname[6:], value))
-
             # get more info about the class, and the current set of
             # form props for it
             if not all_propdef.has_key(cn):
@@ -1235,32 +1586,48 @@ class Client:
             if not all_props.has_key(this):
                 all_props[this] = {}
             props = all_props[this]
+            if not got_props.has_key(this):
+                got_props[this] = {}
+
+            # is this a link command?
+            if d['link']:
+                value = []
+                for entry in extractFormList(form[key]):
+                    m = self.FV_DESIGNATOR.match(entry)
+                    if not m:
+                        raise FormError, \
+                            'link "%s" value "%s" not a designator'%(key, entry)
+                    value.append((m.group(1), m.group(2)))
+
+                # make sure the link property is valid
+                if (not isinstance(propdef[propname], hyperdb.Multilink) and
+                        not isinstance(propdef[propname], hyperdb.Link)):
+                    raise FormError, '%s %s is not a link or '\
+                        'multilink property'%(cn, propname)
+
+                all_links.append((cn, nodeid, propname, value))
+                continue
 
             # detect the special ":required" variable
-            if self.FV_REQUIRED.match(key):
+            if d['required']:
                 all_required[this] = extractFormList(form[key])
                 continue
 
-            # get the required values list
-            if not all_required.has_key(this):
-                all_required[this] = []
-            required = all_required[this]
-
             # see if we're performing a special multilink action
             mlaction = 'set'
-            if self.FV_REMOVE.match(propname):
-                propname = propname[8:]
+            if d['remove']:
                 mlaction = 'remove'
-            elif self.FV_ADD.match(propname):
-                propname = propname[5:]
+            elif d['add']:
                 mlaction = 'add'
 
             # does the property exist?
             if not propdef.has_key(propname):
                 if mlaction != 'set':
-                    raise ValueError, 'You have submitted a %s action for'\
+                    raise FormError, 'You have submitted a %s action for'\
                         ' the property "%s" which doesn\'t exist'%(mlaction,
                         propname)
+                # the form element is probably just something we don't care
+                # about - ignore it
                 continue
             proptype = propdef[propname]
 
@@ -1274,7 +1641,7 @@ class Client:
             else:
                 # multiple values are not OK
                 if isinstance(value, type([])):
-                    raise ValueError, 'You have submitted more than one value'\
+                    raise FormError, 'You have submitted more than one value'\
                         ' for the %s property'%propname
                 # value might be a file upload...
                 if not hasattr(value, 'filename') or value.filename is None:
@@ -1283,7 +1650,7 @@ class Client:
 
             # now that we have the props field, we need a teensy little
             # extra bit of help for the old :note field...
-            if key == ':note' and value:
+            if d['note'] and value:
                 props['author'] = self.db.getuid()
                 props['date'] = date.Date()
 
@@ -1292,18 +1659,18 @@ class Client:
                 if not value:
                     # ignore empty password values
                     continue
-                for key in keys:
-                    if self.FV_CONFIRM.match(key):
+                for key, d in matches:
+                    if d['confirm'] and d['propname'] == propname:
                         confirm = form[key]
                         break
                 else:
-                    raise ValueError, 'Password and confirmation text do '\
+                    raise FormError, 'Password and confirmation text do '\
                         'not match'
                 if isinstance(confirm, type([])):
-                    raise ValueError, 'You have submitted more than one value'\
+                    raise FormError, 'You have submitted more than one value'\
                         ' for the %s property'%propname
                 if value != confirm.value:
-                    raise ValueError, 'Password and confirmation text do '\
+                    raise FormError, 'Password and confirmation text do '\
                         'not match'
                 value = password.Password(value)
 
@@ -1321,12 +1688,12 @@ class Client:
                         try:
                             value = db.classes[link].lookup(value)
                         except KeyError:
-                            raise ValueError, _('property "%(propname)s": '
+                            raise FormError, _('property "%(propname)s": '
                                 '%(value)s not a %(classname)s')%{
                                 'propname': propname, 'value': value,
                                 'classname': link}
                         except TypeError, message:
-                            raise ValueError, _('you may only enter ID values '
+                            raise FormError, _('you may only enter ID values '
                                 'for property "%(propname)s": %(message)s')%{
                                 'propname': propname, 'message': message}
             elif isinstance(proptype, hyperdb.Multilink):
@@ -1340,12 +1707,12 @@ class Client:
                         try:
                             entry = link_cl.lookup(entry)
                         except KeyError:
-                            raise ValueError, _('property "%(propname)s": '
+                            raise FormError, _('property "%(propname)s": '
                                 '"%(value)s" not an entry of %(classname)s')%{
                                 'propname': propname, 'value': entry,
                                 'classname': link}
                         except TypeError, message:
-                            raise ValueError, _('you may only enter ID values '
+                            raise FormError, _('you may only enter ID values '
                                 'for property "%(propname)s": %(message)s')%{
                                 'propname': propname, 'message': message}
                     l.append(entry)
@@ -1371,7 +1738,7 @@ class Client:
                             try:
                                 existing.remove(entry)
                             except ValueError:
-                                raise ValueError, _('property "%(propname)s": '
+                                raise FormError, _('property "%(propname)s": '
                                     '"%(value)s" not currently in list')%{
                                     'propname': propname, 'value': entry}
                     else:
@@ -1389,36 +1756,45 @@ class Client:
                 # other types should be None'd if there's no value
                 value = None
             else:
-                if isinstance(proptype, hyperdb.String):
-                    if (hasattr(value, 'filename') and
-                            value.filename is not None):
-                        # skip if the upload is empty
-                        if not value.filename:
-                            continue
-                        # this String is actually a _file_
-                        # try to determine the file content-type
-                        filename = value.filename.split('\\')[-1]
-                        if propdef.has_key('name'):
-                            props['name'] = filename
-                        # use this info as the type/filename properties
-                        if propdef.has_key('type'):
-                            props['type'] = mimetypes.guess_type(filename)[0]
-                            if not props['type']:
-                                props['type'] = "application/octet-stream"
-                        # finally, read the content
-                        value = value.value
-                    else:
-                        # normal String fix the CRLF/CR -> LF stuff
-                        value = fixNewlines(value)
-
-                elif isinstance(proptype, hyperdb.Date):
-                    value = date.Date(value, offset=timezone)
-                elif isinstance(proptype, hyperdb.Interval):
-                    value = date.Interval(value)
-                elif isinstance(proptype, hyperdb.Boolean):
-                    value = value.lower() in ('yes', 'true', 'on', '1')
-                elif isinstance(proptype, hyperdb.Number):
-                    value = float(value)
+                # handle ValueErrors for all these in a similar fashion
+                try:
+                    if isinstance(proptype, hyperdb.String):
+                        if (hasattr(value, 'filename') and
+                                value.filename is not None):
+                            # skip if the upload is empty
+                            if not value.filename:
+                                continue
+                            # this String is actually a _file_
+                            # try to determine the file content-type
+                            fn = value.filename.split('\\')[-1]
+                            if propdef.has_key('name'):
+                                props['name'] = fn
+                            # use this info as the type/filename properties
+                            if propdef.has_key('type'):
+                                props['type'] = mimetypes.guess_type(fn)[0]
+                                if not props['type']:
+                                    props['type'] = "application/octet-stream"
+                            # finally, read the content
+                            value = value.value
+                        else:
+                            # normal String fix the CRLF/CR -> LF stuff
+                            value = fixNewlines(value)
+
+                    elif isinstance(proptype, hyperdb.Date):
+                        value = date.Date(value, offset=timezone)
+                    elif isinstance(proptype, hyperdb.Interval):
+                        value = date.Interval(value)
+                    elif isinstance(proptype, hyperdb.Boolean):
+                        value = value.lower() in ('yes', 'true', 'on', '1')
+                    elif isinstance(proptype, hyperdb.Number):
+                        value = float(value)
+                except ValueError, msg:
+                    raise FormError, _('Error with %s property: %s')%(
+                        propname, msg)
+
+            # register that we got this property
+            if value:
+                got_props[this][propname] = 1
 
             # get the old value
             if nodeid and not nodeid.startswith('-'):
@@ -1429,6 +1805,8 @@ class Client:
                     # no existing value
                     if not propdef.has_key(propname):
                         raise
+                except IndexError, message:
+                    raise FormError(str(message))
 
                 # make sure the existing multilink is sorted
                 if isinstance(proptype, hyperdb.Multilink):
@@ -1460,15 +1838,24 @@ class Client:
 
                 props[propname] = value
 
-            # register this as received if required?
-            if propname in required and value is not None:
-                required.remove(propname)
+        # check to see if we need to specially link a file to the note
+        if have_note and have_file:
+            all_links.append(('msg', '-1', 'files', [('file', '-1')]))
 
         # see if all the required properties have been supplied
         s = []
         for thing, required in all_required.items():
+            # register the values we got
+            got = got_props.get(thing, {})
+            for entry in required[:]:
+                if got.has_key(entry):
+                    required.remove(entry)
+
+            # any required values not present?
             if not required:
                 continue
+
+            # tell the user to entry the values required
             if len(required) > 1:
                 p = 'properties'
             else:
@@ -1476,8 +1863,19 @@ class Client:
             s.append('Required %s %s %s not supplied'%(thing[0], p,
                 ', '.join(required)))
         if s:
-            raise ValueError, '\n'.join(s)
-
+            raise FormError, '\n'.join(s)
+
+        # When creating a FileClass node, it should have a non-empty content
+        # property to be created. When editing a FileClass node, it should
+        # either have a non-empty content property or no property at all. In
+        # the latter case, nothing will change.
+        for (cn, id), props in all_props.items():
+            if isinstance(self.db.classes[cn], hyperdb.FileClass):
+                if id == '-1':
+                      if not props.get('content', ''):
+                            del all_props[(cn, id)]
+                elif props.has_key('content') and not props['content']:
+                      raise FormError, _('File is empty')
         return all_props, all_links
 
 def fixNewlines(text):
@@ -1494,18 +1892,20 @@ def extractFormList(value):
     ''' Extract a list of values from the form value.
 
         It may be one of:
-         [MiniFieldStorage, MiniFieldStorage, ...]
+         [MiniFieldStorage('value'), MiniFieldStorage('value','value',...), ...]
          MiniFieldStorage('value,value,...')
          MiniFieldStorage('value')
     '''
     # multiple values are OK
     if isinstance(value, type([])):
-        # it's a list of MiniFieldStorages
-        value = [i.value.strip() for i in value]
+        # it's a list of MiniFieldStorages - join then into
+        values = ','.join([i.value.strip() for i in value])
     else:
         # it's a MiniFieldStorage, but may be a comma-separated list
         # of values
-        value = [i.strip() for i in value.value.split(',')]
+        values = value.value
+
+    value = [i.strip() for i in values.split(',')]
 
     # filter out the empty bits
     return filter(None, value)