X-Git-Url: https://git.tokkee.org/?a=blobdiff_plain;f=roundup%2Fcgi%2Fclient.py;h=11a22172e9c1f69437c139afa66525d78d6986a1;hb=5bce0be3879628d6bfa0b6a0c844703c9de96b05;hp=f85d695fd09de7391a89eeac621d67cb98024d82;hpb=3bceec4a1de15fb4048c0eaac567de9b50b5f5c6;p=roundup.git

diff --git a/roundup/cgi/client.py b/roundup/cgi/client.py
index f85d695..11a2217 100644
--- a/roundup/cgi/client.py
+++ b/roundup/cgi/client.py
@@ -1,111 +1,255 @@
-# $Id: client.py,v 1.116 2003-05-09 03:32:41 richard Exp $
-
-__doc__ = """
-WWW request handler (also used in the stand-alone server).
+"""WWW request handler (also used in the stand-alone server).
 """
-
-import os, os.path, cgi, StringIO, urlparse, re, traceback, mimetypes, urllib
-import binascii, Cookie, time, random, MimeWriter, smtplib, socket, quopri
-import stat, rfc822, string
-
-from roundup import roundupdb, date, hyperdb, password, token
-from roundup.i18n import _
-from roundup.cgi.templating import Templates, HTMLRequest, NoTemplate
-from roundup.cgi import cgitb
-from roundup.cgi.PageTemplates import PageTemplate
-from roundup.rfc2822 import encode_header
-from roundup.mailgw import uidFromAddress, openSMTPConnection
-
-class HTTPException(Exception):
-      pass
-class  Unauthorised(HTTPException):
-       pass
-class  NotFound(HTTPException):
-       pass
-class  Redirect(HTTPException):
-       pass
-class  NotModified(HTTPException):
-       pass
-
-# set to indicate to roundup not to actually _send_ email
-# this var must contain a file to write the mail to
-SENDMAILDEBUG = os.environ.get('SENDMAILDEBUG', '')
-
-# used by a couple of routines
-chars = string.letters+string.digits
-
-# XXX actually _use_ FormError
-class FormError(ValueError):
-    ''' An "expected" exception occurred during form parsing.
-        - ie. something we know can go wrong, and don't want to alarm the
-          user with
-
-        We trap this at the user interface level and feed back a nice error
-        to the user.
-    '''
-    pass
-
-class SendFile(Exception):
-    ''' Send a file from the database '''
-
-class SendStaticFile(Exception):
-    ''' Send a static file from the instance html directory '''
+__docformat__ = 'restructuredtext'
+
+import base64, binascii, cgi, codecs, mimetypes, os
+import quopri, random, re, rfc822, stat, sys, time
+import socket, errno
+from traceback import format_exc
+
+from roundup import roundupdb, date, hyperdb, password
+from roundup.cgi import templating, cgitb, TranslationService
+from roundup.cgi.actions import *
+from roundup.exceptions import *
+from roundup.cgi.exceptions import *
+from roundup.cgi.form_parser import FormParser
+from roundup.mailer import Mailer, MessageSendError, encode_quopri
+from roundup.cgi import accept_language
+from roundup import xmlrpc
+
+from roundup.anypy.cookie_ import CookieError, BaseCookie, SimpleCookie, \
+    get_cookie_date
+from roundup.anypy.io_ import StringIO
+from roundup.anypy import http_
+from roundup.anypy import urllib_
+
+from email.MIMEBase import MIMEBase
+from email.MIMEText import MIMEText
+from email.MIMEMultipart import MIMEMultipart
 
 def initialiseSecurity(security):
-    ''' Create some Permissions and Roles on the security object
+    '''Create some Permissions and Roles on the security object
 
-        This function is directly invoked by security.Security.__init__()
-        as a part of the Security object instantiation.
+    This function is directly invoked by security.Security.__init__()
+    as a part of the Security object instantiation.
     '''
-    security.addPermission(name="Web Registration",
-        description="User may register through the web")
     p = security.addPermission(name="Web Access",
         description="User may access the web interface")
     security.addPermissionToRole('Admin', p)
 
     # doing Role stuff through the web - make sure Admin can
+    # TODO: deprecate this and use a property-based control
     p = security.addPermission(name="Web Roles",
         description="User may manipulate user Roles through the web")
     security.addPermissionToRole('Admin', p)
 
+# used to clean messages passed through CGI variables - HTML-escape any tag
+# that isn't <a href="">, <i>, <b> and <br> (including XHTML variants) so
+# that people can't pass through nasties like <script>, <iframe>, ...
+CLEAN_MESSAGE_RE = r'(<(/?(.*?)(\s*href="[^"]")?\s*/?)>)'
+def clean_message(message, mc=re.compile(CLEAN_MESSAGE_RE, re.I)):
+    return mc.sub(clean_message_callback, message)
+def clean_message_callback(match, ok={'a':1,'i':1,'b':1,'br':1}):
+    """ Strip all non <a>,<i>,<b> and <br> tags from a string
+    """
+    if match.group(3).lower() in ok:
+        return match.group(1)
+    return '&lt;%s&gt;'%match.group(2)
+
+
+error_message = ''"""<html><head><title>An error has occurred</title></head>
+<body><h1>An error has occurred</h1>
+<p>A problem was encountered processing your request.
+The tracker maintainers have been notified of the problem.</p>
+</body></html>"""
+
+
+class LiberalCookie(SimpleCookie):
+    """ Python's SimpleCookie throws an exception if the cookie uses invalid
+        syntax.  Other applications on the same server may have done precisely
+        this, preventing roundup from working through no fault of roundup.
+        Numerous other python apps have run into the same problem:
+
+        trac: http://trac.edgewall.org/ticket/2256
+        mailman: http://bugs.python.org/issue472646
+
+        This particular implementation comes from trac's solution to the
+        problem. Unfortunately it requires some hackery in SimpleCookie's
+        internals to provide a more liberal __set method.
+    """
+    def load(self, rawdata, ignore_parse_errors=True):
+        if ignore_parse_errors:
+            self.bad_cookies = []
+            self._BaseCookie__set = self._loose_set
+        SimpleCookie.load(self, rawdata)
+        if ignore_parse_errors:
+            self._BaseCookie__set = self._strict_set
+            for key in self.bad_cookies:
+                del self[key]
+
+    _strict_set = BaseCookie._BaseCookie__set
+
+    def _loose_set(self, key, real_value, coded_value):
+        try:
+            self._strict_set(key, real_value, coded_value)
+        except CookieError:
+            self.bad_cookies.append(key)
+            dict.__setitem__(self, key, None)
+
+
+class Session:
+    """
+    Needs DB to be already opened by client
+
+    Session attributes at instantiation:
+
+    - "client" - reference to client for add_cookie function
+    - "session_db" - session DB manager
+    - "cookie_name" - name of the cookie with session id
+    - "_sid" - session id for current user
+    - "_data" - session data cache
+
+    session = Session(client)
+    session.set(name=value)
+    value = session.get(name)
+
+    session.destroy()  # delete current session
+    session.clean_up() # clean up session table
+
+    session.update(set_cookie=True, expire=3600*24*365)
+                       # refresh session expiration time, setting persistent
+                       # cookie if needed to last for 'expire' seconds
+
+    """
+
+    def __init__(self, client):
+        self._data = {}
+        self._sid  = None
+
+        self.client = client
+        self.session_db = client.db.getSessionManager()
+
+        # parse cookies for session id
+        self.cookie_name = 'roundup_session_%s' % \
+            re.sub('[^a-zA-Z]', '', client.instance.config.TRACKER_NAME)
+        cookies = LiberalCookie(client.env.get('HTTP_COOKIE', ''))
+        if self.cookie_name in cookies:
+            if not self.session_db.exists(cookies[self.cookie_name].value):
+                self._sid = None
+                # remove old cookie
+                self.client.add_cookie(self.cookie_name, None)
+            else:
+                self._sid = cookies[self.cookie_name].value
+                self._data = self.session_db.getall(self._sid)
+
+    def _gen_sid(self):
+        """ generate a unique session key """
+        while 1:
+            s = '%s%s'%(time.time(), random.random())
+            s = binascii.b2a_base64(s).strip()
+            if not self.session_db.exists(s):
+                break
+
+        # clean up the base64
+        if s[-1] == '=':
+            if s[-2] == '=':
+                s = s[:-2]
+            else:
+                s = s[:-1]
+        return s
+
+    def clean_up(self):
+        """Remove expired sessions"""
+        self.session_db.clean()
+
+    def destroy(self):
+        self.client.add_cookie(self.cookie_name, None)
+        self._data = {}
+        self.session_db.destroy(self._sid)
+        self.client.db.commit()
+
+    def get(self, name, default=None):
+        return self._data.get(name, default)
+
+    def set(self, **kwargs):
+        self._data.update(kwargs)
+        if not self._sid:
+            self._sid = self._gen_sid()
+            self.session_db.set(self._sid, **self._data)
+            # add session cookie
+            self.update(set_cookie=True)
+
+            # XXX added when patching 1.4.4 for backward compatibility
+            # XXX remove
+            self.client.session = self._sid
+        else:
+            self.session_db.set(self._sid, **self._data)
+            self.client.db.commit()
+
+    def update(self, set_cookie=False, expire=None):
+        """ update timestamp in db to avoid expiration
+
+            if 'set_cookie' is True, set cookie with 'expire' seconds lifetime
+            if 'expire' is None - session will be closed with the browser
+             
+            XXX the session can be purged within a week even if a cookie
+                lifetime is longer
+        """
+        self.session_db.updateTimestamp(self._sid)
+        self.client.db.commit()
+
+        if set_cookie:
+            self.client.add_cookie(self.cookie_name, self._sid, expire=expire)
+
+
+
 class Client:
-    ''' Instantiate to handle one CGI request.
+    """Instantiate to handle one CGI request.
 
     See inner_main for request processing.
 
     Client attributes at instantiation:
-        "path" is the PATH_INFO inside the instance (with no leading '/')
-        "base" is the base URL for the instance
-        "form" is the cgi form, an instance of FieldStorage from the standard
-               cgi module
-        "additional_headers" is a dictionary of additional HTTP headers that
-               should be sent to the client
-        "response_code" is the HTTP response code to send to the client
+
+    - "path" is the PATH_INFO inside the instance (with no leading '/')
+    - "base" is the base URL for the instance
+    - "form" is the cgi form, an instance of FieldStorage from the standard
+      cgi module
+    - "additional_headers" is a dictionary of additional HTTP headers that
+      should be sent to the client
+    - "response_code" is the HTTP response code to send to the client
+    - "translator" is TranslationService instance
 
     During the processing of a request, the following attributes are used:
-        "error_message" holds a list of error messages
-        "ok_message" holds a list of OK messages
-        "session" is the current user session id
-        "user" is the current user's name
-        "userid" is the current user's id
-        "template" is the current :template context
-        "classname" is the current class context name
-        "nodeid" is the current context item id
+
+    - "db" 
+    - "error_message" holds a list of error messages
+    - "ok_message" holds a list of OK messages
+    - "session" is deprecated in favor of session_api (XXX remove)
+    - "session_api" is the interface to store data in session
+    - "user" is the current user's name
+    - "userid" is the current user's id
+    - "template" is the current :template context
+    - "classname" is the current class context name
+    - "nodeid" is the current context item id
 
     User Identification:
-     If the user has no login cookie, then they are anonymous and are logged
+     Users that are absent in session data are anonymous and are logged
      in as that user. This typically gives them all Permissions assigned to the
      Anonymous Role.
 
-     Once a user logs in, they are assigned a session. The Client instance
-     keeps the nodeid of the session as the "session" attribute.
-
+     Every user is assigned a session. "session_api" is the interface to work
+     with session data.
 
     Special form variables:
      Note that in various places throughout this code, special form
      variables of the form :<name> are used. The colon (":") part may
      actually be one of either ":" or "@".
-    '''
+    """
+
+    # charset used for data storage and form templates
+    # Note: must be in lower case for comparisons!
+    # XXX take this from instance.config?
+    STORAGE_CHARSET = 'utf-8'
 
     #
     # special form variables
@@ -114,39 +258,37 @@ class Client:
     FV_OK_MESSAGE = re.compile(r'[@:]ok_message')
     FV_ERROR_MESSAGE = re.compile(r'[@:]error_message')
 
-    FV_QUERYNAME = re.compile(r'[@:]queryname')
-
-    # edit form variable handling (see unit tests)
-    FV_LABELS = r'''
-       ^(
-         (?P<note>[@:]note)|
-         (?P<file>[@:]file)|
-         (
-          ((?P<classname>%s)(?P<id>[-\d]+))?  # optional leading designator
-          ((?P<required>[@:]required$)|       # :required
-           (
-            (
-             (?P<add>[@:]add[@:])|            # :add:<prop>
-             (?P<remove>[@:]remove[@:])|      # :remove:<prop>
-             (?P<confirm>[@:]confirm[@:])|    # :confirm:<prop>
-             (?P<link>[@:]link[@:])|          # :link:<prop>
-             ([@:])                           # just a separator
-            )?
-            (?P<propname>[^@:]+)             # <prop>
-           )
-          )
-         )
-        )$'''
-
     # Note: index page stuff doesn't appear here:
     # columns, sort, sortdir, filter, group, groupdir, search_text,
     # pagesize, startwith
 
-    def __init__(self, instance, request, env, form=None):
-        hyperdb.traceMark()
+    # list of network error codes that shouldn't be reported to tracker admin
+    # (error descriptions from FreeBSD intro(2))
+    IGNORE_NET_ERRORS = (
+        # A write on a pipe, socket or FIFO for which there is
+        # no process to read the data.
+        errno.EPIPE,
+        # A connection was forcibly closed by a peer.
+        # This normally results from a loss of the connection
+        # on the remote socket due to a timeout or a reboot.
+        errno.ECONNRESET,
+        # Software caused connection abort.  A connection abort
+        # was caused internal to your host machine.
+        errno.ECONNABORTED,
+        # A connect or send request failed because the connected party
+        # did not properly respond after a period of time.
+        errno.ETIMEDOUT,
+    )
+
+    def __init__(self, instance, request, env, form=None, translator=None):
+        # re-seed the random number generator
+        random.seed()
+        self.start = time.time()
         self.instance = instance
         self.request = request
         self.env = env
+        self.setTranslator(translator)
+        self.mailer = Mailer(instance.config)
 
         # save off the path
         self.path = env['PATH_INFO']
@@ -154,15 +296,20 @@ class Client:
         # this is the base URL for this tracker
         self.base = self.instance.config.TRACKER_WEB
 
+        # check the tracker_we setting
+        if not self.base.endswith('/'):
+            self.base = self.base + '/'
+
         # this is the "cookie path" for this tracker (ie. the path part of
         # the "base" url)
-        self.cookie_path = urlparse.urlparse(self.base)[2]
-        self.cookie_name = 'roundup_session_' + re.sub('[^a-zA-Z]', '',
-            self.instance.config.TRACKER_NAME)
+        self.cookie_path = urllib_.urlparse(self.base)[2]
+        # cookies to set in http responce
+        # {(path, name): (value, expire)}
+        self._cookies = {}
 
         # see if we need to re-parse the environment for the form (eg Zope)
         if form is None:
-            self.form = cgi.FieldStorage(environ=env)
+            self.form = cgi.FieldStorage(fp=request.rfile, environ=env)
         else:
             self.form = form
 
@@ -181,152 +328,396 @@ class Client:
         self.additional_headers = {}
         self.response_code = 200
 
+        # default character set
+        self.charset = self.STORAGE_CHARSET
+
+        # parse cookies (used for charset lookups)
+        # use our own LiberalCookie to handle bad apps on the same
+        # server that have set cookies that are out of spec
+        self.cookie = LiberalCookie(self.env.get('HTTP_COOKIE', ''))
+
+        self.user = None
+        self.userid = None
+        self.nodeid = None
+        self.classname = None
+        self.template = None
+
+    def setTranslator(self, translator=None):
+        """Replace the translation engine
+
+        'translator'
+           is TranslationService instance.
+           It must define methods 'translate' (TAL-compatible i18n),
+           'gettext' and 'ngettext' (gettext-compatible i18n).
+
+           If omitted, create default TranslationService.
+        """
+        if translator is None:
+            translator = TranslationService.get_translation(
+                language=self.instance.config["TRACKER_LANGUAGE"],
+                tracker_home=self.instance.config["TRACKER_HOME"])
+        self.translator = translator
+        self._ = self.gettext = translator.gettext
+        self.ngettext = translator.ngettext
 
     def main(self):
-        ''' Wrap the real main in a try/finally so we always close off the db.
-        '''
+        """ Wrap the real main in a try/finally so we always close off the db.
+        """
         try:
-            self.inner_main()
+            if self.env.get('CONTENT_TYPE') == 'text/xml':
+                self.handle_xmlrpc()
+            else:
+                self.inner_main()
         finally:
             if hasattr(self, 'db'):
                 self.db.close()
 
+
+    def handle_xmlrpc(self):
+
+        # Pull the raw XML out of the form.  The "value" attribute
+        # will be the raw content of the POST request.
+        assert self.form.file
+        input = self.form.value
+        # So that the rest of Roundup can query the form in the
+        # usual way, we create an empty list of fields.
+        self.form.list = []
+
+        # Set the charset and language, since other parts of
+        # Roundup may depend upon that.
+        self.determine_charset()
+        self.determine_language()
+        # Open the database as the correct user.
+        self.determine_user()
+        self.check_anonymous_access()
+
+        # Call the appropriate XML-RPC method.
+        handler = xmlrpc.RoundupDispatcher(self.db,
+                                           self.instance.actions,
+                                           self.translator,
+                                           allow_none=True)
+        output = handler.dispatch(input)
+
+        self.setHeader("Content-Type", "text/xml")
+        self.setHeader("Content-Length", str(len(output)))
+        self.write(output)
+        
     def inner_main(self):
-        ''' Process a request.
-
-            The most common requests are handled like so:
-            1. figure out who we are, defaulting to the "anonymous" user
-               see determine_user
-            2. figure out what the request is for - the context
-               see determine_context
-            3. handle any requested action (item edit, search, ...)
-               see handle_action
-            4. render a template, resulting in HTML output
-
-            In some situations, exceptions occur:
-            - HTTP Redirect  (generally raised by an action)
-            - SendFile       (generally raised by determine_context)
-              serve up a FileClass "content" property
-            - SendStaticFile (generally raised by determine_context)
-              serve up a file from the tracker "html" directory
-            - Unauthorised   (generally raised by an action)
-              the action is cancelled, the request is rendered and an error
-              message is displayed indicating that permission was not
-              granted for the action to take place
-            - NotFound       (raised wherever it needs to be)
-              percolates up to the CGI interface that called the client
-        '''
+        """Process a request.
+
+        The most common requests are handled like so:
+
+        1. look for charset and language preferences, set up user locale
+           see determine_charset, determine_language
+        2. figure out who we are, defaulting to the "anonymous" user
+           see determine_user
+        3. figure out what the request is for - the context
+           see determine_context
+        4. handle any requested action (item edit, search, ...)
+           see handle_action
+        5. render a template, resulting in HTML output
+
+        In some situations, exceptions occur:
+
+        - HTTP Redirect  (generally raised by an action)
+        - SendFile       (generally raised by determine_context)
+          serve up a FileClass "content" property
+        - SendStaticFile (generally raised by determine_context)
+          serve up a file from the tracker "html" directory
+        - Unauthorised   (generally raised by an action)
+          the action is cancelled, the request is rendered and an error
+          message is displayed indicating that permission was not
+          granted for the action to take place
+        - templating.Unauthorised   (templating action not permitted)
+          raised by an attempted rendering of a template when the user
+          doesn't have permission
+        - NotFound       (raised wherever it needs to be)
+          percolates up to the CGI interface that called the client
+        """
         self.ok_message = []
         self.error_message = []
         try:
-            # make sure we're identified (even anonymously)
-            self.determine_user()
-            # figure out the context and desired content template
-            self.determine_context()
-            # possibly handle a form submit action (may change self.classname
-            # and self.template, and may also append error/ok_messages)
-            self.handle_action()
-
-            # now render the page
-            # we don't want clients caching our dynamic pages
-            self.additional_headers['Cache-Control'] = 'no-cache'
-# Pragma: no-cache makes Mozilla and its ilk double-load all pages!!
-#            self.additional_headers['Pragma'] = 'no-cache'
-
-            # expire this page 5 seconds from now
-            date = rfc822.formatdate(time.time() + 5)
-            self.additional_headers['Expires'] = date
-
-            # render the content
-            self.write(self.renderContext())
+            self.determine_charset()
+            self.determine_language()
+
+            try:
+                # make sure we're identified (even anonymously)
+                self.determine_user()
+
+                # figure out the context and desired content template
+                self.determine_context()
+
+                # if we've made it this far the context is to a bit of
+                # Roundup's real web interface (not a file being served up)
+                # so do the Anonymous Web Acess check now
+                self.check_anonymous_access()
+
+                # possibly handle a form submit action (may change self.classname
+                # and self.template, and may also append error/ok_messages)
+                html = self.handle_action()
+
+                if html:
+                    self.write_html(html)
+                    return
+
+                # now render the page
+                # we don't want clients caching our dynamic pages
+                self.additional_headers['Cache-Control'] = 'no-cache'
+                # Pragma: no-cache makes Mozilla and its ilk
+                # double-load all pages!!
+                #            self.additional_headers['Pragma'] = 'no-cache'
+
+                # pages with messages added expire right now
+                # simple views may be cached for a small amount of time
+                # TODO? make page expire time configurable
+                # <rj> always expire pages, as IE just doesn't seem to do the
+                # right thing here :(
+                date = time.time() - 1
+                #if self.error_message or self.ok_message:
+                #    date = time.time() - 1
+                #else:
+                #    date = time.time() + 5
+                self.additional_headers['Expires'] = rfc822.formatdate(date)
+
+                # render the content
+                self.write_html(self.renderContext())
+            except SendFile, designator:
+                # The call to serve_file may result in an Unauthorised
+                # exception or a NotModified exception.  Those
+                # exceptions will be handled by the outermost set of
+                # exception handlers.
+                self.serve_file(designator)
+            except SendStaticFile, file:
+                self.serve_static_file(str(file))
+            except IOError:
+                # IOErrors here are due to the client disconnecting before
+                # recieving the reply.
+                pass
+
+        except SeriousError, message:
+            self.write_html(str(message))
         except Redirect, url:
             # let's redirect - if the url isn't None, then we need to do
             # the headers, otherwise the headers have been set before the
             # exception was raised
             if url:
-                self.additional_headers['Location'] = url
+                self.additional_headers['Location'] = str(url)
                 self.response_code = 302
-            self.write('Redirecting to <a href="%s">%s</a>'%(url, url))
-        except SendFile, designator:
-            self.serve_file(designator)
-        except SendStaticFile, file:
-            try:
-                self.serve_static_file(str(file))
-            except NotModified:
-                # send the 304 response
-                self.request.send_response(304)
-                self.request.end_headers()
+            self.write_html('Redirecting to <a href="%s">%s</a>'%(url, url))
+        except LoginError, message:
+            # The user tried to log in, but did not provide a valid
+            # username and password.  If we support HTTP
+            # authorization, send back a response that will cause the
+            # browser to prompt the user again.
+            if self.instance.config.WEB_HTTP_AUTH:
+                self.response_code = http_.client.UNAUTHORIZED
+                realm = self.instance.config.TRACKER_NAME
+                self.setHeader("WWW-Authenticate",
+                               "Basic realm=\"%s\"" % realm)
+            else:
+                self.response_code = http_.client.FORBIDDEN
+            self.renderFrontPage(message)
         except Unauthorised, message:
-            self.classname = None
-            self.template = ''
-            self.error_message.append(message)
-            self.write(self.renderContext())
-        except NotFound:
-            # pass through
-            raise
+            # users may always see the front page
+            self.response_code = 403
+            self.renderFrontPage(message)
+        except NotModified:
+            # send the 304 response
+            self.response_code = 304
+            self.header()
+        except NotFound, e:
+            self.response_code = 404
+            self.template = '404'
+            try:
+                cl = self.db.getclass(self.classname)
+                self.write_html(self.renderContext())
+            except KeyError:
+                # we can't map the URL to a class we know about
+                # reraise the NotFound and let roundup_server
+                # handle it
+                raise NotFound(e)
+        except FormError, e:
+            self.error_message.append(self._('Form Error: ') + str(e))
+            self.write_html(self.renderContext())
         except:
-            # everything else
-            self.write(cgitb.html())
+            # Something has gone badly wrong.  Therefore, we should
+            # make sure that the response code indicates failure.
+            if self.response_code == http_.client.OK:
+                self.response_code = http_.client.INTERNAL_SERVER_ERROR
+            # Help the administrator work out what went wrong.
+            html = ("<h1>Traceback</h1>"
+                    + cgitb.html(i18n=self.translator)
+                    + ("<h1>Environment Variables</h1><table>%s</table>"
+                       % cgitb.niceDict("", self.env)))
+            if not self.instance.config.WEB_DEBUG:
+                exc_info = sys.exc_info()
+                subject = "Error: %s" % exc_info[1]
+                self.send_error_to_admin(subject, html, format_exc())
+                self.write_html(self._(error_message))
+            else:
+                self.write_html(html)
 
     def clean_sessions(self):
-        ''' Age sessions, remove when they haven't been used for a week.
-        
-            Do it only once an hour.
+        """Deprecated
+           XXX remove
+        """
+        self.clean_up()
 
-            Note: also cleans One Time Keys, and other "session" based
-            stuff.
-        '''
-        sessions = self.db.sessions
-        last_clean = sessions.get('last_clean', 'last_use') or 0
+    def clean_up(self):
+        """Remove expired sessions and One Time Keys.
 
-        week = 60*60*24*7
+           Do it only once an hour.
+        """
         hour = 60*60
         now = time.time()
-        if now - last_clean > hour:
-            # remove aged sessions
-            for sessid in sessions.list():
-                interval = now - sessions.get(sessid, 'last_use')
-                if interval > week:
-                    sessions.destroy(sessid)
-            # remove aged otks
-            otks = self.db.otks
-            for sessid in otks.list():
-                interval = now - otks.get(sessid, '__time')
-                if interval > week:
-                    otks.destroy(sessid)
-            sessions.set('last_clean', last_use=time.time())
+
+        # XXX: hack - use OTK table to store last_clean time information
+        #      'last_clean' string is used instead of otk key
+        last_clean = self.db.getOTKManager().get('last_clean', 'last_use', 0)
+        if now - last_clean < hour:
+            return
+
+        self.session_api.clean_up()
+        self.db.getOTKManager().clean()
+        self.db.getOTKManager().set('last_clean', last_use=now)
+        self.db.commit(fail_ok=True)
+
+    def determine_charset(self):
+        """Look for client charset in the form parameters or browser cookie.
+
+        If no charset requested by client, use storage charset (utf-8).
+
+        If the charset is found, and differs from the storage charset,
+        recode all form fields of type 'text/plain'
+        """
+        # look for client charset
+        charset_parameter = 0
+        if '@charset' in self.form:
+            charset = self.form['@charset'].value
+            if charset.lower() == "none":
+                charset = ""
+            charset_parameter = 1
+        elif 'roundup_charset' in self.cookie:
+            charset = self.cookie['roundup_charset'].value
+        else:
+            charset = None
+        if charset:
+            # make sure the charset is recognized
+            try:
+                codecs.lookup(charset)
+            except LookupError:
+                self.error_message.append(self._('Unrecognized charset: %r')
+                    % charset)
+                charset_parameter = 0
+            else:
+                self.charset = charset.lower()
+        # If we've got a character set in request parameters,
+        # set the browser cookie to keep the preference.
+        # This is done after codecs.lookup to make sure
+        # that we aren't keeping a wrong value.
+        if charset_parameter:
+            self.add_cookie('roundup_charset', charset)
+
+        # if client charset is different from the storage charset,
+        # recode form fields
+        # XXX this requires FieldStorage from Python library.
+        #   mod_python FieldStorage is not supported!
+        if self.charset != self.STORAGE_CHARSET:
+            decoder = codecs.getdecoder(self.charset)
+            encoder = codecs.getencoder(self.STORAGE_CHARSET)
+            re_charref = re.compile('&#([0-9]+|x[0-9a-f]+);', re.IGNORECASE)
+            def _decode_charref(matchobj):
+                num = matchobj.group(1)
+                if num[0].lower() == 'x':
+                    uc = int(num[1:], 16)
+                else:
+                    uc = int(num)
+                return unichr(uc)
+
+            for field_name in self.form:
+                field = self.form[field_name]
+                if (field.type == 'text/plain') and not field.filename:
+                    try:
+                        value = decoder(field.value)[0]
+                    except UnicodeError:
+                        continue
+                    value = re_charref.sub(_decode_charref, value)
+                    field.value = encoder(value)[0]
+
+    def determine_language(self):
+        """Determine the language"""
+        # look for language parameter
+        # then for language cookie
+        # last for the Accept-Language header
+        if "@language" in self.form:
+            language = self.form["@language"].value
+            if language.lower() == "none":
+                language = ""
+            self.add_cookie("roundup_language", language)
+        elif "roundup_language" in self.cookie:
+            language = self.cookie["roundup_language"].value
+        elif self.instance.config["WEB_USE_BROWSER_LANGUAGE"]:
+            hal = self.env.get('HTTP_ACCEPT_LANGUAGE')
+            language = accept_language.parse(hal)
+        else:
+            language = ""
+
+        self.language = language
+        if language:
+            self.setTranslator(TranslationService.get_translation(
+                    language,
+                    tracker_home=self.instance.config["TRACKER_HOME"]))
 
     def determine_user(self):
-        ''' Determine who the user is
-        '''
-        # determine the uid to use
+        """Determine who the user is"""
         self.opendb('admin')
-        # clean age sessions
-        self.clean_sessions()
-        # make sure we have the session Class
-        sessions = self.db.sessions
-
-        # look up the user session cookie
-        cookie = Cookie.SimpleCookie(self.env.get('HTTP_COOKIE', ''))
-        user = 'anonymous'
-
-        # bump the "revision" of the cookie since the format changed
-        if (cookie.has_key(self.cookie_name) and
-                cookie[self.cookie_name].value != 'deleted'):
-
-            # get the session key from the cookie
-            self.session = cookie[self.cookie_name].value
-            # get the user from the session
-            try:
-                # update the lifetime datestamp
-                sessions.set(self.session, last_use=time.time())
-                sessions.commit()
-                user = sessions.get(self.session, 'user')
-            except KeyError:
-                user = 'anonymous'
 
-        # sanity check on the user still being valid, getting the userid
-        # at the same time
+        # get session data from db
+        # XXX: rename
+        self.session_api = Session(self)
+
+        # take the opportunity to cleanup expired sessions and otks
+        self.clean_up()
+
+        user = None
+        # first up, try http authorization if enabled
+        if self.instance.config['WEB_HTTP_AUTH']:
+            if 'REMOTE_USER' in self.env:
+                # we have external auth (e.g. by Apache)
+                user = self.env['REMOTE_USER']
+            elif self.env.get('HTTP_AUTHORIZATION', ''):
+                # try handling Basic Auth ourselves
+                auth = self.env['HTTP_AUTHORIZATION']
+                scheme, challenge = auth.split(' ', 1)
+                if scheme.lower() == 'basic':
+                    try:
+                        decoded = base64.decodestring(challenge)
+                    except TypeError:
+                        # invalid challenge
+                        pass
+                    username, password = decoded.split(':')
+                    try:
+                        login = self.get_action_class('login')(self)
+                        login.verifyLogin(username, password)
+                    except LoginError, err:
+                        self.make_user_anonymous()
+                        raise
+                    user = username
+
+        # if user was not set by http authorization, try session lookup
+        if not user:
+            user = self.session_api.get('user')
+            if user:
+                # update session lifetime datestamp
+                self.session_api.update()
+
+        # if no user name set by http authorization or session lookup
+        # the user is anonymous
+        if not user:
+            user = 'anonymous'
+
+        # sanity check on the user still being valid,
+        # getting the userid at the same time
         try:
             self.userid = self.db.user.lookup(user)
         except (KeyError, TypeError):
@@ -341,56 +732,125 @@ class Client:
         # reopen the database as the correct user
         self.opendb(self.user)
 
-    def determine_context(self, dre=re.compile(r'([^\d]+)(\d+)')):
-        ''' Determine the context of this page from the URL:
-
-            The URL path after the instance identifier is examined. The path
-            is generally only one entry long.
-
-            - if there is no path, then we are in the "home" context.
-            * if the path is "_file", then the additional path entry
-              specifies the filename of a static file we're to serve up
-              from the instance "html" directory. Raises a SendStaticFile
-              exception.
-            - if there is something in the path (eg "issue"), it identifies
-              the tracker class we're to display.
-            - if the path is an item designator (eg "issue123"), then we're
-              to display a specific item.
-            * if the path starts with an item designator and is longer than
-              one entry, then we're assumed to be handling an item of a
-              FileClass, and the extra path information gives the filename
-              that the client is going to label the download with (ie
-              "file123/image.png" is nicer to download than "file123"). This
-              raises a SendFile exception.
-
-            Both of the "*" types of contexts stop before we bother to
-            determine the template we're going to use. That's because they
-            don't actually use templates.
-
-            The template used is specified by the :template CGI variable,
-            which defaults to:
-
-             only classname suplied:          "index"
-             full item designator supplied:   "item"
-
-            We set:
+    def check_anonymous_access(self):
+        """Check that the Anonymous user is actually allowed to use the web
+        interface and short-circuit all further processing if they're not.
+        """
+        # allow Anonymous to use the "login" and "register" actions (noting
+        # that "register" has its own "Register" permission check)
+
+        if ':action' in self.form:
+            action = self.form[':action']
+        elif '@action' in self.form:
+            action = self.form['@action']
+        else:
+            action = ''
+        if isinstance(action, list):
+            raise SeriousError('broken form: multiple @action values submitted')
+        elif action != '':
+            action = action.value.lower()
+        if action in ('login', 'register'):
+            return
+
+        # allow Anonymous to view the "user" "register" template if they're
+        # allowed to register
+        if (self.db.security.hasPermission('Register', self.userid, 'user')
+                and self.classname == 'user' and self.template == 'register'):
+            return
+
+        # otherwise for everything else
+        if self.user == 'anonymous':
+            if not self.db.security.hasPermission('Web Access', self.userid):
+                raise Unauthorised(self._("Anonymous users are not "
+                    "allowed to use the web interface"))
+
+    def opendb(self, username):
+        """Open the database and set the current user.
+
+        Opens a database once. On subsequent calls only the user is set on
+        the database object the instance.optimize is set. If we are in
+        "Development Mode" (cf. roundup_server) then the database is always
+        re-opened.
+        """
+        # don't do anything if the db is open and the user has not changed
+        if hasattr(self, 'db') and self.db.isCurrentUser(username):
+            return
+
+        # open the database or only set the user
+        if not hasattr(self, 'db'):
+            self.db = self.instance.open(username)
+        else:
+            if self.instance.optimize:
+                self.db.setCurrentUser(username)
+            else:
+                self.db.close()
+                self.db = self.instance.open(username)
+                # The old session API refers to the closed database;
+                # we can no longer use it.
+                self.session_api = Session(self)
+ 
+
+    def determine_context(self, dre=re.compile(r'([^\d]+)0*(\d+)')):
+        """Determine the context of this page from the URL:
+
+        The URL path after the instance identifier is examined. The path
+        is generally only one entry long.
+
+        - if there is no path, then we are in the "home" context.
+        - if the path is "_file", then the additional path entry
+          specifies the filename of a static file we're to serve up
+          from the instance "html" directory. Raises a SendStaticFile
+          exception.(*)
+        - if there is something in the path (eg "issue"), it identifies
+          the tracker class we're to display.
+        - if the path is an item designator (eg "issue123"), then we're
+          to display a specific item.
+        - if the path starts with an item designator and is longer than
+          one entry, then we're assumed to be handling an item of a
+          FileClass, and the extra path information gives the filename
+          that the client is going to label the download with (ie
+          "file123/image.png" is nicer to download than "file123"). This
+          raises a SendFile exception.(*)
+
+        Both of the "*" types of contexts stop before we bother to
+        determine the template we're going to use. That's because they
+        don't actually use templates.
+
+        The template used is specified by the :template CGI variable,
+        which defaults to:
+
+        - only classname suplied:          "index"
+        - full item designator supplied:   "item"
+
+        We set:
+
              self.classname  - the class to display, can be None
+
              self.template   - the template to render the current context with
+
              self.nodeid     - the nodeid of the class we're displaying
-        '''
+        """
         # default the optional variables
         self.classname = None
         self.nodeid = None
 
         # see if a template or messages are specified
         template_override = ok_message = error_message = None
-        for key in self.form.keys():
+        for key in self.form:
             if self.FV_TEMPLATE.match(key):
                 template_override = self.form[key].value
             elif self.FV_OK_MESSAGE.match(key):
                 ok_message = self.form[key].value
+                ok_message = clean_message(ok_message)
             elif self.FV_ERROR_MESSAGE.match(key):
                 error_message = self.form[key].value
+                error_message = clean_message(error_message)
+
+        # see if we were passed in a message
+        if ok_message:
+            self.ok_message.append(ok_message)
+        if error_message:
+            self.error_message.append(error_message)
 
         # determine the classname and possibly nodeid
         path = self.path.split('/')
@@ -400,21 +860,25 @@ class Client:
             else:
                 self.template = ''
             return
-        elif path[0] == '_file':
-            raise SendStaticFile, os.path.join(*path[1:])
+        elif path[0] in ('_file', '@@file'):
+            raise SendStaticFile(os.path.join(*path[1:]))
         else:
             self.classname = path[0]
             if len(path) > 1:
                 # send the file identified by the designator in path[0]
-                raise SendFile, path[0]
+                raise SendFile(path[0])
 
         # see if we got a designator
         m = dre.match(self.classname)
         if m:
             self.classname = m.group(1)
             self.nodeid = m.group(2)
-            if not self.db.getclass(self.classname).hasnode(self.nodeid):
-                raise NotFound, '%s/%s'%(self.classname, self.nodeid)
+            try:
+                klass = self.db.getclass(self.classname)
+            except KeyError:
+                raise NotFound('%s/%s'%(self.classname, self.nodeid))
+            if not klass.hasnode(self.nodeid):
+                raise NotFound('%s/%s'%(self.classname, self.nodeid))
             # with a designator, we default to item view
             self.template = 'item'
         else:
@@ -425,67 +889,172 @@ class Client:
         try:
             self.db.getclass(self.classname)
         except KeyError:
-            raise NotFound, self.classname
+            raise NotFound(self.classname)
 
         # see if we have a template override
         if template_override is not None:
             self.template = template_override
 
-        # see if we were passed in a message
-        if ok_message:
-            self.ok_message.append(ok_message)
-        if error_message:
-            self.error_message.append(error_message)
-
     def serve_file(self, designator, dre=re.compile(r'([^\d]+)(\d+)')):
-        ''' Serve the file from the content property of the designated item.
-        '''
+        """ Serve the file from the content property of the designated item.
+        """
         m = dre.match(str(designator))
         if not m:
-            raise NotFound, str(designator)
+            raise NotFound(str(designator))
         classname, nodeid = m.group(1), m.group(2)
-        if classname != 'file':
-            raise NotFound, designator
 
-        # we just want to serve up the file named
-        file = self.db.file
-        self.additional_headers['Content-Type'] = file.get(nodeid, 'type')
-        self.write(file.get(nodeid, 'content'))
+        try:
+            klass = self.db.getclass(classname)
+        except KeyError:
+            # The classname was not valid.
+            raise NotFound(str(designator))
+            
+        # perform the Anonymous user access check
+        self.check_anonymous_access()
+
+        # make sure we have the appropriate properties
+        props = klass.getprops()
+        if 'type' not in props:
+            raise NotFound(designator)
+        if 'content' not in props:
+            raise NotFound(designator)
+
+        # make sure we have permission
+        if not self.db.security.hasPermission('View', self.userid,
+                classname, 'content', nodeid):
+            raise Unauthorised(self._("You are not allowed to view "
+                "this file."))
+
+        try:
+            mime_type = klass.get(nodeid, 'type')
+        except IndexError, e:
+            raise NotFound(e)
+        # Can happen for msg class:
+        if not mime_type:
+            mime_type = 'text/plain'
+
+        # if the mime_type is HTML-ish then make sure we're allowed to serve up
+        # HTML-ish content
+        if mime_type in ('text/html', 'text/x-html'):
+            if not self.instance.config['WEB_ALLOW_HTML_FILE']:
+                # do NOT serve the content up as HTML
+                mime_type = 'application/octet-stream'
+
+        # If this object is a file (i.e., an instance of FileClass),
+        # see if we can find it in the filesystem.  If so, we may be
+        # able to use the more-efficient request.sendfile method of
+        # sending the file.  If not, just get the "content" property
+        # in the usual way, and use that.
+        content = None
+        filename = None
+        if isinstance(klass, hyperdb.FileClass):
+            try:
+                filename = self.db.filename(classname, nodeid)
+            except AttributeError:
+                # The database doesn't store files in the filesystem
+                # and therefore doesn't provide the "filename" method.
+                pass
+            except IOError:
+                # The file does not exist.
+                pass
+        if not filename:
+            content = klass.get(nodeid, 'content')
+        
+        lmt = klass.get(nodeid, 'activity').timestamp()
+
+        self._serve_file(lmt, mime_type, content, filename)
 
     def serve_static_file(self, file):
+        """ Serve up the file named from the templates dir
+        """
+        # figure the filename - try STATIC_FILES, then TEMPLATES dir
+        for dir_option in ('STATIC_FILES', 'TEMPLATES'):
+            prefix = self.instance.config[dir_option]
+            if not prefix:
+                continue
+            # ensure the load doesn't try to poke outside
+            # of the static files directory
+            prefix = os.path.normpath(prefix)
+            filename = os.path.normpath(os.path.join(prefix, file))
+            if os.path.isfile(filename) and filename.startswith(prefix):
+                break
+        else:
+            raise NotFound(file)
+
+        # last-modified time
+        lmt = os.stat(filename)[stat.ST_MTIME]
+
+        # detemine meta-type
+        file = str(file)
+        mime_type = mimetypes.guess_type(file)[0]
+        if not mime_type:
+            if file.endswith('.css'):
+                mime_type = 'text/css'
+            else:
+                mime_type = 'text/plain'
+
+        self._serve_file(lmt, mime_type, '', filename)
+
+    def _serve_file(self, lmt, mime_type, content=None, filename=None):
+        """ guts of serve_file() and serve_static_file()
+        """
+
+        # spit out headers
+        self.additional_headers['Content-Type'] = mime_type
+        self.additional_headers['Last-Modified'] = rfc822.formatdate(lmt)
+
         ims = None
         # see if there's an if-modified-since...
-        if hasattr(self.request, 'headers'):
-            ims = self.request.headers.getheader('if-modified-since')
-        elif self.env.has_key('HTTP_IF_MODIFIED_SINCE'):
+        # XXX see which interfaces set this
+        #if hasattr(self.request, 'headers'):
+            #ims = self.request.headers.getheader('if-modified-since')
+        if 'HTTP_IF_MODIFIED_SINCE' in self.env:
             # cgi will put the header in the env var
             ims = self.env['HTTP_IF_MODIFIED_SINCE']
-        filename = os.path.join(self.instance.config.TEMPLATES, file)
-        lmt = os.stat(filename)[stat.ST_MTIME]
         if ims:
             ims = rfc822.parsedate(ims)[:6]
             lmtt = time.gmtime(lmt)[:6]
             if lmtt <= ims:
                 raise NotModified
 
-        # we just want to serve up the file named
-        file = str(file)
-        mt = mimetypes.guess_type(file)[0]
-        if not mt:
-            if file.endswith('.css'):
-                mt = 'text/css'
-            else:
-                mt = 'text/plain'
-        self.additional_headers['Content-Type'] = mt
-        self.additional_headers['Last-Modifed'] = rfc822.formatdate(lmt)
-        self.write(open(filename, 'rb').read())
+        if filename:
+            self.write_file(filename)
+        else:
+            self.additional_headers['Content-Length'] = str(len(content))
+            self.write(content)
+
+    def send_error_to_admin(self, subject, html, txt):
+        """Send traceback information to admin via email.
+           We send both, the formatted html (with more information) and
+           the text version of the traceback. We use
+           multipart/alternative so the receiver can chose which version
+           to display.
+        """
+        to = [self.mailer.config.ADMIN_EMAIL]
+        message = MIMEMultipart('alternative')
+        self.mailer.set_message_attributes(message, to, subject)
+        part = MIMEBase('text', 'html')
+        part.set_charset('utf-8')
+        part.set_payload(html)
+        encode_quopri(part)
+        message.attach(part)
+        part = MIMEText(txt)
+        message.attach(part)
+        self.mailer.smtp_send(to, message.as_string())
+    
+    def renderFrontPage(self, message):
+        """Return the front page of the tracker."""
+    
+        self.classname = self.nodeid = None
+        self.template = ''
+        self.error_message.append(message)
+        self.write_html(self.renderContext())
 
     def renderContext(self):
-        ''' Return a PageTemplate for the named page
-        '''
+        """ Return a PageTemplate for the named page
+        """
         name = self.classname
         extension = self.template
-        pt = Templates(self.instance.config.TEMPLATES).get(name, extension)
 
         # catch errors so we can handle PT rendering errors more nicely
         args = {
@@ -493,1396 +1062,490 @@ class Client:
             'error_message': self.error_message
         }
         try:
+            pt = self.instance.templates.get(name, extension)
             # let the template render figure stuff out
-            return pt.render(self, None, None, **args)
-        except NoTemplate, message:
-            return '<strong>%s</strong>'%message
+            result = pt.render(self, None, None, **args)
+            self.additional_headers['Content-Type'] = pt.content_type
+            if self.env.get('CGI_SHOW_TIMING', ''):
+                if self.env['CGI_SHOW_TIMING'].upper() == 'COMMENT':
+                    timings = {'starttag': '<!-- ', 'endtag': ' -->'}
+                else:
+                    timings = {'starttag': '<p>', 'endtag': '</p>'}
+                timings['seconds'] = time.time()-self.start
+                s = self._('%(starttag)sTime elapsed: %(seconds)fs%(endtag)s\n'
+                    ) % timings
+                if hasattr(self.db, 'stats'):
+                    timings.update(self.db.stats)
+                    s += self._("%(starttag)sCache hits: %(cache_hits)d,"
+                        " misses %(cache_misses)d."
+                        " Loading items: %(get_items)f secs."
+                        " Filtering: %(filtering)f secs."
+                        "%(endtag)s\n") % timings
+                s += '</body>'
+                result = result.replace('</body>', s)
+            return result
+        except templating.NoTemplate, message:
+            return '<strong>%s</strong>'%cgi.escape(str(message))
+        except templating.Unauthorised, message:
+            raise Unauthorised(cgi.escape(str(message)))
         except:
             # everything else
-            return cgitb.pt_html()
+            if self.instance.config.WEB_DEBUG:
+                return cgitb.pt_html(i18n=self.translator)
+            exc_info = sys.exc_info()
+            try:
+                # If possible, send the HTML page template traceback
+                # to the administrator.
+                subject = "Templating Error: %s" % exc_info[1]
+                self.send_error_to_admin(subject, cgitb.pt_html(), format_exc())
+                # Now report the error to the user.
+                return self._(error_message)
+            except:
+                # Reraise the original exception.  The user will
+                # receive an error message, and the adminstrator will
+                # receive a traceback, albeit with less information
+                # than the one we tried to generate above.
+                raise exc_info[0](exc_info[1]).with_traceback(exc_info[2])
 
     # these are the actions that are available
     actions = (
-        ('edit',     'editItemAction'),
-        ('editcsv',  'editCSVAction'),
-        ('new',      'newItemAction'),
-        ('register', 'registerAction'),
-        ('confrego', 'confRegoAction'),
-        ('passrst',  'passResetAction'),
-        ('login',    'loginAction'),
-        ('logout',   'logout_action'),
-        ('search',   'searchAction'),
-        ('retire',   'retireAction'),
-        ('show',     'showAction'),
+        ('edit',        EditItemAction),
+        ('editcsv',     EditCSVAction),
+        ('new',         NewItemAction),
+        ('register',    RegisterAction),
+        ('confrego',    ConfRegoAction),
+        ('passrst',     PassResetAction),
+        ('login',       LoginAction),
+        ('logout',      LogoutAction),
+        ('search',      SearchAction),
+        ('retire',      RetireAction),
+        ('show',        ShowAction),
+        ('export_csv',  ExportCSVAction),
     )
     def handle_action(self):
-        ''' Determine whether there should be an Action called.
+        """ Determine whether there should be an Action called.
 
             The action is defined by the form variable :action which
             identifies the method on this object to call. The actions
             are defined in the "actions" sequence on this class.
-        '''
-        if self.form.has_key(':action'):
-            action = self.form[':action'].value.lower()
-        elif self.form.has_key('@action'):
-            action = self.form['@action'].value.lower()
-        else:
-            return None
-        try:
-            # get the action, validate it
-            for name, method in self.actions:
-                if name == action:
-                    break
-            else:
-                raise ValueError, 'No such action "%s"'%action
-            # call the mapped action
-            getattr(self, method)()
-        except Redirect:
-            raise
-        except Unauthorised:
-            raise
-
-    def write(self, content):
-        if not self.headers_done:
-            self.header()
-        self.request.wfile.write(content)
-
-    def header(self, headers=None, response=None):
-        '''Put up the appropriate header.
-        '''
-        if headers is None:
-            headers = {'Content-Type':'text/html'}
-        if response is None:
-            response = self.response_code
 
-        # update with additional info
-        headers.update(self.additional_headers)
+            Actions may return a page (by default HTML) to return to the
+            user, bypassing the usual template rendering.
 
-        if not headers.has_key('Content-Type'):
-            headers['Content-Type'] = 'text/html'
-        self.request.send_response(response)
-        for entry in headers.items():
-            self.request.send_header(*entry)
-        self.request.end_headers()
-        self.headers_done = 1
-        if self.debug:
-            self.headers_sent = headers
-
-    def set_cookie(self, user):
-        ''' Set up a session cookie for the user and store away the user's
-            login info against the session.
-        '''
-        # TODO generate a much, much stronger session key ;)
-        self.session = binascii.b2a_base64(repr(random.random())).strip()
-
-        # clean up the base64
-        if self.session[-1] == '=':
-            if self.session[-2] == '=':
-                self.session = self.session[:-2]
-            else:
-                self.session = self.session[:-1]
-
-        # insert the session in the sessiondb
-        self.db.sessions.set(self.session, user=user, last_use=time.time())
-
-        # and commit immediately
-        self.db.sessions.commit()
-
-        # expire us in a long, long time
-        expire = Cookie._getdate(86400*365)
-
-        # generate the cookie path - make sure it has a trailing '/'
-        self.additional_headers['Set-Cookie'] = \
-          '%s=%s; expires=%s; Path=%s;'%(self.cookie_name, self.session,
-            expire, self.cookie_path)
-
-    def make_user_anonymous(self):
-        ''' Make us anonymous
-
-            This method used to handle non-existence of the 'anonymous'
-            user, but that user is mandatory now.
-        '''
-        self.userid = self.db.user.lookup('anonymous')
-        self.user = 'anonymous'
-
-    def opendb(self, user):
-        ''' Open the database.
-        '''
-        # open the db if the user has changed
-        if not hasattr(self, 'db') or user != self.db.journaltag:
-            if hasattr(self, 'db'):
-                self.db.close()
-            self.db = self.instance.open(user)
-
-    #
-    # Actions
-    #
-    def loginAction(self):
-        ''' Attempt to log a user in.
-
-            Sets up a session for the user which contains the login
-            credentials.
-        '''
-        # we need the username at a minimum
-        if not self.form.has_key('__login_name'):
-            self.error_message.append(_('Username required'))
-            return
+            We explicitly catch Reject and ValueError exceptions and
+            present their messages to the user.
+        """
+        if ':action' in self.form:
+            action = self.form[':action']
+        elif '@action' in self.form:
+            action = self.form['@action']
+        else:
+            return None
 
-        # get the login info
-        self.user = self.form['__login_name'].value
-        if self.form.has_key('__login_password'):
-            password = self.form['__login_password'].value
+        if isinstance(action, list):
+            raise SeriousError('broken form: multiple @action values submitted')
         else:
-            password = ''
+            action = action.value.lower()
 
-        # make sure the user exists
         try:
-            self.userid = self.db.user.lookup(self.user)
-        except KeyError:
-            name = self.user
-            self.error_message.append(_('No such user "%(name)s"')%locals())
-            self.make_user_anonymous()
-            return
-
-        # verify the password
-        if not self.verifyPassword(self.userid, password):
-            self.make_user_anonymous()
-            self.error_message.append(_('Incorrect password'))
-            return
-
-        # make sure we're allowed to be here
-        if not self.loginPermission():
-            self.make_user_anonymous()
-            self.error_message.append(_("You do not have permission to login"))
-            return
-
-        # now we're OK, re-open the database for real, using the user
-        self.opendb(self.user)
+            action_klass = self.get_action_class(action)
 
-        # set the session cookie
-        self.set_cookie(self.user)
-
-    def verifyPassword(self, userid, password):
-        ''' Verify the password that the user has supplied
-        '''
-        stored = self.db.user.get(self.userid, 'password')
-        if password == stored:
-            return 1
-        if not password and not stored:
-            return 1
-        return 0
-
-    def loginPermission(self):
-        ''' Determine whether the user has permission to log in.
-
-            Base behaviour is to check the user has "Web Access".
-        ''' 
-        if not self.db.security.hasPermission('Web Access', self.userid):
-            return 0
-        return 1
-
-    def logout_action(self):
-        ''' Make us really anonymous - nuke the cookie too
-        '''
-        # log us out
-        self.make_user_anonymous()
-
-        # construct the logout cookie
-        now = Cookie._getdate()
-        self.additional_headers['Set-Cookie'] = \
-           '%s=deleted; Max-Age=0; expires=%s; Path=%s;'%(self.cookie_name,
-            now, self.cookie_path)
+            # call the mapped action
+            if isinstance(action_klass, type('')):
+                # old way of specifying actions
+                return getattr(self, action_klass)()
+            else:
+                return action_klass(self).execute()
 
-        # Let the user know what's going on
-        self.ok_message.append(_('You are logged out'))
+        except (ValueError, Reject), err:
+            self.error_message.append(str(err))
 
-    def registerAction(self):
-        '''Attempt to create a new user based on the contents of the form
-        and then set the cookie.
+    def get_action_class(self, action_name):
+        if (hasattr(self.instance, 'cgi_actions') and
+                action_name in self.instance.cgi_actions):
+            # tracker-defined action
+            action_klass = self.instance.cgi_actions[action_name]
+        else:
+            # go with a default
+            for name, action_klass in self.actions:
+                if name == action_name:
+                    break
+            else:
+                raise ValueError('No such action "%s"'%action_name)
+        return action_klass
 
-        return 1 on successful login
-        '''
-        # parse the props from the form
-        try:
-            props = self.parsePropsFromForm()[0][('user', None)]
-        except (ValueError, KeyError), message:
-            self.error_message.append(_('Error: ') + str(message))
-            return
+    def _socket_op(self, call, *args, **kwargs):
+        """Execute socket-related operation, catch common network errors
 
-        # make sure we're allowed to register
-        if not self.registerPermission(props):
-            raise Unauthorised, _("You do not have permission to register")
+        Parameters:
+            call: a callable to execute
+            args, kwargs: call arguments
 
+        """
         try:
-            self.db.user.lookup(props['username'])
-            self.error_message.append('Error: A user with the username "%s" '
-                'already exists'%props['username'])
-            return
-        except KeyError:
+            call(*args, **kwargs)
+        except socket.error, err:
+            err_errno = getattr (err, 'errno', None)
+            if err_errno is None:
+                try:
+                    err_errno = err[0]
+                except TypeError:
+                    pass
+            if err_errno not in self.IGNORE_NET_ERRORS:
+                raise
+        except IOError:
+            # Apache's mod_python will raise IOError -- without an
+            # accompanying errno -- when a write to the client fails.
+            # A common case is that the client has closed the
+            # connection.  There's no way to be certain that this is
+            # the situation that has occurred here, but that is the
+            # most likely case.
             pass
 
-        # generate the one-time-key and store the props for later
-        otk = ''.join([random.choice(chars) for x in range(32)])
-        for propname, proptype in self.db.user.getprops().items():
-            value = props.get(propname, None)
-            if value is None:
-                pass
-            elif isinstance(proptype, hyperdb.Date):
-                props[propname] = str(value)
-            elif isinstance(proptype, hyperdb.Interval):
-                props[propname] = str(value)
-            elif isinstance(proptype, hyperdb.Password):
-                props[propname] = str(value)
-        props['__time'] = time.time()
-        self.db.otks.set(otk, **props)
-
-        # send the email
-        tracker_name = self.db.config.TRACKER_NAME
-        subject = 'Complete your registration to %s'%tracker_name
-        body = '''
-To complete your registration of the user "%(name)s" with %(tracker)s,
-please visit the following URL:
-
-   %(url)s?@action=confrego&otk=%(otk)s
-'''%{'name': props['username'], 'tracker': tracker_name, 'url': self.base,
-                'otk': otk}
-        if not self.sendEmail(props['address'], subject, body):
-            return
-
-        # commit changes to the database
-        self.db.commit()
-
-        # redirect to the "you're almost there" page
-        raise Redirect, '%suser?@template=rego_progress'%self.base
-
-    def sendEmail(self, to, subject, content):
-        # send email to the user's email address
-        message = StringIO.StringIO()
-        writer = MimeWriter.MimeWriter(message)
-        tracker_name = self.db.config.TRACKER_NAME
-        writer.addheader('Subject', encode_header(subject))
-        writer.addheader('To', to)
-        writer.addheader('From', roundupdb.straddr((tracker_name,
-            self.db.config.ADMIN_EMAIL)))
-        writer.addheader('Date', time.strftime("%a, %d %b %Y %H:%M:%S +0000",
-            time.gmtime()))
-        # add a uniquely Roundup header to help filtering
-        writer.addheader('X-Roundup-Name', tracker_name)
-        # avoid email loops
-        writer.addheader('X-Roundup-Loop', 'hello')
-        writer.addheader('Content-Transfer-Encoding', 'quoted-printable')
-        body = writer.startbody('text/plain; charset=utf-8')
-
-        # message body, encoded quoted-printable
-        content = StringIO.StringIO(content)
-        quopri.encode(content, body, 0)
-
-        if SENDMAILDEBUG:
-            # don't send - just write to a file
-            open(SENDMAILDEBUG, 'a').write('FROM: %s\nTO: %s\n%s\n'%(
-                self.db.config.ADMIN_EMAIL,
-                ', '.join(to),message.getvalue()))
-        else:
-            # now try to send the message
-            try:
-                # send the message as admin so bounces are sent there
-                # instead of to roundup
-                smtp = openSMTPConnection(self.db.config)
-                smtp.sendmail(self.db.config.ADMIN_EMAIL, [to],
-                    message.getvalue())
-            except socket.error, value:
-                self.error_message.append("Error: couldn't send email: "
-                    "mailhost %s"%value)
-                return 0
-            except smtplib.SMTPException, msg:
-                self.error_message.append("Error: couldn't send email: %s"%msg)
-                return 0
-        return 1
+    def write(self, content):
+        if not self.headers_done:
+            self.header()
+        if self.env['REQUEST_METHOD'] != 'HEAD':
+            self._socket_op(self.request.wfile.write, content)
 
-    def registerPermission(self, props):
-        ''' Determine whether the user has permission to register
+    def write_html(self, content):
+        if not self.headers_done:
+            # at this point, we are sure about Content-Type
+            if 'Content-Type' not in self.additional_headers:
+                self.additional_headers['Content-Type'] = \
+                    'text/html; charset=%s' % self.charset
+            self.header()
 
-            Base behaviour is to check the user has "Web Registration".
-        '''
-        # registration isn't allowed to supply roles
-        if props.has_key('roles'):
-            return 0
-        if self.db.security.hasPermission('Web Registration', self.userid):
-            return 1
-        return 0
-
-    def confRegoAction(self):
-        ''' Grab the OTK, use it to load up the new user details
-        '''
-        # pull the rego information out of the otk database
-        otk = self.form['otk'].value
-        props = self.db.otks.getall(otk)
-        for propname, proptype in self.db.user.getprops().items():
-            value = props.get(propname, None)
-            if value is None:
-                pass
-            elif isinstance(proptype, hyperdb.Date):
-                props[propname] = date.Date(value)
-            elif isinstance(proptype, hyperdb.Interval):
-                props[propname] = date.Interval(value)
-            elif isinstance(proptype, hyperdb.Password):
-                props[propname] = password.Password()
-                props[propname].unpack(value)
-
-        # re-open the database as "admin"
-        if self.user != 'admin':
-            self.opendb('admin')
-
-        # create the new user
-        cl = self.db.user
-# XXX we need to make the "default" page be able to display errors!
-        try:
-            props['roles'] = self.instance.config.NEW_WEB_USER_ROLES
-            del props['__time']
-            self.userid = cl.create(**props)
-            # clear the props from the otk database
-            self.db.otks.destroy(otk)
-            self.db.commit()
-        except (ValueError, KeyError), message:
-            self.error_message.append(str(message))
+        if self.env['REQUEST_METHOD'] == 'HEAD':
+            # client doesn't care about content
             return
 
-        # log the new user in
-        self.user = cl.get(self.userid, 'username')
-        # re-open the database for real, using the user
-        self.opendb(self.user)
+        if self.charset != self.STORAGE_CHARSET:
+            # recode output
+            content = content.decode(self.STORAGE_CHARSET, 'replace')
+            content = content.encode(self.charset, 'xmlcharrefreplace')
 
-        # if we have a session, update it
-        if hasattr(self, 'session'):
-            self.db.sessions.set(self.session, user=self.user,
-                last_use=time.time())
-        else:
-            # new session cookie
-            self.set_cookie(self.user)
-
-        # nice message
-        message = _('You are now registered, welcome!')
-
-        # redirect to the user's page
-        raise Redirect, '%suser%s?@ok_message=%s'%(self.base,
-            self.userid, urllib.quote(message))
-
-    def passResetAction(self):
-        ''' Handle password reset requests.
-
-            Presence of either "name" or "address" generate email.
-            Presense of "otk" performs the reset.
-        '''
-        if self.form.has_key('otk'):
-            # pull the rego information out of the otk database
-            otk = self.form['otk'].value
-            uid = self.db.otks.get(otk, 'uid')
-            if uid is None:
-                self.error_message.append('Invalid One Time Key!')
-                return
+        # and write
+        self._socket_op(self.request.wfile.write, content)
 
-            # re-open the database as "admin"
-            if self.user != 'admin':
-                self.opendb('admin')
+    def http_strip(self, content):
+        """Remove HTTP Linear White Space from 'content'.
 
-            # change the password
-            newpw = password.generatePassword()
+        'content' -- A string.
 
-            cl = self.db.user
-# XXX we need to make the "default" page be able to display errors!
-            try:
-                # set the password
-                cl.set(uid, password=password.Password(newpw))
-                # clear the props from the otk database
-                self.db.otks.destroy(otk)
-                self.db.commit()
-            except (ValueError, KeyError), message:
-                self.error_message.append(str(message))
-                return
+        returns -- 'content', with all leading and trailing LWS
+        removed."""
 
-            # user info
-            address = self.db.user.get(uid, 'address')
-            name = self.db.user.get(uid, 'username')
+        # RFC 2616 2.2: Basic Rules
+        #
+        # LWS = [CRLF] 1*( SP | HT )
+        return content.strip(" \r\n\t")
 
-            # send the email
-            tracker_name = self.db.config.TRACKER_NAME
-            subject = 'Password reset for %s'%tracker_name
-            body = '''
-The password has been reset for username "%(name)s".
+    def http_split(self, content):
+        """Split an HTTP list.
 
-Your password is now: %(password)s
-'''%{'name': name, 'password': newpw}
-            if not self.sendEmail(address, subject, body):
-                return
-
-            self.ok_message.append('Password reset and email sent to %s'%address)
-            return
-
-        # no OTK, so now figure the user
-        if self.form.has_key('username'):
-            name = self.form['username'].value
-            try:
-                uid = self.db.user.lookup(name)
-            except KeyError:
-                self.error_message.append('Unknown username')
-                return
-            address = self.db.user.get(uid, 'address')
-        elif self.form.has_key('address'):
-            address = self.form['address'].value
-            uid = uidFromAddress(self.db, ('', address), create=0)
-            if not uid:
-                self.error_message.append('Unknown email address')
-                return
-            name = self.db.user.get(uid, 'username')
-        else:
-            self.error_message.append('You need to specify a username '
-                'or address')
-            return
-
-        # generate the one-time-key and store the props for later
-        otk = ''.join([random.choice(chars) for x in range(32)])
-        self.db.otks.set(otk, uid=uid, __time=time.time())
+        'content' -- A string, giving a list of items.
 
-        # send the email
-        tracker_name = self.db.config.TRACKER_NAME
-        subject = 'Confirm reset of password for %s'%tracker_name
-        body = '''
-Someone, perhaps you, has requested that the password be changed for your
-username, "%(name)s". If you wish to proceed with the change, please follow
-the link below:
+        returns -- A sequence of strings, containing the elements of
+        the list."""
 
-  %(url)suser?@template=forgotten&@action=passrst&otk=%(otk)s
+        # RFC 2616 2.1: Augmented BNF
+        #
+        # Grammar productions of the form "#rule" indicate a
+        # comma-separated list of elements matching "rule".  LWS
+        # is then removed from each element, and empty elements
+        # removed.
 
-You should then receive another email with the new password.
-'''%{'name': name, 'tracker': tracker_name, 'url': self.base, 'otk': otk}
-        if not self.sendEmail(address, subject, body):
-            return
-
-        self.ok_message.append('Email sent to %s'%address)
-
-    def editItemAction(self):
-        ''' Perform an edit of an item in the database.
-
-           See parsePropsFromForm and _editnodes for special variables
-        '''
-        # parse the props from the form
-        try:
-            props, links = self.parsePropsFromForm()
-        except (ValueError, KeyError), message:
-            self.error_message.append(_('Error: ') + str(message))
-            return
-
-        # handle the props
-        try:
-            message = self._editnodes(props, links)
-        except (ValueError, KeyError, IndexError), message:
-            self.error_message.append(_('Error: ') + str(message))
-            return
-
-        # commit now that all the tricky stuff is done
-        self.db.commit()
-
-        # redirect to the item's edit page
-        raise Redirect, '%s%s%s?@ok_message=%s&@template=%s'%(self.base,
-            self.classname, self.nodeid, urllib.quote(message),
-            urllib.quote(self.template))
-
-    def editItemPermission(self, props):
-        ''' Determine whether the user has permission to edit this item.
-
-            Base behaviour is to check the user can edit this class. If we're
-            editing the "user" class, users are allowed to edit their own
-            details. Unless it's the "roles" property, which requires the
-            special Permission "Web Roles".
-        '''
-        # if this is a user node and the user is editing their own node, then
-        # we're OK
-        has = self.db.security.hasPermission
-        if self.classname == 'user':
-            # reject if someone's trying to edit "roles" and doesn't have the
-            # right permission.
-            if props.has_key('roles') and not has('Web Roles', self.userid,
-                    'user'):
-                return 0
-            # if the item being edited is the current user, we're ok
-            if self.nodeid == self.userid:
-                return 1
-        if self.db.security.hasPermission('Edit', self.userid, self.classname):
-            return 1
-        return 0
-
-    def newItemAction(self):
-        ''' Add a new item to the database.
-
-            This follows the same form as the editItemAction, with the same
-            special form values.
-        '''
-        # parse the props from the form
+        # Split at commas.
+        elements = content.split(",")
+        # Remove linear whitespace at either end of the string.
+        elements = [self.http_strip(e) for e in elements]
+        # Remove any now-empty elements.
+        return [e for e in elements if e]
+        
+    def handle_range_header(self, length, etag):
+        """Handle the 'Range' and 'If-Range' headers.
+
+        'length' -- the length of the content available for the
+        resource.
+
+        'etag' -- the entity tag for this resources.
+
+        returns -- If the request headers (including 'Range' and
+        'If-Range') indicate that only a portion of the entity should
+        be returned, then the return value is a pair '(offfset,
+        length)' indicating the first byte and number of bytes of the
+        content that should be returned to the client.  In addition,
+        this method will set 'self.response_code' to indicate Partial
+        Content.  In all other cases, the return value is 'None'.  If
+        appropriate, 'self.response_code' will be
+        set to indicate 'REQUESTED_RANGE_NOT_SATISFIABLE'.  In that
+        case, the caller should not send any data to the client."""
+
+        # RFC 2616 14.35: Range
+        #
+        # See if the Range header is present.
+        ranges_specifier = self.env.get("HTTP_RANGE")
+        if ranges_specifier is None:
+            return None
+        # RFC 2616 14.27: If-Range
+        #
+        # Check to see if there is an If-Range header.
+        # Because the specification says:
+        #
+        #  The If-Range header ... MUST be ignored if the request
+        #  does not include a Range header, we check for If-Range
+        #  after checking for Range.
+        if_range = self.env.get("HTTP_IF_RANGE")
+        if if_range:
+            # The grammar for the If-Range header is:
+            # 
+            #   If-Range = "If-Range" ":" ( entity-tag | HTTP-date )
+            #   entity-tag = [ weak ] opaque-tag
+            #   weak = "W/"
+            #   opaque-tag = quoted-string
+            #
+            # We only support strong entity tags.
+            if_range = self.http_strip(if_range)
+            if (not if_range.startswith('"')
+                or not if_range.endswith('"')):
+                return None
+            # If the condition doesn't match the entity tag, then we
+            # must send the client the entire file.
+            if if_range != etag:
+                return
+        # The grammar for the Range header value is:
+        #
+        #   ranges-specifier = byte-ranges-specifier
+        #   byte-ranges-specifier = bytes-unit "=" byte-range-set
+        #   byte-range-set = 1#( byte-range-spec | suffix-byte-range-spec )
+        #   byte-range-spec = first-byte-pos "-" [last-byte-pos]
+        #   first-byte-pos = 1*DIGIT
+        #   last-byte-pos = 1*DIGIT
+        #   suffix-byte-range-spec = "-" suffix-length
+        #   suffix-length = 1*DIGIT
+        #
+        # Look for the "=" separating the units from the range set.
+        specs = ranges_specifier.split("=", 1)
+        if len(specs) != 2:
+            return None
+        # Check that the bytes-unit is in fact "bytes".  If it is not,
+        # we do not know how to process this range.
+        bytes_unit = self.http_strip(specs[0])
+        if bytes_unit != "bytes":
+            return None
+        # Seperate the range-set into range-specs.
+        byte_range_set = self.http_strip(specs[1])
+        byte_range_specs = self.http_split(byte_range_set)
+        # We only handle exactly one range at this time.
+        if len(byte_range_specs) != 1:
+            return None
+        # Parse the spec.
+        byte_range_spec = byte_range_specs[0]
+        pos = byte_range_spec.split("-", 1)
+        if len(pos) != 2:
+            return None
+        # Get the first and last bytes.
+        first = self.http_strip(pos[0])
+        last = self.http_strip(pos[1])
+        # We do not handle suffix ranges.
+        if not first:
+            return None
+       # Convert the first and last positions to integers.
         try:
-            props, links = self.parsePropsFromForm()
-        except (ValueError, KeyError), message:
-            self.error_message.append(_('Error: ') + str(message))
+            first = int(first)
+            if last:
+                last = int(last)
+            else:
+                last = length - 1
+        except:
+            # The positions could not be parsed as integers.
+            return None
+        # Check that the range makes sense.
+        if (first < 0 or last < 0 or last < first):
+            return None
+        if last >= length:
+            # RFC 2616 10.4.17: 416 Requested Range Not Satisfiable
+            #
+            # If there is an If-Range header, RFC 2616 says that we
+            # should just ignore the invalid Range header.
+            if if_range:
+                return None
+            # Return code 416 with a Content-Range header giving the
+            # allowable range.
+            self.response_code = http_.client.REQUESTED_RANGE_NOT_SATISFIABLE
+            self.setHeader("Content-Range", "bytes */%d" % length)
+            return None
+        # RFC 2616 10.2.7: 206 Partial Content
+        #
+        # Tell the client that we are honoring the Range request by
+        # indicating that we are providing partial content.
+        self.response_code = http_.client.PARTIAL_CONTENT
+        # RFC 2616 14.16: Content-Range
+        #
+        # Tell the client what data we are providing.
+        #
+        #   content-range-spec = byte-content-range-spec
+        #   byte-content-range-spec = bytes-unit SP
+        #                             byte-range-resp-spec "/"
+        #                             ( instance-length | "*" )
+        #   byte-range-resp-spec = (first-byte-pos "-" last-byte-pos)
+        #                          | "*"
+        #   instance-length      = 1 * DIGIT
+        self.setHeader("Content-Range",
+                       "bytes %d-%d/%d" % (first, last, length))
+        return (first, last - first + 1)
+
+    def write_file(self, filename):
+        """Send the contents of 'filename' to the user."""
+
+        # Determine the length of the file.
+        stat_info = os.stat(filename)
+        length = stat_info[stat.ST_SIZE]
+        # Assume we will return the entire file.
+        offset = 0
+        # If the headers have not already been finalized, 
+        if not self.headers_done:
+            # RFC 2616 14.19: ETag
+            #
+            # Compute the entity tag, in a format similar to that
+            # used by Apache.
+            etag = '"%x-%x-%x"' % (stat_info[stat.ST_INO],
+                                   length,
+                                   stat_info[stat.ST_MTIME])
+            self.setHeader("ETag", etag)
+            # RFC 2616 14.5: Accept-Ranges
+            #
+            # Let the client know that we will accept range requests.
+            self.setHeader("Accept-Ranges", "bytes")
+            # RFC 2616 14.35: Range
+            #
+            # If there is a Range header, we may be able to avoid
+            # sending the entire file.
+            content_range = self.handle_range_header(length, etag)
+            if content_range:
+                offset, length = content_range
+            # RFC 2616 14.13: Content-Length
+            #
+            # Tell the client how much data we are providing.
+            self.setHeader("Content-Length", str(length))
+            # Send the HTTP header.
+            self.header()
+        # If the client doesn't actually want the body, or if we are
+        # indicating an invalid range.
+        if (self.env['REQUEST_METHOD'] == 'HEAD'
+            or self.response_code == http_.client.REQUESTED_RANGE_NOT_SATISFIABLE):
             return
-
-        # handle the props - edit or create
-        try:
-            # when it hits the None element, it'll set self.nodeid
-            messages = self._editnodes(props, links)
-
-        except (ValueError, KeyError, IndexError), message:
-            # these errors might just be indicative of user dumbness
-            self.error_message.append(_('Error: ') + str(message))
+        # Use the optimized "sendfile" operation, if possible.
+        if hasattr(self.request, "sendfile"):
+            self._socket_op(self.request.sendfile, filename, offset, length)
             return
-
-        # commit now that all the tricky stuff is done
-        self.db.commit()
-
-        # redirect to the new item's page
-        raise Redirect, '%s%s%s?@ok_message=%s&@template=%s'%(self.base,
-            self.classname, self.nodeid, urllib.quote(messages),
-            urllib.quote(self.template))
-
-    def newItemPermission(self, props):
-        ''' Determine whether the user has permission to create (edit) this
-            item.
-
-            Base behaviour is to check the user can edit this class. No
-            additional property checks are made. Additionally, new user items
-            may be created if the user has the "Web Registration" Permission.
-        '''
-        has = self.db.security.hasPermission
-        if self.classname == 'user' and has('Web Registration', self.userid,
-                'user'):
-            return 1
-        if has('Edit', self.userid, self.classname):
-            return 1
-        return 0
-
-
-    #
-    #  Utility methods for editing
-    #
-    def _editnodes(self, all_props, all_links, newids=None):
-        ''' Use the props in all_props to perform edit and creation, then
-            use the link specs in all_links to do linking.
-        '''
-        # figure dependencies and re-work links
-        deps = {}
-        links = {}
-        for cn, nodeid, propname, vlist in all_links:
-            if not all_props.has_key((cn, nodeid)):
-                # link item to link to doesn't (and won't) exist
-                continue
-            for value in vlist:
-                if not all_props.has_key(value):
-                    # link item to link to doesn't (and won't) exist
-                    continue
-                deps.setdefault((cn, nodeid), []).append(value)
-                links.setdefault(value, []).append((cn, nodeid, propname))
-
-        # figure chained dependencies ordering
-        order = []
-        done = {}
-        # loop detection
-        change = 0
-        while len(all_props) != len(done):
-            for needed in all_props.keys():
-                if done.has_key(needed):
-                    continue
-                tlist = deps.get(needed, [])
-                for target in tlist:
-                    if not done.has_key(target):
-                        break
-                else:
-                    done[needed] = 1
-                    order.append(needed)
-                    change = 1
-            if not change:
-                raise ValueError, 'linking must not loop!'
-
-        # now, edit / create
-        m = []
-        for needed in order:
-            props = all_props[needed]
-            if not props:
-                # nothing to do
-                continue
-            cn, nodeid = needed
-
-            if nodeid is not None and int(nodeid) > 0:
-                # make changes to the node
-                props = self._changenode(cn, nodeid, props)
-
-                # and some nice feedback for the user
-                if props:
-                    info = ', '.join(props.keys())
-                    m.append('%s %s %s edited ok'%(cn, nodeid, info))
-                else:
-                    m.append('%s %s - nothing changed'%(cn, nodeid))
-            else:
-                assert props
-
-                # make a new node
-                newid = self._createnode(cn, props)
-                if nodeid is None:
-                    self.nodeid = newid
-                nodeid = newid
-
-                # and some nice feedback for the user
-                m.append('%s %s created'%(cn, newid))
-
-            # fill in new ids in links
-            if links.has_key(needed):
-                for linkcn, linkid, linkprop in links[needed]:
-                    props = all_props[(linkcn, linkid)]
-                    cl = self.db.classes[linkcn]
-                    propdef = cl.getprops()[linkprop]
-                    if not props.has_key(linkprop):
-                        if linkid is None or linkid.startswith('-'):
-                            # linking to a new item
-                            if isinstance(propdef, hyperdb.Multilink):
-                                props[linkprop] = [newid]
-                            else:
-                                props[linkprop] = newid
-                        else:
-                            # linking to an existing item
-                            if isinstance(propdef, hyperdb.Multilink):
-                                existing = cl.get(linkid, linkprop)[:]
-                                existing.append(nodeid)
-                                props[linkprop] = existing
-                            else:
-                                props[linkprop] = newid
-
-        return '<br>'.join(m)
-
-    def _changenode(self, cn, nodeid, props):
-        ''' change the node based on the contents of the form
-        '''
-        # check for permission
-        if not self.editItemPermission(props):
-            raise Unauthorised, 'You do not have permission to edit %s'%cn
-
-        # make the changes
-        cl = self.db.classes[cn]
-        return cl.set(nodeid, **props)
-
-    def _createnode(self, cn, props):
-        ''' create a node based on the contents of the form
-        '''
-        # check for permission
-        if not self.newItemPermission(props):
-            raise Unauthorised, 'You do not have permission to create %s'%cn
-
-        # create the node and return its id
-        cl = self.db.classes[cn]
-        return cl.create(**props)
-
-    # 
-    # More actions
-    #
-    def editCSVAction(self):
-        ''' Performs an edit of all of a class' items in one go.
-
-            The "rows" CGI var defines the CSV-formatted entries for the
-            class. New nodes are identified by the ID 'X' (or any other
-            non-existent ID) and removed lines are retired.
-        '''
-        # this is per-class only
-        if not self.editCSVPermission():
-            self.error_message.append(
-                _('You do not have permission to edit %s' %self.classname))
-
-        # get the CSV module
+        # Fallback to the "write" operation.
+        f = open(filename, 'rb')
         try:
-            import csv
-        except ImportError:
-            self.error_message.append(_(
-                'Sorry, you need the csv module to use this function.<br>\n'
-                'Get it from: <a href="http://www.object-craft.com.au/projects/csv/">http://www.object-craft.com.au/projects/csv/'))
-            return
-
-        cl = self.db.classes[self.classname]
-        idlessprops = cl.getprops(protected=0).keys()
-        idlessprops.sort()
-        props = ['id'] + idlessprops
-
-        # do the edit
-        rows = self.form['rows'].value.splitlines()
-        p = csv.parser()
-        found = {}
-        line = 0
-        for row in rows[1:]:
-            line += 1
-            values = p.parse(row)
-            # not a complete row, keep going
-            if not values: continue
-
-            # skip property names header
-            if values == props:
-                continue
-
-            # extract the nodeid
-            nodeid, values = values[0], values[1:]
-            found[nodeid] = 1
-
-            # see if the node exists
-            if cl.hasnode(nodeid):
-                exists = 1
-            else:
-                exists = 0
-
-            # confirm correct weight
-            if len(idlessprops) != len(values):
-                self.error_message.append(
-                    _('Not enough values on line %(line)s')%{'line':line})
-                return
-
-            # extract the new values
-            d = {}
-            for name, value in zip(idlessprops, values):
-                prop = cl.properties[name]
-                value = value.strip()
-                # only add the property if it has a value
-                if value:
-                    # if it's a multilink, split it
-                    if isinstance(prop, hyperdb.Multilink):
-                        value = value.split(':')
-                    d[name] = value
-                elif exists:
-                    # nuke the existing value
-                    if isinstance(prop, hyperdb.Multilink):
-                        d[name] = []
-                    else:
-                        d[name] = None
-
-            # perform the edit
-            if exists:
-                # edit existing
-                cl.set(nodeid, **d)
-            else:
-                # new node
-                found[cl.create(**d)] = 1
-
-        # retire the removed entries
-        for nodeid in cl.list():
-            if not found.has_key(nodeid):
-                cl.retire(nodeid)
-
-        # all OK
-        self.db.commit()
-
-        self.ok_message.append(_('Items edited OK'))
+            if offset:
+                f.seek(offset)
+            content = f.read(length)
+        finally:
+            f.close()
+        self.write(content)
 
-    def editCSVPermission(self):
-        ''' Determine whether the user has permission to edit this class.
+    def setHeader(self, header, value):
+        """Override a header to be returned to the user's browser.
+        """
+        self.additional_headers[header] = value
 
-            Base behaviour is to check the user can edit this class.
-        ''' 
-        if not self.db.security.hasPermission('Edit', self.userid,
-                self.classname):
-            return 0
-        return 1
-
-    def searchAction(self, wcre=re.compile(r'[\s,]+')):
-        ''' Mangle some of the form variables.
-
-            Set the form ":filter" variable based on the values of the
-            filter variables - if they're set to anything other than
-            "dontcare" then add them to :filter.
-
-            Handle the ":queryname" variable and save off the query to
-            the user's query list.
-
-            Split any String query values on whitespace and comma.
-        '''
-        # generic edit is per-class only
-        if not self.searchPermission():
-            self.error_message.append(
-                _('You do not have permission to search %s' %self.classname))
-
-        # add a faked :filter form variable for each filtering prop
-        props = self.db.classes[self.classname].getprops()
-        queryname = ''
-        for key in self.form.keys():
-            # special vars
-            if self.FV_QUERYNAME.match(key):
-                queryname = self.form[key].value.strip()
-                continue
+    def header(self, headers=None, response=None):
+        """Put up the appropriate header.
+        """
+        if headers is None:
+            headers = {'Content-Type':'text/html; charset=utf-8'}
+        if response is None:
+            response = self.response_code
 
-            if not props.has_key(key):
-                continue
-            if isinstance(self.form[key], type([])):
-                # search for at least one entry which is not empty
-                for minifield in self.form[key]:
-                    if minifield.value:
-                        break
-                else:
-                    continue
-            else:
-                if not self.form[key].value:
-                    continue
-                if isinstance(props[key], hyperdb.String):
-                    v = self.form[key].value
-                    l = token.token_split(v)
-                    if len(l) > 1 or l[0] != v:
-                        self.form.value.remove(self.form[key])
-                        # replace the single value with the split list
-                        for v in l:
-                            self.form.value.append(cgi.MiniFieldStorage(key, v))
-
-            self.form.value.append(cgi.MiniFieldStorage('@filter', key))
-
-        # handle saving the query params
-        if queryname:
-            # parse the environment and figure what the query _is_
-            req = HTMLRequest(self)
-            url = req.indexargs_href('', {})
-
-            # handle editing an existing query
-            try:
-                qid = self.db.query.lookup(queryname)
-                self.db.query.set(qid, klass=self.classname, url=url)
-            except KeyError:
-                # create a query
-                qid = self.db.query.create(name=queryname,
-                    klass=self.classname, url=url)
+        # update with additional info
+        headers.update(self.additional_headers)
 
-                # and add it to the user's query multilink
-                queries = self.db.user.get(self.userid, 'queries')
-                queries.append(qid)
-                self.db.user.set(self.userid, queries=queries)
+        if headers.get('Content-Type', 'text/html') == 'text/html':
+            headers['Content-Type'] = 'text/html; charset=utf-8'
 
-            # commit the query change to the database
-            self.db.commit()
+        headers = list(headers.items())
 
-    def searchPermission(self):
-        ''' Determine whether the user has permission to search this class.
+        for ((path, name), (value, expire)) in self._cookies.iteritems():
+            cookie = "%s=%s; Path=%s;"%(name, value, path)
+            if expire is not None:
+                cookie += " expires=%s;"%get_cookie_date(expire)
+            headers.append(('Set-Cookie', cookie))
 
-            Base behaviour is to check the user can view this class.
-        ''' 
-        if not self.db.security.hasPermission('View', self.userid,
-                self.classname):
-            return 0
-        return 1
+        self._socket_op(self.request.start_response, headers, response)
 
+        self.headers_done = 1
+        if self.debug:
+            self.headers_sent = headers
 
-    def retireAction(self):
-        ''' Retire the context item.
-        '''
-        # if we want to view the index template now, then unset the nodeid
-        # context info (a special-case for retire actions on the index page)
-        nodeid = self.nodeid
-        if self.template == 'index':
-            self.nodeid = None
-
-        # generic edit is per-class only
-        if not self.retirePermission():
-            self.error_message.append(
-                _('You do not have permission to retire %s' %self.classname))
-            return
+    def add_cookie(self, name, value, expire=86400*365, path=None):
+        """Set a cookie value to be sent in HTTP headers
+
+        Parameters:
+            name:
+                cookie name
+            value:
+                cookie value
+            expire:
+                cookie expiration time (seconds).
+                If value is empty (meaning "delete cookie"),
+                expiration time is forced in the past
+                and this argument is ignored.
+                If None, the cookie will expire at end-of-session.
+                If omitted, the cookie will be kept for a year.
+            path:
+                cookie path (optional)
+
+        """
+        if path is None:
+            path = self.cookie_path
+        if not value:
+            expire = -1
+        self._cookies[(path, name)] = (value, expire)
+
+    def set_cookie(self, user, expire=None):
+        """Deprecated. Use session_api calls directly
+
+        XXX remove
+        """
+
+        # insert the session in the session db
+        self.session_api.set(user=user)
+        # refresh session cookie
+        self.session_api.update(set_cookie=True, expire=expire)
 
-        # make sure we don't try to retire admin or anonymous
-        if self.classname == 'user' and \
-                self.db.user.get(nodeid, 'username') in ('admin', 'anonymous'):
-            self.error_message.append(
-                _('You may not retire the admin or anonymous user'))
-            return
+    def make_user_anonymous(self):
+        """ Make us anonymous
 
-        # do the retire
-        self.db.getclass(self.classname).retire(nodeid)
-        self.db.commit()
+            This method used to handle non-existence of the 'anonymous'
+            user, but that user is mandatory now.
+        """
+        self.userid = self.db.user.lookup('anonymous')
+        self.user = 'anonymous'
 
-        self.ok_message.append(
-            _('%(classname)s %(itemid)s has been retired')%{
-                'classname': self.classname.capitalize(), 'itemid': nodeid})
+    def standard_message(self, to, subject, body, author=None):
+        """Send a standard email message from Roundup.
 
-    def retirePermission(self):
-        ''' Determine whether the user has permission to retire this class.
+        "to"      - recipients list
+        "subject" - Subject
+        "body"    - Message
+        "author"  - (name, address) tuple or None for admin email
 
-            Base behaviour is to check the user can edit this class.
-        ''' 
-        if not self.db.security.hasPermission('Edit', self.userid,
-                self.classname):
+        Arguments are passed to the Mailer.standard_message code.
+        """
+        try:
+            self.mailer.standard_message(to, subject, body, author)
+        except MessageSendError, e:
+            self.error_message.append(str(e))
             return 0
         return 1
 
+    def parsePropsFromForm(self, create=0):
+        return FormParser(self).parse(create=create)
 
-    def showAction(self, typere=re.compile('[@:]type'),
-            numre=re.compile('[@:]number')):
-        ''' Show a node of a particular class/id
-        '''
-        t = n = ''
-        for key in self.form.keys():
-            if typere.match(key):
-                t = self.form[key].value.strip()
-            elif numre.match(key):
-                n = self.form[key].value.strip()
-        if not t:
-            raise ValueError, 'Invalid %s number'%t
-        url = '%s%s%s'%(self.db.config.TRACKER_WEB, t, n)
-        raise Redirect, url
-
-    def parsePropsFromForm(self, num_re=re.compile('^\d+$')):
-        ''' Pull properties out of the form.
-
-            In the following, <bracketed> values are variable, ":" may be
-            one of ":" or "@", and other text "required" is fixed.
-
-            Properties are specified as form variables:
-
-             <propname>
-              - property on the current context item
-
-             <designator>:<propname>
-              - property on the indicated item
-
-             <classname>-<N>:<propname>
-              - property on the Nth new item of classname
-
-            Once we have determined the "propname", we check to see if it
-            is one of the special form values:
-
-             :required
-              The named property values must be supplied or a ValueError
-              will be raised.
-
-             :remove:<propname>=id(s)
-              The ids will be removed from the multilink property.
-
-             :add:<propname>=id(s)
-              The ids will be added to the multilink property.
-
-             :link:<propname>=<designator>
-              Used to add a link to new items created during edit.
-              These are collected up and returned in all_links. This will
-              result in an additional linking operation (either Link set or
-              Multilink append) after the edit/create is done using
-              all_props in _editnodes. The <propname> on the current item
-              will be set/appended the id of the newly created item of
-              class <designator> (where <designator> must be
-              <classname>-<N>).
-
-            Any of the form variables may be prefixed with a classname or
-            designator.
-
-            The return from this method is a dict of 
-                (classname, id): properties
-            ... this dict _always_ has an entry for the current context,
-            even if it's empty (ie. a submission for an existing issue that
-            doesn't result in any changes would return {('issue','123'): {}})
-            The id may be None, which indicates that an item should be
-            created.
-
-            If a String property's form value is a file upload, then we
-            try to set additional properties "filename" and "type" (if
-            they are valid for the class).
-
-            Two special form values are supported for backwards
-            compatibility:
-             :note - create a message (with content, author and date), link
-                     to the context item. This is ALWAYS desginated "msg-1".
-             :file - create a file, attach to the current item and any
-                     message created by :note. This is ALWAYS designated
-                     "file-1".
-
-            We also check that FileClass items have a "content" property with
-            actual content, otherwise we remove them from all_props before
-            returning.
-        '''
-        # some very useful variables
-        db = self.db
-        form = self.form
-
-        if not hasattr(self, 'FV_SPECIAL'):
-            # generate the regexp for handling special form values
-            classes = '|'.join(db.classes.keys())
-            # specials for parsePropsFromForm
-            # handle the various forms (see unit tests)
-            self.FV_SPECIAL = re.compile(self.FV_LABELS%classes, re.VERBOSE)
-            self.FV_DESIGNATOR = re.compile(r'(%s)([-\d]+)'%classes)
-
-        # these indicate the default class / item
-        default_cn = self.classname
-        default_cl = self.db.classes[default_cn]
-        default_nodeid = self.nodeid
-
-        # we'll store info about the individual class/item edit in these
-        all_required = {}       # one entry per class/item
-        all_props = {}          # one entry per class/item
-        all_propdef = {}        # note - only one entry per class
-        all_links = []          # as many as are required
-
-        # we should always return something, even empty, for the context
-        all_props[(default_cn, default_nodeid)] = {}
-
-        keys = form.keys()
-        timezone = db.getUserTimezone()
-
-        # sentinels for the :note and :file props
-        have_note = have_file = 0
-
-        # extract the usable form labels from the form
-        matches = []
-        for key in keys:
-            m = self.FV_SPECIAL.match(key)
-            if m:
-                matches.append((key, m.groupdict()))
-
-        # now handle the matches
-        for key, d in matches:
-            if d['classname']:
-                # we got a designator
-                cn = d['classname']
-                cl = self.db.classes[cn]
-                nodeid = d['id']
-                propname = d['propname']
-            elif d['note']:
-                # the special note field
-                cn = 'msg'
-                cl = self.db.classes[cn]
-                nodeid = '-1'
-                propname = 'content'
-                all_links.append((default_cn, default_nodeid, 'messages',
-                    [('msg', '-1')]))
-                have_note = 1
-            elif d['file']:
-                # the special file field
-                cn = 'file'
-                cl = self.db.classes[cn]
-                nodeid = '-1'
-                propname = 'content'
-                all_links.append((default_cn, default_nodeid, 'files',
-                    [('file', '-1')]))
-                have_file = 1
-            else:
-                # default
-                cn = default_cn
-                cl = default_cl
-                nodeid = default_nodeid
-                propname = d['propname']
-
-            # the thing this value relates to is...
-            this = (cn, nodeid)
-
-            # get more info about the class, and the current set of
-            # form props for it
-            if not all_propdef.has_key(cn):
-                all_propdef[cn] = cl.getprops()
-            propdef = all_propdef[cn]
-            if not all_props.has_key(this):
-                all_props[this] = {}
-            props = all_props[this]
-
-            # is this a link command?
-            if d['link']:
-                value = []
-                for entry in extractFormList(form[key]):
-                    m = self.FV_DESIGNATOR.match(entry)
-                    if not m:
-                        raise ValueError, \
-                            'link "%s" value "%s" not a designator'%(key, entry)
-                    value.append((m.group(1), m.group(2)))
-
-                # make sure the link property is valid
-                if (not isinstance(propdef[propname], hyperdb.Multilink) and
-                        not isinstance(propdef[propname], hyperdb.Link)):
-                    raise ValueError, '%s %s is not a link or '\
-                        'multilink property'%(cn, propname)
-
-                all_links.append((cn, nodeid, propname, value))
-                continue
-
-            # detect the special ":required" variable
-            if d['required']:
-                all_required[this] = extractFormList(form[key])
-                continue
-
-            # get the required values list
-            if not all_required.has_key(this):
-                all_required[this] = []
-            required = all_required[this]
-
-            # see if we're performing a special multilink action
-            mlaction = 'set'
-            if d['remove']:
-                mlaction = 'remove'
-            elif d['add']:
-                mlaction = 'add'
-
-            # does the property exist?
-            if not propdef.has_key(propname):
-                if mlaction != 'set':
-                    raise ValueError, 'You have submitted a %s action for'\
-                        ' the property "%s" which doesn\'t exist'%(mlaction,
-                        propname)
-                # the form element is probably just something we don't care
-                # about - ignore it
-                continue
-            proptype = propdef[propname]
-
-            # Get the form value. This value may be a MiniFieldStorage or a list
-            # of MiniFieldStorages.
-            value = form[key]
-
-            # handle unpacking of the MiniFieldStorage / list form value
-            if isinstance(proptype, hyperdb.Multilink):
-                value = extractFormList(value)
-            else:
-                # multiple values are not OK
-                if isinstance(value, type([])):
-                    raise ValueError, 'You have submitted more than one value'\
-                        ' for the %s property'%propname
-                # value might be a file upload...
-                if not hasattr(value, 'filename') or value.filename is None:
-                    # nope, pull out the value and strip it
-                    value = value.value.strip()
-
-            # now that we have the props field, we need a teensy little
-            # extra bit of help for the old :note field...
-            if d['note'] and value:
-                props['author'] = self.db.getuid()
-                props['date'] = date.Date()
-
-            # handle by type now
-            if isinstance(proptype, hyperdb.Password):
-                if not value:
-                    # ignore empty password values
-                    continue
-                for key, d in matches:
-                    if d['confirm'] and d['propname'] == propname:
-                        confirm = form[key]
-                        break
-                else:
-                    raise ValueError, 'Password and confirmation text do '\
-                        'not match'
-                if isinstance(confirm, type([])):
-                    raise ValueError, 'You have submitted more than one value'\
-                        ' for the %s property'%propname
-                if value != confirm.value:
-                    raise ValueError, 'Password and confirmation text do '\
-                        'not match'
-                value = password.Password(value)
-
-            elif isinstance(proptype, hyperdb.Link):
-                # see if it's the "no selection" choice
-                if value == '-1' or not value:
-                    # if we're creating, just don't include this property
-                    if not nodeid or nodeid.startswith('-'):
-                        continue
-                    value = None
-                else:
-                    # handle key values
-                    link = proptype.classname
-                    if not num_re.match(value):
-                        try:
-                            value = db.classes[link].lookup(value)
-                        except KeyError:
-                            raise ValueError, _('property "%(propname)s": '
-                                '%(value)s not a %(classname)s')%{
-                                'propname': propname, 'value': value,
-                                'classname': link}
-                        except TypeError, message:
-                            raise ValueError, _('you may only enter ID values '
-                                'for property "%(propname)s": %(message)s')%{
-                                'propname': propname, 'message': message}
-            elif isinstance(proptype, hyperdb.Multilink):
-                # perform link class key value lookup if necessary
-                link = proptype.classname
-                link_cl = db.classes[link]
-                l = []
-                for entry in value:
-                    if not entry: continue
-                    if not num_re.match(entry):
-                        try:
-                            entry = link_cl.lookup(entry)
-                        except KeyError:
-                            raise ValueError, _('property "%(propname)s": '
-                                '"%(value)s" not an entry of %(classname)s')%{
-                                'propname': propname, 'value': entry,
-                                'classname': link}
-                        except TypeError, message:
-                            raise ValueError, _('you may only enter ID values '
-                                'for property "%(propname)s": %(message)s')%{
-                                'propname': propname, 'message': message}
-                    l.append(entry)
-                l.sort()
-
-                # now use that list of ids to modify the multilink
-                if mlaction == 'set':
-                    value = l
-                else:
-                    # we're modifying the list - get the current list of ids
-                    if props.has_key(propname):
-                        existing = props[propname]
-                    elif nodeid and not nodeid.startswith('-'):
-                        existing = cl.get(nodeid, propname, [])
-                    else:
-                        existing = []
-
-                    # now either remove or add
-                    if mlaction == 'remove':
-                        # remove - handle situation where the id isn't in
-                        # the list
-                        for entry in l:
-                            try:
-                                existing.remove(entry)
-                            except ValueError:
-                                raise ValueError, _('property "%(propname)s": '
-                                    '"%(value)s" not currently in list')%{
-                                    'propname': propname, 'value': entry}
-                    else:
-                        # add - easy, just don't dupe
-                        for entry in l:
-                            if entry not in existing:
-                                existing.append(entry)
-                    value = existing
-                    value.sort()
-
-            elif value == '':
-                # if we're creating, just don't include this property
-                if not nodeid or nodeid.startswith('-'):
-                    continue
-                # other types should be None'd if there's no value
-                value = None
-            else:
-                # handle ValueErrors for all these in a similar fashion
-                try:
-                    if isinstance(proptype, hyperdb.String):
-                        if (hasattr(value, 'filename') and
-                                value.filename is not None):
-                            # skip if the upload is empty
-                            if not value.filename:
-                                continue
-                            # this String is actually a _file_
-                            # try to determine the file content-type
-                            fn = value.filename.split('\\')[-1]
-                            if propdef.has_key('name'):
-                                props['name'] = fn
-                            # use this info as the type/filename properties
-                            if propdef.has_key('type'):
-                                props['type'] = mimetypes.guess_type(fn)[0]
-                                if not props['type']:
-                                    props['type'] = "application/octet-stream"
-                            # finally, read the content
-                            value = value.value
-                        else:
-                            # normal String fix the CRLF/CR -> LF stuff
-                            value = fixNewlines(value)
-
-                    elif isinstance(proptype, hyperdb.Date):
-                        value = date.Date(value, offset=timezone)
-                    elif isinstance(proptype, hyperdb.Interval):
-                        value = date.Interval(value)
-                    elif isinstance(proptype, hyperdb.Boolean):
-                        value = value.lower() in ('yes', 'true', 'on', '1')
-                    elif isinstance(proptype, hyperdb.Number):
-                        value = float(value)
-                except ValueError, msg:
-                    raise ValueError, _('Error with %s property: %s')%(
-                        propname, msg)
-
-            # get the old value
-            if nodeid and not nodeid.startswith('-'):
-                try:
-                    existing = cl.get(nodeid, propname)
-                except KeyError:
-                    # this might be a new property for which there is
-                    # no existing value
-                    if not propdef.has_key(propname):
-                        raise
-
-                # make sure the existing multilink is sorted
-                if isinstance(proptype, hyperdb.Multilink):
-                    existing.sort()
-
-                # "missing" existing values may not be None
-                if not existing:
-                    if isinstance(proptype, hyperdb.String) and not existing:
-                        # some backends store "missing" Strings as empty strings
-                        existing = None
-                    elif isinstance(proptype, hyperdb.Number) and not existing:
-                        # some backends store "missing" Numbers as 0 :(
-                        existing = 0
-                    elif isinstance(proptype, hyperdb.Boolean) and not existing:
-                        # likewise Booleans
-                        existing = 0
-
-                # if changed, set it
-                if value != existing:
-                    props[propname] = value
-            else:
-                # don't bother setting empty/unset values
-                if value is None:
-                    continue
-                elif isinstance(proptype, hyperdb.Multilink) and value == []:
-                    continue
-                elif isinstance(proptype, hyperdb.String) and value == '':
-                    continue
-
-                props[propname] = value
-
-            # register this as received if required?
-            if propname in required and value is not None:
-                required.remove(propname)
-
-        # check to see if we need to specially link a file to the note
-        if have_note and have_file:
-            all_links.append(('msg', '-1', 'files', [('file', '-1')]))
-
-        # see if all the required properties have been supplied
-        s = []
-        for thing, required in all_required.items():
-            if not required:
-                continue
-            if len(required) > 1:
-                p = 'properties'
-            else:
-                p = 'property'
-            s.append('Required %s %s %s not supplied'%(thing[0], p,
-                ', '.join(required)))
-        if s:
-            raise ValueError, '\n'.join(s)
-
-        # check that FileClass entries have a "content" property with
-        # content, otherwise remove them
-        for (cn, id), props in all_props.items():
-            cl = self.db.classes[cn]
-            if not isinstance(cl, hyperdb.FileClass):
-                continue
-            # we also don't want to create FileClass items with no content
-            if not props.get('content', ''):
-                del all_props[(cn, id)]
-        return all_props, all_links
-
-def fixNewlines(text):
-    ''' Homogenise line endings.
-
-        Different web clients send different line ending values, but
-        other systems (eg. email) don't necessarily handle those line
-        endings. Our solution is to convert all line endings to LF.
-    '''
-    text = text.replace('\r\n', '\n')
-    return text.replace('\r', '\n')
-
-def extractFormList(value):
-    ''' Extract a list of values from the form value.
-
-        It may be one of:
-         [MiniFieldStorage('value'), MiniFieldStorage('value','value',...), ...]
-         MiniFieldStorage('value,value,...')
-         MiniFieldStorage('value')
-    '''
-    # multiple values are OK
-    if isinstance(value, type([])):
-        # it's a list of MiniFieldStorages - join then into
-        values = ','.join([i.value.strip() for i in value])
-    else:
-        # it's a MiniFieldStorage, but may be a comma-separated list
-        # of values
-        values = value.value
-
-    value = [i.strip() for i in values.split(',')]
-
-    # filter out the empty bits
-    return filter(None, value)
-
+# vim: set et sts=4 sw=4 :