From c1fb0c81c783a6741f47e3f4abcd78a3f1cad1ad Mon Sep 17 00:00:00 2001 From: richard Date: Sat, 1 Dec 2001 07:17:50 +0000 Subject: [PATCH] . We now have basic transaction support! Information is only written to the database when the commit() method is called. Only the anydbm backend is modified in this way - neither of the bsddb backends have been. The mail, admin and cgi interfaces all use commit (except the admin tool doesn't have a commit command, so interactive users can't commit...) . Fixed login/registration forwarding the user to the right page (or not, on a failure) git-svn-id: http://svn.roundup-tracker.org/svnroot/roundup/trunk@442 57a73879-2fb5-44c3-a270-3262357dd7e2 --- CHANGES.txt | 5 + roundup-admin | 9 +- roundup/backends/back_anydbm.py | 120 ++++++++++++++----- roundup/cgi_client.py | 166 ++++++++++++++++++--------- roundup/hyperdb.py | 38 +++--- roundup/mailgw.py | 15 ++- roundup/templates/classic/dbinit.py | 16 ++- roundup/templates/extended/dbinit.py | 15 ++- 8 files changed, 278 insertions(+), 106 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 7bb068a..62bf811 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -10,12 +10,17 @@ Feature: . Some more flexibility in the mail gateway and more error handling. . Login now takes you to the page you back to the were denied access to. . Admin user now can has a user index link on their web interface. + . We now have basic transaction support. Information is only written to + the database when the commit() method is called. Only the anydbm backend + is modified in this way - neither of the bsddb backends have been. Fixed: . Lots of bugs, thanks Roché and others on the devel mailing list! . login_action and newuser_action return values were being ignored . Woohoo! Found that bloody re-login bug that was killing the mail gateway. + . Fixed login/registration forwarding the user to the right page (or not, + on a failure) 2001-11-23 - 0.3.0 diff --git a/roundup-admin b/roundup-admin index 984485e..e5f48a6 100755 --- a/roundup-admin +++ b/roundup-admin @@ -16,7 +16,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: roundup-admin,v 1.48 2001-11-27 22:32:03 richard Exp $ +# $Id: roundup-admin,v 1.49 2001-12-01 07:17:50 richard Exp $ import sys if int(sys.version[0]) < 2: @@ -882,8 +882,8 @@ Command help: self.interactive() else: ret = self.run_command(args) - if self.db: - self.db.close() + if self.db: self.db.commit() + if self.db: self.db.close() return ret @@ -893,6 +893,9 @@ if __name__ == '__main__': # # $Log: not supported by cvs2svn $ +# Revision 1.48 2001/11/27 22:32:03 richard +# typo +# # Revision 1.47 2001/11/26 22:55:56 richard # Feature: # . Added INSTANCE_NAME to configuration - used in web and email to identify diff --git a/roundup/backends/back_anydbm.py b/roundup/backends/back_anydbm.py index eb2c143..38305b5 100644 --- a/roundup/backends/back_anydbm.py +++ b/roundup/backends/back_anydbm.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -#$Id: back_anydbm.py,v 1.11 2001-11-21 02:34:18 richard Exp $ +#$Id: back_anydbm.py,v 1.12 2001-12-01 07:17:50 richard Exp $ import anydbm, os, marshal from roundup import hyperdb, date, password @@ -24,7 +24,14 @@ from roundup import hyperdb, date, password # Now the database # class Database(hyperdb.Database): - """A database for storing records containing flexible data types.""" + """A database for storing records containing flexible data types. + + Transaction stuff TODO: + . check the timestamp of the class file and nuke the cache if it's + modified. Do some sort of conflict checking on the dirty stuff. + . perhaps detect write collisions (related to above)? + + """ def __init__(self, storagelocator, journaltag=None): """Open a hyperdatabase given a specifier to some storage. @@ -41,8 +48,10 @@ class Database(hyperdb.Database): """ self.dir, self.journaltag = storagelocator, journaltag self.classes = {} + self.cache = {} # cache of nodes loaded or created + self.dirtynodes = {} # keep track of the dirty nodes by class + self.newnodes = {} # keep track of the new nodes by class self.transactions = [] - # # Classes # @@ -95,39 +104,70 @@ class Database(hyperdb.Database): def addnode(self, classname, nodeid, node): ''' add the specified node to its class's db ''' - db = self.getclassdb(classname, 'c') - # now save the marshalled data - db[nodeid] = marshal.dumps(node) - db.close() - setnode = addnode + self.newnodes.setdefault(classname, {})[nodeid] = 1 + self.cache.setdefault(classname, {})[nodeid] = node + self.savenode(classname, nodeid, node) + + def setnode(self, classname, nodeid, node): + ''' change the specified node + ''' + self.dirtynodes.setdefault(classname, {})[nodeid] = 1 + # can't set without having already loaded the node + self.cache[classname][nodeid] = node + self.savenode(classname, nodeid, node) + + def savenode(self, classname, nodeid, node): + ''' perform the saving of data specified by the set/addnode + ''' + self.transactions.append((self._doSaveNode, (classname, nodeid, node))) def getnode(self, classname, nodeid, cldb=None): ''' add the specified node to its class's db ''' + # try the cache + cache = self.cache.setdefault(classname, {}) + if cache.has_key(nodeid): + return cache[nodeid] + + # get from the database and save in the cache db = cldb or self.getclassdb(classname) if not db.has_key(nodeid): raise IndexError, nodeid res = marshal.loads(db[nodeid]) if not cldb: db.close() + cache[nodeid] = res return res def hasnode(self, classname, nodeid, cldb=None): ''' add the specified node to its class's db ''' + # try the cache + cache = self.cache.setdefault(classname, {}) + if cache.has_key(nodeid): + return 1 + + # not in the cache - check the database db = cldb or self.getclassdb(classname) res = db.has_key(nodeid) if not cldb: db.close() return res def countnodes(self, classname, cldb=None): + # include the new nodes not saved to the DB yet + count = len(self.newnodes.get(classname, {})) + + # and count those in the DB db = cldb or self.getclassdb(classname) - return len(db.keys()) + count = count + len(db.keys()) if not cldb: db.close() - return res + return count def getnodeids(self, classname, cldb=None): + # start off with the new nodes + res = self.newnodes.get(classname, {}).keys() + db = cldb or self.getclassdb(classname) - res = db.keys() + res = res + db.keys() if not cldb: db.close() return res @@ -142,17 +182,8 @@ class Database(hyperdb.Database): 'link' or 'unlink' -- 'params' is (classname, nodeid, propname) 'retire' -- 'params' is None ''' - entry = (nodeid, date.Date().get_tuple(), self.journaltag, action, - params) - db = anydbm.open(os.path.join(self.dir, 'journals.%s'%classname), 'c') - if db.has_key(nodeid): - s = db[nodeid] - l = marshal.loads(db[nodeid]) - l.append(entry) - else: - l = [entry] - db[nodeid] = marshal.dumps(l) - db.close() + self.transactions.append((self._doSaveJournal, (classname, nodeid, + action, params))) def getjournal(self, classname, nodeid): ''' get the journal for id @@ -175,9 +206,10 @@ class Database(hyperdb.Database): return res def close(self): - ''' Close the Database - we must release the circular refs so that - we can be del'ed and the underlying anydbm connections closed - cleanly. + ''' Close the Database. + + Commit all data to the database and release circular refs so + the database is closed cleanly. ''' self.classes = {} @@ -189,18 +221,50 @@ class Database(hyperdb.Database): ''' Commit the current transactions. ''' # lock the DB - for action, classname, entry in self.transactions: - # write the node, figure what's changed for the journal. - pass + for method, args in self.transactions: + print method.__name__, args + # TODO: optimise this, duh! + method(*args) # unlock the DB + # all transactions committed, back to normal + self.cache = {} + self.dirtynodes = {} + self.newnodes = {} + self.transactions = [] + + def _doSaveNode(self, classname, nodeid, node): + db = self.getclassdb(classname, 'c') + # now save the marshalled data + db[nodeid] = marshal.dumps(node) + db.close() + + def _doSaveJournal(self, classname, nodeid, action, params): + entry = (nodeid, date.Date().get_tuple(), self.journaltag, action, + params) + db = anydbm.open(os.path.join(self.dir, 'journals.%s'%classname), 'c') + if db.has_key(nodeid): + s = db[nodeid] + l = marshal.loads(db[nodeid]) + l.append(entry) + else: + l = [entry] + db[nodeid] = marshal.dumps(l) + db.close() + def rollback(self): ''' Reverse all actions from the current transaction. ''' + self.cache = {} + self.dirtynodes = {} + self.newnodes = {} self.transactions = [] # #$Log: not supported by cvs2svn $ +#Revision 1.11 2001/11/21 02:34:18 richard +#Added a target version field to the extended issue schema +# #Revision 1.10 2001/10/09 23:58:10 richard #Moved the data stringification up into the hyperdb.Class class' get, set #and create methods. This means that the data is also stringified for the diff --git a/roundup/cgi_client.py b/roundup/cgi_client.py index 66c923a..451ae5b 100644 --- a/roundup/cgi_client.py +++ b/roundup/cgi_client.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: cgi_client.py,v 1.72 2001-11-30 20:47:58 rochecompaan Exp $ +# $Id: cgi_client.py,v 1.73 2001-12-01 07:17:50 richard Exp $ __doc__ = """ WWW request handler (also used in the stand-alone server). @@ -617,6 +617,8 @@ class Client: raise Unauthorised def login(self, message=None, newuser_form=None, action='index'): + '''Display a login page. + ''' self.pagehead(_('Login to roundup'), message) self.write(_(''' @@ -634,9 +636,10 @@ class Client: if self.user is None and self.ANONYMOUS_REGISTER == 'deny': self.write('
') self.pagefoot() - return 1 + return values = {'realname': '', 'organisation': '', 'address': '', - 'phone': '', 'username': '', 'password': '', 'confirm': ''} + 'phone': '', 'username': '', 'password': '', 'confirm': '', + 'action': action} if newuser_form is not None: for key in newuser_form.keys(): values[key] = newuser_form[key].value @@ -645,6 +648,7 @@ class Client: New User Registration marked items are optional...
+ Name: Organisation: @@ -667,8 +671,14 @@ class Client: self.pagefoot() def login_action(self, message=None): + '''Attempt to log a user in and set the cookie + + returns 0 if a page is generated as a result of this call, and + 1 if not (ie. the login is successful + ''' if not self.form.has_key('__login_name'): - return self.login(message=_('Username required')) + self.login(message=_('Username required')) + return 0 self.user = self.form['__login_name'].value if self.form.has_key('__login_password'): password = self.form['__login_password'].value @@ -681,18 +691,44 @@ class Client: name = self.user self.make_user_anonymous() action = self.form['__destination_url'].value - return self.login(message=_('No such user "%(name)s"')%locals(), - action=action) + self.login(message=_('No such user "%(name)s"')%locals(), + action=action) + return 0 # and that the password is correct pw = self.db.user.get(uid, 'password') - if password != self.db.user.get(uid, 'password'): + if password != pw: self.make_user_anonymous() action = self.form['__destination_url'].value - return self.login(message=_('Incorrect password'), action=action) + self.login(message=_('Incorrect password'), action=action) + return 0 self.set_cookie(self.user, password) - return None # make it explicit + return 1 + + def newuser_action(self, message=None): + '''Attempt to create a new user based on the contents of the form + and then set the cookie. + + return 1 on successful login + ''' + # re-open the database as "admin" + self.db.close() + self.db = self.instance.open('admin') + + # TODO: pre-check the required fields and username key property + cl = self.db.user + try: + props, dummy = parsePropsFromForm(self.db, cl, self.form) + uid = cl.create(**props) + except ValueError, message: + action = self.form['__destination_url'].value + self.login(message, action=action) + return 0 + self.user = cl.get(uid, 'username') + password = cl.get(uid, 'password') + self.set_cookie(self.user, self.form['password'].value) + return 1 def set_cookie(self, user, password): # construct the cookie @@ -723,31 +759,33 @@ class Client: self.header({'Set-Cookie': 'roundup_user=deleted; Max-Age=0; expires=%s; Path=%s;'%(now, path)}) - return self.login() + self.login() - def newuser_action(self, message=None): - ''' create a new user based on the contents of the form and then - set the cookie + + def main(self): + '''Wrap the database accesses so we can close the database cleanly ''' - # re-open the database as "admin" - self.db.close() + # determine the uid to use self.db = self.instance.open('admin') + try: + self.main_user() + finally: + self.db.close() - # TODO: pre-check the required fields and username key property - cl = self.db.user + # re-open the database for real, using the user + self.db = self.instance.open(self.user) try: - props, dummy = parsePropsFromForm(self.db, cl, self.form) - uid = cl.create(**props) - except ValueError, message: - return self.login(message, newuser_form=self.form) - self.user = cl.get(uid, 'username') - password = cl.get(uid, 'password') - self.set_cookie(self.user, self.form['password'].value) - return None # make the None explicit + self.main_action() + except: + self.db.close() + raise + self.db.commit() + self.db.close() - def main(self): - # determine the uid to use - self.db = self.instance.open('admin') + + def main_user(self): + '''Figure out who the user is + ''' cookie = Cookie.Cookie(self.env.get('HTTP_COOKIE', '')) user = 'anonymous' if (cookie.has_key('roundup_user') and @@ -775,11 +813,12 @@ class Client: self.make_user_anonymous() else: self.user = user - self.db.close() - # re-open the database for real, using the user - self.db = self.instance.open(self.user) + def main_action(self): + '''Check for appropriate access permission, and then perform the + action the users specifies + ''' # now figure which function to call path = self.split_path @@ -796,15 +835,15 @@ class Client: # everyone is allowed to try to log in if action == 'login_action': - # do the login - ret = self.login_action() - if ret is not None: - return ret + # try to login + if not self.login_action(): + return # figure the resulting page action = self.form['__destination_url'].value if not action: action = 'index' - return self.do_action(action) + self.do_action(action) + return # allow anonymous people to register if action == 'newuser_action': @@ -812,40 +851,46 @@ class Client: # register, then spit up the login form if self.ANONYMOUS_REGISTER == 'deny' and self.user is None: if action == 'login': - return self.login() # go to the index after login + self.login() # go to the index after login else: - return self.login(action=action) - # add the user - ret = self.newuser_action() - if ret is not None: - return ret + self.login(action=action) + return + # try to add the user + if not self.newuser_action(): + return # figure the resulting page action = self.form['__destination_url'].value if not action: action = 'index' - return self.do_action(action) # no login or registration, make sure totally anonymous access is OK - if self.ANONYMOUS_ACCESS == 'deny' and self.user is None: + elif self.ANONYMOUS_ACCESS == 'deny' and self.user is None: if action == 'login': - return self.login() # go to the index after login + self.login() # go to the index after login else: - return self.login(action=action) + self.login(action=action) + return # just a regular action - return self.do_action(action) + self.do_action(action) def do_action(self, action, dre=re.compile(r'([^\d]+)(\d+)'), nre=re.compile(r'new(\w+)')): + '''Figure the user's action and do it. + ''' # here be the "normal" functionality if action == 'index': - return self.index() + self.index() + return if action == 'list_classes': - return self.classes() + self.classes() + return if action == 'login': - return self.login() + self.login() + return if action == 'logout': - return self.logout() + self.logout() + return m = dre.match(action) if m: self.classname = m.group(1) @@ -862,7 +907,8 @@ class Client: func = getattr(self, 'show%s'%self.classname) except AttributeError: raise NotFound - return func() + func() + return m = nre.match(action) if m: self.classname = m.group(1) @@ -870,16 +916,14 @@ class Client: func = getattr(self, 'new%s'%self.classname) except AttributeError: raise NotFound - return func() + func() + return self.classname = action try: self.db.getclass(self.classname) except KeyError: raise NotFound - return self.list() - - def __del__(self): - self.db.close() + self.list() class ExtendedClient(Client): @@ -1027,6 +1071,14 @@ def parsePropsFromForm(db, cl, form, nodeid=0): # # $Log: not supported by cvs2svn $ +# Revision 1.72 2001/11/30 20:47:58 rochecompaan +# Links in page header are now consistent with default sort order. +# +# Fixed bugs: +# - When login failed the list of issues were still rendered. +# - User was redirected to index page and not to his destination url +# if his first login attempt failed. +# # Revision 1.71 2001/11/30 20:28:10 rochecompaan # Property changes are now completely traceable, whether changes are # made through the web or by email diff --git a/roundup/hyperdb.py b/roundup/hyperdb.py index 10eaf38..779db22 100644 --- a/roundup/hyperdb.py +++ b/roundup/hyperdb.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: hyperdb.py,v 1.37 2001-11-28 21:55:35 richard Exp $ +# $Id: hyperdb.py,v 1.38 2001-12-01 07:17:50 richard Exp $ __doc__ = """ Hyperdatabase implementation, especially field types. @@ -244,23 +244,27 @@ class Class: IndexError is raised. 'propname' must be the name of a property of this class or a KeyError is raised. """ - d = self.db.getnode(self.classname, nodeid) - - # convert the marshalled data to instances - for key, prop in self.properties.items(): - if isinstance(prop, Date): - d[key] = date.Date(d[key]) - elif isinstance(prop, Interval): - d[key] = date.Interval(d[key]) - elif isinstance(prop, Password): - p = password.Password() - p.unpack(d[key]) - d[key] = p - if propname == 'id': return nodeid + + # get the node's dict + d = self.db.getnode(self.classname, nodeid) if not d.has_key(propname) and default is not _marker: return default + + # get the value + prop = self.properties[propname] + + # possibly convert the marshalled data to instances + if isinstance(prop, Date): + return date.Date(d[propname]) + elif isinstance(prop, Interval): + return date.Interval(d[propname]) + elif isinstance(prop, Password): + p = password.Password() + p.unpack(d[propname]) + return p + return d[propname] # XXX not in spec @@ -869,6 +873,12 @@ def Choice(name, *options): # # $Log: not supported by cvs2svn $ +# Revision 1.37 2001/11/28 21:55:35 richard +# . login_action and newuser_action return values were being ignored +# . Woohoo! Found that bloody re-login bug that was killing the mail +# gateway. +# (also a minor cleanup in hyperdb) +# # Revision 1.36 2001/11/27 03:16:09 richard # Another place that wasn't handling missing properties. # diff --git a/roundup/mailgw.py b/roundup/mailgw.py index 5ef6e2c..fb4c84b 100644 --- a/roundup/mailgw.py +++ b/roundup/mailgw.py @@ -73,7 +73,7 @@ are calling the create() method to create a new node). If an auditor raises an exception, the original message is bounced back to the sender with the explanatory message given in the exception. -$Id: mailgw.py,v 1.37 2001-11-28 21:55:35 richard Exp $ +$Id: mailgw.py,v 1.38 2001-12-01 07:17:50 richard Exp $ ''' @@ -355,6 +355,8 @@ Subject was: "%s" self.db.close() self.db = self.instance.open(username) + self.handle_message(author, username, + # re-get the class with the new database connection cl = self.db.getclass(classname) @@ -516,6 +518,8 @@ Subject was: "%s" There was a problem with the message you sent: %s '''%message + # commit the changes to the DB + self.db.commit() else: # If just an item class name is found there, we attempt to create a # new item of that class with its "messages" property initialized to @@ -554,6 +558,9 @@ There was a problem with the message you sent: %s '''%message + # commit the new node(s) to the DB + self.db.commit() + def parseContent(content, blank_line=re.compile(r'[\r\n]+\s*[\r\n]+'), eol=re.compile(r'[\r\n]+'), signature=re.compile(r'^[>|\s]*[-_]+\s*$')): ''' The message body is divided into sections by blank lines. @@ -594,6 +601,12 @@ def parseContent(content, blank_line=re.compile(r'[\r\n]+\s*[\r\n]+'), # # $Log: not supported by cvs2svn $ +# Revision 1.37 2001/11/28 21:55:35 richard +# . login_action and newuser_action return values were being ignored +# . Woohoo! Found that bloody re-login bug that was killing the mail +# gateway. +# (also a minor cleanup in hyperdb) +# # Revision 1.36 2001/11/26 22:55:56 richard # Feature: # . Added INSTANCE_NAME to configuration - used in web and email to identify diff --git a/roundup/templates/classic/dbinit.py b/roundup/templates/classic/dbinit.py index bb389d7..4a9e2b0 100644 --- a/roundup/templates/classic/dbinit.py +++ b/roundup/templates/classic/dbinit.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: dbinit.py,v 1.10 2001-11-26 22:55:56 richard Exp $ +# $Id: dbinit.py,v 1.11 2001-12-01 07:17:50 richard Exp $ import os @@ -123,11 +123,23 @@ def init(adminpw): user = db.getclass('user') user.create(username="admin", password=adminpw, address=instance_config.ADMIN_EMAIL) - + db.commit() db.close() # # $Log: not supported by cvs2svn $ +# Revision 1.10 2001/11/26 22:55:56 richard +# Feature: +# . Added INSTANCE_NAME to configuration - used in web and email to identify +# the instance. +# . Added EMAIL_SIGNATURE_POSITION to indicate where to place the roundup +# signature info in e-mails. +# . Some more flexibility in the mail gateway and more error handling. +# . Login now takes you to the page you back to the were denied access to. +# +# Fixed: +# . Lots of bugs, thanks Roché and others on the devel mailing list! +# # Revision 1.9 2001/10/30 00:54:45 richard # Features: # . #467129 ] Lossage when username=e-mail-address diff --git a/roundup/templates/extended/dbinit.py b/roundup/templates/extended/dbinit.py index 2a92ea7..e97c284 100644 --- a/roundup/templates/extended/dbinit.py +++ b/roundup/templates/extended/dbinit.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: dbinit.py,v 1.15 2001-11-26 22:55:56 richard Exp $ +# $Id: dbinit.py,v 1.16 2001-12-01 07:17:50 richard Exp $ import os @@ -174,10 +174,23 @@ def init(adminpw): user.create(username="admin", password=adminpw, address=instance_config.ADMIN_EMAIL) + db.commit() db.close() # # $Log: not supported by cvs2svn $ +# Revision 1.15 2001/11/26 22:55:56 richard +# Feature: +# . Added INSTANCE_NAME to configuration - used in web and email to identify +# the instance. +# . Added EMAIL_SIGNATURE_POSITION to indicate where to place the roundup +# signature info in e-mails. +# . Some more flexibility in the mail gateway and more error handling. +# . Login now takes you to the page you back to the were denied access to. +# +# Fixed: +# . Lots of bugs, thanks Roché and others on the devel mailing list! +# # Revision 1.14 2001/11/21 02:34:18 richard # Added a target version field to the extended issue schema # -- 2.30.2