From ed1f70d9af02711dafa92e950fc9bb79a23548f2 Mon Sep 17 00:00:00 2001 From: schlatterbeck Date: Thu, 7 Oct 2010 12:22:10 +0000 Subject: [PATCH] - refactor: move import_journal to hyperdb -- it was reimplemented in back_anydbm and rdbms_common and was already inconsistent -- and it doen't have any backend dependencies itself. Interestingly this now fails a test for memorydb (but passes for anydbm: Huh?) git-svn-id: http://svn.roundup-tracker.org/svnroot/roundup/roundup/trunk@4538 57a73879-2fb5-44c3-a270-3262357dd7e2 --- CHANGES.txt | 3 +- roundup/backends/back_anydbm.py | 28 ------------------ roundup/backends/rdbms_common.py | 49 ------------------------------- roundup/hyperdb.py | 50 ++++++++++++++++++++++++++++++++ 4 files changed, 51 insertions(+), 79 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 246023e..0121458 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -44,8 +44,7 @@ Fixed: Thanks to Benni Bärmann for reporting. - Allow search_popup macro to work with all db classes, issue2550567 (thanks John Kristensen) -- lower memory footprint for (journal-) import -- only for rdbms - backends, other backends shouldn't have that much data anyway. +- lower memory footprint for (journal-) import 2010-07-12 1.4.15 diff --git a/roundup/backends/back_anydbm.py b/roundup/backends/back_anydbm.py index 463a43f..3d132ec 100644 --- a/roundup/backends/back_anydbm.py +++ b/roundup/backends/back_anydbm.py @@ -2000,34 +2000,6 @@ class Class(hyperdb.Class): repr(action), repr(params)]) return r - def import_journals(self, entries): - """Import a class's journal. - - Uses setjournal() to set the journal for each item.""" - properties = self.getprops() - d = {} - for l in entries: - nodeid, jdate, user, action, params = tuple(map(eval, l)) - r = d.setdefault(nodeid, []) - if action == 'set': - for propname, value in params.iteritems(): - prop = properties[propname] - if value is None: - pass - elif isinstance(prop, hyperdb.Date): - value = date.Date(value) - elif isinstance(prop, hyperdb.Interval): - value = date.Interval(value) - elif isinstance(prop, hyperdb.Password): - pwd = password.Password() - pwd.unpack(value) - value = pwd - params[propname] = value - r.append((nodeid, date.Date(jdate), user, action, params)) - - for nodeid, l in d.iteritems(): - self.db.setjournal(self.classname, nodeid, l) - class FileClass(hyperdb.FileClass, Class): """This class defines a large chunk of data. To support this, it has a mandatory String property "content" which is typically saved off diff --git a/roundup/backends/rdbms_common.py b/roundup/backends/rdbms_common.py index 5b19850..13589a2 100644 --- a/roundup/backends/rdbms_common.py +++ b/roundup/backends/rdbms_common.py @@ -2641,55 +2641,6 @@ class Class(hyperdb.Class): r.append(list(map(repr, l))) return r - def import_journals(self, entries): - """Import a class's journal. - - Uses setjournal() to set the journal for each item. - Strategy for import: Sort first by id, then import journals for - each id, this way the memory footprint is a lot smaller than the - initial implementation which stored everything in a big hash by - id and then proceeded to import journals for each id.""" - properties = self.getprops() - a = [] - for l in entries: - # first element in sorted list is the (numeric) id - # in python2.4 and up we would use sorted with a key... - a.append ((int (l [0].strip ("'")), l)) - a.sort () - - - last = 0 - r = [] - for n, l in a: - nodeid, jdate, user, action, params = map(eval, l) - assert (str(n) == nodeid) - if n != last: - if r: - self.db.setjournal(self.classname, nodeid, r) - last = n - r = [] - - if action == 'set': - for propname, value in params.iteritems(): - prop = properties[propname] - if value is None: - pass - elif isinstance(prop, Date): - value = date.Date(value) - elif isinstance(prop, Interval): - value = date.Interval(value) - elif isinstance(prop, Password): - pwd = password.Password() - pwd.unpack(value) - value = pwd - params[propname] = value - elif action == 'create' and params: - # old tracker with data stored in the create! - params = {} - r.append((nodeid, date.Date(jdate), user, action, params)) - if r: - self.db.setjournal(self.classname, nodeid, r) - class FileClass(hyperdb.FileClass, Class): """This class defines a large chunk of data. To support this, it has a mandatory String property "content" which is typically saved off diff --git a/roundup/hyperdb.py b/roundup/hyperdb.py index 93f93b8..3a7ad59 100644 --- a/roundup/hyperdb.py +++ b/roundup/hyperdb.py @@ -1237,6 +1237,56 @@ class Class: propnames = self.getprops().keys() propnames.sort() return propnames + + def import_journals(self, entries): + """Import a class's journal. + + Uses setjournal() to set the journal for each item. + Strategy for import: Sort first by id, then import journals for + each id, this way the memory footprint is a lot smaller than the + initial implementation which stored everything in a big hash by + id and then proceeded to import journals for each id.""" + properties = self.getprops() + a = [] + for l in entries: + # first element in sorted list is the (numeric) id + # in python2.4 and up we would use sorted with a key... + a.append ((int (l [0].strip ("'")), l)) + a.sort () + + + last = 0 + r = [] + for n, l in a: + nodeid, jdate, user, action, params = map(eval, l) + assert (str(n) == nodeid) + if n != last: + if r: + self.db.setjournal(self.classname, nodeid, r) + last = n + r = [] + + if action == 'set': + for propname, value in params.iteritems(): + prop = properties[propname] + if value is None: + pass + elif isinstance(prop, Date): + value = date.Date(value) + elif isinstance(prop, Interval): + value = date.Interval(value) + elif isinstance(prop, Password): + pwd = password.Password() + pwd.unpack(value) + value = pwd + params[propname] = value + elif action == 'create' and params: + # old tracker with data stored in the create! + params = {} + r.append((nodeid, date.Date(jdate), user, action, params)) + if r: + self.db.setjournal(self.classname, nodeid, r) + # # convenience methods # -- 2.30.2