From: schlatterbeck <schlatterbeck@57a73879-2fb5-44c3-a270-3262357dd7e2>
Date: Thu, 7 Oct 2010 12:02:12 +0000 (+0000)
Subject: - lower memory footprint for (journal-) import -- only for rdbms
X-Git-Url: https://git.tokkee.org/?p=roundup.git;a=commitdiff_plain;h=a6ce8417393e9d362a47a686a8ba958e52fb39c1

- lower memory footprint for (journal-) import -- only for rdbms
  backends, other backends shouldn't have that much data anyway.


git-svn-id: http://svn.roundup-tracker.org/svnroot/roundup/roundup/trunk@4537 57a73879-2fb5-44c3-a270-3262357dd7e2
---

diff --git a/CHANGES.txt b/CHANGES.txt
index 320e28d..246023e 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -44,6 +44,8 @@ Fixed:
   Thanks to Benni Bärmann for reporting.
 - Allow search_popup macro to work with all db classes, issue2550567
   (thanks John Kristensen)
+- lower memory footprint for (journal-) import -- only for rdbms
+  backends, other backends shouldn't have that much data anyway.
 
 
 2010-07-12 1.4.15
diff --git a/roundup/backends/rdbms_common.py b/roundup/backends/rdbms_common.py
index 912dd81..5b19850 100644
--- a/roundup/backends/rdbms_common.py
+++ b/roundup/backends/rdbms_common.py
@@ -2644,12 +2644,31 @@ class Class(hyperdb.Class):
     def import_journals(self, entries):
         """Import a class's journal.
 
-        Uses setjournal() to set the journal for each item."""
+        Uses setjournal() to set the journal for each item.
+        Strategy for import: Sort first by id, then import journals for
+        each id, this way the memory footprint is a lot smaller than the
+        initial implementation which stored everything in a big hash by
+        id and then proceeded to import journals for each id."""
         properties = self.getprops()
-        d = {}
+        a = []
         for l in entries:
+            # first element in sorted list is the (numeric) id
+            # in python2.4 and up we would use sorted with a key...
+            a.append ((int (l [0].strip ("'")), l))
+        a.sort ()
+
+
+        last = 0
+        r = []
+        for n, l in a:
             nodeid, jdate, user, action, params = map(eval, l)
-            r = d.setdefault(nodeid, [])
+            assert (str(n) == nodeid)
+            if n != last:
+                if r:
+                    self.db.setjournal(self.classname, nodeid, r)
+                last = n
+                r = []
+
             if action == 'set':
                 for propname, value in params.iteritems():
                     prop = properties[propname]
@@ -2668,9 +2687,8 @@ class Class(hyperdb.Class):
                 # old tracker with data stored in the create!
                 params = {}
             r.append((nodeid, date.Date(jdate), user, action, params))
-
-        for nodeid, l in d.iteritems():
-            self.db.setjournal(self.classname, nodeid, l)
+        if r:
+            self.db.setjournal(self.classname, nodeid, r)
 
 class FileClass(hyperdb.FileClass, Class):
     """This class defines a large chunk of data. To support this, it has a