From 6d5b3379724487da99af534fabc4966a4b59dd54 Mon Sep 17 00:00:00 2001 From: gmcm Date: Wed, 2 Oct 2002 19:15:46 +0000 Subject: [PATCH] Fix (re)indexing & find in back_metakit. Fix testTransactions in the metakit test in test_db. git-svn-id: http://svn.roundup-tracker.org/svnroot/roundup/trunk@1307 57a73879-2fb5-44c3-a270-3262357dd7e2 --- roundup/backends/back_metakit.py | 62 +++++++++++++++++++++----------- test/test_db.py | 8 ++++- 2 files changed, 48 insertions(+), 22 deletions(-) diff --git a/roundup/backends/back_metakit.py b/roundup/backends/back_metakit.py index e9a5fe2..1cdcda3 100755 --- a/roundup/backends/back_metakit.py +++ b/roundup/backends/back_metakit.py @@ -87,6 +87,7 @@ class _Database(hyperdb.Database): self._db = metakit.storage(self.dbnm, 1) self.hist = self._db.view('history') self.tables = self._db.view('tables') + self.indexer.rollback() self.indexer.datadb = self._db self.dirty = 0 def clearCache(self): @@ -492,7 +493,7 @@ class Class: changes[key] = oldvalue if hasattr(prop, 'isfilename') and prop.isfilename: propvalues[key] = os.path.basename(value) - if prop.indexme and value is not None: + if prop.indexme: self.db.indexer.add_text((self.classname, nodeid, key), value, 'text/plain') @@ -680,16 +681,24 @@ class Class: vws = [] for propname, ids in propspec: if type(ids) is _STRINGTYPE: - ids = {ids:1} + ids = {int(ids):1} + else: + d = {} + for id in ids.keys(): + d[int(id)] = 1 + ids = d prop = self.ruprops[propname] view = self.getview() if isinstance(prop, hyperdb.Multilink): - view = view.flatten(getattr(view, propname)) def ff(row, nm=propname, ids=ids): - return ids.has_key(str(row.fid)) + sv = getattr(row, nm) + for sr in sv: + if ids.has_key(sr.fid): + return 1 + return 0 else: def ff(row, nm=propname, ids=ids): - return ids.has_key(str(getattr(row, nm))) + return ids.has_key(getattr(row, nm)) ndxview = view.filter(ff) vws.append(ndxview.unique()) @@ -700,7 +709,7 @@ class Class: ndxview = vws[0] for v in vws[1:]: ndxview = ndxview.union(v) - view = view.remapwith(ndxview) + view = self.getview().remapwith(ndxview) rslt = [] for row in view: rslt.append(str(row.id)) @@ -1170,12 +1179,13 @@ class IssueClass(Class, roundupdb.IssueClass): properties['superseder'] = hyperdb.Multilink(classname) Class.__init__(self, db, classname, **properties) -CURVERSION = 1 +CURVERSION = 2 class Indexer(indexer.Indexer): disallows = {'THE':1, 'THIS':1, 'ZZZ':1, 'THAT':1, 'WITH':1} def __init__(self, path, datadb): - self.db = metakit.storage(os.path.join(path, 'index.mk4'), 1) + self.path = os.path.join(path, 'index.mk4') + self.db = metakit.storage(self.path, 1) self.datadb = datadb self.reindex = 0 v = self.db.view('version') @@ -1188,7 +1198,7 @@ class Indexer(indexer.Indexer): v[0].vers = CURVERSION self.reindex = 1 if self.reindex: - self.db.getas('ids[tblid:I,nodeid:I,propid:I]') + self.db.getas('ids[tblid:I,nodeid:I,propid:I,ignore:I]') self.db.getas('index[word:S,hits[pos:I]]') self.db.commit() self.reindex = 1 @@ -1225,12 +1235,16 @@ class Indexer(indexer.Indexer): raise KeyError, "unknown class %r"%classname nodeid = int(nodeid) propid = self._getpropid(classname, property) - pos = self.db.view('ids').append(tblid=tblid,nodeid=nodeid,propid=propid) + ids = self.db.view('ids') + oldpos = ids.find(tblid=tblid,nodeid=nodeid,propid=propid,ignore=0) + if oldpos > -1: + ids[oldpos].ignore = 1 + self.changed = 1 + pos = ids.append(tblid=tblid,nodeid=nodeid,propid=propid) - wordlist = re.findall(r'\b\w{3,25}\b', text) + wordlist = re.findall(r'\b\w{2,25}\b', text.upper()) words = {} for word in wordlist: - word = word.upper() if not self.disallows.has_key(word): words[word] = 1 words = words.keys() @@ -1239,16 +1253,16 @@ class Indexer(indexer.Indexer): for word in words: ndx = index.find(word=word) if ndx < 0: - ndx = index.append(word=word) - hits = index[ndx].hits - if len(hits)==0 or hits.find(pos=pos) < 0: - hits.append(pos=pos) - self.changed = 1 + index.append(word=word) + ndx = index.find(word=word) + index[ndx].hits.append(pos=pos) + self.changed = 1 def find(self, wordlist): hits = None index = self.db.view('index').ordered(1) for word in wordlist: + word = word.upper() if not 2 < len(word) < 26: continue ndx = index.find(word=word) @@ -1267,12 +1281,18 @@ class Indexer(indexer.Indexer): tbls = self.datadb.view('tables') for i in range(len(ids)): hit = ids[i] - classname = tbls[hit.tblid].name - nodeid = str(hit.nodeid) - property = self._getpropname(classname, hit.propid) - rslt[i] = (classname, nodeid, property) + if not hit.ignore: + classname = tbls[hit.tblid].name + nodeid = str(hit.nodeid) + property = self._getpropname(classname, hit.propid) + rslt[i] = (classname, nodeid, property) return rslt def save_index(self): if self.changed: self.db.commit() self.changed = 0 + def rollback(self): + if self.changed: + self.db.rollback() + self.db = metakit.storage(self.path, 1) + self.changed = 0 diff --git a/test/test_db.py b/test/test_db.py index bd1b20b..23a11a8 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: test_db.py,v 1.56 2002-09-26 03:04:24 richard Exp $ +# $Id: test_db.py,v 1.57 2002-10-02 19:15:46 gmcm Exp $ import unittest, os, shutil, time @@ -738,15 +738,21 @@ class metakitDBTestCase(anydbmDBTestCase): self.assertNotEqual(num_issues, len(self.db.issue.list())) self.db.file.create(name="test", type="text/plain", content="hi") self.db.rollback() + num_files = len(self.db.file.list()) for i in range(10): self.db.file.create(name="test", type="text/plain", content="hi %d"%(i)) self.db.commit() # TODO: would be good to be able to ensure the file is not on disk after # a rollback... + num_files2 = len(self.db.file.list()) self.assertNotEqual(num_files, num_files2) self.db.file.create(name="test", type="text/plain", content="hi") + num_rfiles = len(os.listdir(self.db.config.DATABASE + '/files/file/0')) self.db.rollback() + num_rfiles2 = len(os.listdir(self.db.config.DATABASE + '/files/file/0')) + self.assertEqual(num_files2, len(self.db.file.list())) + self.assertEqual(num_rfiles2, num_rfiles-1) class metakitReadOnlyDBTestCase(anydbmReadOnlyDBTestCase): def setUp(self): -- 2.30.2