![[tokkee]](http://tokkee.org/images/avatar.png){kind=avatar}
summary | shortlog | log | commit | commitdiff | tree
raw | patch | inline | side by side (parent: b23eeb5)
raw | patch | inline | side by side (parent: b23eeb5)
| author | richard <richard@57a73879-2fb5-44c3-a270-3262357dd7e2> | |
| Fri, 19 Mar 2004 04:47:59 +0000 (04:47 +0000) | ||
| committer | richard <richard@57a73879-2fb5-44c3-a270-3262357dd7e2> | |
| Fri, 19 Mar 2004 04:47:59 +0000 (04:47 +0000) |
1. The current indexer has been moved to backends/indexer_dbm in
anticipation of my writing an indexer_rdbms,
2. Changed indexer invocation during create / set to follow the pattern
set by the metakit backend, which was much cleaner, and
3. The "content" property of FileClass is now mutable in all but the
metakit backend.
Metakit needs to be changed to support the editing of "content". Hey, and
I learnt today that the metakit backend implements its own indexer. How
about that... :)
git-svn-id: http://svn.roundup-tracker.org/svnroot/roundup/trunk@2157 57a73879-2fb5-44c3-a270-3262357dd7e2
anticipation of my writing an indexer_rdbms,
2. Changed indexer invocation during create / set to follow the pattern
set by the metakit backend, which was much cleaner, and
3. The "content" property of FileClass is now mutable in all but the
metakit backend.
Metakit needs to be changed to support the editing of "content". Hey, and
I learnt today that the metakit backend implements its own indexer. How
about that... :)
git-svn-id: http://svn.roundup-tracker.org/svnroot/roundup/trunk@2157 57a73879-2fb5-44c3-a270-3262357dd7e2
15 files changed:
| CHANGES.txt | patch | blob | history | |
| roundup/backends/back_anydbm.py | patch | blob | history | |
| roundup/backends/back_metakit.py | patch | blob | history | |
| roundup/backends/back_mysql.py | patch | blob | history | |
| roundup/backends/back_postgresql.py | patch | blob | history | |
| roundup/backends/blobfiles.py | patch | blob | history | |
| roundup/backends/indexer_dbm.py | [new file with mode: 0644] | patch | blob |
| roundup/backends/rdbms_common.py | patch | blob | history | |
| roundup/backends/sessions_dbm.py | patch | blob | history | |
| roundup/indexer.py | [deleted file] | patch | blob | history |
| roundup/roundupdb.py | patch | blob | history | |
| test/db_test_base.py | patch | blob | history | |
| test/session_common.py | patch | blob | history | |
| test/test_indexer.py | patch | blob | history | |
| test/test_mailgw.py | patch | blob | history |
diff --git a/CHANGES.txt b/CHANGES.txt
index a217160939572296a7dab2cd57b0b8051fffd839..9b15a736d9f6a77afcb581f6092c4e79d1bb2567 100644 (file)
--- a/CHANGES.txt
+++ b/CHANGES.txt
- the mail gateway now searches recursively for the text/plain and the
attachments of a message (sf bug 841241).
- fixed display of feedback messages in some situations (sf bug 739545)
+- fixed ability to edit "content" property (sf bug 914062)
Cleanup:
- replace curuserid attribute on Database with the extended getuid() method.
index 7091d19bd218995e1bd78a529f6c06b49bb207eb..6fb115d5ae1da2555e7cec64177affc3060259cf 100644 (file)
# BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
# SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
#
-#$Id: back_anydbm.py,v 1.138 2004-03-18 01:58:45 richard Exp $
+#$Id: back_anydbm.py,v 1.139 2004-03-19 04:47:59 richard Exp $
'''This module defines a backend that saves the hyperdatabase in a
database chosen by anydbm. It is guaranteed to always be available in python
versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
from roundup import hyperdb, date, password, roundupdb, security
from blobfiles import FileStorage
from sessions_dbm import Sessions, OneTimeKeys
-from roundup.indexer import Indexer
+from indexer_dbm import Indexer
from roundup.backends import locking
from roundup.hyperdb import String, Password, Date, Interval, Link, \
Multilink, DatabaseError, Boolean, Number, Node
elif isinstance(prop, String):
if type(value) != type('') and type(value) != type(u''):
raise TypeError, 'new property "%s" not a string'%key
+ self.db.indexer.add_text((self.classname, newid, key), value)
elif isinstance(prop, Password):
if not isinstance(value, password.Password):
These operations trigger detectors and can be vetoed. Attempts
to modify the "creation" or "activity" properties cause a KeyError.
'''
+ self.fireAuditors('set', nodeid, propvalues)
+ oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
+ propvalues = self.set_inner(nodeid, **propvalues)
+ self.fireReactors('set', nodeid, oldvalues)
+ return propvalues
+
+ def set_inner(self, nodeid, **propvalues):
+ ''' Called by set, in-between the audit and react calls.
+ '''
if not propvalues:
return propvalues
if self.db.journaltag is None:
raise DatabaseError, 'Database open read-only'
- self.fireAuditors('set', nodeid, propvalues)
- # Take a copy of the node dict so that the subsequent set
- # operation doesn't modify the oldvalues structure.
- oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
-
node = self.db.getnode(self.classname, nodeid)
if node.has_key(self.db.RETIRED_FLAG):
raise IndexError
elif isinstance(prop, String):
if value is not None and type(value) != type('') and type(value) != type(u''):
raise TypeError, 'new property "%s" not a string'%propname
+ self.db.indexer.add_text((self.classname, nodeid, propname),
+ value)
elif isinstance(prop, Password):
if not isinstance(value, password.Password):
if self.do_journal:
self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
- self.fireReactors('set', nodeid, oldvalues)
-
- return propvalues
+ return propvalues
def retire(self, nodeid):
'''Retire a node.
self.properties.update(properties)
def index(self, nodeid):
- '''Add (or refresh) the node to search indexes
- '''
+ ''' Add (or refresh) the node to search indexes '''
# find all the String properties that have indexme
for prop, propclass in self.getprops().items():
- if isinstance(propclass, String) and propclass.indexme:
+ if isinstance(propclass, hyperdb.String) and propclass.indexme:
+ # index them under (classname, nodeid, property)
try:
value = str(self.get(nodeid, prop))
except IndexError:
- # node no longer exists - entry should be removed
- self.db.indexer.purge_entry((self.classname, nodeid, prop))
- else:
- # and index them under (classname, nodeid, property)
- self.db.indexer.add_text((self.classname, nodeid, prop),
- value)
+ # node has been destroyed
+ continue
+ self.db.indexer.add_text((self.classname, nodeid, prop), value)
+
#
# Detector interface
content = propvalues['content']
del propvalues['content']
+ # make sure we have a MIME type
+ mime_type = propvalues.get('type', self.default_mime_type)
+
# do the database create
- newid = Class.create_inner(self, **propvalues)
+ newid = self.create_inner(**propvalues)
+
+ # and index!
+ self.db.indexer.add_text((self.classname, newid, 'content'), content,
+ mime_type)
# fire reactors
self.fireReactors('create', newid, None)
else:
return Class.get(self, nodeid, propname)
+ def set(self, itemid, **propvalues):
+ ''' Snarf the "content" propvalue and update it in a file
+ '''
+ self.fireAuditors('set', itemid, propvalues)
+ oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid))
+
+ # now remove the content property so it's not stored in the db
+ content = None
+ if propvalues.has_key('content'):
+ content = propvalues['content']
+ del propvalues['content']
+
+ # do the database create
+ propvalues = self.set_inner(itemid, **propvalues)
+
+ # do content?
+ if content:
+ # store and index
+ self.db.storefile(self.classname, itemid, None, content)
+ mime_type = propvalues.get('type', self.get(itemid, 'type'))
+ if not mime_type:
+ mime_type = self.default_mime_type
+ self.db.indexer.add_text((self.classname, itemid, 'content'),
+ content, mime_type)
+
+ # fire reactors
+ self.fireReactors('set', itemid, oldvalues)
+ return propvalues
+
def getprops(self, protected=1):
''' In addition to the actual properties on the node, these methods
provide the "content" property. If the "protected" flag is true,
d['content'] = hyperdb.String()
return d
- def index(self, nodeid):
- ''' Index the node in the search index.
-
- We want to index the content in addition to the normal String
- property indexing.
- '''
- # perform normal indexing
- Class.index(self, nodeid)
-
- # get the content to index
- content = self.get(nodeid, 'content')
-
- # figure the mime type
- if self.properties.has_key('type'):
- mime_type = self.get(nodeid, 'type')
- else:
- mime_type = self.default_mime_type
-
- # and index!
- self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
- mime_type)
# deviation from spec - was called ItemClass
class IssueClass(Class, roundupdb.IssueClass):
index cf3f59f99c594947f43e5771f2d5deb94f7c2093..ff544738fbf76290784c77ebf71fe138fa5ea1fd 100755 (executable)
-# $Id: back_metakit.py,v 1.62 2004-03-18 01:58:45 richard Exp $
+# $Id: back_metakit.py,v 1.63 2004-03-19 04:47:59 richard Exp $
'''Metakit backend for Roundup, originally by Gordon McMillan.
Known Current Bugs:
import metakit
from sessions_dbm import Sessions, OneTimeKeys
import re, marshal, os, sys, time, calendar
-from roundup import indexer
+from indexer_dbm import Indexer
import locking
from roundup.date import Range
CURVERSION = 2
-class Indexer(indexer.Indexer):
+class Indexer(Indexer):
disallows = {'THE':1, 'THIS':1, 'ZZZ':1, 'THAT':1, 'WITH':1}
def __init__(self, path, datadb):
self.path = os.path.join(path, 'index.mk4')
index 3afd3d36a16847ba89f5f26756837086352a2820..21f17d8761361e4a993dea1e60b001b0ee92ea9d 100644 (file)
'%s_%s_l_idx'%(classname, ml),
'%s_%s_n_idx'%(classname, ml)
]
+ table_name = '%s_%s'%(classname, ml)
for index_name in l:
if not self.sql_index_exists(table_name, index_name):
continue
index a09086b50db3bdb992cc9e91876d338eba21d39b..195bfa76ef2a15834552490497ea09ff95b18ec3 100644 (file)
cols, mls = self.determine_columns(spec.properties.items())
cols.append('id')
cols.append('__retired__')
- scols = ',' . join(['"%s" VARCHAR(255)' % x for x in cols])
+ scols = ',' . join(['"%s" VARCHAR(255)'%x for x in cols])
sql = 'CREATE TABLE "_%s" (%s)' % (spec.classname, scols)
-
if __debug__:
- print >>hyperdb.DEBUG, 'create_class', (self, sql)
-
+ print >>hyperdb.DEBUG, 'create_class_table', (self, sql)
self.cursor.execute(sql)
self.create_class_table_indexes(spec)
return cols, mls
def create_journal_table(self, spec):
- cols = ',' . join(['"%s" VARCHAR(255)' % x
- for x in 'nodeid date tag action params' . split()])
+ cols = ',' . join(['"%s" VARCHAR(255)'%x
+ for x in 'nodeid date tag action params' . split()])
sql = 'CREATE TABLE "%s__journal" (%s)'%(spec.classname, cols)
-
if __debug__:
- print >>hyperdb.DEBUG, 'create_class', (self, sql)
-
+ print >>hyperdb.DEBUG, 'create_journal_table', (self, sql)
self.cursor.execute(sql)
self.create_journal_table_indexes(spec)
def create_multilink_table(self, spec, ml):
sql = '''CREATE TABLE "%s_%s" (linkid VARCHAR(255),
- nodeid VARCHAR(255))''' % (spec.classname, ml)
+ nodeid VARCHAR(255))'''%(spec.classname, ml)
if __debug__:
print >>hyperdb.DEBUG, 'create_class', (self, sql)
index 673bd938557ebe1579e2eed45c7f999ad4cea7e1..c67a0e8eef732535bad04b12d317d1fe9693ce5e 100644 (file)
# BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
# SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
#
-#$Id: blobfiles.py,v 1.11 2004-02-11 23:55:09 richard Exp $
+#$Id: blobfiles.py,v 1.12 2004-03-19 04:47:59 richard Exp $
'''This module exports file storage for roundup backends.
Files are stored into a directory hierarchy.
'''
if not os.path.exists(os.path.dirname(name)):
os.makedirs(os.path.dirname(name))
- # open the temp file for writing
- open(name + '.tmp', 'wb').write(content)
-
- # save off the commit action
- self.transactions.append((self.doStoreFile, (classname, nodeid,
- property)))
+ # save to a temp file
+ name = name + '.tmp'
+ # make sure we don't register the rename action more than once
+ if not os.path.exists(name):
+ # save off the rename action
+ self.transactions.append((self.doStoreFile, (classname, nodeid,
+ property)))
+ open(name, 'wb').write(content)
def getfile(self, classname, nodeid, property):
'''Get the content of the file in the database.
# determine the name of the file to write to
name = self.filename(classname, nodeid, property)
+ # content is being updated (and some platforms, eg. win32, won't
+ # let us rename over the top of the old file)
+ if os.path.exists(name):
+ os.remove(name)
+
# the file is currently ".tmp" - move it to its real name to commit
os.rename(name+".tmp", name)
diff --git a/roundup/backends/indexer_dbm.py b/roundup/backends/indexer_dbm.py
--- /dev/null
@@ -0,0 +1,349 @@
+#
+# This module is derived from the module described at:
+# http://gnosis.cx/publish/programming/charming_python_15.txt
+#
+# Author: David Mertz (mertz@gnosis.cx)
+# Thanks to: Pat Knight (p.knight@ktgroup.co.uk)
+# Gregory Popovitch (greg@gpy.com)
+#
+# The original module was released under this license, and remains under
+# it:
+#
+# This file is released to the public domain. I (dqm) would
+# appreciate it if you choose to keep derived works under terms
+# that promote freedom, but obviously am giving up any rights
+# to compel such.
+#
+#$Id: indexer_dbm.py,v 1.1 2004-03-19 04:47:59 richard Exp $
+'''This module provides an indexer class, RoundupIndexer, that stores text
+indices in a roundup instance. This class makes searching the content of
+messages, string properties and text files possible.
+'''
+__docformat__ = 'restructuredtext'
+
+import os, shutil, re, mimetypes, marshal, zlib, errno
+from roundup.hyperdb import Link, Multilink
+
+class Indexer:
+ '''Indexes information from roundup's hyperdb to allow efficient
+ searching.
+
+ Three structures are created by the indexer::
+
+ files {identifier: (fileid, wordcount)}
+ words {word: {fileid: count}}
+ fileids {fileid: identifier}
+
+ where identifier is (classname, nodeid, propertyname)
+ '''
+ def __init__(self, db_path):
+ self.indexdb_path = os.path.join(db_path, 'indexes')
+ self.indexdb = os.path.join(self.indexdb_path, 'index.db')
+ self.reindex = 0
+ self.quiet = 9
+ self.changed = 0
+
+ # see if we need to reindex because of a change in code
+ version = os.path.join(self.indexdb_path, 'version')
+ if (not os.path.exists(self.indexdb_path) or
+ not os.path.exists(version)):
+ # for now the file itself is a flag
+ self.force_reindex()
+ elif os.path.exists(version):
+ version = open(version).read()
+ # check the value and reindex if it's not the latest
+ if version.strip() != '1':
+ self.force_reindex()
+
+ def force_reindex(self):
+ '''Force a reindex condition
+ '''
+ if os.path.exists(self.indexdb_path):
+ shutil.rmtree(self.indexdb_path)
+ os.makedirs(self.indexdb_path)
+ os.chmod(self.indexdb_path, 0775)
+ open(os.path.join(self.indexdb_path, 'version'), 'w').write('1\n')
+ self.reindex = 1
+ self.changed = 1
+
+ def should_reindex(self):
+ '''Should we reindex?
+ '''
+ return self.reindex
+
+ def add_text(self, identifier, text, mime_type='text/plain'):
+ '''Add some text associated with the (classname, nodeid, property)
+ identifier.
+ '''
+ # make sure the index is loaded
+ self.load_index()
+
+ # remove old entries for this identifier
+ if self.files.has_key(identifier):
+ self.purge_entry(identifier)
+
+ # split into words
+ words = self.splitter(text, mime_type)
+
+ # Find new file index, and assign it to identifier
+ # (_TOP uses trick of negative to avoid conflict with file index)
+ self.files['_TOP'] = (self.files['_TOP'][0]-1, None)
+ file_index = abs(self.files['_TOP'][0])
+ self.files[identifier] = (file_index, len(words))
+ self.fileids[file_index] = identifier
+
+ # find the unique words
+ filedict = {}
+ for word in words:
+ if filedict.has_key(word):
+ filedict[word] = filedict[word]+1
+ else:
+ filedict[word] = 1
+
+ # now add to the totals
+ for word in filedict.keys():
+ # each word has a dict of {identifier: count}
+ if self.words.has_key(word):
+ entry = self.words[word]
+ else:
+ # new word
+ entry = {}
+ self.words[word] = entry
+
+ # make a reference to the file for this word
+ entry[file_index] = filedict[word]
+
+ # save needed
+ self.changed = 1
+
+ def splitter(self, text, ftype):
+ '''Split the contents of a text string into a list of 'words'
+ '''
+ if ftype == 'text/plain':
+ words = self.text_splitter(text)
+ else:
+ return []
+ return words
+
+ def text_splitter(self, text):
+ """Split text/plain string into a list of words
+ """
+ # case insensitive
+ text = str(text).upper()
+
+ # Split the raw text, losing anything longer than 25 characters
+ # since that'll be gibberish (encoded text or somesuch) or shorter
+ # than 3 characters since those short words appear all over the
+ # place
+ return re.findall(r'\b\w{2,25}\b', text)
+
+ def search(self, search_terms, klass, ignore={},
+ dre=re.compile(r'([^\d]+)(\d+)')):
+ '''Display search results looking for [search, terms] associated
+ with the hyperdb Class "klass". Ignore hits on {class: property}.
+
+ "dre" is a helper, not an argument.
+ '''
+ # do the index lookup
+ hits = self.find(search_terms)
+ if not hits:
+ return {}
+
+ designator_propname = {}
+ for nm, propclass in klass.getprops().items():
+ if isinstance(propclass, Link) or isinstance(propclass, Multilink):
+ designator_propname[propclass.classname] = nm
+
+ # build a dictionary of nodes and their associated messages
+ # and files
+ nodeids = {} # this is the answer
+ propspec = {} # used to do the klass.find
+ for propname in designator_propname.values():
+ propspec[propname] = {} # used as a set (value doesn't matter)
+ for classname, nodeid, property in hits.values():
+ # skip this result if we don't care about this class/property
+ if ignore.has_key((classname, property)):
+ continue
+
+ # if it's a property on klass, it's easy
+ if classname == klass.classname:
+ if not nodeids.has_key(nodeid):
+ nodeids[nodeid] = {}
+ continue
+
+ # make sure the class is a linked one, otherwise ignore
+ if not designator_propname.has_key(classname):
+ continue
+
+ # it's a linked class - set up to do the klass.find
+ linkprop = designator_propname[classname] # eg, msg -> messages
+ propspec[linkprop][nodeid] = 1
+
+ # retain only the meaningful entries
+ for propname, idset in propspec.items():
+ if not idset:
+ del propspec[propname]
+
+ # klass.find tells me the klass nodeids the linked nodes relate to
+ for resid in klass.find(**propspec):
+ resid = str(resid)
+ if not nodeids.has_key(id):
+ nodeids[resid] = {}
+ node_dict = nodeids[resid]
+ # now figure out where it came from
+ for linkprop in propspec.keys():
+ for nodeid in klass.get(resid, linkprop):
+ if propspec[linkprop].has_key(nodeid):
+ # OK, this node[propname] has a winner
+ if not node_dict.has_key(linkprop):
+ node_dict[linkprop] = [nodeid]
+ else:
+ node_dict[linkprop].append(nodeid)
+ return nodeids
+
+ # we override this to ignore not 2 < word < 25 and also to fix a bug -
+ # the (fail) case.
+ def find(self, wordlist):
+ '''Locate files that match ALL the words in wordlist
+ '''
+ if not hasattr(self, 'words'):
+ self.load_index()
+ self.load_index(wordlist=wordlist)
+ entries = {}
+ hits = None
+ for word in wordlist:
+ if not 2 < len(word) < 25:
+ # word outside the bounds of what we index - ignore
+ continue
+ word = word.upper()
+ entry = self.words.get(word) # For each word, get index
+ entries[word] = entry # of matching files
+ if not entry: # Nothing for this one word (fail)
+ return {}
+ if hits is None:
+ hits = {}
+ for k in entry.keys():
+ if not self.fileids.has_key(k):
+ raise ValueError, 'Index is corrupted: re-generate it'
+ hits[k] = self.fileids[k]
+ else:
+ # Eliminate hits for every non-match
+ for fileid in hits.keys():
+ if not entry.has_key(fileid):
+ del hits[fileid]
+ if hits is None:
+ return {}
+ return hits
+
+ segments = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ#_-!"
+ def load_index(self, reload=0, wordlist=None):
+ # Unless reload is indicated, do not load twice
+ if self.index_loaded() and not reload:
+ return 0
+
+ # Ok, now let's actually load it
+ db = {'WORDS': {}, 'FILES': {'_TOP':(0,None)}, 'FILEIDS': {}}
+
+ # Identify the relevant word-dictionary segments
+ if not wordlist:
+ segments = self.segments
+ else:
+ segments = ['-','#']
+ for word in wordlist:
+ segments.append(word[0].upper())
+
+ # Load the segments
+ for segment in segments:
+ try:
+ f = open(self.indexdb + segment, 'rb')
+ except IOError, error:
+ # probably just nonexistent segment index file
+ if error.errno != errno.ENOENT: raise
+ else:
+ pickle_str = zlib.decompress(f.read())
+ f.close()
+ dbslice = marshal.loads(pickle_str)
+ if dbslice.get('WORDS'):
+ # if it has some words, add them
+ for word, entry in dbslice['WORDS'].items():
+ db['WORDS'][word] = entry
+ if dbslice.get('FILES'):
+ # if it has some files, add them
+ db['FILES'] = dbslice['FILES']
+ if dbslice.get('FILEIDS'):
+ # if it has fileids, add them
+ db['FILEIDS'] = dbslice['FILEIDS']
+
+ self.words = db['WORDS']
+ self.files = db['FILES']
+ self.fileids = db['FILEIDS']
+ self.changed = 0
+
+ def save_index(self):
+ # only save if the index is loaded and changed
+ if not self.index_loaded() or not self.changed:
+ return
+
+ # brutal space saver... delete all the small segments
+ for segment in self.segments:
+ try:
+ os.remove(self.indexdb + segment)
+ except OSError, error:
+ # probably just nonexistent segment index file
+ if error.errno != errno.ENOENT: raise
+
+ # First write the much simpler filename/fileid dictionaries
+ dbfil = {'WORDS':None, 'FILES':self.files, 'FILEIDS':self.fileids}
+ open(self.indexdb+'-','wb').write(zlib.compress(marshal.dumps(dbfil)))
+
+ # The hard part is splitting the word dictionary up, of course
+ letters = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ#_"
+ segdicts = {} # Need batch of empty dicts
+ for segment in letters:
+ segdicts[segment] = {}
+ for word, entry in self.words.items(): # Split into segment dicts
+ initchar = word[0].upper()
+ segdicts[initchar][word] = entry
+
+ # save
+ for initchar in letters:
+ db = {'WORDS':segdicts[initchar], 'FILES':None, 'FILEIDS':None}
+ pickle_str = marshal.dumps(db)
+ filename = self.indexdb + initchar
+ pickle_fh = open(filename, 'wb')
+ pickle_fh.write(zlib.compress(pickle_str))
+ os.chmod(filename, 0664)
+
+ # save done
+ self.changed = 0
+
+ def purge_entry(self, identifier):
+ '''Remove a file from file index and word index
+ '''
+ self.load_index()
+
+ if not self.files.has_key(identifier):
+ return
+
+ file_index = self.files[identifier][0]
+ del self.files[identifier]
+ del self.fileids[file_index]
+
+ # The much harder part, cleanup the word index
+ for key, occurs in self.words.items():
+ if occurs.has_key(file_index):
+ del occurs[file_index]
+
+ # save needed
+ self.changed = 1
+
+ def index_loaded(self):
+ return (hasattr(self,'fileids') and hasattr(self,'files') and
+ hasattr(self,'words'))
+
+
+ def rollback(self):
+ ''' load last saved index info. '''
+ self.load_index(reload=1)
+
+# vim: set filetype=python ts=4 sw=4 et si
index 2894cc63e35a8374b17ed070bd0fd0f6955353a9..8064c3ecd0c75f5b52f04a6fae41248d6946ca2d 100644 (file)
-# $Id: rdbms_common.py,v 1.81 2004-03-18 01:58:45 richard Exp $
+# $Id: rdbms_common.py,v 1.82 2004-03-19 04:47:59 richard Exp $
''' Relational database (SQL) backend common code.
Basics:
# support
from blobfiles import FileStorage
-from roundup.indexer import Indexer
+from indexer_dbm import Indexer
from sessions_rdbms import Sessions, OneTimeKeys
from roundup.date import Range
print >>hyperdb.DEBUG, 'update_class FIRING'
# detect key prop change for potential index change
- keyprop_changes = 0
+ keyprop_changes = {}
if new_spec[0] != old_spec[0]:
keyprop_changes = {'remove': old_spec[0], 'add': new_spec[0]}
if new_has(name):
continue
- if isinstance(prop, Multilink):
+ if prop.find('Multilink to') != -1:
# first drop indexes.
- self.drop_multilink_table_indexes(spec.classname, ml)
+ self.drop_multilink_table_indexes(spec.classname, name)
# now the multilink table itself
- sql = 'drop table %s_%s'%(spec.classname, prop)
+ sql = 'drop table %s_%s'%(spec.classname, name)
else:
# if this is the key prop, drop the index first
if old_spec[0] == prop:
- self.drop_class_table_key_index(spec.classname, prop)
+ self.drop_class_table_key_index(spec.classname, name)
del keyprop_changes['remove']
# drop the column
- sql = 'alter table _%s drop column _%s'%(spec.classname, prop)
+ sql = 'alter table _%s drop column _%s'%(spec.classname, name)
if __debug__:
print >>hyperdb.DEBUG, 'update_class', (self, sql)
''' Load the journal from the database
'''
# now get the journal entries
- sql = 'select %s from %s__journal where nodeid=%s'%(cols, classname,
- self.arg)
+ sql = 'select %s from %s__journal where nodeid=%s order by date'%(
+ cols, classname, self.arg)
if __debug__:
print >>hyperdb.DEBUG, 'load_journal', (self, sql, nodeid)
self.cursor.execute(sql, (nodeid,))
self.sql_commit()
# now, do all the other transaction stuff
- reindex = {}
for method, args in self.transactions:
- reindex[method(*args)] = 1
-
- # reindex the nodes that request it
- for classname, nodeid in filter(None, reindex.keys()):
- print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
- self.getclass(classname).index(nodeid)
+ method(*args)
# save the indexer state
self.indexer.save_index()
elif isinstance(prop, String):
if type(value) != type('') and type(value) != type(u''):
raise TypeError, 'new property "%s" not a string'%key
+ self.db.indexer.add_text((self.classname, newid, key), value)
elif isinstance(prop, Password):
if not isinstance(value, password.Password):
If the value of a Link or Multilink property contains an invalid
node id, a ValueError is raised.
'''
+ self.fireAuditors('set', nodeid, propvalues)
+ oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
+ propvalues = self.set_inner(nodeid, **propvalues)
+ self.fireReactors('set', nodeid, oldvalues)
+ return propvalues
+
+ def set_inner(self, nodeid, **propvalues):
+ ''' Called by set, in-between the audit and react calls.
+ '''
if not propvalues:
return propvalues
if self.db.journaltag is None:
raise DatabaseError, 'Database open read-only'
- self.fireAuditors('set', nodeid, propvalues)
- # Take a copy of the node dict so that the subsequent set
- # operation doesn't modify the oldvalues structure.
- # XXX used to try the cache here first
- oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
-
node = self.db.getnode(self.classname, nodeid)
if self.is_retired(nodeid):
raise IndexError, 'Requested item is retired'
elif isinstance(prop, String):
if value is not None and type(value) != type('') and type(value) != type(u''):
raise TypeError, 'new property "%s" not a string'%propname
+ self.db.indexer.add_text((self.classname, nodeid, propname),
+ value)
elif isinstance(prop, Password):
if not isinstance(value, password.Password):
if self.do_journal:
self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
- self.fireReactors('set', nodeid, oldvalues)
-
return propvalues
def retire(self, nodeid):
# find all the String properties that have indexme
for prop, propclass in self.getprops().items():
if isinstance(propclass, String) and propclass.indexme:
- try:
- value = str(self.get(nodeid, prop))
- except IndexError:
- # node no longer exists - entry should be removed
- self.db.indexer.purge_entry((self.classname, nodeid, prop))
- else:
- # and index them under (classname, nodeid, property)
- self.db.indexer.add_text((self.classname, nodeid, prop),
- value)
+ self.db.indexer.add_text((self.classname, nodeid, prop),
+ str(self.get(nodeid, prop)))
#
del propvalues['content']
# do the database create
- newid = Class.create_inner(self, **propvalues)
+ newid = self.create_inner(**propvalues)
+
+ # figure the mime type
+ mime_type = propvalues.get('type', self.default_mime_type)
+
+ # and index!
+ self.db.indexer.add_text((self.classname, newid, 'content'), content,
+ mime_type)
# fire reactors
self.fireReactors('create', newid, None)
d['content'] = hyperdb.String()
return d
- def index(self, nodeid):
- ''' Index the node in the search index.
-
- We want to index the content in addition to the normal String
- property indexing.
+ def set(self, itemid, **propvalues):
+ ''' Snarf the "content" propvalue and update it in a file
'''
- # perform normal indexing
- Class.index(self, nodeid)
+ self.fireAuditors('set', itemid, propvalues)
+ oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid))
- # get the content to index
- content = self.get(nodeid, 'content')
+ # now remove the content property so it's not stored in the db
+ content = None
+ if propvalues.has_key('content'):
+ content = propvalues['content']
+ del propvalues['content']
- # figure the mime type
- if self.properties.has_key('type'):
- mime_type = self.get(nodeid, 'type')
- else:
- mime_type = self.default_mime_type
+ # do the database create
+ propvalues = self.set_inner(itemid, **propvalues)
+
+ # do content?
+ if content:
+ # store and index
+ self.db.storefile(self.classname, itemid, None, content)
+ mime_type = propvalues.get('type', self.get(itemid, 'type'))
+ if not mime_type:
+ mime_type = self.default_mime_type
+ self.db.indexer.add_text((self.classname, itemid, 'content'),
+ content, mime_type)
- # and index!
- self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
- mime_type)
+ # fire reactors
+ self.fireReactors('set', itemid, oldvalues)
+ return propvalues
# XXX deviation from spec - was called ItemClass
class IssueClass(Class, roundupdb.IssueClass):
index f8f0222a93ffa58e36108d0101f7d39c926a98ff..433db42bc6cf8e7be481188580af88274d0fa5f9 100644 (file)
-#$Id: sessions_dbm.py,v 1.1 2004-03-18 01:58:45 richard Exp $
+#$Id: sessions_dbm.py,v 1.2 2004-03-19 04:47:59 richard Exp $
"""This module defines a very basic store that's used by the CGI interface
to store session and one-time-key information.
db = self.opendb('c')
try:
try:
- return marshal.loads(db[infoid])
+ d = marshal.loads(db[infoid])
+ del d['__timestamp']
+ return d
except KeyError:
raise KeyError, 'No such %s "%s"'%(self.name, infoid)
finally:
diff --git a/roundup/indexer.py b/roundup/indexer.py
--- a/roundup/indexer.py
+++ /dev/null
@@ -1,344 +0,0 @@
-#
-# This module is derived from the module described at:
-# http://gnosis.cx/publish/programming/charming_python_15.txt
-#
-# Author: David Mertz (mertz@gnosis.cx)
-# Thanks to: Pat Knight (p.knight@ktgroup.co.uk)
-# Gregory Popovitch (greg@gpy.com)
-#
-# The original module was released under this license, and remains under
-# it:
-#
-# This file is released to the public domain. I (dqm) would
-# appreciate it if you choose to keep derived works under terms
-# that promote freedom, but obviously am giving up any rights
-# to compel such.
-#
-#$Id: indexer.py,v 1.18 2004-02-11 23:55:08 richard Exp $
-'''This module provides an indexer class, RoundupIndexer, that stores text
-indices in a roundup instance. This class makes searching the content of
-messages, string properties and text files possible.
-'''
-__docformat__ = 'restructuredtext'
-
-import os, shutil, re, mimetypes, marshal, zlib, errno
-from hyperdb import Link, Multilink
-
-class Indexer:
- '''Indexes information from roundup's hyperdb to allow efficient
- searching.
-
- Three structures are created by the indexer::
-
- files {identifier: (fileid, wordcount)}
- words {word: {fileid: count}}
- fileids {fileid: identifier}
-
- where identifier is (classname, nodeid, propertyname)
- '''
- def __init__(self, db_path):
- self.indexdb_path = os.path.join(db_path, 'indexes')
- self.indexdb = os.path.join(self.indexdb_path, 'index.db')
- self.reindex = 0
- self.quiet = 9
- self.changed = 0
-
- # see if we need to reindex because of a change in code
- version = os.path.join(self.indexdb_path, 'version')
- if (not os.path.exists(self.indexdb_path) or
- not os.path.exists(version)):
- # for now the file itself is a flag
- self.force_reindex()
- elif os.path.exists(version):
- version = open(version).read()
- # check the value and reindex if it's not the latest
- if version.strip() != '1':
- self.force_reindex()
-
- def force_reindex(self):
- '''Force a reindex condition
- '''
- if os.path.exists(self.indexdb_path):
- shutil.rmtree(self.indexdb_path)
- os.makedirs(self.indexdb_path)
- os.chmod(self.indexdb_path, 0775)
- open(os.path.join(self.indexdb_path, 'version'), 'w').write('1\n')
- self.reindex = 1
- self.changed = 1
-
- def should_reindex(self):
- '''Should we reindex?
- '''
- return self.reindex
-
- def add_text(self, identifier, text, mime_type='text/plain'):
- '''Add some text associated with the (classname, nodeid, property)
- identifier.
- '''
- # make sure the index is loaded
- self.load_index()
-
- # remove old entries for this identifier
- if self.files.has_key(identifier):
- self.purge_entry(identifier)
-
- # split into words
- words = self.splitter(text, mime_type)
-
- # Find new file index, and assign it to identifier
- # (_TOP uses trick of negative to avoid conflict with file index)
- self.files['_TOP'] = (self.files['_TOP'][0]-1, None)
- file_index = abs(self.files['_TOP'][0])
- self.files[identifier] = (file_index, len(words))
- self.fileids[file_index] = identifier
-
- # find the unique words
- filedict = {}
- for word in words:
- if filedict.has_key(word):
- filedict[word] = filedict[word]+1
- else:
- filedict[word] = 1
-
- # now add to the totals
- for word in filedict.keys():
- # each word has a dict of {identifier: count}
- if self.words.has_key(word):
- entry = self.words[word]
- else:
- # new word
- entry = {}
- self.words[word] = entry
-
- # make a reference to the file for this word
- entry[file_index] = filedict[word]
-
- # save needed
- self.changed = 1
-
- def splitter(self, text, ftype):
- '''Split the contents of a text string into a list of 'words'
- '''
- if ftype == 'text/plain':
- words = self.text_splitter(text)
- else:
- return []
- return words
-
- def text_splitter(self, text):
- """Split text/plain string into a list of words
- """
- # case insensitive
- text = text.upper()
-
- # Split the raw text, losing anything longer than 25 characters
- # since that'll be gibberish (encoded text or somesuch) or shorter
- # than 3 characters since those short words appear all over the
- # place
- return re.findall(r'\b\w{2,25}\b', text)
-
- def search(self, search_terms, klass, ignore={},
- dre=re.compile(r'([^\d]+)(\d+)')):
- '''Display search results looking for [search, terms] associated
- with the hyperdb Class "klass". Ignore hits on {class: property}.
-
- "dre" is a helper, not an argument.
- '''
- # do the index lookup
- hits = self.find(search_terms)
- if not hits:
- return {}
-
- designator_propname = {}
- for nm, propclass in klass.getprops().items():
- if isinstance(propclass, Link) or isinstance(propclass, Multilink):
- designator_propname[propclass.classname] = nm
-
- # build a dictionary of nodes and their associated messages
- # and files
- nodeids = {} # this is the answer
- propspec = {} # used to do the klass.find
- for propname in designator_propname.values():
- propspec[propname] = {} # used as a set (value doesn't matter)
- for classname, nodeid, property in hits.values():
- # skip this result if we don't care about this class/property
- if ignore.has_key((classname, property)):
- continue
-
- # if it's a property on klass, it's easy
- if classname == klass.classname:
- if not nodeids.has_key(nodeid):
- nodeids[nodeid] = {}
- continue
-
- # make sure the class is a linked one, otherwise ignore
- if not designator_propname.has_key(classname):
- continue
-
- # it's a linked class - set up to do the klass.find
- linkprop = designator_propname[classname] # eg, msg -> messages
- propspec[linkprop][nodeid] = 1
-
- # retain only the meaningful entries
- for propname, idset in propspec.items():
- if not idset:
- del propspec[propname]
-
- # klass.find tells me the klass nodeids the linked nodes relate to
- for resid in klass.find(**propspec):
- resid = str(resid)
- if not nodeids.has_key(id):
- nodeids[resid] = {}
- node_dict = nodeids[resid]
- # now figure out where it came from
- for linkprop in propspec.keys():
- for nodeid in klass.get(resid, linkprop):
- if propspec[linkprop].has_key(nodeid):
- # OK, this node[propname] has a winner
- if not node_dict.has_key(linkprop):
- node_dict[linkprop] = [nodeid]
- else:
- node_dict[linkprop].append(nodeid)
- return nodeids
-
- # we override this to ignore not 2 < word < 25 and also to fix a bug -
- # the (fail) case.
- def find(self, wordlist):
- '''Locate files that match ALL the words in wordlist
- '''
- if not hasattr(self, 'words'):
- self.load_index()
- self.load_index(wordlist=wordlist)
- entries = {}
- hits = None
- for word in wordlist:
- if not 2 < len(word) < 25:
- # word outside the bounds of what we index - ignore
- continue
- word = word.upper()
- entry = self.words.get(word) # For each word, get index
- entries[word] = entry # of matching files
- if not entry: # Nothing for this one word (fail)
- return {}
- if hits is None:
- hits = {}
- for k in entry.keys():
- if not self.fileids.has_key(k):
- raise ValueError, 'Index is corrupted: re-generate it'
- hits[k] = self.fileids[k]
- else:
- # Eliminate hits for every non-match
- for fileid in hits.keys():
- if not entry.has_key(fileid):
- del hits[fileid]
- if hits is None:
- return {}
- return hits
-
- segments = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ#_-!"
- def load_index(self, reload=0, wordlist=None):
- # Unless reload is indicated, do not load twice
- if self.index_loaded() and not reload:
- return 0
-
- # Ok, now let's actually load it
- db = {'WORDS': {}, 'FILES': {'_TOP':(0,None)}, 'FILEIDS': {}}
-
- # Identify the relevant word-dictionary segments
- if not wordlist:
- segments = self.segments
- else:
- segments = ['-','#']
- for word in wordlist:
- segments.append(word[0].upper())
-
- # Load the segments
- for segment in segments:
- try:
- f = open(self.indexdb + segment, 'rb')
- except IOError, error:
- # probably just nonexistent segment index file
- if error.errno != errno.ENOENT: raise
- else:
- pickle_str = zlib.decompress(f.read())
- f.close()
- dbslice = marshal.loads(pickle_str)
- if dbslice.get('WORDS'):
- # if it has some words, add them
- for word, entry in dbslice['WORDS'].items():
- db['WORDS'][word] = entry
- if dbslice.get('FILES'):
- # if it has some files, add them
- db['FILES'] = dbslice['FILES']
- if dbslice.get('FILEIDS'):
- # if it has fileids, add them
- db['FILEIDS'] = dbslice['FILEIDS']
-
- self.words = db['WORDS']
- self.files = db['FILES']
- self.fileids = db['FILEIDS']
- self.changed = 0
-
- def save_index(self):
- # only save if the index is loaded and changed
- if not self.index_loaded() or not self.changed:
- return
-
- # brutal space saver... delete all the small segments
- for segment in self.segments:
- try:
- os.remove(self.indexdb + segment)
- except OSError, error:
- # probably just nonexistent segment index file
- if error.errno != errno.ENOENT: raise
-
- # First write the much simpler filename/fileid dictionaries
- dbfil = {'WORDS':None, 'FILES':self.files, 'FILEIDS':self.fileids}
- open(self.indexdb+'-','wb').write(zlib.compress(marshal.dumps(dbfil)))
-
- # The hard part is splitting the word dictionary up, of course
- letters = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ#_"
- segdicts = {} # Need batch of empty dicts
- for segment in letters:
- segdicts[segment] = {}
- for word, entry in self.words.items(): # Split into segment dicts
- initchar = word[0].upper()
- segdicts[initchar][word] = entry
-
- # save
- for initchar in letters:
- db = {'WORDS':segdicts[initchar], 'FILES':None, 'FILEIDS':None}
- pickle_str = marshal.dumps(db)
- filename = self.indexdb + initchar
- pickle_fh = open(filename, 'wb')
- pickle_fh.write(zlib.compress(pickle_str))
- os.chmod(filename, 0664)
-
- # save done
- self.changed = 0
-
- def purge_entry(self, identifier):
- '''Remove a file from file index and word index
- '''
- self.load_index()
-
- if not self.files.has_key(identifier):
- return
-
- file_index = self.files[identifier][0]
- del self.files[identifier]
- del self.fileids[file_index]
-
- # The much harder part, cleanup the word index
- for key, occurs in self.words.items():
- if occurs.has_key(file_index):
- del occurs[file_index]
-
- # save needed
- self.changed = 1
-
- def index_loaded(self):
- return (hasattr(self,'fileids') and hasattr(self,'files') and
- hasattr(self,'words'))
-
-# vim: set filetype=python ts=4 sw=4 et si
diff --git a/roundup/roundupdb.py b/roundup/roundupdb.py
index f286dbd2181ef9331098fe6c77d8cc6bd9f3329d..46f383d1d386178108b65244fc613d0d127e9ef6 100644 (file)
--- a/roundup/roundupdb.py
+++ b/roundup/roundupdb.py
# BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
# SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
#
-# $Id: roundupdb.py,v 1.101 2004-03-15 05:50:19 richard Exp $
+# $Id: roundupdb.py,v 1.102 2004-03-19 04:47:59 richard Exp $
"""Extending hyperdb with types specific to issue-tracking.
"""
return timezone
def confirm_registration(self, otk):
- props = self.otks.getall(otk)
+ props = self.getOTKManager().getall(otk)
for propname, proptype in self.user.getprops().items():
value = props.get(propname, None)
if value is None:
cl = self.user
props['roles'] = self.config.NEW_WEB_USER_ROLES
- del props['__time']
userid = cl.create(**props)
# clear the props from the otk database
- self.otks.destroy(otk)
+ self.getOTKManager().destroy(otk)
self.commit()
return userid
diff --git a/test/db_test_base.py b/test/db_test_base.py
index 2405c54d425734aca23976db511a77d02b5614c7..193e145139f86c1fdbd83ec756e42a0c0ce2a8c0 100644 (file)
--- a/test/db_test_base.py
+++ b/test/db_test_base.py
# BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
# SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
#
-# $Id: db_test_base.py,v 1.17 2004-03-18 01:58:46 richard Exp $
+# $Id: db_test_base.py,v 1.18 2004-03-19 04:47:59 richard Exp $
import unittest, os, shutil, errno, imp, sys, time, pprint
Interval, DatabaseError, Boolean, Number, Node
from roundup import date, password
from roundup import init
-from roundup.indexer import Indexer
def setupSchema(db, create, module):
status = module.Class(db, "status", name=String())
# automatic properties (well, the two easy ones anyway)
#
def testCreatorProperty(self):
- id1 = self.db.issue.create()
+ i = self.db.issue
+ id1 = i.create(title='spam')
self.db.commit()
self.db.close()
self.db = self.module.Database(config, 'fred')
setupSchema(self.db, 0, self.module)
i = self.db.issue
- id2 = i.create()
+ id2 = i.create(title='spam')
self.assertNotEqual(id1, id2)
self.assertNotEqual(i.get(id1, 'creator'), i.get(id2, 'creator'))
def testActorProperty(self):
- id1 = self.db.issue.create()
+ i = self.db.issue
+ id1 = i.create(title='spam')
self.db.commit()
self.db.close()
self.db = self.module.Database(config, 'fred')
id1 = self.db.issue.create(title="spam", status='1')
self.db.issue.set(id1)
+ # String
def testStringChange(self):
for commit in (0,1):
# test set & retrieve
if commit: self.db.commit()
self.assertEqual(self.db.issue.get(nid, "title"), None)
+ # FileClass "content" property (no unset test)
+ def testFileClassContentChange(self):
+ for commit in (0,1):
+ # test set & retrieve
+ nid = self.db.file.create(content="spam")
+ self.assertEqual(self.db.file.get(nid, 'content'), 'spam')
+
+ # change and make sure we retrieve the correct value
+ self.db.file.set(nid, content='eggs')
+ if commit: self.db.commit()
+ self.assertEqual(self.db.file.get(nid, 'content'), 'eggs')
+
+ # Link
def testLinkChange(self):
self.assertRaises(IndexError, self.db.issue.create, title="spam",
status='100')
if commit: self.db.commit()
self.assertEqual(self.db.issue.get(nid, "status"), None)
+ # Multilink
def testMultilinkChange(self):
for commit in (0,1):
self.assertRaises(IndexError, self.db.issue.create, title="spam",
self.assertEqual(self.db.issue.get(nid, "nosy"), [])
self.db.issue.set(nid, nosy=[u1,u2])
if commit: self.db.commit()
- self.assertEqual(self.db.issue.get(nid, "nosy"), [u1,u2])
+ l = [u1,u2]; l.sort()
+ m = self.db.issue.get(nid, "nosy"); m.sort()
+ self.assertEqual(l, m)
+ # Date
def testDateChange(self):
self.assertRaises(TypeError, self.db.issue.create,
title='spam', deadline=1)
if commit: self.db.commit()
self.assertEqual(self.db.issue.get(nid, "deadline"), None)
+ # Interval
def testIntervalChange(self):
self.assertRaises(TypeError, self.db.issue.create,
title='spam', foo=1)
if commit: self.db.commit()
self.assertEqual(self.db.issue.get(nid, "foo"), None)
+ # Boolean
def testBooleanChange(self):
userid = self.db.user.create(username='foo', assignable=1)
self.assertEqual(1, self.db.user.get(userid, 'assignable'))
self.db.user.set(nid, assignable=None)
self.assertEqual(self.db.user.get(nid, "assignable"), None)
+ # Number
def testNumberChange(self):
nid = self.db.user.create(username='foo', age=1)
self.assertEqual(1, self.db.user.get(nid, 'age'))
self.db.user.set(nid, age=None)
self.assertEqual(self.db.user.get(nid, "age"), None)
+ # Password
def testPasswordChange(self):
x = password.Password('x')
userid = self.db.user.create(username='foo', password=x)
self.db.user.set(nid, assignable=None)
self.assertEqual(self.db.user.get(nid, "assignable"), None)
+ # key value
def testKeyValue(self):
self.assertRaises(ValueError, self.db.user.create)
self.assertRaises(TypeError, self.db.issue.lookup, 'fubar')
+ # label property
def testLabelProp(self):
# key prop
self.assertEqual(self.db.status.labelprop(), 'name')
# id
self.assertEqual(self.db.stuff.labelprop(default_to_id=1), 'id')
+ # retirement
def testRetire(self):
self.db.issue.create(title="spam", status='1')
b = self.db.status.get('1', 'name')
def testIndexerSearching(self):
f1 = self.db.file.create(content='hello', type="text/plain")
+ # content='world' has the wrong content-type and won't be indexed
f2 = self.db.file.create(content='world', type="text/frozz",
comment='blah blah')
i1 = self.db.issue.create(files=[f1, f2], title="flebble plop")
{i1: {}, i2: {}})
def testReindexing(self):
- self.db.issue.create(title="frooz")
+ search = self.db.indexer.search
+ issue = self.db.issue
+ i1 = issue.create(title="flebble plop")
+ i2 = issue.create(title="flebble frooz")
self.db.commit()
- self.assertEquals(self.db.indexer.search(['frooz'], self.db.issue),
- {'1': {}})
- self.db.issue.set('1', title="dooble")
+ self.assertEquals(search(['plop'], issue), {i1: {}})
+ self.assertEquals(search(['flebble'], issue), {i1: {}, i2: {}})
+
+ # change i1's title
+ issue.set(i1, title="plop")
self.db.commit()
- self.assertEquals(self.db.indexer.search(['dooble'], self.db.issue),
- {'1': {}})
- self.assertEquals(self.db.indexer.search(['frooz'], self.db.issue), {})
+ self.assertEquals(search(['plop'], issue), {i1: {}})
+ self.assertEquals(search(['flebble'], issue), {i2: {}})
+
+ # unset i1's title
+ issue.set(i1, title="")
+ self.db.commit()
+ self.assertEquals(search(['plop'], issue), {})
+ self.assertEquals(search(['flebble'], issue), {i2: {}})
+
+ def testFileClassReindexing(self):
+ f1 = self.db.file.create(content='hello')
+ f2 = self.db.file.create(content='hello, world')
+ i1 = self.db.issue.create(files=[f1, f2])
+ self.db.commit()
+ d = self.db.indexer.search(['hello'], self.db.issue)
+ d[i1]['files'].sort()
+ self.assertEquals(d, {i1: {'files': [f1, f2]}})
+ self.assertEquals(self.db.indexer.search(['world'], self.db.issue),
+ {i1: {'files': [f2]}})
+ self.db.file.set(f1, content="world")
+ self.db.commit()
+ d = self.db.indexer.search(['world'], self.db.issue)
+ d[i1]['files'].sort()
+ self.assertEquals(d, {i1: {'files': [f1, f2]}})
+ self.assertEquals(self.db.indexer.search(['hello'], self.db.issue),
+ {i1: {'files': [f2]}})
+
def testForcedReindexing(self):
self.db.issue.create(title="flebble frooz")
ae(l, m)
for id, props in items.items():
for name, value in props.items():
- ae(klass.get(id, name), value)
+ l = klass.get(id, name)
+ if isinstance(value, type([])):
+ value.sort()
+ l.sort()
+ ae(l, value)
# make sure the retired items are actually imported
- ae(self.db.user.get('3', 'username'), 'blop')
+ ae(self.db.user.get('4', 'username'), 'blop')
ae(self.db.issue.get('2', 'title'), 'issue two')
# make sure id counters are set correctly
diff --git a/test/session_common.py b/test/session_common.py
index 7029528d3d7836769584b157ddf97b889c8a2642..8a9077017419e2d798842f9ea76c9dc7d31e47e5 100644 (file)
--- a/test/session_common.py
+++ b/test/session_common.py
self.sessions.set('random_key', text='nope')
self.assertEqual(self.sessions.get('random_key', 'text'), 'nope')
- def testSetOTK(self):
- assert 0, 'not implemented'
-
- def testExpiry(self):
- assert 0, 'not implemented'
-
class DBMTest(SessionTest):
import roundup.backends.sessions_dbm as sessions_module
diff --git a/test/test_indexer.py b/test/test_indexer.py
index 18e56f971e81690ae753f155a2d72f31a7000a50..8155577fc6c2780d92563adac9486c3637e8df19 100644 (file)
--- a/test/test_indexer.py
+++ b/test/test_indexer.py
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
-# $Id: test_indexer.py,v 1.3 2003-10-25 22:53:26 richard Exp $
+# $Id: test_indexer.py,v 1.4 2004-03-19 04:47:59 richard Exp $
import os, unittest, shutil
-from roundup.indexer import Indexer
+from roundup.backends.indexer_dbm import Indexer
class IndexerTest(unittest.TestCase):
def setUp(self):
diff --git a/test/test_mailgw.py b/test/test_mailgw.py
index aebded9b1c4c07e6fe64d1e2538c474bea5b3434..dcdcf015e35ec6bd8b62ab21ad8185556aba92ab 100644 (file)
--- a/test/test_mailgw.py
+++ b/test/test_mailgw.py
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
-# $Id: test_mailgw.py,v 1.64 2004-01-20 00:11:51 richard Exp $
+# $Id: test_mailgw.py,v 1.65 2004-03-19 04:47:59 richard Exp $
import unittest, tempfile, os, shutil, errno, imp, sys, difflib, rfc822
def testRegistrationConfirmation(self):
otk = "Aj4euk4LZSAdwePohj90SME5SpopLETL"
- self.db.otks.set(otk, username='johannes', __time='')
+ self.db.getOTKManager().set(otk, username='johannes')
self._handle_mail('''Content-Type: text/plain;
charset="iso-8859-1"
From: Chef <chef@bork.bork.bork>