You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/11/04 10:54:55 UTC
svn commit: r1197469 - in /lucene/dev/trunk: lucene/contrib/
modules/suggest/src/java/org/apache/lucene/search/spell/
modules/suggest/src/test/org/apache/lucene/search/spell/
solr/core/src/java/org/apache/solr/spelling/
Author: rmuir
Date: Fri Nov 4 09:54:55 2011
New Revision: 1197469
URL: http://svn.apache.org/viewvc?rev=1197469&view=rev
Log:
LUCENE-3557: SpellChecker should take IWC, remove sneaky silent optimize() methods
Modified:
lucene/dev/trunk/lucene/contrib/CHANGES.txt
lucene/dev/trunk/modules/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java
lucene/dev/trunk/modules/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java
lucene/dev/trunk/modules/suggest/src/test/org/apache/lucene/search/spell/TestPlainTextDictionary.java
lucene/dev/trunk/modules/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java
Modified: lucene/dev/trunk/lucene/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/CHANGES.txt?rev=1197469&r1=1197468&r2=1197469&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/contrib/CHANGES.txt Fri Nov 4 09:54:55 2011
@@ -88,6 +88,14 @@ Changes in backwards compatibility polic
* LUCENE-3508: Changed some method signatures in decompounding TokenFilters
to make them no longer use the Token class. (Uwe Schindler)
+
+ * LUCENE-3557: The various SpellChecker.indexDictionary methods were removed,
+ and consolidated to one:
+
+ indexDictionary(Dictionary dict, IndexWriterConfig config, boolean optimize)
+
+ Previously, there was no way to specify an IndexWriterConfig, and some
+ of these methods would sneakily pass 'true' to optimize. (Robert Muir)
New Features
Modified: lucene/dev/trunk/modules/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java?rev=1197469&r1=1197468&r2=1197469&view=diff
==============================================================================
--- lucene/dev/trunk/modules/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java (original)
+++ lucene/dev/trunk/modules/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java Fri Nov 4 09:54:55 2011
@@ -31,7 +31,6 @@ import org.apache.lucene.document.String
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.Terms;
@@ -481,18 +480,16 @@ public class SpellChecker implements jav
/**
* Indexes the data from the given {@link Dictionary}.
* @param dict Dictionary to index
- * @param mergeFactor mergeFactor to use when indexing
- * @param ramMB the max amount or memory in MB to use
+ * @param config {@link IndexWriterConfig} to use
* @param optimize whether or not the spellcheck index should be optimized
* @throws AlreadyClosedException if the Spellchecker is already closed
* @throws IOException
*/
- public final void indexDictionary(Dictionary dict, int mergeFactor, int ramMB, boolean optimize) throws IOException {
+ public final void indexDictionary(Dictionary dict, IndexWriterConfig config, boolean optimize) throws IOException {
synchronized (modifyCurrentIndexLock) {
ensureOpen();
final Directory dir = this.spellIndex;
- final IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_CURRENT, null).setRAMBufferSizeMB(ramMB));
- ((TieredMergePolicy) writer.getConfig().getMergePolicy()).setMaxMergeAtOnce(mergeFactor);
+ final IndexWriter writer = new IndexWriter(dir, config);
IndexSearcher indexSearcher = obtainSearcher();
final List<TermsEnum> termsEnums = new ArrayList<TermsEnum>();
@@ -543,32 +540,15 @@ public class SpellChecker implements jav
if (optimize)
writer.optimize();
writer.close();
+ // TODO: this isn't that great, maybe in the future SpellChecker should take
+ // IWC in its ctor / keep its writer open?
+
// also re-open the spell index to see our own changes when the next suggestion
// is fetched:
swapSearcher(dir);
}
}
- /**
- * Indexes the data from the given {@link Dictionary}.
- * @param dict the dictionary to index
- * @param mergeFactor mergeFactor to use when indexing
- * @param ramMB the max amount or memory in MB to use
- * @throws IOException
- */
- public final void indexDictionary(Dictionary dict, int mergeFactor, int ramMB) throws IOException {
- indexDictionary(dict, mergeFactor, ramMB, true);
- }
-
- /**
- * Indexes the data from the given {@link Dictionary}.
- * @param dict the dictionary to index
- * @throws IOException
- */
- public final void indexDictionary(Dictionary dict) throws IOException {
- indexDictionary(dict, 300, (int)IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB);
- }
-
private static int getMin(int l) {
if (l > 5) {
return 3;
Modified: lucene/dev/trunk/modules/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java?rev=1197469&r1=1197468&r2=1197469&view=diff
==============================================================================
--- lucene/dev/trunk/modules/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java (original)
+++ lucene/dev/trunk/modules/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java Fri Nov 4 09:54:55 2011
@@ -195,7 +195,7 @@ public class TestLuceneDictionary extend
Directory dir = newDirectory();
SpellChecker sc = new SpellChecker(dir);
indexReader = IndexReader.open(store, true);
- sc.indexDictionary(new LuceneDictionary(indexReader, "contents"));
+ sc.indexDictionary(new LuceneDictionary(indexReader, "contents"), newIndexWriterConfig(TEST_VERSION_CURRENT, null), false);
String[] suggestions = sc.suggestSimilar("Tam", 1);
assertEquals(1, suggestions.length);
assertEquals("Tom", suggestions[0]);
Modified: lucene/dev/trunk/modules/suggest/src/test/org/apache/lucene/search/spell/TestPlainTextDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/suggest/src/test/org/apache/lucene/search/spell/TestPlainTextDictionary.java?rev=1197469&r1=1197468&r2=1197469&view=diff
==============================================================================
--- lucene/dev/trunk/modules/suggest/src/test/org/apache/lucene/search/spell/TestPlainTextDictionary.java (original)
+++ lucene/dev/trunk/modules/suggest/src/test/org/apache/lucene/search/spell/TestPlainTextDictionary.java Fri Nov 4 09:54:55 2011
@@ -35,7 +35,7 @@ public class TestPlainTextDictionary ext
PlainTextDictionary ptd = new PlainTextDictionary(new StringReader(input));
Directory ramDir = newDirectory();
SpellChecker spellChecker = new SpellChecker(ramDir);
- spellChecker.indexDictionary(ptd);
+ spellChecker.indexDictionary(ptd, newIndexWriterConfig(TEST_VERSION_CURRENT, null), false);
String[] similar = spellChecker.suggestSimilar("treeword", 2);
assertEquals(2, similar.length);
assertEquals(similar[0], "threeword");
Modified: lucene/dev/trunk/modules/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java?rev=1197469&r1=1197468&r2=1197469&view=diff
==============================================================================
--- lucene/dev/trunk/modules/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java (original)
+++ lucene/dev/trunk/modules/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java Fri Nov 4 09:54:55 2011
@@ -331,7 +331,7 @@ public class TestSpellChecker extends Lu
private void addwords(IndexReader r, SpellChecker sc, String field) throws IOException {
long time = System.currentTimeMillis();
- sc.indexDictionary(new LuceneDictionary(r, field));
+ sc.indexDictionary(new LuceneDictionary(r, field), newIndexWriterConfig(TEST_VERSION_CURRENT, null), false);
time = System.currentTimeMillis() - time;
//System.out.println("time to build " + field + ": " + time);
}
@@ -379,7 +379,7 @@ public class TestSpellChecker extends Lu
}
try {
- spellChecker.indexDictionary(new LuceneDictionary(r, field));
+ spellChecker.indexDictionary(new LuceneDictionary(r, field), newIndexWriterConfig(TEST_VERSION_CURRENT, null), false);
fail("spellchecker was already closed");
} catch (AlreadyClosedException e) {
// expected
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java?rev=1197469&r1=1197468&r2=1197469&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java Fri Nov 4 09:54:55 2011
@@ -62,7 +62,7 @@ public class FileBasedSpellChecker exten
try {
loadExternalFileDictionary(core);
spellChecker.clearIndex();
- spellChecker.indexDictionary(dictionary);
+ spellChecker.indexDictionary(dictionary, new IndexWriterConfig(core.getSolrConfig().luceneMatchVersion, null), false);
} catch (IOException e) {
throw new RuntimeException(e);
}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java?rev=1197469&r1=1197468&r2=1197469&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java Fri Nov 4 09:54:55 2011
@@ -17,6 +17,7 @@ package org.apache.solr.spelling;
*/
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.search.spell.HighFrequencyDictionary;
@@ -86,7 +87,7 @@ public class IndexBasedSpellChecker exte
dictionary = new HighFrequencyDictionary(reader, field,
threshold);
spellChecker.clearIndex();
- spellChecker.indexDictionary(dictionary);
+ spellChecker.indexDictionary(dictionary, new IndexWriterConfig(core.getSolrConfig().luceneMatchVersion, null), false);
} catch (IOException e) {
throw new RuntimeException(e);