You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/11/04 11:10:04 UTC
svn commit: r1197478 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/backwards/src/test/ lucene/contrib/ lucene/contrib/analyzers/common/
lucene/contrib/spellchecker/
lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/
lucene/co...
Author: rmuir
Date: Fri Nov 4 10:10:03 2011
New Revision: 1197478
URL: http://svn.apache.org/viewvc?rev=1197478&view=rev
Log:
LUCENE-3557: SpellChecker should take IWC, remove sneaky silent optimize() methods
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SpellChecker.java
lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java
lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestPlainTextDictionary.java
lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestSpellChecker.java
lucene/dev/branches/branch_3x/solr/ (props changed)
lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/SpellCheckerRequestHandler.java
lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java
lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java
lucene/dev/branches/branch_3x/solr/core/src/test/org/apache/solr/analysis/TestPhoneticFilterFactory.java (props changed)
Modified: lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt?rev=1197478&r1=1197477&r2=1197478&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt Fri Nov 4 10:10:03 2011
@@ -12,6 +12,14 @@ Changes in backwards compatibility polic
* LUCENE-3508: Changed some method signatures in decompounding TokenFilters
to make them no longer use the Token class. (Uwe Schindler)
+
+ * LUCENE-3557: The various SpellChecker.indexDictionary methods were removed,
+ and consolidated to one:
+
+ indexDictionary(Dictionary dict, IndexWriterConfig config, boolean optimize)
+
+ Previously, there was no way to specify an IndexWriterConfig, and some
+ of these methods would sneakily pass 'true' to optimize. (Robert Muir)
* LUCENE-3552: Renamed LuceneTaxonomyReader/Writer to DirectoryTR/TW. (Shai Erera)
Modified: lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SpellChecker.java?rev=1197478&r1=1197477&r2=1197478&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SpellChecker.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SpellChecker.java Fri Nov 4 10:10:03 2011
@@ -30,7 +30,6 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.BooleanClause;
@@ -561,18 +560,16 @@ public class SpellChecker implements jav
/**
* Indexes the data from the given {@link Dictionary}.
* @param dict Dictionary to index
- * @param mergeFactor mergeFactor to use when indexing
- * @param ramMB the max amount or memory in MB to use
+ * @param config {@link IndexWriterConfig} to use
* @param optimize whether or not the spellcheck index should be optimized
* @throws AlreadyClosedException if the Spellchecker is already closed
* @throws IOException
*/
- public final void indexDictionary(Dictionary dict, int mergeFactor, int ramMB, boolean optimize) throws IOException {
+ public final void indexDictionary(Dictionary dict, IndexWriterConfig config, boolean optimize) throws IOException {
synchronized (modifyCurrentIndexLock) {
ensureOpen();
final Directory dir = this.spellIndex;
- final IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_CURRENT, new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).setRAMBufferSizeMB(ramMB));
- ((TieredMergePolicy) writer.getConfig().getMergePolicy()).setMaxMergeAtOnce(mergeFactor);
+ final IndexWriter writer = new IndexWriter(dir, config);
IndexSearcher indexSearcher = obtainSearcher();
final List<IndexReader> readers = new ArrayList<IndexReader>();
@@ -614,32 +611,15 @@ public class SpellChecker implements jav
if (optimize)
writer.optimize();
writer.close();
+ // TODO: this isn't that great, maybe in the future SpellChecker should take
+ // IWC in its ctor / keep its writer open?
+
// also re-open the spell index to see our own changes when the next suggestion
// is fetched:
swapSearcher(dir);
}
}
- /**
- * Indexes the data from the given {@link Dictionary}.
- * @param dict the dictionary to index
- * @param mergeFactor mergeFactor to use when indexing
- * @param ramMB the max amount or memory in MB to use
- * @throws IOException
- */
- public final void indexDictionary(Dictionary dict, int mergeFactor, int ramMB) throws IOException {
- indexDictionary(dict, mergeFactor, ramMB, true);
- }
-
- /**
- * Indexes the data from the given {@link Dictionary}.
- * @param dict the dictionary to index
- * @throws IOException
- */
- public final void indexDictionary(Dictionary dict) throws IOException {
- indexDictionary(dict, 300, (int)IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB);
- }
-
private static int getMin(int l) {
if (l > 5) {
return 3;
Modified: lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java?rev=1197478&r1=1197477&r2=1197478&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java Fri Nov 4 10:10:03 2011
@@ -196,7 +196,7 @@ public class TestLuceneDictionary extend
Directory dir = newDirectory();
SpellChecker sc = new SpellChecker(dir);
indexReader = IndexReader.open(store, true);
- sc.indexDictionary(new LuceneDictionary(indexReader, "contents"));
+ sc.indexDictionary(new LuceneDictionary(indexReader, "contents"), newIndexWriterConfig(TEST_VERSION_CURRENT, null), false);
String[] suggestions = sc.suggestSimilar("Tam", 1);
assertEquals(1, suggestions.length);
assertEquals("Tom", suggestions[0]);
Modified: lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestPlainTextDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestPlainTextDictionary.java?rev=1197478&r1=1197477&r2=1197478&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestPlainTextDictionary.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestPlainTextDictionary.java Fri Nov 4 10:10:03 2011
@@ -35,7 +35,7 @@ public class TestPlainTextDictionary ext
PlainTextDictionary ptd = new PlainTextDictionary(new StringReader(input));
Directory ramDir = newDirectory();
SpellChecker spellChecker = new SpellChecker(ramDir);
- spellChecker.indexDictionary(ptd);
+ spellChecker.indexDictionary(ptd, newIndexWriterConfig(TEST_VERSION_CURRENT, null), false);
String[] similar = spellChecker.suggestSimilar("treeword", 2);
assertEquals(2, similar.length);
assertEquals(similar[0], "threeword");
Modified: lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestSpellChecker.java?rev=1197478&r1=1197477&r2=1197478&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestSpellChecker.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestSpellChecker.java Fri Nov 4 10:10:03 2011
@@ -331,7 +331,7 @@ public class TestSpellChecker extends Lu
private void addwords(IndexReader r, SpellChecker sc, String field) throws IOException {
long time = System.currentTimeMillis();
- sc.indexDictionary(new LuceneDictionary(r, field));
+ sc.indexDictionary(new LuceneDictionary(r, field), newIndexWriterConfig(TEST_VERSION_CURRENT, null), false);
time = System.currentTimeMillis() - time;
//System.out.println("time to build " + field + ": " + time);
}
@@ -379,7 +379,7 @@ public class TestSpellChecker extends Lu
}
try {
- spellChecker.indexDictionary(new LuceneDictionary(r, field));
+ spellChecker.indexDictionary(new LuceneDictionary(r, field), newIndexWriterConfig(TEST_VERSION_CURRENT, null), false);
fail("spellchecker was already closed");
} catch (AlreadyClosedException e) {
// expected
Modified: lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/SpellCheckerRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/SpellCheckerRequestHandler.java?rev=1197478&r1=1197477&r2=1197478&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/SpellCheckerRequestHandler.java (original)
+++ lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/SpellCheckerRequestHandler.java Fri Nov 4 10:10:03 2011
@@ -18,6 +18,7 @@
package org.apache.solr.handler;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.spell.Dictionary;
@@ -400,7 +401,7 @@ public class SpellCheckerRequestHandler
Dictionary dictionary = getDictionary(req);
spellChecker.clearIndex();
- spellChecker.indexDictionary(dictionary);
+ spellChecker.indexDictionary(dictionary, new IndexWriterConfig(req.getCore().getSolrConfig().luceneMatchVersion, null), false);
reopen();
}
Modified: lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java?rev=1197478&r1=1197477&r2=1197478&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java (original)
+++ lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java Fri Nov 4 10:10:03 2011
@@ -63,7 +63,7 @@ public class FileBasedSpellChecker exten
try {
loadExternalFileDictionary(core);
spellChecker.clearIndex();
- spellChecker.indexDictionary(dictionary);
+ spellChecker.indexDictionary(dictionary, new IndexWriterConfig(core.getSolrConfig().luceneMatchVersion, null), false);
} catch (IOException e) {
throw new RuntimeException(e);
}
Modified: lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java?rev=1197478&r1=1197477&r2=1197478&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java (original)
+++ lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java Fri Nov 4 10:10:03 2011
@@ -17,6 +17,7 @@ package org.apache.solr.spelling;
*/
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.FSDirectory;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrCore;
@@ -85,7 +86,7 @@ public class IndexBasedSpellChecker exte
dictionary = new HighFrequencyDictionary(reader, field,
threshold);
spellChecker.clearIndex();
- spellChecker.indexDictionary(dictionary);
+ spellChecker.indexDictionary(dictionary, new IndexWriterConfig(core.getSolrConfig().luceneMatchVersion, null), false);
} catch (IOException e) {
throw new RuntimeException(e);