You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2012/08/17 02:42:39 UTC
svn commit: r1374116 - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/analysis/
lucene/analysis/stempel/src/java/org/apache/lucene/analysis/pl/
lucene/analysis/stempel/src/java/org/apache/lucene/analysis/stempel/
lucene/analysis/stempel/src/java/...
Author: rmuir
Date: Fri Aug 17 00:42:39 2012
New Revision: 1374116
URL: http://svn.apache.org/viewvc?rev=1374116&view=rev
Log:
SOLR-3737: fix Stempel factory resource loading, use singleton instance
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/analysis/ (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/apache/lucene/analysis/pl/PolishAnalyzer.java
lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/apache/lucene/analysis/stempel/StempelPolishStemFilterFactory.java
lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/egothor/stemmer/Trie.java
lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/test/org/apache/lucene/analysis/stempel/TestStempelPolishStemFilterFactory.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1374116&r1=1374115&r2=1374116&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Fri Aug 17 00:42:39 2012
@@ -75,6 +75,10 @@ Bug Fixes
encoders / stemmers via the ResourceLoader now instead of Class.forName().
Solr users should now no longer have to embed these in its war. (David Smiley)
+* SOLR-3737: StempelPolishStemFilterFactory loaded its stemmer table incorrectly.
+ Also, ensure immutability and use only one instance of this table in RAM (lazy
+ loaded) since its quite large. (sausarkar, Steven Rowe, Robert Muir)
+
Build
* LUCENE-3985: Upgrade to randomizedtesting 2.0.0. Added support for
Modified: lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/apache/lucene/analysis/pl/PolishAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/apache/lucene/analysis/pl/PolishAnalyzer.java?rev=1374116&r1=1374115&r2=1374116&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/apache/lucene/analysis/pl/PolishAnalyzer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/apache/lucene/analysis/pl/PolishAnalyzer.java Fri Aug 17 00:42:39 2012
@@ -59,6 +59,13 @@ public final class PolishAnalyzer extend
}
/**
+ * Returns an unmodifiable instance of the default stemmer table.
+ */
+ public static Trie getDefaultTable() {
+ return DefaultsHolder.DEFAULT_TABLE;
+ }
+
+ /**
* Atomically loads the DEFAULT_STOP_SET in a lazy fashion once the outer class
* accesses the static final set the first time.;
*/
Modified: lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/apache/lucene/analysis/stempel/StempelPolishStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/apache/lucene/analysis/stempel/StempelPolishStemFilterFactory.java?rev=1374116&r1=1374115&r2=1374116&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/apache/lucene/analysis/stempel/StempelPolishStemFilterFactory.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/apache/lucene/analysis/stempel/StempelPolishStemFilterFactory.java Fri Aug 17 00:42:39 2012
@@ -17,28 +17,17 @@ package org.apache.lucene.analysis.stemp
* limitations under the License.
*/
-import java.io.IOException;
-
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.pl.PolishAnalyzer;
import org.apache.lucene.analysis.stempel.StempelFilter;
import org.apache.lucene.analysis.stempel.StempelStemmer;
-import org.apache.lucene.analysis.util.ResourceLoader;
-import org.apache.lucene.analysis.util.ResourceLoaderAware;
import org.apache.lucene.analysis.util.TokenFilterFactory;
-import org.egothor.stemmer.Trie;
/**
* Factory for {@link StempelFilter} using a Polish stemming table.
*/
-public class StempelPolishStemFilterFactory extends TokenFilterFactory implements ResourceLoaderAware {
- private Trie stemmer = null;
- private static final String STEMTABLE = "/org/apache/lucene/analysis/pl/stemmer_20000.tbl";
-
+public class StempelPolishStemFilterFactory extends TokenFilterFactory {
public TokenStream create(TokenStream input) {
- return new StempelFilter(input, new StempelStemmer(stemmer));
- }
-
- public void inform(ResourceLoader loader) throws IOException {
- stemmer = StempelStemmer.load(loader.openResource(STEMTABLE));
+ return new StempelFilter(input, new StempelStemmer(PolishAnalyzer.getDefaultTable()));
}
}
Modified: lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/egothor/stemmer/Trie.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/egothor/stemmer/Trie.java?rev=1374116&r1=1374115&r2=1374116&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/egothor/stemmer/Trie.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/egothor/stemmer/Trie.java Fri Aug 17 00:42:39 2012
@@ -332,7 +332,7 @@ public class Trie {
* @param key the key
* @param cmd the patch command
*/
- public void add(CharSequence key, CharSequence cmd) {
+ void add(CharSequence key, CharSequence cmd) {
if (key == null || cmd == null) {
return;
}
Modified: lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/test/org/apache/lucene/analysis/stempel/TestStempelPolishStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/test/org/apache/lucene/analysis/stempel/TestStempelPolishStemFilterFactory.java?rev=1374116&r1=1374115&r2=1374116&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/test/org/apache/lucene/analysis/stempel/TestStempelPolishStemFilterFactory.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/test/org/apache/lucene/analysis/stempel/TestStempelPolishStemFilterFactory.java Fri Aug 17 00:42:39 2012
@@ -22,7 +22,6 @@ import java.io.StringReader;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
-import org.apache.lucene.analysis.util.ClasspathResourceLoader;
/**
* Tests for {@link StempelPolishStemFilterFactory}
@@ -31,7 +30,6 @@ public class TestStempelPolishStemFilter
public void testBasics() throws Exception {
StringReader document = new StringReader("studenta studenci");
StempelPolishStemFilterFactory factory = new StempelPolishStemFilterFactory();
- factory.inform(new ClasspathResourceLoader(getClass()));
TokenStream ts = factory.create(new WhitespaceTokenizer(TEST_VERSION_CURRENT, document));
assertTokenStreamContents(ts,
new String[] { "student", "student" });