You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2012/08/17 02:42:39 UTC

svn commit: r1374116 - in /lucene/dev/branches/branch_4x: ./ lucene/ lucene/analysis/ lucene/analysis/stempel/src/java/org/apache/lucene/analysis/pl/ lucene/analysis/stempel/src/java/org/apache/lucene/analysis/stempel/ lucene/analysis/stempel/src/java/...

Author: rmuir
Date: Fri Aug 17 00:42:39 2012
New Revision: 1374116

URL: http://svn.apache.org/viewvc?rev=1374116&view=rev
Log:
SOLR-3737: fix Stempel factory resource loading, use singleton instance

Modified:
    lucene/dev/branches/branch_4x/   (props changed)
    lucene/dev/branches/branch_4x/lucene/   (props changed)
    lucene/dev/branches/branch_4x/lucene/CHANGES.txt   (contents, props changed)
    lucene/dev/branches/branch_4x/lucene/analysis/   (props changed)
    lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/apache/lucene/analysis/pl/PolishAnalyzer.java
    lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/apache/lucene/analysis/stempel/StempelPolishStemFilterFactory.java
    lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/egothor/stemmer/Trie.java
    lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/test/org/apache/lucene/analysis/stempel/TestStempelPolishStemFilterFactory.java

Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1374116&r1=1374115&r2=1374116&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Fri Aug 17 00:42:39 2012
@@ -75,6 +75,10 @@ Bug Fixes
   encoders / stemmers via the ResourceLoader now instead of Class.forName().
   Solr users should now no longer have to embed these in its war. (David Smiley)
 
+* SOLR-3737: StempelPolishStemFilterFactory loaded its stemmer table incorrectly.
+  Also, ensure immutability and use only one instance of this table in RAM (lazy
+  loaded) since its quite large. (sausarkar, Steven Rowe, Robert Muir)
+
 Build
 
 * LUCENE-3985: Upgrade to randomizedtesting 2.0.0. Added support for 

Modified: lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/apache/lucene/analysis/pl/PolishAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/apache/lucene/analysis/pl/PolishAnalyzer.java?rev=1374116&r1=1374115&r2=1374116&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/apache/lucene/analysis/pl/PolishAnalyzer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/apache/lucene/analysis/pl/PolishAnalyzer.java Fri Aug 17 00:42:39 2012
@@ -59,6 +59,13 @@ public final class PolishAnalyzer extend
   }
   
   /**
+   * Returns an unmodifiable instance of the default stemmer table.
+   */
+  public static Trie getDefaultTable() {
+    return DefaultsHolder.DEFAULT_TABLE;
+  }
+  
+  /**
    * Atomically loads the DEFAULT_STOP_SET in a lazy fashion once the outer class 
    * accesses the static final set the first time.;
    */

Modified: lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/apache/lucene/analysis/stempel/StempelPolishStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/apache/lucene/analysis/stempel/StempelPolishStemFilterFactory.java?rev=1374116&r1=1374115&r2=1374116&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/apache/lucene/analysis/stempel/StempelPolishStemFilterFactory.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/apache/lucene/analysis/stempel/StempelPolishStemFilterFactory.java Fri Aug 17 00:42:39 2012
@@ -17,28 +17,17 @@ package org.apache.lucene.analysis.stemp
  * limitations under the License.
  */
 
-import java.io.IOException;
-
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.pl.PolishAnalyzer;
 import org.apache.lucene.analysis.stempel.StempelFilter;
 import org.apache.lucene.analysis.stempel.StempelStemmer;
-import org.apache.lucene.analysis.util.ResourceLoader;
-import org.apache.lucene.analysis.util.ResourceLoaderAware;
 import org.apache.lucene.analysis.util.TokenFilterFactory;
-import org.egothor.stemmer.Trie;
 
 /**
  * Factory for {@link StempelFilter} using a Polish stemming table.
  */
-public class StempelPolishStemFilterFactory extends TokenFilterFactory implements ResourceLoaderAware {
-  private Trie stemmer = null;
-  private static final String STEMTABLE = "/org/apache/lucene/analysis/pl/stemmer_20000.tbl";
-  
+public class StempelPolishStemFilterFactory extends TokenFilterFactory {  
   public TokenStream create(TokenStream input) {
-    return new StempelFilter(input, new StempelStemmer(stemmer));
-  }
-
-  public void inform(ResourceLoader loader) throws IOException {
-    stemmer = StempelStemmer.load(loader.openResource(STEMTABLE));
+    return new StempelFilter(input, new StempelStemmer(PolishAnalyzer.getDefaultTable()));
   }
 }

Modified: lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/egothor/stemmer/Trie.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/egothor/stemmer/Trie.java?rev=1374116&r1=1374115&r2=1374116&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/egothor/stemmer/Trie.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/java/org/egothor/stemmer/Trie.java Fri Aug 17 00:42:39 2012
@@ -332,7 +332,7 @@ public class Trie {
    * @param key the key
    * @param cmd the patch command
    */
-  public void add(CharSequence key, CharSequence cmd) {
+  void add(CharSequence key, CharSequence cmd) {
     if (key == null || cmd == null) {
       return;
     }

Modified: lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/test/org/apache/lucene/analysis/stempel/TestStempelPolishStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/test/org/apache/lucene/analysis/stempel/TestStempelPolishStemFilterFactory.java?rev=1374116&r1=1374115&r2=1374116&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/test/org/apache/lucene/analysis/stempel/TestStempelPolishStemFilterFactory.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/stempel/src/test/org/apache/lucene/analysis/stempel/TestStempelPolishStemFilterFactory.java Fri Aug 17 00:42:39 2012
@@ -22,7 +22,6 @@ import java.io.StringReader;
 import org.apache.lucene.analysis.BaseTokenStreamTestCase;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;
-import org.apache.lucene.analysis.util.ClasspathResourceLoader;
 
 /**
  * Tests for {@link StempelPolishStemFilterFactory}
@@ -31,7 +30,6 @@ public class TestStempelPolishStemFilter
   public void testBasics() throws Exception {
     StringReader document = new StringReader("studenta studenci");
     StempelPolishStemFilterFactory factory = new StempelPolishStemFilterFactory();
-    factory.inform(new ClasspathResourceLoader(getClass()));
     TokenStream ts = factory.create(new WhitespaceTokenizer(TEST_VERSION_CURRENT, document));
     assertTokenStreamContents(ts,
         new String[] { "student", "student" });