svn commit: r1358482 - in /lucene/dev/branches/branch_4x: ./ solr/ solr/core/ solr/core/src/java/org/apache/solr/analysis/ solr/core/src/test-files/solr/collection1/conf/ solr/core/src/test/org/apache/solr/analysis/ solr/test-framework/ solr/test-frame...

[rm...@apache.org]

Author: rmuir
Date: Sat Jul  7 00:39:10 2012
New Revision: 1358482

URL: http://svn.apache.org/viewvc?rev=1358482&view=rev
Log:
LUCENE-4185: don't apply CharFilters twice in Solr

Added:
    lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/schema-charfilters.xml
      - copied unchanged from r1358481, lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-charfilters.xml
    lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/analysis/TestCharFilters.java
      - copied unchanged from r1358481, lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/TestCharFilters.java
    lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/analysis/MockCharFilterFactory.java
      - copied unchanged from r1358481, lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/analysis/MockCharFilterFactory.java
Modified:
    lucene/dev/branches/branch_4x/   (props changed)
    lucene/dev/branches/branch_4x/solr/   (props changed)
    lucene/dev/branches/branch_4x/solr/CHANGES.txt   (contents, props changed)
    lucene/dev/branches/branch_4x/solr/core/   (props changed)
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java
    lucene/dev/branches/branch_4x/solr/test-framework/   (props changed)

Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1358482&r1=1358481&r2=1358482&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Sat Jul  7 00:39:10 2012
@@ -53,6 +53,8 @@ Bug Fixes
   
 * SOLR-3587: After reloading a SolrCore, the original Analyzer is still used rather than a new 
   one. (Alexey Serba, yonik, rmuir, Mark Miller)
+  
+* LUCENE-4185: Fix a bug where CharFilters were wrongly being applied twice. (Michael Froh, rmuir)
 
 Other Changes
 

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java?rev=1358482&r1=1358481&r2=1358482&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java Sat Jul  7 00:39:10 2012
@@ -22,17 +22,12 @@ import org.apache.lucene.analysis.util.C
 import org.apache.lucene.analysis.util.TokenFilterFactory;
 import org.apache.lucene.analysis.util.TokenizerFactory;
 
-import java.io.IOException;
 import java.io.Reader;
 
 /**
- *
+ * An analyzer that uses a tokenizer and a list of token filters to
+ * create a TokenStream.
  */
-
-//
-// An analyzer that uses a tokenizer and a list of token filters to
-// create a TokenStream.
-//
 public final class TokenizerChain extends SolrAnalyzer {
   final private CharFilterFactory[] charFilters;
   final private TokenizerFactory tokenizer;
@@ -52,22 +47,6 @@ public final class TokenizerChain extend
   public TokenizerFactory getTokenizerFactory() { return tokenizer; }
   public TokenFilterFactory[] getTokenFilterFactories() { return filters; }
 
-  class SolrTokenStreamComponents extends TokenStreamComponents {
-    public SolrTokenStreamComponents(final Tokenizer source, final TokenStream result) {
-      super(source, result);
-    }
-
-    // TODO: what is going on here?
-    @Override
-    protected void reset(Reader reader) throws IOException {
-      // the tokenizers are currently reset by the indexing process, so only
-      // the tokenizer needs to be reset.
-      Reader r = initReader(null, reader);
-      super.reset(r);
-    }
-  }
-  
-  
   @Override
   public Reader initReader(String fieldName, Reader reader) {
     if (charFilters != null && charFilters.length > 0) {
@@ -82,12 +61,12 @@ public final class TokenizerChain extend
 
   @Override
   protected TokenStreamComponents createComponents(String fieldName, Reader aReader) {
-    Tokenizer tk = tokenizer.create( initReader(fieldName, aReader) );
+    Tokenizer tk = tokenizer.create( aReader );
     TokenStream ts = tk;
     for (TokenFilterFactory filter : filters) {
       ts = filter.create(ts);
     }
-    return new SolrTokenStreamComponents(tk, ts);
+    return new TokenStreamComponents(tk, ts);
   }
 
   @Override