svn commit: r813668 - in /lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis: CharReader.java CharStream.java Tokenizer.java standard/StandardTokenizer.java

[us...@apache.org]

Author: uschindler
Date: Fri Sep 11 05:42:18 2009
New Revision: 813668

URL: http://svn.apache.org/viewvc?rev=813668&view=rev
Log:
LUCENE-1906: Revert changes of LUCENE-1466 in BW branch

Removed:
    lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/CharReader.java
    lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/CharStream.java
Modified:
    lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/Tokenizer.java
    lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java

Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/Tokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/Tokenizer.java?rev=813668&r1=813667&r2=813668&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/Tokenizer.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/Tokenizer.java Fri Sep 11 05:42:18 2009
@@ -34,14 +34,14 @@
 
 public abstract class Tokenizer extends TokenStream {
   /** The text source for this Tokenizer. */
-  protected CharStream input;
+  protected Reader input;
 
   /** Construct a tokenizer with null input. */
   protected Tokenizer() {}
 
   /** Construct a token stream processing the given input. */
   protected Tokenizer(Reader input) {
-    this.input = CharReader.get(input);
+    this.input = input;
   }
 
   /** By default, closes the input Reader. */
@@ -53,7 +53,7 @@
    *  analyzer (in its reusableTokenStream method) will use
    *  this to re-use a previously created tokenizer. */
   public void reset(Reader input) throws IOException {
-    this.input = CharReader.get(input);
+    this.input = input;
   }
 }
 

Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java?rev=813668&r1=813667&r2=813668&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java Fri Sep 11 05:42:18 2009
@@ -22,8 +22,6 @@
 
 import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.CharReader;
-import org.apache.lucene.analysis.CharStream;
 
 /** A grammar-based tokenizer constructed with JFlex
  *
@@ -89,7 +87,7 @@
   private boolean replaceInvalidAcronym = false;
     
   void setInput(Reader reader) {
-    this.input = CharReader.get(reader);
+    this.input = reader;
   }
 
   private int maxTokenLength = StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH;
@@ -110,7 +108,7 @@
      * <code>input</code> to a newly created JFlex scanner.
      */
     public StandardTokenizer(Reader input) {
-	    this.input = CharReader.get(input);
+	    this.input = input;
 	    this.scanner = new StandardTokenizerImpl(input);
     }
 
@@ -125,7 +123,7 @@
    */
   public StandardTokenizer(Reader input, boolean replaceInvalidAcronym) {
     this.replaceInvalidAcronym = replaceInvalidAcronym;
-    setInput(input);
+    this.input = input;
     this.scanner = new StandardTokenizerImpl(input);
   }
 
@@ -184,7 +182,7 @@
     }
 
     public void reset(Reader reader) throws IOException {
-        setInput(reader);
+        input = reader;
         reset();
     }