svn commit: r1429402 - in /lucene/dev/branches/branch_4x: ./ solr/ solr/CHANGES.txt solr/core/ solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java

[us...@apache.org]

Author: uschindler
Date: Sat Jan  5 22:20:28 2013
New Revision: 1429402

URL: http://svn.apache.org/viewvc?rev=1429402&view=rev
Log:
Merged revision(s) 1429401 from lucene/dev/trunk:
SOLR-4275: Fix TrieTokenizer to no longer throw StringIndexOutOfBoundsException in admin UI / AnalysisRequestHandler when you enter no number to tokenize

Modified:
    lucene/dev/branches/branch_4x/   (props changed)
    lucene/dev/branches/branch_4x/solr/   (props changed)
    lucene/dev/branches/branch_4x/solr/CHANGES.txt   (contents, props changed)
    lucene/dev/branches/branch_4x/solr/core/   (props changed)
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java

Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1429402&r1=1429401&r2=1429402&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Sat Jan  5 22:20:28 2013
@@ -443,6 +443,10 @@ Bug Fixes
 * SOLR-3829: Admin UI Logging events broken if schema.xml defines a catch-all
   dynamicField with type ignored (steffkes)
 
+* SOLR-4275: Fix TrieTokenizer to no longer throw StringIndexOutOfBoundsException
+  in admin UI / AnalysisRequestHandler when you enter no number to tokenize.
+  (Uwe Schindler)
+
 Other Changes
 ----------------------
 

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java?rev=1429402&r1=1429401&r2=1429402&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java Sat Jan  5 22:20:28 2013
@@ -50,39 +50,54 @@ public class TrieTokenizerFactory extend
 
   @Override
   public TrieTokenizer create(Reader input) {
-    return new TrieTokenizer(input, type, precisionStep, TrieTokenizer.getNumericTokenStream(precisionStep));
+    return new TrieTokenizer(input, type, TrieTokenizer.getNumericTokenStream(precisionStep));
   }
 }
 
 final class TrieTokenizer extends Tokenizer {
   protected static final DateField dateField = new DateField();
-  protected final int precisionStep;
   protected final TrieTypes type;
   protected final NumericTokenStream ts;
   
   protected final OffsetAttribute ofsAtt = addAttribute(OffsetAttribute.class);
   protected int startOfs, endOfs;
+  protected boolean hasValue;
+  protected final char[] buf = new char[32];
 
   static NumericTokenStream getNumericTokenStream(int precisionStep) {
     return new NumericTokenStream(precisionStep);
   }
 
-  public TrieTokenizer(Reader input, TrieTypes type, int precisionStep, NumericTokenStream ts) {
+  public TrieTokenizer(Reader input, TrieTypes type, NumericTokenStream ts) {
     // must share the attribute source with the NumericTokenStream we delegate to
     super(ts, input);
     this.type = type;
-    this.precisionStep = precisionStep;
     this.ts = ts;
   }
 
   @Override
   public void reset() {
    try {
-      char[] buf = new char[32];
-      int len = input.read(buf);
+      int upto = 0;
+      while (upto < buf.length) {
+        final int length = input.read(buf, upto, buf.length - upto);
+        if (length == -1) break;
+        upto += length;
+      }
+      // skip remaining data if buffer was too short:
+      if (upto == buf.length) {
+        input.skip(Long.MAX_VALUE);
+      }
+
       this.startOfs = correctOffset(0);
-      this.endOfs = correctOffset(len);
-      String v = new String(buf, 0, len);
+      this.endOfs = correctOffset(upto);
+      
+      if (upto == 0) {
+        hasValue = false;
+        return;
+      }
+
+      final String v = new String(buf, 0, upto);
       try {
         switch (type) {
           case INTEGER:
@@ -107,21 +122,24 @@ final class TrieTokenizer extends Tokeni
         throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, 
                                 "Invalid Number: " + v);
       }
+      hasValue = true;
+      ts.reset();
     } catch (IOException e) {
       throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unable to create TrieIndexTokenizer", e);
     }
-    ts.reset();
   }
 
   @Override
   public void close() throws IOException {
     super.close();
-    ts.close();
+    if (hasValue) {
+      ts.close();
+    }
   }
 
   @Override
   public boolean incrementToken() {
-    if (ts.incrementToken()) {
+    if (hasValue && ts.incrementToken()) {
       ofsAtt.setOffset(startOfs, endOfs);
       return true;
     }
@@ -130,7 +148,9 @@ final class TrieTokenizer extends Tokeni
 
   @Override
   public void end() throws IOException {
-    ts.end();
+    if (hasValue) {
+      ts.end();
+    }
     ofsAtt.setOffset(endOfs, endOfs);
   }
 }