svn commit: r605149 - in /lucene/java/trunk/src: java/org/apache/lucene/analysis/KeywordAnalyzer.java java/org/apache/lucene/analysis/KeywordTokenizer.java test/org/apache/lucene/analysis/TestKeywordAnalyzer.java

[mi...@apache.org]

Author: mikemccand
Date: Tue Dec 18 01:20:04 2007
New Revision: 605149

URL: http://svn.apache.org/viewvc?rev=605149&view=rev
Log:
LUCENE-1092: fix KeywordAnalyzer.reusableTokenStream so it can successfully be reused

Modified:
    lucene/java/trunk/src/java/org/apache/lucene/analysis/KeywordAnalyzer.java
    lucene/java/trunk/src/java/org/apache/lucene/analysis/KeywordTokenizer.java
    lucene/java/trunk/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java

Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/KeywordAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/KeywordAnalyzer.java?rev=605149&r1=605148&r2=605149&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/analysis/KeywordAnalyzer.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/analysis/KeywordAnalyzer.java Tue Dec 18 01:20:04 2007
@@ -17,6 +17,7 @@
  * limitations under the License.
  */
 
+import java.io.IOException;
 import java.io.Reader;
 
 /**
@@ -29,12 +30,13 @@
     return new KeywordTokenizer(reader);
   }
   public TokenStream reusableTokenStream(String fieldName,
-                                         final Reader reader) {
+                                         final Reader reader) throws IOException {
     Tokenizer tokenizer = (Tokenizer) getPreviousTokenStream();
     if (tokenizer == null) {
       tokenizer = new KeywordTokenizer(reader);
       setPreviousTokenStream(tokenizer);
-    }
+    } else
+      	tokenizer.reset(reader);
     return tokenizer;
   }
 }

Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/KeywordTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/KeywordTokenizer.java?rev=605149&r1=605148&r2=605149&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/analysis/KeywordTokenizer.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/analysis/KeywordTokenizer.java Tue Dec 18 01:20:04 2007
@@ -55,4 +55,9 @@
     }
     return null;
   }
+
+  public void reset(Reader input) throws IOException {
+    super.reset(input);
+    this.done = false;
+  }
 }

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java?rev=605149&r1=605148&r2=605149&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java Tue Dec 18 01:20:04 2007
@@ -18,7 +18,10 @@
  */
 
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermDocs;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
@@ -60,5 +63,23 @@
     assertEquals("Q36 kept as-is",
               "+partnum:Q36 +space", query.toString("description"));
     assertEquals("doc found!", 1, hits.length());
+  }
+
+  public void testMutipleDocument() throws Exception {
+    RAMDirectory dir = new RAMDirectory();
+    IndexWriter writer = new IndexWriter(dir,new KeywordAnalyzer(), true);
+    Document doc = new Document();
+    doc.add(new Field("partnum", "Q36", Field.Store.YES, Field.Index.TOKENIZED));
+    writer.addDocument(doc);
+    doc = new Document();
+    doc.add(new Field("partnum", "Q37", Field.Store.YES, Field.Index.TOKENIZED));
+    writer.addDocument(doc);
+    writer.close();
+
+    IndexReader reader = IndexReader.open(dir);
+    TermDocs td = reader.termDocs(new Term("partnum", "Q36"));
+    assertTrue(td.next());
+    td = reader.termDocs(new Term("partnum", "Q37"));
+    assertTrue(td.next());
   }
 }