svn commit: r596440 - in /lucene/java/trunk/src: java/org/apache/lucene/analysis/TokenStream.java test/org/apache/lucene/analysis/TestAnalyzers.java

[yo...@apache.org]

Author: yonik
Date: Mon Nov 19 12:45:09 2007
New Revision: 596440

URL: http://svn.apache.org/viewvc?rev=596440&view=rev
Log:
LUCENE-1057: copy payload in Token.next()

Modified:
    lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenStream.java
    lucene/java/trunk/src/test/org/apache/lucene/analysis/TestAnalyzers.java

Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenStream.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenStream.java?rev=596440&r1=596439&r2=596440&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenStream.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenStream.java Mon Nov 19 12:45:09 2007
@@ -17,6 +17,8 @@
  * limitations under the License.
  */
 
+import org.apache.lucene.index.Payload;
+
 import java.io.IOException;
 
 /** A TokenStream enumerates the sequence of tokens, either from
@@ -41,6 +43,13 @@
    *  than calling {@link #next(Token)} instead.. */
   public Token next() throws IOException {
     Token result = next(new Token());
+
+    if (result != null) {
+      Payload p = result.getPayload();
+      if (p != null)
+        result.setPayload(new Payload(p.toByteArray(), 0, p.length()));
+    }
+
     return result;
   }
 

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestAnalyzers.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestAnalyzers.java?rev=596440&r1=596439&r2=596440&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestAnalyzers.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestAnalyzers.java Mon Nov 19 12:45:09 2007
@@ -18,7 +18,11 @@
  */
 
 import java.io.*;
+import java.util.List;
+import java.util.LinkedList;
+
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.index.Payload;
 
 public class TestAnalyzers extends LuceneTestCase {
 
@@ -86,5 +90,69 @@
     assertAnalyzesTo(a, "foo a bar such FOO THESE BAR", 
                      new String[] { "foo", "bar", "foo", "bar" });
   }
+
+  void verifyPayload(TokenStream ts) throws IOException {
+    Token t = new Token();
+    for(byte b=1;;b++) {
+      t.clear();
+      t = ts.next(t);
+      if (t==null) break;
+      // System.out.println("id="+System.identityHashCode(t) + " " + t);
+      // System.out.println("payload=" + (int)t.getPayload().toByteArray()[0]);
+      assertEquals(b, t.getPayload().toByteArray()[0]);
+    }
+  }
+
+  // Make sure old style next() calls result in a new copy of payloads
+  public void testPayloadCopy() throws IOException {
+    String s = "how now brown cow";
+    TokenStream ts;
+    ts = new WhitespaceTokenizer(new StringReader(s));
+    ts = new BuffTokenFilter(ts);
+    ts = new PayloadSetter(ts);
+    verifyPayload(ts);
+
+    ts = new WhitespaceTokenizer(new StringReader(s));
+    ts = new PayloadSetter(ts);
+    ts = new BuffTokenFilter(ts);
+    verifyPayload(ts);
+  }
+
+}
+
+class BuffTokenFilter extends TokenFilter {
+  List lst;
+
+  public BuffTokenFilter(TokenStream input) {
+    super(input);
+  }
+
+  public Token next() throws IOException {
+    if (lst == null) {
+      lst = new LinkedList<Token>();
+      for(;;) {
+        Token t = input.next();
+        if (t==null) break;
+        lst.add(t);
+      }
+    }
+    return lst.size()==0 ? null : (Token)lst.remove(0);
+  }
 }
 
+class PayloadSetter extends TokenFilter {
+  public  PayloadSetter(TokenStream input) {
+    super(input);
+  }
+
+  byte[] data = new byte[1];
+  Payload p = new Payload(data,0,1);
+
+  public Token next(Token target) throws IOException {
+    target = input.next(target);
+    if (target==null) return null;
+    target.setPayload(p);  // reuse the payload / byte[]
+    data[0]++;
+    return target;
+  }
+}
\ No newline at end of file