You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by mi...@apache.org on 2008/08/20 16:38:11 UTC
svn commit: r687357 [6/6] - in /lucene/java/trunk: ./
contrib/analyzers/src/java/org/apache/lucene/analysis/br/
contrib/analyzers/src/java/org/apache/lucene/analysis/cjk/
contrib/analyzers/src/java/org/apache/lucene/analysis/cn/
contrib/analyzers/src/j...
Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TeeSinkTokenTest.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TeeSinkTokenTest.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TeeSinkTokenTest.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TeeSinkTokenTest.java Wed Aug 20 07:38:07 2008
@@ -16,10 +16,10 @@
* limitations under the License.
*/
-import junit.framework.TestCase;
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.util.English;
+import org.apache.lucene.util.LuceneTestCase;
import java.io.IOException;
import java.io.StringReader;
@@ -29,7 +29,7 @@
/**
* tests for the TeeTokenFilter and SinkTokenizer
*/
-public class TeeSinkTokenTest extends TestCase {
+public class TeeSinkTokenTest extends LuceneTestCase {
protected StringBuffer buffer1;
protected StringBuffer buffer2;
protected String[] tokens1;
@@ -63,23 +63,23 @@
SinkTokenizer sink1 = new SinkTokenizer(null) {
public void add(Token t) {
- if (t != null && t.termText().equalsIgnoreCase("The")) {
+ if (t != null && t.term().equalsIgnoreCase("The")) {
super.add(t);
}
}
};
TokenStream source = new TeeTokenFilter(new WhitespaceTokenizer(new StringReader(buffer1.toString())), sink1);
- Token token = null;
int i = 0;
- while ((token = source.next()) != null) {
- assertTrue(token.termText() + " is not equal to " + tokens1[i], token.termText().equals(tokens1[i]) == true);
+ final Token reusableToken = new Token();
+ for (Token nextToken = source.next(reusableToken); nextToken != null; nextToken = source.next(reusableToken)) {
+ assertTrue(nextToken.term() + " is not equal to " + tokens1[i], nextToken.term().equals(tokens1[i]) == true);
i++;
}
assertTrue(i + " does not equal: " + tokens1.length, i == tokens1.length);
assertTrue("sink1 Size: " + sink1.getTokens().size() + " is not: " + 2, sink1.getTokens().size() == 2);
i = 0;
- while ((token = sink1.next()) != null) {
- assertTrue(token.termText() + " is not equal to " + "The", token.termText().equalsIgnoreCase("The") == true);
+ for (Token token = sink1.next(reusableToken); token != null; token = sink1.next(reusableToken)) {
+ assertTrue(token.term() + " is not equal to " + "The", token.term().equalsIgnoreCase("The") == true);
i++;
}
assertTrue(i + " does not equal: " + sink1.getTokens().size(), i == sink1.getTokens().size());
@@ -88,54 +88,54 @@
public void testMultipleSources() throws Exception {
SinkTokenizer theDetector = new SinkTokenizer(null) {
public void add(Token t) {
- if (t != null && t.termText().equalsIgnoreCase("The")) {
+ if (t != null && t.term().equalsIgnoreCase("The")) {
super.add(t);
}
}
};
SinkTokenizer dogDetector = new SinkTokenizer(null) {
public void add(Token t) {
- if (t != null && t.termText().equalsIgnoreCase("Dogs")) {
+ if (t != null && t.term().equalsIgnoreCase("Dogs")) {
super.add(t);
}
}
};
TokenStream source1 = new CachingTokenFilter(new TeeTokenFilter(new TeeTokenFilter(new WhitespaceTokenizer(new StringReader(buffer1.toString())), theDetector), dogDetector));
TokenStream source2 = new TeeTokenFilter(new TeeTokenFilter(new WhitespaceTokenizer(new StringReader(buffer2.toString())), theDetector), dogDetector);
- Token token = null;
int i = 0;
- while ((token = source1.next()) != null) {
- assertTrue(token.termText() + " is not equal to " + tokens1[i], token.termText().equals(tokens1[i]) == true);
+ final Token reusableToken = new Token();
+ for (Token nextToken = source1.next(reusableToken); nextToken != null; nextToken = source1.next(reusableToken)) {
+ assertTrue(nextToken.term() + " is not equal to " + tokens1[i], nextToken.term().equals(tokens1[i]) == true);
i++;
}
assertTrue(i + " does not equal: " + tokens1.length, i == tokens1.length);
assertTrue("theDetector Size: " + theDetector.getTokens().size() + " is not: " + 2, theDetector.getTokens().size() == 2);
assertTrue("dogDetector Size: " + dogDetector.getTokens().size() + " is not: " + 1, dogDetector.getTokens().size() == 1);
i = 0;
- while ((token = source2.next()) != null) {
- assertTrue(token.termText() + " is not equal to " + tokens2[i], token.termText().equals(tokens2[i]) == true);
+ for (Token nextToken = source2.next(reusableToken); nextToken != null; nextToken = source2.next(reusableToken)) {
+ assertTrue(nextToken.term() + " is not equal to " + tokens2[i], nextToken.term().equals(tokens2[i]) == true);
i++;
}
assertTrue(i + " does not equal: " + tokens2.length, i == tokens2.length);
assertTrue("theDetector Size: " + theDetector.getTokens().size() + " is not: " + 4, theDetector.getTokens().size() == 4);
assertTrue("dogDetector Size: " + dogDetector.getTokens().size() + " is not: " + 2, dogDetector.getTokens().size() == 2);
i = 0;
- while ((token = theDetector.next()) != null) {
- assertTrue(token.termText() + " is not equal to " + "The", token.termText().equalsIgnoreCase("The") == true);
+ for (Token nextToken = theDetector.next(reusableToken); nextToken != null; nextToken = theDetector.next(reusableToken)) {
+ assertTrue(nextToken.term() + " is not equal to " + "The", nextToken.term().equalsIgnoreCase("The") == true);
i++;
}
assertTrue(i + " does not equal: " + theDetector.getTokens().size(), i == theDetector.getTokens().size());
i = 0;
- while ((token = dogDetector.next()) != null) {
- assertTrue(token.termText() + " is not equal to " + "Dogs", token.termText().equalsIgnoreCase("Dogs") == true);
+ for (Token nextToken = dogDetector.next(reusableToken); nextToken != null; nextToken = dogDetector.next(reusableToken)) {
+ assertTrue(nextToken.term() + " is not equal to " + "Dogs", nextToken.term().equalsIgnoreCase("Dogs") == true);
i++;
}
assertTrue(i + " does not equal: " + dogDetector.getTokens().size(), i == dogDetector.getTokens().size());
source1.reset();
TokenStream lowerCasing = new LowerCaseFilter(source1);
i = 0;
- while ((token = lowerCasing.next()) != null) {
- assertTrue(token.termText() + " is not equal to " + tokens1[i].toLowerCase(), token.termText().equals(tokens1[i].toLowerCase()) == true);
+ for (Token nextToken = lowerCasing.next(reusableToken); nextToken != null; nextToken = lowerCasing.next(reusableToken)) {
+ assertTrue(nextToken.term() + " is not equal to " + tokens1[i].toLowerCase(), nextToken.term().equals(tokens1[i].toLowerCase()) == true);
i++;
}
assertTrue(i + " does not equal: " + tokens1.length, i == tokens1.length);
@@ -157,22 +157,21 @@
}
//make sure we produce the same tokens
ModuloSinkTokenizer sink = new ModuloSinkTokenizer(tokCount[k], 100);
- Token next = new Token();
- TokenStream result = new TeeTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))), sink);
- while ((next = result.next(next)) != null) {
+ final Token reusableToken = new Token();
+ TokenStream stream = new TeeTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))), sink);
+ while (stream.next(reusableToken) != null) {
}
- result = new ModuloTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))), 100);
- next = new Token();
+ stream = new ModuloTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))), 100);
List tmp = new ArrayList();
- while ((next = result.next(next)) != null) {
- tmp.add(next.clone());
+ for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+ tmp.add(nextToken.clone());
}
List sinkList = sink.getTokens();
assertTrue("tmp Size: " + tmp.size() + " is not: " + sinkList.size(), tmp.size() == sinkList.size());
for (int i = 0; i < tmp.size(); i++) {
Token tfTok = (Token) tmp.get(i);
Token sinkTok = (Token) sinkList.get(i);
- assertTrue(tfTok.termText() + " is not equal to " + sinkTok.termText() + " at token: " + i, tfTok.termText().equals(sinkTok.termText()) == true);
+ assertTrue(tfTok.term() + " is not equal to " + sinkTok.term() + " at token: " + i, tfTok.term().equals(sinkTok.term()) == true);
}
//simulate two fields, each being analyzed once, for 20 documents
@@ -180,15 +179,13 @@
int tfPos = 0;
long start = System.currentTimeMillis();
for (int i = 0; i < 20; i++) {
- next = new Token();
- result = new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString())));
- while ((next = result.next(next)) != null) {
- tfPos += next.getPositionIncrement();
+ stream = new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString())));
+ for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+ tfPos += nextToken.getPositionIncrement();
}
- next = new Token();
- result = new ModuloTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))), modCounts[j]);
- while ((next = result.next(next)) != null) {
- tfPos += next.getPositionIncrement();
+ stream = new ModuloTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))), modCounts[j]);
+ for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+ tfPos += nextToken.getPositionIncrement();
}
}
long finish = System.currentTimeMillis();
@@ -198,15 +195,14 @@
start = System.currentTimeMillis();
for (int i = 0; i < 20; i++) {
sink = new ModuloSinkTokenizer(tokCount[k], modCounts[j]);
- next = new Token();
- result = new TeeTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))), sink);
- while ((next = result.next(next)) != null) {
- sinkPos += next.getPositionIncrement();
+ stream = new TeeTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))), sink);
+ for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+ sinkPos += nextToken.getPositionIncrement();
}
//System.out.println("Modulo--------");
- result = sink;
- while ((next = result.next(next)) != null) {
- sinkPos += next.getPositionIncrement();
+ stream = sink;
+ for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+ sinkPos += nextToken.getPositionIncrement();
}
}
finish = System.currentTimeMillis();
@@ -232,13 +228,15 @@
int count = 0;
//return every 100 tokens
- public Token next(Token result) throws IOException {
-
- while ((result = input.next(result)) != null && count % modCount != 0) {
+ public Token next(final Token reusableToken) throws IOException {
+ Token nextToken = null;
+ for (nextToken = input.next(reusableToken);
+ nextToken != null && count % modCount != 0;
+ nextToken = input.next(reusableToken)) {
count++;
}
count++;
- return result;
+ return nextToken;
}
}
@@ -254,7 +252,7 @@
public void add(Token t) {
if (t != null && count % modCount == 0) {
- lst.add(t.clone());
+ super.add(t);
}
count++;
}
Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestAnalyzers.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestAnalyzers.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestAnalyzers.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestAnalyzers.java Wed Aug 20 07:38:07 2008
@@ -17,13 +17,14 @@
* limitations under the License.
*/
-import java.io.*;
-import java.util.List;
+import java.io.IOException;
+import java.io.StringReader;
import java.util.LinkedList;
+import java.util.List;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.index.Payload;
import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.index.Payload;
+import org.apache.lucene.util.LuceneTestCase;
public class TestAnalyzers extends LuceneTestCase {
@@ -35,12 +36,13 @@
String input,
String[] output) throws Exception {
TokenStream ts = a.tokenStream("dummy", new StringReader(input));
+ final Token reusableToken = new Token();
for (int i=0; i<output.length; i++) {
- Token t = ts.next();
- assertNotNull(t);
- assertEquals(t.termText(), output[i]);
+ Token nextToken = ts.next(reusableToken);
+ assertNotNull(nextToken);
+ assertEquals(nextToken.term(), output[i]);
}
- assertNull(ts.next());
+ assertNull(ts.next(reusableToken));
ts.close();
}
@@ -93,14 +95,14 @@
}
void verifyPayload(TokenStream ts) throws IOException {
- Token t = new Token();
+ final Token reusableToken = new Token();
for(byte b=1;;b++) {
- t.clear();
- t = ts.next(t);
- if (t==null) break;
- // System.out.println("id="+System.identityHashCode(t) + " " + t);
- // System.out.println("payload=" + (int)t.getPayload().toByteArray()[0]);
- assertEquals(b, t.getPayload().toByteArray()[0]);
+ reusableToken.clear();
+ Token nextToken = ts.next(reusableToken);
+ if (nextToken==null) break;
+ // System.out.println("id="+System.identityHashCode(nextToken) + " " + t);
+ // System.out.println("payload=" + (int)nextToken.getPayload().toByteArray()[0]);
+ assertEquals(b, nextToken.getPayload().toByteArray()[0]);
}
}
@@ -141,13 +143,11 @@
super(input);
}
- public Token next() throws IOException {
+ public Token next(final Token reusableToken) throws IOException {
if (lst == null) {
lst = new LinkedList();
- for(;;) {
- Token t = input.next();
- if (t==null) break;
- lst.add(t);
+ for(Token nextToken = input.next(reusableToken); nextToken != null; nextToken = input.next(reusableToken)) {
+ lst.add(nextToken.clone());
}
}
return lst.size()==0 ? null : (Token)lst.remove(0);
@@ -162,11 +162,12 @@
byte[] data = new byte[1];
Payload p = new Payload(data,0,1);
- public Token next(Token target) throws IOException {
- target = input.next(target);
- if (target==null) return null;
- target.setPayload(p); // reuse the payload / byte[]
+ public Token next(final Token reusableToken) throws IOException {
+ assert reusableToken != null;
+ Token nextToken = input.next(reusableToken);
+ if (nextToken==null) return null;
+ nextToken.setPayload(p); // reuse the payload / byte[]
data[0]++;
- return target;
+ return nextToken;
}
}
\ No newline at end of file
Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java Wed Aug 20 07:38:07 2008
@@ -42,11 +42,12 @@
TokenStream stream = new TokenStream() {
private int index = 0;
- public Token next() throws IOException {
+ public Token next(final Token reusableToken) throws IOException {
+ assert reusableToken != null;
if (index == tokens.length) {
return null;
} else {
- return new Token(tokens[index++], 0, 0);
+ return reusableToken.reinit(tokens[index++], 0, 0);
}
}
@@ -91,10 +92,10 @@
private void checkTokens(TokenStream stream) throws IOException {
int count = 0;
- Token token;
- while ((token = stream.next()) != null) {
+ final Token reusableToken = new Token();
+ for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
assertTrue(count < tokens.length);
- assertEquals(tokens[count], token.termText());
+ assertEquals(tokens[count], nextToken.term());
count++;
}
Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java Wed Aug 20 07:38:07 2008
@@ -25,81 +25,82 @@
public void testU() throws Exception {
TokenStream stream = new WhitespaceTokenizer(new StringReader("Des mot clés à LA CHAÃNE à à à à à Ã
à à à à à à à à à à IJ à à à à à à à à Šà à à à à à Ÿ à á â ã ä Ã¥ æ ç è é ê ë ì à î ï ij ð ñ ò ó ô õ ö ø Šà þ ù ú û ü ý ÿ ï¬ ï¬"));
ISOLatin1AccentFilter filter = new ISOLatin1AccentFilter(stream);
- assertEquals("Des", filter.next().termText());
- assertEquals("mot", filter.next().termText());
- assertEquals("cles", filter.next().termText());
- assertEquals("A", filter.next().termText());
- assertEquals("LA", filter.next().termText());
- assertEquals("CHAINE", filter.next().termText());
- assertEquals("A", filter.next().termText());
- assertEquals("A", filter.next().termText());
- assertEquals("A", filter.next().termText());
- assertEquals("A", filter.next().termText());
- assertEquals("A", filter.next().termText());
- assertEquals("A", filter.next().termText());
- assertEquals("AE", filter.next().termText());
- assertEquals("C", filter.next().termText());
- assertEquals("E", filter.next().termText());
- assertEquals("E", filter.next().termText());
- assertEquals("E", filter.next().termText());
- assertEquals("E", filter.next().termText());
- assertEquals("I", filter.next().termText());
- assertEquals("I", filter.next().termText());
- assertEquals("I", filter.next().termText());
- assertEquals("I", filter.next().termText());
- assertEquals("IJ", filter.next().termText());
- assertEquals("D", filter.next().termText());
- assertEquals("N", filter.next().termText());
- assertEquals("O", filter.next().termText());
- assertEquals("O", filter.next().termText());
- assertEquals("O", filter.next().termText());
- assertEquals("O", filter.next().termText());
- assertEquals("O", filter.next().termText());
- assertEquals("O", filter.next().termText());
- assertEquals("OE", filter.next().termText());
- assertEquals("TH", filter.next().termText());
- assertEquals("U", filter.next().termText());
- assertEquals("U", filter.next().termText());
- assertEquals("U", filter.next().termText());
- assertEquals("U", filter.next().termText());
- assertEquals("Y", filter.next().termText());
- assertEquals("Y", filter.next().termText());
- assertEquals("a", filter.next().termText());
- assertEquals("a", filter.next().termText());
- assertEquals("a", filter.next().termText());
- assertEquals("a", filter.next().termText());
- assertEquals("a", filter.next().termText());
- assertEquals("a", filter.next().termText());
- assertEquals("ae", filter.next().termText());
- assertEquals("c", filter.next().termText());
- assertEquals("e", filter.next().termText());
- assertEquals("e", filter.next().termText());
- assertEquals("e", filter.next().termText());
- assertEquals("e", filter.next().termText());
- assertEquals("i", filter.next().termText());
- assertEquals("i", filter.next().termText());
- assertEquals("i", filter.next().termText());
- assertEquals("i", filter.next().termText());
- assertEquals("ij", filter.next().termText());
- assertEquals("d", filter.next().termText());
- assertEquals("n", filter.next().termText());
- assertEquals("o", filter.next().termText());
- assertEquals("o", filter.next().termText());
- assertEquals("o", filter.next().termText());
- assertEquals("o", filter.next().termText());
- assertEquals("o", filter.next().termText());
- assertEquals("o", filter.next().termText());
- assertEquals("oe", filter.next().termText());
- assertEquals("ss", filter.next().termText());
- assertEquals("th", filter.next().termText());
- assertEquals("u", filter.next().termText());
- assertEquals("u", filter.next().termText());
- assertEquals("u", filter.next().termText());
- assertEquals("u", filter.next().termText());
- assertEquals("y", filter.next().termText());
- assertEquals("y", filter.next().termText());
- assertEquals("fi", filter.next().termText());
- assertEquals("fl", filter.next().termText());
- assertNull(filter.next());
+ final Token reusableToken = new Token();
+ assertEquals("Des", filter.next(reusableToken).term());
+ assertEquals("mot", filter.next(reusableToken).term());
+ assertEquals("cles", filter.next(reusableToken).term());
+ assertEquals("A", filter.next(reusableToken).term());
+ assertEquals("LA", filter.next(reusableToken).term());
+ assertEquals("CHAINE", filter.next(reusableToken).term());
+ assertEquals("A", filter.next(reusableToken).term());
+ assertEquals("A", filter.next(reusableToken).term());
+ assertEquals("A", filter.next(reusableToken).term());
+ assertEquals("A", filter.next(reusableToken).term());
+ assertEquals("A", filter.next(reusableToken).term());
+ assertEquals("A", filter.next(reusableToken).term());
+ assertEquals("AE", filter.next(reusableToken).term());
+ assertEquals("C", filter.next(reusableToken).term());
+ assertEquals("E", filter.next(reusableToken).term());
+ assertEquals("E", filter.next(reusableToken).term());
+ assertEquals("E", filter.next(reusableToken).term());
+ assertEquals("E", filter.next(reusableToken).term());
+ assertEquals("I", filter.next(reusableToken).term());
+ assertEquals("I", filter.next(reusableToken).term());
+ assertEquals("I", filter.next(reusableToken).term());
+ assertEquals("I", filter.next(reusableToken).term());
+ assertEquals("IJ", filter.next(reusableToken).term());
+ assertEquals("D", filter.next(reusableToken).term());
+ assertEquals("N", filter.next(reusableToken).term());
+ assertEquals("O", filter.next(reusableToken).term());
+ assertEquals("O", filter.next(reusableToken).term());
+ assertEquals("O", filter.next(reusableToken).term());
+ assertEquals("O", filter.next(reusableToken).term());
+ assertEquals("O", filter.next(reusableToken).term());
+ assertEquals("O", filter.next(reusableToken).term());
+ assertEquals("OE", filter.next(reusableToken).term());
+ assertEquals("TH", filter.next(reusableToken).term());
+ assertEquals("U", filter.next(reusableToken).term());
+ assertEquals("U", filter.next(reusableToken).term());
+ assertEquals("U", filter.next(reusableToken).term());
+ assertEquals("U", filter.next(reusableToken).term());
+ assertEquals("Y", filter.next(reusableToken).term());
+ assertEquals("Y", filter.next(reusableToken).term());
+ assertEquals("a", filter.next(reusableToken).term());
+ assertEquals("a", filter.next(reusableToken).term());
+ assertEquals("a", filter.next(reusableToken).term());
+ assertEquals("a", filter.next(reusableToken).term());
+ assertEquals("a", filter.next(reusableToken).term());
+ assertEquals("a", filter.next(reusableToken).term());
+ assertEquals("ae", filter.next(reusableToken).term());
+ assertEquals("c", filter.next(reusableToken).term());
+ assertEquals("e", filter.next(reusableToken).term());
+ assertEquals("e", filter.next(reusableToken).term());
+ assertEquals("e", filter.next(reusableToken).term());
+ assertEquals("e", filter.next(reusableToken).term());
+ assertEquals("i", filter.next(reusableToken).term());
+ assertEquals("i", filter.next(reusableToken).term());
+ assertEquals("i", filter.next(reusableToken).term());
+ assertEquals("i", filter.next(reusableToken).term());
+ assertEquals("ij", filter.next(reusableToken).term());
+ assertEquals("d", filter.next(reusableToken).term());
+ assertEquals("n", filter.next(reusableToken).term());
+ assertEquals("o", filter.next(reusableToken).term());
+ assertEquals("o", filter.next(reusableToken).term());
+ assertEquals("o", filter.next(reusableToken).term());
+ assertEquals("o", filter.next(reusableToken).term());
+ assertEquals("o", filter.next(reusableToken).term());
+ assertEquals("o", filter.next(reusableToken).term());
+ assertEquals("oe", filter.next(reusableToken).term());
+ assertEquals("ss", filter.next(reusableToken).term());
+ assertEquals("th", filter.next(reusableToken).term());
+ assertEquals("u", filter.next(reusableToken).term());
+ assertEquals("u", filter.next(reusableToken).term());
+ assertEquals("u", filter.next(reusableToken).term());
+ assertEquals("u", filter.next(reusableToken).term());
+ assertEquals("y", filter.next(reusableToken).term());
+ assertEquals("y", filter.next(reusableToken).term());
+ assertEquals("fi", filter.next(reusableToken).term());
+ assertEquals("fl", filter.next(reusableToken).term());
+ assertNull(filter.next(reusableToken));
}
}
Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestLengthFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestLengthFilter.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestLengthFilter.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestLengthFilter.java Wed Aug 20 07:38:07 2008
@@ -27,10 +27,11 @@
TokenStream stream = new WhitespaceTokenizer(
new StringReader("short toolong evenmuchlongertext a ab toolong foo"));
LengthFilter filter = new LengthFilter(stream, 2, 6);
- assertEquals("short", filter.next().termText());
- assertEquals("ab", filter.next().termText());
- assertEquals("foo", filter.next().termText());
- assertNull(filter.next());
+ final Token reusableToken = new Token();
+ assertEquals("short", filter.next(reusableToken).term());
+ assertEquals("ab", filter.next(reusableToken).term());
+ assertEquals("foo", filter.next(reusableToken).term());
+ assertNull(filter.next(reusableToken));
}
}
Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java Wed Aug 20 07:38:07 2008
@@ -29,16 +29,17 @@
TokenStream tokenStream = analyzer.tokenStream("field",
new StringReader(text));
- Token token = tokenStream.next();
+ final Token reusableToken = new Token();
+ Token nextToken = tokenStream.next(reusableToken);
assertEquals("WhitespaceAnalyzer does not lowercase",
"Qwerty",
- token.termText());
+ nextToken.term());
tokenStream = analyzer.tokenStream("special",
new StringReader(text));
- token = tokenStream.next();
+ nextToken = tokenStream.next(reusableToken);
assertEquals("SimpleAnalyzer lowercases",
"qwerty",
- token.termText());
+ nextToken.term());
}
}
Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java Wed Aug 20 07:38:07 2008
@@ -35,18 +35,19 @@
public void assertAnalyzesTo(Analyzer a, String input, String[] expectedImages, String[] expectedTypes, int[] expectedPosIncrs) throws Exception {
TokenStream ts = a.tokenStream("dummy", new StringReader(input));
+ final Token reusableToken = new Token();
for (int i = 0; i < expectedImages.length; i++) {
- Token t = ts.next();
- assertNotNull(t);
- assertEquals(expectedImages[i], t.termText());
+ Token nextToken = ts.next(reusableToken);
+ assertNotNull(nextToken);
+ assertEquals(expectedImages[i], nextToken.term());
if (expectedTypes != null) {
- assertEquals(expectedTypes[i], t.type());
+ assertEquals(expectedTypes[i], nextToken.type());
}
if (expectedPosIncrs != null) {
- assertEquals(expectedPosIncrs[i], t.getPositionIncrement());
+ assertEquals(expectedPosIncrs[i], nextToken.getPositionIncrement());
}
}
- assertNull(ts.next());
+ assertNull(ts.next(reusableToken));
ts.close();
}
Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java Wed Aug 20 07:38:07 2008
@@ -45,9 +45,9 @@
StringReader reader = new StringReader("This is a test of the english stop analyzer");
TokenStream stream = stop.tokenStream("test", reader);
assertTrue(stream != null);
- Token token = null;
- while ((token = stream.next()) != null) {
- assertFalse(inValidTokens.contains(token.termText()));
+ final Token reusableToken = new Token();
+ for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+ assertFalse(inValidTokens.contains(nextToken.term()));
}
}
@@ -60,11 +60,11 @@
StringReader reader = new StringReader("This is a good test of the english stop analyzer");
TokenStream stream = newStop.tokenStream("test", reader);
assertNotNull(stream);
- Token token = null;
- while ((token = stream.next()) != null) {
- String text = token.termText();
+ final Token reusableToken = new Token();
+ for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+ String text = nextToken.term();
assertFalse(stopWordsSet.contains(text));
- assertEquals(1,token.getPositionIncrement()); // by default stop tokenizer does not apply increments.
+ assertEquals(1,nextToken.getPositionIncrement()); // by default stop tokenizer does not apply increments.
}
}
@@ -81,12 +81,12 @@
int expectedIncr[] = { 1, 1, 1, 3, 1, 1, 1, 2, 1};
TokenStream stream = newStop.tokenStream("test", reader);
assertNotNull(stream);
- Token token = null;
int i = 0;
- while ((token = stream.next()) != null) {
- String text = token.termText();
+ final Token reusableToken = new Token();
+ for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+ String text = nextToken.term();
assertFalse(stopWordsSet.contains(text));
- assertEquals(expectedIncr[i++],token.getPositionIncrement());
+ assertEquals(expectedIncr[i++],nextToken.getPositionIncrement());
}
} finally {
StopFilter.setEnablePositionIncrementsDefault(defaultEnable);
Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStopFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStopFilter.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStopFilter.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStopFilter.java Wed Aug 20 07:38:07 2008
@@ -37,17 +37,19 @@
StringReader reader = new StringReader("Now is The Time");
String[] stopWords = new String[] { "is", "the", "Time" };
TokenStream stream = new StopFilter(new WhitespaceTokenizer(reader), stopWords);
- assertEquals("Now", stream.next().termText());
- assertEquals("The", stream.next().termText());
- assertEquals(null, stream.next());
+ final Token reusableToken = new Token();
+ assertEquals("Now", stream.next(reusableToken).term());
+ assertEquals("The", stream.next(reusableToken).term());
+ assertEquals(null, stream.next(reusableToken));
}
public void testIgnoreCase() throws IOException {
StringReader reader = new StringReader("Now is The Time");
String[] stopWords = new String[] { "is", "the", "Time" };
TokenStream stream = new StopFilter(new WhitespaceTokenizer(reader), stopWords, true);
- assertEquals("Now", stream.next().termText());
- assertEquals(null,stream.next());
+ final Token reusableToken = new Token();
+ assertEquals("Now", stream.next(reusableToken).term());
+ assertEquals(null,stream.next(reusableToken));
}
public void testStopFilt() throws IOException {
@@ -55,9 +57,10 @@
String[] stopWords = new String[] { "is", "the", "Time" };
Set stopSet = StopFilter.makeStopSet(stopWords);
TokenStream stream = new StopFilter(new WhitespaceTokenizer(reader), stopSet);
- assertEquals("Now", stream.next().termText());
- assertEquals("The", stream.next().termText());
- assertEquals(null, stream.next());
+ final Token reusableToken = new Token();
+ assertEquals("Now", stream.next(reusableToken).term());
+ assertEquals("The", stream.next(reusableToken).term());
+ assertEquals(null, stream.next(reusableToken));
}
/**
@@ -109,14 +112,15 @@
private void doTestStopPositons(StopFilter stpf, boolean enableIcrements) throws IOException {
log("---> test with enable-increments-"+(enableIcrements?"enabled":"disabled"));
stpf.setEnablePositionIncrements(enableIcrements);
+ final Token reusableToken = new Token();
for (int i=0; i<20; i+=3) {
- Token t = stpf.next();
- log("Token "+i+": "+t);
+ Token nextToken = stpf.next(reusableToken);
+ log("Token "+i+": "+nextToken);
String w = English.intToEnglish(i).trim();
- assertEquals("expecting token "+i+" to be "+w,w,t.termText());
- assertEquals("all but first token must have position increment of 3",enableIcrements?(i==0?1:3):1,t.getPositionIncrement());
+ assertEquals("expecting token "+i+" to be "+w,w,nextToken.term());
+ assertEquals("all but first token must have position increment of 3",enableIcrements?(i==0?1:3):1,nextToken.getPositionIncrement());
}
- assertNull(stpf.next());
+ assertNull(stpf.next(reusableToken));
}
// print debug info depending on VERBOSE
Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestToken.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestToken.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestToken.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestToken.java Wed Aug 20 07:38:07 2008
@@ -17,7 +17,6 @@
* limitations under the License.
*/
-import java.io.*;
import org.apache.lucene.util.LuceneTestCase;
public class TestToken extends LuceneTestCase {
@@ -26,6 +25,119 @@
super(name);
}
+ public void testCtor() throws Exception {
+ Token t = new Token();
+ char[] content = "hello".toCharArray();
+ t.setTermBuffer(content, 0, content.length);
+ char[] buf = t.termBuffer();
+ assertNotSame(t.termBuffer(), content);
+ assertEquals("hello", t.term());
+ assertEquals("word", t.type());
+ assertEquals(0, t.getFlags());
+
+ t = new Token(6, 22);
+ t.setTermBuffer(content, 0, content.length);
+ assertEquals("hello", t.term());
+ assertEquals("(hello,6,22)", t.toString());
+ assertEquals("word", t.type());
+ assertEquals(0, t.getFlags());
+
+ t = new Token(6, 22, 7);
+ t.setTermBuffer(content, 0, content.length);
+ assertEquals("hello", t.term());
+ assertEquals("(hello,6,22)", t.toString());
+ assertEquals(7, t.getFlags());
+
+ t = new Token(6, 22, "junk");
+ t.setTermBuffer(content, 0, content.length);
+ assertEquals("hello", t.term());
+ assertEquals("(hello,6,22,type=junk)", t.toString());
+ assertEquals(0, t.getFlags());
+ }
+
+ public void testResize() {
+ Token t = new Token();
+ char[] content = "hello".toCharArray();
+ t.setTermBuffer(content, 0, content.length);
+ for (int i = 0; i < 2000; i++)
+ {
+ t.resizeTermBuffer(i);
+ assertTrue(i <= t.termBuffer().length);
+ assertEquals("hello", t.term());
+ }
+ }
+
+ public void testGrow() {
+ Token t = new Token();
+ StringBuffer buf = new StringBuffer("ab");
+ for (int i = 0; i < 20; i++)
+ {
+ char[] content = buf.toString().toCharArray();
+ t.setTermBuffer(content, 0, content.length);
+ assertEquals(buf.length(), t.termLength());
+ assertEquals(buf.toString(), t.term());
+ buf.append(buf.toString());
+ }
+ assertEquals(1048576, t.termLength());
+ assertEquals(1179654, t.termBuffer().length);
+
+ // now as a string, first variant
+ t = new Token();
+ buf = new StringBuffer("ab");
+ for (int i = 0; i < 20; i++)
+ {
+ String content = buf.toString();
+ t.setTermBuffer(content, 0, content.length());
+ assertEquals(content.length(), t.termLength());
+ assertEquals(content, t.term());
+ buf.append(content);
+ }
+ assertEquals(1048576, t.termLength());
+ assertEquals(1179654, t.termBuffer().length);
+
+ // now as a string, second variant
+ t = new Token();
+ buf = new StringBuffer("ab");
+ for (int i = 0; i < 20; i++)
+ {
+ String content = buf.toString();
+ t.setTermBuffer(content);
+ assertEquals(content.length(), t.termLength());
+ assertEquals(content, t.term());
+ buf.append(content);
+ }
+ assertEquals(1048576, t.termLength());
+ assertEquals(1179654, t.termBuffer().length);
+
+ // Test for slow growth to a long term
+ t = new Token();
+ buf = new StringBuffer("a");
+ for (int i = 0; i < 20000; i++)
+ {
+ String content = buf.toString();
+ t.setTermBuffer(content);
+ assertEquals(content.length(), t.termLength());
+ assertEquals(content, t.term());
+ buf.append("a");
+ }
+ assertEquals(20000, t.termLength());
+ assertEquals(20331, t.termBuffer().length);
+
+ // Test for slow growth to a long term
+ t = new Token();
+ buf = new StringBuffer("a");
+ for (int i = 0; i < 20000; i++)
+ {
+ String content = buf.toString();
+ t.setTermBuffer(content);
+ assertEquals(content.length(), t.termLength());
+ assertEquals(content, t.term());
+ buf.append("a");
+ }
+ assertEquals(20000, t.termLength());
+ assertEquals(20331, t.termBuffer().length);
+ }
+
public void testToString() throws Exception {
char[] b = {'a', 'l', 'o', 'h', 'a'};
Token t = new Token("", 0, 5);
@@ -40,10 +152,10 @@
Token t = new Token("hello", 0, 5);
assertEquals(t.termText(), "hello");
assertEquals(t.termLength(), 5);
- assertEquals(new String(t.termBuffer(), 0, 5), "hello");
+ assertEquals(t.term(), "hello");
t.setTermText("hello2");
assertEquals(t.termLength(), 6);
- assertEquals(new String(t.termBuffer(), 0, 6), "hello2");
+ assertEquals(t.term(), "hello2");
t.setTermBuffer("hello3".toCharArray(), 0, 6);
assertEquals(t.termText(), "hello3");
@@ -53,4 +165,13 @@
buffer[1] = 'o';
assertEquals(t.termText(), "hollo3");
}
+
+ public void testClone() throws Exception {
+ Token t = new Token(0, 5);
+ char[] content = "hello".toCharArray();
+ t.setTermBuffer(content, 0, 5);
+ char[] buf = t.termBuffer();
+ Token copy = (Token) t.clone();
+ assertNotSame(buf, copy.termBuffer());
+ }
}
Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestDocumentWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestDocumentWriter.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestDocumentWriter.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestDocumentWriter.java Wed Aug 20 07:38:07 2008
@@ -17,21 +17,27 @@
* limitations under the License.
*/
-import org.apache.lucene.analysis.*;
+import java.io.IOException;
+import java.io.Reader;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.SimpleAnalyzer;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.WhitespaceAnalyzer;
+import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Fieldable;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.Field.TermVector;
-import org.apache.lucene.document.Fieldable;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
-import java.io.IOException;
-import java.io.Reader;
-
public class TestDocumentWriter extends LuceneTestCase {
private RAMDirectory dir;
@@ -134,34 +140,30 @@
boolean first=true;
Token buffered;
- public Token next() throws IOException {
- return input.next();
- }
-
- public Token next(Token result) throws IOException {
+ public Token next(final Token reusableToken) throws IOException {
if (buffered != null) {
- Token t = buffered;
+ Token nextToken = buffered;
buffered=null;
- return t;
+ return nextToken;
}
- Token t = input.next(result);
- if (t==null) return null;
- if (Character.isDigit(t.termBuffer()[0])) {
- t.setPositionIncrement(t.termBuffer()[0] - '0');
+ Token nextToken = input.next(reusableToken);
+ if (nextToken==null) return null;
+ if (Character.isDigit(nextToken.termBuffer()[0])) {
+ nextToken.setPositionIncrement(nextToken.termBuffer()[0] - '0');
}
if (first) {
// set payload on first position only
- t.setPayload(new Payload(new byte[]{100}));
+ nextToken.setPayload(new Payload(new byte[]{100}));
first = false;
}
// index a "synonym" for every token
- buffered = (Token)t.clone();
+ buffered = (Token)nextToken.clone();
buffered.setPayload(null);
buffered.setPositionIncrement(0);
buffered.setTermBuffer(new char[]{'b'}, 0, 1);
- return t;
+ return nextToken;
}
};
}
@@ -199,11 +201,12 @@
private String[] tokens = new String[] {"term1", "term2", "term3", "term2"};
private int index = 0;
- public Token next() throws IOException {
+ public Token next(final Token reusableToken) throws IOException {
+ assert reusableToken != null;
if (index == tokens.length) {
return null;
} else {
- return new Token(tokens[index++], 0, 0);
+ return reusableToken.reinit(tokens[index++], 0, 0);
}
}
Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java Wed Aug 20 07:38:07 2008
@@ -1786,11 +1786,11 @@
return new TokenFilter(new StandardTokenizer(reader)) {
private int count = 0;
- public Token next() throws IOException {
+ public Token next(final Token reusableToken) throws IOException {
if (count++ == 5) {
throw new IOException();
}
- return input.next();
+ return input.next(reusableToken);
}
};
}
@@ -1909,10 +1909,10 @@
this.fieldName = fieldName;
}
- public Token next(Token result) throws IOException {
+ public Token next(final Token reusableToken) throws IOException {
if (this.fieldName.equals("crash") && count++ >= 4)
throw new IOException("I'm experiencing problems");
- return input.next(result);
+ return input.next(reusableToken);
}
public void reset() throws IOException {
@@ -3574,13 +3574,13 @@
public void testNegativePositions() throws Throwable {
SinkTokenizer tokens = new SinkTokenizer();
Token t = new Token();
- t.setTermText("a");
+ t.setTermBuffer("a");
t.setPositionIncrement(0);
tokens.add(t);
- t.setTermText("b");
+ t.setTermBuffer("b");
t.setPositionIncrement(1);
tokens.add(t);
- t.setTermText("c");
+ t.setTermBuffer("c");
tokens.add(t);
MockRAMDirectory dir = new MockRAMDirectory();
Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java Wed Aug 20 07:38:07 2008
@@ -103,12 +103,13 @@
super(input);
}
- public Token next() throws IOException {
- Token t = input.next();
- if (t != null) {
- t.setPayload(new Payload(new byte[] { (byte) count++ }));
+ public Token next(final Token reusableToken) throws IOException {
+ assert reusableToken != null;
+ Token nextToken = input.next(reusableToken);
+ if (nextToken != null) {
+ nextToken.setPayload(new Payload(new byte[] { (byte) count++ }));
}
- return t;
+ return nextToken;
}
}
Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestPayloads.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestPayloads.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestPayloads.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestPayloads.java Wed Aug 20 07:38:07 2008
@@ -450,23 +450,24 @@
this.offset = offset;
}
- public Token next(Token token) throws IOException {
- token = input.next(token);
- if (token != null) {
+ public Token next(final Token reusableToken) throws IOException {
+ assert reusableToken != null;
+ Token nextToken = input.next(reusableToken);
+ if (nextToken != null) {
if (offset + length <= data.length) {
Payload p = null;
if (p == null) {
p = new Payload();
- token.setPayload(p);
+ nextToken.setPayload(p);
}
p.setData(data, offset, length);
offset += length;
} else {
- token.setPayload(null);
+ nextToken.setPayload(null);
}
}
- return token;
+ return nextToken;
}
}
@@ -536,11 +537,11 @@
first = true;
}
- public Token next() throws IOException {
- if (!first) return null;
- Token t = new Token(term, 0, 0);
- t.setPayload(new Payload(payload));
- return t;
+ public Token next(final Token reusableToken) throws IOException {
+ if (!first) return null;
+ reusableToken.reinit(term, 0, 0);
+ reusableToken.setPayload(new Payload(payload));
+ return reusableToken;
}
public void close() throws IOException {
Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestTermVectorsReader.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestTermVectorsReader.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestTermVectorsReader.java Wed Aug 20 07:38:07 2008
@@ -118,20 +118,17 @@
private class MyTokenStream extends TokenStream {
int tokenUpto;
- public Token next() {
+ public Token next(final Token reusableToken) {
if (tokenUpto >= tokens.length)
return null;
else {
- final Token t = new Token();
final TestToken testToken = tokens[tokenUpto++];
- t.setTermText(testToken.text);
+ reusableToken.reinit(testToken.text, testToken.startOffset, testToken.endOffset);
if (tokenUpto > 1)
- t.setPositionIncrement(testToken.pos - tokens[tokenUpto-2].pos);
+ reusableToken.setPositionIncrement(testToken.pos - tokens[tokenUpto-2].pos);
else
- t.setPositionIncrement(testToken.pos+1);
- t.setStartOffset(testToken.startOffset);
- t.setEndOffset(testToken.endOffset);
- return t;
+ reusableToken.setPositionIncrement(testToken.pos+1);
+ return reusableToken;
}
}
}
Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestTermdocPerf.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestTermdocPerf.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestTermdocPerf.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestTermdocPerf.java Wed Aug 20 07:38:07 2008
@@ -40,11 +40,12 @@
Token t;
public RepeatingTokenStream(String val) {
- t = new Token(val,0,val.length());
+ t = new Token(0,val.length());
+ t.setTermBuffer(val);
}
- public Token next() throws IOException {
- return --num<0 ? null : t;
+ public Token next(final Token reusableToken) throws IOException {
+ return --num<0 ? null : (Token) t.clone();
}
}
Modified: lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java Wed Aug 20 07:38:07 2008
@@ -141,34 +141,34 @@
private final class TestFilter extends TokenFilter {
- private org.apache.lucene.analysis.Token prevToken;
+ private Token prevToken;
public TestFilter(TokenStream in) {
super(in);
}
- public final org.apache.lucene.analysis.Token next() throws java.io.IOException {
+ public final Token next(final Token reusableToken) throws java.io.IOException {
if (multiToken > 0) {
- org.apache.lucene.analysis.Token token =
- new org.apache.lucene.analysis.Token("multi"+(multiToken+1), prevToken.startOffset(),
- prevToken.endOffset(), prevToken.type());
- token.setPositionIncrement(0);
+ reusableToken.reinit("multi"+(multiToken+1), prevToken.startOffset(), prevToken.endOffset(), prevToken.type());
+ reusableToken.setPositionIncrement(0);
multiToken--;
- return token;
+ return reusableToken;
} else {
- org.apache.lucene.analysis.Token t = input.next();
- prevToken = t;
- if (t == null)
+ Token nextToken = input.next(reusableToken);
+ if (nextToken == null) {
+ prevToken = null;
return null;
- String text = t.termText();
+ }
+ prevToken = (Token) nextToken.clone();
+ String text = nextToken.term();
if (text.equals("triplemulti")) {
multiToken = 2;
- return t;
+ return nextToken;
} else if (text.equals("multi")) {
multiToken = 1;
- return t;
+ return nextToken;
} else {
- return t;
+ return nextToken;
}
}
}
@@ -197,22 +197,16 @@
super(in);
}
- public final org.apache.lucene.analysis.Token next() throws java.io.IOException {
- for (Token t = input.next(); t != null; t = input.next()) {
- if (t.termText().equals("the")) {
+ public final Token next(final Token reusableToken) throws java.io.IOException {
+ for (Token nextToken = input.next(reusableToken); nextToken != null; nextToken = input.next(reusableToken)) {
+ if (nextToken.term().equals("the")) {
// stopword, do nothing
- } else if (t.termText().equals("quick")) {
- org.apache.lucene.analysis.Token token =
- new org.apache.lucene.analysis.Token(t.termText(), t.startOffset(),
- t.endOffset(), t.type());
- token.setPositionIncrement(2);
- return token;
+ } else if (nextToken.term().equals("quick")) {
+ nextToken.setPositionIncrement(2);
+ return nextToken;
} else {
- org.apache.lucene.analysis.Token token =
- new org.apache.lucene.analysis.Token(t.termText(), t.startOffset(),
- t.endOffset(), t.type());
- token.setPositionIncrement(1);
- return token;
+ nextToken.setPositionIncrement(1);
+ return nextToken;
}
}
return null;
Modified: lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiFieldQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiFieldQueryParser.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiFieldQueryParser.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiFieldQueryParser.java Wed Aug 20 07:38:07 2008
@@ -319,7 +319,7 @@
}
private static class EmptyTokenStream extends TokenStream {
- public Token next() {
+ public Token next(final Token reusableToken) {
return null;
}
}
Modified: lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestQueryParser.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestQueryParser.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestQueryParser.java Wed Aug 20 07:38:07 2008
@@ -75,19 +75,20 @@
boolean inPhrase = false;
int savedStart = 0, savedEnd = 0;
- public Token next() throws IOException {
+ public Token next(final Token reusableToken) throws IOException {
+ assert reusableToken != null;
if (inPhrase) {
inPhrase = false;
- return new Token("phrase2", savedStart, savedEnd);
+ return reusableToken.reinit("phrase2", savedStart, savedEnd);
} else
- for (Token token = input.next(); token != null; token = input.next()) {
- if (token.termText().equals("phrase")) {
+ for (Token nextToken = input.next(reusableToken); nextToken != null; nextToken = input.next(reusableToken)) {
+ if (nextToken.term().equals("phrase")) {
inPhrase = true;
- savedStart = token.startOffset();
- savedEnd = token.endOffset();
- return new Token("phrase1", savedStart, savedEnd);
- } else if (!token.termText().equals("stop"))
- return token;
+ savedStart = nextToken.startOffset();
+ savedEnd = nextToken.endOffset();
+ return nextToken.reinit("phrase1", savedStart, savedEnd);
+ } else if (!nextToken.term().equals("stop"))
+ return nextToken;
}
return null;
}
Modified: lucene/java/trunk/src/test/org/apache/lucene/search/TestPositionIncrement.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/search/TestPositionIncrement.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/search/TestPositionIncrement.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/search/TestPositionIncrement.java Wed Aug 20 07:38:07 2008
@@ -49,13 +49,14 @@
private final int[] INCREMENTS = {1, 2, 1, 0, 1};
private int i = 0;
- public Token next() {
+ public Token next(final Token reusableToken) {
+ assert reusableToken != null;
if (i == TOKENS.length)
return null;
- Token t = new Token(TOKENS[i], i, i);
- t.setPositionIncrement(INCREMENTS[i]);
+ reusableToken.reinit(TOKENS[i], i, i);
+ reusableToken.setPositionIncrement(INCREMENTS[i]);
i++;
- return t;
+ return reusableToken;
}
};
}
@@ -204,11 +205,9 @@
Analyzer analyzer = new WhitespaceAnalyzer();
TokenStream ts = analyzer.tokenStream("field",
new StringReader("one two three four five"));
-
- while (true) {
- Token token = ts.next();
- if (token == null) break;
- assertEquals(token.termText(), 1, token.getPositionIncrement());
+ final Token reusableToken = new Token();
+ for (Token nextToken = ts.next(reusableToken); nextToken != null; nextToken = ts.next(reusableToken)) {
+ assertEquals(nextToken.term(), 1, nextToken.getPositionIncrement());
}
}
}
Modified: lucene/java/trunk/src/test/org/apache/lucene/search/payloads/TestBoostingTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/search/payloads/TestBoostingTermQuery.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/search/payloads/TestBoostingTermQuery.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/search/payloads/TestBoostingTermQuery.java Wed Aug 20 07:38:07 2008
@@ -16,21 +16,31 @@
* limitations under the License.
*/
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.analysis.*;
+import java.io.IOException;
+import java.io.Reader;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.LowerCaseTokenizer;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Payload;
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.*;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.CheckHits;
+import org.apache.lucene.search.DefaultSimilarity;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.spans.Spans;
import org.apache.lucene.search.spans.TermSpans;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.English;
-
-import java.io.IOException;
-import java.io.Reader;
+import org.apache.lucene.util.LuceneTestCase;
public class TestBoostingTermQuery extends LuceneTestCase {
private IndexSearcher searcher;
@@ -62,22 +72,23 @@
this.fieldName = fieldName;
}
- public Token next() throws IOException {
- Token result = input.next();
- if (result != null) {
+ public Token next(final Token reusableToken) throws IOException {
+ assert reusableToken != null;
+ Token nextToken = input.next(reusableToken);
+ if (nextToken != null) {
if (fieldName.equals("field")) {
- result.setPayload(new Payload(payloadField));
+ nextToken.setPayload(new Payload(payloadField));
} else if (fieldName.equals("multiField")) {
if (numSeen % 2 == 0) {
- result.setPayload(new Payload(payloadMultiField1));
+ nextToken.setPayload(new Payload(payloadMultiField1));
} else {
- result.setPayload(new Payload(payloadMultiField2));
+ nextToken.setPayload(new Payload(payloadMultiField2));
}
numSeen++;
}
}
- return result;
+ return nextToken;
}
}