You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by gs...@apache.org on 2007/12/20 16:08:00 UTC
svn commit: r605957 -
/lucene/java/trunk/src/test/org/apache/lucene/analysis/TeeSinkTokenTest.java
Author: gsingers
Date: Thu Dec 20 07:07:59 2007
New Revision: 605957
URL: http://svn.apache.org/viewvc?rev=605957&view=rev
Log:
Checking in simple performance test
Modified:
lucene/java/trunk/src/test/org/apache/lucene/analysis/TeeSinkTokenTest.java
Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TeeSinkTokenTest.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TeeSinkTokenTest.java?rev=605957&r1=605956&r2=605957&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TeeSinkTokenTest.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TeeSinkTokenTest.java Thu Dec 20 07:07:59 2007
@@ -17,9 +17,14 @@
*/
import junit.framework.TestCase;
+import org.apache.lucene.analysis.standard.StandardFilter;
+import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.util.English;
-import java.io.StringReader;
import java.io.IOException;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.List;
/**
* tests for the TeeTokenFilter and SinkTokenizer
@@ -56,9 +61,9 @@
public void test() throws IOException {
- SinkTokenizer sink1 = new SinkTokenizer(null){
+ SinkTokenizer sink1 = new SinkTokenizer(null) {
public void add(Token t) {
- if (t != null && t.termText().equalsIgnoreCase("The")){
+ if (t != null && t.termText().equalsIgnoreCase("The")) {
super.add(t);
}
}
@@ -66,14 +71,14 @@
TokenStream source = new TeeTokenFilter(new WhitespaceTokenizer(new StringReader(buffer1.toString())), sink1);
Token token = null;
int i = 0;
- while ((token = source.next()) != null){
+ while ((token = source.next()) != null) {
assertTrue(token.termText() + " is not equal to " + tokens1[i], token.termText().equals(tokens1[i]) == true);
i++;
}
assertTrue(i + " does not equal: " + tokens1.length, i == tokens1.length);
assertTrue("sink1 Size: " + sink1.getTokens().size() + " is not: " + 2, sink1.getTokens().size() == 2);
i = 0;
- while ((token = sink1.next()) != null){
+ while ((token = sink1.next()) != null) {
assertTrue(token.termText() + " is not equal to " + "The", token.termText().equalsIgnoreCase("The") == true);
i++;
}
@@ -81,16 +86,16 @@
}
public void testMultipleSources() throws Exception {
- SinkTokenizer theDetector = new SinkTokenizer(null){
+ SinkTokenizer theDetector = new SinkTokenizer(null) {
public void add(Token t) {
- if (t != null && t.termText().equalsIgnoreCase("The")){
+ if (t != null && t.termText().equalsIgnoreCase("The")) {
super.add(t);
}
}
};
- SinkTokenizer dogDetector = new SinkTokenizer(null){
+ SinkTokenizer dogDetector = new SinkTokenizer(null) {
public void add(Token t) {
- if (t != null && t.termText().equalsIgnoreCase("Dogs")){
+ if (t != null && t.termText().equalsIgnoreCase("Dogs")) {
super.add(t);
}
}
@@ -99,7 +104,7 @@
TokenStream source2 = new TeeTokenFilter(new TeeTokenFilter(new WhitespaceTokenizer(new StringReader(buffer2.toString())), theDetector), dogDetector);
Token token = null;
int i = 0;
- while ((token = source1.next()) != null){
+ while ((token = source1.next()) != null) {
assertTrue(token.termText() + " is not equal to " + tokens1[i], token.termText().equals(tokens1[i]) == true);
i++;
}
@@ -107,7 +112,7 @@
assertTrue("theDetector Size: " + theDetector.getTokens().size() + " is not: " + 2, theDetector.getTokens().size() == 2);
assertTrue("dogDetector Size: " + dogDetector.getTokens().size() + " is not: " + 1, dogDetector.getTokens().size() == 1);
i = 0;
- while ((token = source2.next()) != null){
+ while ((token = source2.next()) != null) {
assertTrue(token.termText() + " is not equal to " + tokens2[i], token.termText().equals(tokens2[i]) == true);
i++;
}
@@ -115,13 +120,13 @@
assertTrue("theDetector Size: " + theDetector.getTokens().size() + " is not: " + 4, theDetector.getTokens().size() == 4);
assertTrue("dogDetector Size: " + dogDetector.getTokens().size() + " is not: " + 2, dogDetector.getTokens().size() == 2);
i = 0;
- while ((token = theDetector.next()) != null){
+ while ((token = theDetector.next()) != null) {
assertTrue(token.termText() + " is not equal to " + "The", token.termText().equalsIgnoreCase("The") == true);
i++;
}
assertTrue(i + " does not equal: " + theDetector.getTokens().size(), i == theDetector.getTokens().size());
i = 0;
- while ((token = dogDetector.next()) != null){
+ while ((token = dogDetector.next()) != null) {
assertTrue(token.termText() + " is not equal to " + "Dogs", token.termText().equalsIgnoreCase("Dogs") == true);
i++;
}
@@ -129,10 +134,130 @@
source1.reset();
TokenStream lowerCasing = new LowerCaseFilter(source1);
i = 0;
- while ((token = lowerCasing.next()) != null){
+ while ((token = lowerCasing.next()) != null) {
assertTrue(token.termText() + " is not equal to " + tokens1[i].toLowerCase(), token.termText().equals(tokens1[i].toLowerCase()) == true);
i++;
}
assertTrue(i + " does not equal: " + tokens1.length, i == tokens1.length);
}
-}
\ No newline at end of file
+
+ /**
+ * Not an explicit test, just useful to print out some info on performance
+ *
+ * @throws Exception
+ */
+ public void testPerformance() throws Exception {
+ int[] tokCount = {100, 500, 1000, 2000, 5000, 10000};
+ int[] modCounts = {1, 2, 5, 10, 20, 50, 100, 200, 500};
+ for (int k = 0; k < tokCount.length; k++) {
+ StringBuffer buffer = new StringBuffer();
+ System.out.println("-----Tokens: " + tokCount[k] + "-----");
+ for (int i = 0; i < tokCount[k]; i++) {
+ buffer.append(English.intToEnglish(i).toUpperCase()).append(' ');
+ }
+ //make sure we produce the same tokens
+ ModuloSinkTokenizer sink = new ModuloSinkTokenizer(tokCount[k], 100);
+ Token next = new Token();
+ TokenStream result = new TeeTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))), sink);
+ while ((next = result.next(next)) != null) {
+ }
+ result = new ModuloTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))), 100);
+ next = new Token();
+ List tmp = new ArrayList();
+ while ((next = result.next(next)) != null) {
+ tmp.add(next.clone());
+ }
+ List sinkList = sink.getTokens();
+ assertTrue("tmp Size: " + tmp.size() + " is not: " + sinkList.size(), tmp.size() == sinkList.size());
+ for (int i = 0; i < tmp.size(); i++) {
+ Token tfTok = (Token) tmp.get(i);
+ Token sinkTok = (Token) sinkList.get(i);
+ assertTrue(tfTok.termText() + " is not equal to " + sinkTok.termText() + " at token: " + i, tfTok.termText().equals(sinkTok.termText()) == true);
+ }
+ //simulate two fields, each being analyzed once, for 20 documents
+
+ for (int j = 0; j < modCounts.length; j++) {
+ int tfPos = 0;
+ long start = System.currentTimeMillis();
+ for (int i = 0; i < 20; i++) {
+ next = new Token();
+ result = new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString())));
+ while ((next = result.next(next)) != null) {
+ tfPos += next.getPositionIncrement();
+ }
+ next = new Token();
+ result = new ModuloTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))), modCounts[j]);
+ while ((next = result.next(next)) != null) {
+ tfPos += next.getPositionIncrement();
+ }
+ }
+ long finish = System.currentTimeMillis();
+ System.out.println("ModCount: " + modCounts[j] + " Two fields took " + (finish - start) + " ms");
+ int sinkPos = 0;
+ //simulate one field with one sink
+ start = System.currentTimeMillis();
+ for (int i = 0; i < 20; i++) {
+ sink = new ModuloSinkTokenizer(tokCount[k], modCounts[j]);
+ next = new Token();
+ result = new TeeTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))), sink);
+ while ((next = result.next(next)) != null) {
+ sinkPos += next.getPositionIncrement();
+ }
+ //System.out.println("Modulo--------");
+ result = sink;
+ while ((next = result.next(next)) != null) {
+ sinkPos += next.getPositionIncrement();
+ }
+ }
+ finish = System.currentTimeMillis();
+ System.out.println("ModCount: " + modCounts[j] + " Tee fields took " + (finish - start) + " ms");
+ assertTrue(sinkPos + " does not equal: " + tfPos, sinkPos == tfPos);
+
+ }
+ System.out.println("- End Tokens: " + tokCount[k] + "-----");
+ }
+
+ }
+
+
+ class ModuloTokenFilter extends TokenFilter {
+
+ int modCount;
+
+ ModuloTokenFilter(TokenStream input, int mc) {
+ super(input);
+ modCount = mc;
+ }
+
+ int count = 0;
+
+ //return every 100 tokens
+ public Token next(Token result) throws IOException {
+
+ while ((result = input.next(result)) != null && count % modCount != 0) {
+ count++;
+ }
+ count++;
+ return result;
+ }
+ }
+
+ class ModuloSinkTokenizer extends SinkTokenizer {
+ int count = 0;
+ int modCount;
+
+
+ ModuloSinkTokenizer(int numToks, int mc) {
+ modCount = mc;
+ lst = new ArrayList(numToks % mc);
+ }
+
+ public void add(Token t) {
+ if (t != null && count % modCount == 0) {
+ lst.add(t.clone());
+ }
+ count++;
+ }
+ }
+}
+