You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by rm...@apache.org on 2009/08/16 14:37:07 UTC
svn commit: r804680 [2/2] - in /lucene/java/trunk/contrib: ./
analyzers/common/src/java/org/apache/lucene/analysis/ar/
analyzers/common/src/java/org/apache/lucene/analysis/br/
analyzers/common/src/java/org/apache/lucene/analysis/cjk/
analyzers/common/s...
Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java?rev=804680&r1=804679&r2=804680&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java Sun Aug 16 12:37:05 2009
@@ -17,12 +17,16 @@
* limitations under the License.
*/
+import java.io.Reader;
import java.io.StringReader;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.LetterTokenizer;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.WhitespaceTokenizer;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.document.Document;
@@ -200,4 +204,90 @@
int[] ranks = new int[] { 1, 2, 0 };
compareRanks(hits, ranks);
}
+
+ public void testReusableTokenStream() throws Exception {
+ Analyzer a = new ShingleAnalyzerWrapper(new WhitespaceAnalyzer(), 2);
+ assertAnalyzesToReuse(a, "please divide into shingles",
+ new String[] { "please", "please divide", "divide", "divide into", "into", "into shingles", "shingles" },
+ new int[] { 0, 0, 7, 7, 14, 14, 19 },
+ new int[] { 6, 13, 13, 18, 18, 27, 27 },
+ new int[] { 1, 0, 1, 0, 1, 0, 1 });
+ assertAnalyzesToReuse(a, "divide me up again",
+ new String[] { "divide", "divide me", "me", "me up", "up", "up again", "again" },
+ new int[] { 0, 0, 7, 7, 10, 10, 13 },
+ new int[] { 6, 9, 9, 12, 12, 18, 18 },
+ new int[] { 1, 0, 1, 0, 1, 0, 1 });
+ }
+
+ /**
+ * subclass that acts just like whitespace analyzer for testing
+ */
+ private class ShingleWrapperSubclassAnalyzer extends ShingleAnalyzerWrapper {
+ public TokenStream tokenStream(String fieldName, Reader reader) {
+ return new WhitespaceTokenizer(reader);
+ }
+ };
+
+ public void testLUCENE1678BWComp() throws Exception {
+ Analyzer a = new ShingleWrapperSubclassAnalyzer();
+ assertAnalyzesToReuse(a, "this is a test",
+ new String[] { "this", "is", "a", "test" },
+ new int[] { 0, 5, 8, 10 },
+ new int[] { 4, 7, 9, 14 },
+ new int[] { 1, 1, 1, 1 });
+ }
+
+ /*
+ * analyzer that does not support reuse
+ * it is LetterTokenizer on odd invocations, WhitespaceTokenizer on even.
+ */
+ private class NonreusableAnalyzer extends Analyzer {
+ int invocationCount = 0;
+ public TokenStream tokenStream(String fieldName, Reader reader) {
+ if (++invocationCount % 2 == 0)
+ return new WhitespaceTokenizer(reader);
+ else
+ return new LetterTokenizer(reader);
+ }
+ }
+
+ public void testWrappedAnalyzerDoesNotReuse() throws Exception {
+ Analyzer a = new ShingleAnalyzerWrapper(new NonreusableAnalyzer());
+ assertAnalyzesToReuse(a, "please divide into shingles.",
+ new String[] { "please", "please divide", "divide", "divide into", "into", "into shingles", "shingles" },
+ new int[] { 0, 0, 7, 7, 14, 14, 19 },
+ new int[] { 6, 13, 13, 18, 18, 27, 27 },
+ new int[] { 1, 0, 1, 0, 1, 0, 1 });
+ assertAnalyzesToReuse(a, "please divide into shingles.",
+ new String[] { "please", "please divide", "divide", "divide into", "into", "into shingles.", "shingles." },
+ new int[] { 0, 0, 7, 7, 14, 14, 19 },
+ new int[] { 6, 13, 13, 18, 18, 28, 28 },
+ new int[] { 1, 0, 1, 0, 1, 0, 1 });
+ assertAnalyzesToReuse(a, "please divide into shingles.",
+ new String[] { "please", "please divide", "divide", "divide into", "into", "into shingles", "shingles" },
+ new int[] { 0, 0, 7, 7, 14, 14, 19 },
+ new int[] { 6, 13, 13, 18, 18, 27, 27 },
+ new int[] { 1, 0, 1, 0, 1, 0, 1 });
+ }
+
+ private void assertAnalyzesToReuse(Analyzer a, String input, String[] output,
+ int[] startOffsets, int[] endOffsets, int[] posIncr) throws Exception {
+ TokenStream ts = a.reusableTokenStream("dummy", new StringReader(input));
+ TermAttribute termAtt = (TermAttribute) ts
+ .getAttribute(TermAttribute.class);
+ OffsetAttribute offsetAtt = (OffsetAttribute) ts
+ .getAttribute(OffsetAttribute.class);
+ PositionIncrementAttribute posIncAtt = (PositionIncrementAttribute) ts
+ .getAttribute(PositionIncrementAttribute.class);
+
+ for (int i = 0; i < output.length; i++) {
+ assertTrue(ts.incrementToken());
+ assertEquals(output[i], termAtt.term());
+ assertEquals(startOffsets[i], offsetAtt.startOffset());
+ assertEquals(endOffsets[i], offsetAtt.endOffset());
+ assertEquals(posIncr[i], posIncAtt.getPositionIncrement());
+ }
+
+ assertFalse(ts.incrementToken());
+ }
}
Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java?rev=804680&r1=804679&r2=804680&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java Sun Aug 16 12:37:05 2009
@@ -18,10 +18,13 @@
*/
import java.io.IOException;
+import java.io.StringReader;
import junit.framework.TestCase;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
@@ -182,6 +185,20 @@
TRI_GRAM_POSITION_INCREMENTS, TRI_GRAM_TYPES);
}
+ public void testReset() throws Exception {
+ Tokenizer wsTokenizer = new WhitespaceTokenizer(new StringReader("please divide this sentence"));
+ TokenStream filter = new ShingleFilter(wsTokenizer, 2);
+ TermAttribute termAtt = (TermAttribute) filter.getAttribute(TermAttribute.class);
+ assertTrue(filter.incrementToken());
+ assertEquals("(please,0,6)", termAtt.toString());
+ assertTrue(filter.incrementToken());
+ assertEquals("(please divide,0,13,type=shingle,posIncr=0)", termAtt.toString());
+ wsTokenizer.reset(new StringReader("please divide this sentence"));
+ filter.reset();
+ assertTrue(filter.incrementToken());
+ assertEquals("(please,0,6)", termAtt.toString());
+ }
+
protected void shingleFilterTest(int maxSize, Token[] tokensToShingle, Token[] tokensToCompare,
int[] positionIncrements, String[] types)
throws IOException {
Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java?rev=804680&r1=804679&r2=804680&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java Sun Aug 16 12:37:05 2009
@@ -17,12 +17,14 @@
* limitations under the License.
*/
+import java.io.Reader;
import java.io.StringReader;
import junit.framework.TestCase;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
@@ -91,6 +93,23 @@
ts.close();
}
+ public void assertAnalyzesToReuse(Analyzer a, String input, String[] output)
+ throws Exception {
+
+ TokenStream ts = a.reusableTokenStream("dummy", new StringReader(input));
+ TermAttribute termAtt = (TermAttribute) ts
+ .getAttribute(TermAttribute.class);
+ OffsetAttribute offsetAtt = (OffsetAttribute) ts
+ .getAttribute(OffsetAttribute.class);
+ TypeAttribute typeAtt = (TypeAttribute) ts
+ .getAttribute(TypeAttribute.class);
+ for (int i = 0; i < output.length; i++) {
+ assertTrue(ts.incrementToken());
+ assertEquals(termAtt.term(), output[i]);
+ }
+ assertFalse(ts.incrementToken());
+ }
+
public void assertAnalyzesTo(Analyzer a, String input, String[] output) throws Exception {
assertAnalyzesTo(a, input, output, null, null, null);
}
@@ -124,4 +143,33 @@
"à¸à¸£à¸°à¹à¸¢à¸à¸§à¹à¸² The quick brown fox jumped over the lazy dogs",
new String[] { "à¸à¸£à¸°à¹à¸¢à¸", "วà¹à¸²", "quick", "brown", "fox", "jumped", "over", "lazy", "dogs" });
}
+
+ public void testReusableTokenStream() throws Exception {
+ ThaiAnalyzer analyzer = new ThaiAnalyzer();
+ assertAnalyzesToReuse(analyzer, "", new String[] {});
+
+ assertAnalyzesToReuse(
+ analyzer,
+ "à¸à¸²à¸£à¸à¸µà¹à¹à¸à¹à¸à¹à¸à¸à¹à¸ªà¸à¸à¸§à¹à¸²à¸à¸²à¸à¸à¸µ",
+ new String[] { "à¸à¸²à¸£", "à¸à¸µà¹", "à¹à¸à¹", "à¸à¹à¸à¸", "à¹à¸ªà¸à¸", "วà¹à¸²", "à¸à¸²à¸", "à¸à¸µ"});
+
+ assertAnalyzesToReuse(
+ analyzer,
+ "à¸à¸£à¸´à¸©à¸±à¸à¸à¸·à¹à¸ XY&Z - à¸à¸¸à¸¢à¸à¸±à¸ xyz@demo.com",
+ new String[] { "à¸à¸£à¸´à¸©à¸±à¸", "à¸à¸·à¹à¸", "xy&z", "à¸à¸¸à¸¢", "à¸à¸±à¸", "xyz@demo.com" });
+ }
+
+ /**
+ * subclass that acts just like whitespace analyzer for testing
+ */
+ private class ThaiSubclassAnalyzer extends ThaiAnalyzer {
+ public TokenStream tokenStream(String fieldName, Reader reader) {
+ return new WhitespaceTokenizer(reader);
+ }
+ }
+
+ public void testLUCENE1678BWComp() throws Exception {
+ ThaiSubclassAnalyzer a = new ThaiSubclassAnalyzer();
+ assertAnalyzesToReuse(a, "à¸à¸²à¸£à¸à¸µà¹à¹à¸à¹à¸à¹à¸à¸à¹à¸ªà¸à¸à¸§à¹à¸²à¸à¸²à¸à¸à¸µ", new String[] { "à¸à¸²à¸£à¸à¸µà¹à¹à¸à¹à¸à¹à¸à¸à¹à¸ªà¸à¸à¸§à¹à¸²à¸à¸²à¸à¸à¸µ" });
+ }
}
Modified: lucene/java/trunk/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/SmartChineseAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/SmartChineseAnalyzer.java?rev=804680&r1=804679&r2=804680&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/SmartChineseAnalyzer.java (original)
+++ lucene/java/trunk/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/SmartChineseAnalyzer.java Sun Aug 16 12:37:05 2009
@@ -151,6 +151,7 @@
}
} else {
streams.tokenStream.reset(reader);
+ streams.filteredTokenStream.reset(); // reset WordTokenFilter's state
}
return streams.filteredTokenStream;
Modified: lucene/java/trunk/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java?rev=804680&r1=804679&r2=804680&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java (original)
+++ lucene/java/trunk/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java Sun Aug 16 12:37:05 2009
@@ -102,4 +102,13 @@
}
}
+ public void reset() throws IOException {
+ super.reset();
+ tokenStart = tokenEnd = 0;
+ }
+
+ public void reset(Reader input) throws IOException {
+ super.reset(input);
+ reset();
+ }
}
Modified: lucene/java/trunk/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordTokenFilter.java?rev=804680&r1=804679&r2=804680&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordTokenFilter.java (original)
+++ lucene/java/trunk/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordTokenFilter.java Sun Aug 16 12:37:05 2009
@@ -81,4 +81,9 @@
typeAtt.setType("word");
return true;
}
+
+ public void reset() throws IOException {
+ super.reset();
+ tokenIter = null;
+ }
}
Modified: lucene/java/trunk/contrib/analyzers/smartcn/src/test/org/apache/lucene/analysis/cn/TestSmartChineseAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/smartcn/src/test/org/apache/lucene/analysis/cn/TestSmartChineseAnalyzer.java?rev=804680&r1=804679&r2=804680&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/smartcn/src/test/org/apache/lucene/analysis/cn/TestSmartChineseAnalyzer.java (original)
+++ lucene/java/trunk/contrib/analyzers/smartcn/src/test/org/apache/lucene/analysis/cn/TestSmartChineseAnalyzer.java Sun Aug 16 12:37:05 2009
@@ -108,6 +108,33 @@
new int[] { 1, 3, 4, 6, 7, 9 });
}
+ public void testReusableTokenStream() throws Exception {
+ Analyzer a = new SmartChineseAnalyzer();
+ assertAnalyzesToReuse(a, "æè´ä¹° Tests äºéå
·åæè£
",
+ new String[] { "æ", "è´ä¹°", "test", "äº", "éå
·", "å", "æè£
"},
+ new int[] { 0, 1, 4, 10, 11, 13, 14 },
+ new int[] { 1, 3, 9, 11, 13, 14, 16 });
+ assertAnalyzesToReuse(a, "æè´ä¹°äºéå
·åæè£
ã",
+ new String[] { "æ", "è´ä¹°", "äº", "éå
·", "å", "æè£
" },
+ new int[] { 0, 1, 3, 4, 6, 7 },
+ new int[] { 1, 3, 4, 6, 7, 9 });
+ }
+
+ public void assertAnalyzesToReuse(Analyzer a, String input, String[] output,
+ int startOffsets[], int endOffsets[]) throws Exception {
+
+ TokenStream ts = a.reusableTokenStream("dummy", new StringReader(input));
+ TermAttribute termAtt = (TermAttribute) ts.getAttribute(TermAttribute.class);
+ OffsetAttribute offsetAtt = (OffsetAttribute) ts.getAttribute(OffsetAttribute.class);
+ for (int i = 0; i < output.length; i++) {
+ assertTrue(ts.incrementToken());
+ assertEquals(termAtt.term(), output[i]);
+ assertEquals(offsetAtt.startOffset(), startOffsets[i]);
+ assertEquals(offsetAtt.endOffset(), endOffsets[i]);
+ }
+ assertFalse(ts.incrementToken());
+ }
+
public void assertAnalyzesTo(Analyzer a, String input, String[] output, int startOffsets[], int endOffsets[], String types[])
throws Exception {
Modified: lucene/java/trunk/contrib/memory/src/java/org/apache/lucene/index/memory/SynonymTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/memory/src/java/org/apache/lucene/index/memory/SynonymTokenFilter.java?rev=804680&r1=804679&r2=804680&view=diff
==============================================================================
--- lucene/java/trunk/contrib/memory/src/java/org/apache/lucene/index/memory/SynonymTokenFilter.java (original)
+++ lucene/java/trunk/contrib/memory/src/java/org/apache/lucene/index/memory/SynonymTokenFilter.java Sun Aug 16 12:37:05 2009
@@ -141,5 +141,12 @@
arr[i + r] = tmp;
}
}
-
+
+ public void reset() throws IOException {
+ super.reset();
+ stack = null;
+ index = 0;
+ current = null;
+ todo = 0;
+ }
}
Added: lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/TestSynonymTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/TestSynonymTokenFilter.java?rev=804680&view=auto
==============================================================================
--- lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/TestSynonymTokenFilter.java (added)
+++ lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/TestSynonymTokenFilter.java Sun Aug 16 12:37:05 2009
@@ -0,0 +1,160 @@
+package org.apache.lucene.index.memory;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.LowerCaseFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.WhitespaceTokenizer;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+
+import junit.framework.TestCase;
+
+public class TestSynonymTokenFilter extends TestCase {
+ File dataDir = new File(System.getProperty("dataDir", "./bin"));
+ File testFile = new File(dataDir, "org/apache/lucene/index/memory/testSynonyms.txt");
+
+ public void testSynonyms() throws Exception {
+ SynonymMap map = new SynonymMap(new FileInputStream(testFile));
+ /* all expansions */
+ Analyzer analyzer = new SynonymWhitespaceAnalyzer(map, Integer.MAX_VALUE);
+ assertAnalyzesTo(analyzer, "Lost in the woods",
+ new String[] { "lost", "in", "the", "woods", "forest", "wood" },
+ new int[] { 0, 5, 8, 12, 12, 12 },
+ new int[] { 4, 7, 11, 17, 17, 17 },
+ new int[] { 1, 1, 1, 1, 0, 0 });
+ }
+
+ public void testSynonymsLimitedAmount() throws Exception {
+ SynonymMap map = new SynonymMap(new FileInputStream(testFile));
+ /* limit to one synonym expansion */
+ Analyzer analyzer = new SynonymWhitespaceAnalyzer(map, 1);
+ assertAnalyzesTo(analyzer, "Lost in the woods",
+ /* wood comes before forest due to
+ * the input file, not lexicographic order
+ */
+ new String[] { "lost", "in", "the", "woods", "wood" },
+ new int[] { 0, 5, 8, 12, 12 },
+ new int[] { 4, 7, 11, 17, 17 },
+ new int[] { 1, 1, 1, 1, 0 });
+ }
+
+ public void testReusableTokenStream() throws Exception {
+ SynonymMap map = new SynonymMap(new FileInputStream(testFile));
+ /* limit to one synonym expansion */
+ Analyzer analyzer = new SynonymWhitespaceAnalyzer(map, 1);
+ assertAnalyzesToReuse(analyzer, "Lost in the woods",
+ new String[] { "lost", "in", "the", "woods", "wood" },
+ new int[] { 0, 5, 8, 12, 12 },
+ new int[] { 4, 7, 11, 17, 17 },
+ new int[] { 1, 1, 1, 1, 0 });
+ assertAnalyzesToReuse(analyzer, "My wolfish dog went to the forest",
+ new String[] { "my", "wolfish", "ravenous", "dog", "went", "to",
+ "the", "forest", "woods" },
+ new int[] { 0, 3, 3, 11, 15, 20, 23, 27, 27 },
+ new int[] { 2, 10, 10, 14, 19, 22, 26, 33, 33 },
+ new int[] { 1, 1, 0, 1, 1, 1, 1, 1, 0 });
+ }
+
+ private class SynonymWhitespaceAnalyzer extends Analyzer {
+ private SynonymMap synonyms;
+ private int maxSynonyms;
+
+ public SynonymWhitespaceAnalyzer(SynonymMap synonyms, int maxSynonyms) {
+ this.synonyms = synonyms;
+ this.maxSynonyms = maxSynonyms;
+ }
+
+ public TokenStream tokenStream(String fieldName, Reader reader) {
+ TokenStream ts = new WhitespaceTokenizer(reader);
+ ts = new LowerCaseFilter(ts);
+ ts = new SynonymTokenFilter(ts, synonyms, maxSynonyms);
+ return ts;
+ }
+
+ private class SavedStreams {
+ Tokenizer source;
+ TokenStream result;
+ };
+
+ public TokenStream reusableTokenStream(String fieldName, Reader reader)
+ throws IOException {
+ SavedStreams streams = (SavedStreams) getPreviousTokenStream();
+ if (streams == null) {
+ streams = new SavedStreams();
+ streams.source = new WhitespaceTokenizer(reader);
+ streams.result = new LowerCaseFilter(streams.source);
+ streams.result = new SynonymTokenFilter(streams.result, synonyms, maxSynonyms);
+ } else {
+ streams.source.reset(reader);
+ streams.result.reset(); // reset the SynonymTokenFilter
+ }
+ return streams.result;
+ }
+ }
+
+ public void assertAnalyzesTo(Analyzer a, String input, String[] output,
+ int startOffsets[], int endOffsets[], int posIncs[]) throws Exception {
+
+ TokenStream ts = a.tokenStream("dummy", new StringReader(input));
+ TermAttribute termAtt = (TermAttribute) ts
+ .getAttribute(TermAttribute.class);
+ OffsetAttribute offsetAtt = (OffsetAttribute) ts
+ .getAttribute(OffsetAttribute.class);
+ PositionIncrementAttribute posIncAtt = (PositionIncrementAttribute) ts
+ .getAttribute(PositionIncrementAttribute.class);
+ for (int i = 0; i < output.length; i++) {
+ assertTrue(ts.incrementToken());
+ assertEquals(termAtt.term(), output[i]);
+ assertEquals(offsetAtt.startOffset(), startOffsets[i]);
+ assertEquals(offsetAtt.endOffset(), endOffsets[i]);
+ assertEquals(posIncAtt.getPositionIncrement(), posIncs[i]);
+ }
+ assertFalse(ts.incrementToken());
+ ts.close();
+ }
+
+ public void assertAnalyzesToReuse(Analyzer a, String input, String[] output,
+ int startOffsets[], int endOffsets[], int posIncs[]) throws Exception {
+
+ TokenStream ts = a.reusableTokenStream("dummy", new StringReader(input));
+ TermAttribute termAtt = (TermAttribute) ts
+ .getAttribute(TermAttribute.class);
+ OffsetAttribute offsetAtt = (OffsetAttribute) ts
+ .getAttribute(OffsetAttribute.class);
+ PositionIncrementAttribute posIncAtt = (PositionIncrementAttribute) ts
+ .getAttribute(PositionIncrementAttribute.class);
+ for (int i = 0; i < output.length; i++) {
+ assertTrue(ts.incrementToken());
+ assertEquals(termAtt.term(), output[i]);
+ assertEquals(offsetAtt.startOffset(), startOffsets[i]);
+ assertEquals(offsetAtt.endOffset(), endOffsets[i]);
+ assertEquals(posIncAtt.getPositionIncrement(), posIncs[i]);
+ }
+ assertFalse(ts.incrementToken());
+ }
+}
Propchange: lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/TestSynonymTokenFilter.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/testSynonyms.txt
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/testSynonyms.txt?rev=804680&view=auto
==============================================================================
--- lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/testSynonyms.txt (added)
+++ lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/testSynonyms.txt Sun Aug 16 12:37:05 2009
@@ -0,0 +1,5 @@
+s(100000001,1,'woods',n,1,0).
+s(100000001,2,'wood',n,1,0).
+s(100000001,3,'forest',n,1,0).
+s(100000002,1,'wolfish',n,1,0).
+s(100000002,2,'ravenous',n,1,0).
Propchange: lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/testSynonyms.txt
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/java/trunk/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballAnalyzer.java?rev=804680&r1=804679&r2=804680&view=diff
==============================================================================
--- lucene/java/trunk/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballAnalyzer.java (original)
+++ lucene/java/trunk/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballAnalyzer.java Sun Aug 16 12:37:05 2009
@@ -20,6 +20,7 @@
import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.standard.*;
+import java.io.IOException;
import java.io.Reader;
import java.util.Set;
@@ -37,6 +38,7 @@
/** Builds the named analyzer with no stop words. */
public SnowballAnalyzer(String name) {
this.name = name;
+ setOverridesTokenStreamMethod(SnowballAnalyzer.class);
}
/** Builds the named analyzer with the given stop words. */
@@ -46,7 +48,8 @@
}
/** Constructs a {@link StandardTokenizer} filtered by a {@link
- StandardFilter}, a {@link LowerCaseFilter} and a {@link StopFilter}. */
+ StandardFilter}, a {@link LowerCaseFilter}, a {@link StopFilter},
+ and a {@link SnowballFilter} */
public TokenStream tokenStream(String fieldName, Reader reader) {
TokenStream result = new StandardTokenizer(reader);
result = new StandardFilter(result);
@@ -56,4 +59,37 @@
result = new SnowballFilter(result, name);
return result;
}
+
+ private class SavedStreams {
+ Tokenizer source;
+ TokenStream result;
+ };
+
+ /** Returns a (possibly reused) {@link StandardTokenizer} filtered by a
+ * {@link StandardFilter}, a {@link LowerCaseFilter},
+ * a {@link StopFilter}, and a {@link SnowballFilter} */
+ public TokenStream reusableTokenStream(String fieldName, Reader reader)
+ throws IOException {
+ if (overridesTokenStreamMethod) {
+ // LUCENE-1678: force fallback to tokenStream() if we
+ // have been subclassed and that subclass overrides
+ // tokenStream but not reusableTokenStream
+ return tokenStream(fieldName, reader);
+ }
+
+ SavedStreams streams = (SavedStreams) getPreviousTokenStream();
+ if (streams == null) {
+ streams = new SavedStreams();
+ streams.source = new StandardTokenizer(reader);
+ streams.result = new StandardFilter(streams.source);
+ streams.result = new LowerCaseFilter(streams.result);
+ if (stopSet != null)
+ streams.result = new StopFilter(streams.result, stopSet);
+ streams.result = new SnowballFilter(streams.result, name);
+ setPreviousTokenStream(streams);
+ } else {
+ streams.source.reset(reader);
+ }
+ return streams.result;
+ }
}
Modified: lucene/java/trunk/contrib/snowball/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/snowball/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java?rev=804680&r1=804679&r2=804680&view=diff
==============================================================================
--- lucene/java/trunk/contrib/snowball/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java (original)
+++ lucene/java/trunk/contrib/snowball/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java Sun Aug 16 12:37:05 2009
@@ -17,11 +17,13 @@
* limitations under the License.
*/
+import java.io.Reader;
import java.io.StringReader;
import junit.framework.TestCase;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.lucene.index.Payload;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
@@ -45,6 +47,18 @@
assertFalse(ts.incrementToken());
ts.close();
}
+
+ public void assertAnalyzesToReuse(Analyzer a,
+ String input,
+ String[] output) throws Exception {
+ TokenStream ts = a.reusableTokenStream("dummy", new StringReader(input));
+ TermAttribute termAtt = (TermAttribute) ts.getAttribute(TermAttribute.class);
+ for (int i = 0; i < output.length; i++) {
+ assertTrue(ts.incrementToken());
+ assertEquals(output[i], termAtt.term());
+ }
+ assertFalse(ts.incrementToken());
+ }
public void testEnglish() throws Exception {
Analyzer a = new SnowballAnalyzer("English");
@@ -52,7 +66,33 @@
new String[]{"he", "abhor", "accent"});
}
-
+ public void testReusableTokenStream() throws Exception {
+ Analyzer a = new SnowballAnalyzer("English");
+ assertAnalyzesToReuse(a, "he abhorred accents",
+ new String[]{"he", "abhor", "accent"});
+ assertAnalyzesToReuse(a, "she abhorred him",
+ new String[]{"she", "abhor", "him"});
+ }
+
+ /**
+ * subclass that acts just like whitespace analyzer for testing
+ */
+ private class SnowballSubclassAnalyzer extends SnowballAnalyzer {
+ public SnowballSubclassAnalyzer(String name) {
+ super(name);
+ }
+
+ public TokenStream tokenStream(String fieldName, Reader reader) {
+ return new WhitespaceTokenizer(reader);
+ }
+ }
+
+ public void testLUCENE1678BWComp() throws Exception {
+ Analyzer a = new SnowballSubclassAnalyzer("English");
+ assertAnalyzesToReuse(a, "he abhorred accents",
+ new String[]{"he", "abhorred", "accents"});
+ }
+
public void testFilterTokens() throws Exception {
SnowballFilter filter = new SnowballFilter(new TestTokenStream(), "English");
TermAttribute termAtt = (TermAttribute) filter.getAttribute(TermAttribute.class);