You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/01/17 18:23:44 UTC
svn commit: r1559196 [7/19] - in /lucene/dev/branches/lucene5376: ./
dev-tools/ dev-tools/idea/solr/contrib/morphlines-cell/
dev-tools/maven/lucene/facet/ lucene/ lucene/analysis/
lucene/analysis/common/ lucene/analysis/common/src/java/org/apache/lucen...
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianCharFilter.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianCharFilter.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianCharFilter.java Fri Jan 17 17:23:33 2014
@@ -26,8 +26,8 @@ import org.apache.lucene.analysis.MockTo
public class TestPersianCharFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new MockTokenizer(reader));
+ protected TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new MockTokenizer());
}
@Override
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilter.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilter.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilter.java Fri Jan 17 17:23:33 2014
@@ -58,7 +58,7 @@ public class TestPersianNormalizationFil
}
private void check(final String input, final String expected) throws IOException {
- MockTokenizer tokenStream = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
+ MockTokenizer tokenStream = whitespaceMockTokenizer(input);
PersianNormalizationFilter filter = new PersianNormalizationFilter(
tokenStream);
assertTokenStreamContents(filter, new String[]{expected});
@@ -67,8 +67,8 @@ public class TestPersianNormalizationFil
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new PersianNormalizationFilter(tokenizer));
}
};
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilterFactory.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilterFactory.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilterFactory.java Fri Jan 17 17:23:33 2014
@@ -33,7 +33,7 @@ public class TestPersianNormalizationFil
*/
public void testNormalization() throws Exception {
Reader reader = new StringReader("ÙØ§Û");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("PersianNormalization").create(stream);
assertTokenStreamContents(stream, new String[] { "ÙØ§Ù" });
}
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java Fri Jan 17 17:23:33 2014
@@ -18,7 +18,6 @@ package org.apache.lucene.analysis.fi;
*/
import java.io.IOException;
-import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
@@ -37,9 +36,8 @@ import static org.apache.lucene.analysis
public class TestFinnishLightStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new FinnishLightStemFilter(source));
}
};
@@ -53,8 +51,8 @@ public class TestFinnishLightStemFilter
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("edeltäjistään"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new FinnishLightStemFilter(sink));
}
@@ -70,8 +68,8 @@ public class TestFinnishLightStemFilter
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new FinnishLightStemFilter(tokenizer));
}
};
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilterFactory.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilterFactory.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilterFactory.java Fri Jan 17 17:23:33 2014
@@ -22,6 +22,7 @@ import java.io.StringReader;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
@@ -30,7 +31,8 @@ import org.apache.lucene.analysis.util.B
public class TestFinnishLightStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("aseistettujen");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("FinnishLightStem").create(stream);
assertTokenStreamContents(stream, new String[] { "aseistet" });
}
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java Fri Jan 17 17:23:33 2014
@@ -37,9 +37,8 @@ import static org.apache.lucene.analysis
public class TestFrenchLightStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer( MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new FrenchLightStemFilter(source));
}
};
@@ -183,8 +182,8 @@ public class TestFrenchLightStemFilter e
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("chevaux"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new FrenchLightStemFilter(sink));
}
@@ -200,8 +199,8 @@ public class TestFrenchLightStemFilter e
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new FrenchLightStemFilter(tokenizer));
}
};
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilterFactory.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilterFactory.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilterFactory.java Fri Jan 17 17:23:33 2014
@@ -30,7 +30,7 @@ import org.apache.lucene.analysis.util.B
public class TestFrenchLightStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("administrativement");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("FrenchLightStem").create(stream);
assertTokenStreamContents(stream, new String[] { "administratif" });
}
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java Fri Jan 17 17:23:33 2014
@@ -37,9 +37,8 @@ import static org.apache.lucene.analysis
public class TestFrenchMinimalStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new FrenchMinimalStemFilter(source));
}
};
@@ -62,8 +61,8 @@ public class TestFrenchMinimalStemFilter
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("chevaux"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer( MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new FrenchMinimalStemFilter(sink));
}
@@ -84,8 +83,8 @@ public class TestFrenchMinimalStemFilter
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new FrenchMinimalStemFilter(tokenizer));
}
};
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilterFactory.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilterFactory.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilterFactory.java Fri Jan 17 17:23:33 2014
@@ -30,7 +30,7 @@ import org.apache.lucene.analysis.util.B
public class TestFrenchMinimalStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("chevaux");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("FrenchMinimalStem").create(stream);
assertTokenStreamContents(stream, new String[] { "cheval" });
}
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilter.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilter.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilter.java Fri Jan 17 17:23:33 2014
@@ -18,12 +18,8 @@ package org.apache.lucene.analysis.ga;
*/
import java.io.IOException;
-import java.io.Reader;
-import java.io.StringReader;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
@@ -37,8 +33,7 @@ public class TestIrishLowerCaseFilter ex
* Test lowercase
*/
public void testIrishLowerCaseFilter() throws Exception {
- TokenStream stream = new MockTokenizer(new StringReader(
- "nAthair tUISCE hARD"), MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer("nAthair tUISCE hARD");
IrishLowerCaseFilter filter = new IrishLowerCaseFilter(stream);
assertTokenStreamContents(filter, new String[] {"n-athair", "t-uisce",
"hard",});
@@ -47,8 +42,8 @@ public class TestIrishLowerCaseFilter ex
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new IrishLowerCaseFilter(tokenizer));
}
};
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilterFactory.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilterFactory.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilterFactory.java Fri Jan 17 17:23:33 2014
@@ -30,7 +30,7 @@ import org.apache.lucene.analysis.util.B
public class TestIrishLowerCaseFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testCasing() throws Exception {
Reader reader = new StringReader("nAthair tUISCE hARD");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("IrishLowerCase").create(stream);
assertTokenStreamContents(stream, new String[] { "n-athair", "t-uisce", "hard" });
}
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java Fri Jan 17 17:23:33 2014
@@ -35,8 +35,8 @@ import org.apache.lucene.analysis.util.C
public class TestGalicianMinimalStemFilter extends BaseTokenStreamTestCase {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new GalicianMinimalStemFilter(tokenizer));
}
};
@@ -57,8 +57,8 @@ public class TestGalicianMinimalStemFilt
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("elefantes"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new GalicianMinimalStemFilter(sink));
}
@@ -74,8 +74,8 @@ public class TestGalicianMinimalStemFilt
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new GalicianMinimalStemFilter(tokenizer));
}
};
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilterFactory.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilterFactory.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilterFactory.java Fri Jan 17 17:23:33 2014
@@ -30,7 +30,7 @@ import org.apache.lucene.analysis.util.B
public class TestGalicianMinimalStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("elefantes");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("GalicianMinimalStem").create(stream);
assertTokenStreamContents(stream, new String[] { "elefante" });
}
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilter.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilter.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilter.java Fri Jan 17 17:23:33 2014
@@ -36,9 +36,8 @@ import org.apache.lucene.analysis.standa
public class TestGalicianStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new StandardTokenizer(TEST_VERSION_CURRENT);
TokenStream result = new LowerCaseFilter(TEST_VERSION_CURRENT, source);
return new TokenStreamComponents(source, new GalicianStemFilter(result));
}
@@ -53,8 +52,8 @@ public class TestGalicianStemFilter exte
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new GalicianStemFilter(tokenizer));
}
};
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilterFactory.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilterFactory.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilterFactory.java Fri Jan 17 17:23:33 2014
@@ -30,7 +30,7 @@ import org.apache.lucene.analysis.util.B
public class TestGalicianStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("cariñosa");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("GalicianStem").create(stream);
assertTokenStreamContents(stream, new String[] { "cariñ" });
}
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiFilters.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiFilters.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiFilters.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiFilters.java Fri Jan 17 17:23:33 2014
@@ -21,7 +21,9 @@ import java.io.Reader;
import java.io.StringReader;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
+import org.apache.lucene.analysis.util.TokenizerFactory;
/**
* Simple tests to ensure the Hindi filter Factories are working.
@@ -32,7 +34,8 @@ public class TestHindiFilters extends Ba
*/
public void testIndicNormalizer() throws Exception {
Reader reader = new StringReader("তà§â à¤
ाà¥à¤°");
- TokenStream stream = tokenizerFactory("Standard").create(reader);
+ TokenStream stream = tokenizerFactory("Standard").create();
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("IndicNormalization").create(stream);
assertTokenStreamContents(stream, new String[] { "à§", "à¤à¤°" });
}
@@ -42,7 +45,8 @@ public class TestHindiFilters extends Ba
*/
public void testHindiNormalizer() throws Exception {
Reader reader = new StringReader("à¥à¤¿à¤¤à¤¾à¤¬");
- TokenStream stream = tokenizerFactory("Standard").create(reader);
+ TokenStream stream = tokenizerFactory("Standard").create();
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("IndicNormalization").create(stream);
stream = tokenFilterFactory("HindiNormalization").create(stream);
assertTokenStreamContents(stream, new String[] {"à¤à¤¿à¤¤à¤¾à¤¬"});
@@ -53,7 +57,8 @@ public class TestHindiFilters extends Ba
*/
public void testStemmer() throws Exception {
Reader reader = new StringReader("à¤à¤¿à¤¤à¤¾à¤¬à¥à¤");
- TokenStream stream = tokenizerFactory("Standard").create(reader);
+ TokenStream stream = tokenizerFactory("Standard").create();
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("IndicNormalization").create(stream);
stream = tokenFilterFactory("HindiNormalization").create(stream);
stream = tokenFilterFactory("HindiStem").create(stream);
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiNormalizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiNormalizer.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiNormalizer.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiNormalizer.java Fri Jan 17 17:23:33 2014
@@ -62,7 +62,7 @@ public class TestHindiNormalizer extends
check("à¤à¤à¤à¥ ॡà¤à¤à¥à¥à¥à¥£à¥à¥", "à¤
à¤à¤à¤à¤à¤à¤à¤¿à¥à¥à¥¢à¥à¥");
}
private void check(String input, String output) throws IOException {
- Tokenizer tokenizer = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
+ Tokenizer tokenizer = whitespaceMockTokenizer(input);
TokenFilter tf = new HindiNormalizationFilter(tokenizer);
assertTokenStreamContents(tf, new String[] { output });
}
@@ -70,8 +70,8 @@ public class TestHindiNormalizer extends
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new HindiNormalizationFilter(tokenizer));
}
};
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiStemmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiStemmer.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiStemmer.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiStemmer.java Fri Jan 17 17:23:33 2014
@@ -18,12 +18,9 @@ package org.apache.lucene.analysis.hi;
*/
import java.io.IOException;
-import java.io.Reader;
-import java.io.StringReader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
@@ -84,7 +81,7 @@ public class TestHindiStemmer extends Ba
}
private void check(String input, String output) throws IOException {
- Tokenizer tokenizer = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
+ Tokenizer tokenizer = whitespaceMockTokenizer(input);
TokenFilter tf = new HindiStemFilter(tokenizer);
assertTokenStreamContents(tf, new String[] { output });
}
@@ -92,8 +89,8 @@ public class TestHindiStemmer extends Ba
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new HindiStemFilter(tokenizer));
}
};
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java Fri Jan 17 17:23:33 2014
@@ -37,9 +37,8 @@ import static org.apache.lucene.analysis
public class TestHungarianLightStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new HungarianLightStemFilter(source));
}
};
@@ -53,8 +52,8 @@ public class TestHungarianLightStemFilte
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("babakocsi"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new HungarianLightStemFilter(sink));
}
@@ -65,8 +64,8 @@ public class TestHungarianLightStemFilte
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new HungarianLightStemFilter(tokenizer));
}
};
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilterFactory.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilterFactory.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilterFactory.java Fri Jan 17 17:23:33 2014
@@ -30,7 +30,7 @@ import org.apache.lucene.analysis.util.B
public class TestHungarianLightStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("házakat");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("HungarianLightStem").create(stream);
assertTokenStreamContents(stream, new String[] { "haz" });
}
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java Fri Jan 17 17:23:33 2014
@@ -18,8 +18,6 @@ package org.apache.lucene.analysis.hunsp
import java.io.IOException;
import java.io.InputStream;
-import java.io.Reader;
-import java.io.StringReader;
import java.text.ParseException;
import java.util.Arrays;
@@ -56,13 +54,13 @@ public class HunspellStemFilterTest ext
* Simple test for KeywordAttribute
*/
public void testKeywordAttribute() throws IOException {
- MockTokenizer tokenizer = new MockTokenizer(new StringReader("lucene is awesome"), MockTokenizer.WHITESPACE, true);
+ MockTokenizer tokenizer = whitespaceMockTokenizer("lucene is awesome");
tokenizer.setEnableChecks(true);
HunspellStemFilter filter = new HunspellStemFilter(tokenizer, DICTIONARY, _TestUtil.nextInt(random(), 1, 3));
assertTokenStreamContents(filter, new String[]{"lucene", "lucen", "is", "awesome"}, new int[] {1, 0, 1, 1});
// assert with keywork marker
- tokenizer = new MockTokenizer(new StringReader("lucene is awesome"), MockTokenizer.WHITESPACE, true);
+ tokenizer = whitespaceMockTokenizer("lucene is awesome");
CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, Arrays.asList("Lucene"), true);
filter = new HunspellStemFilter(new SetKeywordMarkerFilter(tokenizer, set), DICTIONARY, _TestUtil.nextInt(random(), 1, 3));
assertTokenStreamContents(filter, new String[]{"lucene", "is", "awesome"}, new int[] {1, 1, 1});
@@ -73,8 +71,8 @@ public class HunspellStemFilterTest ext
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new HunspellStemFilter(tokenizer, DICTIONARY, _TestUtil.nextInt(random(), 1, 3)));
}
};
@@ -84,8 +82,8 @@ public class HunspellStemFilterTest ext
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new HunspellStemFilter(tokenizer, DICTIONARY, _TestUtil.nextInt(random(), 1, 3)));
}
};
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilterFactory.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilterFactory.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilterFactory.java Fri Jan 17 17:23:33 2014
@@ -30,7 +30,7 @@ import org.apache.lucene.analysis.util.B
public class TestHunspellStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("abc");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("HunspellStem",
"dictionary", "test.dic",
"affix", "test.aff").create(stream);
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemFilterFactory.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemFilterFactory.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemFilterFactory.java Fri Jan 17 17:23:33 2014
@@ -22,6 +22,7 @@ import java.io.StringReader;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
@@ -33,7 +34,8 @@ public class TestIndonesianStemFilterFac
*/
public void testStemming() throws Exception {
Reader reader = new StringReader("dibukukannya");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("IndonesianStem").create(stream);
assertTokenStreamContents(stream, new String[] { "buku" });
}
@@ -43,7 +45,8 @@ public class TestIndonesianStemFilterFac
*/
public void testStemmingInflectional() throws Exception {
Reader reader = new StringReader("dibukukannya");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("IndonesianStem", "stemDerivational", "false").create(stream);
assertTokenStreamContents(stream, new String[] { "dibukukan" });
}
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemmer.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemmer.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemmer.java Fri Jan 17 17:23:33 2014
@@ -32,8 +32,8 @@ public class TestIndonesianStemmer exten
/* full stemming, no stopwords */
Analyzer a = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new IndonesianStemFilter(tokenizer));
}
};
@@ -113,8 +113,8 @@ public class TestIndonesianStemmer exten
/* inflectional-only stemming */
Analyzer b = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new IndonesianStemFilter(tokenizer, false));
}
};
@@ -136,8 +136,8 @@ public class TestIndonesianStemmer exten
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new IndonesianStemFilter(tokenizer));
}
};
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/in/TestIndicNormalizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/in/TestIndicNormalizer.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/in/TestIndicNormalizer.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/in/TestIndicNormalizer.java Fri Jan 17 17:23:33 2014
@@ -47,7 +47,8 @@ public class TestIndicNormalizer extends
}
private void check(String input, String output) throws IOException {
- Tokenizer tokenizer = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);;
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);;
+ tokenizer.setReader(new StringReader(input));
TokenFilter tf = new IndicNormalizationFilter(tokenizer);
assertTokenStreamContents(tf, new String[] { output });
}
@@ -55,8 +56,8 @@ public class TestIndicNormalizer extends
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new IndicNormalizationFilter(tokenizer));
}
};
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java Fri Jan 17 17:23:33 2014
@@ -34,9 +34,8 @@ import static org.apache.lucene.analysis
public class TestItalianLightStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new ItalianLightStemFilter(source));
}
};
@@ -54,8 +53,8 @@ public class TestItalianLightStemFilter
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new ItalianLightStemFilter(tokenizer));
}
};
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilterFactory.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilterFactory.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilterFactory.java Fri Jan 17 17:23:33 2014
@@ -30,7 +30,7 @@ import org.apache.lucene.analysis.util.B
public class TestItalianLightStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("ragazzo ragazzi");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("ItalianLightStem").create(stream);
assertTokenStreamContents(stream, new String[] { "ragazz", "ragazz" });
}
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemFilterFactory.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemFilterFactory.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemFilterFactory.java Fri Jan 17 17:23:33 2014
@@ -22,6 +22,7 @@ import java.io.StringReader;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
@@ -30,7 +31,8 @@ import org.apache.lucene.analysis.util.B
public class TestLatvianStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("tirgiem tirgus");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("LatvianStem").create(stream);
assertTokenStreamContents(stream, new String[] { "tirg", "tirg" });
}
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemmer.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemmer.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemmer.java Fri Jan 17 17:23:33 2014
@@ -32,8 +32,8 @@ import org.apache.lucene.analysis.core.K
public class TestLatvianStemmer extends BaseTokenStreamTestCase {
private Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new LatvianStemFilter(tokenizer));
}
};
@@ -273,8 +273,8 @@ public class TestLatvianStemmer extends
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new LatvianStemFilter(tokenizer));
}
};
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java Fri Jan 17 17:23:33 2014
@@ -26,8 +26,6 @@ import org.apache.lucene.analysis.core.K
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import java.io.IOException;
-import java.io.Reader;
-import java.io.StringReader;
import java.util.List;
import java.util.ArrayList;
import java.util.Iterator;
@@ -36,10 +34,9 @@ public class TestASCIIFoldingFilter exte
// testLain1Accents() is a copy of TestLatin1AccentFilter.testU().
public void testLatin1Accents() throws Exception {
- TokenStream stream = new MockTokenizer(new StringReader
- ("Des mot clés à LA CHAÃNE à à à à à Ã
à à à à à à à à à à IJ à Ã"
+ TokenStream stream = whitespaceMockTokenizer("Des mot clés à LA CHAÃNE à à à à à Ã
à à à à à à à à à à IJ à Ã"
+" à à à à à à Šà à à à à à Ÿ à á â ã ä å æ ç è é ê ë ì à î ï ij"
- +" ð ñ ò ó ô õ ö ø Šà þ ù ú û ü ý ÿ ï¬ ï¬"), MockTokenizer.WHITESPACE, false);
+ +" ð ñ ò ó ô õ ö ø Šà þ ù ú û ü ý ÿ ï¬ ï¬");
ASCIIFoldingFilter filter = new ASCIIFoldingFilter(stream);
CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
@@ -1897,7 +1894,7 @@ public class TestASCIIFoldingFilter exte
expectedOutputTokens.add(expected.toString());
}
- TokenStream stream = new MockTokenizer(new StringReader(inputText.toString()), MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(inputText.toString());
ASCIIFoldingFilter filter = new ASCIIFoldingFilter(stream);
CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
Iterator<String> expectedIter = expectedOutputTokens.iterator();
@@ -1918,8 +1915,8 @@ public class TestASCIIFoldingFilter exte
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new ASCIIFoldingFilter(tokenizer));
}
};
@@ -1929,8 +1926,8 @@ public class TestASCIIFoldingFilter exte
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new ASCIIFoldingFilter(tokenizer));
}
};
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilter.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilter.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilter.java Fri Jan 17 17:23:33 2014
@@ -107,8 +107,10 @@ public class TestCapitalizationFilter ex
boolean onlyFirstWord, CharArraySet keep, boolean forceFirstLetter,
Collection<char[]> okPrefix, int minWordLength, int maxWordCount,
int maxTokenLength) throws IOException {
- assertCapitalizesTo(new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false),
- expected, onlyFirstWord, keep, forceFirstLetter, okPrefix, minWordLength,
+ final MockTokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ tokenizer.setReader(new StringReader(input));
+ assertCapitalizesTo(tokenizer,
+ expected, onlyFirstWord, keep, forceFirstLetter, okPrefix, minWordLength,
maxWordCount, maxTokenLength);
}
@@ -116,7 +118,9 @@ public class TestCapitalizationFilter ex
boolean onlyFirstWord, CharArraySet keep, boolean forceFirstLetter,
Collection<char[]> okPrefix, int minWordLength, int maxWordCount,
int maxTokenLength) throws IOException {
- assertCapitalizesTo(new MockTokenizer(new StringReader(input), MockTokenizer.KEYWORD, false),
+ final MockTokenizer tokenizer = new MockTokenizer(MockTokenizer.KEYWORD, false);
+ tokenizer.setReader(new StringReader(input));
+ assertCapitalizesTo(tokenizer,
new String[] { expected }, onlyFirstWord, keep, forceFirstLetter, okPrefix,
minWordLength, maxWordCount, maxTokenLength);
}
@@ -126,8 +130,8 @@ public class TestCapitalizationFilter ex
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new CapitalizationFilter(tokenizer));
}
};
@@ -138,8 +142,8 @@ public class TestCapitalizationFilter ex
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new CapitalizationFilter(tokenizer));
}
};
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilterFactory.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilterFactory.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilterFactory.java Fri Jan 17 17:23:33 2014
@@ -28,7 +28,7 @@ public class TestCapitalizationFilterFac
public void testCapitalization() throws Exception {
Reader reader = new StringReader("kiTTEN");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "true").create(stream);
@@ -37,7 +37,7 @@ public class TestCapitalizationFilterFac
public void testCapitalization2() throws Exception {
Reader reader = new StringReader("and");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "true",
@@ -48,7 +48,7 @@ public class TestCapitalizationFilterFac
/** first is forced, but it's not a keep word, either */
public void testCapitalization3() throws Exception {
Reader reader = new StringReader("AnD");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "true",
@@ -58,7 +58,7 @@ public class TestCapitalizationFilterFac
public void testCapitalization4() throws Exception {
Reader reader = new StringReader("AnD");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "true",
@@ -68,7 +68,7 @@ public class TestCapitalizationFilterFac
public void testCapitalization5() throws Exception {
Reader reader = new StringReader("big");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "true",
@@ -78,7 +78,7 @@ public class TestCapitalizationFilterFac
public void testCapitalization6() throws Exception {
Reader reader = new StringReader("BIG");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "true",
@@ -88,7 +88,7 @@ public class TestCapitalizationFilterFac
public void testCapitalization7() throws Exception {
Reader reader = new StringReader("Hello thEre my Name is Ryan");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
+ TokenStream stream = keywordMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "true",
@@ -98,7 +98,7 @@ public class TestCapitalizationFilterFac
public void testCapitalization8() throws Exception {
Reader reader = new StringReader("Hello thEre my Name is Ryan");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "false",
@@ -108,7 +108,7 @@ public class TestCapitalizationFilterFac
public void testCapitalization9() throws Exception {
Reader reader = new StringReader("Hello thEre my Name is Ryan");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "false",
@@ -119,7 +119,7 @@ public class TestCapitalizationFilterFac
public void testCapitalization10() throws Exception {
Reader reader = new StringReader("McKinley");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "false",
@@ -131,7 +131,7 @@ public class TestCapitalizationFilterFac
/** using "McK" as okPrefix */
public void testCapitalization11() throws Exception {
Reader reader = new StringReader("McKinley");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "false",
@@ -144,7 +144,7 @@ public class TestCapitalizationFilterFac
/** test with numbers */
public void testCapitalization12() throws Exception {
Reader reader = new StringReader("1st 2nd third");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "false",
@@ -156,7 +156,7 @@ public class TestCapitalizationFilterFac
public void testCapitalization13() throws Exception {
Reader reader = new StringReader("the The the");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
+ TokenStream stream = keywordMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "false",
@@ -168,7 +168,7 @@ public class TestCapitalizationFilterFac
public void testKeepIgnoreCase() throws Exception {
Reader reader = new StringReader("kiTTEN");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
+ TokenStream stream = keywordMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "kitten",
"keepIgnoreCase", "true",
@@ -180,7 +180,7 @@ public class TestCapitalizationFilterFac
public void testKeepIgnoreCase2() throws Exception {
Reader reader = new StringReader("kiTTEN");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
+ TokenStream stream = keywordMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "kitten",
"keepIgnoreCase", "true",
@@ -192,7 +192,7 @@ public class TestCapitalizationFilterFac
public void testKeepIgnoreCase3() throws Exception {
Reader reader = new StringReader("kiTTEN");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
+ TokenStream stream = keywordMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keepIgnoreCase", "true",
"onlyFirstWord", "true",
@@ -208,7 +208,7 @@ public class TestCapitalizationFilterFac
*/
public void testMinWordLength() throws Exception {
Reader reader = new StringReader("helo testing");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"onlyFirstWord", "true",
"minWordLength", "5").create(stream);
@@ -221,7 +221,7 @@ public class TestCapitalizationFilterFac
*/
public void testMaxWordCount() throws Exception {
Reader reader = new StringReader("one two three four");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"maxWordCount", "2").create(stream);
assertTokenStreamContents(stream, new String[] { "One", "Two", "Three", "Four" });
@@ -232,7 +232,7 @@ public class TestCapitalizationFilterFac
*/
public void testMaxWordCount2() throws Exception {
Reader reader = new StringReader("one two three four");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
+ TokenStream stream = keywordMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"maxWordCount", "2").create(stream);
assertTokenStreamContents(stream, new String[] { "one two three four" });
@@ -245,7 +245,7 @@ public class TestCapitalizationFilterFac
*/
public void testMaxTokenLength() throws Exception {
Reader reader = new StringReader("this is a test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"maxTokenLength", "2").create(stream);
assertTokenStreamContents(stream, new String[] { "this", "is", "A", "test" });
@@ -256,7 +256,7 @@ public class TestCapitalizationFilterFac
*/
public void testForceFirstLetterWithKeep() throws Exception {
Reader reader = new StringReader("kitten");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "kitten",
"forceFirstLetter", "true").create(stream);
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCodepointCountFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCodepointCountFilter.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCodepointCountFilter.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCodepointCountFilter.java Fri Jan 17 17:23:33 2014
@@ -31,8 +31,7 @@ import org.apache.lucene.util._TestUtil;
public class TestCodepointCountFilter extends BaseTokenStreamTestCase {
public void testFilterWithPosIncr() throws Exception {
- TokenStream stream = new MockTokenizer(
- new StringReader("short toolong evenmuchlongertext a ab toolong foo"), MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer("short toolong evenmuchlongertext a ab toolong foo");
CodepointCountFilter filter = new CodepointCountFilter(TEST_VERSION_CURRENT, stream, 2, 6);
assertTokenStreamContents(filter,
new String[]{"short", "ab", "foo"},
@@ -43,8 +42,8 @@ public class TestCodepointCountFilter ex
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new CodepointCountFilter(TEST_VERSION_CURRENT, tokenizer, 0, 5));
}
};
@@ -58,7 +57,8 @@ public class TestCodepointCountFilter ex
int max = _TestUtil.nextInt(random(), 0, 100);
int count = text.codePointCount(0, text.length());
boolean expected = count >= min && count <= max;
- TokenStream stream = new KeywordTokenizer(new StringReader(text));
+ TokenStream stream = new KeywordTokenizer();
+ ((Tokenizer)stream).setReader(new StringReader(text));
stream = new CodepointCountFilter(TEST_VERSION_CURRENT, stream, min, max);
stream.reset();
assertEquals(expected, stream.incrementToken());
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCodepointCountFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCodepointCountFilterFactory.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCodepointCountFilterFactory.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCodepointCountFilterFactory.java Fri Jan 17 17:23:33 2014
@@ -22,13 +22,15 @@ import java.io.StringReader;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
public class TestCodepointCountFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testPositionIncrements() throws Exception {
Reader reader = new StringReader("foo foobar super-duper-trooper");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("CodepointCount",
"min", "4",
"max", "10").create(stream);
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestHyphenatedWordsFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestHyphenatedWordsFilter.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestHyphenatedWordsFilter.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestHyphenatedWordsFilter.java Fri Jan 17 17:23:33 2014
@@ -35,7 +35,8 @@ public class TestHyphenatedWordsFilter e
public void testHyphenatedWords() throws Exception {
String input = "ecologi-\r\ncal devel-\r\n\r\nop compre-\u0009hensive-hands-on and ecologi-\ncal";
// first test
- TokenStream ts = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
+ TokenStream ts = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)ts).setReader(new StringReader(input));
ts = new HyphenatedWordsFilter(ts);
assertTokenStreamContents(ts,
new String[] { "ecological", "develop", "comprehensive-hands-on", "and", "ecological" });
@@ -45,17 +46,19 @@ public class TestHyphenatedWordsFilter e
* Test that HyphenatedWordsFilter behaves correctly with a final hyphen
*/
public void testHyphenAtEnd() throws Exception {
- String input = "ecologi-\r\ncal devel-\r\n\r\nop compre-\u0009hensive-hands-on and ecology-";
+ String input = "ecologi-\r\ncal devel-\r\n\r\nop compre-\u0009hensive-hands-on and ecology-";
// first test
- TokenStream ts = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
- ts = new HyphenatedWordsFilter(ts);
- assertTokenStreamContents(ts,
- new String[] { "ecological", "develop", "comprehensive-hands-on", "and", "ecology-" });
+ TokenStream ts = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)ts).setReader(new StringReader(input));
+ ts = new HyphenatedWordsFilter(ts);
+ assertTokenStreamContents(ts,
+ new String[] { "ecological", "develop", "comprehensive-hands-on", "and", "ecology-" });
}
public void testOffsets() throws Exception {
String input = "abc- def geh 1234- 5678-";
- TokenStream ts = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
+ TokenStream ts = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)ts).setReader(new StringReader(input));
ts = new HyphenatedWordsFilter(ts);
assertTokenStreamContents(ts,
new String[] { "abcdef", "geh", "12345678-" },
@@ -68,8 +71,8 @@ public class TestHyphenatedWordsFilter e
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new HyphenatedWordsFilter(tokenizer));
}
};
@@ -80,8 +83,8 @@ public class TestHyphenatedWordsFilter e
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new HyphenatedWordsFilter(tokenizer));
}
};
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeepWordFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeepWordFilter.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeepWordFilter.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeepWordFilter.java Fri Jan 17 17:23:33 2014
@@ -17,8 +17,6 @@
package org.apache.lucene.analysis.miscellaneous;
-import java.io.Reader;
-import java.io.StringReader;
import java.util.HashSet;
import java.util.Set;
@@ -41,12 +39,12 @@ public class TestKeepWordFilter extends
String input = "xxx yyy aaa zzz BBB ccc ddd EEE";
// Test Stopwords
- TokenStream stream = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(input);
stream = new KeepWordFilter(TEST_VERSION_CURRENT, stream, new CharArraySet(TEST_VERSION_CURRENT, words, true));
assertTokenStreamContents(stream, new String[] { "aaa", "BBB" }, new int[] { 3, 2 });
// Now force case
- stream = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
+ stream = whitespaceMockTokenizer(input);
stream = new KeepWordFilter(TEST_VERSION_CURRENT, stream, new CharArraySet(TEST_VERSION_CURRENT,words, false));
assertTokenStreamContents(stream, new String[] { "aaa" }, new int[] { 3 });
}
@@ -60,8 +58,8 @@ public class TestKeepWordFilter extends
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream stream = new KeepWordFilter(TEST_VERSION_CURRENT, tokenizer, new CharArraySet(TEST_VERSION_CURRENT, words, true));
return new TokenStreamComponents(tokenizer, stream);
}
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordMarkerFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordMarkerFilter.java?rev=1559196&r1=1559195&r2=1559196&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordMarkerFilter.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordMarkerFilter.java Fri Jan 17 17:23:33 2014
@@ -43,16 +43,13 @@ public class TestKeywordMarkerFilter ext
String[] output = new String[] { "the", "quick", "brown", "LuceneFox",
"jumps" };
assertTokenStreamContents(new LowerCaseFilterMock(
- new SetKeywordMarkerFilter(new MockTokenizer(new StringReader(
- "The quIck browN LuceneFox Jumps"), MockTokenizer.WHITESPACE, false), set)), output);
+ new SetKeywordMarkerFilter(whitespaceMockTokenizer("The quIck browN LuceneFox Jumps"), set)), output);
CharArraySet mixedCaseSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("LuceneFox"), false);
assertTokenStreamContents(new LowerCaseFilterMock(
- new SetKeywordMarkerFilter(new MockTokenizer(new StringReader(
- "The quIck browN LuceneFox Jumps"), MockTokenizer.WHITESPACE, false), mixedCaseSet)), output);
+ new SetKeywordMarkerFilter(whitespaceMockTokenizer("The quIck browN LuceneFox Jumps"), mixedCaseSet)), output);
CharArraySet set2 = set;
assertTokenStreamContents(new LowerCaseFilterMock(
- new SetKeywordMarkerFilter(new MockTokenizer(new StringReader(
- "The quIck browN LuceneFox Jumps"), MockTokenizer.WHITESPACE, false), set2)), output);
+ new SetKeywordMarkerFilter(whitespaceMockTokenizer("The quIck browN LuceneFox Jumps"), set2)), output);
}
@Test
@@ -60,15 +57,13 @@ public class TestKeywordMarkerFilter ext
String[] output = new String[] { "the", "quick", "brown", "LuceneFox",
"jumps" };
assertTokenStreamContents(new LowerCaseFilterMock(
- new PatternKeywordMarkerFilter(new MockTokenizer(new StringReader(
- "The quIck browN LuceneFox Jumps"), MockTokenizer.WHITESPACE, false), Pattern.compile("[a-zA-Z]+[fF]ox"))), output);
+ new PatternKeywordMarkerFilter(whitespaceMockTokenizer("The quIck browN LuceneFox Jumps"), Pattern.compile("[a-zA-Z]+[fF]ox"))), output);
output = new String[] { "the", "quick", "brown", "lucenefox",
"jumps" };
assertTokenStreamContents(new LowerCaseFilterMock(
- new PatternKeywordMarkerFilter(new MockTokenizer(new StringReader(
- "The quIck browN LuceneFox Jumps"), MockTokenizer.WHITESPACE, false), Pattern.compile("[a-zA-Z]+[f]ox"))), output);
+ new PatternKeywordMarkerFilter(whitespaceMockTokenizer("The quIck browN LuceneFox Jumps"), Pattern.compile("[a-zA-Z]+[f]ox"))), output);
}
// LUCENE-2901
@@ -76,7 +71,7 @@ public class TestKeywordMarkerFilter ext
TokenStream ts = new LowerCaseFilterMock(
new SetKeywordMarkerFilter(
new SetKeywordMarkerFilter(
- new MockTokenizer(new StringReader("Dogs Trees Birds Houses"), MockTokenizer.WHITESPACE, false),
+ whitespaceMockTokenizer("Dogs Trees Birds Houses"),
new CharArraySet(TEST_VERSION_CURRENT, asSet("Birds", "Houses"), false)),
new CharArraySet(TEST_VERSION_CURRENT, asSet("Dogs", "Trees"), false)));
@@ -85,7 +80,7 @@ public class TestKeywordMarkerFilter ext
ts = new LowerCaseFilterMock(
new PatternKeywordMarkerFilter(
new PatternKeywordMarkerFilter(
- new MockTokenizer(new StringReader("Dogs Trees Birds Houses"), MockTokenizer.WHITESPACE, false),
+ whitespaceMockTokenizer("Dogs Trees Birds Houses"),
Pattern.compile("Birds|Houses")),
Pattern.compile("Dogs|Trees")));
@@ -94,7 +89,7 @@ public class TestKeywordMarkerFilter ext
ts = new LowerCaseFilterMock(
new SetKeywordMarkerFilter(
new PatternKeywordMarkerFilter(
- new MockTokenizer(new StringReader("Dogs Trees Birds Houses"), MockTokenizer.WHITESPACE, false),
+ whitespaceMockTokenizer("Dogs Trees Birds Houses"),
Pattern.compile("Birds|Houses")),
new CharArraySet(TEST_VERSION_CURRENT, asSet("Dogs", "Trees"), false)));