You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2011/09/26 18:25:54 UTC
svn commit: r1175937 [2/3] - in /lucene/dev/branches/solrcloud: ./
dev-tools/idea/lucene/contrib/ lucene/
lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/
lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/ l...
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/CommonGramsQueryFilterFactoryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/CommonGramsQueryFilterFactoryTest.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/CommonGramsQueryFilterFactoryTest.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/CommonGramsQueryFilterFactoryTest.java Mon Sep 26 16:25:51 2011
@@ -16,9 +16,9 @@
*/
package org.apache.solr.analysis;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.solr.common.ResourceLoader;
import org.apache.solr.core.SolrResourceLoader;
@@ -76,7 +76,7 @@ public class CommonGramsQueryFilterFacto
Set<?> words = factory.getCommonWords();
assertTrue("words is null and it shouldn't be", words != null);
assertTrue(words.contains("the"));
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader("testing the factory"));
+ Tokenizer tokenizer = new MockTokenizer(new StringReader("testing the factory"), MockTokenizer.WHITESPACE, false);
TokenStream stream = factory.create(tokenizer);
assertTokenStreamContents(stream,
new String[] { "testing_the", "the_factory" });
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/DoubleMetaphoneFilterFactoryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/DoubleMetaphoneFilterFactoryTest.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/DoubleMetaphoneFilterFactoryTest.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/DoubleMetaphoneFilterFactoryTest.java Mon Sep 26 16:25:51 2011
@@ -20,8 +20,8 @@ import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.phonetic.DoubleMetaphoneFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -30,7 +30,7 @@ public class DoubleMetaphoneFilterFactor
public void testDefaults() throws Exception {
DoubleMetaphoneFilterFactory factory = new DoubleMetaphoneFilterFactory();
factory.init(new HashMap<String, String>());
- TokenStream inputStream = new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader("international"));
+ TokenStream inputStream = new MockTokenizer(new StringReader("international"), MockTokenizer.WHITESPACE, false);
TokenStream filteredStream = factory.create(inputStream);
assertEquals(DoubleMetaphoneFilter.class, filteredStream.getClass());
@@ -44,7 +44,7 @@ public class DoubleMetaphoneFilterFactor
parameters.put("maxCodeLength", "8");
factory.init(parameters);
- TokenStream inputStream = new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader("international"));
+ TokenStream inputStream = new MockTokenizer(new StringReader("international"), MockTokenizer.WHITESPACE, false);
TokenStream filteredStream = factory.create(inputStream);
assertEquals(DoubleMetaphoneFilter.class, filteredStream.getClass());
@@ -57,12 +57,13 @@ public class DoubleMetaphoneFilterFactor
public void testReset() throws Exception {
DoubleMetaphoneFilterFactory factory = new DoubleMetaphoneFilterFactory();
factory.init(new HashMap<String, String>());
- TokenStream inputStream = new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader("international"));
+ TokenStream inputStream = new MockTokenizer(new StringReader("international"), MockTokenizer.WHITESPACE, false);
TokenStream filteredStream = factory.create(inputStream);
CharTermAttribute termAtt = filteredStream.addAttribute(CharTermAttribute.class);
assertEquals(DoubleMetaphoneFilter.class, filteredStream.getClass());
+ filteredStream.reset();
assertTrue(filteredStream.incrementToken());
assertEquals(13, termAtt.length());
assertEquals("international", termAtt.toString());
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/LengthFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/LengthFilterTest.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/LengthFilterTest.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/LengthFilterTest.java Mon Sep 26 16:25:51 2011
@@ -21,8 +21,8 @@ import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
public class LengthFilterTest extends BaseTokenTestCase {
@@ -34,7 +34,7 @@ public class LengthFilterTest extends Ba
// default: args.put("enablePositionIncrements", "false");
factory.init(args);
String test = "foo foobar super-duper-trooper";
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader(test)));
+ TokenStream stream = factory.create(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "foobar" }, new int[] { 1 });
factory = new LengthFilterFactory();
@@ -43,7 +43,7 @@ public class LengthFilterTest extends Ba
args.put(LengthFilterFactory.MAX_KEY, String.valueOf(10));
args.put("enablePositionIncrements", "true");
factory.init(args);
- stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader(test)));
+ stream = factory.create(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "foobar" }, new int[] { 2 });
}
}
\ No newline at end of file
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/SnowballPorterFilterFactoryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/SnowballPorterFilterFactoryTest.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/SnowballPorterFilterFactoryTest.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/SnowballPorterFilterFactoryTest.java Mon Sep 26 16:25:51 2011
@@ -16,9 +16,9 @@ package org.apache.solr.analysis;
* limitations under the License.
*/
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.solr.common.ResourceLoader;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.core.SolrResourceLoader;
@@ -52,8 +52,8 @@ public class SnowballPorterFilterFactory
factory.init(args);
factory.inform(new LinesMockSolrResourceLoader(new ArrayList<String>()));
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION,
- new StringReader(StrUtils.join(Arrays.asList(test), ' ')));
+ Tokenizer tokenizer = new MockTokenizer(
+ new StringReader(StrUtils.join(Arrays.asList(test), ' ')), MockTokenizer.WHITESPACE, false);
TokenStream stream = factory.create(tokenizer);
assertTokenStreamContents(stream, gold);
}
@@ -90,7 +90,7 @@ public class SnowballPorterFilterFactory
factory.init(args);
factory.inform(loader);
Reader reader = new StringReader("ridding of some stemming");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
TokenStream stream = factory.create(tokenizer);
assertTokenStreamContents(stream, new String[] { "ridding", "of", "some", "stem" });
}
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestBrazilianStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestBrazilianStemFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestBrazilianStemFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestBrazilianStemFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -20,9 +20,9 @@ package org.apache.solr.analysis;
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
/**
* Simple tests to ensure the Brazilian stem filter factory is working.
@@ -33,7 +33,7 @@ public class TestBrazilianStemFilterFact
*/
public void testStemming() throws Exception {
Reader reader = new StringReader("BrasÃlia");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
BrazilianStemFilterFactory factory = new BrazilianStemFilterFactory();
TokenStream stream = factory.create(tokenizer);
assertTokenStreamContents(stream, new String[] { "brasil" });
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestBulgarianStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestBulgarianStemFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestBulgarianStemFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestBulgarianStemFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -20,9 +20,9 @@ package org.apache.solr.analysis;
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
/**
* Simple tests to ensure the Bulgarian stem filter factory is working.
@@ -33,7 +33,7 @@ public class TestBulgarianStemFilterFact
*/
public void testStemming() throws Exception {
Reader reader = new StringReader("компÑÑÑи");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
BulgarianStemFilterFactory factory = new BulgarianStemFilterFactory();
TokenStream stream = factory.create(tokenizer);
assertTokenStreamContents(stream, new String[] { "компÑÑÑ" });
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestCapitalizationFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestCapitalizationFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestCapitalizationFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestCapitalizationFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -21,10 +21,9 @@ import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
/**
@@ -41,56 +40,56 @@ public class TestCapitalizationFilterFac
CapitalizationFilterFactory factory = new CapitalizationFilterFactory();
factory.init( args );
assertTokenStreamContents(factory.create(
- new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader("kiTTEN"))),
+ new MockTokenizer(new StringReader("kiTTEN"), MockTokenizer.WHITESPACE, false)),
new String[] { "Kitten" });
factory.forceFirstLetter = true;
assertTokenStreamContents(factory.create(
- new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader("and"))),
+ new MockTokenizer(new StringReader("and"), MockTokenizer.WHITESPACE, false)),
new String[] { "And" });
//first is forced, but it's not a keep word, either
assertTokenStreamContents(factory.create(
- new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader("AnD"))),
+ new MockTokenizer(new StringReader("AnD"), MockTokenizer.WHITESPACE, false)),
new String[] { "And" });
factory.forceFirstLetter = false;
//first is not forced, but it's not a keep word, either
assertTokenStreamContents(factory.create(
- new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader("AnD"))),
+ new MockTokenizer(new StringReader("AnD"), MockTokenizer.WHITESPACE, false)),
new String[] { "And" });
factory.forceFirstLetter = true;
assertTokenStreamContents(factory.create(
- new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader("big"))),
+ new MockTokenizer(new StringReader("big"), MockTokenizer.WHITESPACE, false)),
new String[] { "Big" });
assertTokenStreamContents(factory.create(
- new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader("BIG"))),
+ new MockTokenizer(new StringReader("BIG"), MockTokenizer.WHITESPACE, false)),
new String[] { "BIG" });
assertTokenStreamContents(factory.create(
- new KeywordTokenizer(new StringReader("Hello thEre my Name is Ryan"))),
+ new MockTokenizer(new StringReader("Hello thEre my Name is Ryan"), MockTokenizer.KEYWORD, false)),
new String[] { "Hello there my name is ryan" });
// now each token
factory.onlyFirstWord = false;
assertTokenStreamContents(factory.create(
- new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader("Hello thEre my Name is Ryan"))),
+ new MockTokenizer(new StringReader("Hello thEre my Name is Ryan"), MockTokenizer.WHITESPACE, false)),
new String[] { "Hello", "There", "My", "Name", "Is", "Ryan" });
// now only the long words
factory.minWordLength = 3;
assertTokenStreamContents(factory.create(
- new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader("Hello thEre my Name is Ryan"))),
+ new MockTokenizer(new StringReader("Hello thEre my Name is Ryan"), MockTokenizer.WHITESPACE, false)),
new String[] { "Hello", "There", "my", "Name", "is", "Ryan" });
// without prefix
assertTokenStreamContents(factory.create(
- new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader("McKinley"))),
+ new MockTokenizer(new StringReader("McKinley"), MockTokenizer.WHITESPACE, false)),
new String[] { "Mckinley" });
// Now try some prefixes
@@ -98,19 +97,19 @@ public class TestCapitalizationFilterFac
args.put( "okPrefix", "McK" ); // all words
factory.init( args );
assertTokenStreamContents(factory.create(
- new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader("McKinley"))),
+ new MockTokenizer(new StringReader("McKinley"), MockTokenizer.WHITESPACE, false)),
new String[] { "McKinley" });
// now try some stuff with numbers
factory.forceFirstLetter = false;
factory.onlyFirstWord = false;
assertTokenStreamContents(factory.create(
- new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader("1st 2nd third"))),
+ new MockTokenizer(new StringReader("1st 2nd third"), MockTokenizer.WHITESPACE, false)),
new String[] { "1st", "2nd", "Third" });
factory.forceFirstLetter = true;
assertTokenStreamContents(factory.create(
- new KeywordTokenizer(new StringReader("the The the"))),
+ new MockTokenizer(new StringReader("the The the"), MockTokenizer.KEYWORD, false)),
new String[] { "The The the" });
}
@@ -124,17 +123,17 @@ public class TestCapitalizationFilterFac
factory.init( args );
factory.forceFirstLetter = true;
assertTokenStreamContents(factory.create(
- new KeywordTokenizer(new StringReader("kiTTEN"))),
+ new MockTokenizer(new StringReader("kiTTEN"), MockTokenizer.KEYWORD, false)),
new String[] { "KiTTEN" });
factory.forceFirstLetter = false;
assertTokenStreamContents(factory.create(
- new KeywordTokenizer(new StringReader("kiTTEN"))),
+ new MockTokenizer(new StringReader("kiTTEN"), MockTokenizer.KEYWORD, false)),
new String[] { "kiTTEN" });
factory.keep = null;
assertTokenStreamContents(factory.create(
- new KeywordTokenizer(new StringReader("kiTTEN"))),
+ new MockTokenizer(new StringReader("kiTTEN"), MockTokenizer.KEYWORD, false)),
new String[] { "Kitten" });
}
@@ -149,8 +148,8 @@ public class TestCapitalizationFilterFac
args.put(CapitalizationFilterFactory.MIN_WORD_LENGTH, "5");
CapitalizationFilterFactory factory = new CapitalizationFilterFactory();
factory.init(args);
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader(
- "helo testing"));
+ Tokenizer tokenizer = new MockTokenizer(new StringReader(
+ "helo testing"), MockTokenizer.WHITESPACE, false);
TokenStream ts = factory.create(tokenizer);
assertTokenStreamContents(ts, new String[] {"helo", "Testing"});
}
@@ -164,8 +163,8 @@ public class TestCapitalizationFilterFac
args.put(CapitalizationFilterFactory.MAX_WORD_COUNT, "2");
CapitalizationFilterFactory factory = new CapitalizationFilterFactory();
factory.init(args);
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader(
- "one two three four"));
+ Tokenizer tokenizer = new MockTokenizer(new StringReader(
+ "one two three four"), MockTokenizer.WHITESPACE, false);
TokenStream ts = factory.create(tokenizer);
assertTokenStreamContents(ts, new String[] {"One", "Two", "Three", "Four"});
}
@@ -178,8 +177,8 @@ public class TestCapitalizationFilterFac
args.put(CapitalizationFilterFactory.MAX_WORD_COUNT, "2");
CapitalizationFilterFactory factory = new CapitalizationFilterFactory();
factory.init(args);
- Tokenizer tokenizer = new KeywordTokenizer(new StringReader(
- "one two three four"));
+ Tokenizer tokenizer = new MockTokenizer(new StringReader(
+ "one two three four"), MockTokenizer.KEYWORD, false);
TokenStream ts = factory.create(tokenizer);
assertTokenStreamContents(ts, new String[] {"one two three four"});
}
@@ -194,8 +193,8 @@ public class TestCapitalizationFilterFac
args.put(CapitalizationFilterFactory.MAX_TOKEN_LENGTH, "2");
CapitalizationFilterFactory factory = new CapitalizationFilterFactory();
factory.init(args);
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader(
- "this is a test"));
+ Tokenizer tokenizer = new MockTokenizer(new StringReader(
+ "this is a test"), MockTokenizer.WHITESPACE, false);
TokenStream ts = factory.create(tokenizer);
assertTokenStreamContents(ts, new String[] {"this", "is", "A", "test"});
}
@@ -209,7 +208,7 @@ public class TestCapitalizationFilterFac
args.put(CapitalizationFilterFactory.FORCE_FIRST_LETTER, "true");
CapitalizationFilterFactory factory = new CapitalizationFilterFactory();
factory.init(args);
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader("kitten"));
+ Tokenizer tokenizer = new MockTokenizer(new StringReader("kitten"), MockTokenizer.WHITESPACE, false);
TokenStream ts = factory.create(tokenizer);
assertTokenStreamContents(ts, new String[] {"Kitten"});
}
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestChineseFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestChineseFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestChineseFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestChineseFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -20,9 +20,9 @@ package org.apache.solr.analysis;
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
/**
* Simple tests to ensure the Chinese filter factory is working.
@@ -33,7 +33,7 @@ public class TestChineseFilterFactory ex
*/
public void testFiltering() throws Exception {
Reader reader = new StringReader("this 1234 Is such a silly filter");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
ChineseFilterFactory factory = new ChineseFilterFactory();
TokenStream stream = factory.create(tokenizer);
assertTokenStreamContents(stream, new String[] { "Is", "silly", "filter" });
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestCollationKeyFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestCollationKeyFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestCollationKeyFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestCollationKeyFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -28,8 +28,8 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.solr.common.ResourceLoader;
@@ -51,9 +51,9 @@ public class TestCollationKeyFilterFacto
factory.init(args);
factory.inform(new StringMockSolrResourceLoader(""));
TokenStream tsUpper = factory.create(
- new KeywordTokenizer(new StringReader(turkishUpperCase)));
+ new MockTokenizer(new StringReader(turkishUpperCase), MockTokenizer.KEYWORD, false));
TokenStream tsLower = factory.create(
- new KeywordTokenizer(new StringReader(turkishLowerCase)));
+ new MockTokenizer(new StringReader(turkishLowerCase), MockTokenizer.KEYWORD, false));
assertCollatesToSame(tsUpper, tsLower);
}
@@ -71,9 +71,9 @@ public class TestCollationKeyFilterFacto
factory.init(args);
factory.inform(new StringMockSolrResourceLoader(""));
TokenStream tsUpper = factory.create(
- new KeywordTokenizer(new StringReader(turkishUpperCase)));
+ new MockTokenizer(new StringReader(turkishUpperCase), MockTokenizer.KEYWORD, false));
TokenStream tsLower = factory.create(
- new KeywordTokenizer(new StringReader(turkishLowerCase)));
+ new MockTokenizer(new StringReader(turkishLowerCase), MockTokenizer.KEYWORD, false));
assertCollatesToSame(tsUpper, tsLower);
}
@@ -92,9 +92,9 @@ public class TestCollationKeyFilterFacto
factory.init(args);
factory.inform(new StringMockSolrResourceLoader(""));
TokenStream tsFull = factory.create(
- new KeywordTokenizer(new StringReader(fullWidth)));
+ new MockTokenizer(new StringReader(fullWidth), MockTokenizer.KEYWORD, false));
TokenStream tsHalf = factory.create(
- new KeywordTokenizer(new StringReader(halfWidth)));
+ new MockTokenizer(new StringReader(halfWidth), MockTokenizer.KEYWORD, false));
assertCollatesToSame(tsFull, tsHalf);
}
@@ -112,9 +112,9 @@ public class TestCollationKeyFilterFacto
factory.init(args);
factory.inform(new StringMockSolrResourceLoader(""));
TokenStream tsUpper = factory.create(
- new KeywordTokenizer(new StringReader(upperCase)));
+ new MockTokenizer(new StringReader(upperCase), MockTokenizer.KEYWORD, false));
TokenStream tsLower = factory.create(
- new KeywordTokenizer(new StringReader(lowerCase)));
+ new MockTokenizer(new StringReader(lowerCase), MockTokenizer.KEYWORD, false));
assertCollatesToSame(tsUpper, tsLower);
}
@@ -148,9 +148,9 @@ public class TestCollationKeyFilterFacto
factory.init(args);
factory.inform(new StringMockSolrResourceLoader(tailoredRules));
TokenStream tsUmlaut = factory.create(
- new KeywordTokenizer(new StringReader(germanUmlaut)));
+ new MockTokenizer(new StringReader(germanUmlaut), MockTokenizer.KEYWORD, false));
TokenStream tsOE = factory.create(
- new KeywordTokenizer(new StringReader(germanOE)));
+ new MockTokenizer(new StringReader(germanOE), MockTokenizer.KEYWORD, false));
assertCollatesToSame(tsUmlaut, tsOE);
}
@@ -177,6 +177,8 @@ public class TestCollationKeyFilterFacto
private void assertCollatesToSame(TokenStream stream1, TokenStream stream2)
throws IOException {
+ stream1.reset();
+ stream2.reset();
CharTermAttribute term1 = stream1
.addAttribute(CharTermAttribute.class);
CharTermAttribute term2 = stream2
@@ -186,5 +188,9 @@ public class TestCollationKeyFilterFacto
assertEquals(term1.toString(), term2.toString());
assertFalse(stream1.incrementToken());
assertFalse(stream2.incrementToken());
+ stream1.end();
+ stream2.end();
+ stream1.close();
+ stream2.close();
}
}
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestCzechStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestCzechStemFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestCzechStemFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestCzechStemFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -20,9 +20,9 @@ package org.apache.solr.analysis;
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
/**
* Simple tests to ensure the Czech stem filter factory is working.
@@ -33,7 +33,7 @@ public class TestCzechStemFilterFactory
*/
public void testStemming() throws Exception {
Reader reader = new StringReader("angliÄtÃ");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
CzechStemFilterFactory factory = new CzechStemFilterFactory();
TokenStream stream = factory.create(tokenizer);
assertTokenStreamContents(stream, new String[] { "anglick" });
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestDelimitedPayloadTokenFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestDelimitedPayloadTokenFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestDelimitedPayloadTokenFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestDelimitedPayloadTokenFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -21,8 +21,8 @@ import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.payloads.DelimitedPayloadTokenFilter;
import org.apache.lucene.analysis.payloads.FloatEncoder;
import org.apache.lucene.analysis.payloads.PayloadHelper;
@@ -40,8 +40,9 @@ public class TestDelimitedPayloadTokenFi
ResourceLoader loader = new SolrResourceLoader(null, null);
factory.inform(loader);
- TokenStream input = new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader("the|0.1 quick|0.1 red|0.1"));
+ TokenStream input = new MockTokenizer(new StringReader("the|0.1 quick|0.1 red|0.1"), MockTokenizer.WHITESPACE, false);
DelimitedPayloadTokenFilter tf = factory.create(input);
+ tf.reset();
while (tf.incrementToken()){
PayloadAttribute payAttr = tf.getAttribute(PayloadAttribute.class);
assertTrue("payAttr is null and it shouldn't be", payAttr != null);
@@ -62,8 +63,9 @@ public class TestDelimitedPayloadTokenFi
ResourceLoader loader = new SolrResourceLoader(null, null);
factory.inform(loader);
- TokenStream input = new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader("the*0.1 quick*0.1 red*0.1"));
+ TokenStream input = new MockTokenizer(new StringReader("the*0.1 quick*0.1 red*0.1"), MockTokenizer.WHITESPACE, false);
DelimitedPayloadTokenFilter tf = factory.create(input);
+ tf.reset();
while (tf.incrementToken()){
PayloadAttribute payAttr = tf.getAttribute(PayloadAttribute.class);
assertTrue("payAttr is null and it shouldn't be", payAttr != null);
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestDictionaryCompoundWordTokenFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestDictionaryCompoundWordTokenFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestDictionaryCompoundWordTokenFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestDictionaryCompoundWordTokenFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -22,9 +22,9 @@ import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.solr.common.ResourceLoader;
import org.apache.solr.core.SolrResourceLoader;
@@ -37,7 +37,7 @@ public class TestDictionaryCompoundWordT
*/
public void testDecompounding() throws Exception {
Reader reader = new StringReader("I like to play softball");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
DictionaryCompoundWordTokenFilterFactory factory = new DictionaryCompoundWordTokenFilterFactory();
ResourceLoader loader = new SolrResourceLoader(null, null);
Map<String,String> args = new HashMap<String,String>(DEFAULT_VERSION_PARAM);
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestElisionFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestElisionFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestElisionFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestElisionFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -22,9 +22,9 @@ import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.solr.common.ResourceLoader;
import org.apache.solr.core.SolrResourceLoader;
@@ -37,7 +37,7 @@ public class TestElisionFilterFactory ex
*/
public void testElision() throws Exception {
Reader reader = new StringReader("l'avion");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
ElisionFilterFactory factory = new ElisionFilterFactory();
factory.init(DEFAULT_VERSION_PARAM);
ResourceLoader loader = new SolrResourceLoader(null, null);
@@ -54,7 +54,7 @@ public class TestElisionFilterFactory ex
*/
public void testDefaultArticles() throws Exception {
Reader reader = new StringReader("l'avion");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
ElisionFilterFactory factory = new ElisionFilterFactory();
factory.init(DEFAULT_VERSION_PARAM);
ResourceLoader loader = new SolrResourceLoader(null, null);
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestEnglishMinimalStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestEnglishMinimalStemFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestEnglishMinimalStemFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestEnglishMinimalStemFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -20,8 +20,8 @@ package org.apache.solr.analysis;
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
/**
* Simple tests to ensure the English minimal stem factory is working.
@@ -30,7 +30,7 @@ public class TestEnglishMinimalStemFilte
public void testStemming() throws Exception {
Reader reader = new StringReader("bricks");
EnglishMinimalStemFilterFactory factory = new EnglishMinimalStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "brick" });
}
}
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestFinnishLightStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestFinnishLightStemFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestFinnishLightStemFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestFinnishLightStemFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -20,8 +20,8 @@ package org.apache.solr.analysis;
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
/**
* Simple tests to ensure the Finnish light stem factory is working.
@@ -30,7 +30,7 @@ public class TestFinnishLightStemFilterF
public void testStemming() throws Exception {
Reader reader = new StringReader("aseistettujen");
FinnishLightStemFilterFactory factory = new FinnishLightStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "aseistet" });
}
}
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestFrenchLightStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestFrenchLightStemFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestFrenchLightStemFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestFrenchLightStemFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -20,8 +20,8 @@ package org.apache.solr.analysis;
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
/**
* Simple tests to ensure the French light stem factory is working.
@@ -30,7 +30,7 @@ public class TestFrenchLightStemFilterFa
public void testStemming() throws Exception {
Reader reader = new StringReader("administrativement");
FrenchLightStemFilterFactory factory = new FrenchLightStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "administratif" });
}
}
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestFrenchMinimalStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestFrenchMinimalStemFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestFrenchMinimalStemFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestFrenchMinimalStemFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -20,8 +20,8 @@ package org.apache.solr.analysis;
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
/**
* Simple tests to ensure the French minimal stem factory is working.
@@ -30,7 +30,7 @@ public class TestFrenchMinimalStemFilter
public void testStemming() throws Exception {
Reader reader = new StringReader("chevaux");
FrenchMinimalStemFilterFactory factory = new FrenchMinimalStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "cheval" });
}
}
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestGalicianStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestGalicianStemFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestGalicianStemFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestGalicianStemFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -20,8 +20,8 @@ package org.apache.solr.analysis;
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
/**
* Simple tests to ensure the Galician stem factory is working.
@@ -30,7 +30,7 @@ public class TestGalicianStemFilterFacto
public void testStemming() throws Exception {
Reader reader = new StringReader("cariñosa");
GalicianStemFilterFactory factory = new GalicianStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "cariñ" });
}
}
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestGermanLightStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestGermanLightStemFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestGermanLightStemFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestGermanLightStemFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -20,8 +20,8 @@ package org.apache.solr.analysis;
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
/**
* Simple tests to ensure the German light stem factory is working.
@@ -30,7 +30,7 @@ public class TestGermanLightStemFilterFa
public void testStemming() throws Exception {
Reader reader = new StringReader("häuser");
GermanLightStemFilterFactory factory = new GermanLightStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "haus" });
}
}
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestGermanMinimalStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestGermanMinimalStemFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestGermanMinimalStemFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestGermanMinimalStemFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -20,8 +20,8 @@ package org.apache.solr.analysis;
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
/**
* Simple tests to ensure the German minimal stem factory is working.
@@ -30,7 +30,7 @@ public class TestGermanMinimalStemFilter
public void testStemming() throws Exception {
Reader reader = new StringReader("bilder");
GermanMinimalStemFilterFactory factory = new GermanMinimalStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "bild" });
}
}
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestGermanStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestGermanStemFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestGermanStemFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestGermanStemFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -20,9 +20,9 @@ package org.apache.solr.analysis;
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
/**
* Simple tests to ensure the German stem filter factory is working.
@@ -33,7 +33,7 @@ public class TestGermanStemFilterFactory
*/
public void testStemming() throws Exception {
Reader reader = new StringReader("Tischen");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
GermanStemFilterFactory factory = new GermanStemFilterFactory();
TokenStream stream = factory.create(tokenizer);
assertTokenStreamContents(stream, new String[] { "tisch" });
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestGreekLowerCaseFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestGreekLowerCaseFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestGreekLowerCaseFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestGreekLowerCaseFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -20,9 +20,9 @@ package org.apache.solr.analysis;
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
/**
* Simple tests to ensure the Greek lowercase filter factory is working.
@@ -33,7 +33,7 @@ public class TestGreekLowerCaseFilterFac
*/
public void testNormalization() throws Exception {
Reader reader = new StringReader("ÎάÏÎ¿Ï ÎÎΪÎΣ");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
GreekLowerCaseFilterFactory factory = new GreekLowerCaseFilterFactory();
factory.init(DEFAULT_VERSION_PARAM);
TokenStream stream = factory.create(tokenizer);
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestGreekStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestGreekStemFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestGreekStemFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestGreekStemFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -3,9 +3,9 @@ package org.apache.solr.analysis;
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.el.GreekLowerCaseFilter;
/**
@@ -31,7 +31,7 @@ import org.apache.lucene.analysis.el.Gre
public class TestGreekStemFilterFactory extends BaseTokenTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("άνθÏÏÏοÏ");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
TokenStream normalized = new GreekLowerCaseFilter(DEFAULT_VERSION, tokenizer);
GreekStemFilterFactory factory = new GreekStemFilterFactory();
TokenStream stream = factory.create(normalized);
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestHungarianLightStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestHungarianLightStemFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestHungarianLightStemFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestHungarianLightStemFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -20,8 +20,8 @@ package org.apache.solr.analysis;
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
/**
* Simple tests to ensure the Hungarian light stem factory is working.
@@ -30,7 +30,7 @@ public class TestHungarianLightStemFilte
public void testStemming() throws Exception {
Reader reader = new StringReader("házakat");
HungarianLightStemFilterFactory factory = new HungarianLightStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "haz" });
}
}
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestHunspellStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestHunspellStemFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestHunspellStemFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestHunspellStemFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -22,8 +22,8 @@ import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.schema.IndexSchema;
@@ -41,7 +41,7 @@ public class TestHunspellStemFilterFacto
factory.inform(new SolrResourceLoader("solr"));
Reader reader = new StringReader("abc");
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "ab" });
}
}
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestHyphenationCompoundWordTokenFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestHyphenationCompoundWordTokenFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestHyphenationCompoundWordTokenFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestHyphenationCompoundWordTokenFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -22,9 +22,9 @@ import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.solr.common.ResourceLoader;
import org.apache.solr.core.SolrResourceLoader;
@@ -37,7 +37,7 @@ public class TestHyphenationCompoundWord
*/
public void testHyphenationWithDictionary() throws Exception {
Reader reader = new StringReader("min veninde som er lidt af en læsehest");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
HyphenationCompoundWordTokenFilterFactory factory = new HyphenationCompoundWordTokenFilterFactory();
ResourceLoader loader = new SolrResourceLoader(null, null);
Map<String,String> args = new HashMap<String,String>(DEFAULT_VERSION_PARAM);
@@ -60,7 +60,7 @@ public class TestHyphenationCompoundWord
*/
public void testHyphenationOnly() throws Exception {
Reader reader = new StringReader("basketballkurv");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
HyphenationCompoundWordTokenFilterFactory factory = new HyphenationCompoundWordTokenFilterFactory();
ResourceLoader loader = new SolrResourceLoader(null, null);
Map<String,String> args = new HashMap<String,String>(DEFAULT_VERSION_PARAM);
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestIndonesianStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestIndonesianStemFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestIndonesianStemFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestIndonesianStemFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -22,9 +22,9 @@ import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
/**
* Simple tests to ensure the Indonesian stem filter factory is working.
@@ -35,7 +35,7 @@ public class TestIndonesianStemFilterFac
*/
public void testStemming() throws Exception {
Reader reader = new StringReader("dibukukannya");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
IndonesianStemFilterFactory factory = new IndonesianStemFilterFactory();
Map<String,String> args = new HashMap<String,String>();
factory.init(args);
@@ -48,7 +48,7 @@ public class TestIndonesianStemFilterFac
*/
public void testStemmingInflectional() throws Exception {
Reader reader = new StringReader("dibukukannya");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
IndonesianStemFilterFactory factory = new IndonesianStemFilterFactory();
Map<String,String> args = new HashMap<String,String>();
args.put("stemDerivational", "false");
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestItalianLightStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestItalianLightStemFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestItalianLightStemFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestItalianLightStemFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -20,8 +20,8 @@ package org.apache.solr.analysis;
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
/**
* Simple tests to ensure the Italian light stem factory is working.
@@ -30,7 +30,7 @@ public class TestItalianLightStemFilterF
public void testStemming() throws Exception {
Reader reader = new StringReader("ragazzo ragazzi");
ItalianLightStemFilterFactory factory = new ItalianLightStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "ragazz", "ragazz" });
}
}
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestKStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestKStemFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestKStemFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestKStemFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -3,8 +3,8 @@ package org.apache.solr.analysis;
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -30,7 +30,7 @@ public class TestKStemFilterFactory exte
public void testStemming() throws Exception {
Reader reader = new StringReader("bricks");
KStemFilterFactory factory = new KStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "brick" });
}
}
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestKeywordMarkerFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestKeywordMarkerFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestKeywordMarkerFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestKeywordMarkerFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -23,8 +23,8 @@ import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.en.PorterStemFilter;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.solr.common.ResourceLoader;
@@ -36,7 +36,7 @@ import org.apache.solr.core.SolrResource
public class TestKeywordMarkerFilterFactory extends BaseTokenTestCase {
public void testKeywords() throws IOException {
Reader reader = new StringReader("dogs cats");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
KeywordMarkerFilterFactory factory = new KeywordMarkerFilterFactory();
Map<String,String> args = new HashMap<String,String>(DEFAULT_VERSION_PARAM);
ResourceLoader loader = new SolrResourceLoader(null, null);
@@ -50,7 +50,7 @@ public class TestKeywordMarkerFilterFact
public void testKeywordsCaseInsensitive() throws IOException {
Reader reader = new StringReader("dogs cats Cats");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
KeywordMarkerFilterFactory factory = new KeywordMarkerFilterFactory();
Map<String,String> args = new HashMap<String,String>(DEFAULT_VERSION_PARAM);
ResourceLoader loader = new SolrResourceLoader(null, null);
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestLatvianStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestLatvianStemFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestLatvianStemFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestLatvianStemFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -20,8 +20,8 @@ package org.apache.solr.analysis;
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
/**
* Simple tests to ensure the Latvian stem factory is working.
@@ -30,7 +30,7 @@ public class TestLatvianStemFilterFactor
public void testStemming() throws Exception {
Reader reader = new StringReader("tirgiem tirgus");
LatvianStemFilterFactory factory = new LatvianStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "tirg", "tirg" });
}
}
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestMultiWordSynonyms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestMultiWordSynonyms.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestMultiWordSynonyms.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestMultiWordSynonyms.java Mon Sep 26 16:25:51 2011
@@ -19,7 +19,6 @@ package org.apache.solr.analysis;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.solr.common.ResourceLoader;
import java.io.ByteArrayInputStream;
@@ -46,7 +45,7 @@ public class TestMultiWordSynonyms exten
SlowSynonymMap synMap = new SlowSynonymMap(true);
SlowSynonymFilterFactory.parseRules(rules, synMap, "=>", ",", true, null);
- SlowSynonymFilter ts = new SlowSynonymFilter(new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader("a e")), synMap);
+ SlowSynonymFilter ts = new SlowSynonymFilter(new MockTokenizer(new StringReader("a e"), MockTokenizer.WHITESPACE, false), synMap);
// This fails because ["e","e"] is the value of the token stream
assertTokenStreamContents(ts, new String[] { "a", "e" });
}
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestNGramFilters.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestNGramFilters.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestNGramFilters.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestNGramFilters.java Mon Sep 26 16:25:51 2011
@@ -22,9 +22,9 @@ import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
/**
* Simple tests to ensure the NGram filter factories are working.
@@ -64,7 +64,7 @@ public class TestNGramFilters extends Ba
Map<String,String> args = new HashMap<String,String>();
NGramFilterFactory factory = new NGramFilterFactory();
factory.init(args);
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream,
new String[] { "t", "e", "s", "t", "te", "es", "st" });
}
@@ -78,7 +78,7 @@ public class TestNGramFilters extends Ba
args.put("maxGramSize", "3");
NGramFilterFactory factory = new NGramFilterFactory();
factory.init(args);
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream,
new String[] { "te", "es", "st", "tes", "est" });
}
@@ -129,7 +129,7 @@ public class TestNGramFilters extends Ba
Map<String,String> args = new HashMap<String,String>();
EdgeNGramFilterFactory factory = new EdgeNGramFilterFactory();
factory.init(args);
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream,
new String[] { "t" });
}
@@ -143,7 +143,7 @@ public class TestNGramFilters extends Ba
args.put("maxGramSize", "2");
EdgeNGramFilterFactory factory = new EdgeNGramFilterFactory();
factory.init(args);
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream,
new String[] { "t", "te" });
}
@@ -156,7 +156,7 @@ public class TestNGramFilters extends Ba
args.put("side", "back");
EdgeNGramFilterFactory factory = new EdgeNGramFilterFactory();
factory.init(args);
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream,
new String[] { "y" });
}
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPatternReplaceCharFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPatternReplaceCharFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPatternReplaceCharFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPatternReplaceCharFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -24,8 +24,8 @@ import java.util.Map;
import org.apache.lucene.analysis.CharReader;
import org.apache.lucene.analysis.CharStream;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
/**
* Simple tests to ensure this factory is working
@@ -44,7 +44,7 @@ public class TestPatternReplaceCharFilte
factory.init(args);
CharStream cs = factory.create(
CharReader.get( new StringReader( BLOCK ) ) );
- TokenStream ts = new WhitespaceTokenizer(DEFAULT_VERSION, cs );
+ TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
assertTokenStreamContents(ts,
new String[] { "this", "is", "test." },
new int[] { 0, 5, 8 },
@@ -61,8 +61,11 @@ public class TestPatternReplaceCharFilte
factory.init(args);
CharStream cs = factory.create(
CharReader.get( new StringReader( BLOCK ) ) );
- TokenStream ts = new WhitespaceTokenizer(DEFAULT_VERSION, cs );
+ TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ ts.reset();
assertFalse(ts.incrementToken());
+ ts.end();
+ ts.close();
}
// 012345678
@@ -77,7 +80,7 @@ public class TestPatternReplaceCharFilte
factory.init(args);
CharStream cs = factory.create(
CharReader.get( new StringReader( BLOCK ) ) );
- TokenStream ts = new WhitespaceTokenizer(DEFAULT_VERSION, cs );
+ TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
assertTokenStreamContents(ts,
new String[] { "aa#bb#cc" },
new int[] { 0 },
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPatternReplaceFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPatternReplaceFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPatternReplaceFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPatternReplaceFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -17,8 +17,8 @@
package org.apache.solr.analysis;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import java.io.StringReader;
import java.util.HashMap;
@@ -37,7 +37,7 @@ public class TestPatternReplaceFilterFac
args.put("replacement", "-");
factory.init(args);
TokenStream ts = factory.create
- (new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader(input)));
+ (new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(ts,
new String[] { "-foo-foo-foo-", "-", "c-" });
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPersianNormalizationFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPersianNormalizationFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPersianNormalizationFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPersianNormalizationFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -20,9 +20,9 @@ package org.apache.solr.analysis;
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
/**
* Simple tests to ensure the Persian normalization factory is working.
@@ -33,7 +33,7 @@ public class TestPersianNormalizationFil
*/
public void testNormalization() throws Exception {
Reader reader = new StringReader("ÙاÛ");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
PersianNormalizationFilterFactory factory = new PersianNormalizationFilterFactory();
TokenStream stream = factory.create(tokenizer);
assertTokenStreamContents(stream, new String[] { "ÙاÙ" });
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPhoneticFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPhoneticFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPhoneticFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPhoneticFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -22,9 +22,9 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.commons.codec.language.Metaphone;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
/**
@@ -89,8 +89,7 @@ public class TestPhoneticFilterFactory e
static void assertAlgorithm(String algName, String inject, String input,
String[] expected) throws Exception {
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION,
- new StringReader(input));
+ Tokenizer tokenizer = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
Map<String,String> args = new HashMap<String,String>();
args.put("encoder", algName);
args.put("inject", inject);
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPorterStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPorterStemFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPorterStemFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPorterStemFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -20,9 +20,9 @@ package org.apache.solr.analysis;
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
/**
* Simple tests to ensure the Porter stem filter factory is working.
@@ -33,7 +33,7 @@ public class TestPorterStemFilterFactory
*/
public void testStemming() throws Exception {
Reader reader = new StringReader("dogs");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
PorterStemFilterFactory factory = new PorterStemFilterFactory();
TokenStream stream = factory.create(tokenizer);
assertTokenStreamContents(stream, new String[] { "dog" });
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPortugueseLightStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPortugueseLightStemFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPortugueseLightStemFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPortugueseLightStemFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -20,8 +20,8 @@ package org.apache.solr.analysis;
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
/**
* Simple tests to ensure the Portuguese Light stem factory is working.
@@ -30,7 +30,7 @@ public class TestPortugueseLightStemFilt
public void testStemming() throws Exception {
Reader reader = new StringReader("evidentemente");
PortugueseLightStemFilterFactory factory = new PortugueseLightStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "evident" });
}
}
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPortugueseMinimalStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPortugueseMinimalStemFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPortugueseMinimalStemFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPortugueseMinimalStemFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -20,8 +20,8 @@ package org.apache.solr.analysis;
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
/**
* Simple tests to ensure the Portuguese Minimal stem factory is working.
@@ -30,7 +30,7 @@ public class TestPortugueseMinimalStemFi
public void testStemming() throws Exception {
Reader reader = new StringReader("questões");
PortugueseMinimalStemFilterFactory factory = new PortugueseMinimalStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "questão" });
}
}
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPortugueseStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPortugueseStemFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPortugueseStemFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestPortugueseStemFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -20,8 +20,8 @@ package org.apache.solr.analysis;
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
/**
* Simple tests to ensure the Portuguese stem factory is working.
@@ -30,7 +30,7 @@ public class TestPortugueseStemFilterFac
public void testStemming() throws Exception {
Reader reader = new StringReader("maluquice");
PortugueseStemFilterFactory factory = new PortugueseStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "maluc" });
}
}
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestReverseStringFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestReverseStringFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestReverseStringFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestReverseStringFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -20,9 +20,9 @@ package org.apache.solr.analysis;
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
/**
* Simple tests to ensure the Reverse string filter factory is working.
@@ -33,7 +33,7 @@ public class TestReverseStringFilterFact
*/
public void testReversing() throws Exception {
Reader reader = new StringReader("simple test");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
ReverseStringFilterFactory factory = new ReverseStringFilterFactory();
factory.init(DEFAULT_VERSION_PARAM);
TokenStream stream = factory.create(tokenizer);
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -25,8 +25,8 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.search.AutomatonQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.automaton.Automaton;
@@ -66,7 +66,7 @@ public class TestReversedWildcardFilterF
String text = "simple text";
args.put("withOriginal", "true");
factory.init(args);
- TokenStream input = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader(text)));
+ TokenStream input = factory.create(new MockTokenizer(new StringReader(text), MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(input,
new String[] { "\u0001elpmis", "simple", "\u0001txet", "text" },
new int[] { 1, 0, 1, 0 });
@@ -74,7 +74,7 @@ public class TestReversedWildcardFilterF
// now without original tokens
args.put("withOriginal", "false");
factory.init(args);
- input = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader(text)));
+ input = factory.create(new MockTokenizer(new StringReader(text), MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(input,
new String[] { "\u0001elpmis", "\u0001txet" },
new int[] { 1, 1 });
@@ -86,7 +86,7 @@ public class TestReversedWildcardFilterF
String text = "one two three si\uD834\uDD1Ex";
// field one
- TokenStream input = a.tokenStream("one", new StringReader(text));
+ TokenStream input = a.reusableTokenStream("one", new StringReader(text));
assertTokenStreamContents(input,
new String[] { "\u0001eno", "one", "\u0001owt", "two",
"\u0001eerht", "three", "\u0001x\uD834\uDD1Eis", "si\uD834\uDD1Ex" },
@@ -95,7 +95,7 @@ public class TestReversedWildcardFilterF
new int[] { 1, 0, 1, 0, 1, 0, 1, 0 }
);
// field two
- input = a.tokenStream("two", new StringReader(text));
+ input = a.reusableTokenStream("two", new StringReader(text));
assertTokenStreamContents(input,
new String[] { "\u0001eno", "\u0001owt",
"\u0001eerht", "\u0001x\uD834\uDD1Eis" },
@@ -104,7 +104,7 @@ public class TestReversedWildcardFilterF
new int[] { 1, 1, 1, 1 }
);
// field three
- input = a.tokenStream("three", new StringReader(text));
+ input = a.reusableTokenStream("three", new StringReader(text));
assertTokenStreamContents(input,
new String[] { "one", "two", "three", "si\uD834\uDD1Ex" },
new int[] { 0, 4, 8, 14 },
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestRussianLightStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestRussianLightStemFilterFactory.java?rev=1175937&r1=1175936&r2=1175937&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestRussianLightStemFilterFactory.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestRussianLightStemFilterFactory.java Mon Sep 26 16:25:51 2011
@@ -20,8 +20,8 @@ package org.apache.solr.analysis;
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
/**
* Simple tests to ensure the Russian light stem factory is working.
@@ -30,7 +30,7 @@ public class TestRussianLightStemFilterF
public void testStemming() throws Exception {
Reader reader = new StringReader("жÑÑналÑ");
RussianLightStemFilterFactory factory = new RussianLightStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "жÑÑнал" });
}
}