You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ja...@apache.org on 2013/05/30 09:53:46 UTC
svn commit: r1487777 [9/50] - in /lucene/dev/branches/security: ./
dev-tools/ dev-tools/eclipse/dot.settings/ dev-tools/idea/.idea/
dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/replicator/
dev-tools/maven/ dev-tools/maven/lucene/ dev-tools/mav...
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java Thu May 30 07:53:18 2013
@@ -54,41 +54,35 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.wikipedia.WikipediaTokenizer;
import org.apache.lucene.analysis.ValidatingTokenFilter;
import org.apache.lucene.analysis.charfilter.NormalizeCharMap;
import org.apache.lucene.analysis.cjk.CJKBigramFilter;
import org.apache.lucene.analysis.commongrams.CommonGramsFilter;
import org.apache.lucene.analysis.commongrams.CommonGramsQueryFilter;
-import org.apache.lucene.analysis.compound.DictionaryCompoundWordTokenFilter;
import org.apache.lucene.analysis.compound.HyphenationCompoundWordTokenFilter;
import org.apache.lucene.analysis.compound.TestCompoundWordTokenFilter;
import org.apache.lucene.analysis.compound.hyphenation.HyphenationTree;
import org.apache.lucene.analysis.hunspell.HunspellDictionary;
import org.apache.lucene.analysis.hunspell.HunspellDictionaryTest;
import org.apache.lucene.analysis.miscellaneous.HyphenatedWordsFilter;
-import org.apache.lucene.analysis.miscellaneous.KeepWordFilter;
-import org.apache.lucene.analysis.miscellaneous.LengthFilter;
import org.apache.lucene.analysis.miscellaneous.LimitTokenCountFilter;
-import org.apache.lucene.analysis.miscellaneous.TrimFilter;
+import org.apache.lucene.analysis.miscellaneous.LimitTokenPositionFilter;
+import org.apache.lucene.analysis.miscellaneous.StemmerOverrideFilter;
+import org.apache.lucene.analysis.miscellaneous.StemmerOverrideFilter.StemmerOverrideMap;
import org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter;
-import org.apache.lucene.analysis.ngram.EdgeNGramTokenFilter;
-import org.apache.lucene.analysis.ngram.EdgeNGramTokenizer;
-import org.apache.lucene.analysis.ngram.NGramTokenFilter;
-import org.apache.lucene.analysis.ngram.NGramTokenizer;
import org.apache.lucene.analysis.path.PathHierarchyTokenizer;
import org.apache.lucene.analysis.path.ReversePathHierarchyTokenizer;
import org.apache.lucene.analysis.payloads.IdentityEncoder;
import org.apache.lucene.analysis.payloads.PayloadEncoder;
-import org.apache.lucene.analysis.position.PositionFilter;
import org.apache.lucene.analysis.snowball.TestSnowball;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.synonym.SynonymMap;
import org.apache.lucene.analysis.th.ThaiWordFilter;
import org.apache.lucene.analysis.util.CharArrayMap;
import org.apache.lucene.analysis.util.CharArraySet;
-import org.apache.lucene.util.AttributeSource.AttributeFactory;
+import org.apache.lucene.analysis.wikipedia.WikipediaTokenizer;
import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.AttributeSource.AttributeFactory;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.Rethrow;
import org.apache.lucene.util.Version;
@@ -131,6 +125,18 @@ public class TestRandomChains extends Ba
return !((Boolean) args[2]); // args are broken if consumeAllTokens is false
}
});
+ brokenConstructors.put(
+ LimitTokenPositionFilter.class.getConstructor(TokenStream.class, int.class),
+ ALWAYS);
+ brokenConstructors.put(
+ LimitTokenPositionFilter.class.getConstructor(TokenStream.class, int.class, boolean.class),
+ new Predicate<Object[]>() {
+ @Override
+ public boolean apply(Object[] args) {
+ assert args.length == 3;
+ return !((Boolean) args[2]); // args are broken if consumeAllTokens is false
+ }
+ });
for (Class<?> c : Arrays.<Class<?>>asList(
// TODO: can we promote some of these to be only
// offsets offenders?
@@ -139,21 +145,6 @@ public class TestRandomChains extends Ba
// Not broken: we forcefully add this, so we shouldn't
// also randomly pick it:
ValidatingTokenFilter.class,
- // NOTE: these by themselves won't cause any 'basic assertions' to fail.
- // but see https://issues.apache.org/jira/browse/LUCENE-3920, if any
- // tokenfilter that combines words (e.g. shingles) comes after them,
- // this will create bogus offsets because their 'offsets go backwards',
- // causing shingle or whatever to make a single token with a
- // startOffset thats > its endOffset
- // (see LUCENE-3738 for a list of other offenders here)
- // broken!
- NGramTokenizer.class,
- // broken!
- NGramTokenFilter.class,
- // broken!
- EdgeNGramTokenizer.class,
- // broken!
- EdgeNGramTokenFilter.class,
// broken!
WordDelimiterFilter.class)) {
for (Constructor<?> ctor : c.getConstructors()) {
@@ -170,62 +161,9 @@ public class TestRandomChains extends Ba
private static final Map<Constructor<?>,Predicate<Object[]>> brokenOffsetsConstructors = new HashMap<Constructor<?>, Predicate<Object[]>>();
static {
try {
- brokenOffsetsConstructors.put(
- TrimFilter.class.getConstructor(TokenStream.class, boolean.class),
- new Predicate<Object[]>() {
- @Override
- public boolean apply(Object[] args) {
- assert args.length == 2;
- return (Boolean) args[1]; // args are broken if updateOffsets is true
- }
- });
- brokenOffsetsConstructors.put(
- TypeTokenFilter.class.getConstructor(boolean.class, TokenStream.class, Set.class, boolean.class),
- new Predicate<Object[]>() {
- @Override
- public boolean apply(Object[] args) {
- assert args.length == 4;
- // LUCENE-4065: only if you pass 'false' to enablePositionIncrements!
- return !(Boolean) args[0];
- }
- });
- brokenOffsetsConstructors.put(
- TypeTokenFilter.class.getConstructor(boolean.class, TokenStream.class, Set.class),
- new Predicate<Object[]>() {
- @Override
- public boolean apply(Object[] args) {
- assert args.length == 3;
- // LUCENE-4065: only if you pass 'false' to enablePositionIncrements!
- return !(Boolean) args[0];
- }
- });
- brokenOffsetsConstructors.put(
- LengthFilter.class.getConstructor(boolean.class, TokenStream.class, int.class, int.class),
- new Predicate<Object[]>() {
- @Override
- public boolean apply(Object[] args) {
- assert args.length == 4;
- // LUCENE-4065: only if you pass 'false' to enablePositionIncrements!
- return !(Boolean) args[0];
- }
- });
- brokenOffsetsConstructors.put(
- KeepWordFilter.class.getConstructor(boolean.class, TokenStream.class, CharArraySet.class),
- new Predicate<Object[]>() {
- @Override
- public boolean apply(Object[] args) {
- assert args.length == 3;
- // LUCENE-4065: only if you pass 'false' to enablePositionIncrements!
- return !(Boolean) args[0];
- }
- });
for (Class<?> c : Arrays.<Class<?>>asList(
ReversePathHierarchyTokenizer.class,
PathHierarchyTokenizer.class,
- HyphenationCompoundWordTokenFilter.class,
- DictionaryCompoundWordTokenFilter.class,
- // TODO: corrumpts graphs (offset consistency check):
- PositionFilter.class,
// TODO: it seems to mess up offsets!?
WikipediaTokenizer.class,
// TODO: doesn't handle graph inputs
@@ -234,6 +172,8 @@ public class TestRandomChains extends Ba
CJKBigramFilter.class,
// TODO: doesn't handle graph inputs (or even look at positionIncrement)
HyphenatedWordsFilter.class,
+ // TODO: LUCENE-4983
+ CommonGramsFilter.class,
// TODO: doesn't handle graph inputs
CommonGramsQueryFilter.class)) {
for (Constructor<?> ctor : c.getConstructors()) {
@@ -252,14 +192,12 @@ public class TestRandomChains extends Ba
tokenfilters = new ArrayList<Constructor<? extends TokenFilter>>();
charfilters = new ArrayList<Constructor<? extends CharFilter>>();
for (final Class<?> c : analysisClasses) {
- // TODO: Fix below code to use c.isAnnotationPresent(). It was changed
- // to the null check to work around a bug in JDK 8 b78 (see LUCENE-4808).
final int modifiers = c.getModifiers();
if (
// don't waste time with abstract classes or deprecated known-buggy ones
Modifier.isAbstract(modifiers) || !Modifier.isPublic(modifiers)
|| c.isSynthetic() || c.isAnonymousClass() || c.isMemberClass() || c.isInterface()
- || c.getAnnotation(Deprecated.class) != null
+ || c.isAnnotationPresent(Deprecated.class)
|| !(Tokenizer.class.isAssignableFrom(c) || TokenFilter.class.isAssignableFrom(c) || CharFilter.class.isAssignableFrom(c))
) {
continue;
@@ -267,7 +205,7 @@ public class TestRandomChains extends Ba
for (final Constructor<?> ctor : c.getConstructors()) {
// don't test synthetic or deprecated ctors, they likely have known bugs:
- if (ctor.isSynthetic() || ctor.getAnnotation(Deprecated.class) != null || brokenConstructors.get(ctor) == ALWAYS) {
+ if (ctor.isSynthetic() || ctor.isAnnotationPresent(Deprecated.class) || brokenConstructors.get(ctor) == ALWAYS) {
continue;
}
if (Tokenizer.class.isAssignableFrom(c)) {
@@ -457,6 +395,12 @@ public class TestRandomChains extends Ba
return Pattern.compile("a");
}
});
+
+ put(Pattern[].class, new ArgProducer() {
+ @Override public Object create(Random random) {
+ return new Pattern[] {Pattern.compile("([a-z]+)"), Pattern.compile("([0-9]+)")};
+ }
+ });
put(PayloadEncoder.class, new ArgProducer() {
@Override public Object create(Random random) {
return new IdentityEncoder(); // the other encoders will throw exceptions if tokens arent numbers?
@@ -475,20 +419,6 @@ public class TestRandomChains extends Ba
}
}
});
- put(EdgeNGramTokenizer.Side.class, new ArgProducer() {
- @Override public Object create(Random random) {
- return random.nextBoolean()
- ? EdgeNGramTokenizer.Side.FRONT
- : EdgeNGramTokenizer.Side.BACK;
- }
- });
- put(EdgeNGramTokenFilter.Side.class, new ArgProducer() {
- @Override public Object create(Random random) {
- return random.nextBoolean()
- ? EdgeNGramTokenFilter.Side.FRONT
- : EdgeNGramTokenFilter.Side.BACK;
- }
- });
put(HyphenationTree.class, new ArgProducer() {
@Override public Object create(Random random) {
// TODO: make nastier
@@ -567,6 +497,29 @@ public class TestRandomChains extends Ba
return map;
}
});
+ put(StemmerOverrideMap.class, new ArgProducer() {
+ @Override public Object create(Random random) {
+ int num = random.nextInt(10);
+ StemmerOverrideFilter.Builder builder = new StemmerOverrideFilter.Builder(random.nextBoolean());
+ for (int i = 0; i < num; i++) {
+ String input = "";
+ do {
+ input = _TestUtil.randomRealisticUnicodeString(random);
+ } while(input.isEmpty());
+ String out = ""; _TestUtil.randomSimpleString(random);
+ do {
+ out = _TestUtil.randomRealisticUnicodeString(random);
+ } while(out.isEmpty());
+ builder.add(input, out);
+ }
+ try {
+ return builder.build();
+ } catch (Exception ex) {
+ Rethrow.rethrow(ex);
+ return null; // unreachable code
+ }
+ }
+ });
put(SynonymMap.class, new ArgProducer() {
@Override public Object create(Random random) {
SynonymMap.Builder b = new SynonymMap.Builder(random.nextBoolean());
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilter.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilter.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilter.java Thu May 30 07:53:18 2013
@@ -17,17 +17,13 @@ package org.apache.lucene.analysis.core;
*/
import java.io.IOException;
-import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
-import java.util.Set;
-import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.util.CharArraySet;
@@ -72,11 +68,7 @@ public class TestStopFilter extends Base
// with increments
StringReader reader = new StringReader(sb.toString());
StopFilter stpf = new StopFilter(Version.LUCENE_40, new MockTokenizer(reader, MockTokenizer.WHITESPACE, false), stopSet);
- doTestStopPositons(stpf,true);
- // without increments
- reader = new StringReader(sb.toString());
- stpf = new StopFilter(TEST_VERSION_CURRENT, new MockTokenizer(reader, MockTokenizer.WHITESPACE, false), stopSet);
- doTestStopPositons(stpf,false);
+ doTestStopPositons(stpf);
// with increments, concatenating two stop filters
ArrayList<String> a0 = new ArrayList<String>();
ArrayList<String> a1 = new ArrayList<String>();
@@ -95,14 +87,11 @@ public class TestStopFilter extends Base
CharArraySet stopSet1 = StopFilter.makeStopSet(TEST_VERSION_CURRENT, stopWords1);
reader = new StringReader(sb.toString());
StopFilter stpf0 = new StopFilter(TEST_VERSION_CURRENT, new MockTokenizer(reader, MockTokenizer.WHITESPACE, false), stopSet0); // first part of the set
- stpf0.setEnablePositionIncrements(true);
StopFilter stpf01 = new StopFilter(TEST_VERSION_CURRENT, stpf0, stopSet1); // two stop filters concatenated!
- doTestStopPositons(stpf01,true);
+ doTestStopPositons(stpf01);
}
- private void doTestStopPositons(StopFilter stpf, boolean enableIcrements) throws IOException {
- log("---> test with enable-increments-"+(enableIcrements?"enabled":"disabled"));
- stpf.setEnablePositionIncrements(enableIcrements);
+ private void doTestStopPositons(StopFilter stpf) throws IOException {
CharTermAttribute termAtt = stpf.getAttribute(CharTermAttribute.class);
PositionIncrementAttribute posIncrAtt = stpf.getAttribute(PositionIncrementAttribute.class);
stpf.reset();
@@ -111,7 +100,7 @@ public class TestStopFilter extends Base
log("Token "+i+": "+stpf);
String w = English.intToEnglish(i).trim();
assertEquals("expecting token "+i+" to be "+w,w,termAtt.toString());
- assertEquals("all but first token must have position increment of 3",enableIcrements?(i==0?1:3):1,posIncrAtt.getPositionIncrement());
+ assertEquals("all but first token must have position increment of 3",i==0?1:3,posIncrAtt.getPositionIncrement());
}
assertFalse(stpf.incrementToken());
stpf.end();
@@ -159,21 +148,5 @@ public class TestStopFilter extends Base
bufferedState = null;
}
}
-
- public void testFirstPosInc() throws Exception {
- Analyzer analyzer = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- TokenFilter filter = new MockSynonymFilter(tokenizer);
- StopFilter stopfilter = new StopFilter(TEST_VERSION_CURRENT, filter, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
- stopfilter.setEnablePositionIncrements(false);
- return new TokenStreamComponents(tokenizer, stopfilter);
- }
- };
-
- assertAnalyzesTo(analyzer, "the quick brown fox",
- new String[] { "hte", "quick", "brown", "fox" },
- new int[] { 1, 1, 1, 1} );
- }
+
}
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilterFactory.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilterFactory.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilterFactory.java Thu May 30 07:53:18 2013
@@ -17,51 +17,36 @@ package org.apache.lucene.analysis.core;
* limitations under the License.
*/
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.ClasspathResourceLoader;
import org.apache.lucene.analysis.util.ResourceLoader;
-import java.util.Map;
-import java.util.HashMap;
-
-/**
- *
- *
- **/
-public class TestStopFilterFactory extends BaseTokenStreamTestCase {
+public class TestStopFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testInform() throws Exception {
ResourceLoader loader = new ClasspathResourceLoader(getClass());
assertTrue("loader is null and it shouldn't be", loader != null);
- StopFilterFactory factory = new StopFilterFactory();
- Map<String, String> args = new HashMap<String, String>();
- args.put("words", "stop-1.txt");
- args.put("ignoreCase", "true");
- factory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
- factory.init(args);
- factory.inform(loader);
+ StopFilterFactory factory = (StopFilterFactory) tokenFilterFactory("Stop",
+ "words", "stop-1.txt",
+ "ignoreCase", "true");
CharArraySet words = factory.getStopWords();
assertTrue("words is null and it shouldn't be", words != null);
assertTrue("words Size: " + words.size() + " is not: " + 2, words.size() == 2);
assertTrue(factory.isIgnoreCase() + " does not equal: " + true, factory.isIgnoreCase() == true);
- factory = new StopFilterFactory();
- args.put("words", "stop-1.txt, stop-2.txt");
- factory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
- factory.init(args);
- factory.inform(loader);
+ factory = (StopFilterFactory) tokenFilterFactory("Stop",
+ "words", "stop-1.txt, stop-2.txt",
+ "ignoreCase", "true");
words = factory.getStopWords();
assertTrue("words is null and it shouldn't be", words != null);
assertTrue("words Size: " + words.size() + " is not: " + 4, words.size() == 4);
assertTrue(factory.isIgnoreCase() + " does not equal: " + true, factory.isIgnoreCase() == true);
- factory = new StopFilterFactory();
- factory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
- args.put("words", "stop-snowball.txt");
- args.put("format", "snowball");
- factory.init(args);
- factory.inform(loader);
+ factory = (StopFilterFactory) tokenFilterFactory("Stop",
+ "words", "stop-snowball.txt",
+ "format", "snowball",
+ "ignoreCase", "true");
words = factory.getStopWords();
assertEquals(8, words.size());
assertTrue(words.contains("he"));
@@ -73,4 +58,14 @@ public class TestStopFilterFactory exten
assertTrue(words.contains("hers"));
assertTrue(words.contains("herself"));
}
+
+ /** Test that bogus arguments result in exception */
+ public void testBogusArguments() throws Exception {
+ try {
+ tokenFilterFactory("Stop", "bogusArg", "bogusValue");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("Unknown parameters"));
+ }
+ }
}
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilter.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilter.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilter.java Thu May 30 07:53:18 2013
@@ -17,6 +17,11 @@ package org.apache.lucene.analysis.core;
* limitations under the License.
*/
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.Collections;
+import java.util.Set;
+
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardTokenizer;
@@ -25,18 +30,13 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.util.English;
-import java.io.IOException;
-import java.io.StringReader;
-import java.util.Collections;
-import java.util.Set;
-
public class TestTypeTokenFilter extends BaseTokenStreamTestCase {
public void testTypeFilter() throws IOException {
StringReader reader = new StringReader("121 is palindrome, while 123 is not");
Set<String> stopTypes = asSet("<NUM>");
- TokenStream stream = new TypeTokenFilter(true, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopTypes);
+ TokenStream stream = new TypeTokenFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopTypes);
assertTokenStreamContents(stream, new String[]{"is", "palindrome", "while", "is", "not"});
}
@@ -59,12 +59,7 @@ public class TestTypeTokenFilter extends
// with increments
StringReader reader = new StringReader(sb.toString());
- TypeTokenFilter typeTokenFilter = new TypeTokenFilter(true, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopSet);
- testPositons(typeTokenFilter);
-
- // without increments
- reader = new StringReader(sb.toString());
- typeTokenFilter = new TypeTokenFilter(false, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopSet);
+ TypeTokenFilter typeTokenFilter = new TypeTokenFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopSet);
testPositons(typeTokenFilter);
}
@@ -74,11 +69,10 @@ public class TestTypeTokenFilter extends
CharTermAttribute termAttribute = stpf.getAttribute(CharTermAttribute.class);
PositionIncrementAttribute posIncrAtt = stpf.getAttribute(PositionIncrementAttribute.class);
stpf.reset();
- boolean enablePositionIncrements = stpf.getEnablePositionIncrements();
while (stpf.incrementToken()) {
log("Token: " + termAttribute.toString() + ": " + typeAtt.type() + " - " + posIncrAtt.getPositionIncrement());
assertEquals("if position increment is enabled the positionIncrementAttribute value should be 3, otherwise 1",
- posIncrAtt.getPositionIncrement(), enablePositionIncrements ? 3 : 1);
+ posIncrAtt.getPositionIncrement(), 3);
}
stpf.end();
stpf.close();
@@ -87,7 +81,7 @@ public class TestTypeTokenFilter extends
public void testTypeFilterWhitelist() throws IOException {
StringReader reader = new StringReader("121 is palindrome, while 123 is not");
Set<String> stopTypes = Collections.singleton("<NUM>");
- TokenStream stream = new TypeTokenFilter(true, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopTypes, true);
+ TokenStream stream = new TypeTokenFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopTypes, true);
assertTokenStreamContents(stream, new String[]{"121", "123"});
}
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilterFactory.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilterFactory.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilterFactory.java Thu May 30 07:53:18 2013
@@ -17,88 +17,67 @@ package org.apache.lucene.analysis.core;
* limitations under the License.
*/
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.NumericTokenStream;
-import org.apache.lucene.analysis.util.ClasspathResourceLoader;
-import org.apache.lucene.analysis.util.ResourceLoader;
-import org.junit.Test;
+import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
+import org.apache.lucene.analysis.util.TokenFilterFactory;
-import java.util.HashMap;
-import java.util.Map;
import java.util.Set;
/**
* Testcase for {@link TypeTokenFilterFactory}
*/
-public class TestTypeTokenFilterFactory extends BaseTokenStreamTestCase {
+public class TestTypeTokenFilterFactory extends BaseTokenStreamFactoryTestCase {
- @Test
public void testInform() throws Exception {
- ResourceLoader loader = new ClasspathResourceLoader(getClass());
- TypeTokenFilterFactory factory = new TypeTokenFilterFactory();
- Map<String, String> args = new HashMap<String, String>();
- args.put("types", "stoptypes-1.txt");
- args.put("enablePositionIncrements", "true");
- factory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
- factory.init(args);
- factory.inform(loader);
+ TypeTokenFilterFactory factory = (TypeTokenFilterFactory) tokenFilterFactory("Type",
+ "types", "stoptypes-1.txt");
Set<String> types = factory.getStopTypes();
assertTrue("types is null and it shouldn't be", types != null);
assertTrue("types Size: " + types.size() + " is not: " + 2, types.size() == 2);
- assertTrue("enablePositionIncrements was set to true but not correctly parsed", factory.isEnablePositionIncrements());
- factory = new TypeTokenFilterFactory();
- args.put("types", "stoptypes-1.txt, stoptypes-2.txt");
- args.put("enablePositionIncrements", "false");
- args.put("useWhitelist","true");
- factory.init(args);
- factory.inform(loader);
+ factory = (TypeTokenFilterFactory) tokenFilterFactory("Type",
+ "types", "stoptypes-1.txt, stoptypes-2.txt",
+ "useWhitelist", "true");
types = factory.getStopTypes();
assertTrue("types is null and it shouldn't be", types != null);
assertTrue("types Size: " + types.size() + " is not: " + 4, types.size() == 4);
- assertTrue("enablePositionIncrements was set to false but not correctly parsed", !factory.isEnablePositionIncrements());
}
- @Test
public void testCreationWithBlackList() throws Exception {
- TypeTokenFilterFactory typeTokenFilterFactory = new TypeTokenFilterFactory();
- Map<String, String> args = new HashMap<String, String>();
- args.put("types", "stoptypes-1.txt, stoptypes-2.txt");
- args.put("enablePositionIncrements", "false");
- typeTokenFilterFactory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
- typeTokenFilterFactory.init(args);
+ TokenFilterFactory factory = tokenFilterFactory("Type",
+ "types", "stoptypes-1.txt, stoptypes-2.txt");
NumericTokenStream input = new NumericTokenStream();
input.setIntValue(123);
- typeTokenFilterFactory.create(input);
+ factory.create(input);
}
- @Test
- public void testCreationWithWhiteList() throws Exception {
- TypeTokenFilterFactory typeTokenFilterFactory = new TypeTokenFilterFactory();
- Map<String, String> args = new HashMap<String, String>();
- args.put("types", "stoptypes-1.txt, stoptypes-2.txt");
- args.put("enablePositionIncrements", "false");
- args.put("useWhitelist","true");
- typeTokenFilterFactory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
- typeTokenFilterFactory.init(args);
- NumericTokenStream input = new NumericTokenStream();
- input.setIntValue(123);
- typeTokenFilterFactory.create(input);
- }
+ public void testCreationWithWhiteList() throws Exception {
+ TokenFilterFactory factory = tokenFilterFactory("Type",
+ "types", "stoptypes-1.txt, stoptypes-2.txt",
+ "useWhitelist", "true");
+ NumericTokenStream input = new NumericTokenStream();
+ input.setIntValue(123);
+ factory.create(input);
+ }
- @Test
public void testMissingTypesParameter() throws Exception {
try {
- TypeTokenFilterFactory typeTokenFilterFactory = new TypeTokenFilterFactory();
- Map<String, String> args = new HashMap<String, String>();
- args.put("enablePositionIncrements", "false");
- typeTokenFilterFactory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
- typeTokenFilterFactory.init(args);
- typeTokenFilterFactory.inform(new ClasspathResourceLoader(getClass()));
+ tokenFilterFactory("Type");
fail("not supplying 'types' parameter should cause an IllegalArgumentException");
} catch (IllegalArgumentException e) {
// everything ok
}
}
-
+
+ /** Test that bogus arguments result in exception */
+ public void testBogusArguments() throws Exception {
+ try {
+ tokenFilterFactory("Type",
+ "types", "stoptypes-1.txt",
+ "bogusArg", "bogusValue");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("Unknown parameters"));
+ }
+ }
}
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/cz/TestCzechStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/cz/TestCzechStemFilterFactory.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/cz/TestCzechStemFilterFactory.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/cz/TestCzechStemFilterFactory.java Thu May 30 07:53:18 2013
@@ -20,23 +20,31 @@ package org.apache.lucene.analysis.cz;
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
* Simple tests to ensure the Czech stem filter factory is working.
*/
-public class TestCzechStemFilterFactory extends BaseTokenStreamTestCase {
+public class TestCzechStemFilterFactory extends BaseTokenStreamFactoryTestCase {
/**
* Ensure the filter actually stems text.
*/
public void testStemming() throws Exception {
Reader reader = new StringReader("angliÄtÃ");
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- CzechStemFilterFactory factory = new CzechStemFilterFactory();
- TokenStream stream = factory.create(tokenizer);
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("CzechStem").create(stream);
assertTokenStreamContents(stream, new String[] { "anglick" });
}
+
+ /** Test that bogus arguments result in exception */
+ public void testBogusArguments() throws Exception {
+ try {
+ tokenFilterFactory("CzechStem", "bogusArg", "bogusValue");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("Unknown parameters"));
+ }
+ }
}
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/cz/TestCzechStemmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/cz/TestCzechStemmer.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/cz/TestCzechStemmer.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/cz/TestCzechStemmer.java Thu May 30 07:53:18 2013
@@ -26,7 +26,7 @@ import org.apache.lucene.analysis.BaseTo
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.util.CharArraySet;
/**
@@ -281,7 +281,7 @@ public class TestCzechStemmer extends Ba
public void testWithKeywordAttribute() throws IOException {
CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, 1, true);
set.add("hole");
- CzechStemFilter filter = new CzechStemFilter(new KeywordMarkerFilter(
+ CzechStemFilter filter = new CzechStemFilter(new SetKeywordMarkerFilter(
new MockTokenizer(new StringReader("hole desek"), MockTokenizer.WHITESPACE, false), set));
assertTokenStreamContents(filter, new String[] { "hole", "desk" });
}
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanAnalyzer.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanAnalyzer.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanAnalyzer.java Thu May 30 07:53:18 2013
@@ -23,7 +23,7 @@ import java.io.StringReader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.core.LowerCaseTokenizer;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.util.CharArraySet;
public class TestGermanAnalyzer extends BaseTokenStreamTestCase {
@@ -38,7 +38,7 @@ public class TestGermanAnalyzer extends
CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, 1, true);
set.add("fischen");
GermanStemFilter filter = new GermanStemFilter(
- new KeywordMarkerFilter(new LowerCaseTokenizer(TEST_VERSION_CURRENT, new StringReader(
+ new SetKeywordMarkerFilter(new LowerCaseTokenizer(TEST_VERSION_CURRENT, new StringReader(
"Fischen Trinken")), set));
assertTokenStreamContents(filter, new String[] { "fischen", "trink" });
}
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilter.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilter.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilter.java Thu May 30 07:53:18 2013
@@ -26,7 +26,7 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -55,7 +55,7 @@ public class TestGermanLightStemFilter e
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new GermanLightStemFilter(sink));
}
};
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilterFactory.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilterFactory.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilterFactory.java Thu May 30 07:53:18 2013
@@ -20,18 +20,28 @@ package org.apache.lucene.analysis.de;
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
* Simple tests to ensure the German light stem factory is working.
*/
-public class TestGermanLightStemFilterFactory extends BaseTokenStreamTestCase {
+public class TestGermanLightStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("häuser");
- GermanLightStemFilterFactory factory = new GermanLightStemFilterFactory();
- TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("GermanLightStem").create(stream);
assertTokenStreamContents(stream, new String[] { "haus" });
}
+
+ /** Test that bogus arguments result in exception */
+ public void testBogusArguments() throws Exception {
+ try {
+ tokenFilterFactory("GermanLightStem", "bogusArg", "bogusValue");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("Unknown parameters"));
+ }
+ }
}
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java Thu May 30 07:53:18 2013
@@ -26,7 +26,7 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -62,7 +62,7 @@ public class TestGermanMinimalStemFilter
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new GermanMinimalStemFilter(sink));
}
};
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilterFactory.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilterFactory.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilterFactory.java Thu May 30 07:53:18 2013
@@ -20,18 +20,28 @@ package org.apache.lucene.analysis.de;
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
* Simple tests to ensure the German minimal stem factory is working.
*/
-public class TestGermanMinimalStemFilterFactory extends BaseTokenStreamTestCase {
+public class TestGermanMinimalStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("bilder");
- GermanMinimalStemFilterFactory factory = new GermanMinimalStemFilterFactory();
- TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("GermanMinimalStem").create(stream);
assertTokenStreamContents(stream, new String[] { "bild" });
}
+
+ /** Test that bogus arguments result in exception */
+ public void testBogusArguments() throws Exception {
+ try {
+ tokenFilterFactory("GermanMinimalStem", "bogusArg", "bogusValue");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("Unknown parameters"));
+ }
+ }
}
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanNormalizationFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanNormalizationFilterFactory.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanNormalizationFilterFactory.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanNormalizationFilterFactory.java Thu May 30 07:53:18 2013
@@ -20,18 +20,28 @@ package org.apache.lucene.analysis.de;
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
* Simple tests to ensure the German normalization factory is working.
*/
-public class TestGermanNormalizationFilterFactory extends BaseTokenStreamTestCase {
+public class TestGermanNormalizationFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("weiÃbier");
- GermanNormalizationFilterFactory factory = new GermanNormalizationFilterFactory();
- TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("GermanNormalization").create(stream);
assertTokenStreamContents(stream, new String[] { "weissbier" });
}
+
+ /** Test that bogus arguments result in exception */
+ public void testBogusArguments() throws Exception {
+ try {
+ tokenFilterFactory("GermanNormalization", "bogusArg", "bogusValue");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("Unknown parameters"));
+ }
+ }
}
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java Thu May 30 07:53:18 2013
@@ -28,7 +28,7 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.analysis.core.LowerCaseFilter;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -68,7 +68,7 @@ public class TestGermanStemFilter extend
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new GermanStemFilter(sink));
}
};
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilterFactory.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilterFactory.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilterFactory.java Thu May 30 07:53:18 2013
@@ -20,23 +20,31 @@ package org.apache.lucene.analysis.de;
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
* Simple tests to ensure the German stem filter factory is working.
*/
-public class TestGermanStemFilterFactory extends BaseTokenStreamTestCase {
+public class TestGermanStemFilterFactory extends BaseTokenStreamFactoryTestCase {
/**
* Ensure the filter actually stems text.
*/
public void testStemming() throws Exception {
Reader reader = new StringReader("Tischen");
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- GermanStemFilterFactory factory = new GermanStemFilterFactory();
- TokenStream stream = factory.create(tokenizer);
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("GermanStem").create(stream);
assertTokenStreamContents(stream, new String[] { "tisch" });
}
+
+ /** Test that bogus arguments result in exception */
+ public void testBogusArguments() throws Exception {
+ try {
+ tokenFilterFactory("GermanStem", "bogusArg", "bogusValue");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("Unknown parameters"));
+ }
+ }
}
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/el/TestGreekLowerCaseFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/el/TestGreekLowerCaseFilterFactory.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/el/TestGreekLowerCaseFilterFactory.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/el/TestGreekLowerCaseFilterFactory.java Thu May 30 07:53:18 2013
@@ -19,29 +19,32 @@ package org.apache.lucene.analysis.el;
import java.io.Reader;
import java.io.StringReader;
-import java.util.Collections;
-import java.util.Map;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
* Simple tests to ensure the Greek lowercase filter factory is working.
*/
-public class TestGreekLowerCaseFilterFactory extends BaseTokenStreamTestCase {
+public class TestGreekLowerCaseFilterFactory extends BaseTokenStreamFactoryTestCase {
/**
* Ensure the filter actually lowercases (and a bit more) greek text.
*/
public void testNormalization() throws Exception {
Reader reader = new StringReader("ÎάÏÎ¿Ï ÎÎΪÎΣ");
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- GreekLowerCaseFilterFactory factory = new GreekLowerCaseFilterFactory();
- factory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
- Map<String, String> args = Collections.emptyMap();
- factory.init(args);
- TokenStream stream = factory.create(tokenizer);
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("GreekLowerCase").create(stream);
assertTokenStreamContents(stream, new String[] { "μαιοÏ", "μαιοÏ" });
}
+
+ /** Test that bogus arguments result in exception */
+ public void testBogusArguments() throws Exception {
+ try {
+ tokenFilterFactory("GreekLowerCase", "bogusArg", "bogusValue");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("Unknown parameters"));
+ }
+ }
}
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/el/TestGreekStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/el/TestGreekStemFilterFactory.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/el/TestGreekStemFilterFactory.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/el/TestGreekStemFilterFactory.java Thu May 30 07:53:18 2013
@@ -20,22 +20,29 @@ package org.apache.lucene.analysis.el;
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.el.GreekLowerCaseFilter;
+import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
* Simple tests to ensure the Greek stem filter factory is working.
*/
-public class TestGreekStemFilterFactory extends BaseTokenStreamTestCase {
+public class TestGreekStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("άνθÏÏÏοÏ");
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- TokenStream normalized = new GreekLowerCaseFilter(TEST_VERSION_CURRENT, tokenizer);
- GreekStemFilterFactory factory = new GreekStemFilterFactory();
- TokenStream stream = factory.create(normalized);
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("GreekLowerCase").create(stream);
+ stream = tokenFilterFactory("GreekStem").create(stream);
assertTokenStreamContents(stream, new String[] { "ανθÏÏÏ" });
}
+
+ /** Test that bogus arguments result in exception */
+ public void testBogusArguments() throws Exception {
+ try {
+ tokenFilterFactory("GreekStem", "bogusArg", "bogusValue");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("Unknown parameters"));
+ }
+ }
}
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilterFactory.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilterFactory.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilterFactory.java Thu May 30 07:53:18 2013
@@ -20,18 +20,28 @@ package org.apache.lucene.analysis.en;
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
* Simple tests to ensure the English minimal stem factory is working.
*/
-public class TestEnglishMinimalStemFilterFactory extends BaseTokenStreamTestCase {
+public class TestEnglishMinimalStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("bricks");
- EnglishMinimalStemFilterFactory factory = new EnglishMinimalStemFilterFactory();
- TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("EnglishMinimalStem").create(stream);
assertTokenStreamContents(stream, new String[] { "brick" });
}
+
+ /** Test that bogus arguments result in exception */
+ public void testBogusArguments() throws Exception {
+ try {
+ tokenFilterFactory("EnglishMinimalStem", "bogusArg", "bogusValue");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("Unknown parameters"));
+ }
+ }
}
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemFilterFactory.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemFilterFactory.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemFilterFactory.java Thu May 30 07:53:18 2013
@@ -20,18 +20,28 @@ package org.apache.lucene.analysis.en;
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
* Simple tests to ensure the kstem filter factory is working.
*/
-public class TestKStemFilterFactory extends BaseTokenStreamTestCase {
+public class TestKStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("bricks");
- KStemFilterFactory factory = new KStemFilterFactory();
- TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("KStem").create(stream);
assertTokenStreamContents(stream, new String[] { "brick" });
}
+
+ /** Test that bogus arguments result in exception */
+ public void testBogusArguments() throws Exception {
+ try {
+ tokenFilterFactory("KStem", "bogusArg", "bogusValue");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("Unknown parameters"));
+ }
+ }
}
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java Thu May 30 07:53:18 2013
@@ -23,7 +23,7 @@ import java.io.StringReader;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockTokenizer;
@@ -57,7 +57,7 @@ public class TestPorterStemFilter extend
CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, 1, true);
set.add("yourselves");
Tokenizer tokenizer = new MockTokenizer(new StringReader("yourselves yours"), MockTokenizer.WHITESPACE, false);
- TokenStream filter = new PorterStemFilter(new KeywordMarkerFilter(tokenizer, set));
+ TokenStream filter = new PorterStemFilter(new SetKeywordMarkerFilter(tokenizer, set));
assertTokenStreamContents(filter, new String[] {"yourselves", "your"});
}
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilterFactory.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilterFactory.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilterFactory.java Thu May 30 07:53:18 2013
@@ -20,23 +20,31 @@ package org.apache.lucene.analysis.en;
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
* Simple tests to ensure the Porter stem filter factory is working.
*/
-public class TestPorterStemFilterFactory extends BaseTokenStreamTestCase {
+public class TestPorterStemFilterFactory extends BaseTokenStreamFactoryTestCase {
/**
* Ensure the filter actually stems text.
*/
public void testStemming() throws Exception {
Reader reader = new StringReader("dogs");
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- PorterStemFilterFactory factory = new PorterStemFilterFactory();
- TokenStream stream = factory.create(tokenizer);
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("PorterStem").create(stream);
assertTokenStreamContents(stream, new String[] { "dog" });
}
+
+ /** Test that bogus arguments result in exception */
+ public void testBogusArguments() throws Exception {
+ try {
+ tokenFilterFactory("PorterStem", "bogusArg", "bogusValue");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("Unknown parameters"));
+ }
+ }
}
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilterFactory.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilterFactory.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilterFactory.java Thu May 30 07:53:18 2013
@@ -20,18 +20,28 @@ package org.apache.lucene.analysis.es;
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
* Simple tests to ensure the Spanish Light stem factory is working.
*/
-public class TestSpanishLightStemFilterFactory extends BaseTokenStreamTestCase {
+public class TestSpanishLightStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("sociedades");
- SpanishLightStemFilterFactory factory = new SpanishLightStemFilterFactory();
- TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("SpanishLightStem").create(stream);
assertTokenStreamContents(stream, new String[] { "sociedad" });
}
+
+ /** Test that bogus arguments result in exception */
+ public void testBogusArguments() throws Exception {
+ try {
+ tokenFilterFactory("SpanishLightStem", "bogusArg", "bogusValue");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("Unknown parameters"));
+ }
+ }
}
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilterFactory.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilterFactory.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilterFactory.java Thu May 30 07:53:18 2013
@@ -20,23 +20,31 @@ package org.apache.lucene.analysis.fa;
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
* Simple tests to ensure the Persian normalization factory is working.
*/
-public class TestPersianNormalizationFilterFactory extends BaseTokenStreamTestCase {
+public class TestPersianNormalizationFilterFactory extends BaseTokenStreamFactoryTestCase {
/**
* Ensure the filter actually normalizes persian text.
*/
public void testNormalization() throws Exception {
Reader reader = new StringReader("ÙاÛ");
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- PersianNormalizationFilterFactory factory = new PersianNormalizationFilterFactory();
- TokenStream stream = factory.create(tokenizer);
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("PersianNormalization").create(stream);
assertTokenStreamContents(stream, new String[] { "ÙاÙ" });
}
+
+ /** Test that bogus arguments result in exception */
+ public void testBogusArguments() throws Exception {
+ try {
+ tokenFilterFactory("PersianNormalization", "bogusArg", "bogusValue");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("Unknown parameters"));
+ }
+ }
}
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java Thu May 30 07:53:18 2013
@@ -26,7 +26,7 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -55,7 +55,7 @@ public class TestFinnishLightStemFilter
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new FinnishLightStemFilter(sink));
}
};
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilterFactory.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilterFactory.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilterFactory.java Thu May 30 07:53:18 2013
@@ -20,18 +20,28 @@ package org.apache.lucene.analysis.fi;
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
* Simple tests to ensure the Finnish light stem factory is working.
*/
-public class TestFinnishLightStemFilterFactory extends BaseTokenStreamTestCase {
+public class TestFinnishLightStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("aseistettujen");
- FinnishLightStemFilterFactory factory = new FinnishLightStemFilterFactory();
- TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("FinnishLightStem").create(stream);
assertTokenStreamContents(stream, new String[] { "aseistet" });
}
+
+ /** Test that bogus arguments result in exception */
+ public void testBogusArguments() throws Exception {
+ try {
+ tokenFilterFactory("FinnishLightStem", "bogusArg", "bogusValue");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("Unknown parameters"));
+ }
+ }
}
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java Thu May 30 07:53:18 2013
@@ -26,7 +26,7 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -185,7 +185,7 @@ public class TestFrenchLightStemFilter e
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new FrenchLightStemFilter(sink));
}
};
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilterFactory.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilterFactory.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilterFactory.java Thu May 30 07:53:18 2013
@@ -20,18 +20,28 @@ package org.apache.lucene.analysis.fr;
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
* Simple tests to ensure the French light stem factory is working.
*/
-public class TestFrenchLightStemFilterFactory extends BaseTokenStreamTestCase {
+public class TestFrenchLightStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("administrativement");
- FrenchLightStemFilterFactory factory = new FrenchLightStemFilterFactory();
- TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("FrenchLightStem").create(stream);
assertTokenStreamContents(stream, new String[] { "administratif" });
}
+
+ /** Test that bogus arguments result in exception */
+ public void testBogusArguments() throws Exception {
+ try {
+ tokenFilterFactory("FrenchLightStem", "bogusArg", "bogusValue");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("Unknown parameters"));
+ }
+ }
}
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java Thu May 30 07:53:18 2013
@@ -26,7 +26,7 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -64,7 +64,7 @@ public class TestFrenchMinimalStemFilter
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new FrenchMinimalStemFilter(sink));
}
};
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilterFactory.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilterFactory.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilterFactory.java Thu May 30 07:53:18 2013
@@ -20,18 +20,28 @@ package org.apache.lucene.analysis.fr;
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
* Simple tests to ensure the French minimal stem factory is working.
*/
-public class TestFrenchMinimalStemFilterFactory extends BaseTokenStreamTestCase {
+public class TestFrenchMinimalStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("chevaux");
- FrenchMinimalStemFilterFactory factory = new FrenchMinimalStemFilterFactory();
- TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("FrenchMinimalStem").create(stream);
assertTokenStreamContents(stream, new String[] { "cheval" });
}
+
+ /** Test that bogus arguments result in exception */
+ public void testBogusArguments() throws Exception {
+ try {
+ tokenFilterFactory("FrenchMinimalStem", "bogusArg", "bogusValue");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("Unknown parameters"));
+ }
+ }
}
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishAnalyzer.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishAnalyzer.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishAnalyzer.java Thu May 30 07:53:18 2013
@@ -61,7 +61,7 @@ public class TestIrishAnalyzer extends B
Analyzer a = new IrishAnalyzer(TEST_VERSION_CURRENT);
assertAnalyzesTo(a, "n-athair",
new String[] { "athair" },
- new int[] { 1 });
+ new int[] { 2 });
}
/** blast some random strings through the analyzer */
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilterFactory.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilterFactory.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilterFactory.java Thu May 30 07:53:18 2013
@@ -20,18 +20,28 @@ package org.apache.lucene.analysis.ga;
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
* Simple tests to ensure the Irish lowercase filter factory is working.
*/
-public class TestIrishLowerCaseFilterFactory extends BaseTokenStreamTestCase {
+public class TestIrishLowerCaseFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testCasing() throws Exception {
Reader reader = new StringReader("nAthair tUISCE hARD");
- IrishLowerCaseFilterFactory factory = new IrishLowerCaseFilterFactory();
- TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("IrishLowerCase").create(stream);
assertTokenStreamContents(stream, new String[] { "n-athair", "t-uisce", "hard" });
}
+
+ /** Test that bogus arguments result in exception */
+ public void testBogusArguments() throws Exception {
+ try {
+ tokenFilterFactory("IrishLowerCase", "bogusArg", "bogusValue");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("Unknown parameters"));
+ }
+ }
}
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java Thu May 30 07:53:18 2013
@@ -26,7 +26,7 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.util.CharArraySet;
/**
@@ -59,7 +59,7 @@ public class TestGalicianMinimalStemFilt
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new GalicianMinimalStemFilter(sink));
}
};
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilterFactory.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilterFactory.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilterFactory.java Thu May 30 07:53:18 2013
@@ -20,18 +20,28 @@ package org.apache.lucene.analysis.gl;
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
* Simple tests to ensure the Galician plural stem factory is working.
*/
-public class TestGalicianMinimalStemFilterFactory extends BaseTokenStreamTestCase {
+public class TestGalicianMinimalStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("elefantes");
- GalicianMinimalStemFilterFactory factory = new GalicianMinimalStemFilterFactory();
- TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("GalicianMinimalStem").create(stream);
assertTokenStreamContents(stream, new String[] { "elefante" });
}
+
+ /** Test that bogus arguments result in exception */
+ public void testBogusArguments() throws Exception {
+ try {
+ tokenFilterFactory("GalicianMinimalStem", "bogusArg", "bogusValue");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("Unknown parameters"));
+ }
+ }
}
Modified: lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilterFactory.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilterFactory.java (original)
+++ lucene/dev/branches/security/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilterFactory.java Thu May 30 07:53:18 2013
@@ -20,18 +20,28 @@ package org.apache.lucene.analysis.gl;
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
* Simple tests to ensure the Galician stem factory is working.
*/
-public class TestGalicianStemFilterFactory extends BaseTokenStreamTestCase {
+public class TestGalicianStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("cariñosa");
- GalicianStemFilterFactory factory = new GalicianStemFilterFactory();
- TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("GalicianStem").create(stream);
assertTokenStreamContents(stream, new String[] { "cariñ" });
}
+
+ /** Test that bogus arguments result in exception */
+ public void testBogusArguments() throws Exception {
+ try {
+ tokenFilterFactory("GalicianStem", "bogusArg", "bogusValue");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("Unknown parameters"));
+ }
+ }
}