You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2012/04/11 21:54:11 UTC
svn commit: r1324960 [2/2] - in /lucene/dev/trunk: ./ dev-tools/ lucene/
lucene/contrib/ lucene/core/ lucene/test-framework/
lucene/test-framework/src/java/org/apache/lucene/analysis/
modules/analysis/ modules/analysis/common/ modules/analysis/common/s...
Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java?rev=1324960&r1=1324959&r2=1324960&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java Wed Apr 11 19:54:09 2012
@@ -18,17 +18,28 @@ package org.apache.lucene.analysis.core;
*/
import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
import java.io.Reader;
import java.io.StringReader;
import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Modifier;
import java.net.URL;
+import java.nio.CharBuffer;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Enumeration;
+import java.util.HashSet;
+import java.util.IdentityHashMap;
import java.util.List;
+import java.util.Map;
import java.util.Random;
+import java.util.Set;
+import java.util.regex.Pattern;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
@@ -36,67 +47,174 @@ import org.apache.lucene.analysis.Cachin
import org.apache.lucene.analysis.CharReader;
import org.apache.lucene.analysis.CharStream;
import org.apache.lucene.analysis.EmptyTokenizer;
+import org.apache.lucene.analysis.MockGraphTokenFilter;
+import org.apache.lucene.analysis.MockRandomLookaheadTokenFilter;
+import org.apache.lucene.analysis.MockTokenFilter;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.wikipedia.WikipediaTokenizer;
+import org.apache.lucene.analysis.ValidatingTokenFilter;
+import org.apache.lucene.analysis.charfilter.CharFilter;
+import org.apache.lucene.analysis.charfilter.MappingCharFilter;
+import org.apache.lucene.analysis.charfilter.NormalizeCharMap;
+import org.apache.lucene.analysis.cjk.CJKBigramFilter;
+import org.apache.lucene.analysis.commongrams.CommonGramsFilter;
+import org.apache.lucene.analysis.compound.DictionaryCompoundWordTokenFilter;
+import org.apache.lucene.analysis.compound.HyphenationCompoundWordTokenFilter;
+import org.apache.lucene.analysis.compound.TestCompoundWordTokenFilter;
+import org.apache.lucene.analysis.compound.hyphenation.HyphenationTree;
+import org.apache.lucene.analysis.hunspell.HunspellDictionary;
+import org.apache.lucene.analysis.hunspell.HunspellDictionaryTest;
+import org.apache.lucene.analysis.miscellaneous.LimitTokenCountFilter;
+import org.apache.lucene.analysis.miscellaneous.TrimFilter;
+import org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter;
import org.apache.lucene.analysis.ngram.EdgeNGramTokenFilter;
import org.apache.lucene.analysis.ngram.EdgeNGramTokenizer;
import org.apache.lucene.analysis.ngram.NGramTokenFilter;
import org.apache.lucene.analysis.ngram.NGramTokenizer;
+import org.apache.lucene.analysis.path.PathHierarchyTokenizer;
+import org.apache.lucene.analysis.path.ReversePathHierarchyTokenizer;
+import org.apache.lucene.analysis.payloads.IdentityEncoder;
+import org.apache.lucene.analysis.payloads.PayloadEncoder;
+import org.apache.lucene.analysis.position.PositionFilter;
+import org.apache.lucene.analysis.snowball.TestSnowball;
+import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.analysis.synonym.SynonymMap;
+import org.apache.lucene.analysis.th.ThaiWordFilter;
+import org.apache.lucene.analysis.util.CharArrayMap;
+import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.util.AttributeSource.AttributeFactory;
+import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.Rethrow;
import org.apache.lucene.util.Version;
+import org.apache.lucene.util._TestUtil;
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.tartarus.snowball.SnowballProgram;
+import org.xml.sax.InputSource;
/** tests random analysis chains */
public class TestRandomChains extends BaseTokenStreamTestCase {
- static List<Class<? extends Tokenizer>> tokenizers;
- static List<Class<? extends TokenFilter>> tokenfilters;
- static List<Class<? extends CharStream>> charfilters;
+
+ static List<Constructor<? extends Tokenizer>> tokenizers;
+ static List<Constructor<? extends TokenFilter>> tokenfilters;
+ static List<Constructor<? extends CharStream>> charfilters;
+
+ // TODO: fix those and remove
+ private static final Set<Class<?>> brokenComponents = Collections.newSetFromMap(new IdentityHashMap<Class<?>,Boolean>());
+ static {
+ // TODO: can we promote some of these to be only
+ // offsets offenders?
+ Collections.<Class<?>>addAll(brokenComponents,
+ // TODO: fix basetokenstreamtestcase not to trip because this one has no CharTermAtt
+ EmptyTokenizer.class,
+ // doesn't actual reset itself!
+ CachingTokenFilter.class,
+ // doesn't consume whole stream!
+ LimitTokenCountFilter.class,
+ // Not broken: we forcefully add this, so we shouldn't
+ // also randomly pick it:
+ ValidatingTokenFilter.class,
+ // NOTE: these by themselves won't cause any 'basic assertions' to fail.
+ // but see https://issues.apache.org/jira/browse/LUCENE-3920, if any
+ // tokenfilter that combines words (e.g. shingles) comes after them,
+ // this will create bogus offsets because their 'offsets go backwards',
+ // causing shingle or whatever to make a single token with a
+ // startOffset thats > its endOffset
+ // (see LUCENE-3738 for a list of other offenders here)
+ // broken!
+ NGramTokenizer.class,
+ // broken!
+ NGramTokenFilter.class,
+ // broken!
+ EdgeNGramTokenizer.class,
+ // broken!
+ EdgeNGramTokenFilter.class,
+ // broken!
+ WordDelimiterFilter.class,
+ // broken!
+ TrimFilter.class,
+ // TODO: remove this class after we fix its finalOffset bug
+ MappingCharFilter.class
+ );
+ }
+
+ // TODO: also fix these and remove (maybe):
+ // Classes that don't produce consistent graph offsets:
+ private static final Set<Class<?>> brokenOffsetsComponents = Collections.newSetFromMap(new IdentityHashMap<Class<?>,Boolean>());
+ static {
+ Collections.<Class<?>>addAll(brokenOffsetsComponents,
+ ReversePathHierarchyTokenizer.class,
+ PathHierarchyTokenizer.class,
+ HyphenationCompoundWordTokenFilter.class,
+ DictionaryCompoundWordTokenFilter.class,
+ // TODO: corrumpts graphs (offset consistency check):
+ PositionFilter.class,
+ // TODO: it seems to mess up offsets!?
+ WikipediaTokenizer.class,
+ // TODO: doesn't handle graph inputs
+ ThaiWordFilter.class,
+ // TODO: doesn't handle graph inputs
+ CJKBigramFilter.class
+ );
+ }
@BeforeClass
public static void beforeClass() throws Exception {
List<Class<?>> analysisClasses = new ArrayList<Class<?>>();
getClassesForPackage("org.apache.lucene.analysis", analysisClasses);
- tokenizers = new ArrayList<Class<? extends Tokenizer>>();
- tokenfilters = new ArrayList<Class<? extends TokenFilter>>();
- charfilters = new ArrayList<Class<? extends CharStream>>();
- for (Class<?> c : analysisClasses) {
- // don't waste time with abstract classes or deprecated known-buggy ones
+ tokenizers = new ArrayList<Constructor<? extends Tokenizer>>();
+ tokenfilters = new ArrayList<Constructor<? extends TokenFilter>>();
+ charfilters = new ArrayList<Constructor<? extends CharStream>>();
+ for (final Class<?> c : analysisClasses) {
final int modifiers = c.getModifiers();
- if (Modifier.isAbstract(modifiers) || !Modifier.isPublic(modifiers)
- || c.getAnnotation(Deprecated.class) != null
- || c.isSynthetic() || c.isAnonymousClass() || c.isMemberClass() || c.isInterface()
- // TODO: fix basetokenstreamtestcase not to trip because this one has no CharTermAtt
- || c.equals(EmptyTokenizer.class)
- // doesn't actual reset itself!
- || c.equals(CachingTokenFilter.class)
- // broken!
- || c.equals(NGramTokenizer.class)
- // broken!
- || c.equals(NGramTokenFilter.class)
- // broken!
- || c.equals(EdgeNGramTokenizer.class)
- // broken!
- || c.equals(EdgeNGramTokenFilter.class)) {
+ if (
+ // don't waste time with abstract classes or deprecated known-buggy ones
+ Modifier.isAbstract(modifiers) || !Modifier.isPublic(modifiers)
+ || c.isSynthetic() || c.isAnonymousClass() || c.isMemberClass() || c.isInterface()
+ || brokenComponents.contains(c)
+ || c.isAnnotationPresent(Deprecated.class)
+ || !(Tokenizer.class.isAssignableFrom(c) || TokenFilter.class.isAssignableFrom(c) || CharStream.class.isAssignableFrom(c))
+ ) {
continue;
}
- if (Tokenizer.class.isAssignableFrom(c)) {
- tokenizers.add(c.asSubclass(Tokenizer.class));
- } else if (TokenFilter.class.isAssignableFrom(c)) {
- tokenfilters.add(c.asSubclass(TokenFilter.class));
- } else if (CharStream.class.isAssignableFrom(c)) {
- charfilters.add(c.asSubclass(CharStream.class));
+
+ for (final Constructor<?> ctor : c.getConstructors()) {
+ // don't test synthetic or deprecated ctors, they likely have known bugs:
+ if (ctor.isSynthetic() || ctor.isAnnotationPresent(Deprecated.class)) {
+ continue;
+ }
+ if (Tokenizer.class.isAssignableFrom(c)) {
+ assertTrue(ctor.toGenericString() + " has unsupported parameter types",
+ allowedTokenizerArgs.containsAll(Arrays.asList(ctor.getParameterTypes())));
+ tokenizers.add(castConstructor(Tokenizer.class, ctor));
+ } else if (TokenFilter.class.isAssignableFrom(c)) {
+ assertTrue(ctor.toGenericString() + " has unsupported parameter types",
+ allowedTokenFilterArgs.containsAll(Arrays.asList(ctor.getParameterTypes())));
+ tokenfilters.add(castConstructor(TokenFilter.class, ctor));
+ } else if (CharStream.class.isAssignableFrom(c)) {
+ assertTrue(ctor.toGenericString() + " has unsupported parameter types",
+ allowedCharFilterArgs.containsAll(Arrays.asList(ctor.getParameterTypes())));
+ charfilters.add(castConstructor(CharStream.class, ctor));
+ } else {
+ fail("Cannot get here");
+ }
}
}
- final Comparator<Class<?>> classComp = new Comparator<Class<?>>() {
+
+ final Comparator<Constructor<?>> ctorComp = new Comparator<Constructor<?>>() {
@Override
- public int compare(Class<?> arg0, Class<?> arg1) {
- return arg0.getName().compareTo(arg1.getName());
+ public int compare(Constructor<?> arg0, Constructor<?> arg1) {
+ return arg0.toGenericString().compareTo(arg1.toGenericString());
}
};
- Collections.sort(tokenizers, classComp);
- Collections.sort(tokenfilters, classComp);
- Collections.sort(charfilters, classComp);
+ Collections.sort(tokenizers, ctorComp);
+ Collections.sort(tokenfilters, ctorComp);
+ Collections.sort(charfilters, ctorComp);
if (VERBOSE) {
System.out.println("tokenizers = " + tokenizers);
System.out.println("tokenfilters = " + tokenfilters);
@@ -111,19 +229,382 @@ public class TestRandomChains extends Ba
charfilters = null;
}
+ /** Hack to work around the stupidness of Oracle's strict Java backwards compatibility.
+ * {@code Class<T>#getConstructors()} should return unmodifiable {@code List<Constructor<T>>} not array! */
+ @SuppressWarnings("unchecked")
+ private static <T> Constructor<T> castConstructor(Class<T> instanceClazz, Constructor<?> ctor) {
+ return (Constructor<T>) ctor;
+ }
+ private static void getClassesForPackage(String pckgname, List<Class<?>> classes) throws Exception {
+ final ClassLoader cld = TestRandomChains.class.getClassLoader();
+ final String path = pckgname.replace('.', '/');
+ final Enumeration<URL> resources = cld.getResources(path);
+ while (resources.hasMoreElements()) {
+ final File directory = new File(resources.nextElement().toURI());
+ if (directory.exists()) {
+ String[] files = directory.list();
+ for (String file : files) {
+ if (new File(directory, file).isDirectory()) {
+ // recurse
+ String subPackage = pckgname + "." + file;
+ getClassesForPackage(subPackage, classes);
+ }
+ if (file.endsWith(".class")) {
+ String clazzName = file.substring(0, file.length() - 6);
+ // exclude Test classes that happen to be in these packages.
+ // class.ForName'ing some of them can cause trouble.
+ if (!clazzName.endsWith("Test") && !clazzName.startsWith("Test")) {
+ // Don't run static initializers, as we won't use most of them.
+ // Java will do that automatically once accessed/instantiated.
+ classes.add(Class.forName(pckgname + '.' + clazzName, false, cld));
+ }
+ }
+ }
+ }
+ }
+ }
+
+ private static interface ArgProducer {
+ Object create(Random random);
+ }
+
+ private static final Map<Class<?>,ArgProducer> argProducers = new IdentityHashMap<Class<?>,ArgProducer>() {{
+ put(int.class, new ArgProducer() {
+ @Override public Object create(Random random) {
+ // TODO: could cause huge ram usage to use full int range for some filters
+ // (e.g. allocate enormous arrays)
+ // return Integer.valueOf(random.nextInt());
+ return Integer.valueOf(_TestUtil.nextInt(random, -100, 100));
+ }
+ });
+ put(char.class, new ArgProducer() {
+ @Override public Object create(Random random) {
+ // TODO: fix any filters that care to throw IAE instead.
+ // also add a unicode validating filter to validate termAtt?
+ // return Character.valueOf((char)random.nextInt(65536));
+ while(true) {
+ char c = (char)random.nextInt(65536);
+ if (c < '\uD800' || c > '\uDFFF') {
+ return Character.valueOf(c);
+ }
+ }
+ }
+ });
+ put(float.class, new ArgProducer() {
+ @Override public Object create(Random random) {
+ return Float.valueOf(random.nextFloat());
+ }
+ });
+ put(boolean.class, new ArgProducer() {
+ @Override public Object create(Random random) {
+ return Boolean.valueOf(random.nextBoolean());
+ }
+ });
+ put(byte.class, new ArgProducer() {
+ @Override public Object create(Random random) {
+ // this wraps to negative when casting to byte
+ return Byte.valueOf((byte) random.nextInt(256));
+ }
+ });
+ put(byte[].class, new ArgProducer() {
+ @Override public Object create(Random random) {
+ byte bytes[] = new byte[random.nextInt(256)];
+ random.nextBytes(bytes);
+ return bytes;
+ }
+ });
+ put(Random.class, new ArgProducer() {
+ @Override public Object create(Random random) {
+ return new Random(random.nextLong());
+ }
+ });
+ put(Version.class, new ArgProducer() {
+ @Override public Object create(Random random) {
+ // we expect bugs in emulating old versions
+ return TEST_VERSION_CURRENT;
+ }
+ });
+ put(Set.class, new ArgProducer() {
+ @Override public Object create(Random random) {
+ // TypeTokenFilter
+ Set<String> set = new HashSet<String>();
+ int num = random.nextInt(5);
+ for (int i = 0; i < num; i++) {
+ set.add(StandardTokenizer.TOKEN_TYPES[random.nextInt(StandardTokenizer.TOKEN_TYPES.length)]);
+ }
+ return set;
+ }
+ });
+ put(Collection.class, new ArgProducer() {
+ @Override public Object create(Random random) {
+ // CapitalizationFilter
+ Collection<char[]> col = new ArrayList<char[]>();
+ int num = random.nextInt(5);
+ for (int i = 0; i < num; i++) {
+ col.add(_TestUtil.randomSimpleString(random).toCharArray());
+ }
+ return col;
+ }
+ });
+ put(CharArraySet.class, new ArgProducer() {
+ @Override public Object create(Random random) {
+ int num = random.nextInt(10);
+ CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, num, random.nextBoolean());
+ for (int i = 0; i < num; i++) {
+ // TODO: make nastier
+ set.add(_TestUtil.randomSimpleString(random));
+ }
+ return set;
+ }
+ });
+ put(Pattern.class, new ArgProducer() {
+ @Override public Object create(Random random) {
+ // TODO: don't want to make the exponentially slow ones Dawid documents
+ // in TestPatternReplaceFilter, so dont use truly random patterns (for now)
+ return Pattern.compile("a");
+ }
+ });
+ put(PayloadEncoder.class, new ArgProducer() {
+ @Override public Object create(Random random) {
+ return new IdentityEncoder(); // the other encoders will throw exceptions if tokens arent numbers?
+ }
+ });
+ put(HunspellDictionary.class, new ArgProducer() {
+ @Override public Object create(Random random) {
+ // TODO: make nastier
+ InputStream affixStream = HunspellDictionaryTest.class.getResourceAsStream("test.aff");
+ InputStream dictStream = HunspellDictionaryTest.class.getResourceAsStream("test.dic");
+ try {
+ return new HunspellDictionary(affixStream, dictStream, TEST_VERSION_CURRENT);
+ } catch (Exception ex) {
+ Rethrow.rethrow(ex);
+ return null; // unreachable code
+ }
+ }
+ });
+ put(EdgeNGramTokenizer.Side.class, new ArgProducer() {
+ @Override public Object create(Random random) {
+ return random.nextBoolean()
+ ? EdgeNGramTokenizer.Side.FRONT
+ : EdgeNGramTokenizer.Side.BACK;
+ }
+ });
+ put(EdgeNGramTokenFilter.Side.class, new ArgProducer() {
+ @Override public Object create(Random random) {
+ return random.nextBoolean()
+ ? EdgeNGramTokenFilter.Side.FRONT
+ : EdgeNGramTokenFilter.Side.BACK;
+ }
+ });
+ put(HyphenationTree.class, new ArgProducer() {
+ @Override public Object create(Random random) {
+ // TODO: make nastier
+ try {
+ InputSource is = new InputSource(TestCompoundWordTokenFilter.class.getResource("da_UTF8.xml").toExternalForm());
+ HyphenationTree hyphenator = HyphenationCompoundWordTokenFilter.getHyphenationTree(is);
+ return hyphenator;
+ } catch (Exception ex) {
+ Rethrow.rethrow(ex);
+ return null; // unreachable code
+ }
+ }
+ });
+ put(SnowballProgram.class, new ArgProducer() {
+ @Override public Object create(Random random) {
+ try {
+ String lang = TestSnowball.SNOWBALL_LANGS[random.nextInt(TestSnowball.SNOWBALL_LANGS.length)];
+ Class<? extends SnowballProgram> clazz = Class.forName("org.tartarus.snowball.ext." + lang + "Stemmer").asSubclass(SnowballProgram.class);
+ return clazz.newInstance();
+ } catch (Exception ex) {
+ Rethrow.rethrow(ex);
+ return null; // unreachable code
+ }
+ }
+ });
+ put(String.class, new ArgProducer() {
+ @Override public Object create(Random random) {
+ // TODO: make nastier
+ if (random.nextBoolean()) {
+ // a token type
+ return StandardTokenizer.TOKEN_TYPES[random.nextInt(StandardTokenizer.TOKEN_TYPES.length)];
+ } else {
+ return _TestUtil.randomSimpleString(random);
+ }
+ }
+ });
+ put(NormalizeCharMap.class, new ArgProducer() {
+ @Override public Object create(Random random) {
+ NormalizeCharMap map = new NormalizeCharMap();
+ // we can't add duplicate keys, or NormalizeCharMap gets angry
+ Set<String> keys = new HashSet<String>();
+ int num = random.nextInt(5);
+ //System.out.println("NormalizeCharMap=");
+ for (int i = 0; i < num; i++) {
+ String key = _TestUtil.randomSimpleString(random);
+ if (!keys.contains(key)) {
+ String value = _TestUtil.randomSimpleString(random);
+ map.add(key, value);
+ keys.add(key);
+ //System.out.println("mapping: '" + key + "' => '" + value + "'");
+ }
+ }
+ return map;
+ }
+ });
+ put(CharacterRunAutomaton.class, new ArgProducer() {
+ @Override public Object create(Random random) {
+ // TODO: could probably use a purely random automaton
+ switch(random.nextInt(5)) {
+ case 0: return MockTokenizer.KEYWORD;
+ case 1: return MockTokenizer.SIMPLE;
+ case 2: return MockTokenizer.WHITESPACE;
+ case 3: return MockTokenFilter.EMPTY_STOPSET;
+ default: return MockTokenFilter.ENGLISH_STOPSET;
+ }
+ }
+ });
+ put(CharArrayMap.class, new ArgProducer() {
+ @Override public Object create(Random random) {
+ int num = random.nextInt(10);
+ CharArrayMap<String> map = new CharArrayMap<String>(TEST_VERSION_CURRENT, num, random.nextBoolean());
+ for (int i = 0; i < num; i++) {
+ // TODO: make nastier
+ map.put(_TestUtil.randomSimpleString(random), _TestUtil.randomSimpleString(random));
+ }
+ return map;
+ }
+ });
+ put(SynonymMap.class, new ArgProducer() {
+ @Override public Object create(Random random) {
+ SynonymMap.Builder b = new SynonymMap.Builder(random.nextBoolean());
+ final int numEntries = atLeast(10);
+ for (int j = 0; j < numEntries; j++) {
+ addSyn(b, randomNonEmptyString(random), randomNonEmptyString(random), random.nextBoolean());
+ }
+ try {
+ return b.build();
+ } catch (Exception ex) {
+ Rethrow.rethrow(ex);
+ return null; // unreachable code
+ }
+ }
+
+ private void addSyn(SynonymMap.Builder b, String input, String output, boolean keepOrig) {
+ b.add(new CharsRef(input.replaceAll(" +", "\u0000")),
+ new CharsRef(output.replaceAll(" +", "\u0000")),
+ keepOrig);
+ }
+
+ private String randomNonEmptyString(Random random) {
+ while(true) {
+ final String s = _TestUtil.randomUnicodeString(random).trim();
+ if (s.length() != 0 && s.indexOf('\u0000') == -1) {
+ return s;
+ }
+ }
+ }
+ });
+ }};
+
+ static final Set<Class<?>> allowedTokenizerArgs, allowedTokenFilterArgs, allowedCharFilterArgs;
+ static {
+ allowedTokenizerArgs = Collections.newSetFromMap(new IdentityHashMap<Class<?>,Boolean>());
+ allowedTokenizerArgs.addAll(argProducers.keySet());
+ allowedTokenizerArgs.add(Reader.class);
+ allowedTokenizerArgs.add(AttributeFactory.class);
+ allowedTokenizerArgs.add(AttributeSource.class);
+
+ allowedTokenFilterArgs = Collections.newSetFromMap(new IdentityHashMap<Class<?>,Boolean>());
+ allowedTokenFilterArgs.addAll(argProducers.keySet());
+ allowedTokenFilterArgs.add(TokenStream.class);
+ // TODO: fix this one, thats broken:
+ allowedTokenFilterArgs.add(CommonGramsFilter.class);
+
+ allowedCharFilterArgs = Collections.newSetFromMap(new IdentityHashMap<Class<?>,Boolean>());
+ allowedCharFilterArgs.addAll(argProducers.keySet());
+ allowedCharFilterArgs.add(Reader.class);
+ allowedCharFilterArgs.add(CharStream.class);
+ }
+
+ @SuppressWarnings("unchecked")
+ static <T> T newRandomArg(Random random, Class<T> paramType) {
+ final ArgProducer producer = argProducers.get(paramType);
+ assertNotNull("No producer for arguments of type " + paramType.getName() + " found", producer);
+ return (T) producer.create(random);
+ }
+
+ static Object[] newTokenizerArgs(Random random, Reader reader, Class<?>[] paramTypes) {
+ Object[] args = new Object[paramTypes.length];
+ for (int i = 0; i < args.length; i++) {
+ Class<?> paramType = paramTypes[i];
+ if (paramType == Reader.class) {
+ args[i] = reader;
+ } else if (paramType == AttributeFactory.class) {
+ // TODO: maybe the collator one...???
+ args[i] = AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY;
+ } else if (paramType == AttributeSource.class) {
+ // TODO: args[i] = new AttributeSource();
+ // this is currently too scary to deal with!
+ args[i] = null; // force IAE
+ } else {
+ args[i] = newRandomArg(random, paramType);
+ }
+ }
+ return args;
+ }
+
+ static Object[] newCharFilterArgs(Random random, Reader reader, Class<?>[] paramTypes) {
+ Object[] args = new Object[paramTypes.length];
+ for (int i = 0; i < args.length; i++) {
+ Class<?> paramType = paramTypes[i];
+ if (paramType == Reader.class) {
+ args[i] = reader;
+ } else if (paramType == CharStream.class) {
+ args[i] = CharReader.get(reader);
+ } else {
+ args[i] = newRandomArg(random, paramType);
+ }
+ }
+ return args;
+ }
+
+ static Object[] newFilterArgs(Random random, TokenStream stream, Class<?>[] paramTypes) {
+ Object[] args = new Object[paramTypes.length];
+ for (int i = 0; i < args.length; i++) {
+ Class<?> paramType = paramTypes[i];
+ if (paramType == TokenStream.class) {
+ args[i] = stream;
+ } else if (paramType == CommonGramsFilter.class) {
+ // TODO: fix this one, thats broken: CommonGramsQueryFilter takes this one explicitly
+ args[i] = new CommonGramsFilter(TEST_VERSION_CURRENT, stream, newRandomArg(random, CharArraySet.class));
+ } else {
+ args[i] = newRandomArg(random, paramType);
+ }
+ }
+ return args;
+ }
+
static class MockRandomAnalyzer extends Analyzer {
final long seed;
MockRandomAnalyzer(long seed) {
this.seed = seed;
}
+
+ public boolean offsetsAreCorrect() {
+ // TODO: can we not do the full chain here!?
+ Random random = new Random(seed);
+ TokenizerSpec tokenizerSpec = newTokenizer(random, new StringReader(""));
+ TokenFilterSpec filterSpec = newFilterChain(random, tokenizerSpec.tokenizer, tokenizerSpec.offsetsAreCorrect);
+ return filterSpec.offsetsAreCorrect;
+ }
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Random random = new Random(seed);
- TokenizerSpec tokenizerspec = newTokenizer(random, reader);
- TokenFilterSpec filterspec = newFilterChain(random, tokenizerspec.tokenizer);
- return new TokenStreamComponents(tokenizerspec.tokenizer, filterspec.stream);
+ TokenizerSpec tokenizerSpec = newTokenizer(random, reader);
+ TokenFilterSpec filterSpec = newFilterChain(random, tokenizerSpec.tokenizer, tokenizerSpec.offsetsAreCorrect);
+ return new TokenStreamComponents(tokenizerSpec.tokenizer, filterSpec.stream);
}
@Override
@@ -137,43 +618,74 @@ public class TestRandomChains extends Ba
public String toString() {
Random random = new Random(seed);
StringBuilder sb = new StringBuilder();
- CharFilterSpec charfilterSpec = newCharFilterChain(random, new StringReader(""));
+ CharFilterSpec charFilterSpec = newCharFilterChain(random, new StringReader(""));
sb.append("\ncharfilters=");
- sb.append(charfilterSpec.toString);
+ sb.append(charFilterSpec.toString);
// intentional: initReader gets its own separate random
random = new Random(seed);
- TokenizerSpec tokenizerSpec = newTokenizer(random, charfilterSpec.reader);
+ TokenizerSpec tokenizerSpec = newTokenizer(random, charFilterSpec.reader);
sb.append("\n");
sb.append("tokenizer=");
sb.append(tokenizerSpec.toString);
- TokenFilterSpec tokenfilterSpec = newFilterChain(random, tokenizerSpec.tokenizer);
+ TokenFilterSpec tokenFilterSpec = newFilterChain(random, tokenizerSpec.tokenizer, tokenizerSpec.offsetsAreCorrect);
sb.append("\n");
sb.append("filters=");
- sb.append(tokenfilterSpec.toString);
+ sb.append(tokenFilterSpec.toString);
+ sb.append("\n");
+ sb.append("offsetsAreCorrect=" + tokenFilterSpec.offsetsAreCorrect);
return sb.toString();
}
+ private <T> T createComponent(Constructor<T> ctor, Object[] args, StringBuilder descr) {
+ try {
+ final T instance = ctor.newInstance(args);
+ /*
+ if (descr.length() > 0) {
+ descr.append(",");
+ }
+ */
+ descr.append("\n ");
+ descr.append(ctor.getDeclaringClass().getName());
+ String params = Arrays.toString(args);
+ params = params.substring(1, params.length()-1);
+ descr.append("(").append(params).append(")");
+ return instance;
+ } catch (InvocationTargetException ite) {
+ final Throwable cause = ite.getCause();
+ if (cause instanceof IllegalArgumentException ||
+ cause instanceof UnsupportedOperationException) {
+ // thats ok, ignore
+ if (VERBOSE) {
+ System.err.println("Ignoring IAE/UOE from ctor:");
+ cause.printStackTrace(System.err);
+ }
+ } else {
+ Rethrow.rethrow(cause);
+ }
+ } catch (IllegalAccessException iae) {
+ Rethrow.rethrow(iae);
+ } catch (InstantiationException ie) {
+ Rethrow.rethrow(ie);
+ }
+ return null; // no success
+ }
+
// create a new random tokenizer from classpath
private TokenizerSpec newTokenizer(Random random, Reader reader) {
TokenizerSpec spec = new TokenizerSpec();
- boolean success = false;
- while (!success) {
- try {
- // TODO: check Reader+Version,Version+Reader too
- // also look for other variants and handle them special
- int idx = random.nextInt(tokenizers.size());
- try {
- Constructor<? extends Tokenizer> c = tokenizers.get(idx).getConstructor(Version.class, Reader.class);
- spec.tokenizer = c.newInstance(TEST_VERSION_CURRENT, reader);
- } catch (NoSuchMethodException e) {
- Constructor<? extends Tokenizer> c = tokenizers.get(idx).getConstructor(Reader.class);
- spec.tokenizer = c.newInstance(reader);
- }
- spec.toString = tokenizers.get(idx).toString();
- success = true;
- } catch (Exception e) {
- // ignore
+ while (spec.tokenizer == null) {
+ final Constructor<? extends Tokenizer> ctor = tokenizers.get(random.nextInt(tokenizers.size()));
+ final StringBuilder descr = new StringBuilder();
+ final CheckThatYouDidntReadAnythingReaderWrapper wrapper = new CheckThatYouDidntReadAnythingReaderWrapper(reader);
+ final Object args[] = newTokenizerArgs(random, wrapper, ctor.getParameterTypes());
+ spec.tokenizer = createComponent(ctor, args, descr);
+ if (brokenOffsetsComponents.contains(ctor.getDeclaringClass())) {
+ spec.offsetsAreCorrect = false;
+ }
+ if (spec.tokenizer == null) {
+ assertFalse(ctor.getDeclaringClass().getName() + " has read something in ctor but failed with UOE/IAE", wrapper.readSomething);
}
+ spec.toString = descr.toString();
}
return spec;
}
@@ -184,26 +696,13 @@ public class TestRandomChains extends Ba
StringBuilder descr = new StringBuilder();
int numFilters = random.nextInt(3);
for (int i = 0; i < numFilters; i++) {
- boolean success = false;
- while (!success) {
- try {
- // TODO: also look for other variants and handle them special
- int idx = random.nextInt(charfilters.size());
- try {
- Constructor<? extends CharStream> c = charfilters.get(idx).getConstructor(Reader.class);
- spec.reader = c.newInstance(spec.reader);
- } catch (NoSuchMethodException e) {
- Constructor<? extends CharStream> c = charfilters.get(idx).getConstructor(CharStream.class);
- spec.reader = c.newInstance(CharReader.get(spec.reader));
- }
-
- if (descr.length() > 0) {
- descr.append(",");
- }
- descr.append(charfilters.get(idx).toString());
- success = true;
- } catch (Exception e) {
- // ignore
+ while (true) {
+ final Constructor<? extends CharStream> ctor = charfilters.get(random.nextInt(charfilters.size()));
+ final Object args[] = newCharFilterArgs(random, spec.reader, ctor.getParameterTypes());
+ reader = createComponent(ctor, args, descr);
+ if (reader != null) {
+ spec.reader = reader;
+ break;
}
}
}
@@ -211,47 +710,100 @@ public class TestRandomChains extends Ba
return spec;
}
- private TokenFilterSpec newFilterChain(Random random, Tokenizer tokenizer) {
+ private TokenFilterSpec newFilterChain(Random random, Tokenizer tokenizer, boolean offsetsAreCorrect) {
TokenFilterSpec spec = new TokenFilterSpec();
+ spec.offsetsAreCorrect = offsetsAreCorrect;
spec.stream = tokenizer;
StringBuilder descr = new StringBuilder();
int numFilters = random.nextInt(5);
for (int i = 0; i < numFilters; i++) {
- boolean success = false;
- while (!success) {
- try {
- // TODO: also look for other variants and handle them special
- int idx = random.nextInt(tokenfilters.size());
- try {
- Constructor<? extends TokenFilter> c = tokenfilters.get(idx).getConstructor(Version.class, TokenStream.class);
- spec.stream = c.newInstance(TEST_VERSION_CURRENT, spec.stream);
- } catch (NoSuchMethodException e) {
- Constructor<? extends TokenFilter> c = tokenfilters.get(idx).getConstructor(TokenStream.class);
- spec.stream = c.newInstance(spec.stream);
- }
- if (descr.length() > 0) {
- descr.append(",");
+
+ // Insert ValidatingTF after each stage so we can
+ // catch problems right after the TF that "caused"
+ // them:
+ spec.stream = new ValidatingTokenFilter(spec.stream, "stage " + i, spec.offsetsAreCorrect);
+
+ while (true) {
+ final Constructor<? extends TokenFilter> ctor = tokenfilters.get(random.nextInt(tokenfilters.size()));
+
+ // hack: MockGraph/MockLookahead has assertions that will trip if they follow
+ // an offsets violator. so we cant use them after e.g. wikipediatokenizer
+ if (!spec.offsetsAreCorrect &&
+ (ctor.getDeclaringClass().equals(MockGraphTokenFilter.class)
+ || ctor.getDeclaringClass().equals(MockRandomLookaheadTokenFilter.class))) {
+ continue;
+ }
+
+ final Object args[] = newFilterArgs(random, spec.stream, ctor.getParameterTypes());
+ final TokenFilter flt = createComponent(ctor, args, descr);
+ if (flt != null) {
+ if (brokenOffsetsComponents.contains(ctor.getDeclaringClass())) {
+ spec.offsetsAreCorrect = false;
}
- descr.append(tokenfilters.get(idx).toString());
- success = true;
- } catch (Exception e) {
- // ignore
+ spec.stream = flt;
+ break;
}
}
}
+
+ // Insert ValidatingTF after each stage so we can
+ // catch problems right after the TF that "caused"
+ // them:
+ spec.stream = new ValidatingTokenFilter(spec.stream, "last stage", spec.offsetsAreCorrect);
+
spec.toString = descr.toString();
return spec;
}
}
+ static final class CheckThatYouDidntReadAnythingReaderWrapper extends CharFilter {
+ boolean readSomething = false;
+
+ CheckThatYouDidntReadAnythingReaderWrapper(Reader in) {
+ super(CharReader.get(in));
+ }
+
+ @Override
+ public int read(char[] cbuf, int off, int len) throws IOException {
+ readSomething = true;
+ return super.read(cbuf, off, len);
+ }
+
+ @Override
+ public int read() throws IOException {
+ readSomething = true;
+ return super.read();
+ }
+
+ @Override
+ public int read(CharBuffer target) throws IOException {
+ readSomething = true;
+ return super.read(target);
+ }
+
+ @Override
+ public int read(char[] cbuf) throws IOException {
+ readSomething = true;
+ return super.read(cbuf);
+ }
+
+ @Override
+ public long skip(long n) throws IOException {
+ readSomething = true;
+ return super.skip(n);
+ }
+ }
+
static class TokenizerSpec {
Tokenizer tokenizer;
String toString;
+ boolean offsetsAreCorrect = true;
}
static class TokenFilterSpec {
TokenStream stream;
String toString;
+ boolean offsetsAreCorrect = true;
}
static class CharFilterSpec {
@@ -267,40 +819,12 @@ public class TestRandomChains extends Ba
System.out.println("Creating random analyzer:" + a);
}
try {
- checkRandomData(random, a, 1000);
+ checkRandomData(random, a, 1000, 20, false,
+ false /* We already validate our own offsets... */);
} catch (Throwable e) {
System.err.println("Exception from random analyzer: " + a);
throw e;
}
}
}
-
- private static void getClassesForPackage(String pckgname, List<Class<?>> classes) throws Exception {
- final ClassLoader cld = TestRandomChains.class.getClassLoader();
- final String path = pckgname.replace('.', '/');
- final Enumeration<URL> resources = cld.getResources(path);
- while (resources.hasMoreElements()) {
- final File directory = new File(resources.nextElement().toURI());
- if (directory.exists()) {
- String[] files = directory.list();
- for (String file : files) {
- if (new File(directory, file).isDirectory()) {
- // recurse
- String subPackage = pckgname + "." + file;
- getClassesForPackage(subPackage, classes);
- }
- if (file.endsWith(".class")) {
- String clazzName = file.substring(0, file.length() - 6);
- // exclude Test classes that happen to be in these packages.
- // class.ForName'ing some of them can cause trouble.
- if (!clazzName.endsWith("Test") && !clazzName.startsWith("Test")) {
- // Don't run static initializers, as we won't use most of them.
- // Java will do that automatically once accessed/instantiated.
- classes.add(Class.forName(pckgname + '.' + clazzName, false, cld));
- }
- }
- }
- }
- }
- }
}
Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java?rev=1324960&r1=1324959&r2=1324960&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java Wed Apr 11 19:54:09 2012
@@ -65,7 +65,11 @@ public class TestTrimFilter extends Base
new String[] { "a", "b", "c", "" },
new int[] { 1, 0, 1, 3 },
new int[] { 2, 1, 2, 3 },
- new int[] { 1, 1, 1, 1 });
+ null,
+ new int[] { 1, 1, 1, 1 },
+ null,
+ null,
+ false);
}
/**
Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java?rev=1324960&r1=1324959&r2=1324960&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java Wed Apr 11 19:54:09 2012
@@ -72,14 +72,16 @@ public class TestWordDelimiterFilter ext
assertTokenStreamContents(wdf,
new String[] { "foo", "bar", "foobar" },
new int[] { 5, 9, 5 },
- new int[] { 8, 12, 12 });
+ new int[] { 8, 12, 12 },
+ null, null, null, null, false);
wdf = new WordDelimiterFilter(new SingleTokenTokenStream(new Token("foo-bar", 5, 6)), DEFAULT_WORD_DELIM_TABLE, flags, null);
assertTokenStreamContents(wdf,
new String[] { "foo", "bar", "foobar" },
new int[] { 5, 5, 5 },
- new int[] { 6, 6, 6 });
+ new int[] { 6, 6, 6 },
+ null, null, null, null, false);
}
@Test
@@ -123,7 +125,8 @@ public class TestWordDelimiterFilter ext
assertTokenStreamContents(wdf,
new String[] { "foo", "bar", "foobar"},
new int[] { 8, 12, 8 },
- new int[] { 11, 15, 15 });
+ new int[] { 11, 15, 15 },
+ null, null, null, null, false);
}
public void doSplit(final String input, String... output) throws Exception {
@@ -230,18 +233,27 @@ public class TestWordDelimiterFilter ext
assertAnalyzesTo(a, "LUCENE / SOLR", new String[] { "LUCENE", "SOLR" },
new int[] { 0, 9 },
new int[] { 6, 13 },
- new int[] { 1, 1 });
+ null,
+ new int[] { 1, 1 },
+ null,
+ false);
/* only in this case, posInc of 2 ?! */
assertAnalyzesTo(a, "LUCENE / solR", new String[] { "LUCENE", "sol", "R", "solR" },
new int[] { 0, 9, 12, 9 },
new int[] { 6, 12, 13, 13 },
- new int[] { 1, 1, 1, 0 });
+ null,
+ new int[] { 1, 1, 1, 0 },
+ null,
+ false);
assertAnalyzesTo(a, "LUCENE / NUTCH SOLR", new String[] { "LUCENE", "NUTCH", "SOLR" },
new int[] { 0, 9, 15 },
new int[] { 6, 14, 19 },
- new int[] { 1, 1, 1 });
+ null,
+ new int[] { 1, 1, 1 },
+ null,
+ false);
/* analyzer that will consume tokens with large position increments */
Analyzer a2 = new Analyzer() {
@@ -258,24 +270,36 @@ public class TestWordDelimiterFilter ext
assertAnalyzesTo(a2, "LUCENE largegap SOLR", new String[] { "LUCENE", "largegap", "SOLR" },
new int[] { 0, 7, 16 },
new int[] { 6, 15, 20 },
- new int[] { 1, 10, 1 });
+ null,
+ new int[] { 1, 10, 1 },
+ null,
+ false);
/* the "/" had a position increment of 10, where did it go?!?!! */
assertAnalyzesTo(a2, "LUCENE / SOLR", new String[] { "LUCENE", "SOLR" },
new int[] { 0, 9 },
new int[] { 6, 13 },
- new int[] { 1, 11 });
+ null,
+ new int[] { 1, 11 },
+ null,
+ false);
/* in this case, the increment of 10 from the "/" is carried over */
assertAnalyzesTo(a2, "LUCENE / solR", new String[] { "LUCENE", "sol", "R", "solR" },
new int[] { 0, 9, 12, 9 },
new int[] { 6, 12, 13, 13 },
- new int[] { 1, 11, 1, 0 });
+ null,
+ new int[] { 1, 11, 1, 0 },
+ null,
+ false);
assertAnalyzesTo(a2, "LUCENE / NUTCH SOLR", new String[] { "LUCENE", "NUTCH", "SOLR" },
new int[] { 0, 9, 15 },
new int[] { 6, 14, 19 },
- new int[] { 1, 11, 1 });
+ null,
+ new int[] { 1, 11, 1 },
+ null,
+ false);
Analyzer a3 = new Analyzer() {
@Override
@@ -292,14 +316,20 @@ public class TestWordDelimiterFilter ext
new String[] { "lucene", "solr", "lucenesolr" },
new int[] { 0, 7, 0 },
new int[] { 6, 11, 11 },
- new int[] { 1, 1, 0 });
+ null,
+ new int[] { 1, 1, 0 },
+ null,
+ false);
/* the stopword should add a gap here */
assertAnalyzesTo(a3, "the lucene.solr",
new String[] { "lucene", "solr", "lucenesolr" },
new int[] { 4, 11, 4 },
new int[] { 10, 15, 15 },
- new int[] { 2, 1, 0 });
+ null,
+ new int[] { 2, 1, 0 },
+ null,
+ false);
}
/** blast some random strings through the analyzer */
@@ -322,7 +352,7 @@ public class TestWordDelimiterFilter ext
return new TokenStreamComponents(tokenizer, new WordDelimiterFilter(tokenizer, flags, protectedWords));
}
};
- checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random, a, 10000*RANDOM_MULTIPLIER, 20, false, false);
}
}
Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java?rev=1324960&r1=1324959&r2=1324960&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java Wed Apr 11 19:54:09 2012
@@ -94,7 +94,15 @@ public class EdgeNGramTokenFilterTest ex
public void testBackRangeOfNgrams() throws Exception {
EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(input, EdgeNGramTokenFilter.Side.BACK, 1, 3);
- assertTokenStreamContents(tokenizer, new String[]{"e","de","cde"}, new int[]{4,3,2}, new int[]{5,5,5});
+ assertTokenStreamContents(tokenizer,
+ new String[]{"e","de","cde"},
+ new int[]{4,3,2},
+ new int[]{5,5,5},
+ null,
+ null,
+ null,
+ null,
+ false);
}
public void testSmallTokenInStream() throws Exception {
@@ -151,7 +159,7 @@ public class EdgeNGramTokenFilterTest ex
new EdgeNGramTokenFilter(tokenizer, EdgeNGramTokenFilter.Side.BACK, 2, 15));
}
};
- checkRandomData(random, b, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random, b, 10000*RANDOM_MULTIPLIER, 20, false, false);
}
public void testEmptyTerm() throws Exception {
Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java?rev=1324960&r1=1324959&r2=1324960&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java Wed Apr 11 19:54:09 2012
@@ -90,7 +90,7 @@ public class EdgeNGramTokenizerTest exte
public void testBackRangeOfNgrams() throws Exception {
EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(input, EdgeNGramTokenizer.Side.BACK, 1, 3);
- assertTokenStreamContents(tokenizer, new String[]{"e","de","cde"}, new int[]{4,3,2}, new int[]{5,5,5}, 5 /* abcde */);
+ assertTokenStreamContents(tokenizer, new String[]{"e","de","cde"}, new int[]{4,3,2}, new int[]{5,5,5}, null, null, null, 5 /* abcde */, false);
}
public void testReset() throws Exception {
@@ -109,8 +109,8 @@ public class EdgeNGramTokenizerTest exte
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
- checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
- checkRandomData(random, a, 200*RANDOM_MULTIPLIER, 8192);
+ checkRandomData(random, a, 10000*RANDOM_MULTIPLIER, 20, false, false);
+ checkRandomData(random, a, 200*RANDOM_MULTIPLIER, 8192, false, false);
Analyzer b = new Analyzer() {
@Override
@@ -119,7 +119,7 @@ public class EdgeNGramTokenizerTest exte
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
- checkRandomData(random, b, 10000*RANDOM_MULTIPLIER);
- checkRandomData(random, b, 200*RANDOM_MULTIPLIER, 8192);
+ checkRandomData(random, b, 10000*RANDOM_MULTIPLIER, 20, false, false);
+ checkRandomData(random, b, 200*RANDOM_MULTIPLIER, 8192, false, false);
}
}
Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java?rev=1324960&r1=1324959&r2=1324960&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java Wed Apr 11 19:54:09 2012
@@ -77,7 +77,8 @@ public class NGramTokenFilterTest extend
assertTokenStreamContents(filter,
new String[]{"a","b","c","d","e", "ab","bc","cd","de", "abc","bcd","cde"},
new int[]{0,1,2,3,4, 0,1,2,3, 0,1,2},
- new int[]{1,2,3,4,5, 2,3,4,5, 3,4,5}
+ new int[]{1,2,3,4,5, 2,3,4,5, 3,4,5},
+ null, null, null, null, false
);
}
@@ -130,7 +131,7 @@ public class NGramTokenFilterTest extend
new NGramTokenFilter(tokenizer, 2, 15));
}
};
- checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random, a, 10000*RANDOM_MULTIPLIER, 20, false, false);
}
public void testEmptyTerm() throws Exception {
Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java?rev=1324960&r1=1324959&r2=1324960&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java Wed Apr 11 19:54:09 2012
@@ -73,7 +73,11 @@ public class NGramTokenizerTest extends
new String[]{"a","b","c","d","e", "ab","bc","cd","de", "abc","bcd","cde"},
new int[]{0,1,2,3,4, 0,1,2,3, 0,1,2},
new int[]{1,2,3,4,5, 2,3,4,5, 3,4,5},
- 5 /* abcde */
+ null,
+ null,
+ null,
+ 5 /* abcde */,
+ false
);
}
@@ -98,7 +102,7 @@ public class NGramTokenizerTest extends
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
- checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
- checkRandomData(random, a, 200*RANDOM_MULTIPLIER, 8192);
+ checkRandomData(random, a, 10000*RANDOM_MULTIPLIER, 20, false, false);
+ checkRandomData(random, a, 200*RANDOM_MULTIPLIER, 8192, false, false);
}
}
Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java?rev=1324960&r1=1324959&r2=1324960&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java Wed Apr 11 19:54:09 2012
@@ -142,14 +142,16 @@ public class TestSnowball extends BaseTo
}
}
+ /** for testing purposes ONLY */
+ public static String SNOWBALL_LANGS[] = {
+ "Armenian", "Basque", "Catalan", "Danish", "Dutch", "English",
+ "Finnish", "French", "German2", "German", "Hungarian", "Irish",
+ "Italian", "Kp", "Lovins", "Norwegian", "Porter", "Portuguese",
+ "Romanian", "Russian", "Spanish", "Swedish", "Turkish"
+ };
+
public void testEmptyTerm() throws IOException {
- String langs[] = {
- "Armenian", "Basque", "Catalan", "Danish", "Dutch", "English",
- "Finnish", "French", "German2", "German", "Hungarian", "Irish",
- "Italian", "Kp", "Lovins", "Norwegian", "Porter", "Portuguese",
- "Romanian", "Russian", "Spanish", "Swedish", "Turkish"
- };
- for (final String lang : langs) {
+ for (final String lang : SNOWBALL_LANGS) {
Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {