You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2010/11/18 18:28:28 UTC
svn commit: r1036543 - in /lucene/dev/trunk/lucene: ./
contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/
src/java/org/apache/lucene/index/codecs/ src/test/org/apache/lucene/index/
src/test/org/apache/lucene/search/ src/test/org/apach...
Author: simonw
Date: Thu Nov 18 17:28:27 2010
New Revision: 1036543
URL: http://svn.apache.org/viewvc?rev=1036543&view=rev
Log:
LUCENE-2764: Allow tests to use random codec per field
Modified:
lucene/dev/trunk/lucene/common-build.xml
lucene/dev/trunk/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/RandomIndexWriter.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/Test2BTerms.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPrefixRandom.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/LuceneTestCase.java
Modified: lucene/dev/trunk/lucene/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/common-build.xml?rev=1036543&r1=1036542&r2=1036543&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/common-build.xml (original)
+++ lucene/dev/trunk/lucene/common-build.xml Thu Nov 18 17:28:27 2010
@@ -65,6 +65,7 @@
</condition>
<property name="tests.multiplier" value="1" />
<property name="tests.codec" value="random" />
+ <property name="tests.randomCodecPerField" value="true"/>
<property name="tests.locale" value="random" />
<property name="tests.timezone" value="random" />
<property name="tests.directory" value="random" />
Modified: lucene/dev/trunk/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java?rev=1036543&r1=1036542&r2=1036543&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java (original)
+++ lucene/dev/trunk/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java Thu Nov 18 17:28:27 2010
@@ -133,7 +133,6 @@ public class CreateIndexTask extends Per
final String defaultCodec = config.get("default.codec", null);
if (defaultCodec != null) {
- CodecProvider.setDefaultCodec(defaultCodec);
CodecProvider.getDefault().setDefaultFieldCodec(defaultCodec);
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java?rev=1036543&r1=1036542&r2=1036543&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java Thu Nov 18 17:28:27 2010
@@ -39,7 +39,7 @@ import org.apache.lucene.index.codecs.st
public class CodecProvider {
private SegmentInfosWriter infosWriter = new DefaultSegmentInfosWriter();
private SegmentInfosReader infosReader = new DefaultSegmentInfosReader();
- private String defaultFieldCodec = defaultCodec;
+ private String defaultFieldCodec = "Standard";
private final Map<String, String> perFieldMap = new HashMap<String, String>();
@@ -47,7 +47,6 @@ public class CodecProvider {
private final Set<String> knownExtensions = new HashSet<String>();
- private static String defaultCodec = "Standard";
public final static String[] CORE_CODECS = new String[] {"Standard", "Pulsing", "PreFlex", "SimpleText"};
@@ -102,15 +101,6 @@ public class CodecProvider {
public static CodecProvider getDefault() {
return defaultCodecs;
}
-
- /** Used for testing. @lucene.internal */
- public synchronized static void setDefaultCodec(String s) {
- defaultCodec = s;
- }
- /** Used for testing. @lucene.internal */
- public synchronized static String getDefaultCodec() {
- return defaultCodec;
- }
/**
* Sets the {@link Codec} for a given field. Not that setting a fields code is
@@ -175,6 +165,5 @@ class DefaultCodecProvider extends Codec
register(new PreFlexCodec());
register(new PulsingCodec(1));
register(new SimpleTextCodec());
- setDefaultFieldCodec(CodecProvider.getDefaultCodec());
}
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/RandomIndexWriter.java?rev=1036543&r1=1036542&r2=1036543&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/RandomIndexWriter.java Thu Nov 18 17:28:27 2010
@@ -24,7 +24,6 @@ import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
-import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.Version;
@@ -84,7 +83,7 @@ public class RandomIndexWriter implement
flushAt = _TestUtil.nextInt(r, 10, 1000);
if (LuceneTestCase.VERBOSE) {
System.out.println("RIW config=" + w.getConfig());
- System.out.println("codec default=" + CodecProvider.getDefaultCodec());
+ System.out.println("codec default=" + w.getConfig().getCodecProvider().getDefaultFieldCodec());
}
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/Test2BTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/Test2BTerms.java?rev=1036543&r1=1036542&r2=1036543&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/Test2BTerms.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/Test2BTerms.java Thu Nov 18 17:28:27 2010
@@ -123,7 +123,7 @@ public class Test2BTerms extends LuceneT
@Ignore("Takes ~4 hours to run on a fast machine!! And requires that you don't use PreFlex codec.")
public void test2BTerms() throws IOException {
- if ("PreFlex".equals(CodecProvider.getDefaultCodec())) {
+ if ("PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())) {
throw new RuntimeException("thist test cannot run with PreFlex codec");
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java?rev=1036543&r1=1036542&r2=1036543&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java Thu Nov 18 17:28:27 2010
@@ -69,7 +69,7 @@ public class TestLazyProxSkipping extend
int numDocs = 500;
Directory directory = new SeekCountingDirectory(new RAMDirectory());
- IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, true, false)).setMaxBufferedDocs(10));
+ IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, true, false)).setMaxBufferedDocs(10));
((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false);
((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false);
for (int i = 0; i < numDocs; i++) {
@@ -120,7 +120,7 @@ public class TestLazyProxSkipping extend
}
public void testLazySkipping() throws IOException {
- assumeFalse("This test cannot run with SimpleText codec", CodecProvider.getDefaultCodec().equals("SimpleText"));
+ assumeFalse("This test cannot run with SimpleText codec", getRandomFieldCodec(this.field).equals("SimpleText"));
// test whether only the minimum amount of seeks()
// are performed
performTest(5);
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPrefixRandom.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPrefixRandom.java?rev=1036543&r1=1036542&r2=1036543&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPrefixRandom.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPrefixRandom.java Thu Nov 18 17:28:27 2010
@@ -27,7 +27,6 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.RandomIndexWriter;
-import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
@@ -57,7 +56,7 @@ public class TestPrefixRandom extends Lu
// we generate aweful prefixes: good for testing.
// but for preflex codec, the test can be very slow, so use less iterations.
- String codec = CodecProvider.getDefaultCodec();
+ final String codec = getRandomFieldCodec("field");
int num = codec.equals("PreFlex") ? 200 * RANDOM_MULTIPLIER : 2000 * RANDOM_MULTIPLIER;
for (int i = 0; i < num; i++) {
field.setValue(_TestUtil.randomUnicodeString(random, 10));
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java?rev=1036543&r1=1036542&r2=1036543&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java Thu Nov 18 17:28:27 2010
@@ -58,7 +58,6 @@ public class TestRegexpRandom2 extends L
RandomIndexWriter writer = new RandomIndexWriter(random, dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.KEYWORD, false))
.setMaxBufferedDocs(_TestUtil.nextInt(random, 50, 1000)));
-
Document doc = new Document();
Field field = newField("field", "", Field.Store.NO, Field.Index.NOT_ANALYZED);
doc.add(field);
@@ -135,8 +134,7 @@ public class TestRegexpRandom2 extends L
public void testRegexps() throws Exception {
// we generate aweful regexps: good for testing.
// but for preflex codec, the test can be very slow, so use less iterations.
- String codec = CodecProvider.getDefaultCodec();
- int num = codec.equals("PreFlex") ? 100 * RANDOM_MULTIPLIER : 1000 * RANDOM_MULTIPLIER;
+ int num = getRandomFieldCodec("field").equals("PreFlex") ? 100 * RANDOM_MULTIPLIER : 1000 * RANDOM_MULTIPLIER;
for (int i = 0; i < num; i++) {
String reg = AutomatonTestUtil.randomRegexp(random);
assertSame(reg);
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/LuceneTestCase.java?rev=1036543&r1=1036542&r2=1036543&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/LuceneTestCase.java Thu Nov 18 17:28:27 2010
@@ -35,6 +35,8 @@ import org.apache.lucene.index.codecs.mo
import org.apache.lucene.index.codecs.preflex.PreFlexCodec;
import org.apache.lucene.index.codecs.preflexrw.PreFlexRWCodec;
import org.apache.lucene.index.codecs.pulsing.PulsingCodec;
+import org.apache.lucene.index.codecs.simpletext.SimpleTextCodec;
+import org.apache.lucene.index.codecs.standard.StandardCodec;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.FieldCache.CacheEntry;
@@ -73,6 +75,7 @@ import java.lang.reflect.Modifier;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.HashMap;
import java.util.IdentityHashMap;
import java.util.Iterator;
import java.util.List;
@@ -146,6 +149,8 @@ public abstract class LuceneTestCase ext
// tests)
/** Gets the codec to run tests with. */
static final String TEST_CODEC = System.getProperty("tests.codec", "random");
+ /** Get if a random codec per field should be used */ // only use random per field if no explicit codec is set
+ static final boolean TEST_RANDOM_CODEC_PER_FIELD = "random".equals(TEST_CODEC) && Boolean.parseBoolean(System.getProperty("tests.randomCodecPerField", "true"));
/** Gets the locale to run tests with */
static final String TEST_LOCALE = System.getProperty("tests.locale", "random");
/** Gets the timezone to run tests with */
@@ -215,7 +220,7 @@ public abstract class LuceneTestCase ext
static Codec installTestCodecs() {
final CodecProvider cp = CodecProvider.getDefault();
- savedDefaultCodec = CodecProvider.getDefaultCodec();
+ savedDefaultCodec = cp.getDefaultFieldCodec();
String codec = TEST_CODEC;
final boolean codecHasParam;
@@ -235,7 +240,6 @@ public abstract class LuceneTestCase ext
}
}
- CodecProvider.setDefaultCodec(codec);
cp.setDefaultFieldCodec(codec);
if (codec.equals("PreFlex")) {
@@ -268,7 +272,6 @@ public abstract class LuceneTestCase ext
cp.unregister(cp.lookup("MockFixedIntBlock"));
cp.unregister(cp.lookup("MockVariableIntBlock"));
swapCodec(new PulsingCodec(1));
- CodecProvider.setDefaultCodec(savedDefaultCodec);
cp.setDefaultFieldCodec(savedDefaultCodec);
}
@@ -322,6 +325,7 @@ public abstract class LuceneTestCase ext
timeZone = TEST_TIMEZONE.equals("random") ? randomTimeZone(random) : TimeZone.getTimeZone(TEST_TIMEZONE);
TimeZone.setDefault(timeZone);
testsFailed = false;
+ randomCodecProvider = new RandomCodecProvider(random);
}
@AfterClass
@@ -342,11 +346,11 @@ public abstract class LuceneTestCase ext
}
stores = null;
// if tests failed, report some information back
- if (testsFailed)
+ if (testsFailed) {
System.out.println("NOTE: test params are: codec=" + codec +
", locale=" + locale +
- ", timezone=" + (timeZone == null ? "(null)" : timeZone.getID()));
- if (testsFailed) {
+ ", timezone=" + (timeZone == null ? "(null)" : timeZone.getID()) +
+ (TEST_RANDOM_CODEC_PER_FIELD?", "+randomCodecProvider.toString():""));
System.err.println("NOTE: all tests run in this JVM:");
System.err.println(Arrays.toString(testClassesRun.toArray()));
}
@@ -617,8 +621,9 @@ public abstract class LuceneTestCase ext
return newIndexWriterConfig(random, v, a);
}
+ /** create a new index writer config with random defaults */
public static IndexWriterConfig newIndexWriterConfig(Random r, Version v, Analyzer a) {
- IndexWriterConfig c = new IndexWriterConfig(v, a);
+ final IndexWriterConfig c = new IndexWriterConfig(v, a);
if (r.nextBoolean()) {
c.setMergePolicy(new LogDocMergePolicy());
}
@@ -642,7 +647,9 @@ public abstract class LuceneTestCase ext
logmp.setCalibrateSizeByDeletes(r.nextBoolean());
logmp.setMergeFactor(_TestUtil.nextInt(r, 2, 20));
}
-
+ if (TEST_RANDOM_CODEC_PER_FIELD) {
+ c.setCodecProvider(randomCodecProvider);
+ }
c.setReaderPooling(r.nextBoolean());
c.setReaderTermsIndexDivisor(_TestUtil.nextInt(r, 1, 4));
return c;
@@ -844,6 +851,8 @@ public abstract class LuceneTestCase ext
// seed for individual test methods, changed in @before
private long seed;
+ protected static CodecProvider randomCodecProvider;
+
private static final Random seedRand = new Random();
protected static final Random random = new Random();
@@ -944,6 +953,61 @@ public abstract class LuceneTestCase ext
}
}
+ /** Returns the codec for the given field uses in this testcase */
+ public static String getRandomFieldCodec(String field) {
+ final CodecProvider provider = TEST_RANDOM_CODEC_PER_FIELD ? randomCodecProvider : CodecProvider.getDefault();
+ return provider.getFieldCodec(field);
+ }
+
+ static class RandomCodecProvider extends CodecProvider {
+
+ private final Codec[] codecs;
+ private final Map<String, String> perFieldMap = new HashMap<String, String>();
+ private final Random random;
+
+ RandomCodecProvider(Random random) {
+ if (random.nextInt(5) == 0) {
+ /*
+ * We randomly swap in a exclusive PreFlexCodec to provide random test coverage
+ * for preFlex indexes. In realistic a PreFlex-Environment newer codecs don't occur.
+ */
+ this.codecs = new Codec[] { new PreFlexRWCodec() };
+ } else {
+ this.codecs = new Codec[] { new StandardCodec(), new SimpleTextCodec(),
+ new MockSepCodec(), new PulsingCodec(1 + random.nextInt(10)),
+ new MockVariableIntBlockCodec(1 + random.nextInt(10)),
+ new MockFixedIntBlockCodec(1 + random.nextInt(10)), };
+ register(new PreFlexCodec()); // register this for read support
+ }
+ for (int i = 0; i < codecs.length; i++) {
+ register(codecs[i]);
+ }
+
+ this.random = random;
+ }
+
+ @Override
+ public synchronized String getFieldCodec(String name) {
+ if (!perFieldMap.containsKey(name)) { // select a codec at random
+ setFieldCodec(name, codecs[random.nextInt(codecs.length)].name);
+ }
+ return super.getFieldCodec(name);
+ }
+
+ @Override
+ public synchronized void setFieldCodec(String field, String codec) {
+ if (!perFieldMap.containsKey(field)) {
+ perFieldMap.put(field, codec);
+ }
+ super.setFieldCodec(field, codec);
+ }
+
+ @Override
+ public String toString() {
+ return "RandomCodecProvider [perFieldMap=" + perFieldMap + "]";
+ }
+ }
+
@Ignore("just a hack")
public final void alwaysIgnoredTestMethod() {}
}