You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/02/12 17:21:03 UTC
svn commit: r1070106 - in /lucene/dev/trunk/lucene/src:
test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java
test/org/apache/lucene/index/TestDoc.java
test/org/apache/lucene/index/TestIndexReaderCloneNorms.java
Author: mikemccand
Date: Sat Feb 12 16:21:03 2011
New Revision: 1070106
URL: http://svn.apache.org/viewvc?rev=1070106&view=rev
Log:
make MockRandomCodec more evil: now it randomly swaps different int codecs per-file
Modified:
lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDoc.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java
Modified: lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java?rev=1070106&r1=1070105&r2=1070106&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java (original)
+++ lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java Sat Feb 12 16:21:03 2011
@@ -18,15 +18,17 @@ package org.apache.lucene.index.codecs.m
*/
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Iterator;
+import java.util.List;
import java.util.Random;
import java.util.Set;
+import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.codecs.BlockTermsReader;
import org.apache.lucene.index.codecs.BlockTermsWriter;
import org.apache.lucene.index.codecs.Codec;
@@ -46,6 +48,9 @@ import org.apache.lucene.index.codecs.mo
import org.apache.lucene.index.codecs.mocksep.MockSingleIntFactory;
import org.apache.lucene.index.codecs.pulsing.PulsingPostingsReaderImpl;
import org.apache.lucene.index.codecs.pulsing.PulsingPostingsWriterImpl;
+import org.apache.lucene.index.codecs.sep.IntIndexInput;
+import org.apache.lucene.index.codecs.sep.IntIndexOutput;
+import org.apache.lucene.index.codecs.sep.IntStreamFactory;
import org.apache.lucene.index.codecs.sep.SepPostingsReaderImpl;
import org.apache.lucene.index.codecs.sep.SepPostingsWriterImpl;
import org.apache.lucene.index.codecs.standard.StandardPostingsReader;
@@ -71,11 +76,57 @@ public class MockRandomCodec extends Cod
this.seedRandom = new Random(random.nextLong());
}
+ // Chooses random IntStreamFactory depending on file's extension
+ private static class MockIntStreamFactory extends IntStreamFactory {
+ private final int salt;
+ private final List<IntStreamFactory> delegates = new ArrayList<IntStreamFactory>();
+
+ public MockIntStreamFactory(Random random) {
+ salt = random.nextInt();
+ delegates.add(new MockSingleIntFactory());
+ final int blockSize = _TestUtil.nextInt(random, 1, 2000);
+ delegates.add(new MockFixedIntBlockCodec.MockIntFactory(blockSize));
+ final int baseBlockSize = _TestUtil.nextInt(random, 1, 127);
+ delegates.add(new MockVariableIntBlockCodec.MockIntFactory(baseBlockSize));
+ // TODO: others
+ }
+
+ private static String getExtension(String fileName) {
+ final int idx = fileName.indexOf('.');
+ assert idx != -1;
+ return fileName.substring(idx);
+ }
+
+ @Override
+ public IntIndexInput openInput(Directory dir, String fileName, int readBufferSize) throws IOException {
+ // Must only use extension, because IW.addIndexes can
+ // rename segment!
+ final IntStreamFactory f = delegates.get((Math.abs(salt ^ getExtension(fileName).hashCode())) % delegates.size());
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("MockRandomCodec: read using int factory " + f + " from fileName=" + fileName);
+ }
+ return f.openInput(dir, fileName, readBufferSize);
+ }
+
+ @Override
+ public IntIndexOutput createOutput(Directory dir, String fileName) throws IOException {
+ final IntStreamFactory f = delegates.get((Math.abs(salt ^ getExtension(fileName).hashCode())) % delegates.size());
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("MockRandomCodec: write using int factory " + f + " to fileName=" + fileName);
+ }
+ return f.createOutput(dir, fileName);
+ }
+ }
+
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
final long seed = seedRandom.nextLong();
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("MockRandomCodec: writing to seg=" + state.segmentName + " seed=" + seed);
+ }
+
final String seedFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, SEED_EXT);
final IndexOutput out = state.directory.createOutput(seedFileName);
out.writeLong(seed);
@@ -83,25 +134,9 @@ public class MockRandomCodec extends Cod
final Random random = new Random(seed);
PostingsWriterBase postingsWriter;
- final int n = random.nextInt(4);
- if (n == 0) {
- if (LuceneTestCase.VERBOSE) {
- System.out.println("MockRandomCodec: writing MockSep postings");
- }
- postingsWriter = new SepPostingsWriterImpl(state, new MockSingleIntFactory());
- } else if (n == 1) {
- final int blockSize = _TestUtil.nextInt(random, 1, 2000);
- if (LuceneTestCase.VERBOSE) {
- System.out.println("MockRandomCodec: writing MockFixedIntBlock(" + blockSize + ") postings");
- }
- postingsWriter = new SepPostingsWriterImpl(state, new MockFixedIntBlockCodec.MockIntFactory(blockSize));
- } else if (n == 2) {
- final int baseBlockSize = _TestUtil.nextInt(random, 1, 127);
- if (LuceneTestCase.VERBOSE) {
- System.out.println("MockRandomCodec: writing MockVariableIntBlock(" + baseBlockSize + ") postings");
- }
- postingsWriter = new SepPostingsWriterImpl(state, new MockVariableIntBlockCodec.MockIntFactory(baseBlockSize));
+ if (random.nextBoolean()) {
+ postingsWriter = new SepPostingsWriterImpl(state, new MockIntStreamFactory(random));
} else {
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: writing Standard postings");
@@ -190,32 +225,17 @@ public class MockRandomCodec extends Cod
final String seedFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.codecId, SEED_EXT);
final IndexInput in = state.dir.openInput(seedFileName);
final long seed = in.readLong();
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("MockRandomCodec: reading from seg=" + state.segmentInfo.name + " seed=" + seed);
+ }
in.close();
final Random random = new Random(seed);
PostingsReaderBase postingsReader;
- final int n = random.nextInt(4);
- if (n == 0) {
- if (LuceneTestCase.VERBOSE) {
- System.out.println("MockRandomCodec: reading MockSep postings");
- }
- postingsReader = new SepPostingsReaderImpl(state.dir, state.segmentInfo,
- state.readBufferSize, new MockSingleIntFactory(), state.codecId);
- } else if (n == 1) {
- final int blockSize = _TestUtil.nextInt(random, 1, 2000);
- if (LuceneTestCase.VERBOSE) {
- System.out.println("MockRandomCodec: reading MockFixedIntBlock(" + blockSize + ") postings");
- }
- postingsReader = new SepPostingsReaderImpl(state.dir, state.segmentInfo,
- state.readBufferSize, new MockFixedIntBlockCodec.MockIntFactory(blockSize), state.codecId);
- } else if (n == 2) {
- final int baseBlockSize = _TestUtil.nextInt(random, 1, 127);
- if (LuceneTestCase.VERBOSE) {
- System.out.println("MockRandomCodec: reading MockVariableIntBlock(" + baseBlockSize + ") postings");
- }
+ if (random.nextBoolean()) {
postingsReader = new SepPostingsReaderImpl(state.dir, state.segmentInfo,
- state.readBufferSize, new MockVariableIntBlockCodec.MockIntFactory(baseBlockSize), state.codecId);
+ state.readBufferSize, new MockIntStreamFactory(random), state.codecId);
} else {
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: reading Standard postings");
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDoc.java?rev=1070106&r1=1070105&r2=1070106&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDoc.java Sat Feb 12 16:21:03 2011
@@ -57,6 +57,9 @@ public class TestDoc extends LuceneTestC
@Override
public void setUp() throws Exception {
super.setUp();
+ if (VERBOSE) {
+ System.out.println("TEST: setUp");
+ }
workDir = new File(TEMP_DIR,"TestDoc");
workDir.mkdirs();
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java?rev=1070106&r1=1070105&r2=1070106&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java Sat Feb 12 16:21:03 2011
@@ -106,13 +106,17 @@ public class TestIndexReaderCloneNorms e
Directory dir3 = newDirectory();
createIndex(random, dir3);
+ if (VERBOSE) {
+ System.out.println("TEST: now addIndexes/optimize");
+ }
IndexWriter iw = new IndexWriter(
dir3,
newIndexWriterConfig(TEST_VERSION_CURRENT, anlzr).
setOpenMode(OpenMode.APPEND).
setMaxBufferedDocs(5).
- setMergePolicy(newLogMergePolicy(3))
+ setMergePolicy(newLogMergePolicy(3))
);
+ iw.setInfoStream(VERBOSE ? System.out : null);
iw.addIndexes(dir1, dir2);
iw.optimize();
iw.close();
@@ -146,6 +150,9 @@ public class TestIndexReaderCloneNorms e
// try cloning and reopening the norms
private void doTestNorms(Random random, Directory dir) throws IOException {
+ if (VERBOSE) {
+ System.out.println("TEST: now doTestNorms");
+ }
addDocs(random, dir, 12, true);
IndexReader ir = IndexReader.open(dir, false);
verifyIndex(ir);
@@ -237,13 +244,20 @@ public class TestIndexReaderCloneNorms e
}
private void createIndex(Random random, Directory dir) throws IOException {
+ if (VERBOSE) {
+ System.out.println("TEST: createIndex");
+ }
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, anlzr).setOpenMode(OpenMode.CREATE)
.setMaxBufferedDocs(5).setSimilarityProvider(similarityOne).setMergePolicy(newLogMergePolicy()));
+
LogMergePolicy lmp = (LogMergePolicy) iw.getConfig().getMergePolicy();
lmp.setMergeFactor(3);
lmp.setUseCompoundFile(true);
iw.close();
+ if (VERBOSE) {
+ System.out.println("TEST: done createIndex");
+ }
}
private void modifyNormsForF1(IndexReader ir) throws IOException {
@@ -298,6 +312,7 @@ public class TestIndexReaderCloneNorms e
lmp.setMergeFactor(3);
lmp.setUseCompoundFile(compound);
IndexWriter iw = new IndexWriter(dir, conf);
+ iw.setInfoStream(VERBOSE ? System.out : null);
for (int i = 0; i < ndocs; i++) {
iw.addDocument(newDoc());
}