You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/12/28 12:09:08 UTC
svn commit: r1648188 - in
/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene: index/
util/
Author: rmuir
Date: Sun Dec 28 11:09:08 2014
New Revision: 1648188
URL: http://svn.apache.org/r1648188
Log:
LUCENE-6135: renumber fields randomly in tests
Added:
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedDirectoryReader.java (with props)
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedLeafReader.java (with props)
Modified:
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java?rev=1648188&r1=1648187&r2=1648188&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java Sun Dec 28 11:09:08 2014
@@ -56,6 +56,7 @@ import org.apache.lucene.store.MMapDirec
import org.apache.lucene.store.MockDirectoryWrapper.Throttling;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.TestUtil;
import com.carrotsearch.randomizedtesting.generators.RandomInts;
@@ -763,4 +764,51 @@ public abstract class BaseStoredFieldsFo
dir.close();
}
+ /** mix up field numbers, merge, and check that data is correct */
+ public void testMismatchedFields() throws Exception {
+ Directory dirs[] = new Directory[10];
+ for (int i = 0; i < dirs.length; i++) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(null);
+ IndexWriter iw = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ for (int j = 0; j < 10; j++) {
+ // add fields where name=value (e.g. 3=3) so we can detect if stuff gets screwed up.
+ doc.add(new StringField(Integer.toString(j), Integer.toString(j), Field.Store.YES));
+ }
+ for (int j = 0; j < 10; j++) {
+ iw.addDocument(doc);
+ }
+
+ DirectoryReader reader = DirectoryReader.open(iw, true);
+ // mix up fields explicitly
+ if (random().nextBoolean()) {
+ reader = new MismatchedDirectoryReader(reader, random());
+ }
+ dirs[i] = newDirectory();
+ IndexWriter adder = new IndexWriter(dirs[i], new IndexWriterConfig(null));
+ adder.addIndexes(reader);
+ adder.commit();
+ adder.close();
+
+ IOUtils.close(reader, iw, dir);
+ }
+
+ Directory everything = newDirectory();
+ IndexWriter iw = new IndexWriter(everything, new IndexWriterConfig(null));
+ iw.addIndexes(dirs);
+ iw.forceMerge(1);
+
+ LeafReader ir = getOnlySegmentReader(DirectoryReader.open(iw, true));
+ for (int i = 0; i < ir.maxDoc(); i++) {
+ StoredDocument doc = ir.document(i);
+ assertEquals(10, doc.getFields().size());
+ for (int j = 0; j < 10; j++) {
+ assertEquals(Integer.toString(j), doc.get(Integer.toString(j)));
+ }
+ }
+
+ IOUtils.close(iw, ir, everything);
+ IOUtils.close(dirs);
+ }
}
Added: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedDirectoryReader.java?rev=1648188&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedDirectoryReader.java (added)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedDirectoryReader.java Sun Dec 28 11:09:08 2014
@@ -0,0 +1,49 @@
+package org.apache.lucene.index;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Random;
+
+/**
+ * A {@link DirectoryReader} that wraps all its subreaders with
+ * {@link MismatchedLeafReader}
+ */
+public class MismatchedDirectoryReader extends FilterDirectoryReader {
+
+ static class MismatchedSubReaderWrapper extends SubReaderWrapper {
+ final Random random;
+
+ MismatchedSubReaderWrapper(Random random) {
+ this.random = random;
+ }
+
+ @Override
+ public LeafReader wrap(LeafReader reader) {
+ return new MismatchedLeafReader(reader, random);
+ }
+ }
+
+ public MismatchedDirectoryReader(DirectoryReader in, Random random) {
+ super(in, new MismatchedSubReaderWrapper(random));
+ }
+
+ @Override
+ protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) {
+ return new AssertingDirectoryReader(in);
+ }
+}
Added: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedLeafReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedLeafReader.java?rev=1648188&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedLeafReader.java (added)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedLeafReader.java Sun Dec 28 11:09:08 2014
@@ -0,0 +1,132 @@
+package org.apache.lucene.index;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+
+/**
+ * Shuffles field numbers around to try to trip bugs where field numbers
+ * are assumed to always be consistent across segments.
+ */
+public class MismatchedLeafReader extends FilterLeafReader {
+ final FieldInfos shuffled;
+
+ /** Creates a new reader which will renumber fields in {@code in} */
+ public MismatchedLeafReader(LeafReader in, Random random) {
+ super(in);
+ shuffled = shuffleInfos(in.getFieldInfos(), random);
+ }
+
+ @Override
+ public FieldInfos getFieldInfos() {
+ return shuffled;
+ }
+
+ @Override
+ public void document(int docID, StoredFieldVisitor visitor) throws IOException {
+ in.document(docID, new MismatchedVisitor(visitor));
+ }
+
+ static FieldInfos shuffleInfos(FieldInfos infos, Random random) {
+ // first, shuffle the order
+ List<FieldInfo> shuffled = new ArrayList<>();
+ for (FieldInfo info : infos) {
+ shuffled.add(info);
+ }
+ Collections.shuffle(shuffled, random);
+
+ // now renumber:
+ for (int i = 0; i < shuffled.size(); i++) {
+ FieldInfo oldInfo = shuffled.get(i);
+ // TODO: should we introduce "gaps" too?
+ FieldInfo newInfo = new FieldInfo(oldInfo.name, // name
+ i, // number
+ oldInfo.hasVectors(), // storeTermVector
+ oldInfo.omitsNorms(), // omitNorms
+ oldInfo.hasPayloads(), // storePayloads
+ oldInfo.getIndexOptions(), // indexOptions
+ oldInfo.getDocValuesType(), // docValuesType
+ oldInfo.getDocValuesGen(), // dvGen
+ oldInfo.attributes()); // attributes
+ shuffled.set(i, newInfo);
+ }
+
+ return new FieldInfos(shuffled.toArray(new FieldInfo[shuffled.size()]));
+ }
+
+ /**
+ * StoredFieldsVisitor that remaps actual field numbers
+ * to our new shuffled ones.
+ */
+ // TODO: its strange this part of our IR api exposes FieldInfo,
+ // no other "user-accessible" codec apis do this?
+ class MismatchedVisitor extends StoredFieldVisitor {
+ final StoredFieldVisitor in;
+
+ MismatchedVisitor(StoredFieldVisitor in) {
+ this.in = in;
+ }
+
+ @Override
+ public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
+ in.binaryField(renumber(fieldInfo), value);
+ }
+
+ @Override
+ public void stringField(FieldInfo fieldInfo, String value) throws IOException {
+ in.stringField(renumber(fieldInfo), value);
+ }
+
+ @Override
+ public void intField(FieldInfo fieldInfo, int value) throws IOException {
+ in.intField(renumber(fieldInfo), value);
+ }
+
+ @Override
+ public void longField(FieldInfo fieldInfo, long value) throws IOException {
+ in.longField(renumber(fieldInfo), value);
+ }
+
+ @Override
+ public void floatField(FieldInfo fieldInfo, float value) throws IOException {
+ in.floatField(renumber(fieldInfo), value);
+ }
+
+ @Override
+ public void doubleField(FieldInfo fieldInfo, double value) throws IOException {
+ in.doubleField(renumber(fieldInfo), value);
+ }
+
+ @Override
+ public Status needsField(FieldInfo fieldInfo) throws IOException {
+ return in.needsField(renumber(fieldInfo));
+ }
+
+ FieldInfo renumber(FieldInfo original) {
+ FieldInfo renumbered = shuffled.fieldInfo(original.name);
+ if (renumbered == null) {
+ throw new AssertionError("stored fields sending bogus infos!");
+ }
+ return renumbered;
+ }
+ }
+}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java?rev=1648188&r1=1648187&r2=1648188&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java Sun Dec 28 11:09:08 2014
@@ -73,7 +73,7 @@ public class MockRandomMergePolicy exten
// TODO: sometimes make more than 1 merge?
mergeSpec = new MergeSpecification();
final int segsToMerge = TestUtil.nextInt(random, 1, numSegments);
- if (doNonBulkMerges) {
+ if (doNonBulkMerges && random.nextBoolean()) {
mergeSpec.add(new MockRandomOneMerge(segments.subList(0, segsToMerge),random.nextLong()));
} else {
mergeSpec.add(new OneMerge(segments.subList(0, segsToMerge)));
@@ -106,7 +106,7 @@ public class MockRandomMergePolicy exten
while(upto < eligibleSegments.size()) {
int max = Math.min(10, eligibleSegments.size()-upto);
int inc = max <= 2 ? max : TestUtil.nextInt(random, 2, max);
- if (doNonBulkMerges) {
+ if (doNonBulkMerges && random.nextBoolean()) {
mergeSpec.add(new MockRandomOneMerge(eligibleSegments.subList(upto, upto+inc), random.nextLong()));
} else {
mergeSpec.add(new OneMerge(eligibleSegments.subList(upto, upto+inc)));
@@ -151,9 +151,18 @@ public class MockRandomMergePolicy exten
readers = new ArrayList<LeafReader>(super.getMergeReaders());
for (int i = 0; i < readers.size(); i++) {
// wrap it (e.g. prevent bulk merge etc)
- if (r.nextInt(4) == 0) {
+ int thingToDo = r.nextInt(7);
+ if (thingToDo == 0) {
+ // simple no-op FilterReader
readers.set(i, new FilterLeafReader(readers.get(i)));
+ } else if (thingToDo == 1) {
+ // renumber fields
+ // NOTE: currently this only "blocks" bulk merges just by
+ // being a FilterReader. But it might find bugs elsewhere,
+ // and maybe the situation can be improved in the future.
+ readers.set(i, new MismatchedLeafReader(readers.get(i), r));
}
+ // otherwise, reader is unchanged
}
}
return readers;
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1648188&r1=1648187&r2=1648188&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Sun Dec 28 11:09:08 2014
@@ -89,6 +89,8 @@ import org.apache.lucene.index.LogDocMer
import org.apache.lucene.index.LogMergePolicy;
import org.apache.lucene.index.MergePolicy;
import org.apache.lucene.index.MergeScheduler;
+import org.apache.lucene.index.MismatchedDirectoryReader;
+import org.apache.lucene.index.MismatchedLeafReader;
import org.apache.lucene.index.MockRandomMergePolicy;
import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.MultiFields;
@@ -1587,7 +1589,7 @@ public abstract class LuceneTestCase ext
// TODO: remove this, and fix those tests to wrap before putting slow around:
final boolean wasOriginallyAtomic = r instanceof LeafReader;
for (int i = 0, c = random.nextInt(6)+1; i < c; i++) {
- switch(random.nextInt(5)) {
+ switch(random.nextInt(6)) {
case 0:
r = SlowCompositeReaderWrapper.wrap(r);
break;
@@ -1628,6 +1630,13 @@ public abstract class LuceneTestCase ext
r = new AssertingDirectoryReader((DirectoryReader)r);
}
break;
+ case 5:
+ if (r instanceof LeafReader) {
+ r = new MismatchedLeafReader((LeafReader)r, random);
+ } else if (r instanceof DirectoryReader) {
+ r = new MismatchedDirectoryReader((DirectoryReader)r, random);
+ }
+ break;
default:
fail("should not get here");
}