You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2019/03/19 10:17:09 UTC
[lucene-solr] 01/02: LUCENE-8166: Require merge instances to be
consumed in the thread that created them.
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 577bef53dd85734877e598539e7b528b2c1af179
Author: Adrien Grand <jp...@gmail.com>
AuthorDate: Fri Mar 15 14:16:37 2019 +0100
LUCENE-8166: Require merge instances to be consumed in the thread that created them.
---
.../codecs/lucene70/Lucene70NormsProducer.java | 2 +-
.../codecs/memory/DirectDocValuesProducer.java | 4 +-
.../apache/lucene/codecs/DocValuesProducer.java | 5 +-
.../org/apache/lucene/codecs/FieldsProducer.java | 5 +-
.../org/apache/lucene/codecs/NormsProducer.java | 5 +-
.../org/apache/lucene/codecs/PointsReader.java | 5 +-
.../apache/lucene/codecs/StoredFieldsReader.java | 4 +-
.../apache/lucene/codecs/TermVectorsReader.java | 5 +-
.../codecs/lucene80/Lucene80NormsProducer.java | 83 +++++++++++++++++++---
.../codecs/perfield/PerFieldDocValuesFormat.java | 4 +-
.../codecs/perfield/PerFieldPostingsFormat.java | 4 +-
...estLucene50StoredFieldsFormatMergeInstance.java | 29 ++++++++
.../TestLucene80NormsFormatMergeInstance.java | 29 ++++++++
.../apache/lucene/index/TestMultiTermsEnum.java | 2 +-
.../suggest/document/CompletionFieldsProducer.java | 2 +-
.../lucene/codecs/asserting/AssertingCodec.java | 8 +++
.../codecs/asserting/AssertingDocValuesFormat.java | 27 +++++--
.../codecs/asserting/AssertingNormsFormat.java | 15 ++--
.../codecs/asserting/AssertingPointsFormat.java | 15 ++--
.../codecs/asserting/AssertingPostingsFormat.java | 2 +-
.../asserting/AssertingStoredFieldsFormat.java | 16 +++--
.../asserting/AssertingTermVectorsFormat.java | 2 +-
.../apache/lucene/index/AssertingLeafReader.java | 11 ++-
.../lucene/index/BaseIndexFileFormatTestCase.java | 26 ++++++-
.../lucene/index/BaseNormsFormatTestCase.java | 77 ++++++++++++++++++--
.../index/BaseStoredFieldsFormatTestCase.java | 24 +++----
.../apache/lucene/index/MergingCodecReader.java | 75 +++++++++++++++++++
.../index/MergingDirectoryReaderWrapper.java | 50 +++++++++++++
.../org/apache/lucene/util/LuceneTestCase.java | 21 +++++-
29 files changed, 486 insertions(+), 71 deletions(-)
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene70/Lucene70NormsProducer.java b/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene70/Lucene70NormsProducer.java
index c7310e8..82c0233 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene70/Lucene70NormsProducer.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene70/Lucene70NormsProducer.java
@@ -91,7 +91,7 @@ final class Lucene70NormsProducer extends NormsProducer implements Cloneable {
}
@Override
- public NormsProducer getMergeInstance() throws IOException {
+ public NormsProducer getMergeInstance() {
Lucene70NormsProducer clone;
try {
clone = (Lucene70NormsProducer) super.clone();
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java
index 96cd996..baef4db 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java
@@ -80,7 +80,7 @@ class DirectDocValuesProducer extends DocValuesProducer {
static final int VERSION_CURRENT = VERSION_START;
// clone for merge: when merging we don't do any instances.put()s
- DirectDocValuesProducer(DirectDocValuesProducer original) throws IOException {
+ DirectDocValuesProducer(DirectDocValuesProducer original) {
assert Thread.holdsLock(original);
numerics.putAll(original.numerics);
binaries.putAll(original.binaries);
@@ -606,7 +606,7 @@ class DirectDocValuesProducer extends DocValuesProducer {
}
@Override
- public synchronized DocValuesProducer getMergeInstance() throws IOException {
+ public synchronized DocValuesProducer getMergeInstance() {
return new DirectDocValuesProducer(this);
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java
index 9296f19..5fe0b33 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java
@@ -74,10 +74,11 @@ public abstract class DocValuesProducer implements Closeable, Accountable {
public abstract void checkIntegrity() throws IOException;
/**
- * Returns an instance optimized for merging.
+ * Returns an instance optimized for merging. This instance may only be
+ * consumed in the thread that called {@link #getMergeInstance()}.
* <p>
* The default implementation returns {@code this} */
- public DocValuesProducer getMergeInstance() throws IOException {
+ public DocValuesProducer getMergeInstance() {
return this;
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/FieldsProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/FieldsProducer.java
index cd6386c..481b160 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/FieldsProducer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/FieldsProducer.java
@@ -48,10 +48,11 @@ public abstract class FieldsProducer extends Fields implements Closeable, Accoun
public abstract void checkIntegrity() throws IOException;
/**
- * Returns an instance optimized for merging.
+ * Returns an instance optimized for merging. This instance may only be
+ * consumed in the thread that called {@link #getMergeInstance()}.
* <p>
* The default implementation returns {@code this} */
- public FieldsProducer getMergeInstance() throws IOException {
+ public FieldsProducer getMergeInstance() {
return this;
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/NormsProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/NormsProducer.java
index 39f9612..647d9e9 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/NormsProducer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/NormsProducer.java
@@ -49,10 +49,11 @@ public abstract class NormsProducer implements Closeable, Accountable {
public abstract void checkIntegrity() throws IOException;
/**
- * Returns an instance optimized for merging.
+ * Returns an instance optimized for merging. This instance may only be used
+ * from the thread that acquires it.
* <p>
* The default implementation returns {@code this} */
- public NormsProducer getMergeInstance() throws IOException {
+ public NormsProducer getMergeInstance() {
return this;
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/PointsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/PointsReader.java
index b20614a..213b72e 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/PointsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/PointsReader.java
@@ -45,10 +45,11 @@ public abstract class PointsReader implements Closeable, Accountable {
public abstract PointValues getValues(String field) throws IOException;
/**
- * Returns an instance optimized for merging.
+ * Returns an instance optimized for merging. This instance may only be used
+ * in the thread that acquires it.
* <p>
* The default implementation returns {@code this} */
- public PointsReader getMergeInstance() throws IOException {
+ public PointsReader getMergeInstance() {
return this;
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsReader.java
index 6258df5..1f32576 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsReader.java
@@ -52,10 +52,10 @@ public abstract class StoredFieldsReader implements Cloneable, Closeable, Accoun
public abstract void checkIntegrity() throws IOException;
/**
- * Returns an instance optimized for merging.
+ * Returns an instance optimized for merging. This instance may not be cloned.
* <p>
* The default implementation returns {@code this} */
- public StoredFieldsReader getMergeInstance() throws IOException {
+ public StoredFieldsReader getMergeInstance() {
return this;
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsReader.java
index 7104136..dc9115d 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsReader.java
@@ -57,10 +57,11 @@ public abstract class TermVectorsReader implements Cloneable, Closeable, Account
public abstract TermVectorsReader clone();
/**
- * Returns an instance optimized for merging.
+ * Returns an instance optimized for merging. This instance may only be
+ * consumed in the thread that called {@link #getMergeInstance()}.
* <p>
* The default implementation returns {@code this} */
- public TermVectorsReader getMergeInstance() throws IOException {
+ public TermVectorsReader getMergeInstance() {
return this;
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene80/Lucene80NormsProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene80/Lucene80NormsProducer.java
index 66126a2..823cfff 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene80/Lucene80NormsProducer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene80/Lucene80NormsProducer.java
@@ -92,7 +92,7 @@ final class Lucene80NormsProducer extends NormsProducer implements Cloneable {
}
@Override
- public NormsProducer getMergeInstance() throws IOException {
+ public NormsProducer getMergeInstance() {
Lucene80NormsProducer clone;
try {
clone = (Lucene80NormsProducer) super.clone();
@@ -233,18 +233,79 @@ final class Lucene80NormsProducer extends NormsProducer implements Cloneable {
}
private IndexInput getDisiInput(FieldInfo field, NormsEntry entry) throws IOException {
- IndexInput slice = null;
- if (merging) {
- slice = disiInputs.get(field.number);
+ if (merging == false) {
+ return IndexedDISI.createBlockSlice(
+ data, "docs", entry.docsWithFieldOffset, entry.docsWithFieldLength, entry.jumpTableEntryCount);
}
- if (slice == null) {
- slice = IndexedDISI.createBlockSlice(
+
+ IndexInput in = disiInputs.get(field.number);
+ if (in == null) {
+ in = IndexedDISI.createBlockSlice(
data, "docs", entry.docsWithFieldOffset, entry.docsWithFieldLength, entry.jumpTableEntryCount);
- if (merging) {
- disiInputs.put(field.number, slice);
- }
+ disiInputs.put(field.number, in);
}
- return slice;
+
+ final IndexInput inF = in; // same as in but final
+
+ // Wrap so that reads can be interleaved from the same thread if two
+ // norms instances are pulled and consumed in parallel. Merging usually
+ // doesn't need this feature but CheckIndex might, plus we need merge
+ // instances to behave well and not be trappy.
+ return new IndexInput("docs") {
+
+ long offset = 0;
+
+ @Override
+ public void readBytes(byte[] b, int off, int len) throws IOException {
+ inF.seek(offset);
+ offset += len;
+ inF.readBytes(b, off, len);
+ }
+
+ @Override
+ public byte readByte() throws IOException {
+ throw new UnsupportedOperationException("Unused by IndexedDISI");
+ }
+
+ @Override
+ public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
+ throw new UnsupportedOperationException("Unused by IndexedDISI");
+ }
+
+ @Override
+ public short readShort() throws IOException {
+ inF.seek(offset);
+ offset += Short.BYTES;
+ return inF.readShort();
+ }
+
+ @Override
+ public long readLong() throws IOException {
+ inF.seek(offset);
+ offset += Long.BYTES;
+ return inF.readLong();
+ }
+
+ @Override
+ public void seek(long pos) throws IOException {
+ offset = pos;
+ }
+
+ @Override
+ public long length() {
+ throw new UnsupportedOperationException("Unused by IndexedDISI");
+ }
+
+ @Override
+ public long getFilePointer() {
+ return offset;
+ }
+
+ @Override
+ public void close() throws IOException {
+ throw new UnsupportedOperationException("Unused by IndexedDISI");
+ }
+ };
}
private RandomAccessInput getDisiJumpTable(FieldInfo field, NormsEntry entry) throws IOException {
@@ -327,7 +388,7 @@ final class Lucene80NormsProducer extends NormsProducer implements Cloneable {
}
};
}
- final RandomAccessInput slice = data.randomAccessSlice(entry.normsOffset, entry.numDocsWithField * (long) entry.bytesPerNorm);
+ final RandomAccessInput slice = getDataInput(field, entry);
switch (entry.bytesPerNorm) {
case 1:
return new SparseNormsIterator(disi) {
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
index f439699..f2e8940 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
@@ -261,7 +261,7 @@ public abstract class PerFieldDocValuesFormat extends DocValuesFormat {
private final Map<String,DocValuesProducer> formats = new HashMap<>();
// clone for merge
- FieldsReader(FieldsReader other) throws IOException {
+ FieldsReader(FieldsReader other) {
Map<DocValuesProducer,DocValuesProducer> oldToNew = new IdentityHashMap<>();
// First clone all formats
for(Map.Entry<String,DocValuesProducer> ent : other.formats.entrySet()) {
@@ -368,7 +368,7 @@ public abstract class PerFieldDocValuesFormat extends DocValuesFormat {
}
@Override
- public DocValuesProducer getMergeInstance() throws IOException {
+ public DocValuesProducer getMergeInstance() {
return new FieldsReader(this);
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java
index 88ae6da..81bbf72 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java
@@ -247,7 +247,7 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat {
private final String segment;
// clone for merge
- FieldsReader(FieldsReader other) throws IOException {
+ FieldsReader(FieldsReader other) {
Map<FieldsProducer,FieldsProducer> oldToNew = new IdentityHashMap<>();
// First clone all formats
for(Map.Entry<String,FieldsProducer> ent : other.formats.entrySet()) {
@@ -346,7 +346,7 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat {
}
@Override
- public FieldsProducer getMergeInstance() throws IOException {
+ public FieldsProducer getMergeInstance() {
return new FieldsReader(this);
}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatMergeInstance.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatMergeInstance.java
new file mode 100644
index 0000000..d0f3157
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatMergeInstance.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene50;
+
+/**
+ * Test the merge instance of the Lucene50 stored fields format.
+ */
+public class TestLucene50StoredFieldsFormatMergeInstance extends TestLucene50StoredFieldsFormat {
+
+ @Override
+ protected boolean shouldTestMergeInstance() {
+ return true;
+ }
+
+}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene80/TestLucene80NormsFormatMergeInstance.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene80/TestLucene80NormsFormatMergeInstance.java
new file mode 100644
index 0000000..aed0c8b
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene80/TestLucene80NormsFormatMergeInstance.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene80;
+
+/**
+ * Test the merge instance of the Lucene80 norms format.
+ */
+public class TestLucene80NormsFormatMergeInstance extends TestLucene80NormsFormat {
+
+ @Override
+ protected boolean shouldTestMergeInstance() {
+ return true;
+ }
+
+}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestMultiTermsEnum.java b/lucene/core/src/test/org/apache/lucene/index/TestMultiTermsEnum.java
index ffa1f3c..0a3af6f 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestMultiTermsEnum.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestMultiTermsEnum.java
@@ -222,7 +222,7 @@ public class TestMultiTermsEnum extends LuceneTestCase {
}
@Override
- public FieldsProducer getMergeInstance() throws IOException {
+ public FieldsProducer getMergeInstance() {
return create(delegate.getMergeInstance(), newFieldInfo);
}
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsProducer.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsProducer.java
index 7a29b61..b998f8e 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsProducer.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsProducer.java
@@ -133,7 +133,7 @@ final class CompletionFieldsProducer extends FieldsProducer {
}
@Override
- public FieldsProducer getMergeInstance() throws IOException {
+ public FieldsProducer getMergeInstance() {
return new CompletionFieldsProducer(delegateFieldsProducer, readers);
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java
index 15bcfa2..5879118 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java
@@ -33,6 +33,14 @@ import org.apache.lucene.util.TestUtil;
*/
public class AssertingCodec extends FilterCodec {
+ static void assertThread(String object, Thread creationThread) {
+ if (creationThread != Thread.currentThread()) {
+ throw new AssertionError(object + " are only supposed to be consumed in "
+ + "the thread in which they have been acquired. But was acquired in "
+ + creationThread + " and consumed in " + Thread.currentThread() + ".");
+ }
+ }
+
private final PostingsFormat postings = new PerFieldPostingsFormat() {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
index 76ab1df..fd8c246 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
@@ -62,7 +62,7 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
assert state.fieldInfos.hasDocValues();
DocValuesProducer producer = in.fieldsProducer(state);
assert producer != null;
- return new AssertingDocValuesProducer(producer, state.segmentInfo.maxDoc());
+ return new AssertingDocValuesProducer(producer, state.segmentInfo.maxDoc(), false);
}
static class AssertingDocValuesConsumer extends DocValuesConsumer {
@@ -219,10 +219,14 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
static class AssertingDocValuesProducer extends DocValuesProducer {
private final DocValuesProducer in;
private final int maxDoc;
+ private final boolean merging;
+ private final Thread creationThread;
- AssertingDocValuesProducer(DocValuesProducer in, int maxDoc) {
+ AssertingDocValuesProducer(DocValuesProducer in, int maxDoc, boolean merging) {
this.in = in;
this.maxDoc = maxDoc;
+ this.merging = merging;
+ this.creationThread = Thread.currentThread();
// do a few simple checks on init
assert toString() != null;
assert ramBytesUsed() >= 0;
@@ -231,6 +235,9 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
@Override
public NumericDocValues getNumeric(FieldInfo field) throws IOException {
+ if (merging) {
+ AssertingCodec.assertThread("DocValuesProducer", creationThread);
+ }
assert field.getDocValuesType() == DocValuesType.NUMERIC;
NumericDocValues values = in.getNumeric(field);
assert values != null;
@@ -239,6 +246,9 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
@Override
public BinaryDocValues getBinary(FieldInfo field) throws IOException {
+ if (merging) {
+ AssertingCodec.assertThread("DocValuesProducer", creationThread);
+ }
assert field.getDocValuesType() == DocValuesType.BINARY;
BinaryDocValues values = in.getBinary(field);
assert values != null;
@@ -247,6 +257,9 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
@Override
public SortedDocValues getSorted(FieldInfo field) throws IOException {
+ if (merging) {
+ AssertingCodec.assertThread("DocValuesProducer", creationThread);
+ }
assert field.getDocValuesType() == DocValuesType.SORTED;
SortedDocValues values = in.getSorted(field);
assert values != null;
@@ -255,6 +268,9 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
@Override
public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
+ if (merging) {
+ AssertingCodec.assertThread("DocValuesProducer", creationThread);
+ }
assert field.getDocValuesType() == DocValuesType.SORTED_NUMERIC;
SortedNumericDocValues values = in.getSortedNumeric(field);
assert values != null;
@@ -263,6 +279,9 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
@Override
public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
+ if (merging) {
+ AssertingCodec.assertThread("DocValuesProducer", creationThread);
+ }
assert field.getDocValuesType() == DocValuesType.SORTED_SET;
SortedSetDocValues values = in.getSortedSet(field);
assert values != null;
@@ -295,8 +314,8 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
}
@Override
- public DocValuesProducer getMergeInstance() throws IOException {
- return new AssertingDocValuesProducer(in.getMergeInstance(), maxDoc);
+ public DocValuesProducer getMergeInstance() {
+ return new AssertingDocValuesProducer(in.getMergeInstance(), maxDoc, true);
}
@Override
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java
index bf830bf..937b8f6 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java
@@ -50,7 +50,7 @@ public class AssertingNormsFormat extends NormsFormat {
assert state.fieldInfos.hasNorms();
NormsProducer producer = in.normsProducer(state);
assert producer != null;
- return new AssertingNormsProducer(producer, state.segmentInfo.maxDoc());
+ return new AssertingNormsProducer(producer, state.segmentInfo.maxDoc(), false);
}
static class AssertingNormsConsumer extends NormsConsumer {
@@ -88,10 +88,14 @@ public class AssertingNormsFormat extends NormsFormat {
static class AssertingNormsProducer extends NormsProducer {
private final NormsProducer in;
private final int maxDoc;
+ private final boolean merging;
+ private final Thread creationThread;
- AssertingNormsProducer(NormsProducer in, int maxDoc) {
+ AssertingNormsProducer(NormsProducer in, int maxDoc, boolean merging) {
this.in = in;
this.maxDoc = maxDoc;
+ this.merging = merging;
+ this.creationThread = Thread.currentThread();
// do a few simple checks on init
assert toString() != null;
assert ramBytesUsed() >= 0;
@@ -100,6 +104,9 @@ public class AssertingNormsFormat extends NormsFormat {
@Override
public NumericDocValues getNorms(FieldInfo field) throws IOException {
+ if (merging) {
+ AssertingCodec.assertThread("NormsProducer", creationThread);
+ }
assert field.hasNorms();
NumericDocValues values = in.getNorms(field);
assert values != null;
@@ -132,8 +139,8 @@ public class AssertingNormsFormat extends NormsFormat {
}
@Override
- public NormsProducer getMergeInstance() throws IOException {
- return new AssertingNormsProducer(in.getMergeInstance(), maxDoc);
+ public NormsProducer getMergeInstance() {
+ return new AssertingNormsProducer(in.getMergeInstance(), maxDoc, true);
}
@Override
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointsFormat.java
index 4943b99..79dfa5f 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointsFormat.java
@@ -60,17 +60,21 @@ public final class AssertingPointsFormat extends PointsFormat {
@Override
public PointsReader fieldsReader(SegmentReadState state) throws IOException {
- return new AssertingPointsReader(state.segmentInfo.maxDoc(), in.fieldsReader(state));
+ return new AssertingPointsReader(state.segmentInfo.maxDoc(), in.fieldsReader(state), false);
}
static class AssertingPointsReader extends PointsReader {
private final PointsReader in;
private final int maxDoc;
+ private final boolean merging;
+ private final Thread creationThread;
- AssertingPointsReader(int maxDoc, PointsReader in) {
+ AssertingPointsReader(int maxDoc, PointsReader in, boolean merging) {
this.in = in;
this.maxDoc = maxDoc;
+ this.merging = merging;
+ this.creationThread = Thread.currentThread();
// do a few simple checks on init
assert toString() != null;
assert ramBytesUsed() >= 0;
@@ -85,6 +89,9 @@ public final class AssertingPointsFormat extends PointsFormat {
@Override
public PointValues getValues(String field) throws IOException {
+ if (merging) {
+ AssertingCodec.assertThread("PointsReader", creationThread);
+ }
PointValues values = this.in.getValues(field);
if (values == null) {
return null;
@@ -112,8 +119,8 @@ public final class AssertingPointsFormat extends PointsFormat {
}
@Override
- public PointsReader getMergeInstance() throws IOException {
- return new AssertingPointsReader(maxDoc, in.getMergeInstance());
+ public PointsReader getMergeInstance() {
+ return new AssertingPointsReader(maxDoc, in.getMergeInstance(), true);
}
@Override
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java
index e71903d..8446972 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java
@@ -114,7 +114,7 @@ public final class AssertingPostingsFormat extends PostingsFormat {
}
@Override
- public FieldsProducer getMergeInstance() throws IOException {
+ public FieldsProducer getMergeInstance() {
return new AssertingFieldsProducer(in.getMergeInstance());
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java
index e2688a1..f5455f5 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java
@@ -40,7 +40,7 @@ public class AssertingStoredFieldsFormat extends StoredFieldsFormat {
@Override
public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {
- return new AssertingStoredFieldsReader(in.fieldsReader(directory, si, fn, context), si.maxDoc());
+ return new AssertingStoredFieldsReader(in.fieldsReader(directory, si, fn, context), si.maxDoc(), false);
}
@Override
@@ -51,10 +51,14 @@ public class AssertingStoredFieldsFormat extends StoredFieldsFormat {
static class AssertingStoredFieldsReader extends StoredFieldsReader {
private final StoredFieldsReader in;
private final int maxDoc;
+ private final boolean merging;
+ private final Thread creationThread;
- AssertingStoredFieldsReader(StoredFieldsReader in, int maxDoc) {
+ AssertingStoredFieldsReader(StoredFieldsReader in, int maxDoc, boolean merging) {
this.in = in;
this.maxDoc = maxDoc;
+ this.merging = merging;
+ this.creationThread = Thread.currentThread();
// do a few simple checks on init
assert toString() != null;
assert ramBytesUsed() >= 0;
@@ -69,13 +73,15 @@ public class AssertingStoredFieldsFormat extends StoredFieldsFormat {
@Override
public void visitDocument(int n, StoredFieldVisitor visitor) throws IOException {
+ AssertingCodec.assertThread("StoredFieldsReader", creationThread);
assert n >= 0 && n < maxDoc;
in.visitDocument(n, visitor);
}
@Override
public StoredFieldsReader clone() {
- return new AssertingStoredFieldsReader(in.clone(), maxDoc);
+ assert merging == false : "Merge instances do not support cloning";
+ return new AssertingStoredFieldsReader(in.clone(), maxDoc, false);
}
@Override
@@ -98,8 +104,8 @@ public class AssertingStoredFieldsFormat extends StoredFieldsFormat {
}
@Override
- public StoredFieldsReader getMergeInstance() throws IOException {
- return new AssertingStoredFieldsReader(in.getMergeInstance(), maxDoc);
+ public StoredFieldsReader getMergeInstance() {
+ return new AssertingStoredFieldsReader(in.getMergeInstance(), maxDoc, true);
}
@Override
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java
index 000fd6f..8594adc 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java
@@ -97,7 +97,7 @@ public class AssertingTermVectorsFormat extends TermVectorsFormat {
}
@Override
- public TermVectorsReader getMergeInstance() throws IOException {
+ public TermVectorsReader getMergeInstance() {
return new AssertingTermVectorsReader(in.getMergeInstance());
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java b/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java
index aa12de7..96c5ee2 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java
@@ -1028,7 +1028,7 @@ public class AssertingLeafReader extends FilterLeafReader {
/** Wraps a SortedSetDocValues but with additional asserts */
public static class AssertingPointValues extends PointValues {
-
+ private final Thread creationThread = Thread.currentThread();
private final PointValues in;
/** Sole constructor. */
@@ -1048,11 +1048,13 @@ public class AssertingLeafReader extends FilterLeafReader {
@Override
public void intersect(IntersectVisitor visitor) throws IOException {
+ assertThread("Points", creationThread);
in.intersect(new AssertingIntersectVisitor(in.getNumDataDimensions(), in.getNumIndexDimensions(), in.getBytesPerDimension(), visitor));
}
@Override
public long estimatePointCount(IntersectVisitor visitor) {
+ assertThread("Points", creationThread);
long cost = in.estimatePointCount(visitor);
assert cost >= 0;
return cost;
@@ -1060,36 +1062,43 @@ public class AssertingLeafReader extends FilterLeafReader {
@Override
public byte[] getMinPackedValue() throws IOException {
+ assertThread("Points", creationThread);
return Objects.requireNonNull(in.getMinPackedValue());
}
@Override
public byte[] getMaxPackedValue() throws IOException {
+ assertThread("Points", creationThread);
return Objects.requireNonNull(in.getMaxPackedValue());
}
@Override
public int getNumDataDimensions() throws IOException {
+ assertThread("Points", creationThread);
return in.getNumDataDimensions();
}
@Override
public int getNumIndexDimensions() throws IOException {
+ assertThread("Points", creationThread);
return in.getNumIndexDimensions();
}
@Override
public int getBytesPerDimension() throws IOException {
+ assertThread("Points", creationThread);
return in.getBytesPerDimension();
}
@Override
public long size() {
+ assertThread("Points", creationThread);
return in.size();
}
@Override
public int getDocCount() {
+ assertThread("Points", creationThread);
return in.getDocCount();
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
index 780cfa0..5eecc77 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -131,9 +132,15 @@ abstract class BaseIndexFileFormatTestCase extends LuceneTestCase {
queue.addAll(map.values());
v = 2L * map.size() * RamUsageEstimator.NUM_BYTES_OBJECT_REF;
} else {
- v = super.accumulateObject(o, shallowSize, fieldValues, queue);
+ List<Object> references = new ArrayList<>();
+ v = super.accumulateObject(o, shallowSize, fieldValues, references);
+ for (Object r : references) {
+ // AssertingCodec adds Thread references to make sure objects are consumed in the right thread
+ if (r instanceof Thread == false) {
+ queue.add(r);
+ }
+ }
}
- // System.out.println(o.getClass() + "=" + v);
return v;
}
@@ -698,4 +705,19 @@ abstract class BaseIndexFileFormatTestCase extends LuceneTestCase {
Rethrow.rethrow(e);
}
+
+ /**
+ * Returns {@code false} if only the regular fields reader should be tested,
+ * and {@code true} if only the merge instance should be tested.
+ */
+ protected boolean shouldTestMergeInstance() {
+ return false;
+ }
+
+ protected final DirectoryReader maybeWrapWithMergingReader(DirectoryReader r) throws IOException {
+ if (shouldTestMergeInstance()) {
+ r = new MergingDirectoryReaderWrapper(r);
+ }
+ return r;
+ }
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
index e0e1f57..a308f17 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
@@ -37,6 +37,7 @@ import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.TestUtil;
import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
@@ -491,14 +492,14 @@ public abstract class BaseNormsFormatTestCase extends BaseIndexFileFormatTestCas
writer.commit();
// compare
- DirectoryReader ir = DirectoryReader.open(dir);
+ DirectoryReader ir = maybeWrapWithMergingReader(DirectoryReader.open(dir));
checkNormsVsDocValues(ir);
ir.close();
writer.forceMerge(1);
// compare again
- ir = DirectoryReader.open(dir);
+ ir = maybeWrapWithMergingReader(DirectoryReader.open(dir));
checkNormsVsDocValues(ir);
writer.close();
@@ -605,7 +606,7 @@ public abstract class BaseNormsFormatTestCase extends BaseIndexFileFormatTestCas
w.deleteDocuments(new Term("id", ""+id));
}
w.forceMerge(1);
- IndexReader r = w.getReader();
+ IndexReader r = maybeWrapWithMergingReader(w.getReader());
assertFalse(r.hasDeletions());
// Confusingly, norms should exist, and should all be 0, even though we deleted all docs that had the field "content". They should not
@@ -679,7 +680,7 @@ public abstract class BaseNormsFormatTestCase extends BaseIndexFileFormatTestCas
}
}
- DirectoryReader reader = writer.getReader();
+ DirectoryReader reader = maybeWrapWithMergingReader(writer.getReader());
writer.close();
final int numThreads = TestUtil.nextInt(random(), 3, 30);
@@ -711,4 +712,72 @@ public abstract class BaseNormsFormatTestCase extends BaseIndexFileFormatTestCas
reader.close();
dir.close();
}
+
+ public void testIndependantIterators() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig().setMergePolicy(newLogMergePolicy());
+ CannedNormSimilarity sim = new CannedNormSimilarity(new long[] {42, 10, 20});
+ conf.setSimilarity(sim);
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
+ Document doc = new Document();
+ Field indexedField = new TextField("indexed", "a", Field.Store.NO);
+ doc.add(indexedField);
+ for (int i = 0; i < 3; ++i) {
+ writer.addDocument(doc);
+ }
+ writer.forceMerge(1);
+ LeafReader r = getOnlyLeafReader(maybeWrapWithMergingReader(writer.getReader()));
+ NumericDocValues n1 = r.getNormValues("indexed");
+ NumericDocValues n2 = r.getNormValues("indexed");
+ assertEquals(0, n1.nextDoc());
+ assertEquals(42, n1.longValue());
+ assertEquals(1, n1.nextDoc());
+ assertEquals(10, n1.longValue());
+ assertEquals(0, n2.nextDoc());
+ assertEquals(42, n2.longValue());
+ assertEquals(1, n2.nextDoc());
+ assertEquals(10, n2.longValue());
+ assertEquals(2, n2.nextDoc());
+ assertEquals(20, n2.longValue());
+ assertEquals(2, n1.nextDoc());
+ assertEquals(20, n1.longValue());
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, n1.nextDoc());
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, n2.nextDoc());
+ IOUtils.close(r, writer, dir);
+ }
+
+ public void testIndependantSparseIterators() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig().setMergePolicy(newLogMergePolicy());
+ CannedNormSimilarity sim = new CannedNormSimilarity(new long[] {42, 10, 20});
+ conf.setSimilarity(sim);
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
+ Document doc = new Document();
+ Field indexedField = new TextField("indexed", "a", Field.Store.NO);
+ doc.add(indexedField);
+ Document emptyDoc = new Document();
+ for (int i = 0; i < 3; ++i) {
+ writer.addDocument(doc);
+ writer.addDocument(emptyDoc);
+ }
+ writer.forceMerge(1);
+ LeafReader r = getOnlyLeafReader(maybeWrapWithMergingReader(writer.getReader()));
+ NumericDocValues n1 = r.getNormValues("indexed");
+ NumericDocValues n2 = r.getNormValues("indexed");
+ assertEquals(0, n1.nextDoc());
+ assertEquals(42, n1.longValue());
+ assertEquals(2, n1.nextDoc());
+ assertEquals(10, n1.longValue());
+ assertEquals(0, n2.nextDoc());
+ assertEquals(42, n2.longValue());
+ assertEquals(2, n2.nextDoc());
+ assertEquals(10, n2.longValue());
+ assertEquals(4, n2.nextDoc());
+ assertEquals(20, n2.longValue());
+ assertEquals(4, n1.nextDoc());
+ assertEquals(20, n1.longValue());
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, n1.nextDoc());
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, n2.nextDoc());
+ IOUtils.close(r, writer, dir);
+ }
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
index 242abf7..d0d60bf 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
@@ -140,7 +140,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
String[] idsList = docs.keySet().toArray(new String[docs.size()]);
for(int x=0;x<2;x++) {
- IndexReader r = w.getReader();
+ DirectoryReader r = maybeWrapWithMergingReader(w.getReader());
IndexSearcher s = newSearcher(r);
if (VERBOSE) {
@@ -181,7 +181,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
doc.add(newField("aaa", "a b c", customType));
doc.add(newField("zzz", "1 2 3", customType));
w.addDocument(doc);
- IndexReader r = w.getReader();
+ IndexReader r = maybeWrapWithMergingReader(w.getReader());
Document doc2 = r.document(0);
Iterator<IndexableField> it = doc2.getFields().iterator();
assertTrue(it.hasNext());
@@ -280,7 +280,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
doc.add(new NumericDocValuesField("id", id));
w.addDocument(doc);
}
- final DirectoryReader r = w.getReader();
+ final DirectoryReader r = maybeWrapWithMergingReader(w.getReader());
w.close();
assertEquals(numDocs, r.numDocs());
@@ -309,7 +309,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
doc.add(new Field("field", "value", onlyStored));
doc.add(new StringField("field2", "value", Field.Store.YES));
w.addDocument(doc);
- IndexReader r = w.getReader();
+ IndexReader r = maybeWrapWithMergingReader(w.getReader());
w.close();
assertEquals(IndexOptions.NONE, r.document(0).getField("field").fieldType().indexOptions());
assertNotNull(r.document(0).getField("field2").fieldType().indexOptions());
@@ -352,7 +352,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
}
iw.commit();
- final DirectoryReader reader = DirectoryReader.open(dir);
+ final DirectoryReader reader = maybeWrapWithMergingReader(DirectoryReader.open(dir));
final int docID = random().nextInt(100);
for (Field fld : fields) {
String fldName = fld.name();
@@ -383,7 +383,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
iw.addDocument(emptyDoc);
}
iw.commit();
- final DirectoryReader rd = DirectoryReader.open(dir);
+ final DirectoryReader rd = maybeWrapWithMergingReader(DirectoryReader.open(dir));
for (int i = 0; i < numDocs; ++i) {
final Document doc = rd.document(i);
assertNotNull(doc);
@@ -412,7 +412,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
}
iw.commit();
- final DirectoryReader rd = DirectoryReader.open(dir);
+ final DirectoryReader rd = maybeWrapWithMergingReader(DirectoryReader.open(dir));
final IndexSearcher searcher = new IndexSearcher(rd);
final int concurrentReads = atLeast(5);
final int readsPerThread = atLeast(50);
@@ -545,7 +545,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
iw.commit();
- final DirectoryReader ir = DirectoryReader.open(dir);
+ final DirectoryReader ir = maybeWrapWithMergingReader(DirectoryReader.open(dir));
assertTrue(ir.numDocs() > 0);
int numDocs = 0;
for (int i = 0; i < ir.maxDoc(); ++i) {
@@ -649,7 +649,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
w.commit();
w.close();
- DirectoryReader reader = new DummyFilterDirectoryReader(DirectoryReader.open(dir));
+ DirectoryReader reader = new DummyFilterDirectoryReader(maybeWrapWithMergingReader(DirectoryReader.open(dir)));
Directory dir2 = newDirectory();
w = new RandomIndexWriter(random(), dir2);
@@ -657,7 +657,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
reader.close();
dir.close();
- reader = w.getReader();
+ reader = maybeWrapWithMergingReader(w.getReader());
for (int i = 0; i < reader.maxDoc(); ++i) {
final Document doc = reader.document(i);
final int id = doc.getField("id").numericValue().intValue();
@@ -728,7 +728,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
}
iw.commit();
iw.forceMerge(1); // look at what happens when big docs are merged
- final DirectoryReader rd = DirectoryReader.open(dir);
+ final DirectoryReader rd = maybeWrapWithMergingReader(DirectoryReader.open(dir));
final IndexSearcher searcher = new IndexSearcher(rd);
for (int i = 0; i < numDocs; ++i) {
final Query query = new TermQuery(new Term("id", "" + i));
@@ -788,7 +788,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
iw.addDocument(doc);
}
- DirectoryReader reader = DirectoryReader.open(iw);
+ DirectoryReader reader = maybeWrapWithMergingReader(DirectoryReader.open(iw));
// mix up fields explicitly
if (random().nextBoolean()) {
reader = new MismatchedDirectoryReader(reader, random());
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/MergingCodecReader.java b/lucene/test-framework/src/java/org/apache/lucene/index/MergingCodecReader.java
new file mode 100644
index 0000000..41c80ad
--- /dev/null
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/MergingCodecReader.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+import org.apache.lucene.codecs.NormsProducer;
+import org.apache.lucene.codecs.StoredFieldsReader;
+import org.apache.lucene.util.CloseableThreadLocal;
+
+/**
+ * {@link CodecReader} wrapper that performs all reads using the merging
+ * instance of the index formats.
+ */
+public class MergingCodecReader extends FilterCodecReader {
+
+ private final CloseableThreadLocal<StoredFieldsReader> fieldsReader = new CloseableThreadLocal<StoredFieldsReader>() {
+ @Override
+ protected StoredFieldsReader initialValue() {
+ return in.getFieldsReader().getMergeInstance();
+ }
+ };
+ private final CloseableThreadLocal<NormsProducer> normsReader = new CloseableThreadLocal<NormsProducer>() {
+ @Override
+ protected NormsProducer initialValue() {
+ NormsProducer norms = in.getNormsReader();
+ if (norms == null) {
+ return null;
+ } else {
+ return norms.getMergeInstance();
+ }
+ }
+ };
+ // TODO: other formats too
+
+ /** Wrap the given instance. */
+ public MergingCodecReader(CodecReader in) {
+ super(in);
+ }
+
+ @Override
+ public StoredFieldsReader getFieldsReader() {
+ return fieldsReader.get();
+ }
+
+ @Override
+ public NormsProducer getNormsReader() {
+ return normsReader.get();
+ }
+
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ // same content, we can delegate
+ return in.getCoreCacheHelper();
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ // same content, we can delegate
+ return in.getReaderCacheHelper();
+ }
+
+}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/MergingDirectoryReaderWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/index/MergingDirectoryReaderWrapper.java
new file mode 100644
index 0000000..d587bcd
--- /dev/null
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/MergingDirectoryReaderWrapper.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+import java.io.IOException;
+
+/**
+ * {@link DirectoryReader} wrapper that uses the merge instances of the wrapped
+ * {@link CodecReader}s.
+ * NOTE: This class will fail to work if the leaves of the wrapped directory are
+ * not codec readers.
+ */
+public final class MergingDirectoryReaderWrapper extends FilterDirectoryReader {
+
+ /** Wrap the given directory. */
+ public MergingDirectoryReaderWrapper(DirectoryReader in) throws IOException {
+ super(in, new SubReaderWrapper() {
+ @Override
+ public LeafReader wrap(LeafReader reader) {
+ return new MergingCodecReader((CodecReader) reader);
+ }
+ });
+ }
+
+ @Override
+ protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException {
+ return new MergingDirectoryReaderWrapper(in);
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ // doesn't change the content: can delegate
+ return in.getReaderCacheHelper();
+ }
+
+}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
index 1830dfc..eeb0906 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
@@ -1669,7 +1669,7 @@ public abstract class LuceneTestCase extends Assert {
Random random = random();
for (int i = 0, c = random.nextInt(6)+1; i < c; i++) {
- switch(random.nextInt(4)) {
+ switch(random.nextInt(5)) {
case 0:
// will create no FC insanity in atomic case, as ParallelLeafReader has own cache key:
if (VERBOSE) {
@@ -1722,6 +1722,25 @@ public abstract class LuceneTestCase extends Assert {
r = new MismatchedDirectoryReader((DirectoryReader)r, random);
}
break;
+ case 4:
+ if (VERBOSE) {
+ System.out.println("NOTE: LuceneTestCase.wrapReader: wrapping previous reader=" + r + " with MergingCodecReader");
+ }
+ if (r instanceof CodecReader) {
+ r = new MergingCodecReader((CodecReader) r);
+ } else if (r instanceof DirectoryReader) {
+ boolean allLeavesAreCodecReaders = true;
+ for (LeafReaderContext ctx : r.leaves()) {
+ if (ctx.reader() instanceof CodecReader == false) {
+ allLeavesAreCodecReaders = false;
+ break;
+ }
+ }
+ if (allLeavesAreCodecReaders) {
+ r = new MergingDirectoryReaderWrapper((DirectoryReader) r);
+ }
+ }
+ break;
default:
fail("should not get here");
}