You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2019/03/19 12:35:23 UTC
[lucene-solr] branch branch_8x updated (105979f -> 83c3030)
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a change to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
from 105979f SOLR-8033: Remove debug if branch in HdfsTransactionLog
new 1d9f00f LUCENE-8166: Require merge instances to be consumed in the thread that created them.
new 83c3030 LUCENE-8138: Check that dv producers's next/advance and advanceExact impls are consistent.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../codecs/lucene70/Lucene70NormsProducer.java | 76 +++++++++++++++++-
.../codecs/memory/DirectDocValuesProducer.java | 4 +-
.../apache/lucene/codecs/DocValuesProducer.java | 5 +-
.../org/apache/lucene/codecs/FieldsProducer.java | 5 +-
.../org/apache/lucene/codecs/NormsProducer.java | 5 +-
.../org/apache/lucene/codecs/PointsReader.java | 5 +-
.../apache/lucene/codecs/StoredFieldsReader.java | 4 +-
.../apache/lucene/codecs/TermVectorsReader.java | 5 +-
.../codecs/lucene80/Lucene80NormsProducer.java | 83 +++++++++++++++++---
.../codecs/perfield/PerFieldDocValuesFormat.java | 4 +-
.../codecs/perfield/PerFieldPostingsFormat.java | 4 +-
.../java/org/apache/lucene/index/CheckIndex.java | 91 +++++++++++++++-------
...stLucene50StoredFieldsFormatMergeInstance.java} | 14 +++-
.../TestLucene80NormsFormatMergeInstance.java | 11 ++-
.../apache/lucene/index/TestMultiTermsEnum.java | 2 +-
.../suggest/document/CompletionFieldsProducer.java | 2 +-
.../lucene/codecs/asserting/AssertingCodec.java | 8 ++
.../codecs/asserting/AssertingDocValuesFormat.java | 27 ++++++-
.../codecs/asserting/AssertingNormsFormat.java | 15 +++-
.../codecs/asserting/AssertingPointsFormat.java | 15 +++-
.../codecs/asserting/AssertingPostingsFormat.java | 2 +-
.../asserting/AssertingStoredFieldsFormat.java | 16 ++--
.../asserting/AssertingTermVectorsFormat.java | 2 +-
.../apache/lucene/index/AssertingLeafReader.java | 11 ++-
.../lucene/index/BaseIndexFileFormatTestCase.java | 26 ++++++-
.../lucene/index/BaseNormsFormatTestCase.java | 77 +++++++++++++++++-
.../index/BaseStoredFieldsFormatTestCase.java | 24 +++---
.../apache/lucene/index/MergingCodecReader.java | 75 ++++++++++++++++++
...der.java => MergingDirectoryReaderWrapper.java} | 28 +++----
.../org/apache/lucene/util/LuceneTestCase.java | 21 ++++-
30 files changed, 549 insertions(+), 118 deletions(-)
copy lucene/core/src/{java/org/apache/lucene/codecs/lucene50/package-info.java => test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatMergeInstance.java} (77%)
copy solr/core/src/test/org/apache/solr/cloud/ForceLeaderWithTlogReplicasTest.java => lucene/core/src/test/org/apache/lucene/codecs/lucene80/TestLucene80NormsFormatMergeInstance.java (77%)
create mode 100644 lucene/test-framework/src/java/org/apache/lucene/index/MergingCodecReader.java
copy lucene/test-framework/src/java/org/apache/lucene/index/{AssertingDirectoryReader.java => MergingDirectoryReaderWrapper.java} (61%)
[lucene-solr] 01/02: LUCENE-8166: Require merge instances to be
consumed in the thread that created them.
Posted by jp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 1d9f00fd5e7323c0ed082e3c5b31d9c64ffa479d
Author: Adrien Grand <jp...@gmail.com>
AuthorDate: Fri Mar 15 14:16:37 2019 +0100
LUCENE-8166: Require merge instances to be consumed in the thread that created them.
---
.../codecs/lucene70/Lucene70NormsProducer.java | 76 +++++++++++++++++++-
.../codecs/memory/DirectDocValuesProducer.java | 4 +-
.../apache/lucene/codecs/DocValuesProducer.java | 5 +-
.../org/apache/lucene/codecs/FieldsProducer.java | 5 +-
.../org/apache/lucene/codecs/NormsProducer.java | 5 +-
.../org/apache/lucene/codecs/PointsReader.java | 5 +-
.../apache/lucene/codecs/StoredFieldsReader.java | 4 +-
.../apache/lucene/codecs/TermVectorsReader.java | 5 +-
.../codecs/lucene80/Lucene80NormsProducer.java | 83 +++++++++++++++++++---
.../codecs/perfield/PerFieldDocValuesFormat.java | 4 +-
.../codecs/perfield/PerFieldPostingsFormat.java | 4 +-
...estLucene50StoredFieldsFormatMergeInstance.java | 29 ++++++++
.../TestLucene80NormsFormatMergeInstance.java | 29 ++++++++
.../apache/lucene/index/TestMultiTermsEnum.java | 2 +-
.../suggest/document/CompletionFieldsProducer.java | 2 +-
.../lucene/codecs/asserting/AssertingCodec.java | 8 +++
.../codecs/asserting/AssertingDocValuesFormat.java | 27 +++++--
.../codecs/asserting/AssertingNormsFormat.java | 15 ++--
.../codecs/asserting/AssertingPointsFormat.java | 15 ++--
.../codecs/asserting/AssertingPostingsFormat.java | 2 +-
.../asserting/AssertingStoredFieldsFormat.java | 16 +++--
.../asserting/AssertingTermVectorsFormat.java | 2 +-
.../apache/lucene/index/AssertingLeafReader.java | 11 ++-
.../lucene/index/BaseIndexFileFormatTestCase.java | 26 ++++++-
.../lucene/index/BaseNormsFormatTestCase.java | 77 ++++++++++++++++++--
.../index/BaseStoredFieldsFormatTestCase.java | 24 +++----
.../apache/lucene/index/MergingCodecReader.java | 75 +++++++++++++++++++
.../index/MergingDirectoryReaderWrapper.java | 50 +++++++++++++
.../org/apache/lucene/util/LuceneTestCase.java | 21 +++++-
29 files changed, 560 insertions(+), 71 deletions(-)
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene70/Lucene70NormsProducer.java b/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene70/Lucene70NormsProducer.java
index c7310e8..3a2964e 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene70/Lucene70NormsProducer.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene70/Lucene70NormsProducer.java
@@ -91,7 +91,7 @@ final class Lucene70NormsProducer extends NormsProducer implements Cloneable {
}
@Override
- public NormsProducer getMergeInstance() throws IOException {
+ public NormsProducer getMergeInstance() {
Lucene70NormsProducer clone;
try {
clone = (Lucene70NormsProducer) super.clone();
@@ -227,6 +227,80 @@ final class Lucene70NormsProducer extends NormsProducer implements Cloneable {
}
private IndexInput getDisiInput(FieldInfo field, NormsEntry entry) throws IOException {
+ if (merging == false) {
+ return data.slice("docs", entry.docsWithFieldOffset, entry.docsWithFieldLength);
+ }
+
+ IndexInput in = disiInputs.get(field.number);
+ if (in == null) {
+ in = data.slice("docs", entry.docsWithFieldOffset, entry.docsWithFieldLength);
+ disiInputs.put(field.number, in);
+ }
+
+ final IndexInput inF = in; // same as in but final
+
+ // Wrap so that reads can be interleaved from the same thread if two
+ // norms instances are pulled and consumed in parallel. Merging usually
+ // doesn't need this feature but CheckIndex might, plus we need merge
+ // instances to behave well and not be trappy.
+ return new IndexInput("docs") {
+
+ long offset = 0;
+
+ @Override
+ public void readBytes(byte[] b, int off, int len) throws IOException {
+ inF.seek(offset);
+ offset += len;
+ inF.readBytes(b, off, len);
+ }
+
+ @Override
+ public byte readByte() throws IOException {
+ throw new UnsupportedOperationException("Unused by IndexedDISI");
+ }
+
+ @Override
+ public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
+ throw new UnsupportedOperationException("Unused by IndexedDISI");
+ }
+
+ @Override
+ public short readShort() throws IOException {
+ inF.seek(offset);
+ offset += Short.BYTES;
+ return inF.readShort();
+ }
+
+ @Override
+ public long readLong() throws IOException {
+ inF.seek(offset);
+ offset += Long.BYTES;
+ return inF.readLong();
+ }
+
+ @Override
+ public void seek(long pos) throws IOException {
+ offset = pos;
+ }
+
+ @Override
+ public long length() {
+ throw new UnsupportedOperationException("Unused by IndexedDISI");
+ }
+
+ @Override
+ public long getFilePointer() {
+ return offset;
+ }
+
+ @Override
+ public void close() throws IOException {
+ throw new UnsupportedOperationException("Unused by IndexedDISI");
+ }
+ };
+ }
+
+ private IndexInput getDisiInput2(FieldInfo field, NormsEntry entry) throws IOException {
IndexInput slice = null;
if (merging) {
slice = disiInputs.get(field.number);
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java
index 96cd996..baef4db 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java
@@ -80,7 +80,7 @@ class DirectDocValuesProducer extends DocValuesProducer {
static final int VERSION_CURRENT = VERSION_START;
// clone for merge: when merging we don't do any instances.put()s
- DirectDocValuesProducer(DirectDocValuesProducer original) throws IOException {
+ DirectDocValuesProducer(DirectDocValuesProducer original) {
assert Thread.holdsLock(original);
numerics.putAll(original.numerics);
binaries.putAll(original.binaries);
@@ -606,7 +606,7 @@ class DirectDocValuesProducer extends DocValuesProducer {
}
@Override
- public synchronized DocValuesProducer getMergeInstance() throws IOException {
+ public synchronized DocValuesProducer getMergeInstance() {
return new DirectDocValuesProducer(this);
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java
index 9296f19..5fe0b33 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java
@@ -74,10 +74,11 @@ public abstract class DocValuesProducer implements Closeable, Accountable {
public abstract void checkIntegrity() throws IOException;
/**
- * Returns an instance optimized for merging.
+ * Returns an instance optimized for merging. This instance may only be
+ * consumed in the thread that called {@link #getMergeInstance()}.
* <p>
* The default implementation returns {@code this} */
- public DocValuesProducer getMergeInstance() throws IOException {
+ public DocValuesProducer getMergeInstance() {
return this;
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/FieldsProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/FieldsProducer.java
index cd6386c..481b160 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/FieldsProducer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/FieldsProducer.java
@@ -48,10 +48,11 @@ public abstract class FieldsProducer extends Fields implements Closeable, Accoun
public abstract void checkIntegrity() throws IOException;
/**
- * Returns an instance optimized for merging.
+ * Returns an instance optimized for merging. This instance may only be
+ * consumed in the thread that called {@link #getMergeInstance()}.
* <p>
* The default implementation returns {@code this} */
- public FieldsProducer getMergeInstance() throws IOException {
+ public FieldsProducer getMergeInstance() {
return this;
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/NormsProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/NormsProducer.java
index 39f9612..647d9e9 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/NormsProducer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/NormsProducer.java
@@ -49,10 +49,11 @@ public abstract class NormsProducer implements Closeable, Accountable {
public abstract void checkIntegrity() throws IOException;
/**
- * Returns an instance optimized for merging.
+ * Returns an instance optimized for merging. This instance may only be used
+ * from the thread that acquires it.
* <p>
* The default implementation returns {@code this} */
- public NormsProducer getMergeInstance() throws IOException {
+ public NormsProducer getMergeInstance() {
return this;
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/PointsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/PointsReader.java
index b20614a..213b72e 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/PointsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/PointsReader.java
@@ -45,10 +45,11 @@ public abstract class PointsReader implements Closeable, Accountable {
public abstract PointValues getValues(String field) throws IOException;
/**
- * Returns an instance optimized for merging.
+ * Returns an instance optimized for merging. This instance may only be used
+ * in the thread that acquires it.
* <p>
* The default implementation returns {@code this} */
- public PointsReader getMergeInstance() throws IOException {
+ public PointsReader getMergeInstance() {
return this;
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsReader.java
index 6258df5..1f32576 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsReader.java
@@ -52,10 +52,10 @@ public abstract class StoredFieldsReader implements Cloneable, Closeable, Accoun
public abstract void checkIntegrity() throws IOException;
/**
- * Returns an instance optimized for merging.
+ * Returns an instance optimized for merging. This instance may not be cloned.
* <p>
* The default implementation returns {@code this} */
- public StoredFieldsReader getMergeInstance() throws IOException {
+ public StoredFieldsReader getMergeInstance() {
return this;
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsReader.java
index 7104136..dc9115d 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsReader.java
@@ -57,10 +57,11 @@ public abstract class TermVectorsReader implements Cloneable, Closeable, Account
public abstract TermVectorsReader clone();
/**
- * Returns an instance optimized for merging.
+ * Returns an instance optimized for merging. This instance may only be
+ * consumed in the thread that called {@link #getMergeInstance()}.
* <p>
* The default implementation returns {@code this} */
- public TermVectorsReader getMergeInstance() throws IOException {
+ public TermVectorsReader getMergeInstance() {
return this;
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene80/Lucene80NormsProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene80/Lucene80NormsProducer.java
index 66126a2..823cfff 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene80/Lucene80NormsProducer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene80/Lucene80NormsProducer.java
@@ -92,7 +92,7 @@ final class Lucene80NormsProducer extends NormsProducer implements Cloneable {
}
@Override
- public NormsProducer getMergeInstance() throws IOException {
+ public NormsProducer getMergeInstance() {
Lucene80NormsProducer clone;
try {
clone = (Lucene80NormsProducer) super.clone();
@@ -233,18 +233,79 @@ final class Lucene80NormsProducer extends NormsProducer implements Cloneable {
}
private IndexInput getDisiInput(FieldInfo field, NormsEntry entry) throws IOException {
- IndexInput slice = null;
- if (merging) {
- slice = disiInputs.get(field.number);
+ if (merging == false) {
+ return IndexedDISI.createBlockSlice(
+ data, "docs", entry.docsWithFieldOffset, entry.docsWithFieldLength, entry.jumpTableEntryCount);
}
- if (slice == null) {
- slice = IndexedDISI.createBlockSlice(
+
+ IndexInput in = disiInputs.get(field.number);
+ if (in == null) {
+ in = IndexedDISI.createBlockSlice(
data, "docs", entry.docsWithFieldOffset, entry.docsWithFieldLength, entry.jumpTableEntryCount);
- if (merging) {
- disiInputs.put(field.number, slice);
- }
+ disiInputs.put(field.number, in);
}
- return slice;
+
+ final IndexInput inF = in; // same as in but final
+
+ // Wrap so that reads can be interleaved from the same thread if two
+ // norms instances are pulled and consumed in parallel. Merging usually
+ // doesn't need this feature but CheckIndex might, plus we need merge
+ // instances to behave well and not be trappy.
+ return new IndexInput("docs") {
+
+ long offset = 0;
+
+ @Override
+ public void readBytes(byte[] b, int off, int len) throws IOException {
+ inF.seek(offset);
+ offset += len;
+ inF.readBytes(b, off, len);
+ }
+
+ @Override
+ public byte readByte() throws IOException {
+ throw new UnsupportedOperationException("Unused by IndexedDISI");
+ }
+
+ @Override
+ public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
+ throw new UnsupportedOperationException("Unused by IndexedDISI");
+ }
+
+ @Override
+ public short readShort() throws IOException {
+ inF.seek(offset);
+ offset += Short.BYTES;
+ return inF.readShort();
+ }
+
+ @Override
+ public long readLong() throws IOException {
+ inF.seek(offset);
+ offset += Long.BYTES;
+ return inF.readLong();
+ }
+
+ @Override
+ public void seek(long pos) throws IOException {
+ offset = pos;
+ }
+
+ @Override
+ public long length() {
+ throw new UnsupportedOperationException("Unused by IndexedDISI");
+ }
+
+ @Override
+ public long getFilePointer() {
+ return offset;
+ }
+
+ @Override
+ public void close() throws IOException {
+ throw new UnsupportedOperationException("Unused by IndexedDISI");
+ }
+ };
}
private RandomAccessInput getDisiJumpTable(FieldInfo field, NormsEntry entry) throws IOException {
@@ -327,7 +388,7 @@ final class Lucene80NormsProducer extends NormsProducer implements Cloneable {
}
};
}
- final RandomAccessInput slice = data.randomAccessSlice(entry.normsOffset, entry.numDocsWithField * (long) entry.bytesPerNorm);
+ final RandomAccessInput slice = getDataInput(field, entry);
switch (entry.bytesPerNorm) {
case 1:
return new SparseNormsIterator(disi) {
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
index f439699..f2e8940 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
@@ -261,7 +261,7 @@ public abstract class PerFieldDocValuesFormat extends DocValuesFormat {
private final Map<String,DocValuesProducer> formats = new HashMap<>();
// clone for merge
- FieldsReader(FieldsReader other) throws IOException {
+ FieldsReader(FieldsReader other) {
Map<DocValuesProducer,DocValuesProducer> oldToNew = new IdentityHashMap<>();
// First clone all formats
for(Map.Entry<String,DocValuesProducer> ent : other.formats.entrySet()) {
@@ -368,7 +368,7 @@ public abstract class PerFieldDocValuesFormat extends DocValuesFormat {
}
@Override
- public DocValuesProducer getMergeInstance() throws IOException {
+ public DocValuesProducer getMergeInstance() {
return new FieldsReader(this);
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java
index 88ae6da..81bbf72 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java
@@ -247,7 +247,7 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat {
private final String segment;
// clone for merge
- FieldsReader(FieldsReader other) throws IOException {
+ FieldsReader(FieldsReader other) {
Map<FieldsProducer,FieldsProducer> oldToNew = new IdentityHashMap<>();
// First clone all formats
for(Map.Entry<String,FieldsProducer> ent : other.formats.entrySet()) {
@@ -346,7 +346,7 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat {
}
@Override
- public FieldsProducer getMergeInstance() throws IOException {
+ public FieldsProducer getMergeInstance() {
return new FieldsReader(this);
}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatMergeInstance.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatMergeInstance.java
new file mode 100644
index 0000000..d0f3157
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatMergeInstance.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene50;
+
+/**
+ * Test the merge instance of the Lucene50 stored fields format.
+ */
+public class TestLucene50StoredFieldsFormatMergeInstance extends TestLucene50StoredFieldsFormat {
+
+ @Override
+ protected boolean shouldTestMergeInstance() {
+ return true;
+ }
+
+}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene80/TestLucene80NormsFormatMergeInstance.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene80/TestLucene80NormsFormatMergeInstance.java
new file mode 100644
index 0000000..aed0c8b
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene80/TestLucene80NormsFormatMergeInstance.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene80;
+
+/**
+ * Test the merge instance of the Lucene80 norms format.
+ */
+public class TestLucene80NormsFormatMergeInstance extends TestLucene80NormsFormat {
+
+ @Override
+ protected boolean shouldTestMergeInstance() {
+ return true;
+ }
+
+}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestMultiTermsEnum.java b/lucene/core/src/test/org/apache/lucene/index/TestMultiTermsEnum.java
index aa325a1..e7f87bc 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestMultiTermsEnum.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestMultiTermsEnum.java
@@ -222,7 +222,7 @@ public class TestMultiTermsEnum extends LuceneTestCase {
}
@Override
- public FieldsProducer getMergeInstance() throws IOException {
+ public FieldsProducer getMergeInstance() {
return create(delegate.getMergeInstance(), newFieldInfo);
}
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsProducer.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsProducer.java
index 7a29b61..b998f8e 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsProducer.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsProducer.java
@@ -133,7 +133,7 @@ final class CompletionFieldsProducer extends FieldsProducer {
}
@Override
- public FieldsProducer getMergeInstance() throws IOException {
+ public FieldsProducer getMergeInstance() {
return new CompletionFieldsProducer(delegateFieldsProducer, readers);
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java
index 15bcfa2..5879118 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java
@@ -33,6 +33,14 @@ import org.apache.lucene.util.TestUtil;
*/
public class AssertingCodec extends FilterCodec {
+ static void assertThread(String object, Thread creationThread) {
+ if (creationThread != Thread.currentThread()) {
+ throw new AssertionError(object + " are only supposed to be consumed in "
+ + "the thread in which they have been acquired. But was acquired in "
+ + creationThread + " and consumed in " + Thread.currentThread() + ".");
+ }
+ }
+
private final PostingsFormat postings = new PerFieldPostingsFormat() {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
index 76ab1df..fd8c246 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
@@ -62,7 +62,7 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
assert state.fieldInfos.hasDocValues();
DocValuesProducer producer = in.fieldsProducer(state);
assert producer != null;
- return new AssertingDocValuesProducer(producer, state.segmentInfo.maxDoc());
+ return new AssertingDocValuesProducer(producer, state.segmentInfo.maxDoc(), false);
}
static class AssertingDocValuesConsumer extends DocValuesConsumer {
@@ -219,10 +219,14 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
static class AssertingDocValuesProducer extends DocValuesProducer {
private final DocValuesProducer in;
private final int maxDoc;
+ private final boolean merging;
+ private final Thread creationThread;
- AssertingDocValuesProducer(DocValuesProducer in, int maxDoc) {
+ AssertingDocValuesProducer(DocValuesProducer in, int maxDoc, boolean merging) {
this.in = in;
this.maxDoc = maxDoc;
+ this.merging = merging;
+ this.creationThread = Thread.currentThread();
// do a few simple checks on init
assert toString() != null;
assert ramBytesUsed() >= 0;
@@ -231,6 +235,9 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
@Override
public NumericDocValues getNumeric(FieldInfo field) throws IOException {
+ if (merging) {
+ AssertingCodec.assertThread("DocValuesProducer", creationThread);
+ }
assert field.getDocValuesType() == DocValuesType.NUMERIC;
NumericDocValues values = in.getNumeric(field);
assert values != null;
@@ -239,6 +246,9 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
@Override
public BinaryDocValues getBinary(FieldInfo field) throws IOException {
+ if (merging) {
+ AssertingCodec.assertThread("DocValuesProducer", creationThread);
+ }
assert field.getDocValuesType() == DocValuesType.BINARY;
BinaryDocValues values = in.getBinary(field);
assert values != null;
@@ -247,6 +257,9 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
@Override
public SortedDocValues getSorted(FieldInfo field) throws IOException {
+ if (merging) {
+ AssertingCodec.assertThread("DocValuesProducer", creationThread);
+ }
assert field.getDocValuesType() == DocValuesType.SORTED;
SortedDocValues values = in.getSorted(field);
assert values != null;
@@ -255,6 +268,9 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
@Override
public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
+ if (merging) {
+ AssertingCodec.assertThread("DocValuesProducer", creationThread);
+ }
assert field.getDocValuesType() == DocValuesType.SORTED_NUMERIC;
SortedNumericDocValues values = in.getSortedNumeric(field);
assert values != null;
@@ -263,6 +279,9 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
@Override
public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
+ if (merging) {
+ AssertingCodec.assertThread("DocValuesProducer", creationThread);
+ }
assert field.getDocValuesType() == DocValuesType.SORTED_SET;
SortedSetDocValues values = in.getSortedSet(field);
assert values != null;
@@ -295,8 +314,8 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
}
@Override
- public DocValuesProducer getMergeInstance() throws IOException {
- return new AssertingDocValuesProducer(in.getMergeInstance(), maxDoc);
+ public DocValuesProducer getMergeInstance() {
+ return new AssertingDocValuesProducer(in.getMergeInstance(), maxDoc, true);
}
@Override
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java
index bf830bf..937b8f6 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java
@@ -50,7 +50,7 @@ public class AssertingNormsFormat extends NormsFormat {
assert state.fieldInfos.hasNorms();
NormsProducer producer = in.normsProducer(state);
assert producer != null;
- return new AssertingNormsProducer(producer, state.segmentInfo.maxDoc());
+ return new AssertingNormsProducer(producer, state.segmentInfo.maxDoc(), false);
}
static class AssertingNormsConsumer extends NormsConsumer {
@@ -88,10 +88,14 @@ public class AssertingNormsFormat extends NormsFormat {
static class AssertingNormsProducer extends NormsProducer {
private final NormsProducer in;
private final int maxDoc;
+ private final boolean merging;
+ private final Thread creationThread;
- AssertingNormsProducer(NormsProducer in, int maxDoc) {
+ AssertingNormsProducer(NormsProducer in, int maxDoc, boolean merging) {
this.in = in;
this.maxDoc = maxDoc;
+ this.merging = merging;
+ this.creationThread = Thread.currentThread();
// do a few simple checks on init
assert toString() != null;
assert ramBytesUsed() >= 0;
@@ -100,6 +104,9 @@ public class AssertingNormsFormat extends NormsFormat {
@Override
public NumericDocValues getNorms(FieldInfo field) throws IOException {
+ if (merging) {
+ AssertingCodec.assertThread("NormsProducer", creationThread);
+ }
assert field.hasNorms();
NumericDocValues values = in.getNorms(field);
assert values != null;
@@ -132,8 +139,8 @@ public class AssertingNormsFormat extends NormsFormat {
}
@Override
- public NormsProducer getMergeInstance() throws IOException {
- return new AssertingNormsProducer(in.getMergeInstance(), maxDoc);
+ public NormsProducer getMergeInstance() {
+ return new AssertingNormsProducer(in.getMergeInstance(), maxDoc, true);
}
@Override
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointsFormat.java
index 4943b99..79dfa5f 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointsFormat.java
@@ -60,17 +60,21 @@ public final class AssertingPointsFormat extends PointsFormat {
@Override
public PointsReader fieldsReader(SegmentReadState state) throws IOException {
- return new AssertingPointsReader(state.segmentInfo.maxDoc(), in.fieldsReader(state));
+ return new AssertingPointsReader(state.segmentInfo.maxDoc(), in.fieldsReader(state), false);
}
static class AssertingPointsReader extends PointsReader {
private final PointsReader in;
private final int maxDoc;
+ private final boolean merging;
+ private final Thread creationThread;
- AssertingPointsReader(int maxDoc, PointsReader in) {
+ AssertingPointsReader(int maxDoc, PointsReader in, boolean merging) {
this.in = in;
this.maxDoc = maxDoc;
+ this.merging = merging;
+ this.creationThread = Thread.currentThread();
// do a few simple checks on init
assert toString() != null;
assert ramBytesUsed() >= 0;
@@ -85,6 +89,9 @@ public final class AssertingPointsFormat extends PointsFormat {
@Override
public PointValues getValues(String field) throws IOException {
+ if (merging) {
+ AssertingCodec.assertThread("PointsReader", creationThread);
+ }
PointValues values = this.in.getValues(field);
if (values == null) {
return null;
@@ -112,8 +119,8 @@ public final class AssertingPointsFormat extends PointsFormat {
}
@Override
- public PointsReader getMergeInstance() throws IOException {
- return new AssertingPointsReader(maxDoc, in.getMergeInstance());
+ public PointsReader getMergeInstance() {
+ return new AssertingPointsReader(maxDoc, in.getMergeInstance(), true);
}
@Override
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java
index e71903d..8446972 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java
@@ -114,7 +114,7 @@ public final class AssertingPostingsFormat extends PostingsFormat {
}
@Override
- public FieldsProducer getMergeInstance() throws IOException {
+ public FieldsProducer getMergeInstance() {
return new AssertingFieldsProducer(in.getMergeInstance());
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java
index e2688a1..f5455f5 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java
@@ -40,7 +40,7 @@ public class AssertingStoredFieldsFormat extends StoredFieldsFormat {
@Override
public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {
- return new AssertingStoredFieldsReader(in.fieldsReader(directory, si, fn, context), si.maxDoc());
+ return new AssertingStoredFieldsReader(in.fieldsReader(directory, si, fn, context), si.maxDoc(), false);
}
@Override
@@ -51,10 +51,14 @@ public class AssertingStoredFieldsFormat extends StoredFieldsFormat {
static class AssertingStoredFieldsReader extends StoredFieldsReader {
private final StoredFieldsReader in;
private final int maxDoc;
+ private final boolean merging;
+ private final Thread creationThread;
- AssertingStoredFieldsReader(StoredFieldsReader in, int maxDoc) {
+ AssertingStoredFieldsReader(StoredFieldsReader in, int maxDoc, boolean merging) {
this.in = in;
this.maxDoc = maxDoc;
+ this.merging = merging;
+ this.creationThread = Thread.currentThread();
// do a few simple checks on init
assert toString() != null;
assert ramBytesUsed() >= 0;
@@ -69,13 +73,15 @@ public class AssertingStoredFieldsFormat extends StoredFieldsFormat {
@Override
public void visitDocument(int n, StoredFieldVisitor visitor) throws IOException {
+ AssertingCodec.assertThread("StoredFieldsReader", creationThread);
assert n >= 0 && n < maxDoc;
in.visitDocument(n, visitor);
}
@Override
public StoredFieldsReader clone() {
- return new AssertingStoredFieldsReader(in.clone(), maxDoc);
+ assert merging == false : "Merge instances do not support cloning";
+ return new AssertingStoredFieldsReader(in.clone(), maxDoc, false);
}
@Override
@@ -98,8 +104,8 @@ public class AssertingStoredFieldsFormat extends StoredFieldsFormat {
}
@Override
- public StoredFieldsReader getMergeInstance() throws IOException {
- return new AssertingStoredFieldsReader(in.getMergeInstance(), maxDoc);
+ public StoredFieldsReader getMergeInstance() {
+ return new AssertingStoredFieldsReader(in.getMergeInstance(), maxDoc, true);
}
@Override
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java
index 000fd6f..8594adc 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java
@@ -97,7 +97,7 @@ public class AssertingTermVectorsFormat extends TermVectorsFormat {
}
@Override
- public TermVectorsReader getMergeInstance() throws IOException {
+ public TermVectorsReader getMergeInstance() {
return new AssertingTermVectorsReader(in.getMergeInstance());
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java b/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java
index aa12de7..96c5ee2 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java
@@ -1028,7 +1028,7 @@ public class AssertingLeafReader extends FilterLeafReader {
/** Wraps a SortedSetDocValues but with additional asserts */
public static class AssertingPointValues extends PointValues {
-
+ private final Thread creationThread = Thread.currentThread();
private final PointValues in;
/** Sole constructor. */
@@ -1048,11 +1048,13 @@ public class AssertingLeafReader extends FilterLeafReader {
@Override
public void intersect(IntersectVisitor visitor) throws IOException {
+ assertThread("Points", creationThread);
in.intersect(new AssertingIntersectVisitor(in.getNumDataDimensions(), in.getNumIndexDimensions(), in.getBytesPerDimension(), visitor));
}
@Override
public long estimatePointCount(IntersectVisitor visitor) {
+ assertThread("Points", creationThread);
long cost = in.estimatePointCount(visitor);
assert cost >= 0;
return cost;
@@ -1060,36 +1062,43 @@ public class AssertingLeafReader extends FilterLeafReader {
@Override
public byte[] getMinPackedValue() throws IOException {
+ assertThread("Points", creationThread);
return Objects.requireNonNull(in.getMinPackedValue());
}
@Override
public byte[] getMaxPackedValue() throws IOException {
+ assertThread("Points", creationThread);
return Objects.requireNonNull(in.getMaxPackedValue());
}
@Override
public int getNumDataDimensions() throws IOException {
+ assertThread("Points", creationThread);
return in.getNumDataDimensions();
}
@Override
public int getNumIndexDimensions() throws IOException {
+ assertThread("Points", creationThread);
return in.getNumIndexDimensions();
}
@Override
public int getBytesPerDimension() throws IOException {
+ assertThread("Points", creationThread);
return in.getBytesPerDimension();
}
@Override
public long size() {
+ assertThread("Points", creationThread);
return in.size();
}
@Override
public int getDocCount() {
+ assertThread("Points", creationThread);
return in.getDocCount();
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
index 780cfa0..5eecc77 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -131,9 +132,15 @@ abstract class BaseIndexFileFormatTestCase extends LuceneTestCase {
queue.addAll(map.values());
v = 2L * map.size() * RamUsageEstimator.NUM_BYTES_OBJECT_REF;
} else {
- v = super.accumulateObject(o, shallowSize, fieldValues, queue);
+ List<Object> references = new ArrayList<>();
+ v = super.accumulateObject(o, shallowSize, fieldValues, references);
+ for (Object r : references) {
+ // AssertingCodec adds Thread references to make sure objects are consumed in the right thread
+ if (r instanceof Thread == false) {
+ queue.add(r);
+ }
+ }
}
- // System.out.println(o.getClass() + "=" + v);
return v;
}
@@ -698,4 +705,19 @@ abstract class BaseIndexFileFormatTestCase extends LuceneTestCase {
Rethrow.rethrow(e);
}
+
+ /**
+ * Returns {@code false} if only the regular fields reader should be tested,
+ * and {@code true} if only the merge instance should be tested.
+ */
+ protected boolean shouldTestMergeInstance() {
+ return false;
+ }
+
+ protected final DirectoryReader maybeWrapWithMergingReader(DirectoryReader r) throws IOException {
+ if (shouldTestMergeInstance()) {
+ r = new MergingDirectoryReaderWrapper(r);
+ }
+ return r;
+ }
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
index e0e1f57..a308f17 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
@@ -37,6 +37,7 @@ import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.TestUtil;
import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
@@ -491,14 +492,14 @@ public abstract class BaseNormsFormatTestCase extends BaseIndexFileFormatTestCas
writer.commit();
// compare
- DirectoryReader ir = DirectoryReader.open(dir);
+ DirectoryReader ir = maybeWrapWithMergingReader(DirectoryReader.open(dir));
checkNormsVsDocValues(ir);
ir.close();
writer.forceMerge(1);
// compare again
- ir = DirectoryReader.open(dir);
+ ir = maybeWrapWithMergingReader(DirectoryReader.open(dir));
checkNormsVsDocValues(ir);
writer.close();
@@ -605,7 +606,7 @@ public abstract class BaseNormsFormatTestCase extends BaseIndexFileFormatTestCas
w.deleteDocuments(new Term("id", ""+id));
}
w.forceMerge(1);
- IndexReader r = w.getReader();
+ IndexReader r = maybeWrapWithMergingReader(w.getReader());
assertFalse(r.hasDeletions());
// Confusingly, norms should exist, and should all be 0, even though we deleted all docs that had the field "content". They should not
@@ -679,7 +680,7 @@ public abstract class BaseNormsFormatTestCase extends BaseIndexFileFormatTestCas
}
}
- DirectoryReader reader = writer.getReader();
+ DirectoryReader reader = maybeWrapWithMergingReader(writer.getReader());
writer.close();
final int numThreads = TestUtil.nextInt(random(), 3, 30);
@@ -711,4 +712,72 @@ public abstract class BaseNormsFormatTestCase extends BaseIndexFileFormatTestCas
reader.close();
dir.close();
}
+
+ public void testIndependantIterators() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig().setMergePolicy(newLogMergePolicy());
+ CannedNormSimilarity sim = new CannedNormSimilarity(new long[] {42, 10, 20});
+ conf.setSimilarity(sim);
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
+ Document doc = new Document();
+ Field indexedField = new TextField("indexed", "a", Field.Store.NO);
+ doc.add(indexedField);
+ for (int i = 0; i < 3; ++i) {
+ writer.addDocument(doc);
+ }
+ writer.forceMerge(1);
+ LeafReader r = getOnlyLeafReader(maybeWrapWithMergingReader(writer.getReader()));
+ NumericDocValues n1 = r.getNormValues("indexed");
+ NumericDocValues n2 = r.getNormValues("indexed");
+ assertEquals(0, n1.nextDoc());
+ assertEquals(42, n1.longValue());
+ assertEquals(1, n1.nextDoc());
+ assertEquals(10, n1.longValue());
+ assertEquals(0, n2.nextDoc());
+ assertEquals(42, n2.longValue());
+ assertEquals(1, n2.nextDoc());
+ assertEquals(10, n2.longValue());
+ assertEquals(2, n2.nextDoc());
+ assertEquals(20, n2.longValue());
+ assertEquals(2, n1.nextDoc());
+ assertEquals(20, n1.longValue());
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, n1.nextDoc());
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, n2.nextDoc());
+ IOUtils.close(r, writer, dir);
+ }
+
+ public void testIndependantSparseIterators() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig().setMergePolicy(newLogMergePolicy());
+ CannedNormSimilarity sim = new CannedNormSimilarity(new long[] {42, 10, 20});
+ conf.setSimilarity(sim);
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
+ Document doc = new Document();
+ Field indexedField = new TextField("indexed", "a", Field.Store.NO);
+ doc.add(indexedField);
+ Document emptyDoc = new Document();
+ for (int i = 0; i < 3; ++i) {
+ writer.addDocument(doc);
+ writer.addDocument(emptyDoc);
+ }
+ writer.forceMerge(1);
+ LeafReader r = getOnlyLeafReader(maybeWrapWithMergingReader(writer.getReader()));
+ NumericDocValues n1 = r.getNormValues("indexed");
+ NumericDocValues n2 = r.getNormValues("indexed");
+ assertEquals(0, n1.nextDoc());
+ assertEquals(42, n1.longValue());
+ assertEquals(2, n1.nextDoc());
+ assertEquals(10, n1.longValue());
+ assertEquals(0, n2.nextDoc());
+ assertEquals(42, n2.longValue());
+ assertEquals(2, n2.nextDoc());
+ assertEquals(10, n2.longValue());
+ assertEquals(4, n2.nextDoc());
+ assertEquals(20, n2.longValue());
+ assertEquals(4, n1.nextDoc());
+ assertEquals(20, n1.longValue());
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, n1.nextDoc());
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, n2.nextDoc());
+ IOUtils.close(r, writer, dir);
+ }
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
index 242abf7..d0d60bf 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
@@ -140,7 +140,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
String[] idsList = docs.keySet().toArray(new String[docs.size()]);
for(int x=0;x<2;x++) {
- IndexReader r = w.getReader();
+ DirectoryReader r = maybeWrapWithMergingReader(w.getReader());
IndexSearcher s = newSearcher(r);
if (VERBOSE) {
@@ -181,7 +181,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
doc.add(newField("aaa", "a b c", customType));
doc.add(newField("zzz", "1 2 3", customType));
w.addDocument(doc);
- IndexReader r = w.getReader();
+ IndexReader r = maybeWrapWithMergingReader(w.getReader());
Document doc2 = r.document(0);
Iterator<IndexableField> it = doc2.getFields().iterator();
assertTrue(it.hasNext());
@@ -280,7 +280,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
doc.add(new NumericDocValuesField("id", id));
w.addDocument(doc);
}
- final DirectoryReader r = w.getReader();
+ final DirectoryReader r = maybeWrapWithMergingReader(w.getReader());
w.close();
assertEquals(numDocs, r.numDocs());
@@ -309,7 +309,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
doc.add(new Field("field", "value", onlyStored));
doc.add(new StringField("field2", "value", Field.Store.YES));
w.addDocument(doc);
- IndexReader r = w.getReader();
+ IndexReader r = maybeWrapWithMergingReader(w.getReader());
w.close();
assertEquals(IndexOptions.NONE, r.document(0).getField("field").fieldType().indexOptions());
assertNotNull(r.document(0).getField("field2").fieldType().indexOptions());
@@ -352,7 +352,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
}
iw.commit();
- final DirectoryReader reader = DirectoryReader.open(dir);
+ final DirectoryReader reader = maybeWrapWithMergingReader(DirectoryReader.open(dir));
final int docID = random().nextInt(100);
for (Field fld : fields) {
String fldName = fld.name();
@@ -383,7 +383,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
iw.addDocument(emptyDoc);
}
iw.commit();
- final DirectoryReader rd = DirectoryReader.open(dir);
+ final DirectoryReader rd = maybeWrapWithMergingReader(DirectoryReader.open(dir));
for (int i = 0; i < numDocs; ++i) {
final Document doc = rd.document(i);
assertNotNull(doc);
@@ -412,7 +412,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
}
iw.commit();
- final DirectoryReader rd = DirectoryReader.open(dir);
+ final DirectoryReader rd = maybeWrapWithMergingReader(DirectoryReader.open(dir));
final IndexSearcher searcher = new IndexSearcher(rd);
final int concurrentReads = atLeast(5);
final int readsPerThread = atLeast(50);
@@ -545,7 +545,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
iw.commit();
- final DirectoryReader ir = DirectoryReader.open(dir);
+ final DirectoryReader ir = maybeWrapWithMergingReader(DirectoryReader.open(dir));
assertTrue(ir.numDocs() > 0);
int numDocs = 0;
for (int i = 0; i < ir.maxDoc(); ++i) {
@@ -649,7 +649,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
w.commit();
w.close();
- DirectoryReader reader = new DummyFilterDirectoryReader(DirectoryReader.open(dir));
+ DirectoryReader reader = new DummyFilterDirectoryReader(maybeWrapWithMergingReader(DirectoryReader.open(dir)));
Directory dir2 = newDirectory();
w = new RandomIndexWriter(random(), dir2);
@@ -657,7 +657,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
reader.close();
dir.close();
- reader = w.getReader();
+ reader = maybeWrapWithMergingReader(w.getReader());
for (int i = 0; i < reader.maxDoc(); ++i) {
final Document doc = reader.document(i);
final int id = doc.getField("id").numericValue().intValue();
@@ -728,7 +728,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
}
iw.commit();
iw.forceMerge(1); // look at what happens when big docs are merged
- final DirectoryReader rd = DirectoryReader.open(dir);
+ final DirectoryReader rd = maybeWrapWithMergingReader(DirectoryReader.open(dir));
final IndexSearcher searcher = new IndexSearcher(rd);
for (int i = 0; i < numDocs; ++i) {
final Query query = new TermQuery(new Term("id", "" + i));
@@ -788,7 +788,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
iw.addDocument(doc);
}
- DirectoryReader reader = DirectoryReader.open(iw);
+ DirectoryReader reader = maybeWrapWithMergingReader(DirectoryReader.open(iw));
// mix up fields explicitly
if (random().nextBoolean()) {
reader = new MismatchedDirectoryReader(reader, random());
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/MergingCodecReader.java b/lucene/test-framework/src/java/org/apache/lucene/index/MergingCodecReader.java
new file mode 100644
index 0000000..41c80ad
--- /dev/null
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/MergingCodecReader.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+import org.apache.lucene.codecs.NormsProducer;
+import org.apache.lucene.codecs.StoredFieldsReader;
+import org.apache.lucene.util.CloseableThreadLocal;
+
+/**
+ * {@link CodecReader} wrapper that performs all reads using the merging
+ * instance of the index formats.
+ */
+public class MergingCodecReader extends FilterCodecReader {
+
+ private final CloseableThreadLocal<StoredFieldsReader> fieldsReader = new CloseableThreadLocal<StoredFieldsReader>() {
+ @Override
+ protected StoredFieldsReader initialValue() {
+ return in.getFieldsReader().getMergeInstance();
+ }
+ };
+ private final CloseableThreadLocal<NormsProducer> normsReader = new CloseableThreadLocal<NormsProducer>() {
+ @Override
+ protected NormsProducer initialValue() {
+ NormsProducer norms = in.getNormsReader();
+ if (norms == null) {
+ return null;
+ } else {
+ return norms.getMergeInstance();
+ }
+ }
+ };
+ // TODO: other formats too
+
+ /** Wrap the given instance. */
+ public MergingCodecReader(CodecReader in) {
+ super(in);
+ }
+
+ @Override
+ public StoredFieldsReader getFieldsReader() {
+ return fieldsReader.get();
+ }
+
+ @Override
+ public NormsProducer getNormsReader() {
+ return normsReader.get();
+ }
+
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ // same content, we can delegate
+ return in.getCoreCacheHelper();
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ // same content, we can delegate
+ return in.getReaderCacheHelper();
+ }
+
+}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/MergingDirectoryReaderWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/index/MergingDirectoryReaderWrapper.java
new file mode 100644
index 0000000..d587bcd
--- /dev/null
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/MergingDirectoryReaderWrapper.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+import java.io.IOException;
+
+/**
+ * {@link DirectoryReader} wrapper that uses the merge instances of the wrapped
+ * {@link CodecReader}s.
+ * NOTE: This class will fail to work if the leaves of the wrapped directory are
+ * not codec readers.
+ */
+public final class MergingDirectoryReaderWrapper extends FilterDirectoryReader {
+
+ /** Wrap the given directory. */
+ public MergingDirectoryReaderWrapper(DirectoryReader in) throws IOException {
+ super(in, new SubReaderWrapper() {
+ @Override
+ public LeafReader wrap(LeafReader reader) {
+ return new MergingCodecReader((CodecReader) reader);
+ }
+ });
+ }
+
+ @Override
+ protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException {
+ return new MergingDirectoryReaderWrapper(in);
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ // doesn't change the content: can delegate
+ return in.getReaderCacheHelper();
+ }
+
+}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
index 2ffb1bf..7c63a81 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
@@ -1670,7 +1670,7 @@ public abstract class LuceneTestCase extends Assert {
Random random = random();
for (int i = 0, c = random.nextInt(6)+1; i < c; i++) {
- switch(random.nextInt(4)) {
+ switch(random.nextInt(5)) {
case 0:
// will create no FC insanity in atomic case, as ParallelLeafReader has own cache key:
if (VERBOSE) {
@@ -1723,6 +1723,25 @@ public abstract class LuceneTestCase extends Assert {
r = new MismatchedDirectoryReader((DirectoryReader)r, random);
}
break;
+ case 4:
+ if (VERBOSE) {
+ System.out.println("NOTE: LuceneTestCase.wrapReader: wrapping previous reader=" + r + " with MergingCodecReader");
+ }
+ if (r instanceof CodecReader) {
+ r = new MergingCodecReader((CodecReader) r);
+ } else if (r instanceof DirectoryReader) {
+ boolean allLeavesAreCodecReaders = true;
+ for (LeafReaderContext ctx : r.leaves()) {
+ if (ctx.reader() instanceof CodecReader == false) {
+ allLeavesAreCodecReaders = false;
+ break;
+ }
+ }
+ if (allLeavesAreCodecReaders) {
+ r = new MergingDirectoryReaderWrapper((DirectoryReader) r);
+ }
+ }
+ break;
default:
fail("should not get here");
}
[lucene-solr] 02/02: LUCENE-8138: Check that dv producers's
next/advance and advanceExact impls are consistent.
Posted by jp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 83c30303edb8886e146f60d2625fd4e524cf38b6
Author: Adrien Grand <jp...@gmail.com>
AuthorDate: Tue Mar 19 10:53:10 2019 +0100
LUCENE-8138: Check that dv producers's next/advance and advanceExact impls are consistent.
---
.../java/org/apache/lucene/index/CheckIndex.java | 91 +++++++++++++++-------
1 file changed, 65 insertions(+), 26 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
index aa799f2..8193b5f 100644
--- a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
+++ b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
@@ -984,7 +984,7 @@ public final class CheckIndex implements Closeable {
}
for (FieldInfo info : reader.getFieldInfos()) {
if (info.hasNorms()) {
- checkNumericDocValues(info.name, normsReader.getNorms(info));
+ checkNumericDocValues(info.name, normsReader.getNorms(info), normsReader.getNorms(info));
++status.totFields;
}
}
@@ -2312,27 +2312,33 @@ public final class CheckIndex implements Closeable {
}
}
- private static void checkBinaryDocValues(String fieldName, int maxDoc, BinaryDocValues bdv) throws IOException {
- int doc;
+ private static void checkBinaryDocValues(String fieldName, int maxDoc, BinaryDocValues bdv, BinaryDocValues bdv2) throws IOException {
if (bdv.docID() != -1) {
throw new RuntimeException("binary dv iterator for field: " + fieldName + " should start at docID=-1, but got " + bdv.docID());
}
// TODO: we could add stats to DVs, e.g. total doc count w/ a value for this field
- while ((doc = bdv.nextDoc()) != NO_MORE_DOCS) {
+ for (int doc = bdv.nextDoc(); doc != NO_MORE_DOCS; doc = bdv.nextDoc()) {
BytesRef value = bdv.binaryValue();
value.isValid();
+
+ if (bdv2.advanceExact(doc) == false) {
+ throw new RuntimeException("advanceExact did not find matching doc ID: " + doc);
+ }
+ BytesRef value2 = bdv2.binaryValue();
+ if (value.equals(value2) == false) {
+ throw new RuntimeException("nextDoc and advanceExact report different values: " + value + " != " + value2);
+ }
}
}
- private static void checkSortedDocValues(String fieldName, int maxDoc, SortedDocValues dv) throws IOException {
+ private static void checkSortedDocValues(String fieldName, int maxDoc, SortedDocValues dv, SortedDocValues dv2) throws IOException {
if (dv.docID() != -1) {
throw new RuntimeException("sorted dv iterator for field: " + fieldName + " should start at docID=-1, but got " + dv.docID());
}
final int maxOrd = dv.getValueCount()-1;
FixedBitSet seenOrds = new FixedBitSet(dv.getValueCount());
int maxOrd2 = -1;
- int docID;
- while ((docID = dv.nextDoc()) != NO_MORE_DOCS) {
+ for (int doc = dv.nextDoc(); doc != NO_MORE_DOCS; doc = dv.nextDoc()) {
int ord = dv.ordValue();
if (ord == -1) {
throw new RuntimeException("dv for field: " + fieldName + " has -1 ord");
@@ -2342,6 +2348,14 @@ public final class CheckIndex implements Closeable {
maxOrd2 = Math.max(maxOrd2, ord);
seenOrds.set(ord);
}
+
+ if (dv2.advanceExact(doc) == false) {
+ throw new RuntimeException("advanceExact did not find matching doc ID: " + doc);
+ }
+ int ord2 = dv2.ordValue();
+ if (ord != ord2) {
+ throw new RuntimeException("nextDoc and advanceExact report different ords: " + ord + " != " + ord2);
+ }
}
if (maxOrd != maxOrd2) {
throw new RuntimeException("dv for field: " + fieldName + " reports wrong maxOrd=" + maxOrd + " but this is not the case: " + maxOrd2);
@@ -2362,16 +2376,22 @@ public final class CheckIndex implements Closeable {
}
}
- private static void checkSortedSetDocValues(String fieldName, int maxDoc, SortedSetDocValues dv) throws IOException {
+ private static void checkSortedSetDocValues(String fieldName, int maxDoc, SortedSetDocValues dv, SortedSetDocValues dv2) throws IOException {
final long maxOrd = dv.getValueCount()-1;
LongBitSet seenOrds = new LongBitSet(dv.getValueCount());
long maxOrd2 = -1;
- int docID;
- while ((docID = dv.nextDoc()) != NO_MORE_DOCS) {
+ for (int docID = dv.nextDoc(); docID != NO_MORE_DOCS; docID = dv.nextDoc()) {
+ if (dv2.advanceExact(docID) == false) {
+ throw new RuntimeException("advanceExact did not find matching doc ID: " + docID);
+ }
long lastOrd = -1;
long ord;
int ordCount = 0;
while ((ord = dv.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
+ long ord2 = dv2.nextOrd();
+ if (ord != ord2) {
+ throw new RuntimeException("nextDoc and advanceExact report different ords: " + ord + " != " + ord2);
+ }
if (ord <= lastOrd) {
throw new RuntimeException("ords out of order: " + ord + " <= " + lastOrd + " for doc: " + docID);
}
@@ -2386,6 +2406,10 @@ public final class CheckIndex implements Closeable {
if (ordCount == 0) {
throw new RuntimeException("dv for field: " + fieldName + " returned docID=" + docID + " yet has no ordinals");
}
+ long ord2 = dv2.nextOrd();
+ if (ord != ord2) {
+ throw new RuntimeException("nextDoc and advanceExact report different ords: " + ord + " != " + ord2);
+ }
}
if (maxOrd != maxOrd2) {
throw new RuntimeException("dv for field: " + fieldName + " reports wrong maxOrd=" + maxOrd + " but this is not the case: " + maxOrd2);
@@ -2407,19 +2431,22 @@ public final class CheckIndex implements Closeable {
}
}
- private static void checkSortedNumericDocValues(String fieldName, int maxDoc, SortedNumericDocValues ndv) throws IOException {
+ private static void checkSortedNumericDocValues(String fieldName, int maxDoc, SortedNumericDocValues ndv, SortedNumericDocValues ndv2) throws IOException {
if (ndv.docID() != -1) {
throw new RuntimeException("dv iterator for field: " + fieldName + " should start at docID=-1, but got " + ndv.docID());
}
- while (true) {
- int docID = ndv.nextDoc();
- if (docID == NO_MORE_DOCS) {
- break;
- }
+ for (int docID = ndv.nextDoc(); docID != NO_MORE_DOCS; docID = ndv.nextDoc()) {
int count = ndv.docValueCount();
if (count == 0) {
throw new RuntimeException("sorted numeric dv for field: " + fieldName + " returned docValueCount=0 for docID=" + docID);
}
+ if (ndv2.advanceExact(docID) == false) {
+ throw new RuntimeException("advanceExact did not find matching doc ID: " + docID);
+ }
+ int count2 = ndv2.docValueCount();
+ if (count != count2) {
+ throw new RuntimeException("advanceExact reports different value count: " + count + " != " + count2);
+ }
long previous = Long.MIN_VALUE;
for (int j = 0; j < count; j++) {
long value = ndv.nextValue();
@@ -2427,18 +2454,30 @@ public final class CheckIndex implements Closeable {
throw new RuntimeException("values out of order: " + value + " < " + previous + " for doc: " + docID);
}
previous = value;
+
+ long value2 = ndv2.nextValue();
+ if (value != value2) {
+ throw new RuntimeException("advanceExact reports different value: " + value + " != " + value2);
+ }
}
}
}
- private static void checkNumericDocValues(String fieldName, NumericDocValues ndv) throws IOException {
- int doc;
+ private static void checkNumericDocValues(String fieldName, NumericDocValues ndv, NumericDocValues ndv2) throws IOException {
if (ndv.docID() != -1) {
throw new RuntimeException("dv iterator for field: " + fieldName + " should start at docID=-1, but got " + ndv.docID());
}
// TODO: we could add stats to DVs, e.g. total doc count w/ a value for this field
- while ((doc = ndv.nextDoc()) != NO_MORE_DOCS) {
- ndv.longValue();
+ for (int doc = ndv.nextDoc(); doc != NO_MORE_DOCS; doc = ndv.nextDoc()) {
+ long value = ndv.longValue();
+
+ if (ndv2.advanceExact(doc) == false) {
+ throw new RuntimeException("advanceExact did not find matching doc ID: " + doc);
+ }
+ long value2 = ndv2.longValue();
+ if (value != value2) {
+ throw new RuntimeException("advanceExact reports different value: " + value + " != " + value2);
+ }
}
}
@@ -2447,28 +2486,28 @@ public final class CheckIndex implements Closeable {
case SORTED:
status.totalSortedFields++;
checkDVIterator(fi, maxDoc, dvReader::getSorted);
- checkBinaryDocValues(fi.name, maxDoc, dvReader.getSorted(fi));
- checkSortedDocValues(fi.name, maxDoc, dvReader.getSorted(fi));
+ checkBinaryDocValues(fi.name, maxDoc, dvReader.getSorted(fi), dvReader.getSorted(fi));
+ checkSortedDocValues(fi.name, maxDoc, dvReader.getSorted(fi), dvReader.getSorted(fi));
break;
case SORTED_NUMERIC:
status.totalSortedNumericFields++;
checkDVIterator(fi, maxDoc, dvReader::getSortedNumeric);
- checkSortedNumericDocValues(fi.name, maxDoc, dvReader.getSortedNumeric(fi));
+ checkSortedNumericDocValues(fi.name, maxDoc, dvReader.getSortedNumeric(fi), dvReader.getSortedNumeric(fi));
break;
case SORTED_SET:
status.totalSortedSetFields++;
checkDVIterator(fi, maxDoc, dvReader::getSortedSet);
- checkSortedSetDocValues(fi.name, maxDoc, dvReader.getSortedSet(fi));
+ checkSortedSetDocValues(fi.name, maxDoc, dvReader.getSortedSet(fi), dvReader.getSortedSet(fi));
break;
case BINARY:
status.totalBinaryFields++;
checkDVIterator(fi, maxDoc, dvReader::getBinary);
- checkBinaryDocValues(fi.name, maxDoc, dvReader.getBinary(fi));
+ checkBinaryDocValues(fi.name, maxDoc, dvReader.getBinary(fi), dvReader.getBinary(fi));
break;
case NUMERIC:
status.totalNumericFields++;
checkDVIterator(fi, maxDoc, dvReader::getNumeric);
- checkNumericDocValues(fi.name, dvReader.getNumeric(fi));
+ checkNumericDocValues(fi.name, dvReader.getNumeric(fi), dvReader.getNumeric(fi));
break;
default:
throw new AssertionError();