You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2026/04/22 14:37:41 UTC
(lucene) branch branch_10x updated: Store DocValuesSkippers in a separate file (#15976)
This is an automated email from the ASF dual-hosted git repository.
romseygeek pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_10x by this push:
new 71a214ce53c Store DocValuesSkippers in a separate file (#15976)
71a214ce53c is described below
commit 71a214ce53c4c065a34d7aecc81e8836bf1e01a0
Author: Alan Woodward <ro...@apache.org>
AuthorDate: Wed Apr 22 15:30:56 2026 +0100
Store DocValuesSkippers in a separate file (#15976)
Skippers are very useful for excluding large chunks of documents
from range queries. However, the way they are currently stored
interleaved with dv values makes it difficult to cleanly load
only skip values, and interacts badly with page caches and
memory mapping, especially as the skip data for an individual
field tends to be fairly small and often doesn't fill an OS page.
This updates the Lucene90DocValuesFormat to store skip data in
a separate file, which can be mapped or preloaded separately from
the DV data.
---
lucene/CHANGES.txt | 2 +
.../codecs/lucene90/Lucene90DocValuesConsumer.java | 41 +++++++++++------
.../codecs/lucene90/Lucene90DocValuesFormat.java | 23 ++++++++--
.../codecs/lucene90/Lucene90DocValuesProducer.java | 45 ++++++++++++++++--
.../lucene90/TestLucene90DocValuesFormat.java | 53 ++++++++++++++++++++++
5 files changed, 145 insertions(+), 19 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 621f35c26a3..b1d74308ecb 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -56,6 +56,8 @@ Improvements
buffer was controlled by an explicit parameter (set as `resultSimilarity` - `traversalSimilarity`), which is now
deprecated. (Kaival Parikh)
+* GITHUB#15976: Store DocValuesSkipper data in their own file. (Alan Woodward)
+
Optimizations
---------------------
* GITHUB#15861: Optimise PhraseScorer by short circuiting non competitive documents in TOP_SCORES mode. (Prithvi S)
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java
index 13f1463d0db..993ae503995 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java
@@ -63,7 +63,7 @@ import org.apache.lucene.util.packed.DirectWriter;
/** writer for {@link Lucene90DocValuesFormat} */
final class Lucene90DocValuesConsumer extends DocValuesConsumer {
- IndexOutput data, meta;
+ IndexOutput data, meta, skipIndex;
final int maxDoc;
private byte[] termsDictBuffer;
private final int skipIndexIntervalSize;
@@ -75,7 +75,9 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
String dataCodec,
String dataExtension,
String metaCodec,
- String metaExtension)
+ String metaExtension,
+ String skipIndexCodec,
+ String skipIndexExtension)
throws IOException {
this.termsDictBuffer = new byte[1 << 14];
boolean success = false;
@@ -100,6 +102,16 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
Lucene90DocValuesFormat.VERSION_CURRENT,
state.segmentInfo.getId(),
state.segmentSuffix);
+ String skipIndexName =
+ IndexFileNames.segmentFileName(
+ state.segmentInfo.name, state.segmentSuffix, skipIndexExtension);
+ skipIndex = state.directory.createOutput(skipIndexName, state.context);
+ CodecUtil.writeIndexHeader(
+ skipIndex,
+ skipIndexCodec,
+ Lucene90DocValuesFormat.VERSION_CURRENT,
+ state.segmentInfo.getId(),
+ state.segmentSuffix);
maxDoc = state.segmentInfo.maxDoc();
this.skipIndexIntervalSize = skipIndexIntervalSize;
success = true;
@@ -121,14 +133,17 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
if (data != null) {
CodecUtil.writeFooter(data); // write checksum
}
+ if (skipIndex != null) {
+ CodecUtil.writeFooter(skipIndex);
+ }
success = true;
} finally {
if (success) {
- IOUtils.close(data, meta);
+ IOUtils.close(data, meta, skipIndex);
} else {
- IOUtils.closeWhileHandlingException(data, meta);
+ IOUtils.closeWhileHandlingException(data, meta, skipIndex);
}
- meta = data = null;
+ meta = data = skipIndex = null;
}
}
@@ -250,7 +265,7 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
private void writeSkipIndex(FieldInfo field, DocValuesProducer valuesProducer)
throws IOException {
assert field.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE;
- final long start = data.getFilePointer();
+ final long start = skipIndex.getFilePointer();
final SortedNumericDocValues values = valuesProducer.getSortedNumeric(field);
long globalMaxValue = Long.MIN_VALUE;
long globalMinValue = Long.MAX_VALUE;
@@ -292,7 +307,7 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
writeLevels(accumulators);
}
meta.writeLong(start); // record the start in meta
- meta.writeLong(data.getFilePointer() - start); // record the length
+ meta.writeLong(skipIndex.getFilePointer() - start); // record the length
assert globalDocCount == 0 || globalMaxValue >= globalMinValue;
meta.writeLong(globalMaxValue);
meta.writeLong(globalMinValue);
@@ -312,17 +327,17 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
// compute how many levels we need to write for the current accumulator
final int levels = getLevels(index, totalAccumulators);
// write the number of levels
- data.writeByte((byte) levels);
+ skipIndex.writeByte((byte) levels);
// write intervals in reverse order. This is done so we don't
// need to read all of them in case of slipping
for (int level = levels - 1; level >= 0; level--) {
final SkipAccumulator accumulator =
accumulatorsLevels.get(level).get(index >> (SKIP_INDEX_LEVEL_SHIFT * level));
- data.writeInt(accumulator.maxDocID);
- data.writeInt(accumulator.minDocID);
- data.writeLong(accumulator.maxValue);
- data.writeLong(accumulator.minValue);
- data.writeInt(accumulator.docCount);
+ skipIndex.writeInt(accumulator.maxDocID);
+ skipIndex.writeInt(accumulator.minDocID);
+ skipIndex.writeLong(accumulator.maxValue);
+ skipIndex.writeLong(accumulator.minValue);
+ skipIndex.writeInt(accumulator.docCount);
}
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java
index 922bc2481c9..cd4bb7ef076 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java
@@ -132,6 +132,7 @@ import org.apache.lucene.util.packed.DirectWriter;
* <ol>
* <li><code>.dvd</code>: DocValues data
* <li><code>.dvm</code>: DocValues metadata
+ * <li><code>.dvs</code>: DocValues skip index (since version 1)
* </ol>
*
* @lucene.experimental
@@ -158,21 +159,37 @@ public final class Lucene90DocValuesFormat extends DocValuesFormat {
@Override
public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
return new Lucene90DocValuesConsumer(
- state, skipIndexIntervalSize, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION);
+ state,
+ skipIndexIntervalSize,
+ DATA_CODEC,
+ DATA_EXTENSION,
+ META_CODEC,
+ META_EXTENSION,
+ SKIP_INDEX_CODEC,
+ SKIP_INDEX_EXTENSION);
}
@Override
public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOException {
return new Lucene90DocValuesProducer(
- state, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION);
+ state,
+ DATA_CODEC,
+ DATA_EXTENSION,
+ META_CODEC,
+ META_EXTENSION,
+ SKIP_INDEX_CODEC,
+ SKIP_INDEX_EXTENSION);
}
static final String DATA_CODEC = "Lucene90DocValuesData";
static final String DATA_EXTENSION = "dvd";
static final String META_CODEC = "Lucene90DocValuesMetadata";
static final String META_EXTENSION = "dvm";
+ static final String SKIP_INDEX_CODEC = "Lucene90DocValuesSkipIndex";
+ static final String SKIP_INDEX_EXTENSION = "dvs";
static final int VERSION_START = 0;
- static final int VERSION_CURRENT = VERSION_START;
+ static final int VERSION_SKIPPER_SEPARATE_FILE = 1;
+ static final int VERSION_CURRENT = VERSION_SKIPPER_SEPARATE_FILE;
// indicates docvalues type
static final byte NUMERIC = 0;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java
index cb8764eb875..131f430f6b4 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java
@@ -66,6 +66,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
private final IntObjectHashMap<SortedNumericEntry> sortedNumerics;
private final IntObjectHashMap<DocValuesSkipperEntry> skippers;
private final IndexInput data;
+ private final IndexInput skipIndexData;
private final int maxDoc;
private int version = -1;
private final boolean merging;
@@ -76,7 +77,9 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
String dataCodec,
String dataExtension,
String metaCodec,
- String metaExtension)
+ String metaExtension,
+ String skipIndexCodec,
+ String skipIndexExtension)
throws IOException {
String metaName =
IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
@@ -143,6 +146,35 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
IOUtils.closeWhileHandlingException(this.data);
}
}
+
+ if (version >= Lucene90DocValuesFormat.VERSION_SKIPPER_SEPARATE_FILE) {
+ String skipIndexName =
+ IndexFileNames.segmentFileName(
+ state.segmentInfo.name, state.segmentSuffix, skipIndexExtension);
+ this.skipIndexData =
+ state.directory.openInput(skipIndexName, state.context.withHints(FileTypeHint.INDEX));
+ try {
+ final int skipVersion =
+ CodecUtil.checkIndexHeader(
+ skipIndexData,
+ skipIndexCodec,
+ Lucene90DocValuesFormat.VERSION_SKIPPER_SEPARATE_FILE,
+ Lucene90DocValuesFormat.VERSION_CURRENT,
+ state.segmentInfo.getId(),
+ state.segmentSuffix);
+ if (version != skipVersion) {
+ throw new CorruptIndexException(
+ "Format versions mismatch: meta=" + version + ", skipIndex=" + skipVersion,
+ skipIndexData);
+ }
+ CodecUtil.retrieveChecksum(skipIndexData);
+ } catch (Throwable t) {
+ IOUtils.closeWhileSuppressingExceptions(t, data, skipIndexData);
+ throw t;
+ }
+ } else {
+ this.skipIndexData = null;
+ }
}
// Used for cloning
@@ -154,6 +186,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
IntObjectHashMap<SortedNumericEntry> sortedNumerics,
IntObjectHashMap<DocValuesSkipperEntry> skippers,
IndexInput data,
+ IndexInput skipIndexData,
int maxDoc,
int version,
boolean merging) {
@@ -164,6 +197,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
this.sortedNumerics = sortedNumerics;
this.skippers = skippers;
this.data = data.clone();
+ this.skipIndexData = skipIndexData != null ? skipIndexData.clone() : null;
this.maxDoc = maxDoc;
this.version = version;
this.merging = merging;
@@ -179,6 +213,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
sortedNumerics,
skippers,
data,
+ skipIndexData,
maxDoc,
version,
true);
@@ -353,7 +388,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
@Override
public void close() throws IOException {
- data.close();
+ IOUtils.close(data, skipIndexData);
}
private record DocValuesSkipperEntry(
@@ -1788,6 +1823,9 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
@Override
public void checkIntegrity() throws IOException {
CodecUtil.checksumEntireFile(data);
+ if (skipIndexData != null) {
+ CodecUtil.checksumEntireFile(skipIndexData);
+ }
}
/**
@@ -1868,7 +1906,8 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
public DocValuesSkipper getSkipper(FieldInfo field) throws IOException {
final DocValuesSkipperEntry entry = skippers.get(field.number);
- final IndexInput input = data.slice("doc value skipper", entry.offset, entry.length);
+ final IndexInput skipperSource = skipIndexData != null ? skipIndexData : data;
+ final IndexInput input = skipperSource.slice("doc value skipper", entry.offset, entry.length);
// TODO: should we write to disk the actual max level for this segment?
return new DocValuesSkipper() {
final int[] minDocID = new int[SKIP_INDEX_MAX_LEVEL];
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormat.java
index 84952b41b4a..f377e2fb1e6 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormat.java
@@ -45,6 +45,7 @@ import org.apache.lucene.document.StringField;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.DocValuesSkipper;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
@@ -64,6 +65,10 @@ import org.apache.lucene.index.TermsEnum.SeekStatus;
import org.apache.lucene.store.ByteBuffersDataInput;
import org.apache.lucene.store.ByteBuffersDataOutput;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FilterDirectory;
+import org.apache.lucene.store.FilterIndexInput;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.codecs.asserting.AssertingCodec;
import org.apache.lucene.tests.index.BaseCompressingDocValuesFormatTestCase;
@@ -1015,4 +1020,52 @@ public class TestLucene90DocValuesFormat extends BaseCompressingDocValuesFormatT
reader.close();
directory.close();
}
+
+ public void testSkipIndexStoredSeparately() throws IOException {
+ try (Directory dir = newDirectory()) {
+ IndexWriterConfig conf = newIndexWriterConfig();
+ conf.setCodec(getCodec());
+ conf.setUseCompoundFile(false);
+ try (IndexWriter writer = new IndexWriter(dir, conf)) {
+ for (int i = 0; i < 100; i++) {
+ Document doc = new Document();
+ doc.add(NumericDocValuesField.indexedField("dv", i));
+ writer.addDocument(doc);
+ }
+ writer.forceMerge(1);
+ }
+
+ // Wrap the directory so that any attempt to slice() the .dvd file throws.
+ // The producer constructor opens .dvd for header/checksum validation (sequential reads),
+ // but getSkipper() should only slice the .dvs file, never the .dvd file.
+ Directory noDataSliceDir =
+ new FilterDirectory(dir) {
+ @Override
+ public IndexInput openInput(String name, IOContext context) throws IOException {
+ IndexInput in = super.openInput(name, context);
+ if (name.endsWith("." + Lucene90DocValuesFormat.DATA_EXTENSION)) {
+ return new FilterIndexInput("no-slice-" + name, in) {
+ @Override
+ public IndexInput slice(String sliceDescription, long offset, long length) {
+ throw new AssertionError(
+ "should not slice .dvd file for skip index access: " + sliceDescription);
+ }
+ };
+ }
+ return in;
+ }
+ };
+
+ try (DirectoryReader reader = DirectoryReader.open(noDataSliceDir)) {
+ LeafReader leaf = getOnlyLeafReader(reader);
+ DocValuesSkipper skipper = leaf.getDocValuesSkipper("dv");
+ assertNotNull(skipper);
+ assertEquals(100, skipper.docCount());
+ skipper.advance(0);
+ assertEquals(0, skipper.minDocID(0));
+ assertTrue(skipper.maxDocID(0) >= 0);
+ assertTrue(skipper.minValue() <= skipper.maxValue());
+ }
+ }
+ }
}