You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2026/02/20 14:49:25 UTC
(lucene) branch main updated: Skip per-document stored field reads on sorted indices when no stored fields are present (#15718)
This is an automated email from the ASF dual-hosted git repository.
romseygeek pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new 91af8ee142b Skip per-document stored field reads on sorted indices when no stored fields are present (#15718)
91af8ee142b is described below
commit 91af8ee142b5cda6ecfea66db4504377976ed59d
Author: Francisco Fernández Castaño <fr...@gmail.com>
AuthorDate: Fri Feb 20 15:49:19 2026 +0100
Skip per-document stored field reads on sorted indices when no stored fields are present (#15718)
Avoid unnecessary seeks and reads on the stored fields file during
flush in SortingStoredFieldsConsumer for segments that do not contain
stored fields.
Closes #15713
---
lucene/CHANGES.txt | 2 +
.../lucene/index/SortingStoredFieldsConsumer.java | 19 ++-
.../index/TestSortingStoredFieldsConsumer.java | 136 +++++++++++++++++++++
3 files changed, 156 insertions(+), 1 deletion(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 312d40534af..04687a14d16 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -202,6 +202,8 @@ Optimizations
---------------------
* GITHUB#15637: Lazily allocate ByteArrayDataInputs in SegmentTermsEnumFrame#15637 (Misha Dmitriev)
+* GITHUB#15718 Skip per-document stored field reads on sorted indices when no stored fields are present (Francisco Fernández Castaño)
+
Bug Fixes
---------------------
* GITHUB#15668: Fix UnsupportedOperationException in WeightedSpanTermExtractor by
diff --git a/lucene/core/src/java/org/apache/lucene/index/SortingStoredFieldsConsumer.java b/lucene/core/src/java/org/apache/lucene/index/SortingStoredFieldsConsumer.java
index 63dccc8ee3a..510e22eb43f 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SortingStoredFieldsConsumer.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SortingStoredFieldsConsumer.java
@@ -27,6 +27,7 @@ import org.apache.lucene.codecs.compressing.CompressionMode;
import org.apache.lucene.codecs.compressing.Compressor;
import org.apache.lucene.codecs.compressing.Decompressor;
import org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsFormat;
+import org.apache.lucene.document.StoredValue;
import org.apache.lucene.store.ByteBuffersDataInput;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.DataOutput;
@@ -80,6 +81,12 @@ final class SortingStoredFieldsConsumer extends StoredFieldsConsumer {
"TempStoredFields", NO_COMPRESSION, 128 * 1024, 1, 10);
TrackingTmpOutputDirectoryWrapper tmpDirectory;
+ /**
+ * Tracks whether any stored field value was written; used to skip stored field reads at flush
+ * time when segments do not contain any stored fields.
+ */
+ private boolean hasStoredFields;
+
SortingStoredFieldsConsumer(Codec codec, Directory directory, SegmentInfo info) {
super(codec, directory, info);
}
@@ -92,6 +99,12 @@ final class SortingStoredFieldsConsumer extends StoredFieldsConsumer {
}
}
+ @Override
+ void writeField(FieldInfo info, StoredValue value) throws IOException {
+ super.writeField(info, value);
+ hasStoredFields = true;
+ }
+
@Override
void flush(SegmentWriteState state, Sorter.DocMap sortMap) throws IOException {
super.flush(state, sortMap);
@@ -111,7 +124,11 @@ final class SortingStoredFieldsConsumer extends StoredFieldsConsumer {
CopyVisitor visitor = new CopyVisitor(sortWriter);
for (int docID = 0; docID < state.segmentInfo.maxDoc(); docID++) {
sortWriter.startDocument();
- reader.document(sortMap == null ? docID : sortMap.newToOld(docID), visitor);
+ // Skip per-document stored field reads when no stored fields were written,
+ // avoiding unnecessary seeks and reads on the stored fields file.
+ if (hasStoredFields) {
+ reader.document(sortMap == null ? docID : sortMap.newToOld(docID), visitor);
+ }
sortWriter.finishDocument();
}
sortWriter.finish(state.segmentInfo.maxDoc());
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSortingStoredFieldsConsumer.java b/lucene/core/src/test/org/apache/lucene/index/TestSortingStoredFieldsConsumer.java
new file mode 100644
index 00000000000..78749fdbed9
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/index/TestSortingStoredFieldsConsumer.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.index;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.StoredFieldsReader;
+import org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsWriter;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FilterDirectory;
+import org.apache.lucene.store.FilterIndexInput;
+import org.apache.lucene.store.FlushInfo;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+import org.apache.lucene.util.StringHelper;
+import org.apache.lucene.util.Version;
+
+public class TestSortingStoredFieldsConsumer extends LuceneTestCase {
+
+ /**
+ * Verifies that when no stored fields are written, flushing the {@link
+ * SortingStoredFieldsConsumer} skips per-document seeks and reads on the stored fields file. Only
+ * the initial seek for checksum retrieval should occur.
+ */
+ public void testFlushWithNoStoredFieldsSkipsPerDocumentSeeks() throws IOException {
+ try (Directory dir = newDirectory()) {
+ IndexWriterConfig iwc = new IndexWriterConfig();
+ Codec codec = iwc.getCodec();
+ SegmentInfo si =
+ new SegmentInfo(
+ dir,
+ Version.LATEST,
+ null,
+ "_0",
+ -1,
+ false,
+ false,
+ codec,
+ Collections.emptyMap(),
+ StringHelper.randomId(),
+ new HashMap<>(),
+ null);
+
+ AtomicInteger numSeeks = new AtomicInteger();
+ Directory trackingDir =
+ new FilterDirectory(dir) {
+ @Override
+ public IndexInput openInput(String name, IOContext context) throws IOException {
+ if (name.contains(Lucene90CompressingStoredFieldsWriter.FIELDS_EXTENSION)) {
+ return new FilterIndexInput(name, super.openInput(name, context)) {
+ @Override
+ public void seek(long pos) throws IOException {
+ numSeeks.incrementAndGet();
+ super.seek(pos);
+ }
+ };
+ }
+ return super.openInput(name, context);
+ }
+ };
+
+ SortingStoredFieldsConsumer consumer =
+ new SortingStoredFieldsConsumer(codec, trackingDir, si);
+
+ int numDocs = TestUtil.nextInt(random(), 1, 100);
+ for (int i = 0; i < numDocs; i++) {
+ consumer.startDocument(i);
+ consumer.finishDocument();
+ }
+ consumer.finish(numDocs);
+
+ si.setMaxDoc(numDocs);
+ SegmentWriteState state =
+ new SegmentWriteState(
+ null,
+ dir,
+ si,
+ new FieldInfos(new FieldInfo[0]),
+ null,
+ IOContext.flush(new FlushInfo(numDocs, 10)));
+
+ // Identity mapping: doc order is unchanged
+ Sorter.DocMap identityMap =
+ new Sorter.DocMap() {
+ @Override
+ public int oldToNew(int docID) {
+ return docID;
+ }
+
+ @Override
+ public int newToOld(int docID) {
+ return docID;
+ }
+
+ @Override
+ public int size() {
+ return numDocs;
+ }
+ };
+ consumer.flush(state, identityMap);
+
+ try (StoredFieldsReader reader =
+ codec.storedFieldsFormat().fieldsReader(dir, si, state.fieldInfos, IOContext.READONCE)) {
+ for (int i = 0; i < numDocs; i++) {
+ Document document = reader.document(i);
+ assertNotNull(document);
+ assertTrue(document.getFields().isEmpty());
+ }
+ }
+
+ assertEquals(
+ "Expected only 1 seek (for checksum retrieval), not one per document", 1, numSeeks.get());
+ }
+ }
+}