You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dn...@apache.org on 2024/09/18 17:25:19 UTC

(lucene) 01/01: Copy stored fields during flush with index-sort

This is an automated email from the ASF dual-hosted git repository.

dnhatn pushed a commit to branch copy-stored-fields
in repository https://gitbox.apache.org/repos/asf/lucene.git

commit f4a297f641c57e12b72456ba9f48979cbb615ba1
Author: Nhat Nguyen <nh...@elastic.co>
AuthorDate: Wed Sep 18 10:24:50 2024 -0700

    Copy stored fields during flush with index-sort
---
 .../Lucene90CompressingStoredFieldsWriter.java     |   3 +-
 .../Lucene90SortingStoredFieldsConsumer.java       | 108 +++++++++++++++++++++
 .../org/apache/lucene/index/IndexingChain.java     |   4 +-
 .../lucene/index/SortingStoredFieldsConsumer.java  |  96 +++++++-----------
 .../lucene/index/SortingTermVectorsConsumer.java   |   8 +-
 5 files changed, 152 insertions(+), 67 deletions(-)

diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.java
index a7afdbf30ab..ebc3894202e 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.java
@@ -507,8 +507,7 @@ public final class Lucene90CompressingStoredFieldsWriter extends StoredFieldsWri
     BULK_MERGE_ENABLED = v;
   }
 
-  private void copyOneDoc(Lucene90CompressingStoredFieldsReader reader, int docID)
-      throws IOException {
+  void copyOneDoc(Lucene90CompressingStoredFieldsReader reader, int docID) throws IOException {
     assert reader.getVersion() == VERSION_CURRENT;
     SerializedDocument doc = reader.serializedDocument(docID);
     startDocument();
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90SortingStoredFieldsConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90SortingStoredFieldsConsumer.java
new file mode 100644
index 00000000000..ee61ca41d25
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90SortingStoredFieldsConsumer.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene90.compressing;
+
+import java.io.IOException;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.codecs.StoredFieldsReader;
+import org.apache.lucene.codecs.StoredFieldsWriter;
+import org.apache.lucene.codecs.compressing.CompressionMode;
+import org.apache.lucene.codecs.compressing.Compressor;
+import org.apache.lucene.codecs.compressing.Decompressor;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.index.Sorter;
+import org.apache.lucene.index.SortingStoredFieldsConsumer;
+import org.apache.lucene.store.ByteBuffersDataInput;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BytesRef;
+
+/** A {@link SortingStoredFieldsConsumer} using Lucene90StoredFields format. */
+public final class Lucene90SortingStoredFieldsConsumer extends SortingStoredFieldsConsumer {
+  /** A mode that appends bytes without applying compression. */
+  public static final CompressionMode NO_COMPRESSION =
+      new CompressionMode() {
+        @Override
+        public Compressor newCompressor() {
+          return new Compressor() {
+            @Override
+            public void close() {}
+
+            @Override
+            public void compress(ByteBuffersDataInput buffersInput, DataOutput out)
+                throws IOException {
+              out.copyBytes(buffersInput, buffersInput.length());
+            }
+          };
+        }
+
+        @Override
+        public Decompressor newDecompressor() {
+          return new Decompressor() {
+            @Override
+            public void decompress(
+                DataInput in, int originalLength, int offset, int length, BytesRef bytes)
+                throws IOException {
+              bytes.bytes = ArrayUtil.growNoCopy(bytes.bytes, length);
+              in.skipBytes(offset);
+              in.readBytes(bytes.bytes, 0, length);
+              bytes.offset = 0;
+              bytes.length = length;
+            }
+
+            @Override
+            public Decompressor clone() {
+              return this;
+            }
+          };
+        }
+      };
+
+  private static final StoredFieldsFormat LUCENE90_NO_COMPRESSION_STORED_FIELDS_FORMAT =
+      new Lucene90CompressingStoredFieldsFormat(
+          "Lucene90TempStoredFields", NO_COMPRESSION, 128 * 1024, 1, 10);
+
+  public Lucene90SortingStoredFieldsConsumer(Codec codec, Directory directory, SegmentInfo info) {
+    super(codec, LUCENE90_NO_COMPRESSION_STORED_FIELDS_FORMAT, directory, info);
+  }
+
+  @Override
+  protected void copyDocs(
+      SegmentWriteState state,
+      Sorter.DocMap sortMap,
+      StoredFieldsWriter writer,
+      StoredFieldsReader reader)
+      throws IOException {
+    if (reader instanceof Lucene90CompressingStoredFieldsReader compressReader
+        && compressReader.getVersion() == Lucene90CompressingStoredFieldsWriter.VERSION_CURRENT) {
+      if (writer instanceof Lucene90CompressingStoredFieldsWriter compressWriter) {
+        for (int docID = 0; docID < state.segmentInfo.maxDoc(); docID++) {
+          compressWriter.copyOneDoc(
+              compressReader, sortMap == null ? docID : sortMap.newToOld(docID));
+        }
+        return;
+      }
+    } else {
+      assert false : "the temp stored fields format wasn't used?";
+    }
+    super.copyDocs(state, sortMap, writer, reader);
+  }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
index bb81a1fe577..95e017bc30a 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
@@ -39,6 +39,7 @@ import org.apache.lucene.codecs.NormsFormat;
 import org.apache.lucene.codecs.NormsProducer;
 import org.apache.lucene.codecs.PointsFormat;
 import org.apache.lucene.codecs.PointsWriter;
+import org.apache.lucene.codecs.lucene90.compressing.Lucene90SortingStoredFieldsConsumer;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.InvertableType;
 import org.apache.lucene.document.KnnByteVectorField;
@@ -127,7 +128,8 @@ final class IndexingChain implements Accountable {
               indexWriterConfig.getCodec());
     } else {
       storedFieldsConsumer =
-          new SortingStoredFieldsConsumer(indexWriterConfig.getCodec(), directory, segmentInfo);
+          new Lucene90SortingStoredFieldsConsumer(
+              indexWriterConfig.getCodec(), directory, segmentInfo);
       termVectorsWriter =
           new SortingTermVectorsConsumer(
               intBlockAllocator,
diff --git a/lucene/core/src/java/org/apache/lucene/index/SortingStoredFieldsConsumer.java b/lucene/core/src/java/org/apache/lucene/index/SortingStoredFieldsConsumer.java
index 8044a4cc387..8b8d567536f 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SortingStoredFieldsConsumer.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SortingStoredFieldsConsumer.java
@@ -23,93 +23,48 @@ import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.StoredFieldsFormat;
 import org.apache.lucene.codecs.StoredFieldsReader;
 import org.apache.lucene.codecs.StoredFieldsWriter;
-import org.apache.lucene.codecs.compressing.CompressionMode;
-import org.apache.lucene.codecs.compressing.Compressor;
-import org.apache.lucene.codecs.compressing.Decompressor;
-import org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsFormat;
-import org.apache.lucene.store.ByteBuffersDataInput;
 import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.DataOutput;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
-import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
 
-final class SortingStoredFieldsConsumer extends StoredFieldsConsumer {
-
-  static final CompressionMode NO_COMPRESSION =
-      new CompressionMode() {
-        @Override
-        public Compressor newCompressor() {
-          return new Compressor() {
-            @Override
-            public void close() throws IOException {}
-
-            @Override
-            public void compress(ByteBuffersDataInput buffersInput, DataOutput out)
-                throws IOException {
-              out.copyBytes(buffersInput, buffersInput.length());
-            }
-          };
-        }
-
-        @Override
-        public Decompressor newDecompressor() {
-          return new Decompressor() {
-            @Override
-            public void decompress(
-                DataInput in, int originalLength, int offset, int length, BytesRef bytes)
-                throws IOException {
-              bytes.bytes = ArrayUtil.growNoCopy(bytes.bytes, length);
-              in.skipBytes(offset);
-              in.readBytes(bytes.bytes, 0, length);
-              bytes.offset = 0;
-              bytes.length = length;
-            }
-
-            @Override
-            public Decompressor clone() {
-              return this;
-            }
-          };
-        }
-      };
-  private static final StoredFieldsFormat TEMP_STORED_FIELDS_FORMAT =
-      new Lucene90CompressingStoredFieldsFormat(
-          "TempStoredFields", NO_COMPRESSION, 128 * 1024, 1, 10);
-  TrackingTmpOutputDirectoryWrapper tmpDirectory;
-
-  SortingStoredFieldsConsumer(Codec codec, Directory directory, SegmentInfo info) {
+/** A {@link StoredFieldsConsumer} support index sorts */
+public abstract class SortingStoredFieldsConsumer extends StoredFieldsConsumer {
+
+  private TrackingTmpOutputDirectoryWrapper tmpDirectory;
+  final StoredFieldsFormat tmpStoredFieldsFormat;
+
+  protected SortingStoredFieldsConsumer(
+      Codec codec,
+      StoredFieldsFormat tmpStoredFieldsFormat,
+      Directory directory,
+      SegmentInfo info) {
     super(codec, directory, info);
+    this.tmpStoredFieldsFormat = tmpStoredFieldsFormat;
   }
 
   @Override
   protected void initStoredFieldsWriter() throws IOException {
     if (writer == null) {
       this.tmpDirectory = new TrackingTmpOutputDirectoryWrapper(directory);
-      this.writer = TEMP_STORED_FIELDS_FORMAT.fieldsWriter(tmpDirectory, info, IOContext.DEFAULT);
+      this.writer = tmpStoredFieldsFormat.fieldsWriter(tmpDirectory, info, IOContext.DEFAULT);
     }
   }
 
   @Override
-  void flush(SegmentWriteState state, Sorter.DocMap sortMap) throws IOException {
+  final void flush(SegmentWriteState state, Sorter.DocMap sortMap) throws IOException {
     super.flush(state, sortMap);
     StoredFieldsReader reader =
-        TEMP_STORED_FIELDS_FORMAT.fieldsReader(
+        tmpStoredFieldsFormat.fieldsReader(
             tmpDirectory, state.segmentInfo, state.fieldInfos, IOContext.DEFAULT);
-    // Don't pull a merge instance, since merge instances optimize for
-    // sequential access while we consume stored fields in random order here.
     StoredFieldsWriter sortWriter =
         codec.storedFieldsFormat().fieldsWriter(state.directory, state.segmentInfo, state.context);
     try {
       reader.checkIntegrity();
-      CopyVisitor visitor = new CopyVisitor(sortWriter);
-      for (int docID = 0; docID < state.segmentInfo.maxDoc(); docID++) {
-        sortWriter.startDocument();
-        reader.document(sortMap == null ? docID : sortMap.newToOld(docID), visitor);
-        sortWriter.finishDocument();
-      }
+      // Don't pull a merge instance if sortMap is specified, merge instances optimize
+      // for sequential access. In this case, stored fields will be consumed in random order.
+      copyDocs(state, sortMap, sortWriter, sortMap == null ? reader.getMergeInstance() : reader);
       sortWriter.finish(state.segmentInfo.maxDoc());
     } finally {
       IOUtils.close(reader, sortWriter);
@@ -117,6 +72,21 @@ final class SortingStoredFieldsConsumer extends StoredFieldsConsumer {
     }
   }
 
+  /** Copy documents from the temp stored fields reader to the final stored fields writer. */
+  protected void copyDocs(
+      SegmentWriteState state,
+      Sorter.DocMap sortMap,
+      StoredFieldsWriter sortWriter,
+      StoredFieldsReader tempReader)
+      throws IOException {
+    CopyVisitor visitor = new CopyVisitor(sortWriter);
+    for (int docID = 0; docID < state.segmentInfo.maxDoc(); docID++) {
+      sortWriter.startDocument();
+      tempReader.document(sortMap == null ? docID : sortMap.newToOld(docID), visitor);
+      sortWriter.finishDocument();
+    }
+  }
+
   @Override
   void abort() {
     try {
diff --git a/lucene/core/src/java/org/apache/lucene/index/SortingTermVectorsConsumer.java b/lucene/core/src/java/org/apache/lucene/index/SortingTermVectorsConsumer.java
index f328af2c9d3..6546a59c5ea 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SortingTermVectorsConsumer.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SortingTermVectorsConsumer.java
@@ -25,6 +25,7 @@ import org.apache.lucene.codecs.TermVectorsFormat;
 import org.apache.lucene.codecs.TermVectorsReader;
 import org.apache.lucene.codecs.TermVectorsWriter;
 import org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingTermVectorsFormat;
+import org.apache.lucene.codecs.lucene90.compressing.Lucene90SortingStoredFieldsConsumer;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FlushInfo;
@@ -38,7 +39,12 @@ final class SortingTermVectorsConsumer extends TermVectorsConsumer {
 
   private static final TermVectorsFormat TEMP_TERM_VECTORS_FORMAT =
       new Lucene90CompressingTermVectorsFormat(
-          "TempTermVectors", "", SortingStoredFieldsConsumer.NO_COMPRESSION, 8 * 1024, 128, 10);
+          "TempTermVectors",
+          "",
+          Lucene90SortingStoredFieldsConsumer.NO_COMPRESSION,
+          8 * 1024,
+          128,
+          10);
   TrackingTmpOutputDirectoryWrapper tmpDirectory;
 
   SortingTermVectorsConsumer(