You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dn...@apache.org on 2024/09/18 17:25:19 UTC
(lucene) 01/01: Copy stored fields during flush with index-sort
This is an automated email from the ASF dual-hosted git repository.
dnhatn pushed a commit to branch copy-stored-fields
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit f4a297f641c57e12b72456ba9f48979cbb615ba1
Author: Nhat Nguyen <nh...@elastic.co>
AuthorDate: Wed Sep 18 10:24:50 2024 -0700
Copy stored fields during flush with index-sort
---
.../Lucene90CompressingStoredFieldsWriter.java | 3 +-
.../Lucene90SortingStoredFieldsConsumer.java | 108 +++++++++++++++++++++
.../org/apache/lucene/index/IndexingChain.java | 4 +-
.../lucene/index/SortingStoredFieldsConsumer.java | 96 +++++++-----------
.../lucene/index/SortingTermVectorsConsumer.java | 8 +-
5 files changed, 152 insertions(+), 67 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.java
index a7afdbf30ab..ebc3894202e 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.java
@@ -507,8 +507,7 @@ public final class Lucene90CompressingStoredFieldsWriter extends StoredFieldsWri
BULK_MERGE_ENABLED = v;
}
- private void copyOneDoc(Lucene90CompressingStoredFieldsReader reader, int docID)
- throws IOException {
+ void copyOneDoc(Lucene90CompressingStoredFieldsReader reader, int docID) throws IOException {
assert reader.getVersion() == VERSION_CURRENT;
SerializedDocument doc = reader.serializedDocument(docID);
startDocument();
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90SortingStoredFieldsConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90SortingStoredFieldsConsumer.java
new file mode 100644
index 00000000000..ee61ca41d25
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90SortingStoredFieldsConsumer.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene90.compressing;
+
+import java.io.IOException;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.codecs.StoredFieldsReader;
+import org.apache.lucene.codecs.StoredFieldsWriter;
+import org.apache.lucene.codecs.compressing.CompressionMode;
+import org.apache.lucene.codecs.compressing.Compressor;
+import org.apache.lucene.codecs.compressing.Decompressor;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.index.Sorter;
+import org.apache.lucene.index.SortingStoredFieldsConsumer;
+import org.apache.lucene.store.ByteBuffersDataInput;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BytesRef;
+
+/** A {@link SortingStoredFieldsConsumer} using Lucene90StoredFields format. */
+public final class Lucene90SortingStoredFieldsConsumer extends SortingStoredFieldsConsumer {
+ /** A mode that appends bytes without applying compression. */
+ public static final CompressionMode NO_COMPRESSION =
+ new CompressionMode() {
+ @Override
+ public Compressor newCompressor() {
+ return new Compressor() {
+ @Override
+ public void close() {}
+
+ @Override
+ public void compress(ByteBuffersDataInput buffersInput, DataOutput out)
+ throws IOException {
+ out.copyBytes(buffersInput, buffersInput.length());
+ }
+ };
+ }
+
+ @Override
+ public Decompressor newDecompressor() {
+ return new Decompressor() {
+ @Override
+ public void decompress(
+ DataInput in, int originalLength, int offset, int length, BytesRef bytes)
+ throws IOException {
+ bytes.bytes = ArrayUtil.growNoCopy(bytes.bytes, length);
+ in.skipBytes(offset);
+ in.readBytes(bytes.bytes, 0, length);
+ bytes.offset = 0;
+ bytes.length = length;
+ }
+
+ @Override
+ public Decompressor clone() {
+ return this;
+ }
+ };
+ }
+ };
+
+ private static final StoredFieldsFormat LUCENE90_NO_COMPRESSION_STORED_FIELDS_FORMAT =
+ new Lucene90CompressingStoredFieldsFormat(
+ "Lucene90TempStoredFields", NO_COMPRESSION, 128 * 1024, 1, 10);
+
+ public Lucene90SortingStoredFieldsConsumer(Codec codec, Directory directory, SegmentInfo info) {
+ super(codec, LUCENE90_NO_COMPRESSION_STORED_FIELDS_FORMAT, directory, info);
+ }
+
+ @Override
+ protected void copyDocs(
+ SegmentWriteState state,
+ Sorter.DocMap sortMap,
+ StoredFieldsWriter writer,
+ StoredFieldsReader reader)
+ throws IOException {
+ if (reader instanceof Lucene90CompressingStoredFieldsReader compressReader
+ && compressReader.getVersion() == Lucene90CompressingStoredFieldsWriter.VERSION_CURRENT) {
+ if (writer instanceof Lucene90CompressingStoredFieldsWriter compressWriter) {
+ for (int docID = 0; docID < state.segmentInfo.maxDoc(); docID++) {
+ compressWriter.copyOneDoc(
+ compressReader, sortMap == null ? docID : sortMap.newToOld(docID));
+ }
+ return;
+ }
+ } else {
+ assert false : "the temp stored fields format wasn't used?";
+ }
+ super.copyDocs(state, sortMap, writer, reader);
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
index bb81a1fe577..95e017bc30a 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
@@ -39,6 +39,7 @@ import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.NormsProducer;
import org.apache.lucene.codecs.PointsFormat;
import org.apache.lucene.codecs.PointsWriter;
+import org.apache.lucene.codecs.lucene90.compressing.Lucene90SortingStoredFieldsConsumer;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.InvertableType;
import org.apache.lucene.document.KnnByteVectorField;
@@ -127,7 +128,8 @@ final class IndexingChain implements Accountable {
indexWriterConfig.getCodec());
} else {
storedFieldsConsumer =
- new SortingStoredFieldsConsumer(indexWriterConfig.getCodec(), directory, segmentInfo);
+ new Lucene90SortingStoredFieldsConsumer(
+ indexWriterConfig.getCodec(), directory, segmentInfo);
termVectorsWriter =
new SortingTermVectorsConsumer(
intBlockAllocator,
diff --git a/lucene/core/src/java/org/apache/lucene/index/SortingStoredFieldsConsumer.java b/lucene/core/src/java/org/apache/lucene/index/SortingStoredFieldsConsumer.java
index 8044a4cc387..8b8d567536f 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SortingStoredFieldsConsumer.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SortingStoredFieldsConsumer.java
@@ -23,93 +23,48 @@ import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.codecs.StoredFieldsWriter;
-import org.apache.lucene.codecs.compressing.CompressionMode;
-import org.apache.lucene.codecs.compressing.Compressor;
-import org.apache.lucene.codecs.compressing.Decompressor;
-import org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsFormat;
-import org.apache.lucene.store.ByteBuffersDataInput;
import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
-import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
-final class SortingStoredFieldsConsumer extends StoredFieldsConsumer {
-
- static final CompressionMode NO_COMPRESSION =
- new CompressionMode() {
- @Override
- public Compressor newCompressor() {
- return new Compressor() {
- @Override
- public void close() throws IOException {}
-
- @Override
- public void compress(ByteBuffersDataInput buffersInput, DataOutput out)
- throws IOException {
- out.copyBytes(buffersInput, buffersInput.length());
- }
- };
- }
-
- @Override
- public Decompressor newDecompressor() {
- return new Decompressor() {
- @Override
- public void decompress(
- DataInput in, int originalLength, int offset, int length, BytesRef bytes)
- throws IOException {
- bytes.bytes = ArrayUtil.growNoCopy(bytes.bytes, length);
- in.skipBytes(offset);
- in.readBytes(bytes.bytes, 0, length);
- bytes.offset = 0;
- bytes.length = length;
- }
-
- @Override
- public Decompressor clone() {
- return this;
- }
- };
- }
- };
- private static final StoredFieldsFormat TEMP_STORED_FIELDS_FORMAT =
- new Lucene90CompressingStoredFieldsFormat(
- "TempStoredFields", NO_COMPRESSION, 128 * 1024, 1, 10);
- TrackingTmpOutputDirectoryWrapper tmpDirectory;
-
- SortingStoredFieldsConsumer(Codec codec, Directory directory, SegmentInfo info) {
+/** A {@link StoredFieldsConsumer} support index sorts */
+public abstract class SortingStoredFieldsConsumer extends StoredFieldsConsumer {
+
+ private TrackingTmpOutputDirectoryWrapper tmpDirectory;
+ final StoredFieldsFormat tmpStoredFieldsFormat;
+
+ protected SortingStoredFieldsConsumer(
+ Codec codec,
+ StoredFieldsFormat tmpStoredFieldsFormat,
+ Directory directory,
+ SegmentInfo info) {
super(codec, directory, info);
+ this.tmpStoredFieldsFormat = tmpStoredFieldsFormat;
}
@Override
protected void initStoredFieldsWriter() throws IOException {
if (writer == null) {
this.tmpDirectory = new TrackingTmpOutputDirectoryWrapper(directory);
- this.writer = TEMP_STORED_FIELDS_FORMAT.fieldsWriter(tmpDirectory, info, IOContext.DEFAULT);
+ this.writer = tmpStoredFieldsFormat.fieldsWriter(tmpDirectory, info, IOContext.DEFAULT);
}
}
@Override
- void flush(SegmentWriteState state, Sorter.DocMap sortMap) throws IOException {
+ final void flush(SegmentWriteState state, Sorter.DocMap sortMap) throws IOException {
super.flush(state, sortMap);
StoredFieldsReader reader =
- TEMP_STORED_FIELDS_FORMAT.fieldsReader(
+ tmpStoredFieldsFormat.fieldsReader(
tmpDirectory, state.segmentInfo, state.fieldInfos, IOContext.DEFAULT);
- // Don't pull a merge instance, since merge instances optimize for
- // sequential access while we consume stored fields in random order here.
StoredFieldsWriter sortWriter =
codec.storedFieldsFormat().fieldsWriter(state.directory, state.segmentInfo, state.context);
try {
reader.checkIntegrity();
- CopyVisitor visitor = new CopyVisitor(sortWriter);
- for (int docID = 0; docID < state.segmentInfo.maxDoc(); docID++) {
- sortWriter.startDocument();
- reader.document(sortMap == null ? docID : sortMap.newToOld(docID), visitor);
- sortWriter.finishDocument();
- }
+ // Don't pull a merge instance if sortMap is specified, merge instances optimize
+ // for sequential access. In this case, stored fields will be consumed in random order.
+ copyDocs(state, sortMap, sortWriter, sortMap == null ? reader.getMergeInstance() : reader);
sortWriter.finish(state.segmentInfo.maxDoc());
} finally {
IOUtils.close(reader, sortWriter);
@@ -117,6 +72,21 @@ final class SortingStoredFieldsConsumer extends StoredFieldsConsumer {
}
}
+ /** Copy documents from the temp stored fields reader to the final stored fields writer. */
+ protected void copyDocs(
+ SegmentWriteState state,
+ Sorter.DocMap sortMap,
+ StoredFieldsWriter sortWriter,
+ StoredFieldsReader tempReader)
+ throws IOException {
+ CopyVisitor visitor = new CopyVisitor(sortWriter);
+ for (int docID = 0; docID < state.segmentInfo.maxDoc(); docID++) {
+ sortWriter.startDocument();
+ tempReader.document(sortMap == null ? docID : sortMap.newToOld(docID), visitor);
+ sortWriter.finishDocument();
+ }
+ }
+
@Override
void abort() {
try {
diff --git a/lucene/core/src/java/org/apache/lucene/index/SortingTermVectorsConsumer.java b/lucene/core/src/java/org/apache/lucene/index/SortingTermVectorsConsumer.java
index f328af2c9d3..6546a59c5ea 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SortingTermVectorsConsumer.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SortingTermVectorsConsumer.java
@@ -25,6 +25,7 @@ import org.apache.lucene.codecs.TermVectorsFormat;
import org.apache.lucene.codecs.TermVectorsReader;
import org.apache.lucene.codecs.TermVectorsWriter;
import org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingTermVectorsFormat;
+import org.apache.lucene.codecs.lucene90.compressing.Lucene90SortingStoredFieldsConsumer;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FlushInfo;
@@ -38,7 +39,12 @@ final class SortingTermVectorsConsumer extends TermVectorsConsumer {
private static final TermVectorsFormat TEMP_TERM_VECTORS_FORMAT =
new Lucene90CompressingTermVectorsFormat(
- "TempTermVectors", "", SortingStoredFieldsConsumer.NO_COMPRESSION, 8 * 1024, 128, 10);
+ "TempTermVectors",
+ "",
+ Lucene90SortingStoredFieldsConsumer.NO_COMPRESSION,
+ 8 * 1024,
+ 128,
+ 10);
TrackingTmpOutputDirectoryWrapper tmpDirectory;
SortingTermVectorsConsumer(