You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dn...@apache.org on 2024/09/16 18:43:28 UTC
(lucene) branch copy-docs created (now b93a2a1f093)
This is an automated email from the ASF dual-hosted git repository.
dnhatn pushed a change to branch copy-docs
in repository https://gitbox.apache.org/repos/asf/lucene.git
at b93a2a1f093 Allow copy stored documents with index sorts
This branch includes the following new commits:
new b93a2a1f093 Allow copy stored documents with index sorts
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(lucene) 01/01: Allow copy stored documents with index sorts
Posted by dn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dnhatn pushed a commit to branch copy-docs
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit b93a2a1f0935a4e5af8e4d07ada2673c2344520a
Author: Nhat Nguyen <nh...@elastic.co>
AuthorDate: Sun Sep 15 22:39:49 2024 -0700
Allow copy stored documents with index sorts
---
.../apache/lucene/codecs/StoredFieldsWriter.java | 71 ++++++++++++++++++++++
.../Lucene90CompressingStoredFieldsWriter.java | 16 +++++
.../lucene/index/SortingStoredFieldsConsumer.java | 62 +------------------
3 files changed, 90 insertions(+), 59 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java
index 6824b227f68..1a94768b3cf 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java
@@ -28,6 +28,7 @@ import org.apache.lucene.index.DocIDMerger;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.MergeState;
+import org.apache.lucene.index.Sorter;
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.util.Accountable;
@@ -157,6 +158,76 @@ public abstract class StoredFieldsWriter implements Closeable, Accountable {
return docCount;
}
+ /**
+ * Copies stored fields from the documents of the given reader. The documents are reordered based
+ * on the docMap, if provided. Subclasses can override this method to optimize the copy process.
+ *
+ * @param reader the input reader containing the documents
+ * @param docMap the mapping to reorder document IDs, or null
+ * @param numDocs the number of documents to copy
+ * @throws IOException if an I/O error occurs during reading or writing
+ */
+ public void copyDocs(StoredFieldsReader reader, Sorter.DocMap docMap, int numDocs)
+ throws IOException {
+ CopyVisitor visitor = new CopyVisitor(this);
+ for (int docID = 0; docID < numDocs; docID++) {
+ startDocument();
+ reader.document(docMap == null ? docID : docMap.newToOld(docID), visitor);
+ finishDocument();
+ }
+ }
+
+ /** A visitor that copies every field it sees in the provided {@link StoredFieldsWriter}. */
+ private static class CopyVisitor extends StoredFieldVisitor {
+ final StoredFieldsWriter writer;
+
+ CopyVisitor(StoredFieldsWriter writer) {
+ this.writer = writer;
+ }
+
+ @Override
+ public void binaryField(FieldInfo fieldInfo, DataInput value, int length) throws IOException {
+ writer.writeField(fieldInfo, value, length);
+ }
+
+ @Override
+ public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
+ // TODO: can we avoid new BR here?
+ writer.writeField(fieldInfo, new BytesRef(value));
+ }
+
+ @Override
+ public void stringField(FieldInfo fieldInfo, String value) throws IOException {
+ writer.writeField(
+ fieldInfo, Objects.requireNonNull(value, "String value should not be null"));
+ }
+
+ @Override
+ public void intField(FieldInfo fieldInfo, int value) throws IOException {
+ writer.writeField(fieldInfo, value);
+ }
+
+ @Override
+ public void longField(FieldInfo fieldInfo, long value) throws IOException {
+ writer.writeField(fieldInfo, value);
+ }
+
+ @Override
+ public void floatField(FieldInfo fieldInfo, float value) throws IOException {
+ writer.writeField(fieldInfo, value);
+ }
+
+ @Override
+ public void doubleField(FieldInfo fieldInfo, double value) throws IOException {
+ writer.writeField(fieldInfo, value);
+ }
+
+ @Override
+ public Status needsField(FieldInfo fieldInfo) throws IOException {
+ return Status.YES;
+ }
+ }
+
/**
* A visitor that adds every field it sees.
*
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.java
index 0356a1fa448..954ec6ccd68 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.java
@@ -34,6 +34,7 @@ import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.MergeState;
import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.Sorter;
import org.apache.lucene.store.ByteBuffersDataInput;
import org.apache.lucene.store.ByteBuffersDataOutput;
import org.apache.lucene.store.DataInput;
@@ -598,6 +599,21 @@ public final class Lucene90CompressingStoredFieldsWriter extends StoredFieldsWri
}
}
+ @Override
+ public void copyDocs(StoredFieldsReader storedFieldsReader, Sorter.DocMap docMap, int numDocs)
+ throws IOException {
+ if (storedFieldsReader instanceof Lucene90CompressingStoredFieldsReader
+ && ((Lucene90CompressingStoredFieldsReader) storedFieldsReader).getVersion()
+ == VERSION_CURRENT) {
+ var reader = (Lucene90CompressingStoredFieldsReader) storedFieldsReader;
+ for (int i = 0; i < numDocs; i++) {
+ copyOneDoc(reader, docMap != null ? docMap.newToOld(i) : i);
+ }
+ } else {
+ super.copyDocs(storedFieldsReader, docMap, numDocs);
+ }
+ }
+
@Override
public int merge(MergeState mergeState) throws IOException {
final MatchingReaders matchingReaders = new MatchingReaders(mergeState);
diff --git a/lucene/core/src/java/org/apache/lucene/index/SortingStoredFieldsConsumer.java b/lucene/core/src/java/org/apache/lucene/index/SortingStoredFieldsConsumer.java
index c01de610772..5c3f9e13f2c 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SortingStoredFieldsConsumer.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SortingStoredFieldsConsumer.java
@@ -18,7 +18,6 @@
package org.apache.lucene.index;
import java.io.IOException;
-import java.util.Objects;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.StoredFieldsReader;
@@ -104,13 +103,9 @@ final class SortingStoredFieldsConsumer extends StoredFieldsConsumer {
codec.storedFieldsFormat().fieldsWriter(state.directory, state.segmentInfo, state.context);
try {
reader.checkIntegrity();
- CopyVisitor visitor = new CopyVisitor(sortWriter);
- for (int docID = 0; docID < state.segmentInfo.maxDoc(); docID++) {
- sortWriter.startDocument();
- reader.document(sortMap == null ? docID : sortMap.newToOld(docID), visitor);
- sortWriter.finishDocument();
- }
- sortWriter.finish(state.segmentInfo.maxDoc());
+ int maxDocs = state.segmentInfo.maxDoc();
+ sortWriter.copyDocs(reader, sortMap, maxDocs);
+ sortWriter.finish(maxDocs);
} finally {
IOUtils.close(reader, sortWriter);
IOUtils.deleteFiles(tmpDirectory, tmpDirectory.getTemporaryFiles().values());
@@ -128,55 +123,4 @@ final class SortingStoredFieldsConsumer extends StoredFieldsConsumer {
}
}
}
-
- /** A visitor that copies every field it sees in the provided {@link StoredFieldsWriter}. */
- private static class CopyVisitor extends StoredFieldVisitor {
- final StoredFieldsWriter writer;
-
- CopyVisitor(StoredFieldsWriter writer) {
- this.writer = writer;
- }
-
- @Override
- public void binaryField(FieldInfo fieldInfo, DataInput value, int length) throws IOException {
- writer.writeField(fieldInfo, value, length);
- }
-
- @Override
- public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
- // TODO: can we avoid new BR here?
- writer.writeField(fieldInfo, new BytesRef(value));
- }
-
- @Override
- public void stringField(FieldInfo fieldInfo, String value) throws IOException {
- writer.writeField(
- fieldInfo, Objects.requireNonNull(value, "String value should not be null"));
- }
-
- @Override
- public void intField(FieldInfo fieldInfo, int value) throws IOException {
- writer.writeField(fieldInfo, value);
- }
-
- @Override
- public void longField(FieldInfo fieldInfo, long value) throws IOException {
- writer.writeField(fieldInfo, value);
- }
-
- @Override
- public void floatField(FieldInfo fieldInfo, float value) throws IOException {
- writer.writeField(fieldInfo, value);
- }
-
- @Override
- public void doubleField(FieldInfo fieldInfo, double value) throws IOException {
- writer.writeField(fieldInfo, value);
- }
-
- @Override
- public Status needsField(FieldInfo fieldInfo) throws IOException {
- return Status.YES;
- }
- }
}