You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ti...@apache.org on 2026/06/14 19:13:56 UTC
(lucene) branch main updated: Allow multiple addBatch columns per field name (#16219)
This is an automated email from the ASF dual-hosted git repository.
Tim-Brooks pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new 1a6f4040dc6 Allow multiple addBatch columns per field name (#16219)
1a6f4040dc6 is described below
commit 1a6f4040dc647315af01e3693e8684c0b6efb414
Author: Tim Brooks <ti...@timbrooks.org>
AuthorDate: Sun Jun 14 13:13:49 2026 -0600
Allow multiple addBatch columns per field name (#16219)
Relaxes the one-column-per-field-name rule in addBatch so a field can be
built from several columns contributing distinct indexing features — for
example a stored BinaryColumn alongside an inverted BinaryColumn.
processBatch now mirrors processDocument: schemas accumulate across
same-named columns and FieldInfo is frozen once after the loop. Each
feature (inversion, stored, doc values, points, vectors) must still come
from a single column, enforced via a per-field byte bitset of claimed
features keyed off fieldGen.
---
lucene/CHANGES.txt | 2 +
.../apache/lucene/document/column/ColumnBatch.java | 10 +-
.../lucene/document/column/ColumnValidation.java | 65 +++
.../org/apache/lucene/index/IndexingChain.java | 59 ++-
.../column/TestColumnBatchMultiColumnField.java | 471 +++++++++++++++++++++
5 files changed, 585 insertions(+), 22 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 00f6600eb14..44749e3636a 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -633,6 +633,8 @@ Other
* GITHUB#16197: Remove deprecated usages of search(Query, Collector) from monitor module. (Luca Cavanna)
+* GITHUB#16219: Allow multiple addBatch columns per field name. (Tim Brooks)
+
======================= Lucene 10.4.0 =======================
API Changes
diff --git a/lucene/core/src/java/org/apache/lucene/document/column/ColumnBatch.java b/lucene/core/src/java/org/apache/lucene/document/column/ColumnBatch.java
index de39be245d7..18fab36d041 100644
--- a/lucene/core/src/java/org/apache/lucene/document/column/ColumnBatch.java
+++ b/lucene/core/src/java/org/apache/lucene/document/column/ColumnBatch.java
@@ -32,9 +32,13 @@ public abstract class ColumnBatch {
public abstract int numDocs();
/**
- * Returns the columns in this batch. Each column represents a single field across the documents
- * in the batch. Within a batch, column names must be unique: multi-valued fields should be
- * expressed as a single column whose tuple cursor emits multiple values per batch doc-id.
+ * Returns the columns in this batch. Each column covers one indexing feature (inversion, stored,
+ * doc values, points, or vectors) of a field across all documents in the batch.
+ *
+ * <p>Multiple columns may share a field name to combine distinct features — for example, a stored
+ * {@link BinaryColumn} alongside a separate inverted column for the same field — but each feature
+ * must be carried by exactly one column. Multi-valued fields must use a single column whose tuple
+ * cursor emits multiple values per doc-id; a feature must not be split across columns.
*/
public abstract Iterable<Column> columns();
}
diff --git a/lucene/core/src/java/org/apache/lucene/document/column/ColumnValidation.java b/lucene/core/src/java/org/apache/lucene/document/column/ColumnValidation.java
index 6c8666cfe65..d0e9d262be1 100644
--- a/lucene/core/src/java/org/apache/lucene/document/column/ColumnValidation.java
+++ b/lucene/core/src/java/org/apache/lucene/document/column/ColumnValidation.java
@@ -32,6 +32,71 @@ public final class ColumnValidation {
private ColumnValidation() {}
+ /** Inverted-index feature ({@code indexOptions() != NONE}). */
+ static final int FEATURE_INVERSION = 1 << 0;
+
+ /** Stored-fields feature ({@code stored()}). */
+ static final int FEATURE_STORED = 1 << 1;
+
+ /** Doc-values feature ({@code docValuesType() != NONE}). */
+ static final int FEATURE_DOCVALUES = 1 << 2;
+
+ /** Points feature ({@code pointDimensionCount() != 0}). */
+ static final int FEATURE_POINTS = 1 << 3;
+
+ /** KNN-vector feature ({@code vectorDimension() != 0}). */
+ static final int FEATURE_VECTOR = 1 << 4;
+
+ /**
+ * Returns a bitmask of the indexing features ({@code FEATURE_*}) declared by {@code fieldType}.
+ * Used by the column-batch path to enforce that, when several columns share a field name, each
+ * feature is carried by at most one column.
+ */
+ public static int featureMask(IndexableFieldType fieldType) {
+ int mask = 0;
+ if (fieldType.indexOptions() != IndexOptions.NONE) {
+ mask |= FEATURE_INVERSION;
+ }
+ if (fieldType.stored()) {
+ mask |= FEATURE_STORED;
+ }
+ if (fieldType.docValuesType() != DocValuesType.NONE) {
+ mask |= FEATURE_DOCVALUES;
+ }
+ if (fieldType.pointDimensionCount() != 0) {
+ mask |= FEATURE_POINTS;
+ }
+ if (fieldType.vectorDimension() != 0) {
+ mask |= FEATURE_VECTOR;
+ }
+ return mask;
+ }
+
+ /** Returns a human-readable comma-separated list of the feature names set in {@code mask}. */
+ public static String featureNames(int mask) {
+ StringBuilder sb = new StringBuilder();
+ if ((mask & FEATURE_INVERSION) != 0) {
+ sb.append("inversion");
+ }
+ if ((mask & FEATURE_STORED) != 0) {
+ if (!sb.isEmpty()) sb.append(", ");
+ sb.append("stored");
+ }
+ if ((mask & FEATURE_DOCVALUES) != 0) {
+ if (!sb.isEmpty()) sb.append(", ");
+ sb.append("doc values");
+ }
+ if ((mask & FEATURE_POINTS) != 0) {
+ if (!sb.isEmpty()) sb.append(", ");
+ sb.append("points");
+ }
+ if ((mask & FEATURE_VECTOR) != 0) {
+ if (!sb.isEmpty()) sb.append(", ");
+ sb.append("vectors");
+ }
+ return "[" + sb + "]";
+ }
+
/**
* Throws {@link IllegalArgumentException} if {@code fieldType} declares no indexing feature (no
* doc values, no points, not stored, no index options, no vectors).
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
index 8943394e216..f7008fe339b 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
@@ -712,8 +712,10 @@ final class IndexingChain implements Accountable {
boolean hasRowColumns = false;
long batchGen = nextFieldGen++;
- // First pass: validate all column schemas and initialize field infos
+ // First pass: validate all columns and accumulate each field's schema. A batch may carry more
+ // than one column for a field name to combine distinct features (see featureMask).
int columnIdx = 0;
+ int uniqueFieldCount = 0;
for (Column column : columnBatch.columns()) {
final String fieldName = column.name();
final IndexableFieldType fieldType = column.fieldType();
@@ -744,14 +746,34 @@ final class IndexingChain implements Accountable {
}
docFields[columnIdx++] = pf;
- if (pf.fieldGen == batchGen) {
- throw new IllegalArgumentException(
- "ColumnBatch contains more than one column for field \""
- + fieldName
- + "\"; multi-valued fields must use a single column with a sparse cursor");
+ int columnFeatures = ColumnValidation.featureMask(fieldType);
+ if (pf.fieldGen != batchGen) {
+ // First column for this field name in this batch: start a fresh schema and feature set, and
+ // collect the field once so its FieldInfo is initialized/validated after the loop.
+ pf.fieldGen = batchGen;
+ pf.columnFeatures = (byte) columnFeatures;
+ pf.schema.reset(baseDocID);
+ fields[uniqueFieldCount++] = pf;
+ } else {
+ // Each indexing feature must come from a single column for a given field name.
+ int overlap = pf.columnFeatures & columnFeatures;
+ if (overlap != 0) {
+ throw new IllegalArgumentException(
+ "ColumnBatch has multiple columns for field \""
+ + fieldName
+ + "\" claiming the same indexing feature "
+ + ColumnValidation.featureNames(overlap)
+ + "; each feature may appear in at most one column.");
+ }
+ pf.columnFeatures |= (byte) columnFeatures;
}
- pf.fieldGen = batchGen;
- validateColumnSchema(fieldName, pf, fieldType);
+
+ updateDocFieldSchema(fieldName, pf.schema, fieldType);
+ }
+
+ // Initialize field infos / validate schemas once per unique field name in the batch.
+ if (uniqueFieldCount > 0) {
+ initAndValidateFields(uniqueFieldCount);
}
if (parentPf != null) {
@@ -926,17 +948,6 @@ final class IndexingChain implements Accountable {
}
}
- private void validateColumnSchema(String fieldName, PerField pf, IndexableFieldType fieldType)
- throws IOException {
- updateDocFieldSchema(fieldName, pf.schema, fieldType);
- if (pf.fieldInfo == null) {
- initializeFieldInfo(pf);
- pf.trySetValidatedFrozenFieldType();
- } else {
- pf.schema.assertSameSchema(pf.fieldInfo);
- }
- }
-
private static void processLongColumn(
int baseDocID, int numDocs, LongColumn column, PerField pf, IndexableFieldType fieldType)
throws IOException {
@@ -1731,6 +1742,16 @@ final class IndexingChain implements Accountable {
/** We use this to know when a PerField is seen for the first time in the current document. */
long fieldGen = -1;
+ /**
+ * Bit set of indexing features (as returned by {@link ColumnValidation#featureMask}) already
+ * claimed for this field name within the current {@code addBatch} call. A column batch may
+ * carry several columns for one field name to combine distinct features (e.g. a stored column
+ * plus an inverted column), but each feature — inversion, stored, doc values, points, vectors —
+ * must come from a single column. Reset to 0 on the first sighting of the name in a batch
+ * (keyed off {@code fieldGen}).
+ */
+ byte columnFeatures;
+
// Used by the hash table
PerField next;
diff --git a/lucene/core/src/test/org/apache/lucene/document/column/TestColumnBatchMultiColumnField.java b/lucene/core/src/test/org/apache/lucene/document/column/TestColumnBatchMultiColumnField.java
new file mode 100644
index 00000000000..0f546642076
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/document/column/TestColumnBatchMultiColumnField.java
@@ -0,0 +1,471 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.document.column;
+
+import static org.apache.lucene.document.column.ColumnBatchTestUtil.ArrayBinaryColumn;
+import static org.apache.lucene.document.column.ColumnBatchTestUtil.ArrayFloatVectorColumn;
+import static org.apache.lucene.document.column.ColumnBatchTestUtil.ArrayLongColumn;
+import static org.apache.lucene.document.column.ColumnBatchTestUtil.floatVectorType;
+import static org.apache.lucene.document.column.ColumnBatchTestUtil.simpleBatch;
+
+import java.io.IOException;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.IntPoint;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FloatVectorValues;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.KnnVectorValues;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.StoredFields;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TieredMergePolicy;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.analysis.MockAnalyzer;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * Tests for composing multiple columns under a single field name in {@link ColumnBatch#columns()}.
+ */
+public class TestColumnBatchMultiColumnField extends LuceneTestCase {
+
+ /** Returns a frozen {@link FieldType} for an untokenized inverted-only field. */
+ private static FieldType invertedType() {
+ FieldType type = new FieldType();
+ type.setIndexOptions(IndexOptions.DOCS);
+ type.setTokenized(false);
+ type.setOmitNorms(true);
+ type.freeze();
+ return type;
+ }
+
+ /** Returns a frozen {@link FieldType} for a stored-only (non-indexed) field. */
+ private static FieldType storedBinaryType() {
+ FieldType type = new FieldType();
+ type.setStored(true);
+ type.freeze();
+ return type;
+ }
+
+ private IndexWriterConfig stableIwc() {
+ IndexWriterConfig iwc =
+ newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(new TieredMergePolicy());
+ iwc.setRAMBufferSizeMB(16);
+ iwc.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ return iwc;
+ }
+
+ public void testComposeStoredAndInverted() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+
+ BytesRef[] stored = {newBytesRef("Hello World"), newBytesRef("Goodbye World")};
+ // Index the lower-cased tokens so the stored and indexed representations differ.
+ BytesRef[] indexed = {newBytesRef("hello"), newBytesRef("goodbye")};
+
+ w.addBatch(
+ simpleBatch(
+ 2,
+ new ArrayBinaryColumn("body", storedBinaryType(), new int[] {0, 1}, stored),
+ new ArrayBinaryColumn("body", invertedType(), new int[] {0, 1}, indexed)));
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ IndexSearcher s = new IndexSearcher(r);
+
+ // Inverted postings come from the inverted column.
+ assertEquals(1, s.count(new TermQuery(new Term("body", "hello"))));
+ assertEquals(1, s.count(new TermQuery(new Term("body", "goodbye"))));
+ // The upper-cased original is not indexed.
+ assertEquals(0, s.count(new TermQuery(new Term("body", "Hello"))));
+
+ // Stored values come from the stored column.
+ StoredFields storedFields = leaf.storedFields();
+ assertEquals(stored[0], storedFields.document(0).getField("body").binaryValue());
+ assertEquals(stored[1], storedFields.document(1).getField("body").binaryValue());
+
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testComposeDocValuesAndInverted() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+
+ BytesRef[] tokens = {newBytesRef("alpha"), newBytesRef("beta")};
+ w.addBatch(
+ simpleBatch(
+ 2,
+ new ArrayLongColumn(
+ "x", NumericDocValuesField.TYPE, new int[] {0, 1}, new long[] {42L, 99L}),
+ new ArrayBinaryColumn("x", invertedType(), new int[] {0, 1}, tokens)));
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ IndexSearcher s = new IndexSearcher(r);
+
+ // Inverted postings come from the binary column.
+ assertEquals(1, s.count(new TermQuery(new Term("x", "alpha"))));
+ assertEquals(1, s.count(new TermQuery(new Term("x", "beta"))));
+
+ // Numeric doc values come from the long column.
+ NumericDocValues dv = leaf.getNumericDocValues("x");
+ assertNotNull(dv);
+ assertEquals(0, dv.nextDoc());
+ assertEquals(42L, dv.longValue());
+ assertEquals(1, dv.nextDoc());
+ assertEquals(99L, dv.longValue());
+
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testComposeOrderIndependent() throws IOException {
+ BytesRef stored = newBytesRef("Hello World");
+ BytesRef indexed = newBytesRef("hello");
+
+ Directory storedFirst = indexComposedBinaryColumns(true, stored, indexed);
+ Directory invertedFirst = indexComposedBinaryColumns(false, stored, indexed);
+
+ for (Directory d : new Directory[] {storedFirst, invertedFirst}) {
+ DirectoryReader r = DirectoryReader.open(d);
+ LeafReader leaf = getOnlyLeafReader(r);
+ IndexSearcher s = new IndexSearcher(r);
+ assertEquals(1, s.count(new TermQuery(new Term("body", "hello"))));
+ assertEquals(0, s.count(new TermQuery(new Term("body", "Hello"))));
+ assertEquals(stored, leaf.storedFields().document(0).getField("body").binaryValue());
+ r.close();
+ d.close();
+ }
+ }
+
+ private Directory indexComposedBinaryColumns(
+ boolean storedFirst, BytesRef stored, BytesRef indexed) throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+ Column storedCol =
+ new ArrayBinaryColumn("body", storedBinaryType(), new int[] {0}, new BytesRef[] {stored});
+ Column invertedCol =
+ new ArrayBinaryColumn("body", invertedType(), new int[] {0}, new BytesRef[] {indexed});
+ w.addBatch(
+ storedFirst
+ ? simpleBatch(1, storedCol, invertedCol)
+ : simpleBatch(1, invertedCol, storedCol));
+ w.close();
+ return dir;
+ }
+
+ public void testComposeDocValuesAndPoints() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+
+ FieldType pointType = new FieldType();
+ pointType.setDimensions(1, Integer.BYTES);
+ pointType.freeze();
+
+ w.addBatch(
+ simpleBatch(
+ 2,
+ new ArrayLongColumn(
+ "n", NumericDocValuesField.TYPE, new int[] {0, 1}, new long[] {7L, 8L}),
+ new ArrayLongColumn(
+ "n",
+ pointType,
+ LongColumn.NumericKind.INT,
+ new int[] {0, 1},
+ new long[] {7L, 8L})));
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ IndexSearcher s = new IndexSearcher(r);
+
+ // Points come from the point column.
+ assertEquals(1, s.count(IntPoint.newExactQuery("n", 7)));
+ assertEquals(2, s.count(IntPoint.newRangeQuery("n", 7, 8)));
+
+ // NUMERIC doc values come from the doc-values column.
+ NumericDocValues dv = leaf.getNumericDocValues("n");
+ assertNotNull(dv);
+ assertEquals(0, dv.nextDoc());
+ assertEquals(7L, dv.longValue());
+ assertEquals(1, dv.nextDoc());
+ assertEquals(8L, dv.longValue());
+
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testComposeDocValuesAndVector() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+
+ FieldType vectorType = floatVectorType(2, VectorSimilarityFunction.EUCLIDEAN);
+ float[][] vectors = {{1f, 2f}, {3f, 4f}};
+
+ w.addBatch(
+ simpleBatch(
+ 2,
+ new ArrayLongColumn(
+ "v", NumericDocValuesField.TYPE, new int[] {0, 1}, new long[] {11L, 22L}),
+ new ArrayFloatVectorColumn("v", vectorType, new int[] {0, 1}, vectors)));
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+
+ NumericDocValues dv = leaf.getNumericDocValues("v");
+ assertNotNull(dv);
+ assertEquals(0, dv.nextDoc());
+ assertEquals(11L, dv.longValue());
+ assertEquals(1, dv.nextDoc());
+ assertEquals(22L, dv.longValue());
+
+ FloatVectorValues fvv = leaf.getFloatVectorValues("v");
+ assertNotNull(fvv);
+ KnnVectorValues.DocIndexIterator it = fvv.iterator();
+ for (int i = 0; i < 2; i++) {
+ assertEquals(i, it.nextDoc());
+ assertArrayEquals(vectors[i], fvv.vectorValue(it.index()), 0f);
+ }
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, it.nextDoc());
+
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testComposeSparseDisjointDocIds() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+
+ w.addBatch(
+ simpleBatch(
+ 2,
+ new ArrayBinaryColumn(
+ "body", storedBinaryType(), new int[] {0}, new BytesRef[] {newBytesRef("kept")}),
+ new ArrayBinaryColumn(
+ "body", invertedType(), new int[] {1}, new BytesRef[] {newBytesRef("token")})));
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ IndexSearcher s = new IndexSearcher(r);
+
+ // The token is indexed on doc 1 only.
+ assertEquals(1, s.count(new TermQuery(new Term("body", "token"))));
+
+ StoredFields storedFields = leaf.storedFields();
+ // doc 0 carries the stored value; doc 1 has no stored "body".
+ assertEquals(newBytesRef("kept"), storedFields.document(0).getField("body").binaryValue());
+ assertNull(storedFields.document(1).getField("body"));
+
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testComposeAcrossBatches() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, stableIwc());
+
+ w.addBatch(
+ simpleBatch(
+ 1,
+ new ArrayBinaryColumn(
+ "body", storedBinaryType(), new int[] {0}, new BytesRef[] {newBytesRef("Hello")}),
+ new ArrayBinaryColumn(
+ "body", invertedType(), new int[] {0}, new BytesRef[] {newBytesRef("hello")})));
+ w.addBatch(
+ simpleBatch(
+ 1,
+ new ArrayBinaryColumn(
+ "body", storedBinaryType(), new int[] {0}, new BytesRef[] {newBytesRef("World")}),
+ new ArrayBinaryColumn(
+ "body", invertedType(), new int[] {0}, new BytesRef[] {newBytesRef("world")})));
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ IndexSearcher s = new IndexSearcher(r);
+
+ assertEquals(1, s.count(new TermQuery(new Term("body", "hello"))));
+ assertEquals(1, s.count(new TermQuery(new Term("body", "world"))));
+
+ StoredFields storedFields = leaf.storedFields();
+ assertEquals(newBytesRef("Hello"), storedFields.document(0).getField("body").binaryValue());
+ assertEquals(newBytesRef("World"), storedFields.document(1).getField("body").binaryValue());
+
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testComposeMixedWithAddDocument() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, stableIwc());
+
+ // A field type that is both stored and inverted, matching the merged schema produced by
+ // composing storedBinaryType() + invertedType() in a batch.
+ FieldType combined = new FieldType();
+ combined.setStored(true);
+ combined.setIndexOptions(IndexOptions.DOCS);
+ combined.setTokenized(false);
+ combined.setOmitNorms(true);
+ combined.freeze();
+
+ // doc 0 via addDocument establishes the FieldInfo (and the frozen-field fast path).
+ Document doc = new Document();
+ doc.add(new Field("body", "alpha", combined));
+ w.addDocument(doc);
+
+ // doc 1 via a composed batch for the same field name, in the same segment.
+ w.addBatch(
+ simpleBatch(
+ 1,
+ new ArrayBinaryColumn(
+ "body", storedBinaryType(), new int[] {0}, new BytesRef[] {newBytesRef("Beta")}),
+ new ArrayBinaryColumn(
+ "body", invertedType(), new int[] {0}, new BytesRef[] {newBytesRef("beta")})));
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ IndexSearcher s = new IndexSearcher(r);
+
+ assertEquals(1, s.count(new TermQuery(new Term("body", "alpha"))));
+ assertEquals(1, s.count(new TermQuery(new Term("body", "beta"))));
+
+ StoredFields storedFields = leaf.storedFields();
+ assertEquals("alpha", storedFields.document(0).getField("body").stringValue());
+ assertEquals(newBytesRef("Beta"), storedFields.document(1).getField("body").binaryValue());
+
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testRejectsDuplicateInversionFeature() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+ expectThrows(
+ IllegalArgumentException.class,
+ () ->
+ w.addBatch(
+ simpleBatch(
+ 1,
+ new ArrayBinaryColumn(
+ "body", invertedType(), new int[] {0}, new BytesRef[] {newBytesRef("a")}),
+ new ArrayBinaryColumn(
+ "body",
+ invertedType(),
+ new int[] {0},
+ new BytesRef[] {newBytesRef("b")}))));
+ w.close();
+ dir.close();
+ }
+
+ public void testRejectsDuplicateStoredFeature() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+ expectThrows(
+ IllegalArgumentException.class,
+ () ->
+ w.addBatch(
+ simpleBatch(
+ 1,
+ new ArrayBinaryColumn(
+ "body",
+ storedBinaryType(),
+ new int[] {0},
+ new BytesRef[] {newBytesRef("a")}),
+ new ArrayBinaryColumn(
+ "body",
+ storedBinaryType(),
+ new int[] {0},
+ new BytesRef[] {newBytesRef("b")}))));
+ w.close();
+ dir.close();
+ }
+
+ public void testRejectsDuplicateDocValuesFeature() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+ expectThrows(
+ IllegalArgumentException.class,
+ () ->
+ w.addBatch(
+ simpleBatch(
+ 1,
+ new ArrayLongColumn(
+ "x", NumericDocValuesField.TYPE, new int[] {0}, new long[] {1L}),
+ new ArrayLongColumn(
+ "x", NumericDocValuesField.TYPE, new int[] {0}, new long[] {2L}))));
+ w.close();
+ dir.close();
+ }
+
+ public void testRejectsDuplicatePointsFeature() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+ FieldType pointType = new FieldType();
+ pointType.setDimensions(1, Integer.BYTES);
+ pointType.freeze();
+ expectThrows(
+ IllegalArgumentException.class,
+ () ->
+ w.addBatch(
+ simpleBatch(
+ 1,
+ new ArrayLongColumn(
+ "p", pointType, LongColumn.NumericKind.INT, new int[] {0}, new long[] {1L}),
+ new ArrayLongColumn(
+ "p",
+ pointType,
+ LongColumn.NumericKind.INT,
+ new int[] {0},
+ new long[] {2L}))));
+ w.close();
+ dir.close();
+ }
+
+ public void testRejectsDuplicateVectorFeature() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+ FieldType vectorType = floatVectorType(2, VectorSimilarityFunction.EUCLIDEAN);
+ expectThrows(
+ IllegalArgumentException.class,
+ () ->
+ w.addBatch(
+ simpleBatch(
+ 1,
+ new ArrayFloatVectorColumn(
+ "v", vectorType, new int[] {0}, new float[][] {{1f, 2f}}),
+ new ArrayFloatVectorColumn(
+ "v", vectorType, new int[] {0}, new float[][] {{3f, 4f}}))));
+ w.close();
+ dir.close();
+ }
+}