You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2024/01/17 13:10:09 UTC
(lucene) branch branch_9x updated (00e2fe6cacb -> 28b9f2ef190)
This is an automated email from the ASF dual-hosted git repository.
simonw pushed a change to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
from 00e2fe6cacb LUCENE-10366: Override #readVInt and #readVLong for ByteBufferDataInput to avoid the abstraction confusion of #readByte. (#592)
new 0aa88910ca9 Add support for index sorting with document blocks (#12829)
new 28b9f2ef190 Prevent parent field from added to existing index (#13016)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
lucene/CHANGES.txt | 6 +
.../lucene60/Lucene60FieldInfosFormat.java | 3 +-
.../lucene90/Lucene90FieldInfosFormat.java | 3 +-
.../lucene70/TestLucene70SegmentInfoFormat.java | 5 +
.../lucene86/TestLucene86SegmentInfoFormat.java | 5 +
.../lucene90/TestLucene90SegmentInfoFormat.java | 5 +
.../backward_index/TestBackwardsCompatibility.java | 79 +++++++
.../simpletext/SimpleTextFieldInfosFormat.java | 11 +-
.../simpletext/SimpleTextSegmentInfoFormat.java | 9 +-
.../codecs/uniformsplit/TestBlockWriter.java | 3 +-
.../sharedterms/TestSTBlockReader.java | 1 +
.../codecs/lucene94/Lucene94FieldInfosFormat.java | 36 ++-
.../java/org/apache/lucene/index/CheckIndex.java | 29 ++-
.../lucene/index/DocumentsWriterPerThread.java | 57 ++++-
.../java/org/apache/lucene/index/FieldInfo.java | 24 +-
.../java/org/apache/lucene/index/FieldInfos.java | 103 +++++++-
.../java/org/apache/lucene/index/IndexWriter.java | 19 +-
.../org/apache/lucene/index/IndexWriterConfig.java | 16 ++
.../org/apache/lucene/index/IndexingChain.java | 127 +++++++++-
.../apache/lucene/index/LiveIndexWriterConfig.java | 9 +
.../java/org/apache/lucene/index/MultiSorter.java | 18 ++
.../apache/lucene/index/ParallelLeafReader.java | 9 +-
.../org/apache/lucene/index/ReadersAndUpdates.java | 3 +-
.../src/java/org/apache/lucene/index/Sorter.java | 16 +-
.../lucene/internal/tests/IndexPackageAccess.java | 2 +-
.../src/java/org/apache/lucene/search/Sort.java | 2 -
.../org/apache/lucene/index/TestAddIndexes.java | 93 ++++++++
.../test/org/apache/lucene/index/TestCodecs.java | 7 +-
.../src/test/org/apache/lucene/index/TestDoc.java | 2 +-
.../org/apache/lucene/index/TestFieldInfo.java | 5 +-
.../org/apache/lucene/index/TestFieldInfos.java | 5 +-
.../org/apache/lucene/index/TestFieldsReader.java | 7 +-
.../org/apache/lucene/index/TestIndexSorting.java | 261 +++++++++++++++++++++
.../org/apache/lucene/index/TestIndexWriter.java | 165 +++++++++++++
.../lucene/index/TestPendingSoftDeletes.java | 15 +-
.../org/apache/lucene/index/TestSegmentMerger.java | 2 +-
.../apache/lucene/search/TestSortOptimization.java | 3 +-
.../lucene/search/TestTopFieldCollector.java | 8 +-
.../search/highlight/TermVectorLeafReader.java | 1 +
.../apache/lucene/index/memory/MemoryIndex.java | 4 +-
.../compressing/dummy/DummyCompressingCodec.java | 2 +-
.../tests/index/BaseFieldInfoFormatTestCase.java | 20 +-
.../tests/index/BaseIndexFileFormatTestCase.java | 3 +-
.../tests/index/BaseSegmentInfoFormatTestCase.java | 37 ++-
.../lucene/tests/index/MismatchedLeafReader.java | 3 +-
.../lucene/tests/index/RandomIndexWriter.java | 1 +
.../lucene/tests/index/RandomPostingsTester.java | 2 +
47 files changed, 1154 insertions(+), 92 deletions(-)
(lucene) 02/02: Prevent parent field from added to existing index (#13016)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
simonw pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 28b9f2ef19062b7fd8551bdde4d88debd218cc59
Author: Simon Willnauer <si...@apache.org>
AuthorDate: Tue Jan 16 10:53:43 2024 +0100
Prevent parent field from added to existing index (#13016)
This change prevents users from adding a parent field to an existing index.
Parent field must be added before any documents are added to the index to
prevent documents without the parent field from being indexed and later
to be treated as child documents upon merge.
Relates to #12829
---
.../java/org/apache/lucene/index/IndexWriter.java | 7 +++-
.../org/apache/lucene/index/TestIndexWriter.java | 42 ++++++++++++++++++++++
2 files changed, 48 insertions(+), 1 deletion(-)
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index 79d93379868..89e8bcafd75 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -1121,6 +1121,12 @@ public class IndexWriter
// NOTE: this is correct even for an NRT reader because we'll pull FieldInfos even for the
// un-committed segments:
globalFieldNumberMap = getFieldNumberMap();
+ if (create == false
+ && conf.getParentField() != null
+ && globalFieldNumberMap.getFieldNames().contains(conf.getParentField()) == false) {
+ throw new IllegalArgumentException(
+ "can't add a parent field to an already existing index without a parent field");
+ }
validateIndexSort();
@@ -1272,7 +1278,6 @@ public class IndexWriter
map.addOrGet(fi);
}
}
-
return map;
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
index f4ef189112a..aea989720de 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
@@ -4943,6 +4943,48 @@ public class TestIndexWriter extends LuceneTestCase {
}
}
+ public void testParentFieldExistingIndex() throws IOException {
+ try (Directory dir = newDirectory()) {
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ try (IndexWriter writer = new IndexWriter(dir, iwc)) {
+ writer.addDocument(new Document());
+ }
+ IllegalArgumentException iae =
+ expectThrows(
+ IllegalArgumentException.class,
+ () ->
+ new IndexWriter(
+ dir,
+ new IndexWriterConfig(new MockAnalyzer(random()))
+ .setOpenMode(OpenMode.APPEND)
+ .setParentField("foo")));
+ assertEquals(
+ "can't add a parent field to an already existing index without a parent field",
+ iae.getMessage());
+ iae =
+ expectThrows(
+ IllegalArgumentException.class,
+ () ->
+ new IndexWriter(
+ dir,
+ new IndexWriterConfig(new MockAnalyzer(random()))
+ .setOpenMode(OpenMode.CREATE_OR_APPEND)
+ .setParentField("foo")));
+ assertEquals(
+ "can't add a parent field to an already existing index without a parent field",
+ iae.getMessage());
+
+ try (IndexWriter writer =
+ new IndexWriter(
+ dir,
+ new IndexWriterConfig(new MockAnalyzer(random()))
+ .setOpenMode(OpenMode.CREATE)
+ .setParentField("foo"))) {
+ writer.addDocument(new Document());
+ }
+ }
+ }
+
public void testIndexWithParentFieldIsCongruent() throws IOException {
try (Directory dir = newDirectory()) {
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
(lucene) 01/02: Add support for index sorting with document blocks (#12829)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
simonw pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 0aa88910ca9a1032d288996d14203eac4953f2de
Author: Simon Willnauer <si...@apache.org>
AuthorDate: Thu Jan 11 16:11:15 2024 +0100
Add support for index sorting with document blocks (#12829)
Today index sorting will most likely break document blocks added with `IndexWriter#addDocuments(...)` and `#updateDocuments(...)` since the index sorter has no indication of what documents are part of a block. This change automatically adds a marker field to parent documents if configured in `IWC`. These marker documents are optional unless document blocks are indexed and index sorting is configured. In this case indexing blocks will fail unless a parent field is configured. Index so [...]
Relates to #12711
---
lucene/CHANGES.txt | 6 +
.../lucene60/Lucene60FieldInfosFormat.java | 3 +-
.../lucene90/Lucene90FieldInfosFormat.java | 3 +-
.../lucene70/TestLucene70SegmentInfoFormat.java | 5 +
.../lucene86/TestLucene86SegmentInfoFormat.java | 5 +
.../lucene90/TestLucene90SegmentInfoFormat.java | 5 +
.../backward_index/TestBackwardsCompatibility.java | 79 +++++++
.../simpletext/SimpleTextFieldInfosFormat.java | 11 +-
.../simpletext/SimpleTextSegmentInfoFormat.java | 9 +-
.../codecs/uniformsplit/TestBlockWriter.java | 3 +-
.../sharedterms/TestSTBlockReader.java | 1 +
.../codecs/lucene94/Lucene94FieldInfosFormat.java | 36 ++-
.../java/org/apache/lucene/index/CheckIndex.java | 29 ++-
.../lucene/index/DocumentsWriterPerThread.java | 57 ++++-
.../java/org/apache/lucene/index/FieldInfo.java | 24 +-
.../java/org/apache/lucene/index/FieldInfos.java | 103 +++++++-
.../java/org/apache/lucene/index/IndexWriter.java | 12 +-
.../org/apache/lucene/index/IndexWriterConfig.java | 16 ++
.../org/apache/lucene/index/IndexingChain.java | 127 +++++++++-
.../apache/lucene/index/LiveIndexWriterConfig.java | 9 +
.../java/org/apache/lucene/index/MultiSorter.java | 18 ++
.../apache/lucene/index/ParallelLeafReader.java | 9 +-
.../org/apache/lucene/index/ReadersAndUpdates.java | 3 +-
.../src/java/org/apache/lucene/index/Sorter.java | 16 +-
.../lucene/internal/tests/IndexPackageAccess.java | 2 +-
.../src/java/org/apache/lucene/search/Sort.java | 2 -
.../org/apache/lucene/index/TestAddIndexes.java | 93 ++++++++
.../test/org/apache/lucene/index/TestCodecs.java | 7 +-
.../src/test/org/apache/lucene/index/TestDoc.java | 2 +-
.../org/apache/lucene/index/TestFieldInfo.java | 5 +-
.../org/apache/lucene/index/TestFieldInfos.java | 5 +-
.../org/apache/lucene/index/TestFieldsReader.java | 7 +-
.../org/apache/lucene/index/TestIndexSorting.java | 261 +++++++++++++++++++++
.../org/apache/lucene/index/TestIndexWriter.java | 123 ++++++++++
.../lucene/index/TestPendingSoftDeletes.java | 15 +-
.../org/apache/lucene/index/TestSegmentMerger.java | 2 +-
.../apache/lucene/search/TestSortOptimization.java | 3 +-
.../lucene/search/TestTopFieldCollector.java | 8 +-
.../search/highlight/TermVectorLeafReader.java | 1 +
.../apache/lucene/index/memory/MemoryIndex.java | 4 +-
.../compressing/dummy/DummyCompressingCodec.java | 2 +-
.../tests/index/BaseFieldInfoFormatTestCase.java | 20 +-
.../tests/index/BaseIndexFileFormatTestCase.java | 3 +-
.../tests/index/BaseSegmentInfoFormatTestCase.java | 37 ++-
.../lucene/tests/index/MismatchedLeafReader.java | 3 +-
.../lucene/tests/index/RandomIndexWriter.java | 1 +
.../lucene/tests/index/RandomPostingsTester.java | 2 +
47 files changed, 1106 insertions(+), 91 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 4068f28e972..95ceeb47157 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -24,6 +24,12 @@ New Features
better-scoring nodes are available, or the best candidate is below a score of `traversalSimilarity` in the lowest
level. (Aditya Prakash, Kaival Parikh)
+* GITHUB#12829: For indices newly created as of 9.10.0 onwards, IndexWriter preserves document blocks indexed via
+ IndexWriter#addDocuments or IndexWriter#updateDocuments also when index sorting is configured. Document blocks are
+ maintained alongside their parent documents during sort and merge. IndexWriterConfig accepts a parent field that is used
+ to maintain block orders if index sorting is used. Note, this is fully optional in Lucene 9.x while will be mandatory for
+ indices that use document blocks together with index sorting as of 10.0.0. (Simon Willnauer)
+
Improvements
---------------------
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60FieldInfosFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60FieldInfosFormat.java
index 63347e18bab..a3e09db8ae9 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60FieldInfosFormat.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60FieldInfosFormat.java
@@ -217,7 +217,8 @@ public final class Lucene60FieldInfosFormat extends FieldInfosFormat {
0,
VectorEncoding.FLOAT32,
VectorSimilarityFunction.EUCLIDEAN,
- isSoftDeletesField);
+ isSoftDeletesField,
+ false);
} catch (IllegalStateException e) {
throw new CorruptIndexException(
"invalid fieldinfo for field: " + name + ", fieldNumber=" + fieldNumber, input, e);
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90FieldInfosFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90FieldInfosFormat.java
index 182083954c4..d710c68ea1b 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90FieldInfosFormat.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90FieldInfosFormat.java
@@ -194,7 +194,8 @@ public final class Lucene90FieldInfosFormat extends FieldInfosFormat {
vectorDimension,
VectorEncoding.FLOAT32,
vectorDistFunc,
- isSoftDeletesField);
+ isSoftDeletesField,
+ false);
infos[i].checkConsistency();
} catch (IllegalStateException e) {
throw new CorruptIndexException(
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene70/TestLucene70SegmentInfoFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene70/TestLucene70SegmentInfoFormat.java
index 27222ded1c8..c1d3d506999 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene70/TestLucene70SegmentInfoFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene70/TestLucene70SegmentInfoFormat.java
@@ -34,4 +34,9 @@ public class TestLucene70SegmentInfoFormat extends BaseSegmentInfoFormatTestCase
protected Codec getCodec() {
return new Lucene84RWCodec();
}
+
+ @Override
+ protected boolean supportsHasBlocks() {
+ return false;
+ }
}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/TestLucene86SegmentInfoFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/TestLucene86SegmentInfoFormat.java
index c5760dc92ac..5434a7d05ff 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/TestLucene86SegmentInfoFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/TestLucene86SegmentInfoFormat.java
@@ -34,4 +34,9 @@ public class TestLucene86SegmentInfoFormat extends BaseSegmentInfoFormatTestCase
protected Codec getCodec() {
return new Lucene87RWCodec();
}
+
+ @Override
+ protected boolean supportsHasBlocks() {
+ return false;
+ }
}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90SegmentInfoFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90SegmentInfoFormat.java
index 53a8a01a440..9ecc3490641 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90SegmentInfoFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90SegmentInfoFormat.java
@@ -32,4 +32,9 @@ public class TestLucene90SegmentInfoFormat extends BaseSegmentInfoFormatTestCase
protected Codec getCodec() {
return new Lucene90RWCodec();
}
+
+ @Override
+ protected boolean supportsHasBlocks() {
+ return false;
+ }
}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBackwardsCompatibility.java
index 5f85533a979..53dd8cfb86f 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBackwardsCompatibility.java
@@ -98,6 +98,8 @@ import org.apache.lucene.index.TermVectors;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.FieldExistsQuery;
@@ -2187,6 +2189,83 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
}
}
+ public void testSortedIndexAddDocBlocks() throws Exception {
+ for (String name : oldSortedNames) {
+ Path path = createTempDir("sorted");
+ InputStream resource = TestBackwardsCompatibility.class.getResourceAsStream(name + ".zip");
+ assertNotNull("Sorted index index " + name + " not found", resource);
+ TestUtil.unzip(resource, path);
+
+ try (Directory dir = newFSDirectory(path)) {
+ final Sort sort;
+ try (DirectoryReader reader = DirectoryReader.open(dir)) {
+ assertEquals(1, reader.leaves().size());
+ sort = reader.leaves().get(0).reader().getMetaData().getSort();
+ assertNotNull(sort);
+ searchExampleIndex(reader);
+ }
+ // open writer
+ try (IndexWriter writer =
+ new IndexWriter(
+ dir,
+ newIndexWriterConfig(new MockAnalyzer(random()))
+ .setOpenMode(OpenMode.APPEND)
+ .setIndexSort(sort)
+ .setMergePolicy(newLogMergePolicy()))) {
+ // add 10 docs
+ for (int i = 0; i < 10; i++) {
+ Document child = new Document();
+ child.add(new StringField("relation", "child", Field.Store.NO));
+ child.add(new StringField("bid", "" + i, Field.Store.NO));
+ child.add(new NumericDocValuesField("dateDV", i));
+ Document parent = new Document();
+ parent.add(new StringField("relation", "parent", Field.Store.NO));
+ parent.add(new StringField("bid", "" + i, Field.Store.NO));
+ parent.add(new NumericDocValuesField("dateDV", i));
+ writer.addDocuments(Arrays.asList(child, child, parent));
+ if (random().nextBoolean()) {
+ writer.flush();
+ }
+ }
+ if (random().nextBoolean()) {
+ writer.forceMerge(1);
+ }
+ writer.commit();
+ try (IndexReader reader = DirectoryReader.open(dir)) {
+ IndexSearcher searcher = new IndexSearcher(reader);
+ for (int i = 0; i < 10; i++) {
+ TopDocs children =
+ searcher.search(
+ new BooleanQuery.Builder()
+ .add(
+ new TermQuery(new Term("relation", "child")),
+ BooleanClause.Occur.MUST)
+ .add(new TermQuery(new Term("bid", "" + i)), BooleanClause.Occur.MUST)
+ .build(),
+ 2);
+ TopDocs parents =
+ searcher.search(
+ new BooleanQuery.Builder()
+ .add(
+ new TermQuery(new Term("relation", "parent")),
+ BooleanClause.Occur.MUST)
+ .add(new TermQuery(new Term("bid", "" + i)), BooleanClause.Occur.MUST)
+ .build(),
+ 2);
+ assertEquals(2, children.totalHits.value);
+ assertEquals(1, parents.totalHits.value);
+ // make sure it's sorted
+ assertEquals(children.scoreDocs[0].doc + 1, children.scoreDocs[1].doc);
+ assertEquals(children.scoreDocs[1].doc + 1, parents.scoreDocs[0].doc);
+ }
+ }
+ }
+ // This will confirm the docs are really sorted
+ TestUtil.checkIndex(dir);
+ }
+ }
+ }
+
private void searchExampleIndex(DirectoryReader reader) throws IOException {
IndexSearcher searcher = newSearcher(reader);
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java
index 00d5ea17055..381ed34f311 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java
@@ -72,6 +72,7 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat {
static final BytesRef VECTOR_ENCODING = new BytesRef(" vector encoding ");
static final BytesRef VECTOR_SIMILARITY = new BytesRef(" vector similarity ");
static final BytesRef SOFT_DELETES = new BytesRef(" soft-deletes ");
+ static final BytesRef PARENT = new BytesRef(" parent ");
@Override
public FieldInfos read(
@@ -170,6 +171,9 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat {
SimpleTextUtil.readLine(input, scratch);
assert StringHelper.startsWith(scratch.get(), SOFT_DELETES);
boolean isSoftDeletesField = Boolean.parseBoolean(readString(SOFT_DELETES.length, scratch));
+ SimpleTextUtil.readLine(input, scratch);
+ assert StringHelper.startsWith(scratch.get(), PARENT);
+ boolean isParentField = Boolean.parseBoolean(readString(PARENT.length, scratch));
infos[i] =
new FieldInfo(
@@ -188,7 +192,8 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat {
vectorNumDimensions,
vectorEncoding,
vectorDistFunc,
- isSoftDeletesField);
+ isSoftDeletesField,
+ isParentField);
}
SimpleTextUtil.checkFooter(input);
@@ -320,6 +325,10 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat {
SimpleTextUtil.write(out, SOFT_DELETES);
SimpleTextUtil.write(out, Boolean.toString(fi.isSoftDeletesField()), scratch);
SimpleTextUtil.writeNewline(out);
+
+ SimpleTextUtil.write(out, PARENT);
+ SimpleTextUtil.write(out, Boolean.toString(fi.isParentField()), scratch);
+ SimpleTextUtil.writeNewline(out);
}
SimpleTextUtil.writeChecksum(out, scratch);
success = true;
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java
index 5785539c6e3..df2d3f63946 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java
@@ -196,7 +196,13 @@ public class SimpleTextSegmentInfoFormat extends SegmentInfoFormat {
sortField[i] = SortFieldProvider.forName(provider).readSortField(bytes);
assert bytes.eof();
}
- Sort indexSort = sortField.length == 0 ? null : new Sort(sortField);
+
+ final Sort indexSort;
+ if (sortField.length == 0) {
+ indexSort = null;
+ } else {
+ indexSort = new Sort(sortField);
+ }
SimpleTextUtil.checkFooter(input);
@@ -335,7 +341,6 @@ public class SimpleTextSegmentInfoFormat extends SegmentInfoFormat {
SimpleTextUtil.write(output, b.bytes.get().toString(), scratch);
SimpleTextUtil.writeNewline(output);
}
-
SimpleTextUtil.writeChecksum(output, scratch);
}
}
diff --git a/lucene/codecs/src/test/org/apache/lucene/codecs/uniformsplit/TestBlockWriter.java b/lucene/codecs/src/test/org/apache/lucene/codecs/uniformsplit/TestBlockWriter.java
index 46a78341a5c..2ee6b8fd2d2 100644
--- a/lucene/codecs/src/test/org/apache/lucene/codecs/uniformsplit/TestBlockWriter.java
+++ b/lucene/codecs/src/test/org/apache/lucene/codecs/uniformsplit/TestBlockWriter.java
@@ -119,6 +119,7 @@ public class TestBlockWriter extends LuceneTestCase {
0,
VectorEncoding.FLOAT32,
VectorSimilarityFunction.EUCLIDEAN,
- true);
+ true,
+ false);
}
}
diff --git a/lucene/codecs/src/test/org/apache/lucene/codecs/uniformsplit/sharedterms/TestSTBlockReader.java b/lucene/codecs/src/test/org/apache/lucene/codecs/uniformsplit/sharedterms/TestSTBlockReader.java
index 4571a49243d..bf2b0133240 100644
--- a/lucene/codecs/src/test/org/apache/lucene/codecs/uniformsplit/sharedterms/TestSTBlockReader.java
+++ b/lucene/codecs/src/test/org/apache/lucene/codecs/uniformsplit/sharedterms/TestSTBlockReader.java
@@ -206,6 +206,7 @@ public class TestSTBlockReader extends LuceneTestCase {
0,
VectorEncoding.FLOAT32,
VectorSimilarityFunction.EUCLIDEAN,
+ false,
false);
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94FieldInfosFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94FieldInfosFormat.java
index 5a3957aa3c1..0567cebefac 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94FieldInfosFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94FieldInfosFormat.java
@@ -131,13 +131,14 @@ public final class Lucene94FieldInfosFormat extends FieldInfosFormat {
Throwable priorE = null;
FieldInfo[] infos = null;
try {
- CodecUtil.checkIndexHeader(
- input,
- Lucene94FieldInfosFormat.CODEC_NAME,
- Lucene94FieldInfosFormat.FORMAT_START,
- Lucene94FieldInfosFormat.FORMAT_CURRENT,
- segmentInfo.getId(),
- segmentSuffix);
+ int format =
+ CodecUtil.checkIndexHeader(
+ input,
+ Lucene94FieldInfosFormat.CODEC_NAME,
+ Lucene94FieldInfosFormat.FORMAT_START,
+ Lucene94FieldInfosFormat.FORMAT_CURRENT,
+ segmentInfo.getId(),
+ segmentSuffix);
final int size = input.readVInt(); // read in the size
infos = new FieldInfo[size];
@@ -157,6 +158,18 @@ public final class Lucene94FieldInfosFormat extends FieldInfosFormat {
boolean omitNorms = (bits & OMIT_NORMS) != 0;
boolean storePayloads = (bits & STORE_PAYLOADS) != 0;
boolean isSoftDeletesField = (bits & SOFT_DELETES_FIELD) != 0;
+ boolean isParentField =
+ format >= FORMAT_PARENT_FIELD ? (bits & PARENT_FIELD_FIELD) != 0 : false;
+
+ if ((bits & 0xE0) != 0) {
+ throw new CorruptIndexException(
+ "unused bits are set \"" + Integer.toBinaryString(bits) + "\"", input);
+ }
+ if (format < FORMAT_PARENT_FIELD && (bits & 0xF0) != 0) {
+ throw new CorruptIndexException(
+ "parent field bit is set but shouldn't \"" + Integer.toBinaryString(bits) + "\"",
+ input);
+ }
final IndexOptions indexOptions = getIndexOptions(input, input.readByte());
@@ -200,7 +213,8 @@ public final class Lucene94FieldInfosFormat extends FieldInfosFormat {
vectorDimension,
vectorEncoding,
vectorDistFunc,
- isSoftDeletesField);
+ isSoftDeletesField,
+ isParentField);
infos[i].checkConsistency();
} catch (IllegalStateException e) {
throw new CorruptIndexException(
@@ -348,6 +362,7 @@ public final class Lucene94FieldInfosFormat extends FieldInfosFormat {
if (fi.omitsNorms()) bits |= OMIT_NORMS;
if (fi.hasPayloads()) bits |= STORE_PAYLOADS;
if (fi.isSoftDeletesField()) bits |= SOFT_DELETES_FIELD;
+ if (fi.isParentField()) bits |= PARENT_FIELD_FIELD;
output.writeByte(bits);
output.writeByte(indexOptionsByte(fi.getIndexOptions()));
@@ -375,11 +390,14 @@ public final class Lucene94FieldInfosFormat extends FieldInfosFormat {
// Codec header
static final String CODEC_NAME = "Lucene94FieldInfos";
static final int FORMAT_START = 0;
- static final int FORMAT_CURRENT = FORMAT_START;
+ // this doesn't actually change the file format but uses up one more bit an existing bit pattern
+ static final int FORMAT_PARENT_FIELD = 1;
+ static final int FORMAT_CURRENT = FORMAT_PARENT_FIELD;
// Field flags
static final byte STORE_TERMVECTOR = 0x1;
static final byte OMIT_NORMS = 0x2;
static final byte STORE_PAYLOADS = 0x4;
static final byte SOFT_DELETES_FIELD = 0x8;
+ static final byte PARENT_FIELD_FIELD = 0x10;
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
index dd662572de7..884085d1232 100644
--- a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
+++ b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
@@ -1196,34 +1196,39 @@ public final class CheckIndex implements Closeable {
comparators[i] = fields[i].getComparator(1, Pruning.NONE).getLeafComparator(readerContext);
}
- int maxDoc = reader.maxDoc();
-
try {
-
- for (int docID = 1; docID < maxDoc; docID++) {
-
+ LeafMetaData metaData = reader.getMetaData();
+ FieldInfos fieldInfos = reader.getFieldInfos();
+ final DocIdSetIterator iter;
+ if (metaData.hasBlocks() && fieldInfos.getParentField() != null) {
+ iter = reader.getNumericDocValues(fieldInfos.getParentField());
+ } else {
+ iter = DocIdSetIterator.all(reader.maxDoc());
+ }
+ int prevDoc = iter.nextDoc();
+ int nextDoc;
+ while ((nextDoc = iter.nextDoc()) != NO_MORE_DOCS) {
int cmp = 0;
-
for (int i = 0; i < comparators.length; i++) {
- // TODO: would be better if copy() didnt cause a term lookup in TermOrdVal & co,
+ // TODO: would be better if copy() didn't cause a term lookup in TermOrdVal & co,
// the segments are always the same here...
- comparators[i].copy(0, docID - 1);
+ comparators[i].copy(0, prevDoc);
comparators[i].setBottom(0);
- cmp = reverseMul[i] * comparators[i].compareBottom(docID);
+ cmp = reverseMul[i] * comparators[i].compareBottom(nextDoc);
if (cmp != 0) {
break;
}
}
-
if (cmp > 0) {
throw new CheckIndexException(
"segment has indexSort="
+ sort
+ " but docID="
- + (docID - 1)
+ + (prevDoc)
+ " sorts after docID="
- + docID);
+ + nextDoc);
}
+ prevDoc = nextDoc;
}
msg(
infoStream,
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
index 88d64af2ba4..588bcdc9896 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
@@ -21,8 +21,10 @@ import java.text.NumberFormat;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Locale;
+import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@@ -31,6 +33,7 @@ import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.index.DocumentsWriterDeleteQueue.DeleteSlice;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory;
@@ -136,9 +139,10 @@ final class DocumentsWriterPerThread implements Accountable, Lock {
private final ReentrantLock lock = new ReentrantLock();
private int[] deleteDocIDs = new int[0];
private int numDeletedDocIds = 0;
+ private final IndexingChain.ReservedField<NumericDocValuesField> parentField;
DocumentsWriterPerThread(
- int indexVersionCreated,
+ int indexMajorVersionCreated,
String segmentName,
Directory directoryOrig,
Directory directory,
@@ -185,12 +189,19 @@ final class DocumentsWriterPerThread implements Accountable, Lock {
this.enableTestPoints = enableTestPoints;
indexingChain =
new IndexingChain(
- indexVersionCreated,
+ indexMajorVersionCreated,
segmentInfo,
this.directory,
fieldInfos,
indexWriterConfig,
this::onAbortingException);
+ if (indexWriterConfig.getParentField() != null) {
+ this.parentField =
+ indexingChain.markAsReserved(
+ new NumericDocValuesField(indexWriterConfig.getParentField(), -1));
+ } else {
+ this.parentField = null;
+ }
}
final void testPoint(String message) {
@@ -233,7 +244,14 @@ final class DocumentsWriterPerThread implements Accountable, Lock {
final int docsInRamBefore = numDocsInRAM;
boolean allDocsIndexed = false;
try {
- for (Iterable<? extends IndexableField> doc : docs) {
+ final Iterator<? extends Iterable<? extends IndexableField>> iterator = docs.iterator();
+ while (iterator.hasNext()) {
+ Iterable<? extends IndexableField> doc = iterator.next();
+ if (parentField != null) {
+ if (iterator.hasNext() == false) {
+ doc = addParentField(doc, parentField);
+ }
+ }
// Even on exception, the document is still added (but marked
// deleted), so we don't need to un-reserve at that point.
// Aborting exceptions will actually "lose" more than one
@@ -247,10 +265,11 @@ final class DocumentsWriterPerThread implements Accountable, Lock {
onNewDocOnRAM.run();
}
}
- allDocsIndexed = true;
- if (numDocsInRAM - docsInRamBefore > 1) {
+ final int numDocs = numDocsInRAM - docsInRamBefore;
+ if (numDocs > 1) {
segmentInfo.setHasBlocks();
}
+ allDocsIndexed = true;
return finishDocuments(deleteNode, docsInRamBefore);
} finally {
if (!allDocsIndexed && !aborted) {
@@ -264,6 +283,34 @@ final class DocumentsWriterPerThread implements Accountable, Lock {
}
}
+ private Iterable<? extends IndexableField> addParentField(
+ Iterable<? extends IndexableField> doc, IndexableField parentField) {
+ return () -> {
+ final Iterator<? extends IndexableField> first = doc.iterator();
+ return new Iterator<>() {
+ IndexableField additionalField = parentField;
+
+ @Override
+ public boolean hasNext() {
+ return additionalField != null || first.hasNext();
+ }
+
+ @Override
+ public IndexableField next() {
+ if (additionalField != null) {
+ IndexableField field = additionalField;
+ additionalField = null;
+ return field;
+ }
+ if (first.hasNext()) {
+ return first.next();
+ }
+ throw new NoSuchElementException();
+ }
+ };
+ };
+ }
+
private long finishDocuments(DocumentsWriterDeleteQueue.Node<?> deleteNode, int docIdUpTo) {
/*
* here we actually finish the document in two steps 1. push the delete into
diff --git a/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java b/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
index d6da149d13b..fea11b675da 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
@@ -63,6 +63,8 @@ public final class FieldInfo {
// whether this field is used as the soft-deletes field
private final boolean softDeletesField;
+ private final boolean isParentField;
+
/**
* Sole constructor.
*
@@ -84,7 +86,8 @@ public final class FieldInfo {
int vectorDimension,
VectorEncoding vectorEncoding,
VectorSimilarityFunction vectorSimilarityFunction,
- boolean softDeletesField) {
+ boolean softDeletesField,
+ boolean isParentField) {
this.name = Objects.requireNonNull(name);
this.number = number;
this.docValuesType =
@@ -111,6 +114,7 @@ public final class FieldInfo {
this.vectorEncoding = vectorEncoding;
this.vectorSimilarityFunction = vectorSimilarityFunction;
this.softDeletesField = softDeletesField;
+ this.isParentField = isParentField;
this.checkConsistency();
}
@@ -206,6 +210,13 @@ public final class FieldInfo {
throw new IllegalArgumentException(
"vectorDimension must be >=0; got " + vectorDimension + " (field: '" + name + "')");
}
+
+ if (softDeletesField && isParentField) {
+ throw new IllegalArgumentException(
+ "field can't be used as soft-deletes field and parent document field (field: '"
+ + name
+ + "')");
+ }
}
/**
@@ -542,7 +553,8 @@ public final class FieldInfo {
this.vectorDimension,
this.vectorEncoding,
this.vectorSimilarityFunction,
- this.softDeletesField);
+ this.softDeletesField,
+ this.isParentField);
}
/**
@@ -801,4 +813,12 @@ public final class FieldInfo {
public boolean isSoftDeletesField() {
return softDeletesField;
}
+
+ /**
+ * Returns true if this field is configured and used as the parent document field field. See
+ * {@link IndexWriterConfig#setParentField(String)}
+ */
+ public boolean isParentField() {
+ return isParentField;
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
index 85aac5439d3..56ea002db47 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
@@ -31,6 +31,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
@@ -59,6 +60,8 @@ public class FieldInfos implements Iterable<FieldInfo> {
private final boolean hasVectorValues;
private final String softDeletesField;
+ private final String parentField;
+
// used only by fieldInfo(int)
private final FieldInfo[] byNumber;
@@ -78,6 +81,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
boolean hasPointValues = false;
boolean hasVectorValues = false;
String softDeletesField = null;
+ String parentField = null;
int size = 0; // number of elements in byNumberTemp, number of used array slots
FieldInfo[] byNumberTemp = new FieldInfo[10]; // initial array capacity of 10
@@ -132,6 +136,13 @@ public class FieldInfos implements Iterable<FieldInfo> {
}
softDeletesField = info.name;
}
+ if (info.isParentField()) {
+ if (parentField != null && parentField.equals(info.name) == false) {
+ throw new IllegalArgumentException(
+ "multiple parent fields [" + info.name + ", " + parentField + "]");
+ }
+ parentField = info.name;
+ }
}
this.hasVectors = hasVectors;
@@ -145,6 +156,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
this.hasPointValues = hasPointValues;
this.hasVectorValues = hasVectorValues;
this.softDeletesField = softDeletesField;
+ this.parentField = parentField;
List<FieldInfo> valuesTemp = new ArrayList<>();
byNumber = new FieldInfo[size];
@@ -203,8 +215,10 @@ public class FieldInfos implements Iterable<FieldInfo> {
if (indexCreatedVersionMajor == -1) {
indexCreatedVersionMajor = Version.LATEST.major;
}
+ final String parentField = getAndValidateParentField(leaves);
+
final Builder builder =
- new Builder(new FieldNumbers(softDeletesField, indexCreatedVersionMajor));
+ new Builder(new FieldNumbers(softDeletesField, parentField, indexCreatedVersionMajor));
for (final LeafReaderContext ctx : leaves) {
for (FieldInfo fieldInfo : ctx.reader().getFieldInfos()) {
builder.add(fieldInfo);
@@ -214,6 +228,26 @@ public class FieldInfos implements Iterable<FieldInfo> {
}
}
+ private static String getAndValidateParentField(List<LeafReaderContext> leaves) {
+ boolean set = false;
+ String theField = null;
+ for (LeafReaderContext ctx : leaves) {
+ String field = ctx.reader().getFieldInfos().getParentField();
+ if (set && Objects.equals(field, theField) == false) {
+ throw new IllegalStateException(
+ "expected parent doc field to be \""
+ + theField
+ + " \" across all segments but found a segment with different field \""
+ + field
+ + "\"");
+ } else {
+ theField = field;
+ set = true;
+ }
+ }
+ return theField;
+ }
+
/** Returns a set of names of fields that have a terms index. The order is undefined. */
public static Collection<String> getIndexedFields(IndexReader reader) {
return reader.leaves().stream()
@@ -280,6 +314,11 @@ public class FieldInfos implements Iterable<FieldInfo> {
return softDeletesField;
}
+ /** Returns the parent document field name if exists; otherwise returns null */
+ public String getParentField() {
+ return parentField;
+ }
+
/** Returns the number of fields */
public int size() {
return byName.size();
@@ -372,7 +411,11 @@ public class FieldInfos implements Iterable<FieldInfo> {
private final String softDeletesFieldName;
private final boolean strictlyConsistent;
- FieldNumbers(String softDeletesFieldName, int indexCreatedVersionMajor) {
+ // The parent document field from IWC to mark parent document when indexing
+ private final String parentFieldName;
+
+ FieldNumbers(
+ String softDeletesFieldName, String parentFieldName, int indexCreatedVersionMajor) {
this.nameToNumber = new HashMap<>();
this.numberToName = new HashMap<>();
this.indexOptions = new HashMap<>();
@@ -383,11 +426,21 @@ public class FieldInfos implements Iterable<FieldInfo> {
this.storeTermVectors = new HashMap<>();
this.softDeletesFieldName = softDeletesFieldName;
this.strictlyConsistent = indexCreatedVersionMajor >= 9;
+ this.parentFieldName = parentFieldName;
+ if (softDeletesFieldName != null
+ && parentFieldName != null
+ && parentFieldName.equals(softDeletesFieldName)) {
+ throw new IllegalArgumentException(
+ "parent document and soft-deletes field can't be the same field \""
+ + parentFieldName
+ + "\"");
+ }
}
synchronized void verifyFieldInfo(FieldInfo fi) {
String fieldName = fi.getName();
verifySoftDeletedFieldName(fieldName, fi.isSoftDeletesField());
+ verifyParentFieldName(fieldName, fi.isParentField());
if (nameToNumber.containsKey(fieldName)) {
verifySameSchema(fi);
}
@@ -401,6 +454,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
synchronized int addOrGet(FieldInfo fi) {
String fieldName = fi.getName();
verifySoftDeletedFieldName(fieldName, fi.isSoftDeletesField());
+ verifyParentFieldName(fieldName, fi.isParentField());
Integer fieldNumber = nameToNumber.get(fieldName);
if (fieldNumber != null) {
@@ -465,6 +519,33 @@ public class FieldInfos implements Iterable<FieldInfo> {
}
}
+ private void verifyParentFieldName(String fieldName, boolean isParentField) {
+ if (isParentField) {
+ if (parentFieldName == null) {
+ throw new IllegalArgumentException(
+ "can't add field ["
+ + fieldName
+ + "] as parent document field; this IndexWriter has no parent document field configured");
+ } else if (fieldName.equals(parentFieldName) == false) {
+ throw new IllegalArgumentException(
+ "can't add field ["
+ + fieldName
+ + "] as parent document field; this IndexWriter is configured with ["
+ + parentFieldName
+ + "] as parent document field");
+ }
+ } else if (fieldName.equals(parentFieldName)) { // isParent == false
+ // this would be the case if the current index has a parent field that is
+ // not a parent field in the incoming index (think addIndices)
+ throw new IllegalArgumentException(
+ "can't add ["
+ + fieldName
+ + "] as non parent document field; this IndexWriter is configured with ["
+ + parentFieldName
+ + "] as parent document field");
+ }
+ }
+
private void verifySameSchema(FieldInfo fi) {
String fieldName = fi.getName();
IndexOptions currentOpts = this.indexOptions.get(fieldName);
@@ -543,7 +624,8 @@ public class FieldInfos implements Iterable<FieldInfo> {
0,
VectorEncoding.FLOAT32,
VectorSimilarityFunction.EUCLIDEAN,
- (softDeletesFieldName != null && softDeletesFieldName.equals(fieldName)));
+ (softDeletesFieldName != null && softDeletesFieldName.equals(fieldName)),
+ (parentFieldName != null && parentFieldName.equals(fieldName)));
addOrGet(fi);
}
} else {
@@ -609,6 +691,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
if (dvType != dvType0) return null;
boolean isSoftDeletesField = fieldName.equals(softDeletesFieldName);
+ boolean isParentField = fieldName.equals(parentFieldName);
return new FieldInfo(
fieldName,
newFieldNumber,
@@ -625,7 +708,8 @@ public class FieldInfos implements Iterable<FieldInfo> {
0,
VectorEncoding.FLOAT32,
VectorSimilarityFunction.EUCLIDEAN,
- isSoftDeletesField);
+ isSoftDeletesField,
+ isParentField);
}
synchronized void setDocValuesType(int number, String name, DocValuesType dvType) {
@@ -699,6 +783,14 @@ public class FieldInfos implements Iterable<FieldInfo> {
return globalFieldNumbers.softDeletesFieldName;
}
+ /**
+ * Returns the name of the parent document field or <tt>null</tt> if no parent field is
+ * configured
+ */
+ public String getParentFieldName() {
+ return globalFieldNumbers.parentFieldName;
+ }
+
/**
* Adds the provided FieldInfo to this Builder if this field doesn't exist in this Builder. Also
* adds a new field with its schema options to the global FieldNumbers if the field doesn't
@@ -802,7 +894,8 @@ public class FieldInfos implements Iterable<FieldInfo> {
fi.getVectorDimension(),
fi.getVectorEncoding(),
fi.getVectorSimilarityFunction(),
- fi.isSoftDeletesField());
+ fi.isSoftDeletesField(),
+ fi.isParentField());
byName.put(fiNew.getName(), fiNew);
return fiNew;
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index d838b493ac7..79d93379868 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -1262,8 +1262,10 @@ public class IndexWriter
*/
private FieldNumbers getFieldNumberMap() throws IOException {
final FieldNumbers map =
- new FieldNumbers(config.softDeletesField, segmentInfos.getIndexCreatedVersionMajor());
-
+ new FieldNumbers(
+ config.softDeletesField,
+ config.getParentField(),
+ segmentInfos.getIndexCreatedVersionMajor());
for (SegmentCommitInfo info : segmentInfos) {
FieldInfos fis = readFieldInfos(info);
for (FieldInfo fi : fis) {
@@ -6615,11 +6617,13 @@ public class IndexWriter
}
@Override
- public FieldInfosBuilder newFieldInfosBuilder(String softDeletesFieldName) {
+ public FieldInfosBuilder newFieldInfosBuilder(
+ String softDeletesFieldName, String parentFieldName) {
return new FieldInfosBuilder() {
private FieldInfos.Builder builder =
new FieldInfos.Builder(
- new FieldInfos.FieldNumbers(softDeletesFieldName, Version.LATEST.major));
+ new FieldInfos.FieldNumbers(
+ softDeletesFieldName, parentFieldName, Version.LATEST.major));
@Override
public FieldInfosBuilder add(FieldInfo fi) {
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java
index c5e9a8bd224..e20a6371ef2 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java
@@ -545,4 +545,20 @@ public final class IndexWriterConfig extends LiveIndexWriterConfig {
this.eventListener = eventListener;
return this;
}
+
+ /**
+ * Sets the parent document field. If this optional property is set, IndexWriter will add an
+ * internal field to every root document added to the index writer. A document is considered a
+ * parent document if it's the last document in a document block indexed via {@link
+ * IndexWriter#addDocuments(Iterable)} or {@link IndexWriter#updateDocuments(Term, Iterable)} and
+ * its relatives. Additionally, all individual documents added via the single document methods
+ * ({@link IndexWriter#addDocuments(Iterable)} etc.) are also considered parent documents. This
+ * property is optional for all indices that don't use document blocks in combination with index
+ * sorting. In order to maintain the API guarantee that the document order of a block is not
+ * altered by the {@link IndexWriter} a marker for parent documents is required.
+ */
+ public IndexWriterConfig setParentField(String parentField) {
+ this.parentField = parentField;
+ return this;
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
index 00ecb0ad4f4..e6c5ad433b0 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
import java.io.Closeable;
import java.io.IOException;
+import java.io.Reader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -27,6 +28,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
+import java.util.function.Function;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.codecs.DocValuesConsumer;
@@ -38,6 +40,7 @@ import org.apache.lucene.codecs.NormsProducer;
import org.apache.lucene.codecs.PointsFormat;
import org.apache.lucene.codecs.PointsWriter;
import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.InvertableType;
import org.apache.lucene.document.KnnByteVectorField;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.document.StoredValue;
@@ -49,6 +52,7 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BitSet;
import org.apache.lucene.util.ByteBlockPool;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefHash.MaxBytesLengthExceededException;
@@ -219,7 +223,23 @@ final class IndexingChain implements Accountable {
}
LeafReader docValuesReader = getDocValuesLeafReader();
-
+ Function<IndexSorter.DocComparator, IndexSorter.DocComparator> comparatorWrapper =
+ Function.identity();
+
+ if (state.segmentInfo.getHasBlocks() && state.fieldInfos.getParentField() != null) {
+ final DocIdSetIterator readerValues =
+ docValuesReader.getNumericDocValues(state.fieldInfos.getParentField());
+ if (readerValues == null) {
+ throw new CorruptIndexException(
+ "missing doc values for parent field \"" + state.fieldInfos.getParentField() + "\"",
+ "IndexingChain");
+ }
+ BitSet parents = BitSet.of(readerValues, state.segmentInfo.maxDoc());
+ comparatorWrapper =
+ in ->
+ (docID1, docID2) ->
+ in.compare(parents.nextSetBit(docID1), parents.nextSetBit(docID2));
+ }
List<IndexSorter.DocComparator> comparators = new ArrayList<>();
for (int i = 0; i < indexSort.getSort().length; i++) {
SortField sortField = indexSort.getSort()[i];
@@ -227,7 +247,10 @@ final class IndexingChain implements Accountable {
if (sorter == null) {
throw new UnsupportedOperationException("Cannot sort index using sort field " + sortField);
}
- comparators.add(sorter.getDocComparator(docValuesReader, state.segmentInfo.maxDoc()));
+
+ IndexSorter.DocComparator docComparator =
+ sorter.getDocComparator(docValuesReader, state.segmentInfo.maxDoc());
+ comparators.add(comparatorWrapper.apply(docComparator));
}
Sorter sorter = new Sorter(indexSort);
// returns null if the documents are already sorted
@@ -546,7 +569,17 @@ final class IndexingChain implements Accountable {
// build schema for each unique doc field
for (IndexableField field : document) {
IndexableFieldType fieldType = field.fieldType();
- PerField pf = getOrAddPerField(field.name());
+ final boolean isReserved = field.getClass() == ReservedField.class;
+ PerField pf =
+ getOrAddPerField(
+ field.name(), false
+ /* we never add reserved fields during indexing should be done during DWPT setup*/ );
+ if (pf.reserved != isReserved) {
+ throw new IllegalArgumentException(
+ "\""
+ + field.name()
+ + "\" is a reserved field and should not be added to any document");
+ }
if (pf.fieldGen != fieldGen) { // first time we see this field in this document
fields[fieldCount++] = pf;
pf.fieldGen = fieldGen;
@@ -556,7 +589,7 @@ final class IndexingChain implements Accountable {
docFields[docFieldIdx++] = pf;
updateDocFieldSchema(field.name(), pf.schema, fieldType);
}
- // For each field, if it the first time we see this field in this segment,
+ // For each field, if it's the first time we see this field in this segment,
// initialize its FieldInfo.
// If we have already seen this field, verify that its schema
// within the current doc matches its schema in the index.
@@ -646,7 +679,8 @@ final class IndexingChain implements Accountable {
s.vectorDimension,
s.vectorEncoding,
s.vectorSimilarityFunction,
- pf.fieldName.equals(fieldInfos.getSoftDeletesFieldName())));
+ pf.fieldName.equals(fieldInfos.getSoftDeletesFieldName()),
+ pf.fieldName.equals(fieldInfos.getParentFieldName())));
pf.setFieldInfo(fi);
if (fi.getIndexOptions() != IndexOptions.NONE) {
pf.setInvertState();
@@ -741,7 +775,7 @@ final class IndexingChain implements Accountable {
* Returns a previously created {@link PerField}, absorbing the type information from {@link
* FieldType}, and creates a new {@link PerField} if this field name wasn't seen yet.
*/
- private PerField getOrAddPerField(String fieldName) {
+ private PerField getOrAddPerField(String fieldName, boolean reserved) {
final int hashPos = fieldName.hashCode() & hashMask;
PerField pf = fieldHash[hashPos];
while (pf != null && pf.fieldName.equals(fieldName) == false) {
@@ -757,7 +791,8 @@ final class IndexingChain implements Accountable {
schema,
indexWriterConfig.getSimilarity(),
indexWriterConfig.getInfoStream(),
- indexWriterConfig.getAnalyzer());
+ indexWriterConfig.getAnalyzer(),
+ reserved);
pf.next = fieldHash[hashPos];
fieldHash[hashPos] = pf;
totalFieldCount++;
@@ -1028,6 +1063,7 @@ final class IndexingChain implements Accountable {
final String fieldName;
final int indexCreatedVersionMajor;
final FieldSchema schema;
+ final boolean reserved;
FieldInfo fieldInfo;
final Similarity similarity;
@@ -1065,13 +1101,15 @@ final class IndexingChain implements Accountable {
FieldSchema schema,
Similarity similarity,
InfoStream infoStream,
- Analyzer analyzer) {
+ Analyzer analyzer,
+ boolean reserved) {
this.fieldName = fieldName;
this.indexCreatedVersionMajor = indexCreatedVersionMajor;
this.schema = schema;
this.similarity = similarity;
this.infoStream = infoStream;
this.analyzer = analyzer;
+ this.reserved = reserved;
}
void reset(int docId) {
@@ -1518,4 +1556,77 @@ final class IndexingChain implements Accountable {
assertSame("point num bytes", fi.getPointNumBytes(), pointNumBytes);
}
}
+
+ /**
+ * Wraps the given field in a reserved field and registers it as reserved. Only DWPT should do
+ * this to mark fields as private / reserved to prevent this fieldname to be used from the outside
+ * of the IW / DWPT eco-system
+ */
+ <T extends IndexableField> ReservedField<T> markAsReserved(T field) {
+ getOrAddPerField(field.name(), true);
+ return new ReservedField<T>(field);
+ }
+
+ static final class ReservedField<T extends IndexableField> implements IndexableField {
+
+ private final T delegate;
+
+ private ReservedField(T delegate) {
+ this.delegate = delegate;
+ }
+
+ T getDelegate() {
+ return delegate;
+ }
+
+ @Override
+ public String name() {
+ return delegate.name();
+ }
+
+ @Override
+ public IndexableFieldType fieldType() {
+ return delegate.fieldType();
+ }
+
+ @Override
+ public TokenStream tokenStream(Analyzer analyzer, TokenStream reuse) {
+ return delegate.tokenStream(analyzer, reuse);
+ }
+
+ @Override
+ public BytesRef binaryValue() {
+ return delegate.binaryValue();
+ }
+
+ @Override
+ public String stringValue() {
+ return delegate.stringValue();
+ }
+
+ @Override
+ public CharSequence getCharSequenceValue() {
+ return delegate.getCharSequenceValue();
+ }
+
+ @Override
+ public Reader readerValue() {
+ return delegate.readerValue();
+ }
+
+ @Override
+ public Number numericValue() {
+ return delegate.numericValue();
+ }
+
+ @Override
+ public StoredValue storedValue() {
+ return delegate.storedValue();
+ }
+
+ @Override
+ public InvertableType invertableType() {
+ return delegate.invertableType();
+ }
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java b/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java
index 257c429818c..d9ac4f37d84 100644
--- a/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java
+++ b/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java
@@ -98,6 +98,9 @@ public class LiveIndexWriterConfig {
/** The field names involved in the index sort */
protected Set<String> indexSortFields = Collections.emptySet();
+ /** parent document field */
+ protected String parentField = null;
+
/**
* if an indexing thread should check for pending flushes on update in order to help out on a full
* flush
@@ -458,6 +461,11 @@ public class LiveIndexWriterConfig {
return eventListener;
}
+ /** Returns the parent document field name if configured. */
+ public String getParentField() {
+ return parentField;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
@@ -486,6 +494,7 @@ public class LiveIndexWriterConfig {
sb.append("maxFullFlushMergeWaitMillis=").append(getMaxFullFlushMergeWaitMillis()).append("\n");
sb.append("leafSorter=").append(getLeafSorter()).append("\n");
sb.append("eventListener=").append(getIndexWriterEventListener()).append("\n");
+ sb.append("parentField=").append(getParentField()).append("\n");
return sb.toString();
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java b/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
index 2b1fd2ee6b1..ec06fa51178 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
@@ -22,6 +22,7 @@ import java.util.List;
import org.apache.lucene.index.MergeState.DocMap;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
+import org.apache.lucene.util.BitSet;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.PriorityQueue;
import org.apache.lucene.util.packed.PackedInts;
@@ -50,6 +51,23 @@ final class MultiSorter {
"Cannot use sort field " + fields[i] + " for index sorting");
}
comparables[i] = sorter.getComparableProviders(readers);
+ for (int j = 0; j < readers.size(); j++) {
+ CodecReader codecReader = readers.get(j);
+ FieldInfos fieldInfos = codecReader.getFieldInfos();
+ LeafMetaData metaData = codecReader.getMetaData();
+ if (metaData.hasBlocks() && fieldInfos.getParentField() != null) {
+ NumericDocValues parentDocs =
+ codecReader.getNumericDocValues(fieldInfos.getParentField());
+ assert parentDocs != null
+ : "parent field: "
+ + fieldInfos.getParentField()
+ + " must be present if index sorting is used with blocks";
+ BitSet parents = BitSet.of(parentDocs, codecReader.maxDoc());
+ IndexSorter.ComparableProvider[] providers = comparables[i];
+ IndexSorter.ComparableProvider provider = providers[j];
+ providers[j] = docId -> provider.getAsComparableLong(parents.nextSetBit(docId));
+ }
+ }
reverseMuls[i] = fields[i].getReverse() ? -1 : 1;
}
int leafCount = readers.size();
diff --git a/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
index d938ce14a12..1648e627b19 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
@@ -111,6 +111,12 @@ public class ParallelLeafReader extends LeafReader {
.filter(Objects::nonNull)
.findAny()
.orElse(null);
+ final String parentField =
+ completeReaderSet.stream()
+ .map(r -> r.getFieldInfos().getParentField())
+ .filter(Objects::nonNull)
+ .findAny()
+ .orElse(null);
// TODO: make this read-only in a cleaner way?
final int indexCreatedVersionMajor =
completeReaderSet.stream()
@@ -121,8 +127,7 @@ public class ParallelLeafReader extends LeafReader {
.orElse(Version.LATEST.major);
FieldInfos.Builder builder =
new FieldInfos.Builder(
- new FieldInfos.FieldNumbers(softDeletesField, indexCreatedVersionMajor));
-
+ new FieldInfos.FieldNumbers(softDeletesField, parentField, indexCreatedVersionMajor));
Sort indexSort = null;
int createdVersionMajor = -1;
diff --git a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
index 04306c024cb..e5eb05b35bb 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
@@ -720,7 +720,8 @@ final class ReadersAndUpdates {
fi.getVectorDimension(),
fi.getVectorEncoding(),
fi.getVectorSimilarityFunction(),
- fi.isSoftDeletesField());
+ fi.isSoftDeletesField(),
+ fi.isParentField());
}
private SegmentReader createNewReaderWithLatestLiveDocs(SegmentReader reader) throws IOException {
diff --git a/lucene/core/src/java/org/apache/lucene/index/Sorter.java b/lucene/core/src/java/org/apache/lucene/index/Sorter.java
index 7945505be92..a9886db7a2c 100644
--- a/lucene/core/src/java/org/apache/lucene/index/Sorter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/Sorter.java
@@ -17,8 +17,10 @@
package org.apache.lucene.index;
import java.io.IOException;
+import java.util.function.Function;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
+import org.apache.lucene.util.BitSet;
import org.apache.lucene.util.TimSorter;
import org.apache.lucene.util.packed.PackedInts;
import org.apache.lucene.util.packed.PackedLongValues;
@@ -206,13 +208,25 @@ public final class Sorter {
SortField[] fields = sort.getSort();
final IndexSorter.DocComparator[] comparators = new IndexSorter.DocComparator[fields.length];
+ Function<IndexSorter.DocComparator, IndexSorter.DocComparator> comparatorWrapper = in -> in;
+ LeafMetaData metaData = reader.getMetaData();
+ FieldInfos fieldInfos = reader.getFieldInfos();
+ if (metaData.hasBlocks() && fieldInfos.getParentField() != null) {
+ BitSet parents =
+ BitSet.of(reader.getNumericDocValues(fieldInfos.getParentField()), reader.maxDoc());
+ comparatorWrapper =
+ in ->
+ (docID1, docID2) ->
+ in.compare(parents.nextSetBit(docID1), parents.nextSetBit(docID2));
+ }
+
for (int i = 0; i < fields.length; i++) {
IndexSorter sorter = fields[i].getIndexSorter();
if (sorter == null) {
throw new IllegalArgumentException(
"Cannot use sortfield + " + fields[i] + " to sort indexes");
}
- comparators[i] = sorter.getDocComparator(reader, reader.maxDoc());
+ comparators[i] = comparatorWrapper.apply(sorter.getDocComparator(reader, reader.maxDoc()));
}
return sort(reader.maxDoc(), comparators);
}
diff --git a/lucene/core/src/java/org/apache/lucene/internal/tests/IndexPackageAccess.java b/lucene/core/src/java/org/apache/lucene/internal/tests/IndexPackageAccess.java
index 545b5659933..5ca6d9d1c58 100644
--- a/lucene/core/src/java/org/apache/lucene/internal/tests/IndexPackageAccess.java
+++ b/lucene/core/src/java/org/apache/lucene/internal/tests/IndexPackageAccess.java
@@ -31,7 +31,7 @@ public interface IndexPackageAccess {
void setIndexWriterMaxDocs(int limit);
- FieldInfosBuilder newFieldInfosBuilder(String softDeletesFieldName);
+ FieldInfosBuilder newFieldInfosBuilder(String softDeletesFieldName, String parentFieldName);
void checkImpacts(Impacts impacts, int max);
diff --git a/lucene/core/src/java/org/apache/lucene/search/Sort.java b/lucene/core/src/java/org/apache/lucene/search/Sort.java
index 868fbf16360..394dcb1a23d 100644
--- a/lucene/core/src/java/org/apache/lucene/search/Sort.java
+++ b/lucene/core/src/java/org/apache/lucene/search/Sort.java
@@ -85,7 +85,6 @@ public final class Sort {
*/
public Sort rewrite(IndexSearcher searcher) throws IOException {
boolean changed = false;
-
SortField[] rewrittenSortFields = new SortField[fields.length];
for (int i = 0; i < fields.length; i++) {
rewrittenSortFields[i] = fields[i].rewrite(searcher);
@@ -100,7 +99,6 @@ public final class Sort {
@Override
public String toString() {
StringBuilder buffer = new StringBuilder();
-
for (int i = 0; i < fields.length; i++) {
buffer.append(fields[i].toString());
if ((i + 1) < fields.length) buffer.append(',');
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java b/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
index 6b89cbed0f2..8832f4cfde8 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
@@ -1937,4 +1937,97 @@ public class TestAddIndexes extends LuceneTestCase {
targetDir.close();
sourceDir.close();
}
+
+ public void testIllegalParentDocChange() throws Exception {
+ Directory dir1 = newDirectory();
+ IndexWriterConfig iwc1 = newIndexWriterConfig(new MockAnalyzer(random()));
+ iwc1.setParentField("foobar");
+ RandomIndexWriter w1 = new RandomIndexWriter(random(), dir1, iwc1);
+ Document parent = new Document();
+ w1.addDocuments(Arrays.asList(new Document(), new Document(), parent));
+ w1.commit();
+ w1.addDocuments(Arrays.asList(new Document(), new Document(), parent));
+ w1.commit();
+ // so the index sort is in fact burned into the index:
+ w1.forceMerge(1);
+ w1.close();
+
+ Directory dir2 = newDirectory();
+ IndexWriterConfig iwc2 = newIndexWriterConfig(new MockAnalyzer(random()));
+ iwc2.setParentField("foo");
+ RandomIndexWriter w2 = new RandomIndexWriter(random(), dir2, iwc2);
+
+ IndexReader r1 = DirectoryReader.open(dir1);
+ String message =
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> {
+ w2.addIndexes((SegmentReader) getOnlyLeafReader(r1));
+ })
+ .getMessage();
+ assertEquals(
+ "can't add field [foobar] as parent document field; this IndexWriter is configured with [foo] as parent document field",
+ message);
+
+ message =
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> {
+ w2.addIndexes(dir1);
+ })
+ .getMessage();
+ assertEquals(
+ "can't add field [foobar] as parent document field; this IndexWriter is configured with [foo] as parent document field",
+ message);
+
+ Directory dir3 = newDirectory();
+ IndexWriterConfig iwc3 = newIndexWriterConfig(new MockAnalyzer(random()));
+ iwc3.setParentField("foobar");
+ RandomIndexWriter w3 = new RandomIndexWriter(random(), dir3, iwc3);
+
+ w3.addIndexes((SegmentReader) getOnlyLeafReader(r1));
+ w3.addIndexes(dir1);
+
+ IOUtils.close(r1, dir1, w2, dir2, w3, dir3);
+ }
+
+ public void testIllegalNonParentField() throws IOException {
+ Directory dir1 = newDirectory();
+ IndexWriterConfig iwc1 = newIndexWriterConfig(new MockAnalyzer(random()));
+ RandomIndexWriter w1 = new RandomIndexWriter(random(), dir1, iwc1);
+ Document parent = new Document();
+ parent.add(new StringField("foo", "XXX", Field.Store.NO));
+ w1.addDocument(parent);
+ w1.close();
+
+ Directory dir2 = newDirectory();
+ IndexWriterConfig iwc2 = newIndexWriterConfig(new MockAnalyzer(random()));
+ iwc2.setParentField("foo");
+ RandomIndexWriter w2 = new RandomIndexWriter(random(), dir2, iwc2);
+
+ IndexReader r1 = DirectoryReader.open(dir1);
+ String message =
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> {
+ w2.addIndexes((SegmentReader) getOnlyLeafReader(r1));
+ })
+ .getMessage();
+ assertEquals(
+ "can't add [foo] as non parent document field; this IndexWriter is configured with [foo] as parent document field",
+ message);
+
+ message =
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> {
+ w2.addIndexes(dir1);
+ })
+ .getMessage();
+ assertEquals(
+ "can't add [foo] as non parent document field; this IndexWriter is configured with [foo] as parent document field",
+ message);
+
+ IOUtils.close(r1, dir1, w2, dir2);
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java b/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
index 352cf9bbfc0..e90bea123d6 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
@@ -114,6 +114,7 @@ public class TestCodecs extends LuceneTestCase {
0,
VectorEncoding.FLOAT32,
VectorSimilarityFunction.EUCLIDEAN,
+ false,
false));
}
this.terms = terms;
@@ -230,8 +231,7 @@ public class TestCodecs extends LuceneTestCase {
}
final FieldInfos.Builder builder =
- new FieldInfos.Builder(new FieldInfos.FieldNumbers(null, Version.LATEST.major));
-
+ new FieldInfos.Builder(new FieldInfos.FieldNumbers(null, null, Version.LATEST.major));
final FieldData field = new FieldData("field", builder, terms, true, false);
final FieldData[] fields = new FieldData[] {field};
final FieldInfos fieldInfos = builder.finish();
@@ -294,8 +294,7 @@ public class TestCodecs extends LuceneTestCase {
public void testRandomPostings() throws Throwable {
final FieldInfos.Builder builder =
- new FieldInfos.Builder(new FieldInfos.FieldNumbers(null, Version.LATEST.major));
-
+ new FieldInfos.Builder(new FieldInfos.FieldNumbers(null, null, Version.LATEST.major));
final FieldData[] fields = new FieldData[NUM_FIELDS];
for (int i = 0; i < NUM_FIELDS; i++) {
final boolean omitTF = 0 == (i % 3);
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDoc.java b/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
index f93b1d6604c..4445a649772 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
@@ -236,7 +236,7 @@ public class TestDoc extends LuceneTestCase {
si,
InfoStream.getDefault(),
trackingDir,
- new FieldInfos.FieldNumbers(null, Version.LATEST.major),
+ new FieldInfos.FieldNumbers(null, null, Version.LATEST.major),
context);
merger.merge();
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFieldInfo.java b/lucene/core/src/test/org/apache/lucene/index/TestFieldInfo.java
index 42f2c55fbf8..759f18a4b39 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestFieldInfo.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestFieldInfo.java
@@ -224,6 +224,8 @@ public class TestFieldInfo extends LuceneTestCase {
VectorSimilarityFunction vectorSimilarityFunction = VectorSimilarityFunction.EUCLIDEAN;
boolean softDeletesField = false;
+ boolean isParentField = false;
+
FieldInfo get() {
return new FieldInfo(
name,
@@ -241,7 +243,8 @@ public class TestFieldInfo extends LuceneTestCase {
vectorDimension,
vectorEncoding,
vectorSimilarityFunction,
- softDeletesField);
+ softDeletesField,
+ isParentField);
}
public FieldInfoTestBuilder setStoreTermVector(boolean storeTermVector) {
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFieldInfos.java b/lucene/core/src/test/org/apache/lucene/index/TestFieldInfos.java
index a4272cf5633..60bc9bae878 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestFieldInfos.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestFieldInfos.java
@@ -249,7 +249,7 @@ public class TestFieldInfos extends LuceneTestCase {
public void testFieldNumbersAutoIncrement() {
FieldInfos.FieldNumbers fieldNumbers =
- new FieldInfos.FieldNumbers("softDeletes", Version.LATEST.major);
+ new FieldInfos.FieldNumbers("softDeletes", "parentDoc", Version.LATEST.major);
for (int i = 0; i < 10; i++) {
fieldNumbers.addOrGet(
new FieldInfo(
@@ -268,6 +268,7 @@ public class TestFieldInfos extends LuceneTestCase {
0,
VectorEncoding.FLOAT32,
VectorSimilarityFunction.EUCLIDEAN,
+ false,
false));
}
int idx =
@@ -288,6 +289,7 @@ public class TestFieldInfos extends LuceneTestCase {
0,
VectorEncoding.FLOAT32,
VectorSimilarityFunction.EUCLIDEAN,
+ false,
false));
assertEquals("Field numbers 0 through 9 were allocated", 10, idx);
@@ -310,6 +312,7 @@ public class TestFieldInfos extends LuceneTestCase {
0,
VectorEncoding.FLOAT32,
VectorSimilarityFunction.EUCLIDEAN,
+ false,
false));
assertEquals("Field numbers should reset after clear()", 0, idx);
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java b/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java
index 49bbde5dcb6..211c6052a1d 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java
@@ -46,10 +46,8 @@ public class TestFieldsReader extends LuceneTestCase {
@BeforeClass
public static void beforeClass() throws Exception {
testDoc = new Document();
- final String softDeletesFieldName = null;
fieldInfos =
- new FieldInfos.Builder(
- new FieldInfos.FieldNumbers(softDeletesFieldName, Version.LATEST.major));
+ new FieldInfos.Builder(new FieldInfos.FieldNumbers(null, null, Version.LATEST.major));
DocHelper.setupDoc(testDoc);
for (IndexableField field : testDoc.getFields()) {
IndexableFieldType ift = field.fieldType();
@@ -70,7 +68,8 @@ public class TestFieldsReader extends LuceneTestCase {
0,
VectorEncoding.FLOAT32,
VectorSimilarityFunction.EUCLIDEAN,
- field.name().equals(softDeletesFieldName)));
+ false,
+ false));
}
dir = newDirectory();
IndexWriterConfig conf =
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
index e54e7687510..75958e203b1 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
@@ -2121,6 +2121,10 @@ public class TestIndexSorting extends LuceneTestCase {
public void testAddIndexes(boolean withDeletes, boolean useReaders) throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc1 = newIndexWriterConfig();
+ boolean useParent = rarely();
+ if (useParent) {
+ iwc1.setParentField("___parent");
+ }
Sort indexSort =
new Sort(
new SortField("foo", SortField.Type.LONG), new SortField("bar", SortField.Type.LONG));
@@ -2153,6 +2157,9 @@ public class TestIndexSorting extends LuceneTestCase {
} else {
iwc.setIndexSort(indexSort);
}
+ if (useParent) {
+ iwc.setParentField("___parent");
+ }
IndexWriter w2 = new IndexWriter(dir2, iwc);
if (useReaders) {
@@ -3171,4 +3178,258 @@ public class TestIndexSorting extends LuceneTestCase {
reader.close();
dir.close();
}
+
+ public void testBlockContainsParentField() throws IOException {
+ try (Directory dir = newDirectory()) {
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ String parentField = "parent";
+ iwc.setParentField(parentField);
+ Sort indexSort = new Sort(new SortField("foo", SortField.Type.INT));
+ iwc.setIndexSort(indexSort);
+ try (IndexWriter writer = new IndexWriter(dir, iwc)) {
+ List<Runnable> runnabels =
+ Arrays.asList(
+ () -> {
+ IllegalArgumentException ex =
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> {
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("parent", 0));
+ writer.addDocuments(Arrays.asList(doc, new Document()));
+ });
+ assertEquals(
+ "\"parent\" is a reserved field and should not be added to any document",
+ ex.getMessage());
+ },
+ () -> {
+ IllegalArgumentException ex =
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> {
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("parent", 0));
+ writer.addDocuments(Arrays.asList(new Document(), doc));
+ });
+ assertEquals(
+ "\"parent\" is a reserved field and should not be added to any document",
+ ex.getMessage());
+ });
+ Collections.shuffle(runnabels, random());
+ for (Runnable runnable : runnabels) {
+ runnable.run();
+ }
+ }
+ }
+ }
+
+ public void testIndexSortWithBlocks() throws IOException {
+ try (Directory dir = newDirectory()) {
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ AssertingNeedsIndexSortCodec codec = new AssertingNeedsIndexSortCodec();
+ iwc.setCodec(codec);
+ String parentField = "parent";
+ Sort indexSort = new Sort(new SortField("foo", SortField.Type.INT));
+ iwc.setIndexSort(indexSort);
+ iwc.setParentField(parentField);
+ LogMergePolicy policy = newLogMergePolicy();
+ // make sure that merge factor is always > 2
+ if (policy.getMergeFactor() <= 2) {
+ policy.setMergeFactor(3);
+ }
+ iwc.setMergePolicy(policy);
+
+ // add already sorted documents
+ codec.numCalls = 0;
+ codec.needsIndexSort = false;
+ try (IndexWriter w = new IndexWriter(dir, iwc)) {
+ int numDocs = 50 + random().nextInt(50);
+ for (int i = 0; i < numDocs; i++) {
+ Document child1 = new Document();
+ child1.add(new StringField("id", Integer.toString(i), Store.YES));
+ child1.add(new NumericDocValuesField("id", i));
+ child1.add(new NumericDocValuesField("child", 1));
+ child1.add(new NumericDocValuesField("foo", random().nextInt()));
+ Document child2 = new Document();
+ child2.add(new StringField("id", Integer.toString(i), Store.YES));
+ child2.add(new NumericDocValuesField("id", i));
+ child2.add(new NumericDocValuesField("child", 2));
+ child2.add(new NumericDocValuesField("foo", random().nextInt()));
+ Document parent = new Document();
+ parent.add(new StringField("id", Integer.toString(i), Store.YES));
+ parent.add(new NumericDocValuesField("id", i));
+ parent.add(new NumericDocValuesField("foo", random().nextInt()));
+ w.addDocuments(Arrays.asList(child1, child2, parent));
+ if (rarely()) {
+ w.commit();
+ }
+ }
+ w.commit();
+ if (random().nextBoolean()) {
+ w.forceMerge(1, true);
+ }
+ }
+
+ try (DirectoryReader reader = DirectoryReader.open(dir)) {
+ for (LeafReaderContext ctx : reader.leaves()) {
+ LeafReader leaf = ctx.reader();
+ NumericDocValues parentDISI = leaf.getNumericDocValues(parentField);
+ NumericDocValues ids = leaf.getNumericDocValues("id");
+ NumericDocValues children = leaf.getNumericDocValues("child");
+ int doc;
+ int expectedDocID = 2;
+ while ((doc = parentDISI.nextDoc()) != NO_MORE_DOCS) {
+ assertEquals(-1, parentDISI.longValue());
+ assertEquals(expectedDocID, doc);
+ int id = ids.nextDoc();
+ long child1ID = ids.longValue();
+ assertEquals(id, children.nextDoc());
+ long child1 = children.longValue();
+ assertEquals(1, child1);
+
+ id = ids.nextDoc();
+ long child2ID = ids.longValue();
+ assertEquals(id, children.nextDoc());
+ long child2 = children.longValue();
+ assertEquals(2, child2);
+
+ int idParent = ids.nextDoc();
+ assertEquals(id + 1, idParent);
+ long parent = ids.longValue();
+ assertEquals(child1ID, parent);
+ assertEquals(child2ID, parent);
+ expectedDocID += 3;
+ }
+ }
+ }
+ }
+ }
+
+ @SuppressWarnings("fallthrough")
+ public void testMixRandomDocumentsWithBlocks() throws IOException {
+ try (Directory dir = newDirectory()) {
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ AssertingNeedsIndexSortCodec codec = new AssertingNeedsIndexSortCodec();
+ iwc.setCodec(codec);
+ String parentField = "parent";
+ Sort indexSort = new Sort(new SortField("foo", SortField.Type.INT));
+ iwc.setIndexSort(indexSort);
+ iwc.setParentField(parentField);
+ RandomIndexWriter randomIndexWriter = new RandomIndexWriter(random(), dir, iwc);
+ int numDocs = 100 + random().nextInt(900);
+ for (int i = 0; i < numDocs; i++) {
+ if (rarely()) {
+ randomIndexWriter.deleteDocuments(new Term("id", "" + random().nextInt(i + 1)));
+ }
+ List<Document> docs = new ArrayList<>();
+ switch (random().nextInt(100) % 5) {
+ case 4:
+ Document child3 = new Document();
+ child3.add(new StringField("id", Integer.toString(i), Store.YES));
+ child3.add(new NumericDocValuesField("type", 2));
+ child3.add(new NumericDocValuesField("child_ord", 3));
+ child3.add(new NumericDocValuesField("foo", random().nextInt()));
+ docs.add(child3);
+ case 3:
+ Document child2 = new Document();
+ child2.add(new StringField("id", Integer.toString(i), Store.YES));
+ child2.add(new NumericDocValuesField("type", 2));
+ child2.add(new NumericDocValuesField("child_ord", 2));
+ child2.add(new NumericDocValuesField("foo", random().nextInt()));
+ docs.add(child2);
+ case 2:
+ Document child1 = new Document();
+ child1.add(new StringField("id", Integer.toString(i), Store.YES));
+ child1.add(new NumericDocValuesField("type", 2));
+ child1.add(new NumericDocValuesField("child_ord", 1));
+ child1.add(new NumericDocValuesField("foo", random().nextInt()));
+ docs.add(child1);
+ case 1:
+ Document root = new Document();
+ root.add(new StringField("id", Integer.toString(i), Store.YES));
+ root.add(new NumericDocValuesField("type", 1));
+ root.add(new NumericDocValuesField("num_children", docs.size()));
+ root.add(new NumericDocValuesField("foo", random().nextInt()));
+ docs.add(root);
+ randomIndexWriter.addDocuments(docs);
+ break;
+ case 0:
+ Document single = new Document();
+ single.add(new StringField("id", Integer.toString(i), Store.YES));
+ single.add(new NumericDocValuesField("type", 0));
+ single.add(new NumericDocValuesField("foo", random().nextInt()));
+ randomIndexWriter.addDocument(single);
+ }
+ if (rarely()) {
+ randomIndexWriter.forceMerge(1);
+ }
+ randomIndexWriter.commit();
+ }
+
+ randomIndexWriter.close();
+ try (DirectoryReader reader = DirectoryReader.open(dir)) {
+ for (LeafReaderContext ctx : reader.leaves()) {
+ LeafReader leaf = ctx.reader();
+ NumericDocValues parentDISI = leaf.getNumericDocValues(parentField);
+ assertNotNull(parentDISI);
+ NumericDocValues type = leaf.getNumericDocValues("type");
+ NumericDocValues childOrd = leaf.getNumericDocValues("child_ord");
+ NumericDocValues numChildren = leaf.getNumericDocValues("num_children");
+ int numCurrentChildren = 0;
+ int totalPendingChildren = 0;
+ String childId = null;
+ for (int i = 0; i < leaf.maxDoc(); i++) {
+ if (leaf.getLiveDocs() == null || leaf.getLiveDocs().get(i)) {
+ assertTrue(type.advanceExact(i));
+ int typeValue = (int) type.longValue();
+ switch (typeValue) {
+ case 2:
+ assertFalse(parentDISI.advanceExact(i));
+ assertTrue(childOrd.advanceExact(i));
+ if (numCurrentChildren == 0) { // first child
+ childId = leaf.storedFields().document(i).get("id");
+ totalPendingChildren = (int) childOrd.longValue() - 1;
+ } else {
+ assertNotNull(childId);
+ assertEquals(totalPendingChildren--, childOrd.longValue());
+ assertEquals(childId, leaf.storedFields().document(i).get("id"));
+ }
+ numCurrentChildren++;
+ break;
+ case 1:
+ assertTrue(parentDISI.advanceExact(i));
+ assertEquals(-1, parentDISI.longValue());
+ if (childOrd != null) {
+ assertFalse(childOrd.advanceExact(i));
+ }
+ assertTrue(numChildren.advanceExact(i));
+ assertEquals(0, totalPendingChildren);
+ assertEquals(numCurrentChildren, numChildren.longValue());
+ if (numCurrentChildren > 0) {
+ assertEquals(childId, leaf.storedFields().document(i).get("id"));
+ } else {
+ assertNull(childId);
+ }
+ numCurrentChildren = 0;
+ childId = null;
+ break;
+ case 0:
+ assertEquals(-1, parentDISI.longValue());
+ assertTrue(parentDISI.advanceExact(i));
+ if (childOrd != null) {
+ assertFalse(childOrd.advanceExact(i));
+ }
+ if (numChildren != null) {
+ assertFalse(numChildren.advanceExact(i));
+ }
+ break;
+ default:
+ fail();
+ }
+ }
+ }
+ }
+ }
+ }
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
index fc2805d726f..f4ef189112a 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
@@ -1765,6 +1765,44 @@ public class TestIndexWriter extends LuceneTestCase {
}
}
+ public void testSingleDocsDoNotTriggerHasBlocks() throws IOException {
+ try (Directory dir = newDirectory()) {
+ try (IndexWriter w =
+ new IndexWriter(
+ dir,
+ new IndexWriterConfig(new MockAnalyzer(random()))
+ .setMaxBufferedDocs(Integer.MAX_VALUE)
+ .setRAMBufferSizeMB(100))) {
+
+ int docs = 1 + random().nextInt(100);
+ for (int i = 0; i < docs; i++) {
+ Document doc = new Document();
+ doc.add(new StringField("id", "" + i, Field.Store.NO));
+ w.addDocuments(Arrays.asList(doc));
+ }
+ w.commit();
+ SegmentInfos si = w.cloneSegmentInfos();
+ assertEquals(1, si.size());
+ assertFalse(si.asList().get(0).info.getHasBlocks());
+
+ Document doc = new Document();
+ doc.add(new StringField("id", "XXX", Field.Store.NO));
+ w.addDocuments(Arrays.asList(doc, doc));
+ w.commit();
+ si = w.cloneSegmentInfos();
+ assertEquals(2, si.size());
+ assertFalse(si.asList().get(0).info.getHasBlocks());
+ assertTrue(si.asList().get(1).info.getHasBlocks());
+ w.forceMerge(1);
+
+ w.commit();
+ si = w.cloneSegmentInfos();
+ assertEquals(1, si.size());
+ assertTrue(si.asList().get(0).info.getHasBlocks());
+ }
+ }
+ }
+
public void testCarryOverHasBlocks() throws Exception {
try (Directory dir = newDirectory()) {
try (IndexWriter w =
@@ -4890,4 +4928,89 @@ public class TestIndexWriter extends LuceneTestCase {
doc.add(newField(field, "value", storedTextType));
writer.addDocument(doc);
}
+
+ public void testParentAndSoftDeletesAreTheSame() throws IOException {
+ try (Directory dir = newDirectory()) {
+ IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new MockAnalyzer(random()));
+ indexWriterConfig.setSoftDeletesField("foo");
+ indexWriterConfig.setParentField("foo");
+ IllegalArgumentException iae =
+ expectThrows(
+ IllegalArgumentException.class, () -> new IndexWriter(dir, indexWriterConfig));
+ assertEquals(
+ "parent document and soft-deletes field can't be the same field \"foo\"",
+ iae.getMessage());
+ }
+ }
+
+ public void testIndexWithParentFieldIsCongruent() throws IOException {
+ try (Directory dir = newDirectory()) {
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ iwc.setParentField("parent");
+ try (IndexWriter writer = new IndexWriter(dir, iwc)) {
+ if (random().nextBoolean()) {
+ Document child1 = new Document();
+ child1.add(new StringField("id", Integer.toString(1), Field.Store.YES));
+ Document child2 = new Document();
+ child2.add(new StringField("id", Integer.toString(1), Field.Store.YES));
+ Document parent = new Document();
+ parent.add(new StringField("id", Integer.toString(1), Field.Store.YES));
+ writer.addDocuments(Arrays.asList(child1, child2, parent));
+ writer.flush();
+ if (random().nextBoolean()) {
+ writer.addDocuments(Arrays.asList(child1, child2, parent));
+ }
+ } else {
+ writer.addDocument(new Document());
+ }
+ writer.commit();
+ }
+ IllegalArgumentException ex =
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> {
+ IndexWriterConfig config = new IndexWriterConfig(new MockAnalyzer(random()));
+ config.setParentField("someOtherField");
+ new IndexWriter(dir, config);
+ });
+ assertEquals(
+ "can't add field [parent] as parent document field; this IndexWriter is configured with [someOtherField] as parent document field",
+ ex.getMessage());
+ ex =
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> {
+ IndexWriterConfig config = new IndexWriterConfig(new MockAnalyzer(random()));
+ new IndexWriter(dir, config);
+ });
+ assertEquals(
+ "can't add field [parent] as parent document field; this IndexWriter has no parent document field configured",
+ ex.getMessage());
+ }
+ }
+
+ public void testParentFieldIsAlreadyUsed() throws IOException {
+ try (Directory dir = newDirectory()) {
+
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ try (IndexWriter writer = new IndexWriter(dir, iwc)) {
+ Document doc = new Document();
+ doc.add(new StringField("parent", Integer.toString(1), Field.Store.YES));
+ writer.addDocument(doc);
+ writer.commit();
+ }
+ IllegalArgumentException iae =
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> {
+ IndexWriterConfig config = new IndexWriterConfig(new MockAnalyzer(random()));
+ config.setParentField("parent");
+
+ new IndexWriter(dir, config);
+ });
+ assertEquals(
+ "can't add [parent] as non parent document field; this IndexWriter is configured with [parent] as parent document field",
+ iae.getMessage());
+ }
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java b/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
index 02600f5d59d..3c6d57c5d2f 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
@@ -199,7 +199,8 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
0,
VectorEncoding.FLOAT32,
VectorSimilarityFunction.EUCLIDEAN,
- true);
+ true,
+ false);
List<Integer> docsDeleted = Arrays.asList(1, 3, 7, 8, DocIdSetIterator.NO_MORE_DOCS);
List<DocValuesFieldUpdates> updates = Arrays.asList(singleUpdate(docsDeleted, 10, true));
for (DocValuesFieldUpdates update : updates) {
@@ -237,7 +238,8 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
0,
VectorEncoding.FLOAT32,
VectorSimilarityFunction.EUCLIDEAN,
- true);
+ true,
+ false);
for (DocValuesFieldUpdates update : updates) {
deletes.onDocValuesUpdate(fieldInfo, update.iterator());
}
@@ -301,7 +303,8 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
0,
VectorEncoding.FLOAT32,
VectorSimilarityFunction.EUCLIDEAN,
- true);
+ true,
+ false);
List<Integer> docsDeleted = Arrays.asList(1, DocIdSetIterator.NO_MORE_DOCS);
List<DocValuesFieldUpdates> updates = Arrays.asList(singleUpdate(docsDeleted, 3, true));
for (DocValuesFieldUpdates update : updates) {
@@ -370,7 +373,8 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
0,
VectorEncoding.FLOAT32,
VectorSimilarityFunction.EUCLIDEAN,
- true);
+ true,
+ false);
List<DocValuesFieldUpdates> updates =
Arrays.asList(singleUpdate(Arrays.asList(0, 1, DocIdSetIterator.NO_MORE_DOCS), 3, false));
for (DocValuesFieldUpdates update : updates) {
@@ -407,7 +411,8 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
0,
VectorEncoding.FLOAT32,
VectorSimilarityFunction.EUCLIDEAN,
- true);
+ true,
+ false);
updates = Arrays.asList(singleUpdate(Arrays.asList(1, DocIdSetIterator.NO_MORE_DOCS), 3, true));
for (DocValuesFieldUpdates update : updates) {
deletes.onDocValuesUpdate(fieldInfo, update.iterator());
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java b/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
index be25166815f..3f09061c6e9 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
@@ -104,7 +104,7 @@ public class TestSegmentMerger extends LuceneTestCase {
si,
InfoStream.getDefault(),
mergedDir,
- new FieldInfos.FieldNumbers(null, Version.LATEST.major),
+ new FieldInfos.FieldNumbers(null, null, Version.LATEST.major),
newIOContext(random(), new IOContext(new MergeInfo(-1, -1, false, -1))));
MergeState mergeState = merger.merge();
int docsMerged = mergeState.segmentInfo.maxDoc();
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java b/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java
index 765c355e534..401bcb3e5c6 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java
@@ -1297,7 +1297,8 @@ public class TestSortOptimization extends LuceneTestCase {
0,
VectorEncoding.FLOAT32,
VectorSimilarityFunction.DOT_PRODUCT,
- fi.isSoftDeletesField());
+ fi.isSoftDeletesField(),
+ fi.isParentField());
newInfos[i] = noIndexFI;
i++;
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java b/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java
index 2d192e0e910..fd75bb2882b 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java
@@ -182,9 +182,13 @@ public class TestTopFieldCollector extends LuceneTestCase {
dir, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE).setIndexSort(sort));
Document doc = new Document();
doc.add(new NumericDocValuesField("foo", 3));
- w.addDocuments(Arrays.asList(doc, doc, doc, doc));
+ for (Document d : Arrays.asList(doc, doc, doc, doc)) {
+ w.addDocument(d);
+ }
w.flush();
- w.addDocuments(Arrays.asList(doc, doc, doc, doc, doc, doc));
+ for (Document d : Arrays.asList(doc, doc, doc, doc, doc, doc)) {
+ w.addDocument(d);
+ }
w.flush();
IndexReader reader = DirectoryReader.open(w);
assertEquals(2, reader.leaves().size());
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java
index 5a39d49d747..ec9dfa5f612 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java
@@ -103,6 +103,7 @@ public class TermVectorLeafReader extends LeafReader {
0,
VectorEncoding.FLOAT32,
VectorSimilarityFunction.EUCLIDEAN,
+ false,
false);
fieldInfos = new FieldInfos(new FieldInfo[] {fieldInfo});
}
diff --git a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
index 86e1e457eae..172f494c40a 100644
--- a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
+++ b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
@@ -736,6 +736,7 @@ public class MemoryIndex {
fieldType.vectorDimension(),
fieldType.vectorEncoding(),
fieldType.vectorSimilarityFunction(),
+ false,
false);
}
@@ -789,7 +790,8 @@ public class MemoryIndex {
info.fieldInfo.getVectorDimension(),
info.fieldInfo.getVectorEncoding(),
info.fieldInfo.getVectorSimilarityFunction(),
- info.fieldInfo.isSoftDeletesField());
+ info.fieldInfo.isSoftDeletesField(),
+ info.fieldInfo.isParentField());
} else if (existingDocValuesType != docValuesType) {
throw new IllegalArgumentException(
"Can't add ["
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/compressing/dummy/DummyCompressingCodec.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/compressing/dummy/DummyCompressingCodec.java
index 9d7cd7e7742..ef9f58e1bb3 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/compressing/dummy/DummyCompressingCodec.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/compressing/dummy/DummyCompressingCodec.java
@@ -78,7 +78,7 @@ public class DummyCompressingCodec extends CompressingCodec {
@Override
public void compress(ByteBuffersDataInput buffersInput, DataOutput out) throws IOException {
- out.copyBytes(buffersInput, buffersInput.size());
+ out.copyBytes(buffersInput, buffersInput.length());
}
@Override
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseFieldInfoFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseFieldInfoFormatTestCase.java
index 5c9d2dac408..1087eb15c5d 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseFieldInfoFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseFieldInfoFormatTestCase.java
@@ -68,7 +68,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes
FieldInfo fi = createFieldInfo();
addAttributes(fi);
- FieldInfos infos = INDEX_PACKAGE_ACCESS.newFieldInfosBuilder(null).add(fi).finish();
+ FieldInfos infos = INDEX_PACKAGE_ACCESS.newFieldInfosBuilder(null, null).add(fi).finish();
codec.fieldInfosFormat().write(dir, segmentInfo, "", infos, IOContext.DEFAULT);
@@ -96,7 +96,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes
fi.putAttribute("foo", "bar");
fi.putAttribute("bar", "baz");
- FieldInfos infos = INDEX_PACKAGE_ACCESS.newFieldInfosBuilder(null).add(fi).finish();
+ FieldInfos infos = INDEX_PACKAGE_ACCESS.newFieldInfosBuilder(null, null).add(fi).finish();
codec.fieldInfosFormat().write(dir, segmentInfo, "", infos, IOContext.DEFAULT);
@@ -136,7 +136,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes
FieldInfo fi = createFieldInfo();
addAttributes(fi);
- FieldInfos infos = INDEX_PACKAGE_ACCESS.newFieldInfosBuilder(null).add(fi).finish();
+ FieldInfos infos = INDEX_PACKAGE_ACCESS.newFieldInfosBuilder(null, null).add(fi).finish();
fail.setDoFail();
expectThrows(
@@ -171,7 +171,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes
FieldInfo fi = createFieldInfo();
addAttributes(fi);
- FieldInfos infos = INDEX_PACKAGE_ACCESS.newFieldInfosBuilder(null).add(fi).finish();
+ FieldInfos infos = INDEX_PACKAGE_ACCESS.newFieldInfosBuilder(null, null).add(fi).finish();
fail.setDoFail();
expectThrows(
@@ -206,7 +206,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes
FieldInfo fi = createFieldInfo();
addAttributes(fi);
- FieldInfos infos = INDEX_PACKAGE_ACCESS.newFieldInfosBuilder(null).add(fi).finish();
+ FieldInfos infos = INDEX_PACKAGE_ACCESS.newFieldInfosBuilder(null, null).add(fi).finish();
codec.fieldInfosFormat().write(dir, segmentInfo, "", infos, IOContext.DEFAULT);
@@ -243,7 +243,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes
FieldInfo fi = createFieldInfo();
addAttributes(fi);
- FieldInfos infos = INDEX_PACKAGE_ACCESS.newFieldInfosBuilder(null).add(fi).finish();
+ FieldInfos infos = INDEX_PACKAGE_ACCESS.newFieldInfosBuilder(null, null).add(fi).finish();
codec.fieldInfosFormat().write(dir, segmentInfo, "", infos, IOContext.DEFAULT);
@@ -276,7 +276,9 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes
String softDeletesField =
random().nextBoolean() ? TestUtil.randomUnicodeString(random()) : null;
- var builder = INDEX_PACKAGE_ACCESS.newFieldInfosBuilder(softDeletesField);
+ String parentField = random().nextBoolean() ? TestUtil.randomUnicodeString(random()) : null;
+
+ var builder = INDEX_PACKAGE_ACCESS.newFieldInfosBuilder(softDeletesField, parentField);
for (String field : fieldNames) {
IndexableFieldType fieldType = randomFieldType(random(), field);
@@ -307,7 +309,8 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes
fieldType.vectorDimension(),
fieldType.vectorEncoding(),
fieldType.vectorSimilarityFunction(),
- field.equals(softDeletesField));
+ field.equals(softDeletesField),
+ field.equals(parentField));
addAttributes(fi);
builder.add(fi);
}
@@ -431,6 +434,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes
0,
VectorEncoding.FLOAT32,
VectorSimilarityFunction.EUCLIDEAN,
+ false,
false);
}
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseIndexFileFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseIndexFileFormatTestCase.java
index 19b8ba60ea7..c639d6a2e73 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseIndexFileFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseIndexFileFormatTestCase.java
@@ -365,7 +365,8 @@ abstract class BaseIndexFileFormatTestCase extends LuceneTestCase {
proto.getVectorDimension(),
proto.getVectorEncoding(),
proto.getVectorSimilarityFunction(),
- proto.isSoftDeletesField());
+ proto.isSoftDeletesField(),
+ proto.isParentField());
FieldInfos fieldInfos = new FieldInfos(new FieldInfo[] {field});
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseSegmentInfoFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseSegmentInfoFormatTestCase.java
index 0ebb3fcffdc..1d65e3245dc 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseSegmentInfoFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseSegmentInfoFormatTestCase.java
@@ -80,6 +80,33 @@ public abstract class BaseSegmentInfoFormatTestCase extends BaseIndexFileFormatT
dir.close();
}
+ public void testHasBlocks() throws IOException {
+ assumeTrue("test requires a codec that can read/write hasBlocks", supportsHasBlocks());
+
+ Directory dir = newDirectory();
+ Codec codec = getCodec();
+ byte[] id = StringHelper.randomId();
+ SegmentInfo info =
+ new SegmentInfo(
+ dir,
+ getVersions()[0],
+ getVersions()[0],
+ "_123",
+ 1,
+ false,
+ random().nextBoolean(),
+ codec,
+ Collections.emptyMap(),
+ id,
+ Collections.emptyMap(),
+ null);
+ info.setFiles(Collections.<String>emptySet());
+ codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
+ SegmentInfo info2 = codec.segmentInfoFormat().read(dir, "_123", id, IOContext.DEFAULT);
+ assertEquals(info.getHasBlocks(), info2.getHasBlocks());
+ dir.close();
+ }
+
/** Tests SI writer adds itself to files... */
public void testAddsSelfToFiles() throws Exception {
Directory dir = newDirectory();
@@ -260,6 +287,10 @@ public abstract class BaseSegmentInfoFormatTestCase extends BaseIndexFileFormatT
return true;
}
+ protected boolean supportsHasBlocks() {
+ return true;
+ }
+
private SortField randomIndexSortField() {
boolean reversed = random().nextBoolean();
SortField sortField;
@@ -360,7 +391,11 @@ public abstract class BaseSegmentInfoFormatTestCase extends BaseIndexFileFormatT
for (int j = 0; j < numSortFields; ++j) {
sortFields[j] = randomIndexSortField();
}
- sort = new Sort(sortFields);
+ if (supportsHasBlocks()) {
+ sort = new Sort(sortFields);
+ } else {
+ sort = new Sort(sortFields);
+ }
}
Directory dir = newDirectory();
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/MismatchedLeafReader.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/MismatchedLeafReader.java
index 538fa7c8c15..fe0622c7da6 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/MismatchedLeafReader.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/MismatchedLeafReader.java
@@ -117,7 +117,8 @@ public class MismatchedLeafReader extends FilterLeafReader {
oldInfo.getVectorEncoding(), // numeric type of vector samples
// distance function for calculating similarity of the field's vector
oldInfo.getVectorSimilarityFunction(),
- oldInfo.isSoftDeletesField()); // used as soft-deletes field
+ oldInfo.isSoftDeletesField(), // used as soft-deletes field
+ oldInfo.isParentField());
shuffled.set(i, newInfo);
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomIndexWriter.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomIndexWriter.java
index 2988957ba29..f6443c5c498 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomIndexWriter.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomIndexWriter.java
@@ -158,6 +158,7 @@ public class RandomIndexWriter implements Closeable {
} else {
softDeletesRatio = 0d;
}
+
w = mockIndexWriter(dir, c, r);
config = w.getConfig();
flushAt = TestUtil.nextInt(r, 10, 1000);
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomPostingsTester.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomPostingsTester.java
index 55d3c0887f6..d64b99912d5 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomPostingsTester.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomPostingsTester.java
@@ -165,6 +165,7 @@ public class RandomPostingsTester {
0,
VectorEncoding.FLOAT32,
VectorSimilarityFunction.EUCLIDEAN,
+ false,
false);
fieldUpto++;
@@ -738,6 +739,7 @@ public class RandomPostingsTester {
0,
VectorEncoding.FLOAT32,
VectorSimilarityFunction.EUCLIDEAN,
+ false,
false);
}