You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by be...@apache.org on 2024/05/15 13:45:42 UTC
(lucene) branch branch_9x updated: Fix weird NRT bug #13353 (#13369)
This is an automated email from the ASF dual-hosted git repository.
benwtrent pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new a30d7285cfb Fix weird NRT bug #13353 (#13369)
a30d7285cfb is described below
commit a30d7285cfb0e67c93bff3b38b4e395ee14b9513
Author: Benjamin Trent <be...@gmail.com>
AuthorDate: Wed May 15 09:37:19 2024 -0400
Fix weird NRT bug #13353 (#13369)
The issue outlines the problem. When we have point value dimensions, segment core readers assume that there will be point files.
However, when allowing soft deletes and a document fails indexing failed before a point field could be written, this assumption fails. Consequently, the NRT fails to open. I settled on always flushing a point file if the field info says there are point fields, even if there aren't any docs in the buffer.
closes #13353
---
lucene/CHANGES.txt | 3 ++
.../org/apache/lucene/index/IndexingChain.java | 2 +-
.../lucene/index/TestIndexWriterExceptions.java | 33 ++++++++++++++++++++++
3 files changed, 37 insertions(+), 1 deletion(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index ccebd69bb62..274c5b5ba13 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -141,6 +141,9 @@ Bug Fixes
* GITHUB#13366: Disallow NaN and Inf values in scalar quantization and better handle extreme cases. (Ben Trent)
+* GITHUB#13369: Fix NRT opening failure when soft deletes are enabled and the document fails to index before a point
+ field is written (Ben Trent)
+
Build
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
index e6c5ad433b0..128fdae2095 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
@@ -366,7 +366,7 @@ final class IndexingChain implements Accountable {
while (perField != null) {
if (perField.pointValuesWriter != null) {
// We could have initialized pointValuesWriter, but failed to write even a single doc
- if (perField.pointValuesWriter.getNumDocs() > 0) {
+ if (perField.fieldInfo.getPointDimensionCount() > 0) {
if (pointsWriter == null) {
// lazy init
PointsFormat fmt = state.segmentInfo.getCodec().pointsFormat();
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
index 1b36eeca29b..a57fddb726a 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
@@ -47,6 +47,7 @@ import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.ByteBuffersDirectory;
@@ -2309,4 +2310,36 @@ public class TestIndexWriterExceptions extends LuceneTestCase {
DirectoryReader.open(dir).close();
}
}
+
+ public void testExceptionJustBeforeFlushWithPointValues() throws Exception {
+ Directory dir = newDirectory();
+ Analyzer analyzer =
+ new Analyzer(Analyzer.PER_FIELD_REUSE_STRATEGY) {
+ @Override
+ public TokenStreamComponents createComponents(String fieldName) {
+ MockTokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ tokenizer.setEnableChecks(
+ false); // disable workflow checking as we forcefully close() in exceptional cases.
+ TokenStream stream = new CrashingFilter(fieldName, tokenizer);
+ return new TokenStreamComponents(tokenizer, stream);
+ }
+ };
+ IndexWriterConfig iwc =
+ newIndexWriterConfig(analyzer).setCommitOnClose(false).setMaxBufferedDocs(3);
+ MergePolicy mp = iwc.getMergePolicy();
+ iwc.setMergePolicy(
+ new SoftDeletesRetentionMergePolicy("soft_delete", MatchAllDocsQuery::new, mp));
+ IndexWriter w = RandomIndexWriter.mockIndexWriter(dir, iwc, random());
+ Document newdoc = new Document();
+ newdoc.add(newTextField("crash", "do it on token 4", Field.Store.NO));
+ newdoc.add(new IntPoint("int", 42));
+ expectThrows(IOException.class, () -> w.addDocument(newdoc));
+ DirectoryReader r = w.getReader(false, false);
+ LeafReader onlyReader = getOnlyLeafReader(r);
+ // we mark the failed doc as deleted
+ assertEquals(onlyReader.numDeletedDocs(), 1);
+ onlyReader.close();
+ w.close();
+ dir.close();
+ }
}