You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by be...@apache.org on 2024/05/15 13:45:42 UTC

(lucene) branch branch_9x updated: Fix weird NRT bug #13353 (#13369)

This is an automated email from the ASF dual-hosted git repository.

benwtrent pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git


The following commit(s) were added to refs/heads/branch_9x by this push:
     new a30d7285cfb Fix weird NRT bug #13353 (#13369)
a30d7285cfb is described below

commit a30d7285cfb0e67c93bff3b38b4e395ee14b9513
Author: Benjamin Trent <be...@gmail.com>
AuthorDate: Wed May 15 09:37:19 2024 -0400

    Fix weird NRT bug #13353 (#13369)
    
    The issue outlines the problem. When we have point value dimensions, segment core readers assume that there will be point files.
    
    However, when allowing soft deletes and a document fails indexing failed before a point field could be written, this assumption fails. Consequently, the NRT fails to open. I settled on always flushing a point file if the field info says there are point fields, even if there aren't any docs in the buffer.
    
    closes #13353
---
 lucene/CHANGES.txt                                 |  3 ++
 .../org/apache/lucene/index/IndexingChain.java     |  2 +-
 .../lucene/index/TestIndexWriterExceptions.java    | 33 ++++++++++++++++++++++
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index ccebd69bb62..274c5b5ba13 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -141,6 +141,9 @@ Bug Fixes
 
 * GITHUB#13366: Disallow NaN and Inf values in scalar quantization and better handle extreme cases. (Ben Trent)
 
+* GITHUB#13369: Fix NRT opening failure when soft deletes are enabled and the document fails to index before a point
+  field is written (Ben Trent)
+
 Build
 ---------------------
 
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
index e6c5ad433b0..128fdae2095 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
@@ -366,7 +366,7 @@ final class IndexingChain implements Accountable {
         while (perField != null) {
           if (perField.pointValuesWriter != null) {
             // We could have initialized pointValuesWriter, but failed to write even a single doc
-            if (perField.pointValuesWriter.getNumDocs() > 0) {
+            if (perField.fieldInfo.getPointDimensionCount() > 0) {
               if (pointsWriter == null) {
                 // lazy init
                 PointsFormat fmt = state.segmentInfo.getCodec().pointsFormat();
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
index 1b36eeca29b..a57fddb726a 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
@@ -47,6 +47,7 @@ import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.ByteBuffersDirectory;
@@ -2309,4 +2310,36 @@ public class TestIndexWriterExceptions extends LuceneTestCase {
       DirectoryReader.open(dir).close();
     }
   }
+
+  public void testExceptionJustBeforeFlushWithPointValues() throws Exception {
+    Directory dir = newDirectory();
+    Analyzer analyzer =
+        new Analyzer(Analyzer.PER_FIELD_REUSE_STRATEGY) {
+          @Override
+          public TokenStreamComponents createComponents(String fieldName) {
+            MockTokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+            tokenizer.setEnableChecks(
+                false); // disable workflow checking as we forcefully close() in exceptional cases.
+            TokenStream stream = new CrashingFilter(fieldName, tokenizer);
+            return new TokenStreamComponents(tokenizer, stream);
+          }
+        };
+    IndexWriterConfig iwc =
+        newIndexWriterConfig(analyzer).setCommitOnClose(false).setMaxBufferedDocs(3);
+    MergePolicy mp = iwc.getMergePolicy();
+    iwc.setMergePolicy(
+        new SoftDeletesRetentionMergePolicy("soft_delete", MatchAllDocsQuery::new, mp));
+    IndexWriter w = RandomIndexWriter.mockIndexWriter(dir, iwc, random());
+    Document newdoc = new Document();
+    newdoc.add(newTextField("crash", "do it on token 4", Field.Store.NO));
+    newdoc.add(new IntPoint("int", 42));
+    expectThrows(IOException.class, () -> w.addDocument(newdoc));
+    DirectoryReader r = w.getReader(false, false);
+    LeafReader onlyReader = getOnlyLeafReader(r);
+    // we mark the failed doc as deleted
+    assertEquals(onlyReader.numDeletedDocs(), 1);
+    onlyReader.close();
+    w.close();
+    dir.close();
+  }
 }