You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2023/10/24 14:57:12 UTC

[lucene] branch main updated: Run merge-on-full-flush even though no changes got flushed. (#12549)

This is an automated email from the ASF dual-hosted git repository.

jpountz pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git


The following commit(s) were added to refs/heads/main by this push:
     new 71c4ea74ba2 Run merge-on-full-flush even though no changes got flushed. (#12549)
71c4ea74ba2 is described below

commit 71c4ea74ba2270911dc924602ac8110a574e4f1e
Author: Adrien Grand <jp...@gmail.com>
AuthorDate: Tue Oct 24 16:57:05 2023 +0200

    Run merge-on-full-flush even though no changes got flushed. (#12549)
    
    Currently, merge-on-full-flush only checks if merges need to run if changes
    have been flushed to disk. This prevents from having different merging logic
    for refreshes and commits, since the merge policy would not be checked upon
    commit if no new documents got indexed since the previous refresh.
---
 .../src/java/org/apache/lucene/index/IndexWriter.java     |  3 +--
 .../apache/lucene/index/TestDirectoryReaderReopen.java    |  3 ++-
 .../src/test/org/apache/lucene/index/TestIndexWriter.java |  5 ++---
 .../org/apache/lucene/index/TestIndexWriterDelete.java    |  2 +-
 .../apache/lucene/index/TestIndexWriterMergePolicy.java   | 15 ++++++---------
 .../org/apache/lucene/index/TestPendingSoftDeletes.java   |  2 ++
 6 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index aef273348fd..4e235ae7d8e 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -3680,7 +3680,7 @@ public class IndexWriter
               // merge completes which would otherwise have
               // removed the files we are now syncing.
               deleter.incRef(toCommit.files(false));
-              if (anyChanges && maxCommitMergeWaitMillis > 0) {
+              if (maxCommitMergeWaitMillis > 0) {
                 // we can safely call preparePointInTimeMerge since writeReaderPool(true) above
                 // wrote all
                 // necessary files to disk and checkpointed them.
@@ -4232,7 +4232,6 @@ public class IndexWriter
           flushSuccess = true;
         } finally {
           assert Thread.holdsLock(fullFlushLock);
-          ;
           docWriter.finishFullFlush(flushSuccess);
           processEvents(false);
         }
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
index 8221d5399ab..6e993a17473 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
@@ -654,7 +654,8 @@ public class TestDirectoryReaderReopen extends LuceneTestCase {
   public void testOverDecRefDuringReopen() throws Exception {
     MockDirectoryWrapper dir = newMockDirectory();
 
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+    IndexWriterConfig iwc =
+        new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE);
     iwc.setCodec(TestUtil.getDefaultCodec());
     IndexWriter w = new IndexWriter(dir, iwc);
     Document doc = new Document();
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
index 4b8ce1f70e2..bd52a84f7e6 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
@@ -519,11 +519,10 @@ public class TestIndexWriter extends LuceneTestCase {
     doc.add(newField("field", "aaa", customType));
     for (int i = 0; i < 19; i++) writer.addDocument(doc);
     writer.flush(false, true);
-    writer.close();
-    SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
     // Since we flushed w/o allowing merging we should now
     // have 10 segments
-    assertEquals(10, sis.size());
+    assertEquals(10, writer.getSegmentCount());
+    writer.close();
     dir.close();
   }
 
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
index 58f844247d6..468c91fc6ec 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
@@ -1314,7 +1314,7 @@ public class TestIndexWriterDelete extends LuceneTestCase {
     w.addDocument(doc);
     w.close();
 
-    iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+    iwc = new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE);
     iwc.setOpenMode(IndexWriterConfig.OpenMode.APPEND);
     w = new IndexWriter(d, iwc);
     IndexReader r = DirectoryReader.open(w, false, false);
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
index 8eabef5dae5..727f9ce3bd5 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
@@ -394,14 +394,13 @@ public class TestIndexWriterMergePolicy extends LuceneTestCase {
             .setMaxFullFlushMergeWaitMillis(Integer.MAX_VALUE);
 
     IndexWriter writerWithMergePolicy = new IndexWriter(dir, iwc);
-    writerWithMergePolicy.commit(); // No changes. Commit doesn't trigger a merge.
 
+    // No changes. Refresh doesn't trigger a merge.
     DirectoryReader unmergedReader = DirectoryReader.open(writerWithMergePolicy);
     assertEquals(5, unmergedReader.leaves().size());
     unmergedReader.close();
 
-    TestIndexWriter.addDoc(writerWithMergePolicy);
-    writerWithMergePolicy.commit(); // Doc added, do merge on commit.
+    writerWithMergePolicy.commit(); // Do merge on commit.
     assertEquals(1, writerWithMergePolicy.getSegmentCount()); //
 
     DirectoryReader mergedReader = DirectoryReader.open(writerWithMergePolicy);
@@ -410,8 +409,8 @@ public class TestIndexWriterMergePolicy extends LuceneTestCase {
 
     try (IndexReader reader = DirectoryReader.open(writerWithMergePolicy)) {
       IndexSearcher searcher = new IndexSearcher(reader);
-      assertEquals(6, reader.numDocs());
-      assertEquals(6, searcher.count(new MatchAllDocsQuery()));
+      assertEquals(5, reader.numDocs());
+      assertEquals(5, searcher.count(new MatchAllDocsQuery()));
     }
 
     writerWithMergePolicy.close();
@@ -445,16 +444,14 @@ public class TestIndexWriterMergePolicy extends LuceneTestCase {
             .setIndexWriterEventListener(eventListener);
 
     IndexWriter writerWithMergePolicy = new IndexWriter(dir, iwc);
-    writerWithMergePolicy.commit(); // No changes. Commit doesn't trigger a merge.
 
+    // No changes. Refresh doesn't trigger a merge.
     DirectoryReader unmergedReader = DirectoryReader.open(writerWithMergePolicy);
     assertEquals(5, unmergedReader.leaves().size());
     unmergedReader.close();
 
-    TestIndexWriter.addDoc(writerWithMergePolicy);
-
     assertFalse(eventListener.isEventsRecorded());
-    writerWithMergePolicy.commit(); // Doc added, do merge on commit.
+    writerWithMergePolicy.commit(); // Do merge on commit.
     assertEquals(1, writerWithMergePolicy.getSegmentCount()); //
     assertTrue(eventListener.isEventsRecorded());
 
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java b/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
index 8554dd3f40a..02600f5d59d 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
@@ -263,6 +263,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
             newIndexWriterConfig()
                 .setSoftDeletesField("_soft_deletes")
                 .setMaxBufferedDocs(3) // make sure we write one segment
+                .setMergePolicy(NoMergePolicy.INSTANCE) // prevent deletes from triggering merges
                 .setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH));
     Document doc = new Document();
     doc.add(new StringField("id", "1", Field.Store.YES));
@@ -331,6 +332,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
             newIndexWriterConfig()
                 .setSoftDeletesField("_soft_deletes")
                 .setMaxBufferedDocs(3) // make sure we write one segment
+                .setMergePolicy(NoMergePolicy.INSTANCE) // prevent deletes from triggering merges
                 .setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH));
     Document doc = new Document();
     doc.add(new StringField("id", "1", Field.Store.YES));