You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2024/07/18 10:05:37 UTC

(lucene) branch branch_9x updated (b2d3a2b37e0 -> bbcd65fa913)

This is an automated email from the ASF dual-hosted git repository.

jpountz pushed a change to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git


    from b2d3a2b37e0 [9.x] Ensure to use IOContext.READONCE when reading segment files (#13578)
     new 3c3e6c556e8 Stop requiring MaxScoreBulkScorer's outer window from having at least INNER_WINDOW_SIZE docs. (#13582)
     new bbcd65fa913 Add a `targetSearchConcurrency` parameter to `LogMergePolicy`. (#13517)

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 lucene/CHANGES.txt                                 |  7 +++++
 .../org/apache/lucene/index/LogMergePolicy.java    | 35 ++++++++++++++++++++++
 .../apache/lucene/search/MaxScoreBulkScorer.java   |  5 ----
 .../apache/lucene/TestMergeSchedulerExternal.java  |  2 +-
 .../lucene/index/TestConcurrentMergeScheduler.java |  1 +
 .../lucene/index/TestIndexWriterExceptions.java    |  1 +
 .../lucene/index/TestIndexWriterMergePolicy.java   |  4 ++-
 .../apache/lucene/index/TestIndexWriterReader.java |  8 +++--
 .../apache/lucene/index/TestLogMergePolicy.java    |  6 ++--
 .../apache/lucene/tests/util/LuceneTestCase.java   |  5 ++--
 10 files changed, 61 insertions(+), 13 deletions(-)


(lucene) 01/02: Stop requiring MaxScoreBulkScorer's outer window from having at least INNER_WINDOW_SIZE docs. (#13582)

Posted by jp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jpountz pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git

commit 3c3e6c556e80a1fc3a7443cbd30be517988ee30d
Author: Adrien Grand <jp...@gmail.com>
AuthorDate: Thu Jul 18 11:23:19 2024 +0200

    Stop requiring MaxScoreBulkScorer's outer window from having at least INNER_WINDOW_SIZE docs. (#13582)
    
    Currently `MaxScoreBulkScorer` requires its "outer" window to be at least
    `WINDOW_SIZE`. The intuition there was that we should make sure we should use
    the whole range of the bit set that we are using to collect matches. The
    downside is that it may force us to use an upper level in the skip list that
    has worse upper bounds for the scores.
---
 lucene/CHANGES.txt                                                   | 3 +++
 .../core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java   | 5 -----
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index e7fa3d9f2ec..db5735fa6cb 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -52,6 +52,9 @@ Optimizations
 * GITHUB#12100: WordBreakSpellChecker.suggestWordBreaks now does a breadth first search, allowing it to return
   better matches with fewer evaluations (hossman)
 
+* GITHUB#13582: Stop requiring MaxScoreBulkScorer's outer window from having at
+  least INNER_WINDOW_SIZE docs. (Adrien Grand)
+
 Changes in runtime behavior
 ---------------------
 
diff --git a/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java b/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java
index f640f345c4a..ef8627b7ab4 100644
--- a/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java
@@ -271,11 +271,6 @@ final class MaxScoreBulkScorer extends BulkScorer {
       windowMax = (int) Math.min(windowMax, upTo + 1L); // upTo is inclusive
     }
 
-    // Score at least an entire inner window of docs
-    windowMax =
-        Math.max(
-            windowMax, (int) Math.min(Integer.MAX_VALUE, (long) windowMin + INNER_WINDOW_SIZE));
-
     return windowMax;
   }
 


(lucene) 02/02: Add a `targetSearchConcurrency` parameter to `LogMergePolicy`. (#13517)

Posted by jp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jpountz pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git

commit bbcd65fa9139f99583cd06f8e381b04c2ce6d1c1
Author: Adrien Grand <jp...@gmail.com>
AuthorDate: Thu Jul 18 11:28:35 2024 +0200

    Add a `targetSearchConcurrency` parameter to `LogMergePolicy`. (#13517)
    
    This adds the same `targetSearchConcurrency` parameter to `LogMergePolicy`
    that #13430 is adding to `TieredMergePolicy`.
---
 lucene/CHANGES.txt                                 |  4 +++
 .../org/apache/lucene/index/LogMergePolicy.java    | 35 ++++++++++++++++++++++
 .../apache/lucene/TestMergeSchedulerExternal.java  |  2 +-
 .../lucene/index/TestConcurrentMergeScheduler.java |  1 +
 .../lucene/index/TestIndexWriterExceptions.java    |  1 +
 .../lucene/index/TestIndexWriterMergePolicy.java   |  4 ++-
 .../apache/lucene/index/TestIndexWriterReader.java |  8 +++--
 .../apache/lucene/index/TestLogMergePolicy.java    |  6 ++--
 .../apache/lucene/tests/util/LuceneTestCase.java   |  5 ++--
 9 files changed, 58 insertions(+), 8 deletions(-)

diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index db5735fa6cb..2492eef9cf7 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -20,6 +20,10 @@ New Features
   merge policy to try to have at least this number of segments on the highest
   tier. (Adrien Grand, Carlos Delgado)
 
+* GITHUB#13517: Allow configuring the search concurrency on LogDocMergePolicy
+  and LogByteSizeMergePolicy via a new #setTargetConcurrency setter.
+  (Adrien Grand)
+
 Improvements
 ---------------------
 
diff --git a/lucene/core/src/java/org/apache/lucene/index/LogMergePolicy.java b/lucene/core/src/java/org/apache/lucene/index/LogMergePolicy.java
index f7c5011d9c6..e629ef91515 100644
--- a/lucene/core/src/java/org/apache/lucene/index/LogMergePolicy.java
+++ b/lucene/core/src/java/org/apache/lucene/index/LogMergePolicy.java
@@ -90,6 +90,12 @@ public abstract class LogMergePolicy extends MergePolicy {
   /** If true, we pro-rate a segment's size by the percentage of non-deleted documents. */
   protected boolean calibrateSizeByDeletes = true;
 
+  /**
+   * Target search concurrency. This merge policy will avoid creating segments that have more than
+   * {@code maxDoc / targetSearchConcurrency} documents.
+   */
+  protected int targetSearchConcurrency = 1;
+
   /** Sole constructor. (For invocation by subclass constructors, typically implicit.) */
   public LogMergePolicy() {
     super(DEFAULT_NO_CFS_RATIO, MergePolicy.DEFAULT_MAX_CFS_SEGMENT_SIZE);
@@ -131,6 +137,28 @@ public abstract class LogMergePolicy extends MergePolicy {
     return calibrateSizeByDeletes;
   }
 
+  /**
+   * Sets the target search concurrency. This prevents creating segments that are bigger than
+   * maxDoc/targetSearchConcurrency, which in turn makes the work parallelizable into
+   * targetSearchConcurrency slices of similar doc counts.
+   *
+   * <p><b>NOTE:</b> Configuring a value greater than 1 will increase the number of segments in the
+   * index linearly with the value of {@code targetSearchConcurrency} and also increase write
+   * amplification.
+   */
+  public void setTargetSearchConcurrency(int targetSearchConcurrency) {
+    if (targetSearchConcurrency < 1) {
+      throw new IllegalArgumentException(
+          "targetSearchConcurrency must be >= 1 (got " + targetSearchConcurrency + ")");
+    }
+    this.targetSearchConcurrency = targetSearchConcurrency;
+  }
+
+  /** Returns the target search concurrency. */
+  public int getTargetSearchConcurrency() {
+    return targetSearchConcurrency;
+  }
+
   /**
    * Return the number of documents in the provided {@link SegmentCommitInfo}, pro-rated by
    * percentage of non-deleted documents if {@link #setCalibrateSizeByDeletes} is set.
@@ -484,8 +512,10 @@ public abstract class LogMergePolicy extends MergePolicy {
 
     final Set<SegmentCommitInfo> mergingSegments = mergeContext.getMergingSegments();
 
+    int totalDocCount = 0;
     for (int i = 0; i < numSegments; i++) {
       final SegmentCommitInfo info = infos.info(i);
+      totalDocCount += sizeDocs(info, mergeContext);
       long size = size(info, mergeContext);
 
       // Floor tiny segments
@@ -575,6 +605,11 @@ public abstract class LogMergePolicy extends MergePolicy {
             mergeContext);
       }
 
+      final int maxMergeDocs =
+          Math.min(
+              this.maxMergeDocs,
+              (totalDocCount + targetSearchConcurrency - 1) / targetSearchConcurrency);
+
       // Finally, record all merges that are viable at this level:
       int end = start + mergeFactor;
       while (end <= 1 + upto) {
diff --git a/lucene/core/src/test/org/apache/lucene/TestMergeSchedulerExternal.java b/lucene/core/src/test/org/apache/lucene/TestMergeSchedulerExternal.java
index da72c3273d1..4921703b6a7 100644
--- a/lucene/core/src/test/org/apache/lucene/TestMergeSchedulerExternal.java
+++ b/lucene/core/src/test/org/apache/lucene/TestMergeSchedulerExternal.java
@@ -131,7 +131,7 @@ public class TestMergeSchedulerExternal extends LuceneTestCase {
     logMP.setMergeFactor(10);
 
     try {
-      for (int i = 0; i < 20; i++) {
+      for (int i = 0; i < 60; i++) {
         writer.addDocument(doc);
       }
     } catch (
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java b/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
index ec2edbe4453..e3e1093f4d0 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
@@ -804,6 +804,7 @@ public class TestConcurrentMergeScheduler extends LuceneTestCase {
     iwc.setMaxBufferedDocs(2);
     LogMergePolicy lmp = newLogMergePolicy();
     lmp.setMergeFactor(2);
+    lmp.setTargetSearchConcurrency(1);
     iwc.setMergePolicy(lmp);
 
     IndexWriter w = new IndexWriter(dir, iwc);
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
index c60cb1df19b..f282d806bae 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
@@ -471,6 +471,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase {
     cms.setSuppressExceptions();
     conf.setMergeScheduler(cms);
     ((LogMergePolicy) conf.getMergePolicy()).setMergeFactor(2);
+    ((LogMergePolicy) conf.getMergePolicy()).setTargetSearchConcurrency(1);
     TestPoint3 testPoint = new TestPoint3();
     IndexWriter w = RandomIndexWriter.mockIndexWriter(random(), dir, conf, testPoint);
     testPoint.doFail = true;
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
index 1835a4ff309..c56feb62d84 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
@@ -900,8 +900,10 @@ public class TestIndexWriterMergePolicy extends LuceneTestCase {
   }
 
   public void testSetDiagnostics() throws IOException {
+    LogMergePolicy logMp = newLogMergePolicy(4);
+    logMp.setTargetSearchConcurrency(1);
     MergePolicy myMergePolicy =
-        new FilterMergePolicy(newLogMergePolicy(4)) {
+        new FilterMergePolicy(logMp) {
           @Override
           public MergeSpecification findMerges(
               MergeTrigger mergeTrigger, SegmentInfos segmentInfos, MergeContext mergeContext)
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java
index f70f14a9f1f..b568cda329d 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java
@@ -1004,6 +1004,8 @@ public class TestIndexWriterReader extends LuceneTestCase {
   public void testSegmentWarmer() throws Exception {
     Directory dir = newDirectory();
     final AtomicBoolean didWarm = new AtomicBoolean();
+    LogMergePolicy mp = newLogMergePolicy(10);
+    mp.setTargetSearchConcurrency(1);
     IndexWriter w =
         new IndexWriter(
             dir,
@@ -1017,7 +1019,7 @@ public class TestIndexWriterReader extends LuceneTestCase {
                       assertEquals(20, count);
                       didWarm.set(true);
                     })
-                .setMergePolicy(newLogMergePolicy(10)));
+                .setMergePolicy(mp));
 
     Document doc = new Document();
     doc.add(newStringField("foo", "bar", Field.Store.NO));
@@ -1050,6 +1052,8 @@ public class TestIndexWriterReader extends LuceneTestCase {
             return true;
           }
         };
+    LogMergePolicy mp = newLogMergePolicy(10);
+    mp.setTargetSearchConcurrency(1);
     IndexWriter w =
         new IndexWriter(
             dir,
@@ -1058,7 +1062,7 @@ public class TestIndexWriterReader extends LuceneTestCase {
                 .setReaderPooling(true)
                 .setInfoStream(infoStream)
                 .setMergedSegmentWarmer(new SimpleMergedSegmentWarmer(infoStream))
-                .setMergePolicy(newLogMergePolicy(10)));
+                .setMergePolicy(mp));
 
     Document doc = new Document();
     doc.add(newStringField("foo", "bar", Field.Store.NO));
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestLogMergePolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestLogMergePolicy.java
index 0b00a20a17f..ea60f9b1e09 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestLogMergePolicy.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestLogMergePolicy.java
@@ -249,8 +249,10 @@ public class TestLogMergePolicy extends BaseMergePolicyTestCase {
     SegmentInfos segmentInfos = new SegmentInfos(Version.LATEST.major);
 
     LogMergePolicy mp = mergePolicy();
+    // Number of segments guaranteed to trigger a merge.
+    int numSegmentsForMerging = mp.getMergeFactor() + mp.getTargetSearchConcurrency();
 
-    for (int i = 0; i < mp.getMergeFactor(); ++i) {
+    for (int i = 0; i < numSegmentsForMerging; ++i) {
       segmentInfos.add(
           makeSegmentCommitInfo(
               "_" + segNameGenerator.getAndIncrement(),
@@ -266,6 +268,6 @@ public class TestLogMergePolicy extends BaseMergePolicyTestCase {
       segmentInfos =
           applyMerge(segmentInfos, merge, "_" + segNameGenerator.getAndIncrement(), stats);
     }
-    assertEquals(1, segmentInfos.size());
+    assertTrue(segmentInfos.size() < numSegmentsForMerging);
   }
 }
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCase.java
index e411f2271a0..3acf2afa0e0 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCase.java
@@ -1104,6 +1104,7 @@ public abstract class LuceneTestCase extends Assert {
   public static LogMergePolicy newLogMergePolicy(Random r) {
     LogMergePolicy logmp = r.nextBoolean() ? new LogDocMergePolicy() : new LogByteSizeMergePolicy();
     logmp.setCalibrateSizeByDeletes(r.nextBoolean());
+    logmp.setTargetSearchConcurrency(TestUtil.nextInt(random(), 1, 16));
     if (rarely(r)) {
       logmp.setMergeFactor(TestUtil.nextInt(r, 2, 9));
     } else {
@@ -1163,14 +1164,14 @@ public abstract class LuceneTestCase extends Assert {
     return logmp;
   }
 
-  public static MergePolicy newLogMergePolicy(boolean useCFS, int mergeFactor) {
+  public static LogMergePolicy newLogMergePolicy(boolean useCFS, int mergeFactor) {
     LogMergePolicy logmp = newLogMergePolicy();
     logmp.setNoCFSRatio(useCFS ? 1.0 : 0.0);
     logmp.setMergeFactor(mergeFactor);
     return logmp;
   }
 
-  public static MergePolicy newLogMergePolicy(int mergeFactor) {
+  public static LogMergePolicy newLogMergePolicy(int mergeFactor) {
     LogMergePolicy logmp = newLogMergePolicy();
     logmp.setMergeFactor(mergeFactor);
     return logmp;