You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2025/02/01 16:50:45 UTC
(lucene) branch branch_10x updated (8b4c500889b -> 0debeed5cf5)
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a change to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git
from 8b4c500889b Do not enable security manager on JDK 24+ (#14179)
new 2c3717447e2 Fix refill logic in nextDoc(). (#14185)
new 0debeed5cf5 Allow `LogMergePolicy` to merge more than `mergeFactor` segments together when the merge is below the min merge size. (#14166)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
lucene/CHANGES.txt | 4 ++++
.../codecs/lucene101/Lucene101PostingsReader.java | 20 +++++++----------
.../org/apache/lucene/index/LogMergePolicy.java | 25 ++++++++++++++++++++++
.../apache/lucene/index/TestLogMergePolicy.java | 7 +++++-
4 files changed, 43 insertions(+), 13 deletions(-)
(lucene) 02/02: Allow `LogMergePolicy` to merge more than `mergeFactor` segments together when the merge is below the min merge size. (#14166)
Posted by jp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 0debeed5cf599c60717fe6a97dee50531a860571
Author: Adrien Grand <jp...@gmail.com>
AuthorDate: Sat Feb 1 17:43:15 2025 +0100
Allow `LogMergePolicy` to merge more than `mergeFactor` segments together when the merge is below the min merge size. (#14166)
This is essentially porting #266 to `LogMergePolicy`. By allowing more than
`mergeFactor` segments to be merged together for small merges, the merge policy
gets a lower write amplification and indexes have fewer small segments.
---
lucene/CHANGES.txt | 4 ++++
.../org/apache/lucene/index/LogMergePolicy.java | 25 ++++++++++++++++++++++
.../apache/lucene/index/TestLogMergePolicy.java | 7 +++++-
3 files changed, 35 insertions(+), 1 deletion(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index faebe6ec8a3..ef8d89b31b8 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -37,6 +37,10 @@ Improvements
individual and bulk data retrieval overloads; avoid double buffering with
slices. (Chris Hegarty)
+* GITHUB#14166: Log(ByteSize|Doc)MergePolicy now allow merging more than
+ mergeFactor segments together when the merge is below the min merge size.
+ (Adrien Grand)
+
Optimizations
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/LogMergePolicy.java b/lucene/core/src/java/org/apache/lucene/index/LogMergePolicy.java
index 881ae099d5f..b6dc9848c9d 100644
--- a/lucene/core/src/java/org/apache/lucene/index/LogMergePolicy.java
+++ b/lucene/core/src/java/org/apache/lucene/index/LogMergePolicy.java
@@ -636,6 +636,31 @@ public abstract class LogMergePolicy extends MergePolicy {
mergeDocs += segmentDocs;
}
+ if (end - start >= mergeFactor
+ && minMergeSize < maxMergeSize
+ && mergeSize < minMergeSize
+ && anyMerging == false) {
+ // If the merge has mergeFactor segments but is still smaller than the min merged segment
+ // size, keep packing candidate segments.
+ while (end < 1 + upto) {
+ final SegmentInfoAndLevel segLevel = levels.get(end);
+ final SegmentCommitInfo info = segLevel.info;
+ if (mergingSegments.contains(info)) {
+ anyMerging = true;
+ break;
+ }
+ long segmentSize = size(info, mergeContext);
+ long segmentDocs = sizeDocs(info, mergeContext);
+ if (mergeSize + segmentSize > minMergeSize || mergeDocs + segmentDocs > maxMergeDocs) {
+ break;
+ }
+
+ mergeSize += segmentSize;
+ mergeDocs += segmentDocs;
+ end++;
+ }
+ }
+
if (anyMerging || end - start <= 1) {
// skip: there is an ongoing merge at the current level or the computed merge has a single
// segment and this merge policy doesn't do singleton merges
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestLogMergePolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestLogMergePolicy.java
index ea60f9b1e09..8c121c1f6a0 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestLogMergePolicy.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestLogMergePolicy.java
@@ -50,8 +50,13 @@ public class TestLogMergePolicy extends BaseMergePolicyTestCase {
@Override
protected void assertMerge(MergePolicy policy, MergeSpecification merge) throws IOException {
LogMergePolicy lmp = (LogMergePolicy) policy;
+ MergeContext mockMergeContext = new MockMergeContext(SegmentCommitInfo::getDelCount);
for (OneMerge oneMerge : merge.merges) {
- assertTrue(oneMerge.segments.size() <= lmp.getMergeFactor());
+ long mergeSize = 0;
+ for (SegmentCommitInfo info : oneMerge.segments) {
+ mergeSize += lmp.size(info, mockMergeContext);
+ }
+ assertTrue(mergeSize < lmp.minMergeSize || oneMerge.segments.size() <= lmp.getMergeFactor());
}
}
(lucene) 01/02: Fix refill logic in nextDoc(). (#14185)
Posted by jp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 2c3717447e2fb52c2b5129712dceaeae761c47fd
Author: Adrien Grand <jp...@gmail.com>
AuthorDate: Sat Feb 1 17:41:53 2025 +0100
Fix refill logic in nextDoc(). (#14185)
The recent optimization from #14164 interfered in a bad way with a prior
optimization.
---
.../codecs/lucene101/Lucene101PostingsReader.java | 20 ++++++++------------
1 file changed, 8 insertions(+), 12 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsReader.java
index 04b39c23974..b73e6316a7d 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsReader.java
@@ -888,16 +888,7 @@ public final class Lucene101PostingsReader extends PostingsReaderBase {
public void advanceShallow(int target) throws IOException {
if (target > level0LastDocID) { // advance level 0 skip data
doAdvanceShallow(target);
-
- // If we are on the last doc ID of a block and we are advancing on the doc ID just beyond
- // this block, then we decode the block. This may not be necessary, but this helps avoid
- // having to check whether we are in a block that is not decoded yet in #nextDoc().
- if (docBufferUpto == BLOCK_SIZE && target == doc + 1) {
- refillDocs();
- needsRefilling = false;
- } else {
- needsRefilling = true;
- }
+ needsRefilling = true;
}
}
@@ -914,8 +905,13 @@ public final class Lucene101PostingsReader extends PostingsReaderBase {
@Override
public int nextDoc() throws IOException {
- if (doc == level0LastDocID) {
- moveToNextLevel0Block();
+ if (doc == level0LastDocID || needsRefilling) {
+ if (needsRefilling) {
+ refillDocs();
+ needsRefilling = false;
+ } else {
+ moveToNextLevel0Block();
+ }
}
switch (encoding) {