You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by to...@apache.org on 2017/10/12 13:39:47 UTC
svn commit: r1811961 - in /jackrabbit/oak/trunk/oak-lucene/src:
main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/
test/java/org/apache/jackrabbit/oak/plugins/index/lucene/
Author: tommaso
Date: Thu Oct 12 13:39:47 2017
New Revision: 1811961
URL: http://svn.apache.org/viewvc?rev=1811961&view=rev
Log:
OAK-6710 - set a segment threshold for mitigating merges
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/CommitMitigatingTieredMergePolicy.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneWritesOnSegmentStatsTest.java
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/CommitMitigatingTieredMergePolicy.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/CommitMitigatingTieredMergePolicy.java?rev=1811961&r1=1811960&r2=1811961&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/CommitMitigatingTieredMergePolicy.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/CommitMitigatingTieredMergePolicy.java Thu Oct 12 13:39:47 2017
@@ -54,6 +54,7 @@ public class CommitMitigatingTieredMerge
private static final double DEFAULT_MAX_COMMIT_RATE_DOCS = 1000;
private static final double DEFAULT_MAX_COMMIT_RATE_MB = 5;
+ private static final int DEFAULT_MAX_NO_OF_SEGS = 30;
private int maxMergeAtOnce = 10;
private long maxMergedSegmentBytes = 5 * 1024 * 1024 * 1024L;
@@ -66,6 +67,7 @@ public class CommitMitigatingTieredMerge
private double maxCommitRateDocs = DEFAULT_MAX_COMMIT_RATE_DOCS;
private double maxCommitRateMB = DEFAULT_MAX_COMMIT_RATE_MB;
+ private int maxNoOfSegs = DEFAULT_MAX_NO_OF_SEGS;
private double docCount = 0d;
private double mb = 0d;
@@ -330,8 +332,7 @@ public class CommitMitigatingTieredMerge
long now = System.currentTimeMillis();
double timeDelta = (now / 1000d) - (time / 1000d);
double commitRate = Math.abs(docCount - infos.totalDocCount()) / timeDelta;
- log.debug("committing {} docs/sec", commitRate);
- System.out.printf("committing %s docs/sec\n", commitRate);
+ log.debug("committing {} docs/sec ({} segs)", commitRate);
docCount = infos.totalDocCount();
time = now;
@@ -340,7 +341,8 @@ public class CommitMitigatingTieredMerge
message(commitRate + "doc/s (max: " + maxCommitRateDocs + "doc/s)");
}
- if (commitRate > maxCommitRateDocs) {
+ // set a maxSegmentsBarrier
+ if (commitRate > maxCommitRateDocs && infos.size() < maxNoOfSegs) {
return null;
}
@@ -431,15 +433,14 @@ public class CommitMitigatingTieredMerge
double bytes = idxBytes - this.mb;
double mbRate = bytes / timeDelta;
- log.debug("committing {} MBs/sec", mbRate);
- System.out.printf("committing %s MBs/sec\n", mbRate);
+ log.debug("committing {} MBs/sec ({} segs)", mbRate, infos.size());
if (verbose()) {
message(mbRate + "mb/s (max: " + maxCommitRateMB + "mb/s)");
}
this.mb = idxBytes;
- if (mbRate > maxCommitRateMB) {
+ if (mbRate > maxCommitRateMB && infos.size() < maxNoOfSegs) {
return null;
}
Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneWritesOnSegmentStatsTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneWritesOnSegmentStatsTest.java?rev=1811961&r1=1811960&r2=1811961&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneWritesOnSegmentStatsTest.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneWritesOnSegmentStatsTest.java Thu Oct 12 13:39:47 2017
@@ -75,7 +75,7 @@ import org.junit.runners.Parameterized;
* Tests for checking impacts of Lucene writes wrt storage / configuration adjustments on the
* {@link org.apache.jackrabbit.oak.segment.SegmentNodeStore}.
*/
-@Ignore("this is meant to be a benchmark, it shouldn't be part of everyday builds")
+//@Ignore("this is meant to be a benchmark, it shouldn't be part of everyday builds")
@RunWith(Parameterized.class)
public class LuceneWritesOnSegmentStatsTest extends AbstractQueryTest {
@@ -119,16 +119,16 @@ public class LuceneWritesOnSegmentStatsT
return Arrays.asList(new Object[][]{
{false, "oakCodec", false, 4000, "tiered"},
{false, "oakCodec", false, 4000, "mitigated"},
- {false, "oakCodec", false, 4000, "no"},
- {false, "Lucene46", false, 4000, "tiered"},
- {false, "Lucene46", false, 4000, "mitigated"},
- {false, "Lucene46", false, 4000, "no"},
- {false, "oakCodec", false, -1, "tiered"},
- {false, "oakCodec", false, -1, "mitigated"},
- {false, "oakCodec", false, -1, "no"},
- {false, "Lucene46", false, -1, "tiered"},
- {false, "Lucene46", false, -1, "mitigated"},
- {false, "Lucene46", false, -1, "no"},
+// {false, "oakCodec", false, 4000, "no"},
+// {false, "Lucene46", false, 4000, "tiered"},
+// {false, "Lucene46", false, 4000, "mitigated"},
+// {false, "Lucene46", false, 4000, "no"},
+// {false, "oakCodec", false, -1, "tiered"},
+// {false, "oakCodec", false, -1, "mitigated"},
+// {false, "oakCodec", false, -1, "no"},
+// {false, "Lucene46", false, -1, "tiered"},
+// {false, "Lucene46", false, -1, "mitigated"},
+// {false, "Lucene46", false, -1, "no"},
});
}