You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by ji...@apache.org on 2019/04/03 18:48:08 UTC
[incubator-druid] branch master updated: Overwrite index task
maxTotalRows with computed maxRowsPerSegments (#7338)
This is an automated email from the ASF dual-hosted git repository.
jihoonson pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 6b741f0 Overwrite index task maxTotalRows with computed maxRowsPerSegments (#7338)
6b741f0 is described below
commit 6b741f080091034b56ff9e04ea60a847ceff9f3d
Author: lxqfy <yi...@gmail.com>
AuthorDate: Thu Apr 4 02:47:59 2019 +0800
Overwrite index task maxTotalRows with computed maxRowsPerSegments (#7338)
fixed identation
set maxTotalRows to higher value in case mutiple segments in appendrator
with comments
---
.../druid/indexing/common/task/CompactionTask.java | 5 ++++-
.../druid/indexing/common/task/IndexTask.java | 22 ++++++++++++++++++++++
.../indexing/common/task/CompactionTaskTest.java | 2 +-
3 files changed, 27 insertions(+), 2 deletions(-)
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java
index 7608f88..156d0f6 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java
@@ -867,8 +867,11 @@ public class CompactionTask extends AbstractTask
avgRowsPerByte,
nonNullTargetCompactionSizeBytes
);
+ // Setting maxTotalRows to Long.MAX_VALUE to respect the computed maxRowsPerSegment.
+ // If this is set to something too small, compactionTask can generate small segments
+ // which need to be compacted again, which in turn making auto compaction stuck in the same interval.
return (tuningConfig == null ? IndexTuningConfig.createDefault() : tuningConfig)
- .withMaxRowsPerSegment(maxRowsPerSegment);
+ .withMaxRowsPerSegment(maxRowsPerSegment).withMaxTotalRows(Long.MAX_VALUE);
} else {
return tuningConfig;
}
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/IndexTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/IndexTask.java
index 2f52b69..5161f9b 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/IndexTask.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/IndexTask.java
@@ -1437,6 +1437,28 @@ public class IndexTask extends AbstractTask implements ChatHandler
);
}
+ public IndexTuningConfig withMaxTotalRows(Long maxTotalRows)
+ {
+ return new IndexTuningConfig(
+ maxRowsPerSegment,
+ maxRowsInMemory,
+ maxBytesInMemory,
+ maxTotalRows,
+ numShards,
+ partitionDimensions,
+ indexSpec,
+ maxPendingPersists,
+ forceGuaranteedRollup,
+ reportParseExceptions,
+ pushTimeout,
+ basePersistDirectory,
+ segmentWriteOutMediumFactory,
+ logParseExceptions,
+ maxParseExceptions,
+ maxSavedParseExceptions
+ );
+ }
+
public IndexTuningConfig withMaxRowsPerSegment(int maxRowsPerSegment)
{
return new IndexTuningConfig(
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java
index cbd52f5..74b8b4b 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java
@@ -1220,7 +1220,7 @@ public class CompactionTaskTest
41943040, // automatically computed targetPartitionSize
500000,
1000000L,
- null,
+ Long.MAX_VALUE,
null,
null,
null,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org