You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by as...@apache.org on 2019/02/28 03:07:43 UTC
[incubator-druid] branch master updated: Improve error message for
integer overflow in compaction task (#7131)
This is an automated email from the ASF dual-hosted git repository.
asdf2014 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new cacdc83 Improve error message for integer overflow in compaction task (#7131)
cacdc83 is described below
commit cacdc83cad98a2707591257705a2dd3f951b3095
Author: Jihoon Son <ji...@apache.org>
AuthorDate: Wed Feb 27 19:07:37 2019 -0800
Improve error message for integer overflow in compaction task (#7131)
* improve error message for integer overflow in compaction task
* fix build
---
.../org/apache/druid/java/util/common/Numbers.java | 8 ++++++++
.../druid/indexing/common/task/CompactionTask.java | 13 ++++++++++++-
.../indexing/common/task/CompactionTaskTest.java | 22 ++++++++++++++++++++++
3 files changed, 42 insertions(+), 1 deletion(-)
diff --git a/core/src/main/java/org/apache/druid/java/util/common/Numbers.java b/core/src/main/java/org/apache/druid/java/util/common/Numbers.java
index d3ccf26..c9d40d7 100644
--- a/core/src/main/java/org/apache/druid/java/util/common/Numbers.java
+++ b/core/src/main/java/org/apache/druid/java/util/common/Numbers.java
@@ -92,6 +92,14 @@ public final class Numbers
}
}
+ public static int toIntExact(long value, String error)
+ {
+ if ((int) value != value) {
+ throw new ArithmeticException(error);
+ }
+ return (int) value;
+ }
+
private Numbers()
{
}
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java
index c897de3..53c3641 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java
@@ -54,8 +54,10 @@ import org.apache.druid.indexing.firehose.IngestSegmentFirehoseFactory;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.JodaUtils;
+import org.apache.druid.java.util.common.Numbers;
import org.apache.druid.java.util.common.Pair;
import org.apache.druid.java.util.common.RE;
+import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.java.util.common.granularity.Granularity;
import org.apache.druid.java.util.common.granularity.GranularityType;
@@ -845,7 +847,16 @@ public class CompactionTask extends AbstractTask
}
final double avgRowsPerByte = totalNumRows / (double) totalSizeBytes;
- final int maxRowsPerSegment = Math.toIntExact(Math.round(avgRowsPerByte * nonNullTargetCompactionSizeBytes));
+ final long maxRowsPerSegmentLong = Math.round(avgRowsPerByte * nonNullTargetCompactionSizeBytes);
+ final int maxRowsPerSegment = Numbers.toIntExact(
+ maxRowsPerSegmentLong,
+ StringUtils.format(
+ "Estimated maxRowsPerSegment[%s] is out of integer value range. "
+ + "Please consider reducing targetCompactionSizeBytes[%s].",
+ maxRowsPerSegmentLong,
+ targetCompactionSizeBytes
+ )
+ );
Preconditions.checkState(maxRowsPerSegment > 0, "Negative maxRowsPerSegment[%s]", maxRowsPerSegment);
log.info(
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java
index 5117c1a..283dee8 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java
@@ -62,6 +62,7 @@ import org.apache.druid.jackson.DefaultObjectMapper;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.Intervals;
+import org.apache.druid.java.util.common.Pair;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.java.util.common.granularity.Granularity;
@@ -1135,6 +1136,27 @@ public class CompactionTaskTest
.build();
}
+ @Test
+ public void testHugeTargetCompactionSize()
+ {
+ final PartitionConfigurationManager manager = new PartitionConfigurationManager(Long.MAX_VALUE, TUNING_CONFIG);
+ final TestIndexIO indexIO = (TestIndexIO) toolbox.getIndexIO();
+ final Map<File, QueryableIndex> queryableIndexMap = indexIO.getQueryableIndexMap();
+ final List<Pair<QueryableIndex, DataSegment>> segments = new ArrayList<>();
+
+ for (Entry<DataSegment, File> entry : segmentMap.entrySet()) {
+ final DataSegment segment = entry.getKey();
+ final File file = entry.getValue();
+ segments.add(Pair.of(Preconditions.checkNotNull(queryableIndexMap.get(file)), segment));
+ }
+
+ expectedException.expect(ArithmeticException.class);
+ expectedException.expectMessage(
+ CoreMatchers.startsWith("Estimated maxRowsPerSegment[922337203685477632] is out of integer value range.")
+ );
+ manager.computeTuningConfig(segments);
+ }
+
private static List<DimensionsSpec> getExpectedDimensionsSpecForAutoGeneration(boolean keepSegmentGranularity)
{
if (keepSegmentGranularity) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org