You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/11/19 08:32:46 UTC
[2/3] kylin git commit: KYLIN-2201 fix combination check overflow
error
KYLIN-2201 fix combination check overflow error
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a1bc8353
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a1bc8353
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a1bc8353
Branch: refs/heads/v1.6.0-rc2
Commit: a1bc83537b0a62af1f451db96c0c253c143458bd
Parents: 791f570
Author: Roger Shi <ro...@hotmail.com>
Authored: Thu Nov 17 11:07:17 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sat Nov 19 15:56:53 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/model/CubeDesc.java | 6 +-
.../validation/rule/AggregationGroupRule.java | 4 +-
.../kylin/cube/AggregationGroupRuleTest.java | 11 +
.../org/apache/kylin/cube/CubeDescTest.java | 7 +
.../ut_cube_desc_combination_int_overflow.json | 398 +++++++++++++++++++
5 files changed, 421 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1bc8353/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 47e9be8..6cbed80 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -568,7 +568,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
throw new IllegalStateException("Aggregation group " + index + " select rule field not set");
}
- int combination = 1;
+ long combination = 1;
Set<String> includeDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
getDims(includeDims, agg.getIncludes());
@@ -586,7 +586,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
Set<String> jointDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
getDims(jointDimsList, jointDims, agg.getSelectRule().joint_dims);
if (jointDimsList.size() > 0) {
- combination = combination * (1 << jointDimsList.size());
+ combination = combination * (1L << jointDimsList.size());
}
if (!includeDims.containsAll(mandatoryDims) || !includeDims.containsAll(hierarchyDims) || !includeDims.containsAll(jointDims)) {
@@ -607,7 +607,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
normalDims.removeAll(hierarchyDims);
normalDims.removeAll(jointDims);
- combination = combination * (1 << normalDims.size());
+ combination = combination * (1L << normalDims.size());
if (combination > config.getCubeAggrGroupMaxCombination()) {
String msg = "Aggregation group " + index + " has too many combinations, use 'mandatory'/'hierarchy'/'joint' to optimize; or update 'kylin.cube.aggrgroup.max.combination' to a bigger value.";
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1bc8353/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
index c1a062a..a5cf407 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
@@ -62,7 +62,7 @@ public class AggregationGroupRule implements IValidatorRule<CubeDesc> {
continue;
}
- int combination = 1;
+ long combination = 1;
Set<String> includeDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
if (agg.getIncludes() != null) {
for (String include : agg.getIncludes()) {
@@ -115,7 +115,7 @@ public class AggregationGroupRule implements IValidatorRule<CubeDesc> {
normalDims.removeAll(hierarchyDims);
normalDims.removeAll(jointDims);
- combination = combination * (1 << normalDims.size());
+ combination = combination * (1L << normalDims.size());
if (CollectionUtils.containsAny(mandatoryDims, hierarchyDims)) {
Set<String> intersection = new HashSet<>(mandatoryDims);
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1bc8353/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
index 0ebb249..563e139 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.FileInputStream;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Arrays;
@@ -115,6 +116,16 @@ public class AggregationGroupRuleTest {
assertEquals("Aggregation group 0 joint dimensions has overlap with more than 1 dimensions in same hierarchy: [CATEG_LVL2_NAME, META_CATEG_NAME]", (vContext.getResults()[0].getMessage()));
}
+ @Test
+ public void testCombinationIntOverflow() throws IOException {
+ ValidateContext vContext = new ValidateContext();
+ CubeDesc desc = JsonUtil.readValue(new FileInputStream(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + "/cube_desc/ut_cube_desc_combination_int_overflow.json"), CubeDesc.class);
+
+ IValidatorRule<CubeDesc> rule = getAggregationGroupRule();
+ rule.validate(desc, vContext);
+ assertEquals(1, vContext.getResults().length);
+ }
+
public AggregationGroupRule getAggregationGroupRule() {
AggregationGroupRule rule = new AggregationGroupRule() {
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1bc8353/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
index 417ad46..ca7ae50 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
@@ -193,6 +193,13 @@ public class CubeDescTest extends LocalFileMetadataTestCase {
}
@Test
+ public void testCombinationIntOverflow() throws Exception {
+ thrown.expect(IllegalStateException.class);
+ CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ut_cube_desc_combination_int_overflow");
+ cubeDesc.init(getTestConfig());
+ }
+
+ @Test
public void testSerialize() throws Exception {
CubeDesc desc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc");
String str = JsonUtil.writeValueAsIndentString(desc);
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1bc8353/examples/test_case_data/localmeta/cube_desc/ut_cube_desc_combination_int_overflow.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ut_cube_desc_combination_int_overflow.json b/examples/test_case_data/localmeta/cube_desc/ut_cube_desc_combination_int_overflow.json
new file mode 100644
index 0000000..ec6f8f4
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube_desc/ut_cube_desc_combination_int_overflow.json
@@ -0,0 +1,398 @@
+{
+ "uuid" : "9e89e128-f13e-4209-82d2-973985114793",
+ "last_modified" : 1479207711845,
+ "name" : "ut_cube_desc_combination_int_overflow",
+ "model_name" : "ut_large_dimension_number",
+ "description" : "",
+ "null_string" : null,
+ "dimensions" : [ {
+ "name" : "DEFAULT.WIDE_TABLE.A",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "A",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.B",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "B",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.C",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "C",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.D",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "D",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.E",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "E",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.F",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "F",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.G",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "G",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.H",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "H",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.I",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "I",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.J",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "J",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.K",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "K",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.L",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "L",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.M",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "M",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.N",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "N",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.O",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "O",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.P",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "P",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.Q",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "Q",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.R",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "R",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.S",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "S",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.T",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "T",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.U",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "U",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.V",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "V",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.W",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "W",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.X",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "X",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.Y",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "Y",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.Z",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "Z",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.AA",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "AA",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.AB",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "AB",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.AC",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "AC",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.AD",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "AD",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.AE",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "AE",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.AF",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "AF",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.AG",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "AG",
+ "derived" : null
+ },{
+ "name" : "DEFAULT.WIDE_TABLE.AH",
+ "table" : "DEFAULT.WIDE_TABLE",
+ "column" : "AH",
+ "derived" : null
+ } ],
+ "measures" : [ {
+ "name" : "_COUNT_",
+ "function" : {
+ "expression" : "COUNT",
+ "parameter" : {
+ "type" : "constant",
+ "value" : "1",
+ "next_parameter" : null
+ },
+ "returntype" : "bigint"
+ },
+ "dependent_measure_ref" : null
+ } ],
+ "dictionaries" : [ ],
+ "rowkey" : {
+ "rowkey_columns" : [ {
+ "column" : "A",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "B",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "C",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "D",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "E",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "F",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "G",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "H",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "I",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "J",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "K",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "L",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "M",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "N",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "O",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "P",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "Q",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "R",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "S",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "T",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "U",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "V",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "W",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "X",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "Y",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "Z",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "AA",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "AB",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "AC",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "AD",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "AE",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "AF",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "AG",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }, {
+ "column" : "AH",
+ "encoding" : "dict",
+ "isShardBy" : false,
+ "index" : "eq"
+ }]
+ },
+ "hbase_mapping" : {
+ "column_family" : [ {
+ "name" : "F1",
+ "columns" : [ {
+ "qualifier" : "M",
+ "measure_refs" : [ "_COUNT_" ]
+ } ]
+ } ]
+ },
+ "aggregation_groups" : [ {
+ "includes" : [ "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "AA", "AB", "AC", "AD", "AE", "AF", "AG", "AH"],
+ "select_rule" : {
+ "hierarchy_dims" : [ ],
+ "mandatory_dims" : [ ],
+ "joint_dims" : [ ]
+ }
+ }],
+ "signature" : "4NqJVcVIYRF5PkJ0jS+9Rg==",
+ "notify_list" : [ ],
+ "status_need_notify" : [ "ERROR", "DISCARDED", "SUCCEED" ],
+ "partition_date_start" : 1469923200000,
+ "partition_date_end" : 3153600000000,
+ "auto_merge_time_ranges" : [ 604800000, 2419200000 ],
+ "retention_range" : 0,
+ "engine_type" : 100,
+ "storage_type" : 100,
+ "override_kylin_properties" : {
+ "kylin.cube.algorithm" : "auto",
+ "kylin.cube.aggrgroup.max.combination" : "4096",
+ "kylin.job.cubing.inmem.sampling.percent" : "100",
+ "kylin.hbase.default.compression.codec" : "none"
+ }
+}
\ No newline at end of file