You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/04/22 06:12:16 UTC
[47/50] [abbrv] kylin git commit: KYLIN-1556 remove the left hybrid
and refine candidate priority for IT
KYLIN-1556 remove the left hybrid and refine candidate priority for IT
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/30d60e22
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/30d60e22
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/30d60e22
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 30d60e2280969ef9bb58801cc2d2132cbf68b778
Parents: e46c367
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Apr 13 11:09:55 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Thu Apr 21 17:35:42 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/cube/CubeCapabilityChecker.java | 2 +-
.../kylin/metadata/model/FunctionDesc.java | 12 +++++++++++-
.../metadata/realization/CapabilityResult.java | 2 +-
.../hybrid/test_kylin_hybrid_left_join.json | 14 --------------
.../localmeta/project/default.json | 5 -----
.../apache/kylin/query/ITCombinationTest.java | 12 ++++++++++++
.../org/apache/kylin/query/ITIIQueryTest.java | 8 +-------
.../org/apache/kylin/query/ITKylinQueryTest.java | 11 +++++++++++
.../apache/kylin/query/routing/Candidate.java | 19 ++++++++++++-------
9 files changed, 49 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/30d60e22/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index e21dc2b..bd45d07 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -177,7 +177,7 @@ public class CubeCapabilityChecker {
// calcite can do aggregation from columns on-the-fly
List<TblColRef> neededCols = functionDesc.getParameter().getColRefs();
- if (neededCols.size() > 0 && cubeDesc.listDimensionColumnsIncludingDerived().containsAll(neededCols)) {
+ if (neededCols.size() > 0 && cubeDesc.listDimensionColumnsIncludingDerived().containsAll(neededCols) && FunctionDesc.BUILT_IN_AGGREGATIONS.contains(functionDesc.getExpression())) {
result.influences.add(new CapabilityResult.DimensionAsMeasure(functionDesc));
it.remove();
continue;
http://git-wip-us.apache.org/repos/asf/kylin/blob/30d60e22/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index 9e3effb..f3a81d6 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -21,6 +21,7 @@ package org.apache.kylin.metadata.model;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
+import java.util.Set;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.measure.MeasureTypeFactory;
@@ -31,6 +32,7 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
/**
*/
@@ -41,6 +43,14 @@ public class FunctionDesc {
public static final String FUNC_MIN = "MIN";
public static final String FUNC_MAX = "MAX";
public static final String FUNC_COUNT = "COUNT";
+ public static final Set<String> BUILT_IN_AGGREGATIONS = Sets.newHashSet();
+
+ static {
+ BUILT_IN_AGGREGATIONS.add(FUNC_COUNT);
+ BUILT_IN_AGGREGATIONS.add(FUNC_MAX);
+ BUILT_IN_AGGREGATIONS.add(FUNC_MIN);
+ BUILT_IN_AGGREGATIONS.add(FUNC_SUM);
+ }
public static final String PARAMETER_TYPE_CONSTANT = "constant";
public static final String PARAMETER_TYPE_COLUMN = "column";
@@ -67,7 +77,7 @@ public class FunctionDesc {
ArrayList<TblColRef> colRefs = Lists.newArrayList();
for (ParameterDesc p = parameter; p != null; p = p.getNextParameter()) {
if (p.isColumnType()) {
- ColumnDesc sourceColumn = findColumn(factTable,lookupTables,p.getValue());
+ ColumnDesc sourceColumn = findColumn(factTable, lookupTables, p.getValue());
TblColRef colRef = new TblColRef(sourceColumn);
colRefs.add(colRef);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/30d60e22/core-metadata/src/main/java/org/apache/kylin/metadata/realization/CapabilityResult.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/CapabilityResult.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/CapabilityResult.java
index 3b55728..2241e6a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/CapabilityResult.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/CapabilityResult.java
@@ -53,7 +53,7 @@ public class CapabilityResult {
@Override
public double suggestCostMultiplier() {
- return 1.0;
+ return 1;
}
public FunctionDesc getMeasureFunction() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/30d60e22/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json
deleted file mode 100644
index 3a18f87..0000000
--- a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
- "uuid": "5ca78590-64b6-4367-8fb5-7500eb95fd9c",
-
- "name": "test_kylin_hybrid_left_join",
- "realizations": [
- {
- "type": "CUBE",
- "realization": "test_kylin_cube_with_slr_left_join_empty"
- }
- ],
- "cost": 100,
- "last_modified": 1420016227424,
- "create_time": null
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/30d60e22/examples/test_case_data/localmeta/project/default.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/project/default.json b/examples/test_case_data/localmeta/project/default.json
index 6359ff5..41ac6da 100644
--- a/examples/test_case_data/localmeta/project/default.json
+++ b/examples/test_case_data/localmeta/project/default.json
@@ -39,11 +39,6 @@
"realization": "test_streaming_table_cube"
},
{
- "name": "test_kylin_hybrid_left_join",
- "type": "HYBRID",
- "realization": "test_kylin_hybrid_left_join"
- },
- {
"name": "test_kylin_hybrid_inner_join",
"type": "HYBRID",
"realization": "test_kylin_hybrid_inner_join"
http://git-wip-us.apache.org/repos/asf/kylin/blob/30d60e22/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
index 1845587..4191462 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
@@ -21,7 +21,10 @@ package org.apache.kylin.query;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Map;
+import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.query.routing.Candidate;
import org.apache.kylin.storage.hbase.HBaseStorage;
import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
import org.junit.AfterClass;
@@ -29,6 +32,8 @@ import org.junit.BeforeClass;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+import com.google.common.collect.Maps;
+
/**
*/
@RunWith(Parameterized.class)
@@ -36,6 +41,12 @@ public class ITCombinationTest extends ITKylinQueryTest {
@BeforeClass
public static void setUp() throws SQLException {
+ Map<RealizationType, Integer> priorities = Maps.newHashMap();
+ priorities.put(RealizationType.INVERTED_INDEX, 2);
+ priorities.put(RealizationType.HYBRID, 0);
+ priorities.put(RealizationType.CUBE, 0);
+ Candidate.setPriorities(priorities);
+
printInfo("setUp in ITCombinationTest");
}
@@ -44,6 +55,7 @@ public class ITCombinationTest extends ITKylinQueryTest {
printInfo("tearDown in ITCombinationTest");
clean();
HBaseStorage.overwriteStorageQuery = null;
+ Candidate.restorePriorities();
}
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/30d60e22/kylin-it/src/test/java/org/apache/kylin/query/ITIIQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITIIQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITIIQueryTest.java
index 67033dd..3f470cb 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITIIQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITIIQueryTest.java
@@ -46,18 +46,12 @@ public class ITIIQueryTest extends ITKylinQueryTest {
priorities.put(RealizationType.HYBRID, 1);
priorities.put(RealizationType.CUBE, 2);
Candidate.setPriorities(priorities);
-
}
@AfterClass
public static void tearDown() throws Exception {
ITKylinQueryTest.tearDown();//invoke super class
-
- Map<RealizationType, Integer> priorities = Maps.newHashMap();
- priorities.put(RealizationType.INVERTED_INDEX, 1);
- priorities.put(RealizationType.CUBE, 0);
- priorities.put(RealizationType.HYBRID, 0);
- Candidate.setPriorities(priorities);
+ Candidate.restorePriorities();
}
@Parameterized.Parameters
http://git-wip-us.apache.org/repos/asf/kylin/blob/30d60e22/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 09fcf4b..7de0484 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -23,14 +23,18 @@ import static org.junit.Assert.assertTrue;
import java.io.File;
import java.sql.DriverManager;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
+import com.google.common.collect.Maps;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.query.enumerator.OLAPQuery;
import org.apache.kylin.query.relnode.OLAPContext;
+import org.apache.kylin.query.routing.Candidate;
import org.apache.kylin.query.schema.OLAPSchemaFactory;
import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
import org.dbunit.database.DatabaseConnection;
@@ -45,6 +49,12 @@ public class ITKylinQueryTest extends KylinTestBase {
@BeforeClass
public static void setUp() throws Exception {
+ Map<RealizationType, Integer> priorities = Maps.newHashMap();
+ priorities.put(RealizationType.INVERTED_INDEX, 2);
+ priorities.put(RealizationType.HYBRID, 0);
+ priorities.put(RealizationType.CUBE, 0);
+ Candidate.setPriorities(priorities);
+
printInfo("setUp in KylinQueryTest");
joinType = "left";
@@ -53,6 +63,7 @@ public class ITKylinQueryTest extends KylinTestBase {
@AfterClass
public static void tearDown() throws Exception {
+ Candidate.restorePriorities();
printInfo("tearDown in KylinQueryTest");
clean();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/30d60e22/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/Candidate.java b/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
index 9be8f5e..28c60a8 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
@@ -29,18 +29,23 @@ import com.google.common.collect.Maps;
public class Candidate implements Comparable<Candidate> {
- static final Map<RealizationType, Integer> PRIORITIES = Maps.newHashMap();
+ static Map<RealizationType, Integer> DEFAULT_PRIORITIES = Maps.newHashMap();
+ static Map<RealizationType, Integer> PRIORITIES = DEFAULT_PRIORITIES;
static {
- PRIORITIES.put(RealizationType.HYBRID, 0);
- PRIORITIES.put(RealizationType.CUBE, 1);
- PRIORITIES.put(RealizationType.INVERTED_INDEX, 2);
+ DEFAULT_PRIORITIES.put(RealizationType.HYBRID, 0);
+ DEFAULT_PRIORITIES.put(RealizationType.CUBE, 1);
+ DEFAULT_PRIORITIES.put(RealizationType.INVERTED_INDEX, 2);
}
/** for test only */
public static void setPriorities(Map<RealizationType, Integer> priorities) {
- PRIORITIES.clear();
- PRIORITIES.putAll(priorities);
+ PRIORITIES = priorities;
+ }
+
+ /** for test only */
+ public static void restorePriorities() {
+ PRIORITIES = DEFAULT_PRIORITIES;
}
// ============================================================================
@@ -87,7 +92,7 @@ public class Candidate implements Comparable<Candidate> {
if (comp != 0) {
return comp;
}
-
+
return 0;
}