You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/01/10 03:44:32 UTC
kylin git commit: KYLIN-2348 allow multiple models serving one query
Repository: kylin
Updated Branches:
refs/heads/master 00c8f31e6 -> 3a36c66a1
KYLIN-2348 allow multiple models serving one query
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3a36c66a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3a36c66a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3a36c66a
Branch: refs/heads/master
Commit: 3a36c66a1c2bbb372b0a3f3243f181332d7ed882
Parents: 00c8f31
Author: Li Yang <li...@apache.org>
Authored: Tue Jan 10 11:32:58 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Tue Jan 10 11:44:17 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/Dictionary.java | 2 -
.../apache/kylin/query/ITKylinQueryTest.java | 16 +++++--
.../org/apache/kylin/query/KylinTestBase.java | 1 +
.../src/test/resources/query/sql/query101.sql | 18 +++++++
.../src/test/resources/query/sql/query102.sql | 18 +++++++
.../src/test/resources/query/sql/query103.sql | 18 +++++++
.../resources/query/sql_multi_model/query01.sql | 49 ++++++++++++++++++++
.../relnode/OLAPToEnumerableConverter.java | 18 +++----
.../kylin/query/routing/ModelChooser.java | 43 +++++++++++++----
.../apache/kylin/query/schema/OLAPTable.java | 8 ++--
10 files changed, 161 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java b/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
index 03996a7..1e172bc 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
@@ -254,6 +254,4 @@ abstract public class Dictionary<T> implements Serializable {
*/
public abstract void readFields(DataInput in) throws IOException;
-
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 840a03f..aaf0925 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -47,7 +47,7 @@ import org.junit.rules.ExpectedException;
import com.google.common.collect.Maps;
-@Ignore("KylinQueryTest is contained by ITCombinationTest")
+//@Ignore("KylinQueryTest is contained by ITCombinationTest")
public class ITKylinQueryTest extends KylinTestBase {
@Rule
@@ -149,7 +149,7 @@ public class ITKylinQueryTest extends KylinTestBase {
@Test
public void testSingleRunQuery() throws Exception {
- String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/temp/sample.sql";
+ String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql_multi_model/query01.sql";
File sqlFile = new File(queryFileName);
if (sqlFile.exists()) {
@@ -270,7 +270,17 @@ public class ITKylinQueryTest extends KylinTestBase {
this.batchExecuteQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_intersect_count");
}
}
-
+
+
+ @Test
+ public void testMultiModelQuery() throws Exception {
+ if ("left".equalsIgnoreCase(joinType)) {
+ joinType = "default";
+ execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_multi_model", null, true);
+ joinType = "left";
+ }
+ }
+
@Test
public void testDimDistinctCountQuery() throws Exception {
execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_distinct_dim", null, true);
http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index 4f1f591..ae69f09 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -335,6 +335,7 @@ public class KylinTestBase {
printInfo("Query Result from H2 - " + queryName);
H2Connection h2Conn = new H2Connection(h2Connection, null);
h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory());
+ h2Conn.getConfig().setFeature(DatabaseConfig.FEATURE_DATATYPE_WARNING, false);
executeQuery(h2Conn, queryName, sql, needSort);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/kylin-it/src/test/resources/query/sql/query101.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query101.sql b/kylin-it/src/test/resources/query/sql/query101.sql
index fb42bca..765d26d 100644
--- a/kylin-it/src/test/resources/query/sql/query101.sql
+++ b/kylin-it/src/test/resources/query/sql/query101.sql
@@ -1,3 +1,21 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
select meta_categ_name, count(1) as cnt, sum(price) as GMV
http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/kylin-it/src/test/resources/query/sql/query102.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query102.sql b/kylin-it/src/test/resources/query/sql/query102.sql
index bd1e15e..6073c98 100644
--- a/kylin-it/src/test/resources/query/sql/query102.sql
+++ b/kylin-it/src/test/resources/query/sql/query102.sql
@@ -1,3 +1,21 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
select meta_categ_name, count(1) as cnt, sum(price) as GMV
http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/kylin-it/src/test/resources/query/sql/query103.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query103.sql b/kylin-it/src/test/resources/query/sql/query103.sql
index c5f9bf9..6c8ce9e 100644
--- a/kylin-it/src/test/resources/query/sql/query103.sql
+++ b/kylin-it/src/test/resources/query/sql/query103.sql
@@ -1,3 +1,21 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
select meta_categ_name, count(1) as cnt, sum(price) as GMV
http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/kylin-it/src/test/resources/query/sql_multi_model/query01.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_multi_model/query01.sql b/kylin-it/src/test/resources/query/sql_multi_model/query01.sql
new file mode 100644
index 0000000..cb0e2f2
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_multi_model/query01.sql
@@ -0,0 +1,49 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select
+ l.cal_dt
+ , sum(left_join_gvm) as left_join_sum
+ , sum(inner_join_gvm) as inner_join_sum
+from
+ (
+ select test_kylin_fact.cal_dt, sum(price) as left_join_gvm
+
+ from test_kylin_fact
+ left JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ left JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+
+ group by test_kylin_fact.cal_dt
+ ) l
+ inner join
+ (
+ select test_kylin_fact.cal_dt, sum(price) as inner_join_gvm
+
+ from test_kylin_fact
+ inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+
+ group by test_kylin_fact.cal_dt
+ ) i
+ on l.cal_dt = i.cal_dt
+group by
+ l.cal_dt
http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
index 8f01a74..0468c91 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
@@ -18,6 +18,7 @@
package org.apache.kylin.query.relnode;
+import java.util.IdentityHashMap;
import java.util.List;
import java.util.Set;
@@ -79,19 +80,11 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab
// identify model
List<OLAPContext> contexts = listContextsHavingScan();
- Set<IRealization> candidates = ModelChooser.selectModel(contexts);
+ IdentityHashMap<OLAPContext, Set<IRealization>> candidates = ModelChooser.selectModel(contexts);
// identify realization for each context
- for (OLAPContext context : OLAPContext.getThreadLocalContexts()) {
-
- // Context has no table scan is created by OLAPJoinRel which looks like
- // (sub-query) as A join (sub-query) as B
- // No realization needed for such context.
- if (context.firstTableScan == null) {
- continue;
- }
-
- IRealization realization = QueryRouter.selectRealization(context, candidates);
+ for (OLAPContext context : contexts) {
+ IRealization realization = QueryRouter.selectRealization(context, candidates.get(context));
context.realization = realization;
doAccessControl(context);
}
@@ -115,6 +108,9 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab
}
private List<OLAPContext> listContextsHavingScan() {
+ // Context has no table scan is created by OLAPJoinRel which looks like
+ // (sub-query) as A join (sub-query) as B
+ // No realization needed for such context.
int size = OLAPContext.getThreadLocalContexts().size();
List<OLAPContext> result = Lists.newArrayListWithCapacity(size);
for (int i = 0; i < size; i++) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
index ccb2900..2517dc5 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
@@ -19,6 +19,7 @@
package org.apache.kylin.query.routing;
import java.util.Comparator;
+import java.util.IdentityHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -39,14 +40,39 @@ import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.relnode.OLAPTableScan;
import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
public class ModelChooser {
- // select a model that satisfies all the contexts
- public static Set<IRealization> selectModel(List<OLAPContext> contexts) {
+ // select models for given contexts, return realization candidates for each context
+ public static IdentityHashMap<OLAPContext, Set<IRealization>> selectModel(List<OLAPContext> contexts) {
+
+ IdentityHashMap<OLAPContext, Set<IRealization>> candidates = new IdentityHashMap<>();
+
+ // attempt one model for all contexts
+ Set<IRealization> reals = attemptSelectModel(contexts);
+ if (reals != null) {
+ for (OLAPContext ctx : contexts) {
+ candidates.put(ctx, reals);
+ }
+ return candidates;
+ }
+
+ // try different model for different context
+ for (OLAPContext ctx : contexts) {
+ reals = attemptSelectModel(ImmutableList.of(ctx));
+ if (reals == null)
+ throw new NoRealizationFoundException("No model found for" + toErrorMsg(ctx));
+
+ candidates.put(ctx, reals);
+ }
+ return candidates;
+ }
+
+ private static Set<IRealization> attemptSelectModel(List<OLAPContext> contexts) {
Map<DataModelDesc, Set<IRealization>> modelMap = makeOrderedModelMap(contexts);
for (DataModelDesc model : modelMap.keySet()) {
@@ -57,17 +83,14 @@ public class ModelChooser {
return modelMap.get(model);
}
}
-
- throw new NoRealizationFoundException("No model found for" + toErrorMsg(contexts));
+ return null;
}
- private static String toErrorMsg(List<OLAPContext> contexts) {
+ private static String toErrorMsg(OLAPContext ctx) {
StringBuilder buf = new StringBuilder();
- for (OLAPContext ctx : contexts) {
- buf.append(", ").append(ctx.firstTableScan);
- for (JoinDesc join : ctx.joins)
- buf.append(", ").append(join);
- }
+ buf.append(ctx.firstTableScan);
+ for (JoinDesc join : ctx.joins)
+ buf.append(", ").append(join);
return buf.toString();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
index 85996f4..fe6a806 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
@@ -186,14 +186,14 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
//to avoid overflow we upgrade x's type to long
//this includes checking two parts:
//1. sum measures in cubes:
- HashSet<ColumnDesc> updateColumns = Sets.newHashSet();
+ HashSet<ColumnDesc> upgradeCols = Sets.newHashSet();
for (MeasureDesc m : mgr.listEffectiveMeasures(olapSchema.getProjectName(), sourceTable.getIdentity())) {
if (m.getFunction().isSum()) {
FunctionDesc func = m.getFunction();
if (func.getReturnDataType() != func.getRewriteFieldType() && //
func.getReturnDataType().isBigInt() && //
func.getRewriteFieldType().isIntegerFamily()) {
- updateColumns.add(func.getParameter().getColRefs().get(0).getColumnDesc());
+ upgradeCols.add(func.getParameter().getColRefs().get(0).getColumnDesc());
}
}
}
@@ -204,12 +204,12 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
for (String metricColumn : model.getMetrics()) {
TblColRef col = model.findColumn(metricColumn);
if (col.getTable().equals(sourceTable.getIdentity()) && col.getType().isIntegerFamily() && !col.getType().isBigInt())
- updateColumns.add(col.getColumnDesc());
+ upgradeCols.add(col.getColumnDesc());
}
}
}
- for (ColumnDesc upgrade : updateColumns) {
+ for (ColumnDesc upgrade : upgradeCols) {
int index = tableColumns.indexOf(upgrade);
if (index < 0) {
throw new IllegalStateException("Metric column " + upgrade + " is not found in the the project's columns");