You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2019/01/07 02:19:38 UTC
[kylin] branch master updated: KYLIN-3722 Disable limit push down
for multi olapcontext
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new d5bfbb1 KYLIN-3722 Disable limit push down for multi olapcontext
d5bfbb1 is described below
commit d5bfbb1293ab549a3182ddda761ad5023f98d175
Author: hit-lacus <hi...@126.com>
AuthorDate: Sun Jan 6 16:45:15 2019 +0800
KYLIN-3722 Disable limit push down for multi olapcontext
---
.../org/apache/kylin/common/KylinConfigBase.java | 4 ++++
.../org/apache/kylin/query/ITKylinQueryTest.java | 17 ++++++++------
.../src/test/resources/query/sql_limit/query06.sql | 26 ++++++++++++++++++++++
.../apache/kylin/query/relnode/OLAPContext.java | 2 ++
.../apache/kylin/query/relnode/OLAPLimitRel.java | 3 ++-
.../org/apache/kylin/query/relnode/OLAPRel.java | 9 ++++++--
6 files changed, 51 insertions(+), 10 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b63062e..0275001 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1929,4 +1929,8 @@ abstract public class KylinConfigBase implements Serializable {
public String getJdbcSourceAdaptor() {
return getOptional("kylin.source.jdbc.adaptor");
}
+
+ public boolean isLimitPushDownEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.storage.limit-push-down-enable", TRUE));
+ }
}
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 4fdc68e..1e7bde4 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -282,8 +282,9 @@ public class ITKylinQueryTest extends KylinTestBase {
@Test
public void testPreciselyDistinctCountRollupQuery() throws Exception {
// the "inner" test cube uses "SegmentAppendTrieDictBuilder" which doesn't support rollup.
- if("left".equalsIgnoreCase(joinType)) {
- execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_distinct_precisely_rollup", null, true);
+ if ("left".equalsIgnoreCase(joinType)) {
+ execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_distinct_precisely_rollup", null,
+ true);
}
}
@@ -373,13 +374,18 @@ public class ITKylinQueryTest extends KylinTestBase {
new File(getQueryFolderPrefix() + "src/test/resources/query/sql_limit"), ".sql");
for (File sqlFile : sqlFiles) {
runSQL(sqlFile, false, false);
- assertTrue(checkFinalPushDownLimit());
+ if (sqlFile.getAbsolutePath().contains("query06.sql")) {
+ assertTrue(!checkFinalPushDownLimit());
+ } else {
+ assertTrue(checkFinalPushDownLimit());
+ }
}
}
@Test
public void testLimitCorrectness() throws Exception {
this.execLimitAndValidate(getQueryFolderPrefix() + "src/test/resources/query/sql");
+
}
@Test
@@ -463,8 +469,6 @@ public class ITKylinQueryTest extends KylinTestBase {
execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_values", null, true);
}
-
-
@Test
public void testPlan() throws Exception {
String originProp = System.getProperty("calcite.debug");
@@ -473,5 +477,4 @@ public class ITKylinQueryTest extends KylinTestBase {
if (originProp == null || "false".equals(originProp))
System.setProperty("calcite.debug", "false");
}
-}
-
\ No newline at end of file
+}
\ No newline at end of file
diff --git a/kylin-it/src/test/resources/query/sql_limit/query06.sql b/kylin-it/src/test/resources/query/sql_limit/query06.sql
new file mode 100644
index 0000000..353fb60
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_limit/query06.sql
@@ -0,0 +1,26 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT sum(price)
+FROM test_kylin_fact
+WHERE CAL_DT >= '2012-02-16' and CAL_DT <= '2012-03-16' and LEAF_CATEG_ID in
+(
+ SELECT DISTINCT LEAF_CATEG_ID
+ FROM TEST_CATEGORY_GROUPINGS
+ WHERE META_CATEG_NAME is not null
+) limit 10
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
index 4daaaab..340967c 100755
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
@@ -135,6 +135,7 @@ public class OLAPContext {
public boolean hasWindow = false;
public boolean groupByExpression = false; // checkout if group by column has operator
public boolean afterOuterAggregate = false;
+ public boolean disableLimitPushdown = !KylinConfig.getInstanceFromEnv().isLimitPushDownEnabled();
// cube metadata
public IRealization realization;
@@ -293,6 +294,7 @@ public class OLAPContext {
}
fixedModel = false;
}
+
public void bindVariable(DataContext dataContext) {
bindVariable(this.filter, dataContext);
}
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
index 7ec489b..1d0654c 100755
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
@@ -82,7 +82,8 @@ public class OLAPLimitRel extends SingleRel implements OLAPRel {
// ignore limit after having clause
// ignore limit after another limit, e.g. select A, count(*) from (select A,B from fact group by A,B limit 100) limit 10
// ignore limit after outer aggregate, e.g. select count(1) from (select A,B from fact group by A,B ) limit 10
- if (!context.afterHavingClauseFilter && !context.afterLimit && !context.afterOuterAggregate) {
+ if (!context.afterHavingClauseFilter && !context.afterLimit && !context.afterOuterAggregate
+ && !context.disableLimitPushdown) {
Number limitValue = (Number) (((RexLiteral) localFetch).getValue());
int limit = limitValue.intValue();
this.context.storageContext.setLimit(limit);
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPRel.java
index 95f6693..a10ba25 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPRel.java
@@ -106,6 +106,10 @@ public interface OLAPRel extends RelNode {
public void allocateContext() {
OLAPContext context = new OLAPContext(ctxSeq++);
+ if (!ctxStack.isEmpty()) {
+ ctxStack.peek().disableLimitPushdown = true;
+ context.disableLimitPushdown = true;
+ }
ctxStack.push(context);
OLAPContext.registerContext(context);
setNewOLAPContextRequired(false);
@@ -224,8 +228,9 @@ public interface OLAPRel extends RelNode {
}
@Override
- public EnumerableRel.Result visitChild(EnumerableRel parent, int ordinal, EnumerableRel child, EnumerableRel.Prefer prefer) {
-
+ public EnumerableRel.Result visitChild(EnumerableRel parent, int ordinal, EnumerableRel child,
+ EnumerableRel.Prefer prefer) {
+
if (calciteDebug) {
OLAPContext context;
if (child instanceof OLAPRel)