You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/05/10 02:45:17 UTC
[2/2] kylin git commit: KYLIN-2598 Should not translate filter to a
in-clause filter with too many elements
KYLIN-2598 Should not translate filter to a in-clause filter with too many elements
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cb472167
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cb472167
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cb472167
Branch: refs/heads/2.0.x
Commit: cb472167ac627ecc01a089fd126facdb6e004a38
Parents: 4b32c58
Author: lidongsjtu <li...@apache.org>
Authored: Wed May 10 10:44:58 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Wed May 10 10:44:58 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 4 +++
.../kylin/dict/BuiltInFunctionTransformer.java | 10 +++++-
.../test/resources/query/sql_like/query01.sql | 6 ++--
.../test/resources/query/sql_like/query05.sql | 33 ++++++++++++++++++++
.../test/resources/query/sql_like/query06.sql | 33 ++++++++++++++++++++
.../test/resources/query/sql_like/query22.sql | 31 ++++++++++++++++++
.../test/resources/query/sql_like/query23.sql | 31 ++++++++++++++++++
7 files changed, 144 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/cb472167/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 760b7da..2404a1e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -829,6 +829,10 @@ abstract public class KylinConfigBase implements Serializable {
return value > 0 ? value : Long.MAX_VALUE;
}
+ public int getTranslatedInClauseMaxSize() {
+ return Integer.parseInt(getOptional("kylin.query.translated-in-clause-max-size", "1024"));
+ }
+
public int getLargeQueryThreshold() {
return Integer.parseInt(getOptional("kylin.query.large-query-threshold", String.valueOf(1000000)));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cb472167/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
index 6ef2f96..4d7b15d 100755
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
@@ -21,6 +21,7 @@ package org.apache.kylin.dict;
import java.util.Collection;
import java.util.ListIterator;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.dimension.IDimensionEncodingMap;
import org.apache.kylin.metadata.filter.BuiltInFunctionTupleFilter;
@@ -93,12 +94,19 @@ public class BuiltInFunctionTransformer implements ITupleFilterTransformer {
translated.addChild(new ColumnTupleFilter(columnRef));
try {
+ int translatedInClauseMaxSize = KylinConfig.getInstanceFromEnv().getTranslatedInClauseMaxSize();
+
for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
Object dictVal = dict.getValueFromId(i);
if ((Boolean) builtInFunctionTupleFilter.invokeFunction(dictVal)) {
translated.addChild(new ConstantTupleFilter(dictVal));
+
+ if (translated.getChildren().size() > translatedInClauseMaxSize) {
+ return null;
+ }
}
}
+
} catch (Exception e) {
logger.debug(e.getMessage());
return null;
@@ -175,4 +183,4 @@ public class BuiltInFunctionTransformer implements ITupleFilterTransformer {
}
return translated;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cb472167/kylin-it/src/test/resources/query/sql_like/query01.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query01.sql b/kylin-it/src/test/resources/query/sql_like/query01.sql
index e1f131d..e3a05ca 100644
--- a/kylin-it/src/test/resources/query/sql_like/query01.sql
+++ b/kylin-it/src/test/resources/query/sql_like/query01.sql
@@ -16,7 +16,7 @@
-- limitations under the License.
--
-select lstg_format_name as lstg_format_name, count(*) as cnt
+select META_CATEG_NAME as META_CATEG_NAME, count(*) as cnt
from test_kylin_fact
inner JOIN edw.test_cal_dt as test_cal_dt
@@ -27,5 +27,5 @@ inner JOIN edw.test_cal_dt as test_cal_dt
ON test_kylin_fact.lstg_site_id = test_sites.site_id
-where lstg_format_name like '%BIN%'
-group by lstg_format_name
\ No newline at end of file
+where META_CATEG_NAME like '%ab%'
+group by META_CATEG_NAME
http://git-wip-us.apache.org/repos/asf/kylin/blob/cb472167/kylin-it/src/test/resources/query/sql_like/query05.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query05.sql b/kylin-it/src/test/resources/query/sql_like/query05.sql
new file mode 100644
index 0000000..6ac9758
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query05.sql
@@ -0,0 +1,33 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select upper(meta_categ_name) as META_CATEG_NAME, count(*) as cnt
+
+
+from test_kylin_fact
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+
+where lower(meta_categ_name)='baby' and substring(meta_categ_name,1,3) in ('Bab') and upper(meta_categ_name) > 'AAAA' and
+lower(meta_categ_name) like '%b%' and char_length(meta_categ_name) < 10 and char_length(meta_categ_name) > 3 and meta_categ_name||'a'='Babya'
+group by meta_categ_name
http://git-wip-us.apache.org/repos/asf/kylin/blob/cb472167/kylin-it/src/test/resources/query/sql_like/query06.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query06.sql b/kylin-it/src/test/resources/query/sql_like/query06.sql
new file mode 100644
index 0000000..6417584
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query06.sql
@@ -0,0 +1,33 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+-- test case for KYLIN-1954
+
+select META_CATEG_NAME as META_CATEG_NAME, count(*) as cnt
+
+ from test_kylin_fact
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+
+where META_CATEG_NAME like '%ab%' and META_CATEG_NAME > 'A'
+group by META_CATEG_NAME
http://git-wip-us.apache.org/repos/asf/kylin/blob/cb472167/kylin-it/src/test/resources/query/sql_like/query22.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query22.sql b/kylin-it/src/test/resources/query/sql_like/query22.sql
new file mode 100644
index 0000000..a962d3f
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query22.sql
@@ -0,0 +1,31 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select META_CATEG_NAME as META_CATEG_NAME, count(*) as cnt
+
+ from test_kylin_fact
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+
+where META_CATEG_NAME not like '%ab%'
+group by META_CATEG_NAME
http://git-wip-us.apache.org/repos/asf/kylin/blob/cb472167/kylin-it/src/test/resources/query/sql_like/query23.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query23.sql b/kylin-it/src/test/resources/query/sql_like/query23.sql
new file mode 100644
index 0000000..b3a78c2
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query23.sql
@@ -0,0 +1,31 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select META_CATEG_NAME as META_CATEG_NAME, count(*) as cnt
+
+ from test_kylin_fact
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+
+where not(META_CATEG_NAME not like '%ab%')
+group by META_CATEG_NAME