You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2023/05/06 06:59:11 UTC
[kylin] 10/38: KYLIN-5523 [FOLLOW UP] fix some function of cc as join key or filter column
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 5657afc7f2a827419d8b8beda4b294c75f6245e0
Author: Pengfei Zhan <de...@gmail.com>
AuthorDate: Wed Feb 15 21:35:56 2023 +0800
KYLIN-5523 [FOLLOW UP] fix some function of cc as join key or filter column
---
.../org/apache/kylin/common/KylinConfigBase.java | 7 +-
.../kylin-backward-compatibility.properties | 1 +
.../org/apache/kylin/job/JoinedFlatTableTest.java | 242 --------------
.../kylin/metadata/model/JoinedFlatTable.java | 371 ---------------------
.../apache/kylin/rest/service/ModelService.java | 53 ++-
.../kylin/rest/service/ModelServiceTest.java | 32 +-
.../org/apache/kylin/query/util/PushDownUtil.java | 59 +++-
.../apache/kylin/query/util/QueryAliasMatcher.java | 25 +-
.../apache/kylin/query/util/CCOnRealModelTest.java | 11 +-
.../apache/kylin/query/util/PushDownUtilTest.java | 171 +++++++++-
10 files changed, 282 insertions(+), 690 deletions(-)
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 0d4796e6e3..f20dca320b 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -2502,6 +2502,7 @@ public abstract class KylinConfigBase implements Serializable {
public int getCalciteBindableCacheSize() {
return Integer.parseInt(getOptional("kylin.query.calcite.bindable.cache.maxSize", "10"));
}
+
public int getCalciteBindableCacheConcurrencyLevel() {
return Integer.parseInt(getOptional("kylin.query.calcite.bindable.cache.concurrencyLevel", "5"));
}
@@ -3456,7 +3457,7 @@ public abstract class KylinConfigBase implements Serializable {
}
public boolean skipCheckFlatTable() {
- return Boolean.parseBoolean(getOptional("kylin.model.skip-flattable-check", FALSE));
+ return Boolean.parseBoolean(getOptional("kylin.model.skip-check-flattable", FALSE));
}
public boolean isQueryExceptionCacheEnabled() {
@@ -3570,10 +3571,6 @@ public abstract class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(getOptional("kylin.table.fast-reload-enabled", TRUE));
}
- public boolean isSkipCheckFlatTable() {
- return Boolean.parseBoolean(getOptional("kylin.model.skip-check-flattable", FALSE));
- }
-
public boolean isUnitOfWorkSimulationEnabled() {
return Boolean.parseBoolean(getOptional("kylin.env.unitofwork-simulation-enabled", FALSE));
}
diff --git a/src/core-common/src/main/resources/kylin-backward-compatibility.properties b/src/core-common/src/main/resources/kylin-backward-compatibility.properties
index d14394d8dd..fc239ae6a8 100644
--- a/src/core-common/src/main/resources/kylin-backward-compatibility.properties
+++ b/src/core-common/src/main/resources/kylin-backward-compatibility.properties
@@ -28,6 +28,7 @@ kylin.realization.providers=kylin.metadata.realization-providers
kylin.cube.dimension.customEncodingFactories=kylin.metadata.custom-dimension-encodings
kylin.cube.measure.customMeasureType.=kylin.metadata.custom-measure-types.
kap.metadata.semi-automatic-mode=kylin.metadata.semi-automatic-mode
+kylin.model.skip-flattable-check=kylin.model.skip-check-flattable
### Dictionary ###
diff --git a/src/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java b/src/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java
deleted file mode 100644
index 52ab091c31..0000000000
--- a/src/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java
+++ /dev/null
@@ -1,242 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job;
-
-import java.util.List;
-
-import org.apache.calcite.avatica.util.Quoting;
-import org.apache.kylin.common.util.RandomUtil;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.JoinTableDesc;
-import org.apache.kylin.metadata.model.NonEquiJoinCondition;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TableRef;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.model.ComputedColumnDesc;
-import org.apache.kylin.metadata.model.JoinedFlatTable;
-import org.apache.kylin.metadata.model.NDataModel;
-import org.assertj.core.util.Lists;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.google.common.collect.ImmutableBiMap;
-
-public class JoinedFlatTableTest {
-
- private static final String QUOTE = Quoting.DOUBLE_QUOTE.string;
- private NDataModel dataModel = new NDataModel();
-
- @Before
- public void setUp() {
- dataModel.setUuid(RandomUtil.randomUUIDStr());
-
- TableDesc lineOrderTableDesc = new TableDesc();
- lineOrderTableDesc.setUuid("665a66d6-08d6-42b8-9be8-d9a0456ee250");
- lineOrderTableDesc.setName("LINEORDER");
- lineOrderTableDesc.setDatabase("SSB");
- lineOrderTableDesc.setSourceType(9);
- lineOrderTableDesc.setTableType("MANAGED");
-
- ColumnDesc loSuppkeyColDesc = new ColumnDesc();
- loSuppkeyColDesc.setId("1");
- loSuppkeyColDesc.setName("LO_SUPPKEY");
- loSuppkeyColDesc.setDatatype("integer");
- loSuppkeyColDesc.setComment("null");
- loSuppkeyColDesc.setTable(lineOrderTableDesc);
-
- ColumnDesc loTaxColDesc = new ColumnDesc();
- loTaxColDesc.setId("2");
- loTaxColDesc.setName("LO_TAX");
- loTaxColDesc.setDatatype("bigint");
- loTaxColDesc.setComment("null");
- loTaxColDesc.setTable(lineOrderTableDesc);
-
- ColumnDesc loRevenueColDesc = new ColumnDesc();
- loRevenueColDesc.setId("3");
- loRevenueColDesc.setName("LO_REVENUE");
- loRevenueColDesc.setDatatype("bigint");
- loRevenueColDesc.setComment("null");
- loRevenueColDesc.setTable(lineOrderTableDesc);
-
- ColumnDesc cc = new ColumnDesc("4", "PROFIT", "bigint", null, null, null,
- "case when `LINEORDER`.`LO_REVENUE` - `LINEORDER`.`LO_TAX` > 0 then 'LINEORDER' else null end");
- cc.setTable(lineOrderTableDesc);
-
- lineOrderTableDesc.setColumns(new ColumnDesc[] { loSuppkeyColDesc, loTaxColDesc, loRevenueColDesc, cc });
- TableRef lineOrderTableRef = new TableRef(dataModel, "LINEORDER", lineOrderTableDesc, false);
-
- TableDesc supplierTableDesc = new TableDesc();
- supplierTableDesc.setUuid("719e9bf4-82de-40ec-9454-ab43ff94eef4");
- supplierTableDesc.setName("SUPPLIER");
- supplierTableDesc.setDatabase("SSB");
- supplierTableDesc.setSourceType(9);
- supplierTableDesc.setTableType("MANAGED");
-
- ColumnDesc sSuppkey = new ColumnDesc();
- sSuppkey.setId("1");
- sSuppkey.setName("S_SUPPKEY");
- sSuppkey.setDatatype("integer");
- sSuppkey.setComment("null");
- sSuppkey.setTable(supplierTableDesc);
-
- ColumnDesc sCity = new ColumnDesc();
- sCity.setId("2");
- sCity.setName("S_CITY");
- sCity.setDatatype("varchar(4096)");
- sCity.setComment("null");
- sCity.setTable(supplierTableDesc);
-
- supplierTableDesc.setColumns(new ColumnDesc[] { sSuppkey, sCity });
- TableRef supplierTableRef = new TableRef(dataModel, "SUPPLIER", supplierTableDesc, false);
-
- ImmutableBiMap.Builder<Integer, TblColRef> effectiveCols = ImmutableBiMap.builder();
- effectiveCols.put(1, lineOrderTableRef.getColumn("LO_SUPPKEY"));
- effectiveCols.put(2, lineOrderTableRef.getColumn("LO_REVENUE"));
- effectiveCols.put(3, lineOrderTableRef.getColumn("LO_TAX"));
- effectiveCols.put(4, lineOrderTableRef.getColumn("PROFIT"));
- effectiveCols.put(5, supplierTableRef.getColumn("S_SUPPKEY"));
- effectiveCols.put(6, supplierTableRef.getColumn("S_CITY"));
- dataModel.setEffectiveCols(effectiveCols.build());
-
- dataModel.setRootFactTableName("SSB.LINEORDER");
- dataModel.setRootFactTableRef(lineOrderTableRef);
-
- JoinDesc joinDesc = new JoinDesc();
- joinDesc.setType("LEFT");
- joinDesc.setPrimaryKey(new String[] { "LINEORDER.LO_SUPPKEY" });
- joinDesc.setForeignKey(new String[] { "SUPPLIER.S_SUPPKEY" });
- joinDesc.setPrimaryTableRef(lineOrderTableRef);
- joinDesc.setPrimaryKeyColumns(new TblColRef[] { new TblColRef(lineOrderTableRef, loSuppkeyColDesc) });
- joinDesc.setForeignKeyColumns(new TblColRef[] { new TblColRef(supplierTableRef, sSuppkey) });
- JoinTableDesc supplierJoinTableDesc = new JoinTableDesc();
- supplierJoinTableDesc.setTable("SSB.SUPPLIER");
- supplierJoinTableDesc.setAlias("SUPPLIER");
- supplierJoinTableDesc.setKind(NDataModel.TableKind.LOOKUP);
- supplierJoinTableDesc.setTableRef(supplierTableRef);
- supplierJoinTableDesc.setJoin(joinDesc);
-
- dataModel.setJoinTables(Lists.newArrayList(supplierJoinTableDesc));
- dataModel.setFilterCondition("SUPPLIER.S_CITY != 'beijing'");
- ComputedColumnDesc cc1 = new ComputedColumnDesc();
- cc1.setExpression(
- "case when \"LINEORDER\".\"LO_REVENUE\" - \"LINEORDER\".\"LO_TAX\" > 0 then 'LINEORDER' else null end");
- cc1.setInnerExpression(
- "case when `LINEORDER`.`LO_REVENUE` - `LINEORDER`.`LO_TAX` > 0 then 'LINEORDER' else null end");
- cc1.setColumnName("PROFIT");
- cc1.setDatatype("bigint");
- dataModel.setComputedColumnDescs(Lists.newArrayList(cc1));
- }
-
- @Test
- public void testQuoteIdentifierInSqlExpr() {
- String cc = JoinedFlatTable.quoteIdentifierInSqlExpr(dataModel, "LINEORDER.LO_REVENUE-LINEORDER.LO_TAX", QUOTE);
- Assert.assertEquals("\"LINEORDER\".\"LO_REVENUE\"-\"LINEORDER\".\"LO_TAX\"", cc);
-
- String where1 = JoinedFlatTable.quoteIdentifierInSqlExpr(dataModel, "LINEORDER.LO_REVENUE-LINEORDER.LO_TAX>0",
- QUOTE);
- Assert.assertEquals("\"LINEORDER\".\"LO_REVENUE\"-\"LINEORDER\".\"LO_TAX\">0", where1);
-
- String where2 = JoinedFlatTable.quoteIdentifierInSqlExpr(dataModel,
- "LINEORDER.LO_REVENUE>100 AND LINEORDER.LO_TAX>0", QUOTE);
- Assert.assertEquals("\"LINEORDER\".\"LO_REVENUE\">100 AND \"LINEORDER\".\"LO_TAX\">0", where2);
-
- String where3 = JoinedFlatTable.quoteIdentifierInSqlExpr(dataModel,
- "`LINEORDER`.`LO_REVENUE`>100 AND `LINEORDER`.`LO_TAX`>0", QUOTE);
- Assert.assertEquals("\"LINEORDER\".\"LO_REVENUE\">100 AND \"LINEORDER\".\"LO_TAX\">0", where3);
- }
-
- @Test
- public void testGenerateSelectDataStatement() {
- String flatTableSql = JoinedFlatTable.generateSelectDataStatement(dataModel, false);
- String expectedSql = "SELECT \n" //
- + "\"LINEORDER\".\"LO_SUPPKEY\" as \"LINEORDER_LO_SUPPKEY\",\n"
- + "\"LINEORDER\".\"LO_REVENUE\" as \"LINEORDER_LO_REVENUE\",\n"
- + "\"LINEORDER\".\"LO_TAX\" as \"LINEORDER_LO_TAX\",\n"
- + "\"SUPPLIER\".\"S_SUPPKEY\" as \"SUPPLIER_S_SUPPKEY\",\n"
- + "\"SUPPLIER\".\"S_CITY\" as \"SUPPLIER_S_CITY\"\n" //
- + "FROM \n" //
- + "\"SSB\".\"LINEORDER\" as \"LINEORDER\" \n" //
- + "LEFT JOIN \"SSB\".\"SUPPLIER\" as \"SUPPLIER\"\n" //
- + "ON \"SUPPLIER\".\"S_SUPPKEY\"=\"LINEORDER\".\"LO_SUPPKEY\"\n" //
- + "WHERE \n" //
- + "1 = 1\n" //
- + " AND (\"SUPPLIER\".\"S_CITY\" != 'beijing')";
- Assert.assertEquals(expectedSql, flatTableSql.trim());
-
- NonEquiJoinCondition nonEquiJoinCondition = new NonEquiJoinCondition();
- nonEquiJoinCondition.setExpr("SUPPLIER.S_SUPPKEY <> LINEORDER.LO_SUPPKEY AND LINEORDER.LO_SUPPKEY > 10");
- dataModel.getJoinTables().get(0).getJoin().setNonEquiJoinCondition(nonEquiJoinCondition);
- String nonEquiJoinConditionSql = JoinedFlatTable.generateSelectDataStatement(dataModel, false);
- Assert.assertTrue(nonEquiJoinConditionSql.contains(
- "ON \"SUPPLIER\".\"S_SUPPKEY\" <> \"LINEORDER\".\"LO_SUPPKEY\" AND \"LINEORDER\".\"LO_SUPPKEY\" > 10"));
- dataModel.getJoinTables().get(0).getJoin().setNonEquiJoinCondition(null);
- }
-
- @Test
- public void testQuoteIdentifier() {
- List<String> tablePatterns = JoinedFlatTable.getTableNameOrAliasPatterns("KYLIN_SALES");
- String exprTable = "KYLIN_SALES.PRICE * KYLIN_SALES.COUNT";
- String expectedExprTable = "\"KYLIN_SALES\".PRICE * \"KYLIN_SALES\".COUNT";
- String quotedExprTable = JoinedFlatTable.quoteIdentifier(exprTable, QUOTE, "KYLIN_SALES", tablePatterns);
- Assert.assertEquals(expectedExprTable, quotedExprTable);
-
- exprTable = "`KYLIN_SALES`.PRICE * KYLIN_SALES.COUNT";
- expectedExprTable = "\"KYLIN_SALES\".PRICE * \"KYLIN_SALES\".COUNT";
- quotedExprTable = JoinedFlatTable.quoteIdentifier(exprTable, QUOTE, "KYLIN_SALES", tablePatterns);
- Assert.assertEquals(expectedExprTable, quotedExprTable);
-
- exprTable = "KYLIN_SALES.PRICE AS KYLIN_SALES_PRICE * KYLIN_SALES.COUNT AS KYLIN_SALES_COUNT";
- expectedExprTable = "\"KYLIN_SALES\".PRICE AS KYLIN_SALES_PRICE * \"KYLIN_SALES\".COUNT AS KYLIN_SALES_COUNT";
- quotedExprTable = JoinedFlatTable.quoteIdentifier(exprTable, QUOTE, "KYLIN_SALES", tablePatterns);
- Assert.assertEquals(expectedExprTable, quotedExprTable);
-
- exprTable = "(KYLIN_SALES.PRICE AS KYLIN_SALES_PRICE > 1 and KYLIN_SALES.COUNT AS KYLIN_SALES_COUNT > 50)";
- expectedExprTable = "(\"KYLIN_SALES\".PRICE AS KYLIN_SALES_PRICE > 1 and \"KYLIN_SALES\".COUNT AS KYLIN_SALES_COUNT > 50)";
- quotedExprTable = JoinedFlatTable.quoteIdentifier(exprTable, QUOTE, "KYLIN_SALES", tablePatterns);
- Assert.assertEquals(expectedExprTable, quotedExprTable);
-
- List<String> columnPatterns = JoinedFlatTable.getColumnNameOrAliasPatterns("PRICE");
- String expr = "KYLIN_SALES.PRICE * KYLIN_SALES.COUNT";
- String expectedExpr = "KYLIN_SALES.\"PRICE\" * KYLIN_SALES.COUNT";
- String quotedExpr = JoinedFlatTable.quoteIdentifier(expr, QUOTE, "PRICE", columnPatterns);
- Assert.assertEquals(expectedExpr, quotedExpr);
-
- expr = "KYLIN_SALES.PRICE / KYLIN_SALES.COUNT";
- expectedExpr = "KYLIN_SALES.\"PRICE\" / KYLIN_SALES.COUNT";
- quotedExpr = JoinedFlatTable.quoteIdentifier(expr, QUOTE, "PRICE", columnPatterns);
- Assert.assertEquals(expectedExpr, quotedExpr);
-
- expr = "KYLIN_SALES.PRICE AS KYLIN_SALES_PRICE * KYLIN_SALES.COUNT AS KYLIN_SALES_COUNT";
- expectedExpr = "KYLIN_SALES.\"PRICE\" AS KYLIN_SALES_PRICE * KYLIN_SALES.COUNT AS KYLIN_SALES_COUNT";
- quotedExpr = JoinedFlatTable.quoteIdentifier(expr, QUOTE, "PRICE", columnPatterns);
- Assert.assertEquals(expectedExpr, quotedExpr);
-
- expr = "(PRICE > 1 AND COUNT > 50)";
- expectedExpr = "(\"PRICE\" > 1 AND COUNT > 50)";
- quotedExpr = JoinedFlatTable.quoteIdentifier(expr, QUOTE, "PRICE", columnPatterns);
- Assert.assertEquals(expectedExpr, quotedExpr);
-
- expr = "PRICE>1 and `PRICE` < 15";
- expectedExpr = "\"PRICE\">1 and \"PRICE\" < 15";
- quotedExpr = JoinedFlatTable.quoteIdentifier(expr, QUOTE, "PRICE", columnPatterns);
- Assert.assertEquals(expectedExpr, quotedExpr);
- }
-}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinedFlatTable.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinedFlatTable.java
deleted file mode 100644
index b222252d63..0000000000
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinedFlatTable.java
+++ /dev/null
@@ -1,371 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.metadata.model;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.function.Function;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.calcite.avatica.util.Quoting;
-import org.apache.commons.lang3.StringUtils;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-public class JoinedFlatTable {
-
- private static final String DATABASE_AND_TABLE = "%s.%s";
-
- private static final String QUOTE = Quoting.DOUBLE_QUOTE.string;
- private static final String UNDER_LINE = "_";
- private static final String DOT = ".";
- private static final Pattern BACK_TICK_DOT_PATTERN = Pattern
- .compile("`[^\\f\\n\\r\\t\\v]+?`\\.`[^\\f\\n\\r\\t\\v]+?`");
-
- private JoinedFlatTable() {
- }
-
- private static String quote(String identifier) {
- return QUOTE + identifier + QUOTE;
- }
-
- private static String colName(TblColRef col) {
- return col.getTableAlias() + UNDER_LINE + col.getName();
- }
-
- private static String quotedTable(TableDesc table) {
- if (table.getCaseSensitiveDatabase().equals("null")) {
- return quote(table.getCaseSensitiveName().toUpperCase(Locale.ROOT));
- }
- return String.format(Locale.ROOT, DATABASE_AND_TABLE,
- quote(table.getCaseSensitiveDatabase().toUpperCase(Locale.ROOT)),
- quote(table.getCaseSensitiveName().toUpperCase(Locale.ROOT)));
- }
-
- private static String quotedColExpressionInSourceDB(NDataModel modelDesc, TblColRef col) {
- Map<String, ComputedColumnDesc> ccMap = Maps.newHashMap();
- modelDesc.getComputedColumnDescs().forEach(cc -> ccMap.putIfAbsent(cc.getColumnName(), cc));
- if (!col.getColumnDesc().isComputedColumn()) {
- return quote(col.getTableAlias()) + DOT + quote(col.getName());
- }
- return quoteIdentifierInSqlExpr(modelDesc, ccMap.get(col.getName()).getInnerExpression(), QUOTE);
- }
-
- private static String appendEffectiveColumnsStatement(NDataModel modelDesc, List<TblColRef> effectiveColumns,
- boolean singleLine, boolean includeCC, Function<TblColRef, String> namingFunction) {
- final String sep = getSepBySingleLineTag(singleLine);
-
- StringBuilder subSql = new StringBuilder();
- if (effectiveColumns.isEmpty()) {
- subSql.append("1");
- return subSql.toString();
- }
-
- effectiveColumns.forEach(col -> {
- if (includeCC || !col.getColumnDesc().isComputedColumn()) {
- if (subSql.length() > 0) {
- subSql.append(",").append(sep);
- }
-
- String colName = namingFunction != null ? namingFunction.apply(col) : colName(col);
- subSql.append(quotedColExpressionInSourceDB(modelDesc, col)).append(" as ").append(quote(colName));
- }
- });
-
- return subSql.toString();
- }
-
- private static String appendWhereStatement(NDataModel modelDesc, boolean singleLine) {
- final String sep = getSepBySingleLineTag(singleLine);
-
- StringBuilder whereBuilder = new StringBuilder();
- whereBuilder.append("1 = 1").append(sep);
-
- if (StringUtils.isNotEmpty(modelDesc.getFilterCondition())) {
- String quotedFilterCondition = quoteIdentifierInSqlExpr(modelDesc, modelDesc.getFilterCondition(), QUOTE);
- whereBuilder.append(" AND (").append(quotedFilterCondition).append(") ").append(sep); // -> filter condition contains special character may cause bug
- }
-
- return whereBuilder.toString();
- }
-
- private static String appendJoinStatement(NDataModel modelDesc, boolean singleLine) {
- final String sep = getSepBySingleLineTag(singleLine);
-
- StringBuilder subSql = new StringBuilder();
-
- Set<TableRef> dimTableCache = new HashSet<>();
- TableRef rootTable = modelDesc.getRootFactTable();
- subSql.append(quotedTable(modelDesc.getRootFactTable().getTableDesc())).append(" as ")
- .append(quote(rootTable.getAlias())).append(" ").append(sep);
-
- for (JoinTableDesc lookupDesc : modelDesc.getJoinTables()) {
- JoinDesc join = lookupDesc.getJoin();
- if (checkJoinDesc(join)) {
- continue;
- }
- String joinType = join.getType().toUpperCase(Locale.ROOT);
- TableRef dimTable = lookupDesc.getTableRef();
- if (dimTableCache.contains(dimTable)) {
- continue;
- }
- subSql.append(joinType).append(" JOIN ").append(quotedTable(dimTable.getTableDesc())).append(" as ")
- .append(quote(dimTable.getAlias())).append(sep);
- subSql.append("ON ");
-
- if (Objects.nonNull(join.getNonEquiJoinCondition())) {
- subSql.append(quoteIdentifierInSqlExpr(modelDesc, join.getNonEquiJoinCondition().getExpr(), QUOTE));
- } else {
- TblColRef[] pk = join.getPrimaryKeyColumns();
- TblColRef[] fk = join.getForeignKeyColumns();
- if (pk.length != fk.length) {
- throw new RuntimeException(
- String.format(Locale.ROOT, "Invalid join condition of lookup table: %s", lookupDesc));
- }
-
- for (int i = 0; i < pk.length; i++) {
- if (i > 0) {
- subSql.append(" AND ");
- }
- subSql.append(quotedColExpressionInSourceDB(modelDesc, fk[i])).append("=")
- .append(quotedColExpressionInSourceDB(modelDesc, pk[i]));
- }
- }
- subSql.append(sep);
- dimTableCache.add(dimTable);
- }
- return subSql.toString();
- }
-
- private static String getSepBySingleLineTag(boolean singleLine) {
- return singleLine ? " " : "\n";
- }
-
- public static String generateSelectDataStatement(NDataModel modelDesc, boolean singleLine) {
- return generateSelectDataStatement(modelDesc, Lists.newArrayList(modelDesc.getEffectiveCols().values()),
- singleLine, false, true, null);
- }
-
- public static String generateSelectDataStatement(NDataModel modelDesc, List<TblColRef> effectiveColumns,
- boolean singleLine, boolean includeCC, boolean includeFilter, Function<TblColRef, String> namingFunction) {
- final String sep = getSepBySingleLineTag(singleLine);
-
- StringBuilder sql = new StringBuilder("SELECT ").append(sep);
- String columnsStatement = appendEffectiveColumnsStatement(modelDesc, effectiveColumns, singleLine, includeCC,
- namingFunction);
- sql.append(columnsStatement.endsWith(sep) ? columnsStatement : columnsStatement + sep);
- sql.append("FROM ").append(sep);
- String joinStatement = appendJoinStatement(modelDesc, singleLine);
- sql.append(joinStatement.endsWith(sep) ? joinStatement : joinStatement + sep);
- if (includeFilter) {
- sql.append("WHERE ").append(sep);
- sql.append(appendWhereStatement(modelDesc, singleLine));
- }
- return sql.toString();
- }
-
- private static boolean checkJoinDesc(JoinDesc join) {
- return join == null || join.getType().equals("");
- }
-
- private static String getColumnAlias(String tableName, String columnName,
- Map<String, Map<String, String>> tableToColumnsMap) {
- Map<String, String> colToAliasMap = getColToColAliasMapInTable(tableName, tableToColumnsMap);
- if (!colToAliasMap.containsKey(columnName)) {
- return null;
- }
- return colToAliasMap.get(columnName);
- }
-
- private static boolean columnHasAlias(String tableName, String columnName,
- Map<String, Map<String, String>> tableToColumnsMap) {
- Map<String, String> colToAliasMap = getColToColAliasMapInTable(tableName, tableToColumnsMap);
- return colToAliasMap.containsKey(columnName);
- }
-
- private static Map<String, String> getColToColAliasMapInTable(String tableName,
- Map<String, Map<String, String>> tableToColumnsMap) {
- if (tableToColumnsMap.containsKey(tableName)) {
- return tableToColumnsMap.get(tableName);
- }
- return Maps.newHashMap();
- }
-
- private static Set<String> listColumnsInTable(String tableName,
- Map<String, Map<String, String>> tableToColumnsMap) {
- Map<String, String> colToAliasMap = getColToColAliasMapInTable(tableName, tableToColumnsMap);
- return colToAliasMap.keySet();
- }
-
- @VisibleForTesting
- public static String quoteIdentifier(String sqlExpr, String quotation, String identifier,
- List<String> identifierPatterns) {
- String quotedIdentifier = quotation + identifier.trim() + quotation;
-
- for (String pattern : identifierPatterns) {
- Matcher matcher = Pattern.compile(pattern, Pattern.CASE_INSENSITIVE | Pattern.DOTALL).matcher(sqlExpr);
- if (matcher.find()) {
- sqlExpr = matcher.replaceAll("$1" + quotedIdentifier + "$3");
- }
- }
- return sqlExpr;
- }
-
- private static boolean isIdentifierNeedToQuote(String sqlExpr, String identifier, List<String> identifierPatterns) {
- if (StringUtils.isBlank(sqlExpr) || StringUtils.isBlank(identifier)) {
- return false;
- }
-
- for (String pattern : identifierPatterns) {
- if (Pattern.compile(pattern, Pattern.CASE_INSENSITIVE | Pattern.DOTALL).matcher(sqlExpr).find()) {
- return true;
- }
- }
- return false;
- }
-
- @VisibleForTesting
- public static List<String> getTableNameOrAliasPatterns(String tableName) {
- Preconditions.checkNotNull(tableName);
- // Pattern must contain these regex groups, and place identifier in sec group ($2)
- List<String> patterns = Lists.newArrayList();
- patterns.add("([+\\-*/%&|^=><\\s,(])(" + tableName.trim() + ")(\\.)");
- patterns.add("([+\\-*/%&|^=><\\s,(])(`" + tableName.trim() + "`)(\\.)");
- patterns.add("([\\.\\s])(" + tableName.trim() + ")([,\\s)])");
- patterns.add("([\\.\\s])(`" + tableName.trim() + "`)([,\\s)])");
- patterns.add("(^)(" + tableName.trim() + ")([\\.])");
- patterns.add("(^)(`" + tableName.trim() + "`)([\\.])");
- return patterns;
- }
-
- @VisibleForTesting
- public static List<String> getColumnNameOrAliasPatterns(String colName) {
- Preconditions.checkNotNull(colName);
- // Pattern must contain these regex groups, and place identifier in sec group ($2)
- List<String> patterns = Lists.newArrayList();
- patterns.add("([\\.\\s(])(" + colName.trim() + ")([+\\-*/%&|^=><\\s,)]|$)");
- patterns.add("([\\.\\s(])(`" + colName.trim() + "`)([+\\-*/%&|^=><\\s,)]|$)");
- patterns.add("(^)(" + colName.trim() + ")([+\\-*/%&|^=><\\s,)])");
- patterns.add("(^)(`" + colName.trim() + "`)([+\\-*/%&|^=><\\s,)])");
- return patterns;
- }
-
- private static Map<String, Map<String, String>> buildTableToColumnsMap(NDataModel modelDesc) {
- Map<String, Map<String, String>> map = Maps.newHashMap();
- Set<TblColRef> colRefs = modelDesc.getEffectiveCols().values();
- for (TblColRef colRef : colRefs) {
- String colName = colRef.getName();
- String tableName = colRef.getTableRef().getTableName();
- String colAlias = colRef.getTableAlias() + "_" + colRef.getName();
- if (map.containsKey(tableName)) {
- map.get(tableName).put(colName, colAlias);
- } else {
- Map<String, String> colToAliasMap = Maps.newHashMap();
- colToAliasMap.put(colName, colAlias);
- map.put(tableName, colToAliasMap);
- }
- }
- return map;
- }
-
- private static Map<String, String> buildTableToTableAliasMap(NDataModel modelDesc) {
- Map<String, String> map = Maps.newHashMap();
- Set<TblColRef> colRefs = modelDesc.getEffectiveCols().values();
- for (TblColRef colRef : colRefs) {
- String tableName = colRef.getTableRef().getTableName();
- String alias = colRef.getTableAlias();
- map.put(tableName, alias);
- }
- return map;
- }
-
- /**
- * Used for quote identifiers in Sql Filter Expression & Computed Column Expression for flat table
- * @param modelDesc
- * @param sqlExpr
- * @param quotation
- * @return
- */
- @VisibleForTesting
- public static String quoteIdentifierInSqlExpr(NDataModel modelDesc, String sqlExpr, String quotation) {
- if (BACK_TICK_DOT_PATTERN.matcher(sqlExpr).find()) {
- return quoteIdentifierInSqlBackTickExpr(sqlExpr, quotation);
- }
- Map<String, String> tabToAliasMap = buildTableToTableAliasMap(modelDesc);
- Map<String, Map<String, String>> tabToColsMap = buildTableToColumnsMap(modelDesc);
-
- boolean tableMatched = false;
- for (Map.Entry<String, String> tableEntry : tabToAliasMap.entrySet()) {
- List<String> tabPatterns = getTableNameOrAliasPatterns(tableEntry.getKey());
- if (isIdentifierNeedToQuote(sqlExpr, tableEntry.getKey(), tabPatterns)) {
- sqlExpr = quoteIdentifier(sqlExpr, quotation, tableEntry.getKey(), tabPatterns);
- tableMatched = true;
- }
-
- String tabAlias = tableEntry.getValue();
- List<String> tabAliasPatterns = getTableNameOrAliasPatterns(tabAlias);
- if (isIdentifierNeedToQuote(sqlExpr, tabAlias, tabAliasPatterns)) {
- sqlExpr = quoteIdentifier(sqlExpr, quotation, tabAlias, tabAliasPatterns);
- tableMatched = true;
- }
-
- if (!tableMatched) {
- continue;
- }
-
- Set<String> columns = listColumnsInTable(tableEntry.getKey(), tabToColsMap);
- for (String column : columns) {
- List<String> colPatterns = getColumnNameOrAliasPatterns(column);
- if (isIdentifierNeedToQuote(sqlExpr, column, colPatterns)) {
- sqlExpr = quoteIdentifier(sqlExpr, quotation, column, colPatterns);
- }
- if (columnHasAlias(tableEntry.getKey(), column, tabToColsMap)) {
- String colAlias = getColumnAlias(tableEntry.getKey(), column, tabToColsMap);
- List<String> colAliasPattern = getColumnNameOrAliasPatterns(colAlias);
- if (isIdentifierNeedToQuote(sqlExpr, colAlias, colAliasPattern)) {
- sqlExpr = quoteIdentifier(sqlExpr, quotation, colAlias, colPatterns);
- }
- }
- }
-
- tableMatched = false; //reset
- }
- return sqlExpr;
- }
-
- private static String quoteIdentifierInSqlBackTickExpr(String sqlExpr, String quotation) {
- String result = sqlExpr;
- Matcher matcher = BACK_TICK_DOT_PATTERN.matcher(sqlExpr);
- while (matcher.find()) {
- String target = matcher.group();
- target = target.replace("`", quotation);
- result = result.replace(matcher.group(), target);
- }
- return result;
- }
-
-}
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
index a7dd882b84..de7d4191f4 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -167,7 +167,6 @@ import org.apache.kylin.metadata.model.FusionModelManager;
import org.apache.kylin.metadata.model.ISourceAware;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.JoinTableDesc;
-import org.apache.kylin.metadata.model.JoinedFlatTable;
import org.apache.kylin.metadata.model.ManagementType;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.MultiPartitionDesc;
@@ -1292,8 +1291,8 @@ public class ModelService extends AbstractModelService implements TableModelSupp
public String getModelSql(String modelId, String project) {
aclEvaluate.checkProjectReadPermission(project);
- NDataModel modelDesc = getManager(NDataModelManager.class, project).getDataModelDesc(modelId);
- return JoinedFlatTable.generateSelectDataStatement(modelDesc, false);
+ NDataModel model = getManager(NDataModelManager.class, project).getDataModelDesc(modelId);
+ return PushDownUtil.generateFlatTableSql(model, project, false);
}
public List<RelatedModelResponse> getRelateModels(String project, String table, String modelId) {
@@ -1712,16 +1711,12 @@ public class ModelService extends AbstractModelService implements TableModelSupp
}
@VisibleForTesting
- public void checkFlatTableSql(NDataModel dataModel) {
- String project = dataModel.getProject();
- ProjectInstance prjInstance = getManager(NProjectManager.class).getProject(project);
- if (KylinConfig.getInstanceFromEnv().isUTEnv() || prjInstance.getConfig().isSkipCheckFlatTable()) {
- return;
- }
- if (getModelConfig(dataModel).skipCheckFlatTable()) {
+ public void checkFlatTableSql(NDataModel model) {
+ if (skipCheckFlatTable(model)) {
return;
}
- long rangePartitionTableCount = dataModel.getAllTableRefs().stream()
+
+ long rangePartitionTableCount = model.getAllTableRefs().stream()
.filter(p -> p.getTableDesc().isRangePartition()).count();
if (rangePartitionTableCount > 0) {
logger.info("Range partitioned tables do not support pushdown, so do not need to perform subsequent logic");
@@ -1729,22 +1724,34 @@ public class ModelService extends AbstractModelService implements TableModelSupp
}
try {
+ String project = model.getProject();
+ ProjectInstance prjInstance = getManager(NProjectManager.class).getProject(project);
if (prjInstance.getSourceType() == ISourceAware.ID_SPARK
- && dataModel.getModelType() == NDataModel.ModelType.BATCH) {
+ && model.getModelType() == NDataModel.ModelType.BATCH) {
SparkSession ss = SparderEnv.getSparkSession();
- String flatTableSql = JoinedFlatTable.generateSelectDataStatement(dataModel, false);
+ String flatTableSql = PushDownUtil.generateFlatTableSql(model, project, false);
QueryParams queryParams = new QueryParams(project, flatTableSql, "default", false);
queryParams.setKylinConfig(prjInstance.getConfig());
queryParams.setAclInfo(AclPermissionUtil.createAclInfo(project, getCurrentUserGroups()));
- String pushdownSql = PushDownUtil.massagePushDownSql(queryParams);
- ss.sql(pushdownSql);
+ ss.sql(PushDownUtil.massagePushDownSql(queryParams));
}
} catch (Exception e) {
- buildExceptionMessage(dataModel, e);
+ buildExceptionMessage(model, e);
+ }
+ }
+
+ private boolean skipCheckFlatTable(NDataModel model) {
+ if (KylinConfig.getInstanceFromEnv().isUTEnv()) {
+ return true;
}
+ IndexPlan indexPlan = getIndexPlan(model.getId(), model.getProject());
+ KylinConfig config = indexPlan == null || indexPlan.getConfig() == null
+ ? NProjectManager.getProjectConfig(model.getProject())
+ : indexPlan.getConfig();
+ return config.skipCheckFlatTable();
}
- private static void buildExceptionMessage(NDataModel dataModel, Exception e) {
+ private void buildExceptionMessage(NDataModel dataModel, Exception e) {
Pattern pattern = Pattern.compile("cannot resolve '(.*?)' given input columns");
Matcher matcher = pattern.matcher(e.getMessage().replace("`", ""));
if (matcher.find()) {
@@ -1761,14 +1768,6 @@ public class ModelService extends AbstractModelService implements TableModelSupp
}
}
- private KylinConfig getModelConfig(NDataModel dataModel) {
- IndexPlan indexPlan = getIndexPlan(dataModel.getId(), dataModel.getProject());
- if (indexPlan == null || indexPlan.getConfig() == null) {
- return getManager(NProjectManager.class).getProject(dataModel.getProject()).getConfig();
- }
- return indexPlan.getConfig();
- }
-
private void validatePartitionDateColumn(ModelRequest modelRequest) {
if (Objects.nonNull(modelRequest.getPartitionDesc())) {
if (StringUtils.isNotEmpty(modelRequest.getPartitionDesc().getPartitionDateColumn())) {
@@ -2350,7 +2349,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
}
public JobInfoResponse fixSegmentHoles(String project, String modelId, List<SegmentTimeRequest> segmentHoles,
- Set<String> ignoredSnapshotTables) throws Exception {
+ Set<String> ignoredSnapshotTables) throws SQLException {
aclEvaluate.checkProjectOperationPermission(project);
NDataModel modelDesc = getManager(NDataModelManager.class, project).getDataModelDesc(modelId);
checkModelAndIndexManually(project, modelId);
@@ -2580,7 +2579,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
boolean buildSegmentOverlapEnable = getIndexPlan(model.getId(), project).getConfig()
.isBuildSegmentOverlapEnabled();
- boolean isBuildAllIndexesFinally = batchIndexIds == null || batchIndexIds.size() == 0
+ boolean isBuildAllIndexesFinally = CollectionUtils.isEmpty(batchIndexIds)
|| batchIndexIds.size() == getIndexPlan(model.getId(), project).getAllIndexes().size();
for (NDataSegment existedSegment : segments) {
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
index 5b2926052f..b69b89585b 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
@@ -84,8 +84,8 @@ import org.apache.calcite.sql.SqlKind;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
@@ -2203,7 +2203,6 @@ public class ModelServiceTest extends SourceTestCase {
request.setEnd("100");
request.setUuid(RandomUtil.randomUUIDStr());
modelService.createModel(request.getProject(), request);
- //TODO modelService.updateModelToResourceStore(deserialized, "default");
List<NDataModelResponse> dataModelDescs = modelService.getModels("nmodel_cc_test", "default", true, null, null,
"", false);
@@ -5161,20 +5160,23 @@ public class ModelServiceTest extends SourceTestCase {
public void testBuildExceptionMessage() {
NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), "default");
NDataModel dataModel = modelManager.getDataModelDesc("a8ba3ff1-83bd-4066-ad54-d2fb3d1f0e94");
- {
- val testException = new RuntimeException("test");
- Assert.assertThrows("model [test_encoding], Something went wrong. test", KylinException.class,
- () -> ReflectionTestUtils.invokeMethod(ModelService.class, "buildExceptionMessage", dataModel,
- testException));
- }
+ String toValidMethodName = "buildExceptionMessage";
+ String expectedMsg = "model [test_encoding], Something went wrong. test";
+ val testException = new RuntimeException("test");
+ Assert.assertThrows(expectedMsg, KylinException.class,
+ () -> ReflectionTestUtils.invokeMethod(modelService, toValidMethodName, dataModel, testException));
+ }
- {
- val testException = new RuntimeException("cannot resolve 'test' given input columns");
- Assert.assertThrows(
- "Can’t save model \"test_encoding\". Please ensure that the used column \"test\" exist in source table \"DEFAULT.TEST_ENCODING\".",
- KylinException.class, () -> ReflectionTestUtils.invokeMethod(ModelService.class,
- "buildExceptionMessage", dataModel, testException));
- }
+ @Test
+ public void testBuildExceptionMessageCausedByResolveProblem() {
+ NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), "default");
+ NDataModel dataModel = modelManager.getDataModelDesc("a8ba3ff1-83bd-4066-ad54-d2fb3d1f0e94");
+ String toValidMethodName = "buildExceptionMessage";
+ String expectedMsg = "Can’t save model \"test_encoding\". Please ensure that the used column \"test\" "
+ + "exist in source table \"DEFAULT.TEST_ENCODING\".";
+ val testException = new RuntimeException("cannot resolve 'test' given input columns");
+ Assert.assertThrows(expectedMsg, KylinException.class,
+ () -> ReflectionTestUtils.invokeMethod(modelService, toValidMethodName, dataModel, testException));
}
@Test
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java b/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
index 9f3be8de77..b902bfe3f5 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
@@ -90,6 +90,7 @@ public class PushDownUtil {
.compile("/\\*\\s*\\+\\s*(?i)MODEL_PRIORITY\\s*\\([\\s\\S]*\\)\\s*\\*/");
public static final String DEFAULT_SCHEMA = "DEFAULT";
private static final String CC_SPLITTER = "'##CC_PUSH_DOWN_TOKEN##'";
+ private static final String UNDER_LINE = "_";
private static final ExecutorService asyncExecutor = Executors.newCachedThreadPool();
private static final Map<String, IPushDownConverter> PUSH_DOWN_CONVERTER_MAP = Maps.newConcurrentMap();
@@ -109,11 +110,8 @@ public class PushDownUtil {
}
public static PushdownResult tryIterQuery(QueryParams queryParams) throws SQLException {
-
- String sql = queryParams.getSql();
- String project = queryParams.getProject();
-
KylinConfig projectConfig = NProjectManager.getProjectConfig(queryParams.getProject());
+ queryParams.setKylinConfig(projectConfig);
if (!projectConfig.isPushDownEnabled()) {
checkPushDownIncapable(queryParams);
return null;
@@ -130,21 +128,17 @@ public class PushDownUtil {
logger.info("Kylin cannot support non-select queries, routing to other engines");
}
+ // Set a push-down engine for query context.
IPushDownRunner runner = (IPushDownRunner) ClassUtil.newInstance(projectConfig.getPushDownRunnerClassName());
- runner.init(projectConfig, project);
+ runner.init(projectConfig, queryParams.getProject());
logger.debug("Query Pushdown runner {}", runner);
-
- // set pushdown engine in query context
-
- // for file source
int sourceType = projectConfig.getDefaultSource();
String engine = sourceType == ISourceAware.ID_SPARK && KapConfig.getInstanceFromEnv().isCloud()
? QueryContext.PUSHDOWN_OBJECT_STORAGE
: runner.getName();
QueryContext.current().setPushdownEngine(engine);
- queryParams.setKylinConfig(projectConfig);
- queryParams.setSql(sql);
+ String sql;
try {
sql = massagePushDownSql(queryParams);
} catch (NoAuthorizedColsError e) {
@@ -154,13 +148,12 @@ public class PushDownUtil {
QueryContext.currentTrace().startSpan(QueryTrace.PREPARE_AND_SUBMIT_JOB);
if (queryParams.isSelect()) {
- PushdownResult result = runner.executeQueryToIterator(sql, project);
+ PushdownResult result = runner.executeQueryToIterator(sql, queryParams.getProject());
if (QueryContext.current().getQueryTagInfo().isAsyncQuery()) {
AsyncQueryUtil.saveMetaDataAndFileInfo(QueryContext.current(), result.getColumnMetas());
}
return result;
}
-
return PushdownResult.emptyResult();
}
@@ -189,6 +182,10 @@ public class PushDownUtil {
}
public static String massagePushDownSql(QueryParams queryParams) {
+ if (queryParams.getSql() == null) {
+ return StringUtils.EMPTY;
+ }
+
String sql = queryParams.getSql();
sql = QueryUtil.trimRightSemiColon(sql);
sql = SQL_HINT_PATTERN.matcher(sql).replaceAll("");
@@ -225,6 +222,41 @@ public class PushDownUtil {
return converters;
}
+ /**
+ * This method is currently only used for verifying the flat-table generated by the model.
+ */
+ public static String generateFlatTableSql(NDataModel model, String project, boolean singleLine) {
+ String sep = singleLine ? " " : "\n";
+ StringBuilder sqlBuilder = new StringBuilder();
+ sqlBuilder.append("SELECT ").append(sep);
+
+ List<TblColRef> tblColRefs = Lists.newArrayList(model.getEffectiveCols().values());
+ if (tblColRefs.isEmpty()) {
+ sqlBuilder.append("1 ").append(sep);
+ } else {
+ String allColStr = tblColRefs.stream() //
+ .filter(colRef -> !colRef.getColumnDesc().isComputedColumn()) //
+ .map(colRef -> {
+ String s = colRef.getTableAlias() + UNDER_LINE + colRef.getName();
+ String colName = StringHelper.doubleQuote(s);
+ return colRef.getDoubleQuoteExp() + " as " + colName + sep;
+ }).collect(Collectors.joining(", "));
+ sqlBuilder.append(allColStr);
+ }
+
+ sqlBuilder.append("FROM ").append(model.getRootFactTable().getTableDesc().getDoubleQuoteIdentity());
+ appendJoinStatement(model, sqlBuilder, singleLine);
+
+ sqlBuilder.append("WHERE ").append(sep);
+ sqlBuilder.append("1 = 1").append(sep);
+ if (StringUtils.isNotEmpty(model.getFilterCondition())) {
+ String filterCondition = massageExpression(model, project, model.getFilterCondition(), null);
+ sqlBuilder.append(" AND (").append(filterCondition).append(") ").append(sep);
+ }
+
+ return new EscapeTransformer().transform(sqlBuilder.toString());
+ }
+
public static String expandComputedColumnExp(NDataModel model, String project, String expression) {
StringBuilder forCC = new StringBuilder();
forCC.append("select ").append(expression).append(" ,").append(CC_SPLITTER) //
@@ -237,6 +269,7 @@ public class PushDownUtil {
// massage nested CC for drafted model
Map<String, NDataModel> modelMap = Maps.newHashMap();
modelMap.put(model.getUuid(), model);
+ ccSql = new EscapeTransformer().transform(ccSql);
ccSql = RestoreFromComputedColumn.convertWithGivenModels(ccSql, project, DEFAULT_SCHEMA, modelMap);
} catch (Exception e) {
logger.warn("Failed to massage SQL expression [{}] with input model {}", ccSql, model.getUuid(), e);
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/QueryAliasMatcher.java b/src/query-common/src/main/java/org/apache/kylin/query/util/QueryAliasMatcher.java
index ab4fb87d2e..b9c1b579ea 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/QueryAliasMatcher.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/QueryAliasMatcher.java
@@ -43,7 +43,6 @@ import org.apache.calcite.sql.util.SqlBasicVisitor;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.KylinConfigExt;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
@@ -72,9 +71,11 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
// match alias in query to alias in model
// Not designed to reuse, re-new per query
+@Slf4j
public class QueryAliasMatcher {
static final ColumnRowType MODEL_VIEW_COLUMN_ROW_TYPE = new ColumnRowType(new ArrayList<>());
private static final ColumnRowType SUBQUERY_TAG = new ColumnRowType(null);
@@ -196,8 +197,7 @@ public class QueryAliasMatcher {
return null;
}
JoinsGraph joinsGraph = new JoinsGraph(firstTable, joinDescs);
- KylinConfigExt projectConfig = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProject(project)
- .getConfig();
+ KylinConfig projectConfig = NProjectManager.getProjectConfig(project);
if (sqlJoinCapturer.foundJoinOnCC) {
// 1st round: dry run without cc expr comparison to collect model alias matching
@@ -241,6 +241,7 @@ public class QueryAliasMatcher {
}
private static class CCJoinEdgeMatcher extends DefaultJoinEdgeMatcher {
+ private static final EscapeTransformer TRANSFORMER = new EscapeTransformer();
transient QueryAliasMatchInfo matchInfo;
boolean compareCCExpr;
@@ -263,12 +264,20 @@ public class QueryAliasMatcher {
|| (!a.isComputedColumn() && b.isComputedColumn())) {
return false;
} else {
- if (!compareCCExpr)
+ if (!compareCCExpr) {
return true;
-
- SqlNode node1 = CalciteParser.getExpNode(a.getComputedColumnExpr());
- SqlNode node2 = CalciteParser.getExpNode(b.getComputedColumnExpr());
- return ExpressionComparator.isNodeEqual(node1, node2, matchInfo, new AliasDeduceImpl(matchInfo));
+ }
+ try {
+ SqlNode node1 = CalciteParser.getExpNode(TRANSFORMER.transform(a.getDoubleQuoteInnerExpr()));
+ SqlNode node2 = CalciteParser.getExpNode(TRANSFORMER.transform(b.getDoubleQuoteInnerExpr()));
+ return ExpressionComparator.isNodeEqual(node1, node2, matchInfo, new AliasDeduceImpl(matchInfo));
+ } catch (Exception e) {
+ // If this situation occurs, it means that there is an error in the parsing of the computed column.
+ // Therefore, we can directly assume that these two computed columns are not equal.
+ log.error("Failed to parse expressions, {} or {}", a.getComputedColumnExpr(),
+ b.getComputedColumnExpr());
+ return false;
+ }
}
}
}
diff --git a/src/query/src/test/java/org/apache/kylin/query/util/CCOnRealModelTest.java b/src/query/src/test/java/org/apache/kylin/query/util/CCOnRealModelTest.java
index 1b337e0569..658d266530 100644
--- a/src/query/src/test/java/org/apache/kylin/query/util/CCOnRealModelTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/util/CCOnRealModelTest.java
@@ -113,7 +113,7 @@ public class CCOnRealModelTest extends NLocalFileMetadataTestCase {
}
// ignored for KAP#16258
- @Ignore
+ @Ignore("historic ignored")
@Test
public void testSubquery() {
{
@@ -198,7 +198,6 @@ public class CCOnRealModelTest extends NLocalFileMetadataTestCase {
}
@Test
- @Ignore("Not support CC on Join condition")
public void testJoinOnCC() {
{
String originSql = "select count(*) from TEST_KYLIN_FACT\n"
@@ -232,11 +231,9 @@ public class CCOnRealModelTest extends NLocalFileMetadataTestCase {
}
@Test
- public void testNoFrom() throws Exception {
- String originSql = "select sum(price * item_count),(SELECT 1 as VERSION) from test_kylin_fact";
- String ccSql = "select sum(TEST_KYLIN_FACT.DEAL_AMOUNT),(SELECT 1 as VERSION) from test_kylin_fact";
-
- check(converter, originSql, ccSql);
+ public void testNoFrom() {
+ check(converter, "select sum(price * item_count),(SELECT 1 as VERSION) from test_kylin_fact",
+ "select sum(TEST_KYLIN_FACT.DEAL_AMOUNT),(SELECT 1 as VERSION) from test_kylin_fact");
}
@Test
diff --git a/src/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java b/src/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java
index 0a57502dcf..fdc7dcf28c 100644
--- a/src/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java
@@ -18,10 +18,22 @@
package org.apache.kylin.query.util;
+import java.util.List;
import java.util.Properties;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.exception.KylinException;
+import org.apache.kylin.common.exception.QueryErrorCode;
+import org.apache.kylin.common.exception.ServerErrorCode;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
+import org.apache.kylin.metadata.model.ComputedColumnDesc;
+import org.apache.kylin.metadata.model.JoinDesc;
+import org.apache.kylin.metadata.model.JoinTableDesc;
+import org.apache.kylin.metadata.model.NDataModel;
+import org.apache.kylin.metadata.model.NDataModelManager;
+import org.apache.kylin.metadata.model.TableRef;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -49,7 +61,8 @@ public class PushDownUtilTest extends NLocalFileMetadataTestCase {
PushDownUtil.tryIterQuery(queryParams);
Assert.fail();
} catch (Exception e) {
- Assert.assertFalse(e instanceof IllegalArgumentException);
+ Assert.assertTrue(e instanceof KylinException);
+ Assert.assertEquals(ServerErrorCode.SPARK_FAILURE.toErrorCode(), ((KylinException) e).getErrorCode());
}
}
@@ -64,7 +77,9 @@ public class PushDownUtilTest extends NLocalFileMetadataTestCase {
PushDownUtil.tryIterQuery(queryParams);
Assert.fail();
} catch (Exception e) {
- Assert.assertFalse(e instanceof IllegalArgumentException);
+ Assert.assertTrue(e instanceof KylinException);
+ Assert.assertEquals(QueryErrorCode.INVALID_PARAMETER_PUSH_DOWN.toErrorCode(),
+ ((KylinException) e).getErrorCode());
}
}
@@ -133,4 +148,156 @@ public class PushDownUtilTest extends NLocalFileMetadataTestCase {
String resSql = "select ab from table where aa = '' and bb = '\\'as\\'n\\''";
Assert.assertEquals(resSql, PushDownUtil.replaceEscapedQuote(sql));
}
+
+ @Test
+ public void testGenerateFlatTableSql() {
+ String project = "default";
+ NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
+ NDataModel model = modelManager.getDataModelDescByAlias("test_bank");
+ String expected = "SELECT\n" //
+ + "\"TEST_BANK_INCOME\".\"COUNTRY\" as \"TEST_BANK_INCOME_COUNTRY\"\n"
+ + ", \"TEST_BANK_INCOME\".\"INCOME\" as \"TEST_BANK_INCOME_INCOME\"\n"
+ + ", \"TEST_BANK_INCOME\".\"NAME\" as \"TEST_BANK_INCOME_NAME\"\n"
+ + ", \"TEST_BANK_INCOME\".\"DT\" as \"TEST_BANK_INCOME_DT\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"COUNTRY\" as \"TEST_BANK_LOCATION_COUNTRY\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"OWNER\" as \"TEST_BANK_LOCATION_OWNER\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"LOCATION\" as \"TEST_BANK_LOCATION_LOCATION\"\n"
+ + "FROM \"DEFAULT\".\"TEST_BANK_INCOME\"\n"
+ + "INNER JOIN \"DEFAULT\".\"TEST_BANK_LOCATION\" as \"TEST_BANK_LOCATION\"\n"
+ + "ON \"TEST_BANK_INCOME\".\"COUNTRY\" = \"TEST_BANK_LOCATION\".\"COUNTRY\"\n" //
+ + "WHERE\n" //
+ + "1 = 1";
+ Assert.assertEquals(expected, PushDownUtil.generateFlatTableSql(model, project, false));
+ }
+
+ @Test
+ public void testGenerateFlatTableSqlWithCCJoin() {
+ String project = "default";
+ NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
+ NDataModel model = modelManager.getDataModelDescByAlias("test_bank");
+ updateModelToAddCC(project, model);
+ // change join condition
+ EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ NDataModelManager modelMgr = NDataModelManager.getInstance(kylinConfig, project);
+ modelMgr.updateDataModel(model.getUuid(), copyForWrite -> {
+ List<JoinTableDesc> joinTables = copyForWrite.getJoinTables();
+ TableRef rootTableRef = copyForWrite.getRootFactTable();
+ TblColRef cc1 = rootTableRef.getColumn("CC1");
+ JoinDesc join = joinTables.get(0).getJoin();
+ join.setForeignKeyColumns(new TblColRef[] { cc1 });
+ join.setForeignKey(new String[] { "TEST_BANK_INCOME.CC1" });
+ });
+ return null;
+ }, project);
+ String expected = "SELECT\n" //
+ + "\"TEST_BANK_INCOME\".\"COUNTRY\" as \"TEST_BANK_INCOME_COUNTRY\"\n"
+ + ", \"TEST_BANK_INCOME\".\"INCOME\" as \"TEST_BANK_INCOME_INCOME\"\n"
+ + ", \"TEST_BANK_INCOME\".\"NAME\" as \"TEST_BANK_INCOME_NAME\"\n"
+ + ", \"TEST_BANK_INCOME\".\"DT\" as \"TEST_BANK_INCOME_DT\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"COUNTRY\" as \"TEST_BANK_LOCATION_COUNTRY\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"OWNER\" as \"TEST_BANK_LOCATION_OWNER\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"LOCATION\" as \"TEST_BANK_LOCATION_LOCATION\"\n"
+ + "FROM \"DEFAULT\".\"TEST_BANK_INCOME\"\n"
+ + "INNER JOIN \"DEFAULT\".\"TEST_BANK_LOCATION\" as \"TEST_BANK_LOCATION\"\n"
+ + "ON SUBSTRING(\"TEST_BANK_INCOME\".\"COUNTRY\", 0, 4) = \"TEST_BANK_LOCATION\".\"COUNTRY\"\n"
+ + "WHERE\n" //
+ + "1 = 1";
+ NDataModel updatedModel = modelManager.getDataModelDesc(model.getUuid());
+ Assert.assertEquals(expected, PushDownUtil.generateFlatTableSql(updatedModel, project, false));
+
+ }
+
+ @Test
+ public void testGenerateFlatTableSqlWithFilterCondition() {
+ String project = "default";
+ NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
+ NDataModel model = modelManager.getDataModelDescByAlias("test_bank");
+ updateModelToAddCC(project, model);
+ // change filter condition
+ EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ NDataModelManager modelMgr = NDataModelManager.getInstance(kylinConfig, project);
+ modelMgr.updateDataModel(model.getUuid(), copyForWrite -> {
+ copyForWrite.setFilterCondition(
+ "SUBSTRING(\"TEST_BANK_INCOME\".\"COUNTRY\", 0, 4) = 'china' and cc1 = 'china'");
+ });
+ return null;
+ }, project);
+ String expected = "SELECT\n" //
+ + "\"TEST_BANK_INCOME\".\"COUNTRY\" as \"TEST_BANK_INCOME_COUNTRY\"\n"
+ + ", \"TEST_BANK_INCOME\".\"INCOME\" as \"TEST_BANK_INCOME_INCOME\"\n"
+ + ", \"TEST_BANK_INCOME\".\"NAME\" as \"TEST_BANK_INCOME_NAME\"\n"
+ + ", \"TEST_BANK_INCOME\".\"DT\" as \"TEST_BANK_INCOME_DT\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"COUNTRY\" as \"TEST_BANK_LOCATION_COUNTRY\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"OWNER\" as \"TEST_BANK_LOCATION_OWNER\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"LOCATION\" as \"TEST_BANK_LOCATION_LOCATION\"\n"
+ + "FROM \"DEFAULT\".\"TEST_BANK_INCOME\"\n"
+ + "INNER JOIN \"DEFAULT\".\"TEST_BANK_LOCATION\" as \"TEST_BANK_LOCATION\"\n"
+ + "ON \"TEST_BANK_INCOME\".\"COUNTRY\" = \"TEST_BANK_LOCATION\".\"COUNTRY\"\n" //
+ + "WHERE\n" //
+ + "1 = 1\n" //
+ + " AND (SUBSTRING(`TEST_BANK_INCOME`.`COUNTRY`, 0, 4) = 'china' and (SUBSTRING(`TEST_BANK_INCOME`.`COUNTRY`, 0, 4)) = 'china')";
+ NDataModel updatedModel = modelManager.getDataModelDesc(model.getUuid());
+ Assert.assertEquals(expected, PushDownUtil.generateFlatTableSql(updatedModel, project, false));
+ }
+
+ @Test
+ public void testGenerateFlatTableSqlWithSpecialFunctions() {
+ String project = "default";
+ NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
+ NDataModel model = modelManager.getDataModelDescByAlias("test_bank");
+ updateModelToAddCC(project, model);
+ // change filter condition
+ EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ NDataModelManager modelMgr = NDataModelManager.getInstance(kylinConfig, project);
+ modelMgr.updateDataModel(model.getUuid(), copyForWrite -> {
+ copyForWrite.setFilterCondition("timestampadd(day, 1, current_date) = '2012-01-01' and cc1 = 'china'");
+ });
+ return null;
+ }, project);
+ String expected = "SELECT\n" //
+ + "\"TEST_BANK_INCOME\".\"COUNTRY\" as \"TEST_BANK_INCOME_COUNTRY\"\n"
+ + ", \"TEST_BANK_INCOME\".\"INCOME\" as \"TEST_BANK_INCOME_INCOME\"\n"
+ + ", \"TEST_BANK_INCOME\".\"NAME\" as \"TEST_BANK_INCOME_NAME\"\n"
+ + ", \"TEST_BANK_INCOME\".\"DT\" as \"TEST_BANK_INCOME_DT\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"COUNTRY\" as \"TEST_BANK_LOCATION_COUNTRY\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"OWNER\" as \"TEST_BANK_LOCATION_OWNER\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"LOCATION\" as \"TEST_BANK_LOCATION_LOCATION\"\n"
+ + "FROM \"DEFAULT\".\"TEST_BANK_INCOME\"\n"
+ + "INNER JOIN \"DEFAULT\".\"TEST_BANK_LOCATION\" as \"TEST_BANK_LOCATION\"\n"
+ + "ON \"TEST_BANK_INCOME\".\"COUNTRY\" = \"TEST_BANK_LOCATION\".\"COUNTRY\"\n" //
+ + "WHERE\n" //
+ + "1 = 1\n" //
+ + " AND (TIMESTAMPADD(day, 1, current_date) = '2012-01-01' and (SUBSTRING(`TEST_BANK_INCOME`.`COUNTRY`, 0, 4)) = 'china')";
+ NDataModel updatedModel = modelManager.getDataModelDesc(model.getUuid());
+ Assert.assertEquals(expected, PushDownUtil.generateFlatTableSql(updatedModel, project, false));
+ }
+
+ private void updateModelToAddCC(String project, NDataModel model) {
+ EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ NDataModelManager modelMgr = NDataModelManager.getInstance(kylinConfig, project);
+ modelMgr.updateDataModel(model.getUuid(), copyForWrite -> {
+ ComputedColumnDesc cc = new ComputedColumnDesc();
+ cc.setColumnName("CC1");
+ cc.setDatatype("int");
+ cc.setExpression("substring(\"TEST_BANK_INCOME\".\"COUNTRY\", 0, 4)");
+ cc.setInnerExpression("SUBSTRING(`TEST_BANK_INCOME`.`COUNTRY`, 0, 4)");
+ cc.setTableAlias("TEST_BANK_INCOME");
+ cc.setTableIdentity(model.getRootFactTableName());
+ copyForWrite.getComputedColumnDescs().add(cc);
+ List<NDataModel.NamedColumn> columns = copyForWrite.getAllNamedColumns();
+ int id = columns.size();
+ NDataModel.NamedColumn namedColumn = new NDataModel.NamedColumn();
+ namedColumn.setName("CC1");
+ namedColumn.setId(id);
+ namedColumn.setAliasDotColumn("TEST_BANK_INCOME.CC1");
+ columns.add(namedColumn);
+ copyForWrite.setAllNamedColumns(columns);
+ });
+ return null;
+ }, project);
+ }
}