You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2023/09/07 08:47:16 UTC
[kylin] 08/24: [DIRTY] fix error when table index answer non select star query
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit af7291cfbd1c18aca54183f82cd96f5f9f9aad55
Author: DorrisRuixuanZhang <do...@hotmail.com>
AuthorDate: Mon May 15 16:47:07 2023 +0800
[DIRTY] fix error when table index answer non select star query
---
.../metadata/cube/cuboid/TableIndexMatcher.java | 5 +++
.../routing/TableIndexAnswerSelectStarTest.java | 45 +++++++++++++++++++++-
.../baa44f37-8481-e78b-5cac-faa7d76767db.json | 4 +-
.../org/apache/kylin/query/schema/OLAPTable.java | 25 ------------
4 files changed, 51 insertions(+), 28 deletions(-)
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/TableIndexMatcher.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/TableIndexMatcher.java
index 444ba8d04e..e9c716f13e 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/TableIndexMatcher.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/TableIndexMatcher.java
@@ -31,6 +31,7 @@ import org.apache.kylin.metadata.cube.model.NDataflow;
import org.apache.kylin.metadata.model.AntiFlatChecker;
import org.apache.kylin.metadata.model.ColExcludedChecker;
import org.apache.kylin.metadata.model.DeriveInfo;
+import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.realization.CapabilityResult;
import org.apache.kylin.metadata.realization.SQLDigest;
@@ -66,6 +67,10 @@ public class TableIndexMatcher extends IndexMatcher {
if (NProjectManager.getProjectConfig(project).useTableIndexAnswerSelectStarEnabled()) {
penaltyFactor = unmatchedCols.size();
unmatchedCols.removeAll(dataflow.getAllColumnsIndex());
+ TableRef tableRef = dataflow.getModel().getTableNameMap().get(sqlDigest.factTable);
+ if (sqlDigest.isRawQuery && (sqlDigest.allColumns.size() == tableRef.getColumns().size())) {
+ unmatchedCols.clear();
+ }
}
goThruDerivedDims(layout.getIndex(), needDerive, unmatchedCols);
boolean isMatch = unmatchedCols.isEmpty();
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/query/routing/TableIndexAnswerSelectStarTest.java b/src/kylin-it/src/test/java/org/apache/kylin/query/routing/TableIndexAnswerSelectStarTest.java
index 07d4dd2e80..5e4ebfc8fe 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/query/routing/TableIndexAnswerSelectStarTest.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/query/routing/TableIndexAnswerSelectStarTest.java
@@ -30,6 +30,7 @@ import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.common.util.TempMetadataBuilder;
import org.apache.kylin.common.util.Unsafe;
import org.apache.kylin.engine.spark.NLocalWithSparkSessionTest;
+import org.apache.kylin.guava30.shaded.common.base.Throwables;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.cube.cuboid.NLayoutCandidate;
@@ -45,6 +46,7 @@ import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NDataModelManager;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
+import org.apache.kylin.query.engine.QueryExec;
import org.apache.kylin.query.engine.SchemaMetaData;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.util.OlapContextTestUtil;
@@ -155,7 +157,8 @@ public class TableIndexAnswerSelectStarTest extends NLocalWithSparkSessionTest {
copyForWrite.getAllNamedColumns().add(newCol);
});
- NIndexPlanManager indexPlanManager = NIndexPlanManager.getInstance(KylinConfig.getInstanceFromEnv(), getProject());
+ NIndexPlanManager indexPlanManager = NIndexPlanManager.getInstance(KylinConfig.getInstanceFromEnv(),
+ getProject());
IndexPlan indexPlan = indexPlanManager.getIndexPlan(modelId);
Long oldBaseAggLayout = indexPlan.getBaseAggLayoutId();
indexPlanManager.updateIndexPlan(indexPlan.getUuid(), copyForWrite -> copyForWrite
@@ -194,6 +197,46 @@ public class TableIndexAnswerSelectStarTest extends NLocalWithSparkSessionTest {
Assert.assertEquals(20000010001L, layoutCandidate.getLayoutEntity().getId());
}
+ @Test
+ public void testNonSelectStarAndNestedQuery() throws Exception {
+ NDataflowManager dfMgr = NDataflowManager.getInstance(getTestConfig(), getProject());
+ dfMgr.updateDataflowStatus("ccb82d81-1497-ca6d-f226-3258a0f0ba4f", RealizationStatusEnum.OFFLINE);
+ dfMgr.updateDataflowStatus("c7a44f37-8481-e78b-5cac-faa7d76767db", RealizationStatusEnum.OFFLINE);
+ String dfID = "baa44f37-8481-e78b-5cac-faa7d76767db";
+ NDataflow dataflow = dfMgr.getDataflow(dfID);
+
+ overwriteSystemProp("kylin.query.use-tableindex-answer-select-star.enabled", "true");
+
+ String sql1 = "select * from (select trans_id from test_kylin_fact)";
+ OLAPContext context = OlapContextTestUtil.getOlapContexts(getProject(), sql1).get(0);
+ Map<String, String> sqlAlias2ModelName = OlapContextTestUtil.matchJoins(dataflow.getModel(), context);
+ context.fixModel(dataflow.getModel(), sqlAlias2ModelName);
+ NLayoutCandidate layoutCandidate = QueryLayoutChooser.selectLayoutCandidate(dataflow,
+ dataflow.getQueryableSegments(), context.getSQLDigest(), null);
+ Assert.assertNotNull(layoutCandidate);
+ Assert.assertEquals(20000010001L, layoutCandidate.getLayoutEntity().getId());
+
+ QueryExec queryExec = new QueryExec(getProject(), getTestConfig());
+ String sql2 = "select price from test_kylin_fact";
+ try {
+ queryExec.executeQuery(sql2);
+ Assert.fail();
+ } catch (Exception e) {
+ Assert.assertTrue(Throwables.getRootCause(e.getCause()).getMessage()
+ .contains("No realization found for OLAPContext"));
+ }
+
+ String sql3 = "select * from (select price from test_kylin_fact)";
+ try {
+ queryExec.executeQuery(sql3);
+ Assert.fail();
+ } catch (Exception e) {
+ Assert.assertTrue(Throwables.getRootCause(e.getCause()).getMessage()
+ .contains("No realization found for OLAPContext"));
+ }
+
+ }
+
@Override
public String getProject() {
return "tableindex_answer_selectstart";
diff --git a/src/kylin-it/src/test/resources/ut_meta/tableindex_answer_selectstart/metadata/tableindex_answer_selectstart/index_plan/baa44f37-8481-e78b-5cac-faa7d76767db.json b/src/kylin-it/src/test/resources/ut_meta/tableindex_answer_selectstart/metadata/tableindex_answer_selectstart/index_plan/baa44f37-8481-e78b-5cac-faa7d76767db.json
index 9e35198cf8..b1c9f7b8be 100644
--- a/src/kylin-it/src/test/resources/ut_meta/tableindex_answer_selectstart/metadata/tableindex_answer_selectstart/index_plan/baa44f37-8481-e78b-5cac-faa7d76767db.json
+++ b/src/kylin-it/src/test/resources/ut_meta/tableindex_answer_selectstart/metadata/tableindex_answer_selectstart/index_plan/baa44f37-8481-e78b-5cac-faa7d76767db.json
@@ -8,12 +8,12 @@
"indexes" : [ {
"id" : 0,
"dimensions" : [ 0, 1, 2, 4, 5, 6 ],
- "measures" : [ 100000, 100001, 100002, 100003, 100004, 100005, 100006, 100007, 100008, 100009, 100010, 100011, 100012 ],
+ "measures" : [ 100000, 100001, 100002, 100003, 100004, 100005, 100006, 100007, 100008, 100010, 100011, 100012 ],
"layouts" : [ {
"id" : 1,
"name" : null,
"owner" : null,
- "col_order" : [ 0, 1, 2, 4, 5, 6, 100000, 100001, 100002, 100003, 100004, 100005, 100006, 100007, 100008, 100009, 100010, 100011, 100012 ],
+ "col_order" : [ 0, 1, 2, 4, 5, 6, 100000, 100001, 100002, 100003, 100004, 100005, 100006, 100007, 100008, 100010, 100011, 100012 ],
"shard_by_columns" : [ ],
"partition_by_columns" : [ ],
"sort_by_columns" : [ ],
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/schema/OLAPTable.java b/src/query-common/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
index 01860665d0..7912d2c698 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
@@ -58,8 +58,6 @@ import org.apache.kylin.common.util.CollectionUtil;
import org.apache.kylin.guava30.shaded.common.collect.Iterables;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.measure.topn.TopNMeasureType;
-import org.apache.kylin.metadata.cube.model.NDataflow;
-import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.ComputedColumnDesc;
@@ -69,7 +67,6 @@ import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.util.ComputedColumnUtil;
import org.apache.kylin.metadata.project.NProjectManager;
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.query.QueryExtension;
import org.apache.kylin.query.enumerator.OLAPQuery;
import org.apache.kylin.query.relnode.OLAPTableScan;
@@ -256,28 +253,6 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
return allColumns;
}
- KylinConfig projectConfig = NProjectManager.getProjectConfig(sourceTable.getProject());
- NDataflowManager dataflowManager = NDataflowManager.getInstance(olapSchema.getConfig(),
- sourceTable.getProject());
- if (projectConfig.useTableIndexAnswerSelectStarEnabled()) {
- Set<ColumnDesc> exposeColumnDescSet = new HashSet<>();
- String tableName = sourceTable.getIdentity();
- List<NDataModel> modelList = modelsMap.get(tableName);
- for (NDataModel dataModel : modelList) {
- NDataflow dataflow = dataflowManager.getDataflow(dataModel.getId());
- if (dataflow.getStatus() == RealizationStatusEnum.ONLINE) {
- dataflow.getAllColumns().forEach(tblColRef -> {
- if (tblColRef.getTable().equalsIgnoreCase(tableName)) {
- exposeColumnDescSet.add(tblColRef.getColumnDesc());
- }
- });
- }
- }
- if (!exposeColumnDescSet.isEmpty()) {
- allColumns = Lists.newArrayList(exposeColumnDescSet);
- }
- }
-
val authorizedCC = getAuthorizedCC();
if (CollectionUtils.isNotEmpty(authorizedCC)) {
val ccAsColumnDesc = ComputedColumnUtil.createComputedColumns(authorizedCC, sourceTable);