You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/01/17 17:02:22 UTC
[doris] branch master updated: [fix](nereids) fix bug of invalid column in olap scan node when a materialized view is selected (#15976)
This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 96b9115286 [fix](nereids) fix bug of invalid column in olap scan node when a materialized view is selected (#15976)
96b9115286 is described below
commit 96b9115286a2d358251f405cf0a7371f9c51f352
Author: starocean999 <40...@users.noreply.github.com>
AuthorDate: Wed Jan 18 01:02:12 2023 +0800
[fix](nereids) fix bug of invalid column in olap scan node when a materialized view is selected (#15976)
if a materialized view is selected, the olap scan node's NonUserVisibleOutput property may contains column from other materialized view. This pr remove invalid column
---
.../glue/translator/PhysicalPlanTranslator.java | 11 +-
.../test_materialized_view_nereids.out | 52 +++++++++
.../test_materialized_view_nereids.groovy | 124 +++++++++++++++++++++
3 files changed, 186 insertions(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 5f66fa72c3..8ad6a9326b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -442,7 +442,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
// Create OlapScanNode
List<Slot> slotList = new ImmutableList.Builder<Slot>()
.addAll(olapScan.getOutput())
- .addAll(olapScan.getNonUserVisibleOutput())
+ .addAll(filterSlotsOfSelectedIndex(olapScan.getNonUserVisibleOutput(), olapScan))
.build();
OlapTable olapTable = olapScan.getTable();
TupleDescriptor tupleDescriptor = generateTupleDesc(slotList, olapTable, context);
@@ -1742,4 +1742,13 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
return resultExpressions;
}
}
+
+ private List<Slot> filterSlotsOfSelectedIndex(List<Slot> slots, PhysicalOlapScan olapScan) {
+ ImmutableSet<Column> selectIndexColumns = ImmutableSet.copyOf(olapScan.getTable()
+ .getIndexIdToMeta().get(olapScan.getSelectedIndexId()).getSchema());
+ return slots.stream()
+ .filter(slot -> ((SlotReference) slot).getColumn().isPresent()
+ && selectIndexColumns.contains(((SlotReference) slot).getColumn().get()))
+ .collect(ImmutableList.toImmutableList());
+ }
}
diff --git a/regression-test/data/nereids_syntax_p0/test_materialized_view_nereids.out b/regression-test/data/nereids_syntax_p0/test_materialized_view_nereids.out
new file mode 100644
index 0000000000..74bac182b1
--- /dev/null
+++ b/regression-test/data/nereids_syntax_p0/test_materialized_view_nereids.out
@@ -0,0 +1,52 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+test_materialized_view1 DUP_KEYS record_id INT Yes true \N true
+ seller_id INT Yes true \N true
+ store_id INT Yes true \N true
+ sale_date DATE Yes false \N NONE true
+ sale_amt BIGINT Yes false \N NONE true
+
+amt_sum AGG_KEYS store_id INT Yes true \N true
+ sale_amt BIGINT Yes false \N SUM true
+
+-- !sql --
+test_materialized_view2 DUP_KEYS record_id INT Yes true \N true
+ seller_id INT Yes true \N true
+ store_id INT Yes true \N true
+ sale_date DATE Yes false \N NONE true
+ sale_amt BIGINT Yes false \N NONE true
+
+seller_id_order DUP_KEYS store_id INT Yes true \N true
+ seller_id INT Yes true \N true
+ sale_amt BIGINT Yes false \N NONE true
+
+-- !sql --
+1 1 1 2020-05-30 100
+2 1 1 2020-05-30 100
+
+-- !sql --
+1 200
+
+-- !sql --
+1 1 1 2020-05-30 100
+2 1 1 2020-05-30 100
+
+-- !sql --
+1 200
+
+-- !sql --
+
+
+ CASE WHEN sale_amt IS NULL THEN 0 ELSE 1 END BIGINT No false \N SUM true
+ sale_amt BIGINT Yes false \N NONE true
+ sale_amt BIGINT Yes false \N SUM true
+ sale_date DATE Yes false \N NONE true
+ seller_id INT Yes true \N true
+ store_id INT Yes true \N true
+amt_count AGG_KEYS store_id INT Yes true \N true
+amt_sum AGG_KEYS store_id INT Yes true \N true
+test_materialized_view1 DUP_KEYS record_id INT Yes true \N true
+
+-- !sql --
+1 2
+
diff --git a/regression-test/suites/nereids_syntax_p0/test_materialized_view_nereids.groovy b/regression-test/suites/nereids_syntax_p0/test_materialized_view_nereids.groovy
new file mode 100644
index 0000000000..d6d990a1c6
--- /dev/null
+++ b/regression-test/suites/nereids_syntax_p0/test_materialized_view_nereids.groovy
@@ -0,0 +1,124 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+suite("test_materialized_view_nereids") {
+ def tbName1 = "test_materialized_view1"
+ def tbName2 = "test_materialized_view2"
+
+ def getJobState = { tableName ->
+ def jobStateResult = sql """ SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1; """
+ return jobStateResult[0][8]
+ }
+ sql "DROP TABLE IF EXISTS ${tbName1} FORCE"
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tbName1}(
+ record_id int,
+ seller_id int,
+ store_id int,
+ sale_date date,
+ sale_amt bigint
+ )
+ DISTRIBUTED BY HASH(record_id) properties("replication_num" = "1");
+ """
+ sql "DROP TABLE IF EXISTS ${tbName2} FORCE"
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tbName2}(
+ record_id int,
+ seller_id int,
+ store_id int,
+ sale_date date,
+ sale_amt bigint
+ )
+ DISTRIBUTED BY HASH(record_id) properties("replication_num" = "1");
+ """
+ sql "CREATE materialized VIEW amt_sum AS SELECT store_id, sum(sale_amt) FROM ${tbName1} GROUP BY store_id;"
+ int max_try_secs = 60
+ while (max_try_secs--) {
+ String res = getJobState(tbName1)
+ if (res == "FINISHED") {
+ break
+ } else {
+ Thread.sleep(2000)
+ if (max_try_secs < 1) {
+ println "test timeout," + "state:" + res
+ assertEquals("FINISHED",res)
+ }
+ }
+ }
+ sql "CREATE materialized VIEW seller_id_order AS SELECT store_id,seller_id, sale_amt FROM ${tbName2} ORDER BY store_id,seller_id;"
+ max_try_secs = 60
+ while (max_try_secs--) {
+ String res = getJobState(tbName2)
+ if (res == "FINISHED") {
+ break
+ } else {
+ Thread.sleep(2000)
+ if (max_try_secs < 1) {
+ println "test timeout," + "state:" + res
+ assertEquals("FINISHED",res)
+ }
+ }
+ }
+ sql "SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName='${tbName1}';"
+ sql "SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName='${tbName2}';"
+ qt_sql "DESC ${tbName1} ALL;"
+ qt_sql "DESC ${tbName2} ALL;"
+ sql "insert into ${tbName1} values(1, 1, 1, '2020-05-30',100);"
+ sql "insert into ${tbName1} values(2, 1, 1, '2020-05-30',100);"
+ sql "insert into ${tbName2} values(1, 1, 1, '2020-05-30',100);"
+ sql "insert into ${tbName2} values(2, 1, 1, '2020-05-30',100);"
+ Thread.sleep(1000)
+
+ sql 'set enable_vectorized_engine=true;'
+ sql 'set enable_fallback_to_original_planner=false;'
+ sql 'set enable_nereids_planner=true;'
+
+ explain{
+ sql("SELECT store_id, sum(sale_amt) FROM ${tbName1} GROUP BY store_id")
+ contains("(amt_sum)")
+ }
+ qt_sql "SELECT * FROM ${tbName1} order by record_id;"
+ qt_sql "SELECT store_id, sum(sale_amt) FROM ${tbName1} GROUP BY store_id order by store_id;"
+ qt_sql "SELECT * FROM ${tbName2} order by record_id;"
+ qt_sql "SELECT store_id, sum(sale_amt) FROM ${tbName2} GROUP BY store_id order by store_id;"
+
+ sql "CREATE materialized VIEW amt_count AS SELECT store_id, count(sale_amt) FROM ${tbName1} GROUP BY store_id;"
+ max_try_secs = 60
+ while (max_try_secs--) {
+ String res = getJobState(tbName1)
+ if (res == "FINISHED") {
+ break
+ } else {
+ Thread.sleep(2000)
+ if (max_try_secs < 1) {
+ println "test timeout," + "state:" + res
+ assertEquals("FINISHED",res)
+ }
+ }
+ }
+ sql "SELECT store_id, count(sale_amt) FROM ${tbName1} GROUP BY store_id;"
+ order_qt_sql "DESC ${tbName1} ALL;"
+
+ qt_sql "SELECT store_id, count(sale_amt) FROM ${tbName1} GROUP BY store_id;"
+
+ explain {
+ sql("SELECT store_id, count(sale_amt) FROM ${tbName1} GROUP BY store_id;")
+ contains "(amt_count)"
+ }
+
+ sql "DROP TABLE ${tbName1} FORCE;"
+ sql "DROP TABLE ${tbName2} FORCE;"
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org