You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by da...@apache.org on 2023/07/18 06:46:36 UTC
[doris] branch branch-2.0 updated (9bb22d480b -> 9357a7f9fa)
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a change to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
from 9bb22d480b [fix](clone) fix cannot further repair clone replica which miss version data (#21382)
new c05c9ded26 [fix](load) fix dead loop in _handle_mem_exceed_limit function when reduce memory for load (#21886)
new d324bd3e82 [fix](planner)need recalculate nullable info of output slots for join node (#21650)
new 9357a7f9fa [fix](planner) push conjuncts into SetOperationStmt inline view (#21718)
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/runtime/load_channel_mgr.cpp | 7 +-
.../apache/doris/analysis/ExprSubstitutionMap.java | 10 ++
.../org/apache/doris/planner/JoinNodeBase.java | 9 ++
.../apache/doris/planner/SingleNodePlanner.java | 23 ++++
.../correctness_p0/test_many_inlineview.groovy | 151 +++++++++++++++++++++
.../test_push_conjuncts_inlineview.groovy | 65 +++++++++
6 files changed, 262 insertions(+), 3 deletions(-)
create mode 100644 regression-test/suites/correctness_p0/test_many_inlineview.groovy
create mode 100644 regression-test/suites/correctness_p0/test_push_conjuncts_inlineview.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 03/03: [fix](planner) push conjuncts into SetOperationStmt inline view (#21718)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 9357a7f9fa216d919e44ebc343b078f22800ad07
Author: starocean999 <40...@users.noreply.github.com>
AuthorDate: Tue Jul 18 14:17:07 2023 +0800
[fix](planner) push conjuncts into SetOperationStmt inline view (#21718)
* [fix](planner)push conjuncts into SetOperationStmt inline view
---
.../apache/doris/planner/SingleNodePlanner.java | 23 ++++++++
.../test_push_conjuncts_inlineview.groovy | 65 ++++++++++++++++++++++
2 files changed, 88 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
index 82495ce243..0d3d65e88c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
@@ -21,6 +21,7 @@
package org.apache.doris.planner;
import org.apache.doris.analysis.AggregateInfo;
+import org.apache.doris.analysis.AnalyticExpr;
import org.apache.doris.analysis.AnalyticInfo;
import org.apache.doris.analysis.Analyzer;
import org.apache.doris.analysis.AssertNumRowsElement;
@@ -272,6 +273,13 @@ public class SingleNodePlanner {
AggregateInfo aggInfo = selectStmt.getAggInfo();
root = analyticPlanner.createSingleNodePlan(root,
aggInfo != null ? aggInfo.getGroupingExprs() : null, inputPartitionExprs);
+ List<Expr> predicates = getBoundPredicates(analyzer,
+ selectStmt.getAnalyticInfo().getOutputTupleDesc());
+ if (!predicates.isEmpty()) {
+ root = new SelectNode(ctx.getNextNodeId(), root, predicates);
+ root.init(analyzer);
+ Preconditions.checkState(root.hasValidStats());
+ }
if (aggInfo != null && !inputPartitionExprs.isEmpty()) {
// analytic computation will benefit from a partition on inputPartitionExprs
aggInfo.setPartitionExprs(inputPartitionExprs);
@@ -1813,6 +1821,17 @@ public class SingleNodePlanner {
e.setIsOnClauseConjunct(false);
}
inlineViewRef.getAnalyzer().registerConjuncts(viewPredicates, inlineViewRef.getAllTupleIds());
+ QueryStmt queryStmt = inlineViewRef.getQueryStmt();
+ if (queryStmt instanceof SetOperationStmt) {
+ // registerConjuncts for every set operand
+ SetOperationStmt setOperationStmt = (SetOperationStmt) queryStmt;
+ for (SetOperationStmt.SetOperand setOperand : setOperationStmt.getOperands()) {
+ setOperand.getAnalyzer().registerConjuncts(
+ Expr.substituteList(viewPredicates, setOperand.getSmap(),
+ setOperand.getAnalyzer(), false),
+ inlineViewRef.getAllTupleIds());
+ }
+ }
// mark (fully resolve) slots referenced by remaining unassigned conjuncts as
// materialized
@@ -2746,6 +2765,10 @@ public class SingleNodePlanner {
}
sourceExpr = slotDesc.getSourceExprs().get(0);
}
+ if (sourceExpr instanceof AnalyticExpr) {
+ isAllSlotReferToGroupBys = false;
+ break;
+ }
// if grouping set is given and column is not in all grouping set list
// we cannot push the predicate since the column value can be null
if (stmt.getGroupByClause() == null) {
diff --git a/regression-test/suites/correctness_p0/test_push_conjuncts_inlineview.groovy b/regression-test/suites/correctness_p0/test_push_conjuncts_inlineview.groovy
new file mode 100644
index 0000000000..80a35194f6
--- /dev/null
+++ b/regression-test/suites/correctness_p0/test_push_conjuncts_inlineview.groovy
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+ // or more contributor license agreements. See the NOTICE file
+ // distributed with this work for additional information
+ // regarding copyright ownership. The ASF licenses this file
+ // to you under the Apache License, Version 2.0 (the
+ // "License"); you may not use this file except in compliance
+ // with the License. You may obtain a copy of the License at
+ //
+ // http://www.apache.org/licenses/LICENSE-2.0
+ //
+ // Unless required by applicable law or agreed to in writing,
+ // software distributed under the License is distributed on an
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ // KIND, either express or implied. See the License for the
+ // specific language governing permissions and limitations
+ // under the License.
+
+suite("test_push_conjuncts_inlineview") {
+ sql """ set enable_nereids_planner=false"""
+ sql """ DROP TABLE IF EXISTS `push_conjunct_table` """
+ sql """
+ CREATE TABLE `push_conjunct_table` (
+ `a_key` varchar(255) NULL ,
+ `d_key` varchar(255) NULL ,
+ `c_key` varchar(32) NULL ,
+ `b_key` date NOT NULL
+ ) ENGINE=OLAP
+ UNIQUE KEY(`a_key`, `d_key`, `c_key`)
+ DISTRIBUTED BY HASH(`a_key`, `d_key`, `c_key`) BUCKETS 4
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "in_memory" = "false",
+ "storage_format" = "V2",
+ "disable_auto_compaction" = "false"
+ );
+ """
+ explain {
+ sql("""select
+ 1
+ from
+ (
+ select
+ rank() over(
+ partition by a_key
+ , c_key
+ , d_key
+ order by
+ b_key desc
+ ) as px
+ from
+ push_conjunct_table a
+
+ union all
+ select 2 as px
+ from
+ push_conjunct_table a
+ )a
+ where
+ a.px = 1;""")
+ contains "4:VSELECT"
+ }
+
+ sql """ DROP TABLE IF EXISTS `push_conjunct_table` """
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/03: [fix](load) fix dead loop in _handle_mem_exceed_limit function when reduce memory for load (#21886)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit c05c9ded2632197da69180cb6c57ac858fd73c85
Author: Xin Liao <li...@126.com>
AuthorDate: Tue Jul 18 09:49:36 2023 +0800
[fix](load) fix dead loop in _handle_mem_exceed_limit function when reduce memory for load (#21886)
---
be/src/runtime/load_channel_mgr.cpp | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/be/src/runtime/load_channel_mgr.cpp b/be/src/runtime/load_channel_mgr.cpp
index 2a0020a9ff..3d9dbc6f19 100644
--- a/be/src/runtime/load_channel_mgr.cpp
+++ b/be/src/runtime/load_channel_mgr.cpp
@@ -371,11 +371,12 @@ void LoadChannelMgr::_handle_mem_exceed_limit() {
continue;
}
all_writers_mem.emplace_back(kv.second, item.first, std::move(item.second));
- size_t pos = all_writers_mem.size() - 1;
- tablets_mem_heap.emplace(std::get<2>(all_writers_mem[pos]).begin(),
- std::get<2>(all_writers_mem[pos]).end(), pos);
}
}
+ for (size_t i = 0; i < all_writers_mem.size(); i++) {
+ tablets_mem_heap.emplace(std::get<2>(all_writers_mem[i]).begin(),
+ std::get<2>(all_writers_mem[i]).end(), i);
+ }
// reduce 1/10 memory every time
int64_t mem_to_flushed = _mem_tracker->consumption() / 10;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 02/03: [fix](planner)need recalculate nullable info of output slots for join node (#21650)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit d324bd3e82836851f5e43f53b9fae20bc3895cbb
Author: starocean999 <40...@users.noreply.github.com>
AuthorDate: Tue Jul 18 12:10:27 2023 +0800
[fix](planner)need recalculate nullable info of output slots for join node (#21650)
* [fix](planner)need recalculate nullable info of output slots for join node
---
.../apache/doris/analysis/ExprSubstitutionMap.java | 10 ++
.../org/apache/doris/planner/JoinNodeBase.java | 9 ++
.../correctness_p0/test_many_inlineview.groovy | 151 +++++++++++++++++++++
3 files changed, 170 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprSubstitutionMap.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprSubstitutionMap.java
index cef2cc75bd..5c346ef576 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprSubstitutionMap.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprSubstitutionMap.java
@@ -346,4 +346,14 @@ public final class ExprSubstitutionMap {
public ExprSubstitutionMap clone() {
return new ExprSubstitutionMap(Expr.cloneList(lhs), Expr.cloneList(rhs));
}
+
+ public void reCalculateNullableInfoForSlotInRhs() {
+ Preconditions.checkState(lhs.size() == rhs.size(), "lhs and rhs must be same size");
+ for (int i = 0; i < rhs.size(); i++) {
+ if (rhs.get(i) instanceof SlotRef) {
+ ((SlotRef) rhs.get(i)).getDesc().setIsNullable(lhs.get(i).isNullable()
+ || ((SlotRef) rhs.get(i)).getDesc().getIsNullable());
+ }
+ }
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/JoinNodeBase.java b/fe/fe-core/src/main/java/org/apache/doris/planner/JoinNodeBase.java
index cc26086144..11e9c31e8a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/JoinNodeBase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/JoinNodeBase.java
@@ -408,6 +408,15 @@ public abstract class JoinNodeBase extends PlanNode {
conjuncts = Expr.substituteList(conjuncts, originToIntermediateSmap, analyzer, false);
// 5. replace tuple is null expr
TupleIsNullPredicate.substitueListForTupleIsNull(vSrcToOutputSMap.getLhs(), originTidsToIntermediateTidMap);
+
+ Preconditions.checkState(vSrcToOutputSMap.getLhs().size() == vOutputTupleDesc.getSlots().size());
+ List<Expr> exprs = vSrcToOutputSMap.getLhs();
+ ArrayList<SlotDescriptor> slots = vOutputTupleDesc.getSlots();
+ for (int i = 0; i < slots.size(); i++) {
+ slots.get(i).setIsNullable(exprs.get(i).isNullable());
+ }
+ vSrcToOutputSMap.reCalculateNullableInfoForSlotInRhs();
+ vOutputTupleDesc.computeMemLayout();
}
protected abstract List<SlotId> computeSlotIdsForJoinConjuncts(Analyzer analyzer);
diff --git a/regression-test/suites/correctness_p0/test_many_inlineview.groovy b/regression-test/suites/correctness_p0/test_many_inlineview.groovy
new file mode 100644
index 0000000000..651b32411e
--- /dev/null
+++ b/regression-test/suites/correctness_p0/test_many_inlineview.groovy
@@ -0,0 +1,151 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Test some keywords that may conflict.
+// For example, "bin" is used for function "bin",
+// and also used "show catalog recycle bin"
+suite("test_many_inlineview") {
+ sql """
+ drop table if exists ods_drp_ch_sys_codelist_et;
+ """
+
+ sql """
+ drop table if exists ods_drp_xpi_storemain;
+ """
+
+ sql """
+ drop table if exists ods_drp_xpi_storechannel;
+ """
+
+ sql """
+ drop table if exists ods_drp_ch_shop_et;
+ """
+
+ sql """
+ CREATE TABLE `ods_drp_ch_sys_codelist_et` (
+ `PARAM_TYPE_CODE` varchar(96) NOT NULL COMMENT '',
+ `PARAM_CODE` varchar(600) NOT NULL COMMENT '',
+ `PARAM_NAME` varchar(600) NOT NULL COMMENT ''
+
+ ) ENGINE=OLAP
+ UNIQUE KEY(`PARAM_TYPE_CODE`)
+ COMMENT 'ods_drp_ch_sys_codelist_et'
+ DISTRIBUTED BY HASH(`PARAM_TYPE_CODE`) BUCKETS 4
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "in_memory" = "false",
+ "storage_format" = "V2",
+ "disable_auto_compaction" = "false"
+ );
+ """
+
+ sql """
+ CREATE TABLE `ods_drp_xpi_storemain` (
+ `fID` varchar(300) NOT NULL COMMENT 'ID',
+ `fStoreCode` varchar(150) NULL COMMENT '店铺编码',
+ ) ENGINE=OLAP
+ UNIQUE KEY(`fID`)
+ DISTRIBUTED BY HASH(`fID`) BUCKETS 4
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "in_memory" = "false",
+ "storage_format" = "V2",
+ "disable_auto_compaction" = "false"
+ );
+ """
+
+ sql """
+ CREATE TABLE `ods_drp_xpi_storechannel` (
+ `fFlowStoreID` varchar(300) NULL COMMENT '',
+ `fBusinessMarket` varchar(150) NULL COMMENT ''
+
+ ) ENGINE=OLAP
+ UNIQUE KEY(`fFlowStoreID`)
+ COMMENT 'ods_drp_xpi_storechannel'
+ DISTRIBUTED BY HASH(`fFlowStoreID`) BUCKETS 4
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "in_memory" = "false",
+ "storage_format" = "V2",
+ "disable_auto_compaction" = "false"
+ );
+ """
+
+ sql """
+ CREATE TABLE `ods_drp_ch_shop_et` (
+ `SHOP_CODE` varchar(60) NULL COMMENT ''
+ ) ENGINE=OLAP
+ UNIQUE KEY(`SHOP_CODE`)
+ COMMENT 'ods_drp_ch_shop_et'
+ DISTRIBUTED BY HASH(`SHOP_CODE`) BUCKETS 4
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "in_memory" = "false",
+ "storage_format" = "V2",
+ "disable_auto_compaction" = "false"
+ );
+ """
+
+ explain {
+ sql(""" verbose
+ SELECT `a`.`shopcode` AS `shop_code` ,
+ `a`.`BusinessMarket` AS `business_market`
+ FROM
+ (SELECT `t1`.`fStoreCode` AS `shopcode` ,
+ ifnull(`sc_c1`.`PARAM_NAME`,
+ `sc`.`fBusinessMarket`) AS `BusinessMarket`
+ FROM `ods_drp_xpi_storemain` t1 left outer
+ JOIN `ods_drp_xpi_storechannel` sc
+ ON `sc`.`fFlowStoreID` = `t1`.`fID` left outer
+ JOIN `ods_drp_ch_sys_codelist_et` sc_c1
+ ON ( `sc`.`fBusinessMarket` = `sc_c1`.`PARAM_CODE` )
+ AND ( `sc_c1`.`PARAM_TYPE_CODE` = 'st_market_segment' )
+ UNION
+ SELECT `t1`.`fStoreCode` AS `shopCode`,
+ ifnull(`sc_c1`.`PARAM_NAME`,
+ `sc`.`fBusinessMarket`) AS `paramgradation`
+ FROM `ods_drp_xpi_storemain` t1 left outer
+ JOIN `ods_drp_xpi_storechannel` sc
+ ON `sc`.`fFlowStoreID` = `t1`.`fID` left outer
+ JOIN `ods_drp_ch_sys_codelist_et` sc_c1
+ ON ( `sc`.`fBusinessMarket` = `sc_c1`.`PARAM_CODE` )
+ AND ( `sc_c1`.`PARAM_TYPE_CODE` = 'st_market_segment' )) a left outer
+ JOIN `ods_drp_ch_shop_et` b
+ ON `a`.`shopcode` = `b`.`shop_code`
+ UNION all
+ SELECT `a`.`SHOP_CODE` AS `shop_code` ,
+ NULL AS `business_market`
+ FROM `ods_drp_ch_shop_et` a;""")
+ notContains "fBusinessMarket, colUniqueId=1, type=varchar(150), nullable=false"
+ }
+
+ sql """
+ drop table if exists ods_drp_ch_sys_codelist_et;
+ """
+
+ sql """
+ drop table if exists ods_drp_xpi_storemain;
+ """
+
+ sql """
+ drop table if exists ods_drp_xpi_storechannel;
+ """
+
+ sql """
+ drop table if exists ods_drp_ch_shop_et;
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org