You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2024/01/10 02:34:24 UTC
(doris) branch branch-2.1 updated (767de7afe86 -> fae7a395de7)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a change to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
from 767de7afe86 Revert "[feature](pipelineX) control exchange sink by memory usage (#28814)" (#29652)
new e2ccca6290e [pipelineX](bug) Fix runtime filter timeout (#29738)
new 3675e0302c6 [fix](nereids) generate correct order for runtime filter when contains NullSafeEquals hash condition (#29726)
new fae7a395de7 [Bug](FS) Fix S3FileSystem::file_size_impl #29720
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/io/fs/s3_file_system.cpp | 5 ++-
be/src/pipeline/exec/hashjoin_build_sink.cpp | 4 +-
.../processor/post/RuntimeFilterGenerator.java | 22 +++++-----
.../runtime_filter/runtime_filter.groovy | 48 ++++++++++++++++++++++
4 files changed, 66 insertions(+), 13 deletions(-)
create mode 100644 regression-test/suites/nereids_p0/runtime_filter/runtime_filter.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 02/03: [fix](nereids) generate correct order for runtime filter when contains NullSafeEquals hash condition (#29726)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 3675e0302c6cd5f8b711eff76a7cbd2a751a08ce
Author: minghong <en...@gmail.com>
AuthorDate: Wed Jan 10 10:23:44 2024 +0800
[fix](nereids) generate correct order for runtime filter when contains NullSafeEquals hash condition (#29726)
Be do not support RF for NullSafeEquals, so fe not generate RF for them.
However, after we support NullSafeEquals as Hash join condition,
the order of RF is wrong when generating RF in FE. this PR fix it.
---
.../processor/post/RuntimeFilterGenerator.java | 22 +++++-----
.../runtime_filter/runtime_filter.groovy | 48 ++++++++++++++++++++++
2 files changed, 60 insertions(+), 10 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java
index 0d9471287d7..b73b6e8bdad 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java
@@ -368,18 +368,20 @@ public class RuntimeFilterGenerator extends PlanPostProcessor {
.filter(type -> (type.getValue() & ctx.getSessionVariable().getRuntimeFilterType()) > 0)
.collect(Collectors.toList());
- List<EqualTo> hashJoinConjuncts = join.getEqualToConjuncts();
+ List<Expression> hashJoinConjuncts = join.getHashJoinConjuncts().stream().collect(Collectors.toList());
for (int i = 0; i < hashJoinConjuncts.size(); i++) {
- EqualTo equalTo = ((EqualTo) JoinUtils.swapEqualToForChildrenOrder(
- hashJoinConjuncts.get(i), join.left().getOutputSet()));
- for (TRuntimeFilterType type : legalTypes) {
- //bitmap rf is generated by nested loop join.
- if (type == TRuntimeFilterType.BITMAP) {
- continue;
+ if (hashJoinConjuncts.get(i) instanceof EqualTo) {
+ EqualTo equalTo = ((EqualTo) JoinUtils.swapEqualToForChildrenOrder(
+ (EqualTo) hashJoinConjuncts.get(i), join.left().getOutputSet()));
+ for (TRuntimeFilterType type : legalTypes) {
+ //bitmap rf is generated by nested loop join.
+ if (type == TRuntimeFilterType.BITMAP) {
+ continue;
+ }
+ long buildSideNdv = getBuildSideNdv(join, equalTo);
+ join.pushDownRuntimeFilter(context, generator, join, equalTo.right(),
+ equalTo.left(), type, buildSideNdv, i);
}
- long buildSideNdv = getBuildSideNdv(join, equalTo);
- join.pushDownRuntimeFilter(context, generator, join, equalTo.right(),
- equalTo.left(), type, buildSideNdv, i);
}
}
}
diff --git a/regression-test/suites/nereids_p0/runtime_filter/runtime_filter.groovy b/regression-test/suites/nereids_p0/runtime_filter/runtime_filter.groovy
new file mode 100644
index 00000000000..905ac6dd87b
--- /dev/null
+++ b/regression-test/suites/nereids_p0/runtime_filter/runtime_filter.groovy
@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("runtime_filter") {
+ sql ''' drop table if exists rf_dws_asset_domain_statistics_daily'''
+ sql '''CREATE TABLE rf_dws_asset_domain_statistics_daily (
+ account_id int(11) NULL,
+ ssp_id int(11) NULL,
+ account_name varchar(500) NULL,
+ d_s date NOT NULL
+ ) ENGINE = OLAP
+ DUPLICATE KEY(account_id, ssp_id, account_name) COMMENT 'OLAP'
+ PARTITION BY RANGE(d_s) (PARTITION p20231220 VALUES [('2023-12-20'), ('2023-12-21')))
+ DISTRIBUTED BY HASH(account_name) BUCKETS 9
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ ); '''
+
+ sql "set runtime_filter_mode=GLOBAL"
+
+ explain {
+ sql """
+ SELECT count(*) FROM
+ rf_dws_asset_domain_statistics_daily t1
+ INNER JOIN (
+ SELECT account_id, account_name
+ FROM dws_asset_domain_statistics_daily
+ WHERE d_s = '2023-12-20'
+ ) t2
+ ON (t1.account_id <=> t2.account_id);
+ """
+ notContains("RFs")
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 01/03: [pipelineX](bug) Fix runtime filter timeout (#29738)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit e2ccca6290e34c6cc8c6c1e657c85267f4b787aa
Author: Gabriel <ga...@gmail.com>
AuthorDate: Tue Jan 9 22:45:59 2024 +0800
[pipelineX](bug) Fix runtime filter timeout (#29738)
---
be/src/pipeline/exec/hashjoin_build_sink.cpp | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/be/src/pipeline/exec/hashjoin_build_sink.cpp b/be/src/pipeline/exec/hashjoin_build_sink.cpp
index f34f835d121..757673c70a8 100644
--- a/be/src/pipeline/exec/hashjoin_build_sink.cpp
+++ b/be/src/pipeline/exec/hashjoin_build_sink.cpp
@@ -388,7 +388,9 @@ HashJoinBuildSinkOperatorX::HashJoinBuildSinkOperatorX(ObjectPool* pool, int ope
_partition_exprs(tnode.__isset.distribute_expr_lists && !_is_broadcast_join
? tnode.distribute_expr_lists[1]
: std::vector<TExpr> {}),
- _use_global_rf(use_global_rf) {}
+ _use_global_rf(use_global_rf) {
+ _runtime_filter_descs = tnode.runtime_filters;
+}
Status HashJoinBuildSinkOperatorX::prepare(RuntimeState* state) {
if (_is_broadcast_join) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 03/03: [Bug](FS) Fix S3FileSystem::file_size_impl #29720
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit fae7a395de7961bd8907a4b0208d8b9328191c89
Author: plat1ko <pl...@gmail.com>
AuthorDate: Wed Jan 10 10:31:40 2024 +0800
[Bug](FS) Fix S3FileSystem::file_size_impl #29720
---
be/src/io/fs/s3_file_system.cpp | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/be/src/io/fs/s3_file_system.cpp b/be/src/io/fs/s3_file_system.cpp
index c7d4e07fd1e..77f303ffdc0 100644
--- a/be/src/io/fs/s3_file_system.cpp
+++ b/be/src/io/fs/s3_file_system.cpp
@@ -299,11 +299,12 @@ Status S3FileSystem::file_size_impl(const Path& file, int64_t* file_size) const
auto outcome = client->HeadObject(request);
s3_bvar::s3_head_total << 1;
- if (outcome.IsSuccess()) {
- *file_size = outcome.GetResult().GetContentLength();
+ if (!outcome.IsSuccess()) {
return s3fs_error(outcome.GetError(),
fmt::format("failed to get file size {}", full_path(key)));
}
+
+ *file_size = outcome.GetResult().GetContentLength();
return Status::OK();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org