You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by st...@apache.org on 2024/01/26 10:45:26 UTC
(doris) branch master updated: [feature](nereids)support mark join (#30133)
This is an automated email from the ASF dual-hosted git repository.
starocean999 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 781477ad0b8 [feature](nereids)support mark join (#30133)
781477ad0b8 is described below
commit 781477ad0b8edb830dc92d326d4d44f1940fc448
Author: starocean999 <40...@users.noreply.github.com>
AuthorDate: Fri Jan 26 18:45:18 2024 +0800
[feature](nereids)support mark join (#30133)
Co-authored-by: Jerry Hu <mr...@gmail.com>
---
be/src/pipeline/exec/hashjoin_build_sink.cpp | 49 +-
be/src/pipeline/exec/hashjoin_build_sink.h | 7 +
be/src/pipeline/exec/hashjoin_probe_operator.cpp | 71 +-
be/src/pipeline/exec/hashjoin_probe_operator.h | 18 +
be/src/pipeline/exec/join_build_sink_operator.cpp | 12 +-
be/src/pipeline/exec/join_probe_operator.cpp | 13 +-
be/src/pipeline/exec/join_probe_operator.h | 2 +
be/src/pipeline/pipeline_x/dependency.h | 1 +
be/src/vec/common/hash_table/hash_map.h | 2 +-
be/src/vec/common/hash_table/join_hash_table.h | 145 +-
.../exec/join/null_aware_left_semi_join_impl.cpp | 26 +
be/src/vec/exec/join/process_hash_table_probe.h | 6 +-
.../vec/exec/join/process_hash_table_probe_impl.h | 155 +-
be/src/vec/exec/join/vhash_join_node.cpp | 112 +-
be/src/vec/exec/join/vhash_join_node.h | 24 +-
be/src/vec/exec/join/vjoin_node_base.cpp | 34 +-
be/src/vec/exec/join/vjoin_node_base.h | 6 +-
be/src/vec/exprs/vexpr_context.cpp | 44 +
be/src/vec/exprs/vexpr_context.h | 4 +
be/src/vec/runtime/shared_hash_table_controller.h | 1 +
.../org/apache/doris/analysis/JoinOperator.java | 4 +-
.../glue/translator/PhysicalPlanTranslator.java | 50 +-
.../org/apache/doris/nereids/rules/RuleType.java | 1 +
.../rules/analysis/CollectJoinConstraint.java | 2 +-
.../nereids/rules/analysis/SubqueryToApply.java | 60 +-
.../rules/expression/ExpressionRewrite.java | 47 +-
.../TrySimplifyPredicateWithMarkJoinSlot.java | 113 ++
.../implementation/LogicalJoinToHashJoin.java | 1 +
.../LogicalJoinToNestedLoopJoin.java | 1 +
.../rules/rewrite/AdjustConjunctsReturnType.java | 5 +-
.../nereids/rules/rewrite/AdjustNullable.java | 16 +-
.../nereids/rules/rewrite/CheckDataTypes.java | 8 +-
.../rules/rewrite/ConvertInnerOrCrossJoin.java | 6 +-
.../rules/rewrite/EliminateDedupJoinCondition.java | 7 +-
.../rules/rewrite/EliminateJoinCondition.java | 8 +-
.../nereids/rules/rewrite/EliminateMarkJoin.java | 59 +
.../rewrite/EliminateNullAwareLeftAntiJoin.java | 5 +-
.../nereids/rules/rewrite/EliminateSemiJoin.java | 2 +
.../rules/rewrite/ExtractFilterFromCrossJoin.java | 4 +-
.../rules/rewrite/FindHashConditionForJoin.java | 1 +
.../doris/nereids/rules/rewrite/InApplyToJoin.java | 66 +-
.../doris/nereids/rules/rewrite/OrExpansion.java | 1 +
.../rules/rewrite/PushDownAliasThroughJoin.java | 3 +-
.../PushDownExpressionsInHashCondition.java | 13 +
.../rules/rewrite/PushDownFilterThroughJoin.java | 1 +
.../rules/rewrite/PushDownJoinOtherCondition.java | 3 +-
.../rewrite/UnCorrelatedApplyAggregateFilter.java | 2 +-
.../rules/rewrite/UnCorrelatedApplyFilter.java | 2 +-
.../rewrite/UnCorrelatedApplyProjectFilter.java | 2 +-
.../apache/doris/nereids/stats/JoinEstimation.java | 12 +-
.../trees/copier/LogicalPlanDeepCopier.java | 8 +-
.../doris/nereids/trees/plans/algebra/Join.java | 23 +-
.../nereids/trees/plans/logical/LogicalApply.java | 33 +-
.../nereids/trees/plans/logical/LogicalJoin.java | 270 ++--
.../nereids/trees/plans/logical/UsingJoin.java | 26 +-
.../trees/plans/physical/AbstractPhysicalJoin.java | 61 +-
.../trees/plans/physical/PhysicalHashJoin.java | 59 +-
.../plans/physical/PhysicalNestedLoopJoin.java | 73 +-
.../apache/doris/nereids/util/ExpressionUtils.java | 72 +
.../org/apache/doris/nereids/util/JoinUtils.java | 17 +-
.../org/apache/doris/planner/HashJoinNode.java | 95 ++
.../apache/doris/planner/NestedLoopJoinNode.java | 17 +
.../rules/rewrite/ExistsApplyToJoinTest.java | 8 +-
.../nereids/trees/plans/PlanToStringTest.java | 2 +-
gensrc/thrift/PlanNodes.thrift | 6 +-
.../nereids_p0/subquery/subquery_unnesting.out | 1494 ++++++++++++++++++++
.../subquery/test_subquery_in_project.out | 120 ++
.../shape/query10.out | 85 +-
.../shape/query35.out | 53 +-
.../noStatsRfPrune/query10.out | 49 +-
.../noStatsRfPrune/query35.out | 51 +-
.../no_stats_shape/query10.out | 49 +-
.../no_stats_shape/query35.out | 51 +-
.../rf_prune/query10.out | 83 +-
.../rf_prune/query35.out | 53 +-
.../nereids_tpcds_shape_sf100_p0/shape/query10.out | 83 +-
.../nereids_tpcds_shape_sf100_p0/shape/query35.out | 53 +-
.../nereids_p0/subquery/subquery_unnesting.groovy | 132 ++
.../subquery/test_subquery_in_project.groovy | 55 +
79 files changed, 3657 insertions(+), 761 deletions(-)
diff --git a/be/src/pipeline/exec/hashjoin_build_sink.cpp b/be/src/pipeline/exec/hashjoin_build_sink.cpp
index 4ea3b006868..0c94ffe8a58 100644
--- a/be/src/pipeline/exec/hashjoin_build_sink.cpp
+++ b/be/src/pipeline/exec/hashjoin_build_sink.cpp
@@ -76,6 +76,12 @@ Status HashJoinBuildSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo
_shared_hash_table_dependency->block();
p._shared_hashtable_controller->append_dependency(p.node_id(),
_shared_hash_table_dependency);
+ } else {
+ if ((p._join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN ||
+ p._join_op == TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN) &&
+ p._have_other_join_conjunct) {
+ _build_indexes_null = std::make_shared<std::vector<uint32_t>>();
+ }
}
_memory_usage_counter = ADD_LABEL_COUNTER(profile(), "MemoryUsage");
@@ -250,44 +256,46 @@ Status HashJoinBuildSinkLocalState::process_build_block(RuntimeState* state,
st = std::visit(
Overload {[&](std::monostate& arg, auto join_op, auto has_null_value,
- auto short_circuit_for_null_in_build_side) -> Status {
+ auto short_circuit_for_null_in_build_side,
+ auto with_other_conjuncts) -> Status {
LOG(FATAL) << "FATAL: uninited hash table";
__builtin_unreachable();
return Status::OK();
},
[&](auto&& arg, auto&& join_op, auto has_null_value,
- auto short_circuit_for_null_in_build_side) -> Status {
+ auto short_circuit_for_null_in_build_side,
+ auto with_other_conjuncts) -> Status {
using HashTableCtxType = std::decay_t<decltype(arg)>;
using JoinOpType = std::decay_t<decltype(join_op)>;
vectorized::ProcessHashTableBuild<HashTableCtxType,
HashJoinBuildSinkLocalState>
hash_table_build_process(rows, block, raw_ptrs, this,
state->batch_size(), state);
- return hash_table_build_process
- .template run<JoinOpType::value, has_null_value,
- short_circuit_for_null_in_build_side>(
- arg,
- has_null_value || short_circuit_for_null_in_build_side
- ? &null_map_val->get_data()
- : nullptr,
- &_shared_state->_has_null_in_build_side);
+ return hash_table_build_process.template run<
+ JoinOpType::value, has_null_value,
+ short_circuit_for_null_in_build_side, with_other_conjuncts>(
+ arg,
+ has_null_value || short_circuit_for_null_in_build_side
+ ? &null_map_val->get_data()
+ : nullptr,
+ &_shared_state->_has_null_in_build_side);
}},
*_shared_state->hash_table_variants, _shared_state->join_op_variants,
vectorized::make_bool_variant(_build_side_ignore_null),
- vectorized::make_bool_variant(p._short_circuit_for_null_in_build_side));
+ vectorized::make_bool_variant(p._short_circuit_for_null_in_build_side),
+ vectorized::make_bool_variant((p._have_other_join_conjunct)));
return st;
}
void HashJoinBuildSinkLocalState::_set_build_ignore_flag(vectorized::Block& block,
const std::vector<int>& res_col_ids) {
+ auto& p = _parent->cast<HashJoinBuildSinkOperatorX>();
for (size_t i = 0; i < _build_expr_ctxs.size(); ++i) {
- if (!_shared_state->is_null_safe_eq_join[i]) {
+ if (!_shared_state->is_null_safe_eq_join[i] && !p._short_circuit_for_null_in_build_side) {
const auto* column = block.get_by_position(res_col_ids[i]).column.get();
if (check_and_get_column<vectorized::ColumnNullable>(*column)) {
- _build_side_ignore_null |= (_parent->cast<HashJoinBuildSinkOperatorX>()._join_op !=
- TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN &&
- !_shared_state->store_null_in_hash_table[i]);
+ _build_side_ignore_null |= !_shared_state->store_null_in_hash_table[i];
}
}
}
@@ -484,6 +492,7 @@ Status HashJoinBuildSinkOperatorX::sink(RuntimeState* state, vectorized::Block*
state, local_state._shared_state->build_block.get(), &local_state, use_global_rf));
RETURN_IF_ERROR(
local_state.process_build_block(state, (*local_state._shared_state->build_block)));
+ local_state._shared_state->build_indexes_null = local_state._build_indexes_null;
if (_shared_hashtable_controller) {
_shared_hash_table_context->status = Status::OK();
// arena will be shared with other instances.
@@ -497,6 +506,8 @@ Status HashJoinBuildSinkOperatorX::sink(RuntimeState* state, vectorized::Block*
_shared_hash_table_context);
}
_shared_hash_table_context->block = local_state._shared_state->build_block;
+ _shared_hash_table_context->build_indexes_null =
+ local_state._shared_state->build_indexes_null;
_shared_hashtable_controller->signal(node_id());
}
} else if (!local_state._should_build_hash_table) {
@@ -527,6 +538,9 @@ Status HashJoinBuildSinkOperatorX::sink(RuntimeState* state, vectorized::Block*
_shared_hash_table_context->hash_table_variants));
local_state._shared_state->build_block = _shared_hash_table_context->block;
+ local_state._build_indexes_null = _shared_hash_table_context->build_indexes_null;
+ local_state._shared_state->build_indexes_null =
+ _shared_hash_table_context->build_indexes_null;
const bool use_global_rf =
local_state._parent->cast<HashJoinBuildSinkOperatorX>()._use_global_rf;
@@ -561,10 +575,11 @@ Status HashJoinBuildSinkOperatorX::sink(RuntimeState* state, vectorized::Block*
local_state.init_short_circuit_for_probe();
if (source_state == SourceState::FINISHED) {
- // Since the comparison of null values is meaningless, null aware left anti join should not output null
+ // Since the comparison of null values is meaningless, null aware left anti/semi join should not output null
// when the build side is not empty.
if (local_state._shared_state->build_block &&
- _join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN) {
+ (_join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN ||
+ _join_op == TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN)) {
local_state._shared_state->probe_ignore_null = true;
}
local_state._dependency->set_ready_to_read();
diff --git a/be/src/pipeline/exec/hashjoin_build_sink.h b/be/src/pipeline/exec/hashjoin_build_sink.h
index 8420719330c..f70d9e67bce 100644
--- a/be/src/pipeline/exec/hashjoin_build_sink.h
+++ b/be/src/pipeline/exec/hashjoin_build_sink.h
@@ -113,6 +113,13 @@ protected:
std::shared_ptr<SharedHashTableDependency> _shared_hash_table_dependency;
std::vector<int> _build_col_ids;
+ /*
+ * For null aware anti/semi join with other join conjuncts, we do need to care about the rows in
+ * build side with null keys,
+ * because the other join conjuncts' result may be changed from null to false(null & false == false).
+ */
+ std::shared_ptr<std::vector<uint32_t>> _build_indexes_null;
+
RuntimeProfile::Counter* _build_table_timer = nullptr;
RuntimeProfile::Counter* _build_expr_call_timer = nullptr;
RuntimeProfile::Counter* _build_table_insert_timer = nullptr;
diff --git a/be/src/pipeline/exec/hashjoin_probe_operator.cpp b/be/src/pipeline/exec/hashjoin_probe_operator.cpp
index c7029614c50..aef2e011fa0 100644
--- a/be/src/pipeline/exec/hashjoin_probe_operator.cpp
+++ b/be/src/pipeline/exec/hashjoin_probe_operator.cpp
@@ -44,6 +44,12 @@ Status HashJoinProbeLocalState::init(RuntimeState* state, LocalStateInfo& info)
for (size_t i = 0; i < _other_join_conjuncts.size(); i++) {
RETURN_IF_ERROR(p._other_join_conjuncts[i]->clone(state, _other_join_conjuncts[i]));
}
+
+ _mark_join_conjuncts.resize(p._mark_join_conjuncts.size());
+ for (size_t i = 0; i < _mark_join_conjuncts.size(); i++) {
+ RETURN_IF_ERROR(p._mark_join_conjuncts[i]->clone(state, _mark_join_conjuncts[i]));
+ }
+
_construct_mutable_join_block();
_probe_column_disguise_null.reserve(_probe_expr_ctxs.size());
_probe_arena_memory_usage =
@@ -83,6 +89,7 @@ void HashJoinProbeLocalState::prepare_for_next() {
_build_index = 0;
_ready_probe = false;
_last_probe_match = -1;
+ _last_probe_null_mark = -1;
_prepare_probe_block();
}
@@ -234,42 +241,6 @@ Status HashJoinProbeOperatorX::pull(doris::RuntimeState* state, vectorized::Bloc
source_state = SourceState::FINISHED;
return Status::OK();
}
- if (local_state._shared_state->_has_null_in_build_side &&
- _short_circuit_for_null_in_build_side && _is_mark_join) {
- /// `_has_null_in_build_side` means have null value in build side.
- /// `_short_circuit_for_null_in_build_side` means short circuit if has null in build side(e.g. null aware left anti join).
- /// We need to create a column as mark with all rows set to NULL.
- auto block_rows = local_state._probe_block.rows();
- if (block_rows == 0) {
- if (local_state._probe_eos) {
- source_state = SourceState::FINISHED;
- }
- return Status::OK();
- }
-
- vectorized::Block temp_block;
- //get probe side output column
- for (int i = 0; i < _left_output_slot_flags.size(); ++i) {
- temp_block.insert(local_state._probe_block.get_by_position(i));
- }
- auto mark_column =
- vectorized::ColumnNullable::create(vectorized::ColumnUInt8::create(block_rows, 0),
- vectorized::ColumnUInt8::create(block_rows, 1));
- temp_block.insert({std::move(mark_column),
- make_nullable(std::make_shared<vectorized::DataTypeUInt8>()), ""});
-
- {
- SCOPED_TIMER(local_state._join_filter_timer);
- RETURN_IF_ERROR(vectorized::VExprContext::filter_block(
- local_state._conjuncts, &temp_block, temp_block.columns()));
- }
-
- RETURN_IF_ERROR(local_state._build_output_block(&temp_block, output_block, false));
- temp_block.clear();
- local_state._probe_block.clear_column_data(_child_x->row_desc().num_materialized_slots());
- local_state.reached_limit(output_block, source_state);
- return Status::OK();
- }
//TODO: this short circuit maybe could refactor, no need to check at here.
if (local_state._shared_state->empty_right_table_need_probe_dispose) {
@@ -330,6 +301,7 @@ Status HashJoinProbeOperatorX::pull(doris::RuntimeState* state, vectorized::Bloc
Status st;
if (local_state._probe_index < local_state._probe_block.rows()) {
+ local_state._build_indexes_null = local_state._shared_state->build_indexes_null;
DCHECK(local_state._has_set_need_null_map_for_probe);
RETURN_IF_CATCH_EXCEPTION({
std::visit(
@@ -540,7 +512,8 @@ Status HashJoinProbeOperatorX::init(const TPlanNode& tnode, RuntimeState* state)
DCHECK(tnode.__isset.hash_join_node);
const bool probe_dispose_null =
_match_all_probe || _build_unique || _join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN ||
- _join_op == TJoinOp::LEFT_ANTI_JOIN || _join_op == TJoinOp::LEFT_SEMI_JOIN;
+ _join_op == TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN || _join_op == TJoinOp::LEFT_ANTI_JOIN ||
+ _join_op == TJoinOp::LEFT_SEMI_JOIN;
const std::vector<TEqJoinCondition>& eq_join_conjuncts = tnode.hash_join_node.eq_join_conjuncts;
std::vector<bool> probe_not_ignore_null(eq_join_conjuncts.size());
size_t conjuncts_index = 0;
@@ -575,6 +548,20 @@ Status HashJoinProbeOperatorX::init(const TPlanNode& tnode, RuntimeState* state)
DCHECK(!_build_unique);
DCHECK(_have_other_join_conjunct);
}
+
+ if (tnode.hash_join_node.__isset.mark_join_conjuncts &&
+ !tnode.hash_join_node.mark_join_conjuncts.empty()) {
+ RETURN_IF_ERROR(vectorized::VExpr::create_expr_trees(
+ tnode.hash_join_node.mark_join_conjuncts, _mark_join_conjuncts));
+ DCHECK(_is_mark_join);
+
+ /// We make mark join conjuncts as equal conjuncts for null aware join,
+ /// so `_mark_join_conjuncts` should be empty if this is null aware join.
+ DCHECK_EQ(_mark_join_conjuncts.empty(),
+ _join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN ||
+ _join_op == TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN);
+ }
+
return Status::OK();
}
@@ -603,6 +590,11 @@ Status HashJoinProbeOperatorX::prepare(RuntimeState* state) {
for (auto& conjunct : _other_join_conjuncts) {
RETURN_IF_ERROR(conjunct->prepare(state, *_intermediate_row_desc));
}
+
+ for (auto& conjunct : _mark_join_conjuncts) {
+ RETURN_IF_ERROR(conjunct->prepare(state, *_intermediate_row_desc));
+ }
+
RETURN_IF_ERROR(vectorized::VExpr::prepare(_probe_expr_ctxs, state, _child_x->row_desc()));
DCHECK(_build_side_child != nullptr);
// right table data types
@@ -621,6 +613,11 @@ Status HashJoinProbeOperatorX::open(RuntimeState* state) {
for (auto& conjunct : _other_join_conjuncts) {
RETURN_IF_ERROR(conjunct->open(state));
}
+
+ for (auto& conjunct : _mark_join_conjuncts) {
+ RETURN_IF_ERROR(conjunct->open(state));
+ }
+
return Status::OK();
}
diff --git a/be/src/pipeline/exec/hashjoin_probe_operator.h b/be/src/pipeline/exec/hashjoin_probe_operator.h
index 18db6acc67f..1bdb9864c40 100644
--- a/be/src/pipeline/exec/hashjoin_probe_operator.h
+++ b/be/src/pipeline/exec/hashjoin_probe_operator.h
@@ -59,6 +59,8 @@ using HashTableCtxVariants = std::variant<
vectorized::ProcessHashTableProbe<TJoinOp::RIGHT_SEMI_JOIN, HashJoinProbeLocalState>,
vectorized::ProcessHashTableProbe<TJoinOp::RIGHT_ANTI_JOIN, HashJoinProbeLocalState>,
vectorized::ProcessHashTableProbe<TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN,
+ HashJoinProbeLocalState>,
+ vectorized::ProcessHashTableProbe<TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN,
HashJoinProbeLocalState>>;
class HashJoinProbeDependency final : public Dependency {
@@ -120,10 +122,23 @@ private:
std::atomic<bool> _probe_inited = false;
int _last_probe_match;
+ // For mark join, last probe index of null mark
+ int _last_probe_null_mark;
+
+ /*
+ * For null aware anti/semi join with other join conjuncts, we do need to care about the rows in
+ * build side with null keys,
+ * because the other join conjuncts' result may be changed from null to false(null & false == false).
+ */
+ std::shared_ptr<std::vector<uint32_t>> _build_indexes_null;
+
vectorized::Block _probe_block;
vectorized::ColumnRawPtrs _probe_columns;
// other expr
vectorized::VExprContextSPtrs _other_join_conjuncts;
+
+ vectorized::VExprContextSPtrs _mark_join_conjuncts;
+
// probe expr
vectorized::VExprContextSPtrs _probe_expr_ctxs;
std::vector<uint16_t> _probe_column_disguise_null;
@@ -188,6 +203,9 @@ private:
const bool _is_broadcast_join;
// other expr
vectorized::VExprContextSPtrs _other_join_conjuncts;
+
+ vectorized::VExprContextSPtrs _mark_join_conjuncts;
+
// probe expr
vectorized::VExprContextSPtrs _probe_expr_ctxs;
bool _probe_ignore_null = false;
diff --git a/be/src/pipeline/exec/join_build_sink_operator.cpp b/be/src/pipeline/exec/join_build_sink_operator.cpp
index 798b5d86b00..73ebfb947a3 100644
--- a/be/src/pipeline/exec/join_build_sink_operator.cpp
+++ b/be/src/pipeline/exec/join_build_sink_operator.cpp
@@ -68,7 +68,8 @@ JoinBuildSinkOperatorX<LocalStateType>::JoinBuildSinkOperatorX(ObjectPool* pool,
_join_op == TJoinOp::RIGHT_SEMI_JOIN),
_is_left_semi_anti(_join_op == TJoinOp::LEFT_ANTI_JOIN ||
_join_op == TJoinOp::LEFT_SEMI_JOIN ||
- _join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN),
+ _join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN ||
+ _join_op == TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN),
_is_outer_join(_match_all_build || _match_all_probe),
_is_mark_join(tnode.__isset.nested_loop_join_node
? (tnode.nested_loop_join_node.__isset.is_mark
@@ -76,11 +77,13 @@ JoinBuildSinkOperatorX<LocalStateType>::JoinBuildSinkOperatorX(ObjectPool* pool,
: false)
: tnode.hash_join_node.__isset.is_mark ? tnode.hash_join_node.is_mark
: false),
- _short_circuit_for_null_in_build_side(_join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN) {
+ _short_circuit_for_null_in_build_side(_join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN &&
+ !_is_mark_join) {
_init_join_op();
if (_is_mark_join) {
DCHECK(_join_op == TJoinOp::LEFT_ANTI_JOIN || _join_op == TJoinOp::LEFT_SEMI_JOIN ||
- _join_op == TJoinOp::CROSS_JOIN || _join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN)
+ _join_op == TJoinOp::CROSS_JOIN || _join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN ||
+ _join_op == TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN)
<< "Mark join is only supported for null aware left semi/anti join and cross join "
"but this is "
<< _join_op;
@@ -97,7 +100,8 @@ JoinBuildSinkOperatorX<LocalStateType>::JoinBuildSinkOperatorX(ObjectPool* pool,
M(CROSS_JOIN) \
M(RIGHT_SEMI_JOIN) \
M(RIGHT_ANTI_JOIN) \
- M(NULL_AWARE_LEFT_ANTI_JOIN)
+ M(NULL_AWARE_LEFT_ANTI_JOIN) \
+ M(NULL_AWARE_LEFT_SEMI_JOIN)
template <typename LocalStateType>
void JoinBuildSinkOperatorX<LocalStateType>::_init_join_op() {
diff --git a/be/src/pipeline/exec/join_probe_operator.cpp b/be/src/pipeline/exec/join_probe_operator.cpp
index 4e94b4a206b..3e71ec41dd4 100644
--- a/be/src/pipeline/exec/join_probe_operator.cpp
+++ b/be/src/pipeline/exec/join_probe_operator.cpp
@@ -65,9 +65,15 @@ void JoinProbeLocalState<DependencyType, Derived>::_construct_mutable_join_block
_join_block.insert({type_ptr->create_column(), type_ptr, slot_desc->col_name()});
}
}
+
if (p._is_mark_join) {
- DCHECK(!p._is_mark_join ||
- _join_block.get_by_position(_join_block.columns() - 1).column->is_nullable());
+ _mark_column_id = _join_block.columns() - 1;
+#ifndef NDEBUG
+ const auto& mark_column = assert_cast<const vectorized::ColumnNullable&>(
+ *_join_block.get_by_position(_mark_column_id).column);
+ auto& nested_column = mark_column.get_nested_column();
+ DCHECK(check_and_get_column<vectorized::ColumnUInt8>(nested_column) != nullptr);
+#endif
}
}
@@ -182,7 +188,8 @@ JoinProbeOperatorX<LocalStateType>::JoinProbeOperatorX(ObjectPool* pool, const T
: false)
: tnode.hash_join_node.__isset.is_mark ? tnode.hash_join_node.is_mark
: false),
- _short_circuit_for_null_in_build_side(_join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN) {
+ _short_circuit_for_null_in_build_side(_join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN &&
+ !_is_mark_join) {
if (tnode.__isset.hash_join_node) {
_intermediate_row_desc.reset(new RowDescriptor(
descs, tnode.hash_join_node.vintermediate_tuple_id_list,
diff --git a/be/src/pipeline/exec/join_probe_operator.h b/be/src/pipeline/exec/join_probe_operator.h
index 9c6280bb64e..5dc6089c348 100644
--- a/be/src/pipeline/exec/join_probe_operator.h
+++ b/be/src/pipeline/exec/join_probe_operator.h
@@ -53,6 +53,8 @@ protected:
vectorized::MutableColumnPtr _tuple_is_null_left_flag_column = nullptr;
vectorized::MutableColumnPtr _tuple_is_null_right_flag_column = nullptr;
+ size_t _mark_column_id = -1;
+
RuntimeProfile::Counter* _probe_timer = nullptr;
RuntimeProfile::Counter* _probe_rows_counter = nullptr;
RuntimeProfile::Counter* _join_filter_timer = nullptr;
diff --git a/be/src/pipeline/pipeline_x/dependency.h b/be/src/pipeline/pipeline_x/dependency.h
index 8a58973be35..5ae3480cd4d 100644
--- a/be/src/pipeline/pipeline_x/dependency.h
+++ b/be/src/pipeline/pipeline_x/dependency.h
@@ -461,6 +461,7 @@ struct HashJoinSharedState : public JoinSharedState {
const std::vector<TupleDescriptor*> build_side_child_desc;
size_t build_exprs_size = 0;
std::shared_ptr<vectorized::Block> build_block;
+ std::shared_ptr<std::vector<uint32_t>> build_indexes_null;
bool probe_ignore_null = false;
};
diff --git a/be/src/vec/common/hash_table/hash_map.h b/be/src/vec/common/hash_table/hash_map.h
index d10b24ade21..0bc7a5abbfe 100644
--- a/be/src/vec/common/hash_table/hash_map.h
+++ b/be/src/vec/common/hash_table/hash_map.h
@@ -216,4 +216,4 @@ using HashMapWithStackMemory = HashMapTable<
HashTableAllocatorWithStackMemory<(1ULL << initial_size_degree) *
sizeof(HashMapCellWithSavedHash<Key, Mapped, Hash>)>>;
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/vec/common/hash_table/join_hash_table.h b/be/src/vec/common/hash_table/join_hash_table.h
index b190d3d89ce..08311989b5d 100644
--- a/be/src/vec/common/hash_table/join_hash_table.h
+++ b/be/src/vec/common/hash_table/join_hash_table.h
@@ -92,14 +92,9 @@ public:
}
}
- if constexpr (with_other_conjuncts) {
- return _find_batch_conjunct<JoinOpType>(keys, build_idx_map, probe_idx, build_idx,
- probe_rows, probe_idxs, build_idxs);
- }
-
- if constexpr (is_mark_join) {
- return _find_batch_mark<JoinOpType>(keys, build_idx_map, probe_idx, probe_rows,
- probe_idxs, build_idxs, mark_column);
+ if constexpr (with_other_conjuncts || is_mark_join) {
+ return _find_batch_conjunct<JoinOpType, need_judge_null>(
+ keys, build_idx_map, probe_idx, build_idx, probe_rows, probe_idxs, build_idxs);
}
if constexpr (JoinOpType == TJoinOp::INNER_JOIN || JoinOpType == TJoinOp::FULL_OUTER_JOIN ||
@@ -122,6 +117,80 @@ public:
return std::tuple {0, 0U, 0};
}
+ /**
+ * Because the equality comparison result of null with any value is null,
+ * in null aware join, if the probe key of a row in the left table(probe side) is null,
+ * then this row will match all rows on the right table(build side) (the match result is null).
+ * If the probe key of a row in the left table does not match any row in right table,
+ * this row will match all rows with null key in the right table.
+ * select 'a' in ('b', null) => 'a' = 'b' or 'a' = null => false or null => null
+ * select 'a' in ('a', 'b', null) => true
+ * select 'a' not in ('b', null) => null => 'a' != 'b' and 'a' != null => true and null => null
+ * select 'a' not in ('a', 'b', null) => false
+ */
+ auto find_null_aware_with_other_conjuncts(
+ const Key* __restrict keys, const uint32_t* __restrict build_idx_map, int probe_idx,
+ uint32_t build_idx, int probe_rows, uint32_t* __restrict probe_idxs,
+ uint32_t* __restrict build_idxs, std::set<uint32_t>& null_result,
+ const std::vector<uint32_t>& build_indexes_null, const size_t build_block_count) {
+ auto matched_cnt = 0;
+ const auto batch_size = max_batch_size;
+
+ bool has_matched = false;
+ auto do_the_probe = [&]() {
+ while (build_idx && matched_cnt < batch_size) {
+ if (build_idx == bucket_size) {
+ /// All rows in build side should be executed with other join conjuncts.
+ for (size_t i = 1; i != build_block_count; ++i) {
+ build_idxs[matched_cnt] = i;
+ probe_idxs[matched_cnt] = probe_idx;
+ matched_cnt++;
+ }
+ null_result.emplace(probe_idx);
+ build_idx = 0;
+ has_matched = true;
+ break;
+ } else if (keys[probe_idx] == build_keys[build_idx]) {
+ build_idxs[matched_cnt] = build_idx;
+ probe_idxs[matched_cnt] = probe_idx;
+ matched_cnt++;
+ has_matched = true;
+ }
+
+ build_idx = next[build_idx];
+ }
+
+ // may over batch_size when emplace 0 into build_idxs
+ if (!build_idx) {
+ if (!has_matched) { // has no any row matched
+ for (auto index : build_indexes_null) {
+ build_idxs[matched_cnt] = index;
+ probe_idxs[matched_cnt] = probe_idx;
+ matched_cnt++;
+ }
+ }
+ probe_idxs[matched_cnt] = probe_idx;
+ build_idxs[matched_cnt] = 0;
+ matched_cnt++;
+ has_matched = false;
+ }
+
+ probe_idx++;
+ };
+
+ if (build_idx) {
+ do_the_probe();
+ }
+
+ while (probe_idx < probe_rows && matched_cnt < batch_size) {
+ build_idx = build_idx_map[probe_idx];
+ do_the_probe();
+ }
+
+ probe_idx -= (build_idx != 0);
+ return std::tuple {probe_idx, build_idx, matched_cnt};
+ }
+
template <int JoinOpType>
bool iterate_map(std::vector<uint32_t>& build_idxs) const {
const auto batch_size = max_batch_size;
@@ -157,42 +226,6 @@ public:
}
private:
- // only LEFT_ANTI_JOIN/LEFT_SEMI_JOIN/NULL_AWARE_LEFT_ANTI_JOIN/CROSS_JOIN support mark join
- template <int JoinOpType>
- auto _find_batch_mark(const Key* __restrict keys, const uint32_t* __restrict build_idx_map,
- int probe_idx, int probe_rows, uint32_t* __restrict probe_idxs,
- uint32_t* __restrict build_idxs,
- vectorized::ColumnFilterHelper* mark_column) {
- auto matched_cnt = 0;
- const auto batch_size = max_batch_size;
-
- while (probe_idx < probe_rows && matched_cnt < batch_size) {
- auto build_idx = build_idx_map[probe_idx] == bucket_size ? 0 : build_idx_map[probe_idx];
-
- while (build_idx && keys[probe_idx] != build_keys[build_idx]) {
- build_idx = next[build_idx];
- }
-
- if (build_idx_map[probe_idx] == bucket_size) {
- // mark result as null when probe row is null
- mark_column->insert_null();
- } else {
- bool matched =
- JoinOpType == TJoinOp::LEFT_SEMI_JOIN ? build_idx != 0 : build_idx == 0;
- if (!matched && _has_null_key) {
- mark_column->insert_null();
- } else {
- mark_column->insert_value(matched);
- }
- }
-
- probe_idxs[matched_cnt] = probe_idx++;
- build_idxs[matched_cnt] = build_idx;
- matched_cnt++;
- }
- return std::tuple {probe_idx, 0U, matched_cnt};
- }
-
template <int JoinOpType, bool with_other_conjuncts, bool is_mark_join>
auto _process_null_aware_left_anti_join_for_empty_build_side(
int probe_idx, int probe_rows, uint32_t* __restrict probe_idxs,
@@ -203,14 +236,13 @@ private:
while (probe_idx < probe_rows && matched_cnt < batch_size) {
probe_idxs[matched_cnt] = probe_idx++;
- if constexpr (is_mark_join) {
- build_idxs[matched_cnt] = 0;
- }
+ build_idxs[matched_cnt] = 0;
++matched_cnt;
}
if constexpr (is_mark_join && !with_other_conjuncts) {
- mark_column->resize_fill(matched_cnt, 1);
+ // we will flip the mark column later for anti join, so here set 0 into mark column.
+ mark_column->resize_fill(matched_cnt, 0);
}
return std::tuple {probe_idx, 0U, matched_cnt};
@@ -260,7 +292,7 @@ private:
return std::tuple {probe_idx, 0U, matched_cnt};
}
- template <int JoinOpType>
+ template <int JoinOpType, bool need_judge_null>
auto _find_batch_conjunct(const Key* __restrict keys, const uint32_t* __restrict build_idx_map,
int probe_idx, uint32_t build_idx, int probe_rows,
uint32_t* __restrict probe_idxs, uint32_t* __restrict build_idxs) {
@@ -276,7 +308,17 @@ private:
build_idxs[matched_cnt] = build_idx;
matched_cnt++;
}
- } else if (keys[probe_idx] == build_keys[build_idx]) {
+ } else if constexpr (need_judge_null) {
+ if (build_idx == bucket_size) {
+ build_idxs[matched_cnt] = build_idx;
+ probe_idxs[matched_cnt] = probe_idx;
+ build_idx = 0;
+ matched_cnt++;
+ break;
+ }
+ }
+
+ if (keys[probe_idx] == build_keys[build_idx]) {
build_idxs[matched_cnt] = build_idx;
probe_idxs[matched_cnt] = probe_idx;
matched_cnt++;
@@ -288,7 +330,8 @@ private:
JoinOpType == TJoinOp::FULL_OUTER_JOIN ||
JoinOpType == TJoinOp::LEFT_SEMI_JOIN ||
JoinOpType == TJoinOp::LEFT_ANTI_JOIN ||
- JoinOpType == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN) {
+ JoinOpType == doris::TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN ||
+ JoinOpType == doris::TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN) {
// may over batch_size when emplace 0 into build_idxs
if (!build_idx) {
probe_idxs[matched_cnt] = probe_idx;
diff --git a/be/src/vec/exec/join/null_aware_left_semi_join_impl.cpp b/be/src/vec/exec/join/null_aware_left_semi_join_impl.cpp
new file mode 100644
index 00000000000..98d39e61479
--- /dev/null
+++ b/be/src/vec/exec/join/null_aware_left_semi_join_impl.cpp
@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gen_cpp/PlanNodes_types.h>
+
+#include "process_hash_table_probe_impl.h"
+
+namespace doris::vectorized {
+
+INSTANTIATION_FOR(TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN);
+
+}
diff --git a/be/src/vec/exec/join/process_hash_table_probe.h b/be/src/vec/exec/join/process_hash_table_probe.h
index 803cc34bd78..02bf242e55a 100644
--- a/be/src/vec/exec/join/process_hash_table_probe.h
+++ b/be/src/vec/exec/join/process_hash_table_probe.h
@@ -44,7 +44,7 @@ struct ProcessHashTableProbe {
// output build side result column
void build_side_output_column(MutableColumns& mcol, const std::vector<bool>& output_slot_flags,
- int size, bool have_other_join_conjunct);
+ int size, bool have_other_join_conjunct, bool is_mark_join);
void probe_side_output_column(MutableColumns& mcol, const std::vector<bool>& output_slot_flags,
int size, int last_probe_index, bool all_match_one,
@@ -70,6 +70,10 @@ struct ProcessHashTableProbe {
Status do_other_join_conjuncts(Block* output_block, bool is_mark_join,
std::vector<uint8_t>& visited, bool has_null_in_build_side);
+ template <bool with_other_conjuncts>
+ Status do_mark_join_conjuncts(Block* output_block, size_t hash_table_bucket_size,
+ const std::set<uint32_t>& null_result);
+
template <typename HashTableType>
typename HashTableType::State _init_probe_side(HashTableType& hash_table_ctx, size_t probe_rows,
bool with_other_join_conjuncts,
diff --git a/be/src/vec/exec/join/process_hash_table_probe_impl.h b/be/src/vec/exec/join/process_hash_table_probe_impl.h
index c9f92c59e08..fd2ac01613d 100644
--- a/be/src/vec/exec/join/process_hash_table_probe_impl.h
+++ b/be/src/vec/exec/join/process_hash_table_probe_impl.h
@@ -62,7 +62,7 @@ ProcessHashTableProbe<JoinOpType, Parent>::ProcessHashTableProbe(Parent* parent,
template <int JoinOpType, typename Parent>
void ProcessHashTableProbe<JoinOpType, Parent>::build_side_output_column(
MutableColumns& mcol, const std::vector<bool>& output_slot_flags, int size,
- bool have_other_join_conjunct) {
+ bool have_other_join_conjunct, bool is_mark_join) {
SCOPED_TIMER(_build_side_output_timer);
constexpr auto is_semi_anti_join = JoinOpType == TJoinOp::RIGHT_ANTI_JOIN ||
JoinOpType == TJoinOp::RIGHT_SEMI_JOIN ||
@@ -73,7 +73,9 @@ void ProcessHashTableProbe<JoinOpType, Parent>::build_side_output_column(
constexpr auto probe_all =
JoinOpType == TJoinOp::LEFT_OUTER_JOIN || JoinOpType == TJoinOp::FULL_OUTER_JOIN;
- if ((!is_semi_anti_join || have_other_join_conjunct) && size) {
+ if ((!is_semi_anti_join || have_other_join_conjunct ||
+ (is_mark_join && !_parent->_mark_join_conjuncts.empty())) &&
+ size) {
for (int i = 0; i < _right_col_len; i++) {
const auto& column = *_build_block->safe_get_by_position(i).column;
if (output_slot_flags[i]) {
@@ -177,7 +179,21 @@ Status ProcessHashTableProbe<JoinOpType, Parent>::do_process(HashTableType& hash
mark_column = std::make_unique<ColumnFilterHelper>(*mcol[mcol.size() - 1]);
}
- {
+ /// `null_result` set which contains the probe indexes of null results.
+ std::set<uint32_t> null_result;
+ if constexpr ((JoinOpType == doris::TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN ||
+ JoinOpType == doris::TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN) &&
+ with_other_conjuncts) {
+ SCOPED_TIMER(_search_hashtable_timer);
+ auto [new_probe_idx, new_build_idx, new_current_offset] =
+ hash_table_ctx.hash_table->find_null_aware_with_other_conjuncts(
+ hash_table_ctx.keys, hash_table_ctx.bucket_nums.data(), probe_index,
+ build_index, probe_rows, _probe_indexs.data(), _build_indexs.data(),
+ null_result, *(_parent->_build_indexes_null), _build_block->rows());
+ probe_index = new_probe_idx;
+ build_index = new_build_idx;
+ current_offset = new_current_offset;
+ } else {
SCOPED_TIMER(_search_hashtable_timer);
auto [new_probe_idx, new_build_idx,
new_current_offset] = hash_table_ctx.hash_table->template find_batch < JoinOpType,
@@ -191,7 +207,8 @@ Status ProcessHashTableProbe<JoinOpType, Parent>::do_process(HashTableType& hash
current_offset = new_current_offset;
}
- build_side_output_column(mcol, *_right_output_slot_flags, current_offset, with_other_conjuncts);
+ build_side_output_column(mcol, *_right_output_slot_flags, current_offset, with_other_conjuncts,
+ is_mark_join);
if constexpr (with_other_conjuncts || (JoinOpType != TJoinOp::RIGHT_SEMI_JOIN &&
JoinOpType != TJoinOp::RIGHT_ANTI_JOIN)) {
@@ -214,7 +231,10 @@ Status ProcessHashTableProbe<JoinOpType, Parent>::do_process(HashTableType& hash
output_block->swap(mutable_block.to_block());
- if constexpr (with_other_conjuncts) {
+ if constexpr (is_mark_join) {
+ return do_mark_join_conjuncts<with_other_conjuncts>(
+ output_block, hash_table_ctx.hash_table->get_bucket_size(), null_result);
+ } else if constexpr (with_other_conjuncts) {
return do_other_join_conjuncts(output_block, is_mark_join,
hash_table_ctx.hash_table->get_visited(),
hash_table_ctx.hash_table->has_null_key());
@@ -223,6 +243,131 @@ Status ProcessHashTableProbe<JoinOpType, Parent>::do_process(HashTableType& hash
return Status::OK();
}
+template <int JoinOpType, typename Parent>
+template <bool with_other_conjuncts>
+Status ProcessHashTableProbe<JoinOpType, Parent>::do_mark_join_conjuncts(
+ Block* output_block, size_t hash_table_bucket_size, const std::set<uint32_t>& null_result) {
+ DCHECK(JoinOpType == TJoinOp::LEFT_ANTI_JOIN ||
+ JoinOpType == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN ||
+ JoinOpType == TJoinOp::LEFT_SEMI_JOIN ||
+ JoinOpType == TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN);
+
+ constexpr bool is_anti_join = JoinOpType == TJoinOp::LEFT_ANTI_JOIN ||
+ JoinOpType == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN;
+ constexpr bool is_null_aware_join = JoinOpType == TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN ||
+ JoinOpType == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN;
+
+ const auto row_count = output_block->rows();
+ auto mark_column_mutable =
+ output_block->get_by_position(_parent->_mark_column_id).column->assume_mutable();
+ auto& mark_column = assert_cast<ColumnNullable&>(*mark_column_mutable);
+ IColumn::Filter& filter = assert_cast<ColumnUInt8&>(mark_column.get_nested_column()).get_data();
+
+ if (_parent->_mark_join_conjuncts.empty()) {
+ // For null aware anti/semi join, if the equal conjuncts was not matched and the build side has null value,
+ // the result should be null. Like:
+ // select 4 not in (2, 3, null) => null, select 4 not in (2, 3) => true
+ // select 4 in (2, 3, null) => null, select 4 in (2, 3) => false
+ const bool should_be_null_if_build_side_has_null = *_has_null_in_build_side;
+
+ mark_column.resize(row_count);
+ auto* filter_data =
+ assert_cast<ColumnUInt8&>(mark_column.get_nested_column()).get_data().data();
+ auto* mark_null_map = mark_column.get_null_map_data().data();
+ int last_probe_matched = -1;
+ for (size_t i = 0; i != row_count; ++i) {
+ filter_data[i] = _build_indexs[i] != 0 && _build_indexs[i] != hash_table_bucket_size;
+ if constexpr (is_null_aware_join) {
+ if constexpr (with_other_conjuncts) {
+ mark_null_map[i] =
+ null_result.contains(_probe_indexs[i]) && _build_indexs[i] != 0;
+ } else {
+ if (filter_data[i]) {
+ last_probe_matched = _probe_indexs[i];
+ mark_null_map[i] = false;
+ } else if (_build_indexs[i] == 0) {
+ mark_null_map[i] = should_be_null_if_build_side_has_null &&
+ last_probe_matched != _probe_indexs[i];
+ } else if (_build_indexs[i] == hash_table_bucket_size) {
+ mark_null_map[i] = true;
+ }
+ }
+ }
+ }
+ if constexpr (!is_null_aware_join) {
+ memset(mark_null_map, 0, row_count);
+ }
+ } else {
+ RETURN_IF_ERROR(VExprContext::execute_conjuncts(_parent->_mark_join_conjuncts, output_block,
+ mark_column.get_null_map_column(), filter));
+ }
+ auto* mark_null_map = mark_column.get_null_map_data().data();
+
+ auto* mark_filter_data = filter.data();
+
+ if constexpr (with_other_conjuncts) {
+ IColumn::Filter other_conjunct_filter(row_count, 1);
+ {
+ bool can_be_filter_all = false;
+ RETURN_IF_ERROR(VExprContext::execute_conjuncts(_parent->_other_join_conjuncts, nullptr,
+ output_block, &other_conjunct_filter,
+ &can_be_filter_all));
+ }
+ DCHECK_EQ(filter.size(), other_conjunct_filter.size());
+ const auto* other_filter_data = other_conjunct_filter.data();
+ for (size_t i = 0; i != filter.size(); ++i) {
+ // null & any(true or false) => null => false
+ mark_filter_data[i] &= (!mark_null_map[i]) & other_filter_data[i];
+
+ // null & true => null
+ // null & false => false
+ mark_null_map[i] &= other_filter_data[i];
+ }
+ }
+
+ auto filter_column = ColumnUInt8::create(row_count, 0);
+ auto* __restrict filter_map = filter_column->get_data().data();
+
+ /**
+ * Here need `!with_other_conjuncts` be true,
+ * because null aware join with other join conjuncts will process the `mark_null_map` after the
+ * other join conjuncts are executed.
+ */
+ const bool should_be_null_if_build_side_has_null =
+ *_has_null_in_build_side && is_null_aware_join && !with_other_conjuncts;
+ for (size_t i = 0; i != row_count; ++i) {
+ bool not_matched_before = _parent->_last_probe_match != _probe_indexs[i];
+ if (_build_indexs[i] == 0) {
+ bool has_null_mark_value = _parent->_last_probe_null_mark == _probe_indexs[i];
+ if (not_matched_before) {
+ filter_map[i] = true;
+ mark_null_map[i] = has_null_mark_value || should_be_null_if_build_side_has_null;
+ mark_filter_data[i] = false;
+ }
+ } else {
+ if (mark_null_map[i]) { // is null
+ _parent->_last_probe_null_mark = _probe_indexs[i];
+ } else {
+ if (mark_filter_data[i] && not_matched_before) {
+ _parent->_last_probe_match = _probe_indexs[i];
+ filter_map[i] = true;
+ }
+ }
+ }
+ }
+
+ if constexpr (is_anti_join) {
+ // flip the mark column
+ for (size_t i = 0; i != row_count; ++i) {
+ mark_filter_data[i] ^= 1;
+ }
+ }
+
+ auto result_column_id = output_block->columns();
+ output_block->insert({std::move(filter_column), std::make_shared<DataTypeUInt8>(), ""});
+ return Block::filter_block(output_block, result_column_id, result_column_id);
+}
+
template <int JoinOpType, typename Parent>
Status ProcessHashTableProbe<JoinOpType, Parent>::do_other_join_conjuncts(
Block* output_block, bool is_mark_join, std::vector<uint8_t>& visited,
diff --git a/be/src/vec/exec/join/vhash_join_node.cpp b/be/src/vec/exec/join/vhash_join_node.cpp
index c7b6b5d0411..271e522cd65 100644
--- a/be/src/vec/exec/join/vhash_join_node.cpp
+++ b/be/src/vec/exec/join/vhash_join_node.cpp
@@ -107,7 +107,8 @@ Status HashJoinNode::init(const TPlanNode& tnode, RuntimeState* state) {
_join_op == TJoinOp::RIGHT_ANTI_JOIN;
const bool probe_dispose_null =
_match_all_probe || _build_unique || _join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN ||
- _join_op == TJoinOp::LEFT_ANTI_JOIN || _join_op == TJoinOp::LEFT_SEMI_JOIN;
+ _join_op == TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN || _join_op == TJoinOp::LEFT_ANTI_JOIN ||
+ _join_op == TJoinOp::LEFT_SEMI_JOIN;
const std::vector<TEqJoinCondition>& eq_join_conjuncts = tnode.hash_join_node.eq_join_conjuncts;
std::vector<bool> probe_not_ignore_null(eq_join_conjuncts.size());
@@ -156,6 +157,35 @@ Status HashJoinNode::init(const TPlanNode& tnode, RuntimeState* state) {
DCHECK(_have_other_join_conjunct);
}
+ if (tnode.hash_join_node.__isset.mark_join_conjuncts) {
+ RETURN_IF_ERROR(VExpr::create_expr_trees(tnode.hash_join_node.mark_join_conjuncts,
+ _mark_join_conjuncts));
+ DCHECK(_is_mark_join);
+
+ /// We make mark join conjuncts as equal conjuncts for null aware join,
+ /// so `_mark_join_conjuncts` should be empty if this is null aware join.
+ DCHECK_EQ(_mark_join_conjuncts.empty(),
+ _join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN ||
+ _join_op == TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN);
+ }
+
+#ifndef NDEBUG
+ /// mark join should be half join
+ if (_is_mark_join) {
+ DCHECK(_join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN ||
+ _join_op == TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN ||
+ _join_op == TJoinOp::LEFT_ANTI_JOIN || _join_op == TJoinOp::LEFT_SEMI_JOIN ||
+ _join_op == TJoinOp::RIGHT_ANTI_JOIN || _join_op == TJoinOp::RIGHT_SEMI_JOIN)
+ << "join(op: " << _join_op << ") should not be mark join";
+ }
+#endif
+
+ if ((_join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN ||
+ _join_op == TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN) &&
+ _have_other_join_conjunct) {
+ _build_indexes_null = std::make_shared<std::vector<uint32_t>>();
+ }
+
_runtime_filters.resize(_runtime_filter_descs.size());
for (size_t i = 0; i < _runtime_filter_descs.size(); i++) {
RETURN_IF_ERROR(state->runtime_filter_mgr()->register_producer_filter(
@@ -250,6 +280,11 @@ Status HashJoinNode::prepare(RuntimeState* state) {
for (auto& conjunct : _other_join_conjuncts) {
RETURN_IF_ERROR(conjunct->prepare(state, *_intermediate_row_desc));
}
+
+ for (auto& conjunct : _mark_join_conjuncts) {
+ RETURN_IF_ERROR(conjunct->prepare(state, *_intermediate_row_desc));
+ }
+
RETURN_IF_ERROR(VExpr::prepare(_output_expr_ctxs, state, *_intermediate_row_desc));
// right table data types
@@ -300,6 +335,7 @@ void HashJoinNode::prepare_for_next() {
_build_index = 0;
_ready_probe = false;
_last_probe_match = -1;
+ _last_probe_null_mark = -1;
_prepare_probe_block();
}
@@ -312,39 +348,6 @@ Status HashJoinNode::pull(doris::RuntimeState* state, vectorized::Block* output_
return Status::OK();
}
- /// `_has_null_in_build_side` means have null value in build side.
- /// `_short_circuit_for_null_in_build_side` means short circuit if has null in build side(e.g. null aware left anti join).
- if (_has_null_in_build_side && _short_circuit_for_null_in_build_side && _is_mark_join) {
- /// We need to create a column as mark with all rows set to NULL.
- auto block_rows = _probe_block.rows();
- if (block_rows == 0) {
- *eos = _probe_eos;
- return Status::OK();
- }
-
- Block temp_block;
- //get probe side output column
- for (int i = 0; i < _left_output_slot_flags.size(); ++i) {
- temp_block.insert(_probe_block.get_by_position(i));
- }
- auto mark_column = ColumnNullable::create(ColumnUInt8::create(block_rows, 0),
- ColumnUInt8::create(block_rows, 1));
- temp_block.insert(
- {std::move(mark_column), make_nullable(std::make_shared<DataTypeUInt8>()), ""});
-
- {
- SCOPED_TIMER(_join_filter_timer);
- RETURN_IF_ERROR(
- VExprContext::filter_block(_conjuncts, &temp_block, temp_block.columns()));
- }
-
- RETURN_IF_ERROR(_build_output_block(&temp_block, output_block, false));
- temp_block.clear();
- release_block_memory(_probe_block);
- reached_limit(output_block, eos);
- return Status::OK();
- }
-
//TODO: this short circuit maybe could refactor, no need to check at here.
if (_empty_right_table_need_probe_dispose) {
// when build table rows is 0 and not have other_join_conjunct and join type is one of LEFT_OUTER_JOIN/FULL_OUTER_JOIN/LEFT_ANTI_JOIN
@@ -667,6 +670,10 @@ Status HashJoinNode::alloc_resource(doris::RuntimeState* state) {
for (auto& conjunct : _other_join_conjuncts) {
RETURN_IF_ERROR(conjunct->open(state));
}
+
+ for (auto& conjunct : _mark_join_conjuncts) {
+ RETURN_IF_ERROR(conjunct->open(state));
+ }
return Status::OK();
}
@@ -754,6 +761,7 @@ Status HashJoinNode::sink(doris::RuntimeState* state, vectorized::Block* in_bloc
// arena will be shared with other instances.
_shared_hash_table_context->arena = _arena;
_shared_hash_table_context->block = _build_block;
+ _shared_hash_table_context->build_indexes_null = _build_indexes_null;
_shared_hash_table_context->hash_table_variants = _hash_table_variants;
_shared_hash_table_context->short_circuit_for_null_in_probe_side =
_has_null_in_build_side;
@@ -786,6 +794,7 @@ Status HashJoinNode::sink(doris::RuntimeState* state, vectorized::Block* in_bloc
*std::static_pointer_cast<HashTableVariants>(
_shared_hash_table_context->hash_table_variants));
_build_block = _shared_hash_table_context->block;
+ _build_indexes_null = _shared_hash_table_context->build_indexes_null;
if (!_shared_hash_table_context->runtime_filters.empty()) {
auto ret = std::visit(
@@ -816,9 +825,10 @@ Status HashJoinNode::sink(doris::RuntimeState* state, vectorized::Block* in_bloc
_process_hashtable_ctx_variants_init(state);
}
- // Since the comparison of null values is meaningless, null aware left anti join should not output null
+ // Since the comparison of null values is meaningless, null aware left anti/semi join should not output null
// when the build side is not empty.
- if (_build_block && _join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN) {
+ if (_build_block && (_join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN ||
+ _join_op == TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN)) {
_probe_ignore_null = true;
}
_init_short_circuit_for_probe();
@@ -917,11 +927,10 @@ bool HashJoinNode::_need_probe_null_map(Block& block, const std::vector<int>& re
void HashJoinNode::_set_build_ignore_flag(Block& block, const std::vector<int>& res_col_ids) {
DCHECK_EQ(_build_expr_ctxs.size(), _probe_expr_ctxs.size());
for (size_t i = 0; i < _build_expr_ctxs.size(); ++i) {
- if (!_is_null_safe_eq_join[i]) {
+ if (!_is_null_safe_eq_join[i] && !_short_circuit_for_null_in_build_side) {
const auto* column = block.get_by_position(res_col_ids[i]).column.get();
if (check_and_get_column<ColumnNullable>(*column)) {
- _build_side_ignore_null |= (_join_op != TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN &&
- !_store_null_in_hash_table[i]);
+ _build_side_ignore_null |= !_store_null_in_hash_table[i];
}
}
}
@@ -962,30 +971,33 @@ Status HashJoinNode::_process_build_block(RuntimeState* state, Block& block) {
st = std::visit(
Overload {[&](std::monostate& arg, auto join_op, auto has_null_value,
- auto short_circuit_for_null_in_build_side) -> Status {
+ auto short_circuit_for_null_in_build_side,
+ auto with_other_conjuncts) -> Status {
LOG(FATAL) << "FATAL: uninited hash table";
__builtin_unreachable();
return Status::OK();
},
[&](auto&& arg, auto&& join_op, auto has_null_value,
- auto short_circuit_for_null_in_build_side) -> Status {
+ auto short_circuit_for_null_in_build_side,
+ auto with_other_conjuncts) -> Status {
using HashTableCtxType = std::decay_t<decltype(arg)>;
using JoinOpType = std::decay_t<decltype(join_op)>;
ProcessHashTableBuild<HashTableCtxType, HashJoinNode>
hash_table_build_process(rows, block, raw_ptrs, this,
state->batch_size(), state);
- return hash_table_build_process
- .template run<JoinOpType::value, has_null_value,
- short_circuit_for_null_in_build_side>(
- arg,
- has_null_value || short_circuit_for_null_in_build_side
- ? &null_map_val->get_data()
- : nullptr,
- &_has_null_in_build_side);
+ return hash_table_build_process.template run<
+ JoinOpType::value, has_null_value,
+ short_circuit_for_null_in_build_side, with_other_conjuncts>(
+ arg,
+ has_null_value || short_circuit_for_null_in_build_side
+ ? &null_map_val->get_data()
+ : nullptr,
+ &_has_null_in_build_side);
}},
*_hash_table_variants, _join_op_variants, make_bool_variant(_build_side_ignore_null),
- make_bool_variant(_short_circuit_for_null_in_build_side));
+ make_bool_variant(_short_circuit_for_null_in_build_side),
+ make_bool_variant(_have_other_join_conjunct));
return st;
}
diff --git a/be/src/vec/exec/join/vhash_join_node.h b/be/src/vec/exec/join/vhash_join_node.h
index 7fdb103d1f2..535db4434dc 100644
--- a/be/src/vec/exec/join/vhash_join_node.h
+++ b/be/src/vec/exec/join/vhash_join_node.h
@@ -110,13 +110,19 @@ struct ProcessHashTableBuild {
_batch_size(batch_size),
_state(state) {}
- template <int JoinOpType, bool ignore_null, bool short_circuit_for_null>
+ template <int JoinOpType, bool ignore_null, bool short_circuit_for_null,
+ bool with_other_conjuncts>
Status run(HashTableContext& hash_table_ctx, ConstNullMapPtr null_map, bool* has_null_key) {
if (short_circuit_for_null || ignore_null) {
// first row is mocked and is null
for (uint32_t i = 1; i < _rows; i++) {
if ((*null_map)[i]) {
*has_null_key = true;
+ if constexpr (with_other_conjuncts &&
+ (JoinOpType == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN ||
+ JoinOpType == TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN)) {
+ _parent->_build_indexes_null->emplace_back(i);
+ }
}
}
if (short_circuit_for_null && *has_null_key) {
@@ -201,7 +207,8 @@ using HashTableCtxVariants =
ProcessHashTableProbe<TJoinOp::CROSS_JOIN, HashJoinNode>,
ProcessHashTableProbe<TJoinOp::RIGHT_SEMI_JOIN, HashJoinNode>,
ProcessHashTableProbe<TJoinOp::RIGHT_ANTI_JOIN, HashJoinNode>,
- ProcessHashTableProbe<TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN, HashJoinNode>>;
+ ProcessHashTableProbe<TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN, HashJoinNode>,
+ ProcessHashTableProbe<TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN, HashJoinNode>>;
class HashJoinNode final : public VJoinNodeBase {
public:
@@ -291,6 +298,9 @@ private:
// other expr
VExprContextSPtrs _other_join_conjuncts;
+ // conjuncts for mark join, which result type is ternary boolean(true, false, null)
+ VExprContextSPtrs _mark_join_conjuncts;
+
// mark the join column whether support null eq
std::vector<bool> _is_null_safe_eq_join;
@@ -304,6 +314,13 @@ private:
std::vector<uint16_t> _probe_column_disguise_null;
std::vector<uint16_t> _probe_column_convert_to_null;
+ /*
+ * For null aware anti/semi join with other join conjuncts, we do need to care about the rows in
+ * build side with null keys,
+ * because the other join conjuncts' result maybe change null to false(null & false == false).
+ */
+ std::shared_ptr<std::vector<uint32_t>> _build_indexes_null;
+
DataTypes _right_table_data_types;
DataTypes _left_table_data_types;
std::vector<std::string> _right_table_column_names;
@@ -352,6 +369,9 @@ private:
bool _probe_eos = false;
int _last_probe_match;
+ // For mark join, last probe index of null mark
+ int _last_probe_null_mark;
+
bool _build_side_ignore_null = false;
bool _is_broadcast_join = false;
diff --git a/be/src/vec/exec/join/vjoin_node_base.cpp b/be/src/vec/exec/join/vjoin_node_base.cpp
index 656810ba7bd..e81851b7d93 100644
--- a/be/src/vec/exec/join/vjoin_node_base.cpp
+++ b/be/src/vec/exec/join/vjoin_node_base.cpp
@@ -63,29 +63,33 @@ VJoinNodeBase::VJoinNodeBase(ObjectPool* pool, const TPlanNode& tnode, const Des
_join_op == TJoinOp::FULL_OUTER_JOIN),
_build_unique(!_have_other_join_conjunct &&
(_join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN ||
+ _join_op == TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN ||
_join_op == TJoinOp::LEFT_ANTI_JOIN ||
_join_op == TJoinOp::LEFT_SEMI_JOIN)),
_is_right_semi_anti(_join_op == TJoinOp::RIGHT_ANTI_JOIN ||
_join_op == TJoinOp::RIGHT_SEMI_JOIN),
_is_left_semi_anti(_join_op == TJoinOp::LEFT_ANTI_JOIN ||
_join_op == TJoinOp::LEFT_SEMI_JOIN ||
- _join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN),
+ _join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN ||
+ _join_op == TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN),
_is_outer_join(_match_all_build || _match_all_probe),
_is_mark_join(tnode.__isset.nested_loop_join_node
- ? (tnode.nested_loop_join_node.__isset.is_mark
- ? tnode.nested_loop_join_node.is_mark
- : false)
- : tnode.hash_join_node.__isset.is_mark ? tnode.hash_join_node.is_mark
- : false),
- _short_circuit_for_null_in_build_side(_join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN) {
+ ? tnode.nested_loop_join_node.__isset.is_mark &&
+ tnode.nested_loop_join_node.is_mark
+ : tnode.hash_join_node.__isset.is_mark &&
+ tnode.hash_join_node.is_mark),
+ _short_circuit_for_null_in_build_side(_join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN &&
+ !_is_mark_join) {
_init_join_op();
if (_is_mark_join) {
DCHECK(_join_op == TJoinOp::LEFT_ANTI_JOIN || _join_op == TJoinOp::LEFT_SEMI_JOIN ||
- _join_op == TJoinOp::CROSS_JOIN || _join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN)
+ _join_op == TJoinOp::CROSS_JOIN || _join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN ||
+ _join_op == TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN)
<< "Mark join is only supported for null aware left semi/anti join and cross join "
"but this is "
<< _join_op;
}
+
if (tnode.__isset.hash_join_node) {
_output_row_desc.reset(
new RowDescriptor(descs, {tnode.hash_join_node.voutput_tuple_id}, {false}));
@@ -144,8 +148,15 @@ void VJoinNodeBase::_construct_mutable_join_block() {
}
}
- DCHECK(!_is_mark_join ||
- _join_block.get_by_position(_join_block.columns() - 1).column->is_nullable());
+ if (_is_mark_join) {
+ _mark_column_id = _join_block.columns() - 1;
+#ifndef NDEBUG
+ const auto& mark_column = assert_cast<const ColumnNullable&>(
+ *_join_block.get_by_position(_mark_column_id).column);
+ auto& nested_column = mark_column.get_nested_column();
+ DCHECK(check_and_get_column<ColumnUInt8>(nested_column) != nullptr);
+#endif
+ }
}
Status VJoinNodeBase::_build_output_block(Block* origin_block, Block* output_block,
@@ -290,7 +301,8 @@ void VJoinNodeBase::_probe_side_open_thread(RuntimeState* state, std::promise<St
M(CROSS_JOIN) \
M(RIGHT_SEMI_JOIN) \
M(RIGHT_ANTI_JOIN) \
- M(NULL_AWARE_LEFT_ANTI_JOIN)
+ M(NULL_AWARE_LEFT_ANTI_JOIN) \
+ M(NULL_AWARE_LEFT_SEMI_JOIN)
void VJoinNodeBase::_init_join_op() {
switch (_join_op) {
diff --git a/be/src/vec/exec/join/vjoin_node_base.h b/be/src/vec/exec/join/vjoin_node_base.h
index c918e26e6fe..95d10a0a07d 100644
--- a/be/src/vec/exec/join/vjoin_node_base.h
+++ b/be/src/vec/exec/join/vjoin_node_base.h
@@ -50,7 +50,8 @@ using JoinOpVariants =
std::integral_constant<TJoinOp::type, TJoinOp::CROSS_JOIN>,
std::integral_constant<TJoinOp::type, TJoinOp::RIGHT_SEMI_JOIN>,
std::integral_constant<TJoinOp::type, TJoinOp::RIGHT_ANTI_JOIN>,
- std::integral_constant<TJoinOp::type, TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN>>;
+ std::integral_constant<TJoinOp::type, TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN>,
+ std::integral_constant<TJoinOp::type, TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN>>;
class VJoinNodeBase : public ExecNode {
public:
@@ -121,6 +122,9 @@ protected:
// for some join, when build side rows is empty, we could return directly by add some additional null data in probe table.
bool _empty_right_table_need_probe_dispose = false;
+
+ size_t _mark_column_id;
+
std::unique_ptr<RowDescriptor> _output_row_desc;
std::unique_ptr<RowDescriptor> _intermediate_row_desc;
// output expr
diff --git a/be/src/vec/exprs/vexpr_context.cpp b/be/src/vec/exprs/vexpr_context.cpp
index cebb7dd2e53..7325ba5f315 100644
--- a/be/src/vec/exprs/vexpr_context.cpp
+++ b/be/src/vec/exprs/vexpr_context.cpp
@@ -223,6 +223,50 @@ Status VExprContext::execute_conjuncts(const VExprContextSPtrs& ctxs,
return Status::OK();
}
+Status VExprContext::execute_conjuncts(const VExprContextSPtrs& conjuncts, Block* block,
+ ColumnUInt8& null_map, IColumn::Filter& filter) {
+ const auto& rows = block->rows();
+ if (rows == 0) {
+ return Status::OK();
+ }
+
+ null_map.resize(rows);
+ auto* final_null_map = null_map.get_data().data();
+ memset(final_null_map, 0, rows);
+ filter.resize_fill(rows, 1);
+ auto* final_filter_ptr = filter.data();
+
+ for (const auto& conjunct : conjuncts) {
+ int result_column_id = -1;
+ RETURN_IF_ERROR(conjunct->execute(block, &result_column_id));
+ auto& filter_column =
+ unpack_if_const(block->get_by_position(result_column_id).column).first;
+ if (auto* nullable_column = check_and_get_column<ColumnNullable>(*filter_column)) {
+ const ColumnPtr& nested_column = nullable_column->get_nested_column_ptr();
+ const IColumn::Filter& result =
+ assert_cast<const ColumnUInt8&>(*nested_column).get_data();
+ auto* __restrict filter_data = result.data();
+ auto* __restrict null_map_data = nullable_column->get_null_map_data().data();
+ DCHECK_EQ(rows, nullable_column->size());
+
+ for (size_t i = 0; i != rows; ++i) {
+ // null and null => null
+ // null and true => null
+ // null and false => false
+ final_null_map[i] = (final_null_map[i] & (null_map_data[i] | filter_data[i])) |
+ (null_map_data[i] & (final_null_map[i] | final_filter_ptr[i]));
+ final_filter_ptr[i] = final_filter_ptr[i] & filter_data[i];
+ }
+ } else {
+ auto* filter_data = assert_cast<const ColumnUInt8&>(*filter_column).get_data().data();
+ for (size_t i = 0; i != rows; ++i) {
+ final_filter_ptr[i] = final_filter_ptr[i] & filter_data[i];
+ }
+ }
+ }
+ return Status::OK();
+}
+
// TODO Performance Optimization
// need exception safety
Status VExprContext::execute_conjuncts_and_filter_block(
diff --git a/be/src/vec/exprs/vexpr_context.h b/be/src/vec/exprs/vexpr_context.h
index 70bd37b1878..423e1aac12a 100644
--- a/be/src/vec/exprs/vexpr_context.h
+++ b/be/src/vec/exprs/vexpr_context.h
@@ -81,6 +81,10 @@ public:
IColumn::Filter* result_filter,
bool* can_filter_all);
+ [[nodiscard]] static Status execute_conjuncts(const VExprContextSPtrs& conjuncts, Block* block,
+ ColumnUInt8& null_map,
+ IColumn::Filter& result_filter);
+
static Status execute_conjuncts(const VExprContextSPtrs& ctxs,
const std::vector<IColumn::Filter*>* filters, Block* block,
IColumn::Filter* result_filter, bool* can_filter_all);
diff --git a/be/src/vec/runtime/shared_hash_table_controller.h b/be/src/vec/runtime/shared_hash_table_controller.h
index b8770e63856..a81fc994f39 100644
--- a/be/src/vec/runtime/shared_hash_table_controller.h
+++ b/be/src/vec/runtime/shared_hash_table_controller.h
@@ -59,6 +59,7 @@ struct SharedHashTableContext {
std::shared_ptr<Arena> arena;
std::shared_ptr<void> hash_table_variants;
std::shared_ptr<Block> block;
+ std::shared_ptr<std::vector<uint32_t>> build_indexes_null;
std::map<int, SharedRuntimeFilterContext> runtime_filters;
std::atomic<bool> signaled = false;
bool short_circuit_for_null_in_probe_side = false;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/JoinOperator.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/JoinOperator.java
index a45835f3bd3..bd8798bd4d4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/JoinOperator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/JoinOperator.java
@@ -37,7 +37,9 @@ public enum JoinOperator {
// NOT IN subqueries. It can have a single equality join conjunct
// that returns TRUE when the rhs is NULL.
NULL_AWARE_LEFT_ANTI_JOIN("NULL AWARE LEFT ANTI JOIN",
- TJoinOp.NULL_AWARE_LEFT_ANTI_JOIN);
+ TJoinOp.NULL_AWARE_LEFT_ANTI_JOIN),
+ NULL_AWARE_LEFT_SEMI_JOIN("NULL AWARE LEFT SEMI JOIN",
+ TJoinOp.NULL_AWARE_LEFT_SEMI_JOIN);
private final String description;
private final TJoinOp thriftJoinOp;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 5acda0f2980..cc8b2483ff8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -1183,9 +1183,22 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
.map(e -> JoinUtils.swapEqualToForChildrenOrder(e, hashJoin.left().getOutputSet()))
.map(e -> ExpressionTranslator.translate(e, context))
.collect(Collectors.toList());
+ List<Expr> markConjuncts = ImmutableList.of();
+ boolean isHashJoinConjunctsEmpty = hashJoin.getHashJoinConjuncts().isEmpty();
+ boolean isMarkJoinConjunctsEmpty = hashJoin.getMarkJoinConjuncts().isEmpty();
+ if (isHashJoinConjunctsEmpty) {
+ // if hash join conjuncts is empty, means mark join conjuncts must be EqualPredicate
+ // BE should use mark join conjuncts to build hash table
+ Preconditions.checkState(!isMarkJoinConjunctsEmpty, "mark join conjuncts should not be empty.");
+ markConjuncts = hashJoin.getMarkJoinConjuncts().stream()
+ .map(EqualPredicate.class::cast)
+ .map(e -> JoinUtils.swapEqualToForChildrenOrder(e, hashJoin.left().getOutputSet()))
+ .map(e -> ExpressionTranslator.translate(e, context))
+ .collect(Collectors.toList());
+ }
HashJoinNode hashJoinNode = new HashJoinNode(context.nextPlanNodeId(), leftPlanRoot,
- rightPlanRoot, JoinType.toJoinOperator(joinType), execEqConjuncts, Lists.newArrayList(),
+ rightPlanRoot, JoinType.toJoinOperator(joinType), execEqConjuncts, Lists.newArrayList(), markConjuncts,
null, null, null, hashJoin.isMarkJoin());
hashJoinNode.setNereidsId(hashJoin.getId());
hashJoinNode.setDistributeExprLists(distributeExprLists);
@@ -1246,6 +1259,15 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
.flatMap(e -> e.getInputSlots().stream())
.map(SlotReference.class::cast)
.forEach(s -> hashOutputSlotReferenceMap.put(s.getExprId(), s));
+ if (!isHashJoinConjunctsEmpty && !isMarkJoinConjunctsEmpty) {
+ // if hash join conjuncts is NOT empty, mark join conjuncts would be processed like other conjuncts
+ // BE should deal with mark join conjuncts differently, its result is 3 value bool(true, false, null)
+ hashJoin.getMarkJoinConjuncts()
+ .stream()
+ .flatMap(e -> e.getInputSlots().stream())
+ .map(SlotReference.class::cast)
+ .forEach(s -> hashOutputSlotReferenceMap.put(s.getExprId(), s));
+ }
hashJoin.getFilterConjuncts().stream()
.filter(e -> !(e.equals(BooleanLiteral.TRUE)))
.flatMap(e -> e.getInputSlots().stream())
@@ -1271,7 +1293,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
List<SlotDescriptor> rightIntermediateSlotDescriptor = Lists.newArrayList();
TupleDescriptor intermediateDescriptor = context.generateTupleDesc();
- if (hashJoin.getOtherJoinConjuncts().isEmpty()
+ if (hashJoin.getOtherJoinConjuncts().isEmpty() && (isHashJoinConjunctsEmpty != isMarkJoinConjunctsEmpty)
&& (joinType == JoinType.LEFT_ANTI_JOIN
|| joinType == JoinType.LEFT_SEMI_JOIN
|| joinType == JoinType.NULL_AWARE_LEFT_ANTI_JOIN)) {
@@ -1294,7 +1316,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
}
leftIntermediateSlotDescriptor.add(sd);
}
- } else if (hashJoin.getOtherJoinConjuncts().isEmpty()
+ } else if (hashJoin.getOtherJoinConjuncts().isEmpty() && (isHashJoinConjunctsEmpty != isMarkJoinConjunctsEmpty)
&& (joinType == JoinType.RIGHT_ANTI_JOIN || joinType == JoinType.RIGHT_SEMI_JOIN)) {
for (SlotDescriptor rightSlotDescriptor : rightSlotDescriptors) {
if (!rightSlotDescriptor.isMaterialized()) {
@@ -1391,6 +1413,15 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
hashJoinNode.setOtherJoinConjuncts(otherJoinConjuncts);
+ if (!isHashJoinConjunctsEmpty && !isMarkJoinConjunctsEmpty) {
+ // add mark join conjuncts to hash join node
+ List<Expr> markJoinConjuncts = hashJoin.getMarkJoinConjuncts()
+ .stream()
+ .map(e -> ExpressionTranslator.translate(e, context))
+ .collect(Collectors.toList());
+ hashJoinNode.setMarkJoinConjuncts(markJoinConjuncts);
+ }
+
hashJoinNode.setvIntermediateTupleDescList(Lists.newArrayList(intermediateDescriptor));
if (hashJoin.isShouldTranslateOutput()) {
@@ -1564,6 +1595,12 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
nestedLoopJoinNode.setJoinConjuncts(joinConjuncts);
+ if (!nestedLoopJoin.getOtherJoinConjuncts().isEmpty()) {
+ List<Expr> markJoinConjuncts = nestedLoopJoin.getMarkJoinConjuncts().stream()
+ .map(e -> ExpressionTranslator.translate(e, context)).collect(Collectors.toList());
+ nestedLoopJoinNode.setMarkJoinConjuncts(markJoinConjuncts);
+ }
+
nestedLoopJoin.getFilterConjuncts().stream()
.filter(e -> !(e.equals(BooleanLiteral.TRUE)))
.map(e -> ExpressionTranslator.translate(e, context))
@@ -1713,6 +1750,13 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
for (Expr expr : otherConjuncts) {
Expr.extractSlots(expr, requiredOtherConjunctsSlotIdSet);
}
+ if (!((HashJoinNode) joinNode).getEqJoinConjuncts().isEmpty()
+ && !((HashJoinNode) joinNode).getMarkJoinConjuncts().isEmpty()) {
+ List<Expr> markConjuncts = ((HashJoinNode) joinNode).getMarkJoinConjuncts();
+ for (Expr expr : markConjuncts) {
+ Expr.extractSlots(expr, requiredOtherConjunctsSlotIdSet);
+ }
+ }
requiredOtherConjunctsSlotIdSet.forEach(e -> requiredExprIds.add(context.findExprId(e)));
requiredSlotIdSet.forEach(e -> requiredExprIds.add(context.findExprId(e)));
for (ExprId exprId : requiredExprIds) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index 5f4c784d7da..6a994c1b6e5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -211,6 +211,7 @@ public enum RuleType {
ELIMINATE_NOT_NULL(RuleTypeClass.REWRITE),
ELIMINATE_UNNECESSARY_PROJECT(RuleTypeClass.REWRITE),
ELIMINATE_OUTER_JOIN(RuleTypeClass.REWRITE),
+ ELIMINATE_MARK_JOIN(RuleTypeClass.REWRITE),
ELIMINATE_GROUP_BY(RuleTypeClass.REWRITE),
ELIMINATE_JOIN_BY_UK(RuleTypeClass.REWRITE),
ELIMINATE_DEDUP_JOIN_CONDITION(RuleTypeClass.REWRITE),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java
index 42bc6d1f9bc..459118435c4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java
@@ -47,7 +47,7 @@ public class CollectJoinConstraint implements RewriteRuleFactory {
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
- logicalJoin().thenApply(ctx -> {
+ logicalJoin().whenNot(LogicalJoin::isMarkJoin).thenApply(ctx -> {
if (!ctx.cascadesContext.isLeadingJoin()) {
return ctx.root;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java
index 12eed92b402..b2f202aad8b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java
@@ -21,6 +21,7 @@ import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.rules.expression.rules.TrySimplifyPredicateWithMarkJoinSlot;
import org.apache.doris.nereids.trees.TreeNode;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.And;
@@ -49,6 +50,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalSort;
+import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
@@ -75,8 +77,6 @@ public class SubqueryToApply implements AnalysisRuleFactory {
RuleType.FILTER_SUBQUERY_TO_APPLY.build(
logicalFilter().thenApply(ctx -> {
LogicalFilter<Plan> filter = ctx.root;
- boolean shouldOutputMarkJoinSlot = filter.getConjuncts().stream()
- .anyMatch(expr -> shouldOutputMarkJoinSlot(expr, SearchState.SearchNot));
ImmutableList<Set<SubqueryExpr>> subqueryExprsList = filter.getConjuncts().stream()
.<Set<SubqueryExpr>>map(e -> e.collect(SubqueryToApply::canConvertToSupply))
.collect(ImmutableList.toImmutableList());
@@ -84,6 +84,11 @@ public class SubqueryToApply implements AnalysisRuleFactory {
.flatMap(Collection::stream).noneMatch(SubqueryExpr.class::isInstance)) {
return filter;
}
+ ImmutableList<Boolean> shouldOutputMarkJoinSlot =
+ filter.getConjuncts().stream()
+ .map(expr -> !(expr instanceof SubqueryExpr)
+ && expr.containsType(SubqueryExpr.class))
+ .collect(ImmutableList.toImmutableList());
List<Expression> oldConjuncts = ImmutableList.copyOf(filter.getConjuncts());
ImmutableList.Builder<Expression> newConjuncts = new ImmutableList.Builder<>();
@@ -101,15 +106,29 @@ public class SubqueryToApply implements AnalysisRuleFactory {
// first step: Replace the subquery of predicate in LogicalFilter
// second step: Replace subquery with LogicalApply
ReplaceSubquery replaceSubquery = new ReplaceSubquery(
- ctx.statementContext, shouldOutputMarkJoinSlot);
+ ctx.statementContext, shouldOutputMarkJoinSlot.get(i));
SubqueryContext context = new SubqueryContext(subqueryExprs);
Expression conjunct = replaceSubquery.replace(oldConjuncts.get(i), context);
+ /*
+ * the idea is replacing each mark join slot with null and false literal
+ * then run FoldConstant rule, if the evaluate result are:
+ * 1. all true
+ * 2. all null and false (in logicalFilter, we discard both null and false values)
+ * the mark slot can be non-nullable boolean
+ * we pass this info to LogicalApply. And in InApplyToJoin rule
+ * if it's semi join with non-null mark slot
+ * we can safely change the mark conjunct to hash conjunct
+ */
+ boolean isMarkSlotNotNull = conjunct.containsType(MarkJoinSlotReference.class)
+ ? ExpressionUtils.canInferNotNullForMarkSlot(
+ TrySimplifyPredicateWithMarkJoinSlot.INSTANCE.rewrite(conjunct, null))
+ : false;
applyPlan = subqueryToApply(subqueryExprs.stream()
.collect(ImmutableList.toImmutableList()), tmpPlan,
context.getSubqueryToMarkJoinSlot(),
ctx.cascadesContext,
- Optional.of(conjunct), false);
+ Optional.of(conjunct), false, isMarkSlotNotNull);
tmpPlan = applyPlan;
newConjuncts.add(conjunct);
}
@@ -120,8 +139,9 @@ public class SubqueryToApply implements AnalysisRuleFactory {
return new LogicalProject<>(applyPlan.getOutput().stream()
.filter(s -> !(s instanceof MarkJoinSlotReference))
.collect(ImmutableList.toImmutableList()), newFilter);
+ } else {
+ return newFilter;
}
- return new LogicalFilter<>(conjuncts, applyPlan);
})
),
RuleType.PROJECT_SUBQUERY_TO_APPLY.build(logicalProject().thenApply(ctx -> {
@@ -155,7 +175,7 @@ public class SubqueryToApply implements AnalysisRuleFactory {
subqueryExprs.stream().collect(ImmutableList.toImmutableList()),
childPlan, context.getSubqueryToMarkJoinSlot(),
ctx.cascadesContext,
- Optional.of(newProject), true);
+ Optional.of(newProject), true, false);
childPlan = applyPlan;
newProjects.add((NamedExpression) newProject);
}
@@ -216,12 +236,25 @@ public class SubqueryToApply implements AnalysisRuleFactory {
ReplaceSubquery replaceSubquery = new ReplaceSubquery(ctx.statementContext, true);
SubqueryContext context = new SubqueryContext(subqueryExprs);
Expression conjunct = replaceSubquery.replace(subqueryConjuncts.get(i), context);
-
+ /*
+ * the idea is replacing each mark join slot with null and false literal
+ * then run FoldConstant rule, if the evaluate result are:
+ * 1. all true
+ * 2. all null and false (in logicalFilter, we discard both null and false values)
+ * the mark slot can be non-nullable boolean
+ * we pass this info to LogicalApply. And in InApplyToJoin rule
+ * if it's semi join with non-null mark slot
+ * we can safely change the mark conjunct to hash conjunct
+ */
+ boolean isMarkSlotNotNull = conjunct.containsType(MarkJoinSlotReference.class)
+ ? ExpressionUtils.canInferNotNullForMarkSlot(
+ TrySimplifyPredicateWithMarkJoinSlot.INSTANCE.rewrite(conjunct, null))
+ : false;
applyPlan = subqueryToApply(
subqueryExprs.stream().collect(ImmutableList.toImmutableList()),
relatedInfoList.get(i) == RelatedInfo.RelatedToLeft ? leftChildPlan : rightChildPlan,
context.getSubqueryToMarkJoinSlot(),
- ctx.cascadesContext, Optional.of(conjunct), false);
+ ctx.cascadesContext, Optional.of(conjunct), false, isMarkSlotNotNull);
if (relatedInfoList.get(i) == RelatedInfo.RelatedToLeft) {
leftChildPlan = applyPlan;
} else {
@@ -282,7 +315,7 @@ public class SubqueryToApply implements AnalysisRuleFactory {
SubqueryExpr subqueryExpr = subqueryExprs.get(0);
List<Slot> correlatedSlots = subqueryExpr.getCorrelateSlots();
if (subqueryExpr instanceof ScalarSubquery) {
- Set<Slot> inputSlots = expression.getInputSlots();
+ Set<Slot> inputSlots = subqueryExpr.getInputSlots();
if (correlatedSlots.isEmpty() && inputSlots.isEmpty()) {
relatedInfo = RelatedInfo.Unrelated;
} else if (leftOutputSlots.containsAll(inputSlots)
@@ -322,7 +355,8 @@ public class SubqueryToApply implements AnalysisRuleFactory {
private LogicalPlan subqueryToApply(List<SubqueryExpr> subqueryExprs, LogicalPlan childPlan,
Map<SubqueryExpr, Optional<MarkJoinSlotReference>> subqueryToMarkJoinSlot,
CascadesContext ctx,
- Optional<Expression> conjunct, boolean isProject) {
+ Optional<Expression> conjunct, boolean isProject,
+ boolean isMarkJoinSlotNotNull) {
LogicalPlan tmpPlan = childPlan;
for (int i = 0; i < subqueryExprs.size(); ++i) {
SubqueryExpr subqueryExpr = subqueryExprs.get(i);
@@ -336,7 +370,7 @@ public class SubqueryToApply implements AnalysisRuleFactory {
if (!ctx.subqueryIsAnalyzed(subqueryExpr)) {
tmpPlan = addApply(subqueryExpr, tmpPlan,
subqueryToMarkJoinSlot, ctx, conjunct,
- isProject, subqueryExprs.size() == 1);
+ isProject, subqueryExprs.size() == 1, isMarkJoinSlotNotNull);
}
}
return tmpPlan;
@@ -354,7 +388,7 @@ public class SubqueryToApply implements AnalysisRuleFactory {
private LogicalPlan addApply(SubqueryExpr subquery, LogicalPlan childPlan,
Map<SubqueryExpr, Optional<MarkJoinSlotReference>> subqueryToMarkJoinSlot,
CascadesContext ctx, Optional<Expression> conjunct,
- boolean isProject, boolean singleSubquery) {
+ boolean isProject, boolean singleSubquery, boolean isMarkJoinSlotNotNull) {
ctx.setSubqueryExprIsAnalyzed(subquery, true);
boolean needAddScalarSubqueryOutputToProjects = isConjunctContainsScalarSubqueryOutput(
subquery, conjunct, isProject, singleSubquery);
@@ -362,7 +396,7 @@ public class SubqueryToApply implements AnalysisRuleFactory {
subquery.getCorrelateSlots(),
subquery, Optional.empty(),
subqueryToMarkJoinSlot.get(subquery),
- needAddScalarSubqueryOutputToProjects, isProject,
+ needAddScalarSubqueryOutputToProjects, isProject, isMarkJoinSlotNotNull,
childPlan, subquery.getQueryPlan());
List<NamedExpression> projects = ImmutableList.<NamedExpression>builder()
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java
index ebe27c0cfed..116cf426a4c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.rules.expression;
+import org.apache.doris.common.Pair;
import org.apache.doris.nereids.properties.OrderKey;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
@@ -39,7 +40,6 @@ import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Lists;
import java.util.ArrayList;
import java.util.List;
@@ -179,34 +179,39 @@ public class ExpressionRewrite implements RewriteRuleFactory {
LogicalJoin<Plan, Plan> join = ctx.root;
List<Expression> hashJoinConjuncts = join.getHashJoinConjuncts();
List<Expression> otherJoinConjuncts = join.getOtherJoinConjuncts();
- if (otherJoinConjuncts.isEmpty() && hashJoinConjuncts.isEmpty()) {
+ List<Expression> markJoinConjuncts = join.getMarkJoinConjuncts();
+ if (otherJoinConjuncts.isEmpty() && hashJoinConjuncts.isEmpty()
+ && markJoinConjuncts.isEmpty()) {
return join;
}
- ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext);
- List<Expression> rewriteHashJoinConjuncts = Lists.newArrayList();
- boolean hashJoinConjunctsChanged = false;
- for (Expression expr : hashJoinConjuncts) {
- Expression newExpr = rewriter.rewrite(expr, context);
- hashJoinConjunctsChanged = hashJoinConjunctsChanged || !newExpr.equals(expr);
- rewriteHashJoinConjuncts.addAll(ExpressionUtils.extractConjunction(newExpr));
- }
- List<Expression> rewriteOtherJoinConjuncts = Lists.newArrayList();
- boolean otherJoinConjunctsChanged = false;
- for (Expression expr : otherJoinConjuncts) {
- Expression newExpr = rewriter.rewrite(expr, context);
- otherJoinConjunctsChanged = otherJoinConjunctsChanged || !newExpr.equals(expr);
- rewriteOtherJoinConjuncts.addAll(ExpressionUtils.extractConjunction(newExpr));
- }
+ ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext);
+ Pair<Boolean, List<Expression>> newHashJoinConjuncts = rewriteConjuncts(hashJoinConjuncts, context);
+ Pair<Boolean, List<Expression>> newOtherJoinConjuncts = rewriteConjuncts(otherJoinConjuncts, context);
+ Pair<Boolean, List<Expression>> newMarkJoinConjuncts = rewriteConjuncts(markJoinConjuncts, context);
- if (!hashJoinConjunctsChanged && !otherJoinConjunctsChanged) {
+ if (!newHashJoinConjuncts.first && !newOtherJoinConjuncts.first
+ && !newMarkJoinConjuncts.first) {
return join;
}
- return new LogicalJoin<>(join.getJoinType(), rewriteHashJoinConjuncts,
- rewriteOtherJoinConjuncts, join.getDistributeHint(), join.getMarkJoinSlotReference(),
- join.children());
+
+ return new LogicalJoin<>(join.getJoinType(), newHashJoinConjuncts.second,
+ newOtherJoinConjuncts.second, newMarkJoinConjuncts.second,
+ join.getDistributeHint(), join.getMarkJoinSlotReference(), join.children());
}).toRule(RuleType.REWRITE_JOIN_EXPRESSION);
}
+
+ private Pair<Boolean, List<Expression>> rewriteConjuncts(List<Expression> conjuncts,
+ ExpressionRewriteContext context) {
+ boolean isChanged = false;
+ ImmutableList.Builder<Expression> rewrittenConjuncts = new ImmutableList.Builder<>();
+ for (Expression expr : conjuncts) {
+ Expression newExpr = rewriter.rewrite(expr, context);
+ isChanged = isChanged || !newExpr.equals(expr);
+ rewrittenConjuncts.addAll(ExpressionUtils.extractConjunction(newExpr));
+ }
+ return Pair.of(isChanged, rewrittenConjuncts.build());
+ }
}
private class SortExpressionRewrite extends OneRewriteRuleFactory {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/TrySimplifyPredicateWithMarkJoinSlot.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/TrySimplifyPredicateWithMarkJoinSlot.java
new file mode 100644
index 00000000000..d4dc6697d80
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/TrySimplifyPredicateWithMarkJoinSlot.java
@@ -0,0 +1,113 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.expression.rules;
+
+import org.apache.doris.nereids.rules.expression.AbstractExpressionRewriteRule;
+import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
+import org.apache.doris.nereids.trees.expressions.And;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference;
+import org.apache.doris.nereids.trees.expressions.Or;
+import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
+
+/**
+ * TrySimplifyPredicateWithMarkJoinSlot
+ */
+public class TrySimplifyPredicateWithMarkJoinSlot extends AbstractExpressionRewriteRule {
+ public static final TrySimplifyPredicateWithMarkJoinSlot INSTANCE =
+ new TrySimplifyPredicateWithMarkJoinSlot();
+
+ @Override
+ public Expression visitAnd(And and, ExpressionRewriteContext context) {
+ /*
+ * predicate(with mark slot) and predicate(no mark slot)
+ * false and TRUE -> false(*) -> discard
+ * false and NULL -> null -> discard
+ * false and FALSE -> false -> discard
+ *
+ * null and TRUE -> null(*) -> discard
+ * null and NULL -> null -> discard
+ * null and FALSE -> false -> discard
+ *
+ * true and TRUE -> true(x) -> keep
+ * true and NULL -> null -> discard
+ * true and FALSE -> false -> discard
+ *
+ * we can see only 'predicate(with mark slot) and TRUE' may produce different results(*)
+ * because in filter predicate, we discard null and false values and only keep true values
+ * we can substitute mark slot with null and false to evaluate the predicate
+ * if the result are true, or result is either false or null, we can use non-nullable mark slot
+ * see ExpressionUtils.canInferNotNullForMarkSlot for more info
+ * we change 'predicate(with mark slot) and predicate(no mark slot)' -> predicate(with mark slot) and true
+ * to evaluate the predicate
+ */
+ Expression left = and.left();
+ Expression newLeft = left.accept(this, context);
+
+ if (newLeft.getInputSlots().stream().noneMatch(MarkJoinSlotReference.class::isInstance)) {
+ newLeft = BooleanLiteral.TRUE;
+ }
+
+ Expression right = and.right();
+ Expression newRight = right.accept(this, context);
+ if (newRight.getInputSlots().stream().noneMatch(MarkJoinSlotReference.class::isInstance)) {
+ newRight = BooleanLiteral.TRUE;
+ }
+ Expression expr = new And(newLeft, newRight);
+ return expr;
+ }
+
+ @Override
+ public Expression visitOr(Or or, ExpressionRewriteContext context) {
+ /*
+ * predicate(with mark slot) or predicate(no mark slot)
+ * false or TRUE -> true -> keep
+ * false or NULL -> null(^) -> discard
+ * false or FALSE -> false(*) -> discard
+ *
+ * null or TRUE -> true -> keep
+ * null or NULL -> null(^) -> discard
+ * null or FALSE -> null(*) -> discard
+ *
+ * true or TRUE -> true -> keep
+ * true or NULL -> true(#) -> keep
+ * true or FALSE -> true(x) -> keep
+ *
+ * like And operator, even there are more differences. we can get the same conclusion.
+ * by substituting mark slot with null and false to evaluate the predicate
+ * if the result are true, or result is either false or null, we can use non-nullable mark slot
+ * we change 'predicate(with mark slot) or predicate(no mark slot)' -> predicate(with mark slot) or false
+ * to evaluate the predicate
+ */
+ Expression left = or.left();
+ Expression newLeft = left.accept(this, context);
+
+ if (newLeft.getInputSlots().stream().noneMatch(MarkJoinSlotReference.class::isInstance)) {
+ newLeft = BooleanLiteral.FALSE;
+ }
+
+ Expression right = or.right();
+ Expression newRight = right.accept(this, context);
+ if (newRight.getInputSlots().stream().noneMatch(MarkJoinSlotReference.class::isInstance)) {
+ newRight = BooleanLiteral.FALSE;
+ }
+ Expression expr = new Or(newLeft, newRight);
+ return expr;
+ }
+
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalJoinToHashJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalJoinToHashJoin.java
index 1ab830d57b4..a8de496af44 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalJoinToHashJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalJoinToHashJoin.java
@@ -34,6 +34,7 @@ public class LogicalJoinToHashJoin extends OneImplementationRuleFactory {
join.getJoinType(),
join.getHashJoinConjuncts(),
join.getOtherJoinConjuncts(),
+ join.getMarkJoinConjuncts(),
join.getDistributeHint(),
join.getMarkJoinSlotReference(),
join.getLogicalProperties(),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalJoinToNestedLoopJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalJoinToNestedLoopJoin.java
index af02f6a2ed1..278d90a7634 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalJoinToNestedLoopJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalJoinToNestedLoopJoin.java
@@ -34,6 +34,7 @@ public class LogicalJoinToNestedLoopJoin extends OneImplementationRuleFactory {
join.getJoinType(),
join.getHashJoinConjuncts(),
join.getOtherJoinConjuncts(),
+ join.getMarkJoinConjuncts(),
join.getMarkJoinSlotReference(),
join.getLogicalProperties(),
join.left(),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustConjunctsReturnType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustConjunctsReturnType.java
index 9c1a60fdf5f..a3c4263b2b2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustConjunctsReturnType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustConjunctsReturnType.java
@@ -64,6 +64,9 @@ public class AdjustConjunctsReturnType extends DefaultPlanRewriter<Void> impleme
List<Expression> otherConjuncts = join.getOtherJoinConjuncts().stream()
.map(expr -> TypeCoercionUtils.castIfNotSameType(expr, BooleanType.INSTANCE))
.collect(Collectors.toList());
- return join.withJoinConjuncts(hashConjuncts, otherConjuncts);
+ List<Expression> markConjuncts = join.getMarkJoinConjuncts().stream()
+ .map(expr -> TypeCoercionUtils.castIfNotSameType(expr, BooleanType.INSTANCE))
+ .collect(Collectors.toList());
+ return join.withJoinConjuncts(hashConjuncts, otherConjuncts, markConjuncts);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java
index 8f70b86e4bd..7bb4251f168 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java
@@ -49,6 +49,7 @@ import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter;
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter;
import org.apache.doris.nereids.util.ExpressionUtils;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
@@ -117,9 +118,22 @@ public class AdjustNullable extends DefaultPlanRewriter<Map<ExprId, Slot>> imple
public Plan visitLogicalJoin(LogicalJoin<? extends Plan, ? extends Plan> join, Map<ExprId, Slot> replaceMap) {
join = (LogicalJoin<? extends Plan, ? extends Plan>) super.visit(join, replaceMap);
List<Expression> hashConjuncts = updateExpressions(join.getHashJoinConjuncts(), replaceMap);
+ List<Expression> markConjuncts;
+ if (hashConjuncts.isEmpty()) {
+ // if hashConjuncts is empty, mark join conjuncts may used to build hash table
+ // so need call updateExpressions for mark join conjuncts before adjust nullable by output slot
+ markConjuncts = updateExpressions(join.getMarkJoinConjuncts(), replaceMap);
+ } else {
+ markConjuncts = null;
+ }
join.getOutputSet().forEach(o -> replaceMap.put(o.getExprId(), o));
+ if (markConjuncts == null) {
+ // hashConjuncts is not empty, mark join conjuncts are processed like other join conjuncts
+ Preconditions.checkState(!hashConjuncts.isEmpty(), "hash conjuncts should not be empty");
+ markConjuncts = updateExpressions(join.getMarkJoinConjuncts(), replaceMap);
+ }
List<Expression> otherConjuncts = updateExpressions(join.getOtherJoinConjuncts(), replaceMap);
- return join.withJoinConjuncts(hashConjuncts, otherConjuncts).recomputeLogicalProperties();
+ return join.withJoinConjuncts(hashConjuncts, otherConjuncts, markConjuncts).recomputeLogicalProperties();
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CheckDataTypes.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CheckDataTypes.java
index d2578684919..875bf42bab2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CheckDataTypes.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CheckDataTypes.java
@@ -33,6 +33,7 @@ import org.apache.doris.nereids.types.UnsupportedType;
import com.google.common.collect.ImmutableSet;
+import java.util.List;
import java.util.Set;
/**
@@ -58,7 +59,12 @@ public class CheckDataTypes implements CustomRewriter {
}
private void checkLogicalJoin(LogicalJoin<? extends Plan, ? extends Plan> plan) {
- plan.getHashJoinConjuncts().forEach(expr -> {
+ List<Expression> conjuncts = plan.getHashJoinConjuncts();
+ if (conjuncts.isEmpty()) {
+ // if hash conjuncts are empty, we may use mark conjuncts to build hash table
+ conjuncts = plan.getMarkJoinConjuncts();
+ }
+ conjuncts.forEach(expr -> {
DataType leftType = expr.child(0).getDataType();
DataType rightType = expr.child(1).getDataType();
if (!leftType.acceptsType(rightType)) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConvertInnerOrCrossJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConvertInnerOrCrossJoin.java
index fd41d2deea1..6f67ce9fc9f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConvertInnerOrCrossJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConvertInnerOrCrossJoin.java
@@ -37,11 +37,13 @@ public class ConvertInnerOrCrossJoin implements RewriteRuleFactory {
public List<Rule> buildRules() {
return ImmutableList.of(
innerLogicalJoin()
- .when(join -> join.getHashJoinConjuncts().size() == 0 && join.getOtherJoinConjuncts().size() == 0)
+ .when(join -> join.getHashJoinConjuncts().isEmpty() && join.getOtherJoinConjuncts().isEmpty()
+ && join.getMarkJoinConjuncts().isEmpty())
.then(join -> join.withJoinType(JoinType.CROSS_JOIN))
.toRule(RuleType.INNER_TO_CROSS_JOIN),
crossLogicalJoin()
- .when(join -> join.getHashJoinConjuncts().size() != 0 || join.getOtherJoinConjuncts().size() != 0)
+ .when(join -> !join.getHashJoinConjuncts().isEmpty() || !join.getOtherJoinConjuncts().isEmpty()
+ || !join.getMarkJoinConjuncts().isEmpty())
.then(join -> join.withJoinType(JoinType.INNER_JOIN))
.toRule(RuleType.CROSS_TO_INNER_JOIN)
);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateDedupJoinCondition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateDedupJoinCondition.java
index eeabd177ffa..f8baaefb39f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateDedupJoinCondition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateDedupJoinCondition.java
@@ -37,11 +37,14 @@ public class EliminateDedupJoinCondition extends OneRewriteRuleFactory {
.distinct().collect(Collectors.toList());
List<Expression> dedupOtherJoinConjuncts = join.getOtherJoinConjuncts().stream()
.distinct().collect(Collectors.toList());
+ List<Expression> dedupMarkJoinConjuncts = join.getMarkJoinConjuncts().stream()
+ .distinct().collect(Collectors.toList());
if (dedupHashJoinConjuncts.size() == join.getHashJoinConjuncts().size()
- && dedupOtherJoinConjuncts.size() == join.getOtherJoinConjuncts().size()) {
+ && dedupOtherJoinConjuncts.size() == join.getOtherJoinConjuncts().size()
+ && dedupMarkJoinConjuncts.size() == join.getMarkJoinConjuncts().size()) {
return null;
}
- return join.withJoinConjuncts(dedupHashJoinConjuncts, dedupOtherJoinConjuncts);
+ return join.withJoinConjuncts(dedupHashJoinConjuncts, dedupOtherJoinConjuncts, dedupMarkJoinConjuncts);
}).toRule(RuleType.ELIMINATE_DEDUP_JOIN_CONDITION);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinCondition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinCondition.java
index 37e4cb85cea..c997ad328ac 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinCondition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinCondition.java
@@ -39,11 +39,15 @@ public class EliminateJoinCondition extends OneRewriteRuleFactory {
List<Expression> otherJoinConjuncts = join.getOtherJoinConjuncts().stream()
.filter(expression -> !expression.equals(BooleanLiteral.TRUE))
.collect(Collectors.toList());
+ List<Expression> markJoinConjuncts = join.getMarkJoinConjuncts().stream()
+ .filter(expression -> !expression.equals(BooleanLiteral.TRUE))
+ .collect(Collectors.toList());
if (hashJoinConjuncts.size() == join.getHashJoinConjuncts().size()
- && otherJoinConjuncts.size() == join.getOtherJoinConjuncts().size()) {
+ && otherJoinConjuncts.size() == join.getOtherJoinConjuncts().size()
+ && markJoinConjuncts.size() == join.getMarkJoinConjuncts().size()) {
return null;
}
- return join.withJoinConjuncts(hashJoinConjuncts, otherJoinConjuncts);
+ return join.withJoinConjuncts(hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts);
}).toRule(RuleType.ELIMINATE_JOIN_CONDITION);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateMarkJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateMarkJoin.java
new file mode 100644
index 00000000000..a5d59291cce
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateMarkJoin.java
@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.rules.expression.rules.TrySimplifyPredicateWithMarkJoinSlot;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
+import org.apache.doris.nereids.util.ExpressionUtils;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.Set;
+
+/**
+ * Eliminate mark join.
+ */
+public class EliminateMarkJoin extends OneRewriteRuleFactory {
+
+ @Override
+ public Rule build() {
+ return logicalFilter(logicalJoin().when(
+ join -> join.getJoinType().isSemiJoin() && !join.getMarkJoinConjuncts().isEmpty()))
+ .when(filter -> canSimplifyMarkJoin(filter.getConjuncts()))
+ .then(filter -> filter.withChildren(eliminateMarkJoin(filter.child())))
+ .toRule(RuleType.ELIMINATE_MARK_JOIN);
+ }
+
+ private boolean canSimplifyMarkJoin(Set<Expression> predicates) {
+ return ExpressionUtils
+ .canInferNotNullForMarkSlot(TrySimplifyPredicateWithMarkJoinSlot.INSTANCE
+ .rewrite(ExpressionUtils.and(predicates), null));
+ }
+
+ private LogicalJoin<Plan, Plan> eliminateMarkJoin(LogicalJoin<Plan, Plan> join) {
+ ImmutableList.Builder<Expression> newHashConjuncts = ImmutableList.builder();
+ newHashConjuncts.addAll(join.getHashJoinConjuncts());
+ newHashConjuncts.addAll(join.getMarkJoinConjuncts());
+ return join.withJoinConjuncts(newHashConjuncts.build(), join.getOtherJoinConjuncts(),
+ ExpressionUtils.EMPTY_CONDITION);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateNullAwareLeftAntiJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateNullAwareLeftAntiJoin.java
index fd62dc6be9d..0621d235758 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateNullAwareLeftAntiJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateNullAwareLeftAntiJoin.java
@@ -31,7 +31,10 @@ public class EliminateNullAwareLeftAntiJoin extends OneRewriteRuleFactory {
@Override
public Rule build() {
return nullAwareLeftAntiLogicalJoin().then(antiJoin -> {
- if (Stream.concat(antiJoin.getHashJoinConjuncts().stream(), antiJoin.getOtherJoinConjuncts().stream())
+ if (Stream.concat(Stream.concat(
+ antiJoin.getHashJoinConjuncts().stream(),
+ antiJoin.getOtherJoinConjuncts().stream()),
+ antiJoin.getMarkJoinConjuncts().stream())
.noneMatch(expression -> expression.nullable())) {
return antiJoin.withJoinType(JoinType.LEFT_ANTI_JOIN);
} else {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateSemiJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateSemiJoin.java
index 3c9f09e9316..f514a1d14e9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateSemiJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateSemiJoin.java
@@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import java.util.List;
@@ -35,6 +36,7 @@ public class EliminateSemiJoin extends OneRewriteRuleFactory {
public Rule build() {
return logicalJoin()
// right will be converted to left
+ .whenNot(LogicalJoin::isMarkJoin)
.when(join -> join.getJoinType().isLeftSemiOrAntiJoin())
.when(join -> join.getHashJoinConjuncts().isEmpty())
.then(join -> {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExtractFilterFromCrossJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExtractFilterFromCrossJoin.java
index 5d360d1a0e8..f5b2e41502c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExtractFilterFromCrossJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExtractFilterFromCrossJoin.java
@@ -40,7 +40,9 @@ public class ExtractFilterFromCrossJoin extends OneRewriteRuleFactory {
return crossLogicalJoin()
.then(join -> {
LogicalJoin<Plan, Plan> newJoin = new LogicalJoin<>(JoinType.CROSS_JOIN,
- ExpressionUtils.EMPTY_CONDITION, ExpressionUtils.EMPTY_CONDITION, join.getDistributeHint(),
+ ExpressionUtils.EMPTY_CONDITION, ExpressionUtils.EMPTY_CONDITION,
+ join.getMarkJoinConjuncts(),
+ join.getDistributeHint(),
join.getMarkJoinSlotReference(), join.children());
Set<Expression> predicates = Stream.concat(join.getHashJoinConjuncts().stream(),
join.getOtherJoinConjuncts().stream())
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/FindHashConditionForJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/FindHashConditionForJoin.java
index aeeeefb593b..a3d2cacfb6e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/FindHashConditionForJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/FindHashConditionForJoin.java
@@ -73,6 +73,7 @@ public class FindHashConditionForJoin extends OneRewriteRuleFactory {
return new LogicalJoin<>(joinType,
combinedHashJoinConjuncts,
remainedNonHashJoinConjuncts,
+ join.getMarkJoinConjuncts(),
join.getDistributeHint(),
join.getMarkJoinSlotReference(),
join.children());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InApplyToJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InApplyToJoin.java
index 054b92c9f79..e3702f2a93f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InApplyToJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InApplyToJoin.java
@@ -100,32 +100,54 @@ public class InApplyToJoin extends OneRewriteRuleFactory {
// TODO: trick here, because when deep copy logical plan the apply right child
// is not same with query plan in subquery expr, since the scan node copy twice
Expression right = inSubquery.getSubqueryOutput((LogicalPlan) apply.right());
- if (apply.isCorrelated()) {
- if (inSubquery.isNot()) {
- predicate = ExpressionUtils.and(ExpressionUtils.or(new EqualTo(left, right),
- new IsNull(left), new IsNull(right)),
- apply.getCorrelationFilter().get());
- } else {
- predicate = ExpressionUtils.and(new EqualTo(left, right),
- apply.getCorrelationFilter().get());
- }
- } else {
+ if (apply.isMarkJoin()) {
+ List<Expression> joinConjuncts = apply.getCorrelationFilter().isPresent()
+ ? ExpressionUtils.extractConjunction(apply.getCorrelationFilter().get())
+ : Lists.newArrayList();
predicate = new EqualTo(left, right);
- }
-
- List<Expression> conjuncts = ExpressionUtils.extractConjunction(predicate);
- if (inSubquery.isNot()) {
+ List<Expression> markConjuncts = Lists.newArrayList(predicate);
+ if (!predicate.nullable() || (apply.isMarkJoinSlotNotNull() && !inSubquery.isNot())) {
+ // we can merge mark conjuncts with hash conjuncts in 2 scenarios
+ // 1. the mark join predicate is not nullable, so no null value would be produced
+ // 2. semi join with non-nullable mark slot.
+ // because semi join only care about mark slot with true value and discard false and null
+ // it's safe the use false instead of null in this case
+ joinConjuncts.addAll(markConjuncts);
+ markConjuncts.clear();
+ }
return new LogicalJoin<>(
- predicate.nullable() && !apply.isCorrelated()
- ? JoinType.NULL_AWARE_LEFT_ANTI_JOIN
- : JoinType.LEFT_ANTI_JOIN,
- Lists.newArrayList(), conjuncts, new DistributeHint(DistributeType.NONE),
- apply.getMarkJoinSlotReference(), apply.children());
- } else {
- return new LogicalJoin<>(JoinType.LEFT_SEMI_JOIN, Lists.newArrayList(),
- conjuncts,
+ inSubquery.isNot() ? JoinType.LEFT_ANTI_JOIN : JoinType.LEFT_SEMI_JOIN,
+ Lists.newArrayList(), joinConjuncts, markConjuncts,
new DistributeHint(DistributeType.NONE), apply.getMarkJoinSlotReference(),
apply.children());
+ } else {
+ if (apply.isCorrelated()) {
+ if (inSubquery.isNot()) {
+ predicate = ExpressionUtils.and(ExpressionUtils.or(new EqualTo(left, right),
+ new IsNull(left), new IsNull(right)),
+ apply.getCorrelationFilter().get());
+ } else {
+ predicate = ExpressionUtils.and(new EqualTo(left, right),
+ apply.getCorrelationFilter().get());
+ }
+ } else {
+ predicate = new EqualTo(left, right);
+ }
+
+ List<Expression> conjuncts = ExpressionUtils.extractConjunction(predicate);
+ if (inSubquery.isNot()) {
+ return new LogicalJoin<>(
+ predicate.nullable() && !apply.isCorrelated()
+ ? JoinType.NULL_AWARE_LEFT_ANTI_JOIN
+ : JoinType.LEFT_ANTI_JOIN,
+ Lists.newArrayList(), conjuncts, new DistributeHint(DistributeType.NONE),
+ apply.getMarkJoinSlotReference(), apply.children());
+ } else {
+ return new LogicalJoin<>(JoinType.LEFT_SEMI_JOIN, Lists.newArrayList(),
+ conjuncts,
+ new DistributeHint(DistributeType.NONE), apply.getMarkJoinSlotReference(),
+ apply.children());
+ }
}
}).toRule(RuleType.IN_APPLY_TO_JOIN);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OrExpansion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OrExpansion.java
index 1e4d7793ba6..3968f817e8e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OrExpansion.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OrExpansion.java
@@ -74,6 +74,7 @@ public class OrExpansion extends OneExplorationRuleFactory {
@Override
public Rule build() {
return logicalJoin(any(), any()).when(JoinUtils::shouldNestedLoopJoin)
+ .whenNot(LogicalJoin::isMarkJoin)
.when(join -> supportJoinType.contains(join.getJoinType())
&& ConnectContext.get().getSessionVariable().getEnablePipelineEngine())
.thenApply(ctx -> {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAliasThroughJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAliasThroughJoin.java
index a544dfa361e..0618ffc5f5c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAliasThroughJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAliasThroughJoin.java
@@ -96,8 +96,9 @@ public class PushDownAliasThroughJoin extends OneRewriteRuleFactory {
List<Expression> newHash = replaceJoinConjuncts(join.getHashJoinConjuncts(), replaceMap);
List<Expression> newOther = replaceJoinConjuncts(join.getOtherJoinConjuncts(), replaceMap);
+ List<Expression> newMark = replaceJoinConjuncts(join.getMarkJoinConjuncts(), replaceMap);
- Plan newJoin = join.withConjunctsChildren(newHash, newOther, left, right);
+ Plan newJoin = join.withConjunctsChildren(newHash, newOther, newMark, left, right);
return project.withProjectsAndChild(newProjects, newJoin);
}).toRule(RuleType.PUSH_DOWN_ALIAS_THROUGH_JOIN);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownExpressionsInHashCondition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownExpressionsInHashCondition.java
index 4aeb47a9d48..594498cf6b5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownExpressionsInHashCondition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownExpressionsInHashCondition.java
@@ -99,6 +99,19 @@ public class PushDownExpressionsInHashCondition extends OneRewriteRuleFactory {
}
});
+ // add mark conjuncts used slots to project exprs
+ join.getMarkJoinConjuncts().stream().flatMap(conjunct ->
+ conjunct.getInputSlots().stream()
+ ).forEach(slot -> {
+ if (leftExprIdSet.contains(slot.getExprId())) {
+ // belong to left child
+ leftProjectExprs.add(slot);
+ } else {
+ // belong to right child
+ rightProjectExprs.add(slot);
+ }
+ });
+
List<Expression> newHashConjuncts = join.getHashJoinConjuncts().stream()
.map(equalTo -> equalTo.withChildren(equalTo.children()
.stream().map(expr -> exprReplaceMap.get(expr).toSlot())
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughJoin.java
index c90723678a0..c31c05c8a78 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughJoin.java
@@ -137,6 +137,7 @@ public class PushDownFilterThroughJoin extends OneRewriteRuleFactory {
new LogicalJoin<>(join.getJoinType(),
join.getHashJoinConjuncts(),
joinConditions,
+ join.getMarkJoinConjuncts(),
join.getDistributeHint(),
join.getMarkJoinSlotReference(),
PlanUtils.filterOrSelf(leftPredicates, join.left()),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOtherCondition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOtherCondition.java
index 3716af8acb5..8fff61988d4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOtherCondition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOtherCondition.java
@@ -89,7 +89,8 @@ public class PushDownJoinOtherCondition extends OneRewriteRuleFactory {
Plan right = PlanUtils.filterOrSelf(rightConjuncts, join.right());
return new LogicalJoin<>(join.getJoinType(), join.getHashJoinConjuncts(),
- remainingOther, join.getDistributeHint(), join.getMarkJoinSlotReference(), left, right);
+ remainingOther, join.getMarkJoinConjuncts(), join.getDistributeHint(),
+ join.getMarkJoinSlotReference(), left, right);
}).toRule(RuleType.PUSH_DOWN_JOIN_OTHER_CONDITION);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/UnCorrelatedApplyAggregateFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/UnCorrelatedApplyAggregateFilter.java
index 15f1b4a554e..b0b62f2e9bf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/UnCorrelatedApplyAggregateFilter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/UnCorrelatedApplyAggregateFilter.java
@@ -89,7 +89,7 @@ public class UnCorrelatedApplyAggregateFilter extends OneRewriteRuleFactory {
ExpressionUtils.optionalAnd(correlatedPredicate),
apply.getMarkJoinSlotReference(),
apply.isNeedAddSubOutputToProjects(),
- apply.isInProject(), apply.left(), newAgg);
+ apply.isInProject(), apply.isMarkJoinSlotNotNull(), apply.left(), newAgg);
}).toRule(RuleType.UN_CORRELATED_APPLY_AGGREGATE_FILTER);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/UnCorrelatedApplyFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/UnCorrelatedApplyFilter.java
index af2c83b2b83..30b5cfcef3e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/UnCorrelatedApplyFilter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/UnCorrelatedApplyFilter.java
@@ -69,7 +69,7 @@ public class UnCorrelatedApplyFilter extends OneRewriteRuleFactory {
return new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryExpr(),
ExpressionUtils.optionalAnd(correlatedPredicate), apply.getMarkJoinSlotReference(),
apply.isNeedAddSubOutputToProjects(),
- apply.isInProject(), apply.left(), child);
+ apply.isInProject(), apply.isMarkJoinSlotNotNull(), apply.left(), child);
}).toRule(RuleType.UN_CORRELATED_APPLY_FILTER);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/UnCorrelatedApplyProjectFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/UnCorrelatedApplyProjectFilter.java
index 911bf0eef0c..82950e14dc7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/UnCorrelatedApplyProjectFilter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/UnCorrelatedApplyProjectFilter.java
@@ -90,7 +90,7 @@ public class UnCorrelatedApplyProjectFilter extends OneRewriteRuleFactory {
return new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryExpr(),
ExpressionUtils.optionalAnd(correlatedPredicate), apply.getMarkJoinSlotReference(),
apply.isNeedAddSubOutputToProjects(),
- apply.isInProject(), apply.left(), newProject);
+ apply.isInProject(), apply.isMarkJoinSlotNotNull(), apply.left(), newProject);
}).toRule(RuleType.UN_CORRELATED_APPLY_PROJECT_FILTER);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java
index 3173b654d48..29e30b30f33 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java
@@ -60,7 +60,7 @@ public class JoinEstimation {
private static boolean hashJoinConditionContainsUnknownColumnStats(Statistics leftStats,
Statistics rightStats, Join join) {
- for (Expression expr : join.getHashJoinConjuncts()) {
+ for (Expression expr : join.getEqualPredicates()) {
for (Slot slot : expr.getInputSlots()) {
ColumnStatistic colStats = leftStats.findColumnStatistics(slot);
if (colStats == null) {
@@ -87,7 +87,7 @@ public class JoinEstimation {
boolean leftBigger = leftStats.getRowCount() > rightStats.getRowCount();
double rightStatsRowCount = StatsMathUtil.nonZeroDivisor(rightStats.getRowCount());
double leftStatsRowCount = StatsMathUtil.nonZeroDivisor(leftStats.getRowCount());
- List<EqualPredicate> trustableConditions = join.getHashJoinConjuncts().stream()
+ List<EqualPredicate> trustableConditions = join.getEqualPredicates().stream()
.map(expression -> (EqualPredicate) expression)
.filter(
expression -> {
@@ -173,7 +173,7 @@ public class JoinEstimation {
private static double computeSelectivityForBuildSideWhenColStatsUnknown(Statistics buildStats, Join join) {
double sel = 1.0;
- for (Expression cond : join.getHashJoinConjuncts()) {
+ for (Expression cond : join.getEqualPredicates()) {
if (cond instanceof EqualTo) {
EqualTo equal = (EqualTo) cond;
if (equal.left() instanceof Slot && equal.right() instanceof Slot) {
@@ -204,7 +204,7 @@ public class JoinEstimation {
}
Statistics innerJoinStats;
- if (join.getHashJoinConjuncts().isEmpty()) {
+ if (join.getEqualPredicates().isEmpty()) {
innerJoinStats = estimateNestLoopJoin(leftStats, rightStats, join);
} else {
innerJoinStats = estimateHashJoin(leftStats, rightStats, join);
@@ -283,7 +283,7 @@ public class JoinEstimation {
}
}
double rowCount = Double.POSITIVE_INFINITY;
- for (Expression conjunct : join.getHashJoinConjuncts()) {
+ for (Expression conjunct : join.getEqualPredicates()) {
double eqRowCount = estimateSemiOrAntiRowCountBySlotsEqual(leftStats, rightStats,
join, (EqualPredicate) conjunct);
if (rowCount > eqRowCount) {
@@ -359,7 +359,7 @@ public class JoinEstimation {
*/
private static Statistics updateJoinResultStatsByHashJoinCondition(Statistics innerStats, Join join) {
Map<Expression, ColumnStatistic> updatedCols = new HashMap<>();
- for (Expression expr : join.getHashJoinConjuncts()) {
+ for (Expression expr : join.getEqualPredicates()) {
EqualPredicate equalTo = (EqualPredicate) expr;
ColumnStatistic leftColStats = ExpressionEstimation.estimate(equalTo.left(), innerStats);
ColumnStatistic rightColStats = ExpressionEstimation.estimate(equalTo.right(), innerStats);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
index 6e803e258ee..ee4e08ac646 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
@@ -115,7 +115,8 @@ public class LogicalPlanDeepCopier extends DefaultPlanRewriter<DeepCopierContext
Optional<MarkJoinSlotReference> markJoinSlotReference = apply.getMarkJoinSlotReference()
.map(m -> (MarkJoinSlotReference) ExpressionDeepCopier.INSTANCE.deepCopy(m, context));
return new LogicalApply<>(correlationSlot, subqueryExpr, correlationFilter,
- markJoinSlotReference, apply.isNeedAddSubOutputToProjects(), apply.isInProject(), left, right);
+ markJoinSlotReference, apply.isNeedAddSubOutputToProjects(), apply.isInProject(),
+ apply.isMarkJoinSlotNotNull(), left, right);
}
@Override
@@ -336,7 +337,10 @@ public class LogicalPlanDeepCopier extends DefaultPlanRewriter<DeepCopierContext
List<Expression> hashJoinConjuncts = join.getHashJoinConjuncts().stream()
.map(c -> ExpressionDeepCopier.INSTANCE.deepCopy(c, context))
.collect(ImmutableList.toImmutableList());
- return new LogicalJoin<>(join.getJoinType(), hashJoinConjuncts, otherJoinConjuncts,
+ List<Expression> markJoinConjuncts = join.getMarkJoinConjuncts().stream()
+ .map(c -> ExpressionDeepCopier.INSTANCE.deepCopy(c, context))
+ .collect(ImmutableList.toImmutableList());
+ return new LogicalJoin<>(join.getJoinType(), hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts,
join.getDistributeHint(), join.getMarkJoinSlotReference(), children);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Join.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Join.java
index b892214b976..39788ced173 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Join.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Join.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.trees.plans.algebra;
import org.apache.doris.nereids.hint.DistributeHint;
+import org.apache.doris.nereids.trees.expressions.EqualPredicate;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference;
@@ -28,6 +29,7 @@ import org.apache.doris.nereids.trees.plans.JoinType;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
/**
* Common interface for logical/physical join.
@@ -42,25 +44,28 @@ public interface Join {
.collect(Collectors.toList());
}
+ default List<EqualPredicate> getEqualPredicates() {
+ return Stream.concat(getHashJoinConjuncts().stream(), getMarkJoinConjuncts().stream())
+ .filter(EqualPredicate.class::isInstance).map(EqualPredicate.class::cast)
+ .collect(Collectors.toList());
+ }
+
List<Expression> getOtherJoinConjuncts();
+ List<Expression> getMarkJoinConjuncts();
+
Optional<Expression> getOnClauseCondition();
DistributeHint getDistributeHint();
boolean isMarkJoin();
+ Optional<MarkJoinSlotReference> getMarkJoinSlotReference();
+
default boolean hasDistributeHint() {
return getDistributeHint().distributeType != DistributeType.NONE;
}
- /**
- * The join plan has join condition or not.
- */
- default boolean hasJoinCondition() {
- return !getHashJoinConjuncts().isEmpty() || !getOtherJoinConjuncts().isEmpty();
- }
-
default JoinDistributeType getLeftHint() {
return JoinDistributeType.NONE;
}
@@ -78,8 +83,4 @@ public interface Join {
return JoinDistributeType.NONE;
}
}
-
- default Optional<MarkJoinSlotReference> getLeftMarkJoinSlotReference() {
- return Optional.empty();
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalApply.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalApply.java
index 04656f0bd1e..517048c209a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalApply.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalApply.java
@@ -62,6 +62,14 @@ public class LogicalApply<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
// Whether adding the subquery's output to projects
private final boolean needAddSubOutputToProjects;
+ /*
+ * This flag is indicate the mark join slot can be non-null or not
+ * in InApplyToJoin rule, if it's semi join with non-null mark slot
+ * we can safely change the mark conjunct to hash conjunct
+ * see SubqueryToApply rule for more info
+ */
+ private final boolean isMarkJoinSlotNotNull;
+
private LogicalApply(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties,
List<Expression> correlationSlot,
@@ -69,6 +77,7 @@ public class LogicalApply<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
Optional<MarkJoinSlotReference> markJoinSlotReference,
boolean needAddSubOutputToProjects,
boolean inProject,
+ boolean isMarkJoinSlotNotNull,
LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) {
super(PlanType.LOGICAL_APPLY, groupExpression, logicalProperties, leftChild, rightChild);
this.correlationSlot = correlationSlot == null ? ImmutableList.of() : ImmutableList.copyOf(correlationSlot);
@@ -77,14 +86,15 @@ public class LogicalApply<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
this.markJoinSlotReference = markJoinSlotReference;
this.needAddSubOutputToProjects = needAddSubOutputToProjects;
this.inProject = inProject;
+ this.isMarkJoinSlotNotNull = isMarkJoinSlotNotNull;
}
public LogicalApply(List<Expression> correlationSlot, SubqueryExpr subqueryExpr,
Optional<Expression> correlationFilter, Optional<MarkJoinSlotReference> markJoinSlotReference,
- boolean needAddSubOutputToProjects, boolean inProject,
+ boolean needAddSubOutputToProjects, boolean inProject, boolean isMarkJoinSlotNotNull,
LEFT_CHILD_TYPE input, RIGHT_CHILD_TYPE subquery) {
this(Optional.empty(), Optional.empty(), correlationSlot, subqueryExpr, correlationFilter,
- markJoinSlotReference, needAddSubOutputToProjects, inProject, input,
+ markJoinSlotReference, needAddSubOutputToProjects, inProject, isMarkJoinSlotNotNull, input,
subquery);
}
@@ -136,6 +146,10 @@ public class LogicalApply<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
return inProject;
}
+ public boolean isMarkJoinSlotNotNull() {
+ return isMarkJoinSlotNotNull;
+ }
+
@Override
public List<Slot> computeOutput() {
return ImmutableList.<Slot>builder()
@@ -153,6 +167,7 @@ public class LogicalApply<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
"correlationSlot", correlationSlot,
"correlationFilter", correlationFilter,
"isMarkJoin", markJoinSlotReference.isPresent(),
+ "isMarkJoinSlotNotNull", isMarkJoinSlotNotNull,
"MarkJoinSlotReference", markJoinSlotReference.isPresent() ? markJoinSlotReference.get() : "empty");
}
@@ -170,14 +185,15 @@ public class LogicalApply<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
&& Objects.equals(correlationFilter, that.getCorrelationFilter())
&& Objects.equals(markJoinSlotReference, that.getMarkJoinSlotReference())
&& needAddSubOutputToProjects == that.needAddSubOutputToProjects
- && inProject == that.inProject;
+ && inProject == that.inProject
+ && isMarkJoinSlotNotNull == that.isMarkJoinSlotNotNull;
}
@Override
public int hashCode() {
return Objects.hash(
correlationSlot, subqueryExpr, correlationFilter,
- markJoinSlotReference, needAddSubOutputToProjects, inProject);
+ markJoinSlotReference, needAddSubOutputToProjects, inProject, isMarkJoinSlotNotNull);
}
@Override
@@ -201,14 +217,15 @@ public class LogicalApply<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
public LogicalApply<Plan, Plan> withSubqueryExprAndChildren(SubqueryExpr subqueryExpr, List<Plan> children) {
return new LogicalApply<>(correlationSlot, subqueryExpr, correlationFilter,
- markJoinSlotReference, needAddSubOutputToProjects, inProject, children.get(0), children.get(1));
+ markJoinSlotReference, needAddSubOutputToProjects, inProject, isMarkJoinSlotNotNull,
+ children.get(0), children.get(1));
}
@Override
public LogicalApply<Plan, Plan> withChildren(List<Plan> children) {
Preconditions.checkArgument(children.size() == 2);
return new LogicalApply<>(correlationSlot, subqueryExpr, correlationFilter,
- markJoinSlotReference, needAddSubOutputToProjects, inProject,
+ markJoinSlotReference, needAddSubOutputToProjects, inProject, isMarkJoinSlotNotNull,
children.get(0), children.get(1));
}
@@ -216,7 +233,7 @@ public class LogicalApply<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new LogicalApply<>(groupExpression, Optional.of(getLogicalProperties()),
correlationSlot, subqueryExpr, correlationFilter, markJoinSlotReference,
- needAddSubOutputToProjects, inProject, left(), right());
+ needAddSubOutputToProjects, inProject, isMarkJoinSlotNotNull, left(), right());
}
@Override
@@ -225,6 +242,6 @@ public class LogicalApply<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
Preconditions.checkArgument(children.size() == 2);
return new LogicalApply<>(groupExpression, logicalProperties, correlationSlot, subqueryExpr,
correlationFilter, markJoinSlotReference,
- needAddSubOutputToProjects, inProject, children.get(0), children.get(1));
+ needAddSubOutputToProjects, inProject, isMarkJoinSlotNotNull, children.get(0), children.get(1));
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java
index ae8db7c97d4..d6d183408a7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java
@@ -67,6 +67,7 @@ public class LogicalJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
private final JoinType joinType;
private final List<Expression> otherJoinConjuncts;
private final List<Expression> hashJoinConjuncts;
+ private final List<Expression> markJoinConjuncts;
// When the predicate condition contains subqueries and disjunctions, the join will be marked as MarkJoin.
private final Optional<MarkJoinSlotReference> markJoinSlotReference;
@@ -80,114 +81,92 @@ public class LogicalJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
public LogicalJoin(JoinType joinType, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) {
this(joinType, ExpressionUtils.EMPTY_CONDITION, ExpressionUtils.EMPTY_CONDITION,
- new DistributeHint(DistributeType.NONE),
- Optional.empty(), Optional.empty(), Optional.empty(), leftChild, rightChild);
+ ExpressionUtils.EMPTY_CONDITION, new DistributeHint(DistributeType.NONE),
+ Optional.empty(), Optional.empty(), Optional.empty(),
+ ImmutableList.of(leftChild, rightChild), null);
}
- public LogicalJoin(JoinType joinType, List<Expression> hashJoinConjuncts, LEFT_CHILD_TYPE leftChild,
- RIGHT_CHILD_TYPE rightChild) {
+ public LogicalJoin(JoinType joinType, List<Expression> hashJoinConjuncts,
+ LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) {
this(joinType, hashJoinConjuncts, ExpressionUtils.EMPTY_CONDITION,
- new DistributeHint(DistributeType.NONE), Optional.empty(),
- Optional.empty(), Optional.empty(), leftChild, rightChild);
+ ExpressionUtils.EMPTY_CONDITION, new DistributeHint(DistributeType.NONE),
+ Optional.empty(), Optional.empty(), Optional.empty(),
+ ImmutableList.of(leftChild, rightChild), null);
}
public LogicalJoin(JoinType joinType, List<Expression> hashJoinConjuncts, List<Expression> otherJoinConjuncts,
LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) {
- this(joinType, hashJoinConjuncts, otherJoinConjuncts,
+ this(joinType, hashJoinConjuncts, otherJoinConjuncts, ExpressionUtils.EMPTY_CONDITION,
new DistributeHint(DistributeType.NONE), Optional.empty(),
- Optional.empty(), Optional.empty(), leftChild, rightChild);
+ Optional.empty(), Optional.empty(), ImmutableList.of(leftChild, rightChild), null);
}
- public LogicalJoin(JoinType joinType, List<Expression> hashJoinConjuncts, List<Expression> otherJoinConjuncts,
- DistributeHint hint, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) {
- this(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, Optional.empty(), Optional.empty(),
- Optional.empty(), leftChild, rightChild);
+ public LogicalJoin(JoinType joinType, List<Expression> hashJoinConjuncts,
+ List<Expression> otherJoinConjuncts, DistributeHint hint, LEFT_CHILD_TYPE leftChild,
+ RIGHT_CHILD_TYPE rightChild) {
+ this(joinType, hashJoinConjuncts, otherJoinConjuncts, ExpressionUtils.EMPTY_CONDITION, hint,
+ Optional.empty(), Optional.empty(), Optional.empty(),
+ ImmutableList.of(leftChild, rightChild), null);
}
- public LogicalJoin(
- JoinType joinType,
- List<Expression> hashJoinConjuncts,
- List<Expression> otherJoinConjuncts,
- DistributeHint hint,
- Optional<MarkJoinSlotReference> markJoinSlotReference,
- LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) {
- this(joinType, hashJoinConjuncts,
- otherJoinConjuncts, hint, markJoinSlotReference,
- Optional.empty(), Optional.empty(), leftChild, rightChild);
- }
-
- public LogicalJoin(
- long bitmap,
- JoinType joinType,
- List<Expression> hashJoinConjuncts,
- List<Expression> otherJoinConjuncts,
- DistributeHint hint,
- Optional<MarkJoinSlotReference> markJoinSlotReference,
- LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) {
- this(joinType, hashJoinConjuncts,
- otherJoinConjuncts, hint, markJoinSlotReference,
- Optional.empty(), Optional.empty(), leftChild, rightChild);
- this.bitmap = LongBitmap.or(this.bitmap, bitmap);
+ public LogicalJoin(JoinType joinType, List<Expression> hashJoinConjuncts,
+ List<Expression> otherJoinConjuncts, DistributeHint hint,
+ Optional<MarkJoinSlotReference> markJoinSlotReference, LEFT_CHILD_TYPE leftChild,
+ RIGHT_CHILD_TYPE rightChild) {
+ this(joinType, hashJoinConjuncts, otherJoinConjuncts, ExpressionUtils.EMPTY_CONDITION, hint,
+ markJoinSlotReference, Optional.empty(), Optional.empty(),
+ ImmutableList.of(leftChild, rightChild), null);
}
- public LogicalJoin(
- JoinType joinType,
- List<Expression> hashJoinConjuncts,
- List<Expression> otherJoinConjuncts,
- DistributeHint hint,
- Optional<MarkJoinSlotReference> markJoinSlotReference,
- List<Plan> children) {
- this(joinType, hashJoinConjuncts,
- otherJoinConjuncts, hint, markJoinSlotReference,
- Optional.empty(), Optional.empty(), children);
+ public LogicalJoin(JoinType joinType, List<Expression> hashJoinConjuncts,
+ List<Expression> otherJoinConjuncts, List<Expression> markJoinConjuncts, DistributeHint hint,
+ Optional<MarkJoinSlotReference> markJoinSlotReference, LEFT_CHILD_TYPE leftChild,
+ RIGHT_CHILD_TYPE rightChild) {
+ this(joinType, hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts, hint,
+ markJoinSlotReference, Optional.empty(), Optional.empty(),
+ ImmutableList.of(leftChild, rightChild), null);
}
- private LogicalJoin(JoinType joinType, List<Expression> hashJoinConjuncts, List<Expression> otherJoinConjuncts,
- DistributeHint hint, Optional<MarkJoinSlotReference> markJoinSlotReference,
- Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
- List<Plan> children, JoinReorderContext joinReorderContext) {
- // Just use in withXXX method. Don't need check/copyOf()
- super(PlanType.LOGICAL_JOIN, groupExpression, logicalProperties, children);
- this.joinType = Objects.requireNonNull(joinType, "joinType can not be null");
- this.hashJoinConjuncts = hashJoinConjuncts;
- this.otherJoinConjuncts = otherJoinConjuncts;
- this.hint = Objects.requireNonNull(hint, "hint can not be null");
- this.joinReorderContext.copyFrom(joinReorderContext);
- this.markJoinSlotReference = markJoinSlotReference;
+ public LogicalJoin(long bitmap, JoinType joinType, List<Expression> hashJoinConjuncts,
+ List<Expression> otherJoinConjuncts, DistributeHint hint,
+ Optional<MarkJoinSlotReference> markJoinSlotReference, LEFT_CHILD_TYPE leftChild,
+ RIGHT_CHILD_TYPE rightChild) {
+ this(joinType, hashJoinConjuncts, otherJoinConjuncts, ExpressionUtils.EMPTY_CONDITION, hint,
+ markJoinSlotReference, Optional.empty(), Optional.empty(),
+ ImmutableList.of(leftChild, rightChild), null);
+ this.bitmap = LongBitmap.or(this.bitmap, bitmap);
}
- private LogicalJoin(
- JoinType joinType,
- List<Expression> hashJoinConjuncts,
- List<Expression> otherJoinConjuncts,
- DistributeHint hint,
- Optional<MarkJoinSlotReference> markJoinSlotReference,
- Optional<GroupExpression> groupExpression,
- Optional<LogicalProperties> logicalProperties,
- LEFT_CHILD_TYPE leftChild,
- RIGHT_CHILD_TYPE rightChild) {
- super(PlanType.LOGICAL_JOIN, groupExpression, logicalProperties, leftChild, rightChild);
- this.joinType = Objects.requireNonNull(joinType, "joinType can not be null");
- this.hashJoinConjuncts = ImmutableList.copyOf(hashJoinConjuncts);
- this.otherJoinConjuncts = ImmutableList.copyOf(otherJoinConjuncts);
- this.hint = Objects.requireNonNull(hint, "hint can not be null");
- this.markJoinSlotReference = markJoinSlotReference;
+ public LogicalJoin(JoinType joinType, List<Expression> hashJoinConjuncts,
+ List<Expression> otherJoinConjuncts, DistributeHint hint,
+ Optional<MarkJoinSlotReference> markJoinSlotReference, List<Plan> children) {
+ this(joinType, hashJoinConjuncts, otherJoinConjuncts, ExpressionUtils.EMPTY_CONDITION, hint,
+ markJoinSlotReference, Optional.empty(), Optional.empty(), children, null);
+ }
+
+ public LogicalJoin(JoinType joinType, List<Expression> hashJoinConjuncts,
+ List<Expression> otherJoinConjuncts, List<Expression> markJoinConjuncts, DistributeHint hint,
+ Optional<MarkJoinSlotReference> markJoinSlotReference, List<Plan> children) {
+ this(joinType, hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts, hint,
+ markJoinSlotReference, Optional.empty(), Optional.empty(), children, null);
}
- private LogicalJoin(
- JoinType joinType,
- List<Expression> hashJoinConjuncts,
- List<Expression> otherJoinConjuncts,
- DistributeHint hint,
- Optional<MarkJoinSlotReference> markJoinSlotReference,
+ private LogicalJoin(JoinType joinType, List<Expression> hashJoinConjuncts,
+ List<Expression> otherJoinConjuncts, List<Expression> markJoinConjuncts,
+ DistributeHint hint, Optional<MarkJoinSlotReference> markJoinSlotReference,
Optional<GroupExpression> groupExpression,
- Optional<LogicalProperties> logicalProperties,
- List<Plan> children) {
+ Optional<LogicalProperties> logicalProperties, List<Plan> children,
+ JoinReorderContext joinReorderContext) {
+ // Just use in withXXX method. Don't need check/copyOf()
super(PlanType.LOGICAL_JOIN, groupExpression, logicalProperties, children);
this.joinType = Objects.requireNonNull(joinType, "joinType can not be null");
this.hashJoinConjuncts = ImmutableList.copyOf(hashJoinConjuncts);
this.otherJoinConjuncts = ImmutableList.copyOf(otherJoinConjuncts);
+ this.markJoinConjuncts = ImmutableList.copyOf(markJoinConjuncts);
this.hint = Objects.requireNonNull(hint, "hint can not be null");
+ if (joinReorderContext != null) {
+ this.joinReorderContext.copyFrom(joinReorderContext);
+ }
this.markJoinSlotReference = markJoinSlotReference;
}
@@ -204,26 +183,56 @@ public class LogicalJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
return hashJoinConjuncts;
}
+ /**
+ * getConditionSlot
+ */
public Set<Slot> getConditionSlot() {
+ // this function is called by rules which reject mark join
+ // so markJoinConjuncts is not processed here
+ Preconditions.checkState(!isMarkJoin(),
+ "shouldn't call mark join's getConditionSlot method");
return Stream.concat(hashJoinConjuncts.stream(), otherJoinConjuncts.stream())
- .flatMap(expr -> expr.getInputSlots().stream()).collect(ImmutableSet.toImmutableSet());
+ .flatMap(expr -> expr.getInputSlots().stream())
+ .collect(ImmutableSet.toImmutableSet());
}
+ /**
+ * getConditionExprId
+ */
public Set<ExprId> getConditionExprId() {
+ // this function is called by rules which reject mark join
+ // so markJoinConjuncts is not processed here
+ Preconditions.checkState(!isMarkJoin(),
+ "shouldn't call mark join's getConditionExprId method");
return Stream.concat(getHashJoinConjuncts().stream(), getOtherJoinConjuncts().stream())
.flatMap(expr -> expr.getInputSlotExprIds().stream()).collect(Collectors.toSet());
}
+ /**
+ * getLeftConditionSlot
+ */
public Set<Slot> getLeftConditionSlot() {
+ // TODO this function is used by TransposeSemiJoinAgg, we assume it can handle mark join correctly.
Set<Slot> leftOutputSet = this.left().getOutputSet();
- return Stream.concat(hashJoinConjuncts.stream(), otherJoinConjuncts.stream())
- .flatMap(expr -> expr.getInputSlots().stream())
- .filter(leftOutputSet::contains)
+ return Stream
+ .concat(Stream.concat(hashJoinConjuncts.stream(), otherJoinConjuncts.stream()),
+ markJoinConjuncts.stream())
+ .flatMap(expr -> expr.getInputSlots().stream()).filter(leftOutputSet::contains)
.collect(ImmutableSet.toImmutableSet());
}
+ /**
+ * getOnClauseCondition
+ */
public Optional<Expression> getOnClauseCondition() {
- return ExpressionUtils.optionalAnd(hashJoinConjuncts, otherJoinConjuncts);
+ // TODO this function is called by AggScalarSubQueryToWindowFunction and InferPredicates
+ // we assume they can handle mark join correctly
+ Optional<Expression> normalJoinConjuncts =
+ ExpressionUtils.optionalAnd(hashJoinConjuncts, otherJoinConjuncts);
+ return normalJoinConjuncts.isPresent()
+ ? ExpressionUtils.optionalAnd(ImmutableList.of(normalJoinConjuncts.get()),
+ markJoinConjuncts)
+ : ExpressionUtils.optionalAnd(markJoinConjuncts);
}
public JoinType getJoinType() {
@@ -242,6 +251,10 @@ public class LogicalJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
return markJoinSlotReference.isPresent();
}
+ public List<Expression> getMarkJoinConjuncts() {
+ return markJoinConjuncts;
+ }
+
public JoinReorderContext getJoinReorderContext() {
return joinReorderContext;
}
@@ -261,7 +274,8 @@ public class LogicalJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
"type", joinType,
"markJoinSlotReference", markJoinSlotReference,
"hashJoinConjuncts", hashJoinConjuncts,
- "otherJoinConjuncts", otherJoinConjuncts);
+ "otherJoinConjuncts", otherJoinConjuncts,
+ "markJoinConjuncts", markJoinConjuncts);
if (hint.distributeType != DistributeType.NONE) {
args.add("hint");
args.add(hint.getExplainString());
@@ -282,12 +296,13 @@ public class LogicalJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
&& hint.equals(that.hint)
&& hashJoinConjuncts.equals(that.hashJoinConjuncts)
&& otherJoinConjuncts.equals(that.otherJoinConjuncts)
+ && markJoinConjuncts.equals(that.markJoinConjuncts)
&& Objects.equals(markJoinSlotReference, that.markJoinSlotReference);
}
@Override
public int hashCode() {
- return Objects.hash(joinType, hashJoinConjuncts, otherJoinConjuncts, markJoinSlotReference);
+ return Objects.hash(joinType, hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts, markJoinSlotReference);
}
@Override
@@ -300,6 +315,7 @@ public class LogicalJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
return new ImmutableList.Builder<Expression>()
.addAll(hashJoinConjuncts)
.addAll(otherJoinConjuncts)
+ .addAll(markJoinConjuncts)
.build();
}
@@ -328,67 +344,86 @@ public class LogicalJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
@Override
public LogicalJoin<Plan, Plan> withChildren(List<Plan> children) {
Preconditions.checkArgument(children.size() == 2);
- return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, markJoinSlotReference,
- Optional.empty(), Optional.empty(), children, joinReorderContext);
+ return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts,
+ hint, markJoinSlotReference, Optional.empty(), Optional.empty(), children,
+ joinReorderContext);
}
@Override
public LogicalJoin<Plan, Plan> withGroupExpression(Optional<GroupExpression> groupExpression) {
- return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, markJoinSlotReference,
- groupExpression, Optional.of(getLogicalProperties()), children, joinReorderContext);
+ return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts,
+ hint, markJoinSlotReference, groupExpression, Optional.of(getLogicalProperties()),
+ children, joinReorderContext);
}
@Override
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
Preconditions.checkArgument(children.size() == 2);
- return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, markJoinSlotReference,
- groupExpression, logicalProperties, children, joinReorderContext);
+ return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts,
+ hint, markJoinSlotReference, groupExpression, logicalProperties, children,
+ joinReorderContext);
}
public LogicalJoin<Plan, Plan> withChildrenNoContext(Plan left, Plan right) {
- return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint,
- markJoinSlotReference, left, right);
+ return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts,
+ hint, markJoinSlotReference, Optional.empty(), Optional.empty(),
+ ImmutableList.of(left, right), null);
}
- public LogicalJoin<Plan, Plan> withJoinConjuncts(
- List<Expression> hashJoinConjuncts, List<Expression> otherJoinConjuncts) {
- return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts,
- hint, markJoinSlotReference, children);
+ public LogicalJoin<Plan, Plan> withJoinConjuncts(List<Expression> hashJoinConjuncts,
+ List<Expression> otherJoinConjuncts) {
+ return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts,
+ hint, markJoinSlotReference, Optional.empty(), Optional.empty(), children, null);
+ }
+
+ public LogicalJoin<Plan, Plan> withJoinConjuncts(List<Expression> hashJoinConjuncts,
+ List<Expression> otherJoinConjuncts,
+ List<Expression> markJoinConjuncts) {
+ return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts,
+ hint, markJoinSlotReference, Optional.empty(), Optional.empty(), children, null);
}
public LogicalJoin<Plan, Plan> withHashJoinConjunctsAndChildren(
List<Expression> hashJoinConjuncts, Plan left, Plan right) {
Preconditions.checkArgument(children.size() == 2);
- return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint,
- markJoinSlotReference, left, right);
+ return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts,
+ hint, markJoinSlotReference, Optional.empty(), Optional.empty(),
+ ImmutableList.of(left, right), null);
}
public LogicalJoin<Plan, Plan> withConjunctsChildren(List<Expression> hashJoinConjuncts,
List<Expression> otherJoinConjuncts, Plan left, Plan right) {
- return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, markJoinSlotReference, left,
- right);
+ return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts,
+ hint, markJoinSlotReference, Optional.empty(), Optional.empty(),
+ ImmutableList.of(left, right), null);
}
- public LogicalJoin<Plan, Plan> withJoinType(JoinType joinType) {
- return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint,
- markJoinSlotReference, children);
+ public LogicalJoin<Plan, Plan> withConjunctsChildren(List<Expression> hashJoinConjuncts,
+ List<Expression> otherJoinConjuncts,
+ List<Expression> markJoinConjuncts, Plan left, Plan right) {
+ return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts,
+ hint, markJoinSlotReference, Optional.empty(), Optional.empty(),
+ ImmutableList.of(left, right), null);
}
- public LogicalJoin<Plan, Plan> withTypeChildren(JoinType joinType, Plan left, Plan right) {
- return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint,
- markJoinSlotReference, left, right);
+ public LogicalJoin<Plan, Plan> withJoinType(JoinType joinType) {
+ return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts,
+ hint, markJoinSlotReference, Optional.empty(), Optional.empty(), children, null);
}
- public LogicalJoin<Plan, Plan> withOtherJoinConjuncts(List<Expression> otherJoinConjuncts) {
- return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint,
- markJoinSlotReference, children);
+ public LogicalJoin<Plan, Plan> withTypeChildren(JoinType joinType, Plan left, Plan right) {
+ return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts,
+ hint, markJoinSlotReference, Optional.empty(), Optional.empty(),
+ ImmutableList.of(left, right), null);
}
/**
* extractNullRejectHashKeys
*/
public @Nullable Pair<Set<Slot>, Set<Slot>> extractNullRejectHashKeys() {
+ // this function is only used by computeFuncDeps, and function dependence calculation is disabled for mark join
+ // so markJoinConjuncts is not processed now
Set<Slot> leftKeys = new HashSet<>();
Set<Slot> rightKeys = new HashSet<>();
for (Expression expression : hashJoinConjuncts) {
@@ -413,6 +448,10 @@ public class LogicalJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
@Override
public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>> outputSupplier) {
+ if (isMarkJoin()) {
+ // TODO disable function dependence calculation for mark join, but need re-think this in future.
+ return FunctionalDependencies.EMPTY_FUNC_DEPS;
+ }
//1. NALAJ and FOJ block functional dependencies
if (joinType.isNullAwareLeftAntiJoin() || joinType.isFullOuterJoin()) {
return FunctionalDependencies.EMPTY_FUNC_DEPS;
@@ -478,6 +517,8 @@ public class LogicalJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
* get Equal slot from join
*/
public ImmutableEqualSet<Slot> getEqualSlots() {
+ // this function is only used by EliminateJoinByFK rule, and EliminateJoinByFK is disabled for mark join
+ // so markJoinConjuncts is not processed now
// TODO: Use fd in the future
if (!joinType.isInnerJoin() && !joinType.isSemiJoin()) {
return ImmutableEqualSet.empty();
@@ -499,6 +540,7 @@ public class LogicalJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
properties.put("JoinType", joinType.toString());
properties.put("HashJoinConjuncts", hashJoinConjuncts.toString());
properties.put("OtherJoinConjuncts", otherJoinConjuncts.toString());
+ properties.put("MarkJoinConjuncts", markJoinConjuncts.toString());
properties.put("DistributeHint", hint.toString());
properties.put("MarkJoinSlotReference", markJoinSlotReference.toString());
logicalJoin.put("Properties", properties);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/UsingJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/UsingJoin.java
index ca9fbbb2f7a..e206b79a6b3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/UsingJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/UsingJoin.java
@@ -33,7 +33,6 @@ import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableList.Builder;
-import org.apache.commons.collections.CollectionUtils;
import java.util.List;
import java.util.Optional;
@@ -48,13 +47,12 @@ public class UsingJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends Pl
private final ImmutableList<Expression> otherJoinConjuncts;
private final ImmutableList<Expression> hashJoinConjuncts;
private final DistributeHint hint;
- private final Optional<MarkJoinSlotReference> markJoinSlotReference;
public UsingJoin(JoinType joinType, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild,
List<Expression> expressions, List<Expression> hashJoinConjuncts,
DistributeHint hint) {
this(joinType, leftChild, rightChild, expressions,
- hashJoinConjuncts, Optional.empty(), Optional.empty(), hint, Optional.empty());
+ hashJoinConjuncts, Optional.empty(), Optional.empty(), hint);
}
/**
@@ -63,13 +61,12 @@ public class UsingJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends Pl
public UsingJoin(JoinType joinType, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild,
List<Expression> expressions, List<Expression> hashJoinConjuncts, Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties,
- DistributeHint hint, Optional<MarkJoinSlotReference> markJoinSlotReference) {
+ DistributeHint hint) {
super(PlanType.LOGICAL_USING_JOIN, groupExpression, logicalProperties, leftChild, rightChild);
this.joinType = joinType;
this.otherJoinConjuncts = ImmutableList.copyOf(expressions);
this.hashJoinConjuncts = ImmutableList.copyOf(hashJoinConjuncts);
this.hint = hint;
- this.markJoinSlotReference = markJoinSlotReference;
}
@Override
@@ -114,20 +111,20 @@ public class UsingJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends Pl
@Override
public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new UsingJoin(joinType, child(0), child(1), otherJoinConjuncts,
- hashJoinConjuncts, groupExpression, Optional.of(getLogicalProperties()), hint, markJoinSlotReference);
+ hashJoinConjuncts, groupExpression, Optional.of(getLogicalProperties()), hint);
}
@Override
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
return new UsingJoin(joinType, children.get(0), children.get(1), otherJoinConjuncts,
- hashJoinConjuncts, groupExpression, logicalProperties, hint, markJoinSlotReference);
+ hashJoinConjuncts, groupExpression, logicalProperties, hint);
}
@Override
public Plan withChildren(List<Plan> children) {
return new UsingJoin(joinType, children.get(0), children.get(1), otherJoinConjuncts,
- hashJoinConjuncts, groupExpression, Optional.of(getLogicalProperties()), hint, markJoinSlotReference);
+ hashJoinConjuncts, groupExpression, Optional.of(getLogicalProperties()), hint);
}
@Override
@@ -160,11 +157,15 @@ public class UsingJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends Pl
}
public boolean isMarkJoin() {
- return markJoinSlotReference.isPresent();
+ return false;
}
public Optional<MarkJoinSlotReference> getMarkJoinSlotReference() {
- return markJoinSlotReference;
+ return Optional.empty();
+ }
+
+ public List<Expression> getMarkJoinConjuncts() {
+ return ExpressionUtils.EMPTY_CONDITION;
}
@Override
@@ -176,9 +177,4 @@ public class UsingJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends Pl
public boolean hasDistributeHint() {
return hint != null;
}
-
- @Override
- public boolean hasJoinCondition() {
- return !CollectionUtils.isEmpty(hashJoinConjuncts);
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalJoin.java
index 7d1c65b5899..e2847970370 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalJoin.java
@@ -36,6 +36,7 @@ import org.apache.doris.nereids.util.JoinUtils;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.statistics.Statistics;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableList.Builder;
import com.google.common.collect.ImmutableSet;
@@ -61,6 +62,7 @@ public abstract class AbstractPhysicalJoin<
protected final JoinType joinType;
protected final List<Expression> hashJoinConjuncts;
protected final List<Expression> otherJoinConjuncts;
+ protected final List<Expression> markJoinConjuncts;
protected final DistributeHint hint;
protected final Optional<MarkJoinSlotReference> markJoinSlotReference;
protected final List<RuntimeFilter> runtimeFilters = Lists.newArrayList();
@@ -81,12 +83,9 @@ public abstract class AbstractPhysicalJoin<
Optional<MarkJoinSlotReference> markJoinSlotReference,
Optional<GroupExpression> groupExpression,
LogicalProperties logicalProperties, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) {
- super(type, groupExpression, logicalProperties, leftChild, rightChild);
- this.joinType = Objects.requireNonNull(joinType, "joinType can not be null");
- this.hashJoinConjuncts = ImmutableList.copyOf(hashJoinConjuncts);
- this.otherJoinConjuncts = ImmutableList.copyOf(otherJoinConjuncts);
- this.hint = Objects.requireNonNull(hint, "hint can not be null");
- this.markJoinSlotReference = markJoinSlotReference;
+ this(type, joinType, hashJoinConjuncts, otherJoinConjuncts, ExpressionUtils.EMPTY_CONDITION,
+ hint, markJoinSlotReference, groupExpression, logicalProperties, null, null,
+ leftChild, rightChild);
}
/**
@@ -105,10 +104,30 @@ public abstract class AbstractPhysicalJoin<
Statistics statistics,
LEFT_CHILD_TYPE leftChild,
RIGHT_CHILD_TYPE rightChild) {
+ this(type, joinType, hashJoinConjuncts, otherJoinConjuncts, ExpressionUtils.EMPTY_CONDITION,
+ hint, markJoinSlotReference, groupExpression, logicalProperties, physicalProperties,
+ statistics, leftChild, rightChild);
+ }
+
+ protected AbstractPhysicalJoin(
+ PlanType type,
+ JoinType joinType,
+ List<Expression> hashJoinConjuncts,
+ List<Expression> otherJoinConjuncts,
+ List<Expression> markJoinConjuncts,
+ DistributeHint hint,
+ Optional<MarkJoinSlotReference> markJoinSlotReference,
+ Optional<GroupExpression> groupExpression,
+ LogicalProperties logicalProperties,
+ PhysicalProperties physicalProperties,
+ Statistics statistics,
+ LEFT_CHILD_TYPE leftChild,
+ RIGHT_CHILD_TYPE rightChild) {
super(type, groupExpression, logicalProperties, physicalProperties, statistics, leftChild, rightChild);
this.joinType = Objects.requireNonNull(joinType, "joinType can not be null");
this.hashJoinConjuncts = ImmutableList.copyOf(hashJoinConjuncts);
this.otherJoinConjuncts = ImmutableList.copyOf(otherJoinConjuncts);
+ this.markJoinConjuncts = ImmutableList.copyOf(markJoinConjuncts);
this.hint = hint;
this.markJoinSlotReference = markJoinSlotReference;
}
@@ -137,11 +156,16 @@ public abstract class AbstractPhysicalJoin<
return markJoinSlotReference.isPresent();
}
+ public List<Expression> getMarkJoinConjuncts() {
+ return markJoinConjuncts;
+ }
+
@Override
public List<? extends Expression> getExpressions() {
return new Builder<Expression>()
.addAll(hashJoinConjuncts)
- .addAll(otherJoinConjuncts).build();
+ .addAll(otherJoinConjuncts)
+ .addAll(markJoinConjuncts).build();
}
// TODO:
@@ -158,13 +182,14 @@ public abstract class AbstractPhysicalJoin<
return joinType == that.joinType
&& hashJoinConjuncts.equals(that.hashJoinConjuncts)
&& otherJoinConjuncts.equals(that.otherJoinConjuncts)
+ && markJoinConjuncts.equals(that.markJoinConjuncts)
&& hint.equals(that.hint)
&& Objects.equals(markJoinSlotReference, that.markJoinSlotReference);
}
@Override
public int hashCode() {
- return Objects.hash(joinType, hashJoinConjuncts, otherJoinConjuncts, markJoinSlotReference);
+ return Objects.hash(joinType, hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts, markJoinSlotReference);
}
/**
@@ -173,7 +198,14 @@ public abstract class AbstractPhysicalJoin<
* @return the combination of hashJoinConjuncts and otherJoinConjuncts
*/
public Optional<Expression> getOnClauseCondition() {
- return ExpressionUtils.optionalAnd(hashJoinConjuncts, otherJoinConjuncts);
+ // TODO this function is called by AggScalarSubQueryToWindowFunction and InferPredicates
+ // we assume they can handle mark join correctly
+ Optional<Expression> normalJoinConjuncts =
+ ExpressionUtils.optionalAnd(hashJoinConjuncts, otherJoinConjuncts);
+ return normalJoinConjuncts.isPresent()
+ ? ExpressionUtils.optionalAnd(ImmutableList.of(normalJoinConjuncts.get()),
+ markJoinConjuncts)
+ : ExpressionUtils.optionalAnd(markJoinConjuncts);
}
@Override
@@ -200,6 +232,7 @@ public abstract class AbstractPhysicalJoin<
properties.put("JoinType", joinType.toString());
properties.put("HashJoinConjuncts", hashJoinConjuncts.toString());
properties.put("OtherJoinConjuncts", otherJoinConjuncts.toString());
+ properties.put("MarkJoinConjuncts", markJoinConjuncts.toString());
properties.put("JoinHint", hint.toString());
properties.put("MarkJoinSlotReference", markJoinSlotReference.toString());
physicalJoin.put("Properties", properties);
@@ -223,7 +256,14 @@ public abstract class AbstractPhysicalJoin<
.build();
}
+ /**
+ * getConditionSlot
+ */
public Set<Slot> getConditionSlot() {
+ // this function is called by rules which reject mark join
+ // so markJoinConjuncts is not processed here
+ Preconditions.checkState(!isMarkJoin(),
+ "shouldn't call mark join's getConditionSlot method");
return Stream.concat(hashJoinConjuncts.stream(), otherJoinConjuncts.stream())
.flatMap(expr -> expr.getInputSlots().stream()).collect(ImmutableSet.toImmutableSet());
}
@@ -233,7 +273,8 @@ public abstract class AbstractPhysicalJoin<
List<Object> args = Lists.newArrayList("type", joinType,
"stats", statistics,
"hashCondition", hashJoinConjuncts,
- "otherCondition", otherJoinConjuncts);
+ "otherCondition", otherJoinConjuncts,
+ "markCondition", markJoinConjuncts);
if (markJoinSlotReference.isPresent()) {
args.add("isMarkJoin");
args.add("true");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java
index 58d3530b690..43b82dcc20f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java
@@ -35,6 +35,7 @@ import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.MutableState;
import org.apache.doris.planner.RuntimeFilterId;
import org.apache.doris.qe.ConnectContext;
@@ -86,14 +87,30 @@ public class PhysicalHashJoin<
Optional<GroupExpression> groupExpression,
LogicalProperties logicalProperties,
LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) {
- super(PlanType.PHYSICAL_HASH_JOIN, joinType, hashJoinConjuncts, otherJoinConjuncts, hint, markJoinSlotReference,
- groupExpression, logicalProperties, leftChild, rightChild);
+ this(joinType, hashJoinConjuncts, otherJoinConjuncts, ExpressionUtils.EMPTY_CONDITION, hint,
+ markJoinSlotReference, groupExpression, logicalProperties, null, null, leftChild,
+ rightChild);
+ }
+
+ public PhysicalHashJoin(
+ JoinType joinType,
+ List<Expression> hashJoinConjuncts,
+ List<Expression> otherJoinConjuncts,
+ List<Expression> markJoinConjuncts,
+ DistributeHint hint,
+ Optional<MarkJoinSlotReference> markJoinSlotReference,
+ LogicalProperties logicalProperties,
+ LEFT_CHILD_TYPE leftChild,
+ RIGHT_CHILD_TYPE rightChild) {
+ this(joinType, hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts, hint, markJoinSlotReference,
+ Optional.empty(), logicalProperties, null, null, leftChild, rightChild);
}
private PhysicalHashJoin(
JoinType joinType,
List<Expression> hashJoinConjuncts,
List<Expression> otherJoinConjuncts,
+ List<Expression> markJoinConjuncts,
DistributeHint hint,
Optional<MarkJoinSlotReference> markJoinSlotReference,
Optional<GroupExpression> groupExpression,
@@ -102,8 +119,9 @@ public class PhysicalHashJoin<
Statistics statistics,
LEFT_CHILD_TYPE leftChild,
RIGHT_CHILD_TYPE rightChild) {
- super(PlanType.PHYSICAL_HASH_JOIN, joinType, hashJoinConjuncts, otherJoinConjuncts, hint, markJoinSlotReference,
- groupExpression, logicalProperties, physicalProperties, statistics, leftChild, rightChild);
+ super(PlanType.PHYSICAL_HASH_JOIN, joinType, hashJoinConjuncts, otherJoinConjuncts,
+ markJoinConjuncts, hint, markJoinSlotReference, groupExpression, logicalProperties,
+ physicalProperties, statistics, leftChild, rightChild);
}
/**
@@ -111,6 +129,12 @@ public class PhysicalHashJoin<
* Return pair of left used slots and right used slots.
*/
public Pair<List<ExprId>, List<ExprId>> getHashConjunctsExprIds() {
+ // TODO this function is only called by addShuffleJoinRequestProperty
+ // currently standalone mark join can only allow broadcast( we can remove this limitation after implement
+ // something like nullaware shuffle to broadcast nulls to all instances
+ // mark join with non-empty hash join conjuncts allow shuffle join by hash join conjuncts
+ Preconditions.checkState(!(isMarkJoin() && hashJoinConjuncts.isEmpty()),
+ "shouldn't call mark join's getHashConjunctsExprIds method for standalone mark join");
int size = hashJoinConjuncts.size();
List<ExprId> exprIds1 = new ArrayList<>(size);
@@ -143,7 +167,7 @@ public class PhysicalHashJoin<
public PhysicalHashJoin<Plan, Plan> withChildren(List<Plan> children) {
Preconditions.checkArgument(children.size() == 2);
PhysicalHashJoin newJoin = new PhysicalHashJoin<>(joinType, hashJoinConjuncts,
- otherJoinConjuncts, hint, markJoinSlotReference,
+ otherJoinConjuncts, markJoinConjuncts, hint, markJoinSlotReference,
Optional.empty(), getLogicalProperties(), physicalProperties, statistics,
children.get(0), children.get(1));
if (groupExpression.isPresent()) {
@@ -155,28 +179,32 @@ public class PhysicalHashJoin<
@Override
public PhysicalHashJoin<LEFT_CHILD_TYPE, RIGHT_CHILD_TYPE> withGroupExpression(
Optional<GroupExpression> groupExpression) {
- return new PhysicalHashJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, markJoinSlotReference,
- groupExpression, getLogicalProperties(), left(), right());
+ return new PhysicalHashJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts,
+ markJoinConjuncts, hint, markJoinSlotReference, groupExpression,
+ getLogicalProperties(), null, null, left(), right());
}
@Override
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
Preconditions.checkArgument(children.size() == 2);
- return new PhysicalHashJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, markJoinSlotReference,
- groupExpression, logicalProperties.get(), children.get(0), children.get(1));
+ return new PhysicalHashJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts,
+ markJoinConjuncts, hint, markJoinSlotReference, groupExpression,
+ logicalProperties.get(), null, null, children.get(0), children.get(1));
}
public PhysicalHashJoin<LEFT_CHILD_TYPE, RIGHT_CHILD_TYPE> withPhysicalPropertiesAndStats(
PhysicalProperties physicalProperties, Statistics statistics) {
- return new PhysicalHashJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, markJoinSlotReference,
- groupExpression, getLogicalProperties(), physicalProperties, statistics, left(), right());
+ return new PhysicalHashJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts,
+ markJoinConjuncts, hint, markJoinSlotReference, groupExpression,
+ getLogicalProperties(), physicalProperties, statistics, left(), right());
}
@Override
public boolean pushDownRuntimeFilter(CascadesContext context, IdGenerator<RuntimeFilterId> generator,
AbstractPhysicalJoin<?, ?> builderNode, Expression srcExpr, Expression probeExpr,
TRuntimeFilterType type, long buildSideNdv, int exprOrder) {
+ // currently, mark join doesn't support RF, so markJoinConjuncts is not processed here
if (RuntimeFilterGenerator.DENIED_JOIN_TYPES.contains(getJoinType()) || isMarkJoin()) {
if (builderNode instanceof PhysicalHashJoin) {
PhysicalHashJoin<?, ?> builderJoin = (PhysicalHashJoin<?, ?>) builderNode;
@@ -253,6 +281,10 @@ public class PhysicalHashJoin<
.sorted().collect(Collectors.joining(" and ", " hashCondition=(", ")")));
builder.append(otherJoinConjuncts.stream().map(cond -> cond.shapeInfo())
.sorted().collect(Collectors.joining(" and ", " otherCondition=(", ")")));
+ if (!markJoinConjuncts.isEmpty()) {
+ builder.append(markJoinConjuncts.stream().map(cond -> cond.shapeInfo()).sorted()
+ .collect(Collectors.joining(" and ", " markCondition=(", ")")));
+ }
if (!runtimeFilters.isEmpty()) {
builder.append(" build RFs:").append(runtimeFilters.stream()
.map(rf -> rf.shapeInfo()).collect(Collectors.joining(";")));
@@ -262,7 +294,8 @@ public class PhysicalHashJoin<
@Override
public PhysicalHashJoin<LEFT_CHILD_TYPE, RIGHT_CHILD_TYPE> resetLogicalProperties() {
- return new PhysicalHashJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, markJoinSlotReference,
- groupExpression, null, physicalProperties, statistics, left(), right());
+ return new PhysicalHashJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts,
+ markJoinConjuncts, hint, markJoinSlotReference, groupExpression, null,
+ physicalProperties, statistics, left(), right());
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalNestedLoopJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalNestedLoopJoin.java
index 1ad2f2b6161..1f4e2bb1712 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalNestedLoopJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalNestedLoopJoin.java
@@ -29,6 +29,7 @@ import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.MutableState;
import org.apache.doris.statistics.Statistics;
@@ -39,6 +40,7 @@ import com.google.common.collect.Sets;
import java.util.List;
import java.util.Optional;
import java.util.Set;
+import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
@@ -82,10 +84,8 @@ public class PhysicalNestedLoopJoin<
Optional<GroupExpression> groupExpression,
LogicalProperties logicalProperties,
LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) {
- super(PlanType.PHYSICAL_NESTED_LOOP_JOIN, joinType, hashJoinConjuncts, otherJoinConjuncts,
- // nested loop join ignores join hints.
- new DistributeHint(DistributeType.NONE), markJoinSlotReference,
- groupExpression, logicalProperties, leftChild, rightChild);
+ this(joinType, hashJoinConjuncts, otherJoinConjuncts, ExpressionUtils.EMPTY_CONDITION, markJoinSlotReference,
+ groupExpression, logicalProperties, null, null, leftChild, rightChild);
}
/**
@@ -105,7 +105,36 @@ public class PhysicalNestedLoopJoin<
Statistics statistics,
LEFT_CHILD_TYPE leftChild,
RIGHT_CHILD_TYPE rightChild) {
- super(PlanType.PHYSICAL_NESTED_LOOP_JOIN, joinType, hashJoinConjuncts, otherJoinConjuncts,
+ this(joinType, hashJoinConjuncts, otherJoinConjuncts, ExpressionUtils.EMPTY_CONDITION, markJoinSlotReference,
+ groupExpression, logicalProperties, physicalProperties, statistics, leftChild, rightChild);
+ }
+
+ public PhysicalNestedLoopJoin(
+ JoinType joinType,
+ List<Expression> hashJoinConjuncts,
+ List<Expression> otherJoinConjuncts,
+ List<Expression> markJoinConjuncts,
+ Optional<MarkJoinSlotReference> markJoinSlotReference,
+ LogicalProperties logicalProperties,
+ LEFT_CHILD_TYPE leftChild,
+ RIGHT_CHILD_TYPE rightChild) {
+ this(joinType, hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts, markJoinSlotReference,
+ Optional.empty(), logicalProperties, null, null, leftChild, rightChild);
+ }
+
+ private PhysicalNestedLoopJoin(
+ JoinType joinType,
+ List<Expression> hashJoinConjuncts,
+ List<Expression> otherJoinConjuncts,
+ List<Expression> markJoinConjuncts,
+ Optional<MarkJoinSlotReference> markJoinSlotReference,
+ Optional<GroupExpression> groupExpression,
+ LogicalProperties logicalProperties,
+ PhysicalProperties physicalProperties,
+ Statistics statistics,
+ LEFT_CHILD_TYPE leftChild,
+ RIGHT_CHILD_TYPE rightChild) {
+ super(PlanType.PHYSICAL_NESTED_LOOP_JOIN, joinType, hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts,
// nested loop join ignores join hints.
new DistributeHint(DistributeType.NONE), markJoinSlotReference,
groupExpression, logicalProperties, physicalProperties, statistics, leftChild, rightChild);
@@ -132,7 +161,7 @@ public class PhysicalNestedLoopJoin<
public PhysicalNestedLoopJoin<Plan, Plan> withChildren(List<Plan> children) {
Preconditions.checkArgument(children.size() == 2);
PhysicalNestedLoopJoin newJoin = new PhysicalNestedLoopJoin<>(joinType,
- hashJoinConjuncts, otherJoinConjuncts, markJoinSlotReference, Optional.empty(),
+ hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts, markJoinSlotReference, Optional.empty(),
getLogicalProperties(), physicalProperties, statistics, children.get(0), children.get(1));
if (groupExpression.isPresent()) {
newJoin.setMutableState(MutableState.KEY_GROUP, groupExpression.get().getOwnerGroup().getGroupId().asInt());
@@ -144,8 +173,8 @@ public class PhysicalNestedLoopJoin<
public PhysicalNestedLoopJoin<LEFT_CHILD_TYPE, RIGHT_CHILD_TYPE> withGroupExpression(
Optional<GroupExpression> groupExpression) {
return new PhysicalNestedLoopJoin<>(joinType,
- hashJoinConjuncts, otherJoinConjuncts, markJoinSlotReference,
- groupExpression, getLogicalProperties(), left(), right());
+ hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts, markJoinSlotReference,
+ groupExpression, getLogicalProperties(), null, null, left(), right());
}
@Override
@@ -153,15 +182,15 @@ public class PhysicalNestedLoopJoin<
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
Preconditions.checkArgument(children.size() == 2);
return new PhysicalNestedLoopJoin<>(joinType,
- hashJoinConjuncts, otherJoinConjuncts, markJoinSlotReference, groupExpression,
- logicalProperties.get(), children.get(0), children.get(1));
+ hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts, markJoinSlotReference, groupExpression,
+ logicalProperties.get(), null, null, children.get(0), children.get(1));
}
@Override
public PhysicalNestedLoopJoin<LEFT_CHILD_TYPE, RIGHT_CHILD_TYPE> withPhysicalPropertiesAndStats(
PhysicalProperties physicalProperties, Statistics statistics) {
return new PhysicalNestedLoopJoin<>(joinType,
- hashJoinConjuncts, otherJoinConjuncts, markJoinSlotReference, groupExpression,
+ hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts, markJoinSlotReference, groupExpression,
getLogicalProperties(), physicalProperties, statistics, left(), right());
}
@@ -177,23 +206,39 @@ public class PhysicalNestedLoopJoin<
return bitMapRuntimeFilterConditions.isEmpty();
}
+ /**
+ * getConditionSlot
+ */
public Set<Slot> getConditionSlot() {
+ // this function is called by rules which reject mark join
+ // so markJoinConjuncts is not processed here
+ Preconditions.checkState(!isMarkJoin(),
+ "shouldn't call mark join's getConditionSlot method");
return Stream.concat(hashJoinConjuncts.stream(), otherJoinConjuncts.stream())
- .flatMap(expr -> expr.getInputSlots().stream()).collect(ImmutableSet.toImmutableSet());
+ .flatMap(expr -> expr.getInputSlots().stream())
+ .collect(ImmutableSet.toImmutableSet());
}
@Override
public String shapeInfo() {
StringBuilder builder = new StringBuilder("NestedLoopJoin");
builder.append("[").append(joinType).append("]");
- otherJoinConjuncts.forEach(expr -> builder.append(expr.shapeInfo()));
+ if (!markJoinConjuncts.isEmpty()) {
+ builder.append(otherJoinConjuncts.stream().map(cond -> cond.shapeInfo()).sorted()
+ .collect(Collectors.joining(" and ", " otherCondition=(", ")")));
+ builder.append(markJoinConjuncts.stream().map(cond -> cond.shapeInfo()).sorted()
+ .collect(Collectors.joining(" and ", " markCondition=(", ")")));
+ } else {
+ otherJoinConjuncts.forEach(expr -> builder.append(expr.shapeInfo()));
+ }
+
return builder.toString();
}
@Override
public PhysicalNestedLoopJoin<LEFT_CHILD_TYPE, RIGHT_CHILD_TYPE> resetLogicalProperties() {
return new PhysicalNestedLoopJoin<>(joinType,
- hashJoinConjuncts, otherJoinConjuncts, markJoinSlotReference, groupExpression,
+ hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts, markJoinSlotReference, groupExpression,
null, physicalProperties, statistics, left(), right());
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
index 63840e8ded3..3134ac6bab6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
@@ -34,6 +34,7 @@ import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.InPredicate;
import org.apache.doris.nereids.trees.expressions.IsNull;
+import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.Or;
@@ -51,6 +52,7 @@ import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewri
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.visitor.ExpressionLineageReplacer;
+import org.apache.doris.nereids.types.BooleanType;
import org.apache.doris.nereids.types.coercion.NumericType;
import com.google.common.base.Preconditions;
@@ -59,6 +61,7 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableList.Builder;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import java.util.Arrays;
@@ -469,6 +472,75 @@ public class ExpressionUtils {
return children.stream().allMatch(c -> c instanceof NullLiteral);
}
+ /**
+ * canInferNotNullForMarkSlot
+ */
+ public static boolean canInferNotNullForMarkSlot(Expression predicate) {
+ /*
+ * assume predicate is from LogicalFilter
+ * the idea is replacing each mark join slot with null and false literal then run FoldConstant rule
+ * if the evaluate result are:
+ * 1. all true
+ * 2. all null and false (in logicalFilter, we discard both null and false values)
+ * the mark slot can be non-nullable boolean
+ * and in semi join, we can safely change the mark conjunct to hash conjunct
+ */
+ ImmutableList<Literal> literals =
+ ImmutableList.of(new NullLiteral(BooleanType.INSTANCE), BooleanLiteral.FALSE);
+ List<MarkJoinSlotReference> markJoinSlotReferenceList =
+ ((Set<MarkJoinSlotReference>) predicate
+ .collect(MarkJoinSlotReference.class::isInstance)).stream()
+ .collect(Collectors.toList());
+ int markSlotSize = markJoinSlotReferenceList.size();
+ int maxMarkSlotCount = 4;
+ // if the conjunct has mark slot, and maximum 4 mark slots(for performance)
+ if (markSlotSize > 0 && markSlotSize <= maxMarkSlotCount) {
+ Map<Expression, Expression> replaceMap = Maps.newHashMap();
+ boolean meetTrue = false;
+ boolean meetNullOrFalse = false;
+ /*
+ * markSlotSize = 1 -> loopCount = 2 ---- 0, 1
+ * markSlotSize = 2 -> loopCount = 4 ---- 00, 01, 10, 11
+ * markSlotSize = 3 -> loopCount = 8 ---- 000, 001, 010, 011, 100, 101, 110, 111
+ * markSlotSize = 4 -> loopCount = 16 ---- 0000, 0001, ... 1111
+ */
+ int loopCount = 2 << markSlotSize;
+ for (int i = 0; i < loopCount; ++i) {
+ replaceMap.clear();
+ /*
+ * replace each mark slot with null or false
+ * literals.get(0) -> NullLiteral(BooleanType.INSTANCE)
+ * literals.get(1) -> BooleanLiteral.FALSE
+ */
+ for (int j = 0; j < markSlotSize; ++j) {
+ replaceMap.put(markJoinSlotReferenceList.get(j), literals.get((i >> j) & 1));
+ }
+ Expression evalResult = FoldConstantRule.INSTANCE.rewrite(
+ ExpressionUtils.replace(predicate, replaceMap),
+ new ExpressionRewriteContext(null));
+
+ if (evalResult.equals(BooleanLiteral.TRUE)) {
+ if (meetNullOrFalse) {
+ return false;
+ } else {
+ meetTrue = true;
+ }
+ } else if ((isNullOrFalse(evalResult))) {
+ if (meetTrue) {
+ return false;
+ } else {
+ meetNullOrFalse = true;
+ }
+ }
+ }
+ }
+ return true;
+ }
+
+ private static boolean isNullOrFalse(Expression expression) {
+ return expression.isNullLiteral() || expression.equals(BooleanLiteral.FALSE);
+ }
+
/**
* infer notNulls slot from predicate
*/
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java
index 9beaa29c433..27125504791 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java
@@ -58,10 +58,17 @@ import java.util.stream.Collectors;
* Utils for join
*/
public class JoinUtils {
+ /**
+ * couldShuffle
+ */
public static boolean couldShuffle(Join join) {
// Cross-join and Null-Aware-Left-Anti-Join only can be broadcast join.
- // Because mark join would consider null value from both build and probe side, so must use broadcast join too.
- return !(join.getJoinType().isCrossJoin() || join.getJoinType().isNullAwareLeftAntiJoin() || join.isMarkJoin());
+ // standalone mark join would consider null value from both build and probe side, so must use broadcast join.
+ // mark join with hash conjuncts can shuffle by hash conjuncts
+ // TODO actually standalone mark join can use shuffle, but need do nullaware shuffle to broadcast null value
+ // to all instances
+ return !(join.getJoinType().isCrossJoin() || join.getJoinType().isNullAwareLeftAntiJoin()
+ || (!join.getMarkJoinConjuncts().isEmpty() && join.getHashJoinConjuncts().isEmpty()));
}
public static boolean couldBroadcast(Join join) {
@@ -173,10 +180,14 @@ public class JoinUtils {
}
public static boolean shouldNestedLoopJoin(Join join) {
- return join.getHashJoinConjuncts().isEmpty();
+ // currently, mark join conjuncts only has one conjunct, so we always get the first element here
+ return join.getHashJoinConjuncts().isEmpty() && (join.getMarkJoinConjuncts().isEmpty()
+ || !(join.getMarkJoinConjuncts().get(0) instanceof EqualPredicate));
}
public static boolean shouldNestedLoopJoin(JoinType joinType, List<Expression> hashConjuncts) {
+ // this function is only called by hyper graph, which reject mark join
+ // so mark join is not processed here
return hashConjuncts.isEmpty();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
index b2ab651eb0a..d96d79ebe7f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
@@ -76,6 +76,8 @@ public class HashJoinNode extends JoinNodeBase {
// join conjuncts from the JOIN clause that aren't equi-join predicates
private List<Expr> otherJoinConjuncts;
+ private List<Expr> markJoinConjuncts;
+
private DistributionMode distrMode;
private boolean isColocate = false; //the flag for colocate join
private String colocateReason = ""; // if can not do colocate join, set reason here
@@ -173,6 +175,61 @@ public class HashJoinNode extends JoinNodeBase {
vSrcToOutputSMap = new ExprSubstitutionMap(srcToOutputList, Collections.emptyList());
}
+ public HashJoinNode(PlanNodeId id, PlanNode outer, PlanNode inner, JoinOperator joinOp,
+ List<Expr> eqJoinConjuncts, List<Expr> otherJoinConjuncts, List<Expr> markJoinConjuncts,
+ List<Expr> srcToOutputList, TupleDescriptor intermediateTuple,
+ TupleDescriptor outputTuple, boolean isMarkJoin) {
+ super(id, "HASH JOIN", StatisticalType.HASH_JOIN_NODE, joinOp, isMarkJoin);
+ Preconditions.checkArgument((eqJoinConjuncts != null && !eqJoinConjuncts.isEmpty())
+ || (markJoinConjuncts != null && !markJoinConjuncts.isEmpty()));
+ Preconditions.checkArgument(otherJoinConjuncts != null);
+ tblRefIds.addAll(outer.getTblRefIds());
+ tblRefIds.addAll(inner.getTblRefIds());
+
+ if (joinOp.equals(JoinOperator.LEFT_ANTI_JOIN) || joinOp.equals(JoinOperator.LEFT_SEMI_JOIN)
+ || joinOp.equals(JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN)) {
+ tupleIds.addAll(outer.getTupleIds());
+ } else if (joinOp.equals(JoinOperator.RIGHT_ANTI_JOIN) || joinOp.equals(JoinOperator.RIGHT_SEMI_JOIN)) {
+ tupleIds.addAll(inner.getTupleIds());
+ } else {
+ tupleIds.addAll(outer.getTupleIds());
+ tupleIds.addAll(inner.getTupleIds());
+ }
+
+ for (Expr eqJoinPredicate : eqJoinConjuncts) {
+ Preconditions.checkArgument(eqJoinPredicate instanceof BinaryPredicate);
+ BinaryPredicate eqJoin = (BinaryPredicate) eqJoinPredicate;
+ if (eqJoin.getOp().equals(BinaryPredicate.Operator.EQ_FOR_NULL)) {
+ Preconditions.checkArgument(eqJoin.getChildren().size() == 2);
+ if (!eqJoin.getChild(0).isNullable() || !eqJoin.getChild(1).isNullable()) {
+ eqJoin.setOp(BinaryPredicate.Operator.EQ);
+ }
+ }
+ this.eqJoinConjuncts.add(eqJoin);
+ }
+ this.distrMode = DistributionMode.NONE;
+ this.otherJoinConjuncts = otherJoinConjuncts;
+ this.markJoinConjuncts = markJoinConjuncts;
+ children.add(outer);
+ children.add(inner);
+
+ // Inherits all the nullable tuple from the children
+ // Mark tuples that form the "nullable" side of the outer join as nullable.
+ nullableTupleIds.addAll(inner.getNullableTupleIds());
+ nullableTupleIds.addAll(outer.getNullableTupleIds());
+ if (joinOp.equals(JoinOperator.FULL_OUTER_JOIN)) {
+ nullableTupleIds.addAll(outer.getTupleIds());
+ nullableTupleIds.addAll(inner.getTupleIds());
+ } else if (joinOp.equals(JoinOperator.LEFT_OUTER_JOIN)) {
+ nullableTupleIds.addAll(inner.getTupleIds());
+ } else if (joinOp.equals(JoinOperator.RIGHT_OUTER_JOIN)) {
+ nullableTupleIds.addAll(outer.getTupleIds());
+ }
+ vIntermediateTupleDescList = Lists.newArrayList(intermediateTuple);
+ vOutputTupleDesc = outputTuple;
+ vSrcToOutputSMap = new ExprSubstitutionMap(srcToOutputList, Collections.emptyList());
+ }
+
public List<BinaryPredicate> getEqJoinConjuncts() {
return eqJoinConjuncts;
}
@@ -717,6 +774,32 @@ public class HashJoinNode extends JoinNodeBase {
msg.hash_join_node.addToOtherJoinConjuncts(e.treeToThrift());
}
+ if (markJoinConjuncts != null) {
+ if (eqJoinConjuncts.isEmpty()) {
+ Preconditions.checkState(joinOp == JoinOperator.LEFT_SEMI_JOIN
+ || joinOp == JoinOperator.LEFT_ANTI_JOIN);
+ if (joinOp == JoinOperator.LEFT_SEMI_JOIN) {
+ msg.hash_join_node.join_op = JoinOperator.NULL_AWARE_LEFT_SEMI_JOIN.toThrift();
+ } else if (joinOp == JoinOperator.LEFT_ANTI_JOIN) {
+ msg.hash_join_node.join_op = JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN.toThrift();
+ }
+ // because eqJoinConjuncts mustn't be empty in thrift
+ // we have to use markJoinConjuncts instead
+ for (Expr e : markJoinConjuncts) {
+ Preconditions.checkState(e instanceof BinaryPredicate,
+ "mark join conjunct must be BinaryPredicate");
+ TEqJoinCondition eqJoinCondition = new TEqJoinCondition(
+ e.getChild(0).treeToThrift(), e.getChild(1).treeToThrift());
+ eqJoinCondition.setOpcode(((BinaryPredicate) e).getOp().getOpcode());
+ msg.hash_join_node.addToEqJoinConjuncts(eqJoinCondition);
+ }
+ } else {
+ for (Expr e : markJoinConjuncts) {
+ msg.hash_join_node.addToMarkJoinConjuncts(e.treeToThrift());
+ }
+ }
+ }
+
if (hashOutputSlotIds != null) {
for (SlotId slotId : hashOutputSlotIds) {
msg.hash_join_node.addToHashOutputSlotIds(slotId.asInt());
@@ -772,6 +855,10 @@ public class HashJoinNode extends JoinNodeBase {
output.append(detailPrefix).append("other join predicates: ")
.append(getExplainString(otherJoinConjuncts)).append("\n");
}
+ if (markJoinConjuncts != null && !markJoinConjuncts.isEmpty()) {
+ output.append(detailPrefix).append("mark join predicates: ")
+ .append(getExplainString(markJoinConjuncts)).append("\n");
+ }
if (!conjuncts.isEmpty()) {
output.append(detailPrefix).append("other predicates: ").append(getExplainString(conjuncts)).append("\n");
}
@@ -849,10 +936,18 @@ public class HashJoinNode extends JoinNodeBase {
this.otherJoinConjuncts = otherJoinConjuncts;
}
+ public void setMarkJoinConjuncts(List<Expr> markJoinConjuncts) {
+ this.markJoinConjuncts = markJoinConjuncts;
+ }
+
public List<Expr> getOtherJoinConjuncts() {
return otherJoinConjuncts;
}
+ public List<Expr> getMarkJoinConjuncts() {
+ return markJoinConjuncts;
+ }
+
SlotRef getMappedInputSlotRef(SlotRef slotRef) {
if (outputSmap != null) {
Expr mappedExpr = outputSmap.mappingForRhsExpr(slotRef);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java
index 05eb34a7815..63cc724001b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java
@@ -65,6 +65,8 @@ public class NestedLoopJoinNode extends JoinNodeBase {
private List<Expr> runtimeFilterExpr = Lists.newArrayList();
private List<Expr> joinConjuncts;
+ private List<Expr> markJoinConjuncts;
+
public NestedLoopJoinNode(PlanNodeId id, PlanNode outer, PlanNode inner, TableRef innerRef) {
super(id, "NESTED LOOP JOIN", StatisticalType.NESTED_LOOP_JOIN_NODE, outer, inner, innerRef);
tupleIds.addAll(outer.getOutputTupleIds());
@@ -81,6 +83,10 @@ public class NestedLoopJoinNode extends JoinNodeBase {
this.joinConjuncts = joinConjuncts;
}
+ public void setMarkJoinConjuncts(List<Expr> markJoinConjuncts) {
+ this.markJoinConjuncts = markJoinConjuncts;
+ }
+
@Override
protected List<SlotId> computeSlotIdsForJoinConjuncts(Analyzer analyzer) {
// conjunct
@@ -171,6 +177,12 @@ public class NestedLoopJoinNode extends JoinNodeBase {
for (Expr conjunct : joinConjuncts) {
msg.nested_loop_join_node.addToJoinConjuncts(conjunct.treeToThrift());
}
+ if (markJoinConjuncts != null) {
+ for (Expr conjunct : markJoinConjuncts) {
+ msg.nested_loop_join_node.addToMarkJoinConjuncts(conjunct.treeToThrift());
+ }
+ }
+
msg.nested_loop_join_node.setIsMark(isMarkJoin());
if (vSrcToOutputSMap != null) {
for (int i = 0; i < vSrcToOutputSMap.size(); i++) {
@@ -230,6 +242,11 @@ public class NestedLoopJoinNode extends JoinNodeBase {
output.append(detailPrefix).append("join conjuncts: ").append(getExplainString(joinConjuncts)).append("\n");
}
+ if (markJoinConjuncts != null && !markJoinConjuncts.isEmpty()) {
+ output.append(detailPrefix).append("mark join predicates: ")
+ .append(getExplainString(markJoinConjuncts)).append("\n");
+ }
+
if (!conjuncts.isEmpty()) {
output.append(detailPrefix).append("predicates: ").append(getExplainString(conjuncts)).append("\n");
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ExistsApplyToJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ExistsApplyToJoinTest.java
index eeb7a505f8b..5f617eaacc4 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ExistsApplyToJoinTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ExistsApplyToJoinTest.java
@@ -48,7 +48,7 @@ class ExistsApplyToJoinTest implements MemoPatternMatchSupported {
LogicalApply<LogicalOlapScan, LogicalOlapScan> apply =
new LogicalApply<>(ImmutableList.of(leftSlots.get(0), rightSlots.get(0)),
exists, Optional.of(equalTo), Optional.empty(),
- false, false, left, right);
+ false, false, false, left, right);
PlanChecker.from(MemoTestUtils.createConnectContext(), apply)
.applyTopDown(new ExistsApplyToJoin())
.matchesFromRoot(logicalJoin(
@@ -68,7 +68,7 @@ class ExistsApplyToJoinTest implements MemoPatternMatchSupported {
LogicalApply<LogicalOlapScan, LogicalOlapScan> apply =
new LogicalApply<>(Collections.emptyList(),
exists, Optional.of(equalTo), Optional.empty(),
- false, false, left, right);
+ false, false, false, left, right);
PlanChecker.from(MemoTestUtils.createConnectContext(), apply)
.applyTopDown(new ExistsApplyToJoin())
.matchesFromRoot(logicalJoin(
@@ -88,7 +88,7 @@ class ExistsApplyToJoinTest implements MemoPatternMatchSupported {
LogicalApply<LogicalOlapScan, LogicalOlapScan> apply =
new LogicalApply<>(Collections.emptyList(),
exists, Optional.of(equalTo), Optional.empty(),
- false, false, left, right);
+ false, false, false, left, right);
PlanChecker.from(MemoTestUtils.createConnectContext(), apply)
.applyTopDown(new ExistsApplyToJoin())
.matchesFromRoot(logicalFilter(logicalJoin(
@@ -109,7 +109,7 @@ class ExistsApplyToJoinTest implements MemoPatternMatchSupported {
LogicalApply<LogicalOlapScan, LogicalOlapScan> apply =
new LogicalApply<>(ImmutableList.of(leftSlots.get(0), rightSlots.get(0)),
exists, Optional.of(equalTo), Optional.empty(),
- false, false, left, right);
+ false, false, false, left, right);
PlanChecker.from(MemoTestUtils.createConnectContext(), apply)
.applyTopDown(new ExistsApplyToJoin())
.matchesFromRoot(logicalJoin(
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java
index a764b9ec343..229b90f00e0 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java
@@ -75,7 +75,7 @@ public class PlanToStringTest {
left, right);
System.out.println(plan.toString());
Assertions.assertTrue(plan.toString().matches(
- "LogicalJoin\\[\\d+\\] \\( type=INNER_JOIN, markJoinSlotReference=Optional.empty, hashJoinConjuncts=\\[\\(a#\\d+ = b#\\d+\\)], otherJoinConjuncts=\\[] \\)"));
+ "LogicalJoin\\[\\d+\\] \\( type=INNER_JOIN, markJoinSlotReference=Optional.empty, hashJoinConjuncts=\\[\\(a#\\d+ = b#\\d+\\)], otherJoinConjuncts=\\[], markJoinConjuncts=\\[] \\)"));
}
@Test
diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
index 911a565c1ae..d5892895e12 100644
--- a/gensrc/thrift/PlanNodes.thrift
+++ b/gensrc/thrift/PlanNodes.thrift
@@ -731,7 +731,8 @@ enum TJoinOp {
// on the build side. Those NULLs are considered candidate matches, and therefore could
// be rejected (ANTI-join), based on the other join conjuncts. This is in contrast
// to LEFT_ANTI_JOIN where NULLs are not matches and therefore always returned.
- NULL_AWARE_LEFT_ANTI_JOIN
+ NULL_AWARE_LEFT_ANTI_JOIN,
+ NULL_AWARE_LEFT_SEMI_JOIN
}
enum TJoinDistributionType {
@@ -774,6 +775,7 @@ struct THashJoinNode {
11: optional bool is_mark
12: optional TJoinDistributionType dist_type
+ 13: optional list<Exprs.TExpr> mark_join_conjuncts
}
struct TNestedLoopJoinNode {
@@ -793,6 +795,8 @@ struct TNestedLoopJoinNode {
7: optional bool is_mark
8: optional list<Exprs.TExpr> join_conjuncts
+
+ 9: optional list<Exprs.TExpr> mark_join_conjuncts
}
struct TMergeJoinNode {
diff --git a/regression-test/data/nereids_p0/subquery/subquery_unnesting.out b/regression-test/data/nereids_p0/subquery/subquery_unnesting.out
new file mode 100644
index 00000000000..f809cbf8bcd
--- /dev/null
+++ b/regression-test/data/nereids_p0/subquery/subquery_unnesting.out
@@ -0,0 +1,1494 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select1 --
+1 \N
+1 2
+1 3
+2 4
+2 5
+3 3
+3 4
+
+-- !select2 --
+1 2
+1 3
+2 4
+3 3
+3 4
+
+-- !select3 --
+1 \N
+1 2
+1 3
+2 4
+2 5
+3 3
+3 4
+20 2
+22 3
+24 4
+
+-- !select4 --
+1 \N
+1 2
+1 3
+2 4
+2 5
+3 3
+3 4
+
+-- !select5 --
+1 \N
+1 2
+1 3
+2 4
+2 5
+3 3
+3 4
+
+-- !select6 --
+2 4
+3 4
+
+-- !select7 --
+1 \N
+1 2
+1 3
+2 4
+2 5
+3 3
+3 4
+
+-- !select8 --
+2 4
+2 5
+3 3
+3 4
+
+-- !select9 --
+1 \N
+1 2
+1 3
+2 4
+2 5
+3 3
+3 4
+
+-- !select10 --
+2 4
+2 5
+3 3
+3 4
+
+-- !select11 --
+\N \N
+\N 1
+1 \N
+1 2
+1 3
+2 4
+2 5
+3 3
+3 4
+20 2
+22 3
+24 4
+
+-- !select12 --
+\N \N
+\N 1
+1 \N
+1 2
+1 3
+2 5
+3 3
+20 2
+22 3
+24 4
+
+-- !select13 --
+\N \N
+1 \N
+1 2
+1 3
+2 4
+2 5
+3 3
+3 4
+20 2
+22 3
+24 4
+
+-- !select14 --
+\N \N
+1 \N
+1 2
+1 3
+2 4
+2 5
+3 4
+20 2
+22 3
+24 4
+
+-- !select15 --
+1 \N
+1 2
+1 3
+2 4
+2 5
+3 3
+3 4
+
+-- !select16 --
+1 \N
+1 2
+1 3
+20 2
+22 3
+24 4
+
+-- !select17 --
+\N 2
+1 \N
+1 2
+1 3
+2 4
+2 5
+3 3
+3 4
+20 2
+22 3
+24 4
+
+-- !select18 --
+\N 2
+1 2
+1 3
+2 4
+3 3
+3 4
+20 2
+22 3
+24 4
+
+-- !select19 --
+1 \N
+1 2
+1 3
+2 4
+2 5
+3 3
+3 4
+22 3
+24 4
+
+-- !select20 --
+1 3
+2 4
+2 5
+3 3
+3 4
+22 3
+24 4
+
+-- !select21 --
+1 \N
+1 2
+1 3
+2 4
+2 5
+3 3
+3 4
+
+-- !select22 --
+\N \N
+\N 1
+\N 2
+1 \N
+1 2
+1 3
+2 4
+2 5
+3 3
+3 4
+20 2
+22 3
+24 4
+
+-- !select23 --
+\N \N
+\N 1
+1 \N
+1 2
+1 3
+2 4
+2 5
+3 3
+3 4
+
+-- !select24 --
+\N \N
+\N 1
+1 \N
+2 5
+
+-- !select25 --
+\N \N
+\N 1
+\N 2
+1 \N
+1 2
+1 3
+2 4
+2 5
+3 3
+3 4
+20 2
+
+-- !select26 --
+\N \N
+1 \N
+
+-- !select27 --
+\N \N
+\N 1
+\N 2
+1 \N
+1 2
+1 3
+2 4
+2 5
+3 3
+3 4
+20 2
+22 3
+24 4
+
+-- !select28 --
+
+-- !select29 --
+\N 2
+1 2
+1 3
+2 4
+3 3
+3 4
+20 2
+22 3
+24 4
+
+-- !select30 --
+1 3
+2 4
+2 5
+3 3
+3 4
+22 3
+24 4
+
+-- !select31 --
+\N \N
+\N 1
+\N 2
+1 \N
+1 2
+1 3
+2 4
+2 5
+3 3
+3 4
+20 2
+22 3
+24 4
+
+-- !select32 --
+\N \N
+\N 1
+\N 2
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+20 2
+22 3
+24 4
+
+-- !select33 --
+\N \N
+\N 1
+\N 2
+1 \N
+1 2
+1 3
+2 4
+2 5
+3 3
+3 4
+20 2
+22 3
+24 4
+
+-- !select34 --
+\N \N
+\N 1
+\N 2
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+20 2
+22 3
+24 4
+
+-- !select35 --
+\N \N
+\N 1
+\N 2
+1 \N
+1 2
+1 2
+1 3
+1 3
+1 3
+2 4
+2 4
+2 5
+3 3
+3 3
+3 3
+3 4
+3 4
+20 2
+22 3
+24 4
+
+-- !select36 --
+\N \N
+\N 1
+\N 2
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+20 2
+22 3
+24 4
+
+-- !select37 --
+\N \N
+\N 1
+\N 2
+1 \N
+1 2
+1 2
+1 3
+2 4
+2 4
+2 5
+3 3
+3 4
+20 2
+22 3
+24 4
+
+-- !select38 --
+\N \N
+\N 1
+\N 2
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+20 2
+22 3
+24 4
+
+-- !select39 --
+\N \N
+\N 1
+\N 2
+1 \N
+1 2
+1 2
+1 3
+1 3
+1 3
+2 4
+2 5
+3 3
+3 3
+3 3
+3 4
+3 4
+20 2
+22 3
+24 4
+
+-- !select40 --
+\N \N
+\N 1
+\N 2
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+20 2
+22 3
+24 4
+
+-- !select41 --
+\N \N
+\N 1
+\N 2
+1 \N
+1 2
+1 2
+1 3
+1 3
+1 3
+2 4
+2 4
+2 5
+3 3
+3 3
+3 3
+3 4
+3 4
+20 2
+22 3
+24 4
+
+-- !select42 --
+\N \N
+\N 1
+\N 2
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+20 2
+20 2
+22 3
+24 4
+24 4
+
+-- !select43 --
+\N \N
+\N 1
+\N 2
+1 \N
+1 2
+1 3
+2 4
+2 5
+3 3
+3 4
+3 4
+20 2
+20 2
+22 3
+24 4
+24 4
+
+-- !select44 --
+\N \N
+\N 1
+\N 2
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+20 2
+22 3
+22 3
+22 3
+24 4
+24 4
+
+-- !select45 --
+\N \N
+\N 1
+\N 2
+1 \N
+1 2
+1 3
+2 4
+2 4
+2 5
+3 3
+3 4
+20 2
+22 3
+22 3
+22 3
+24 4
+24 4
+
+-- !select46 --
+\N \N
+\N 1
+\N 2
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+20 2
+22 3
+24 4
+
+-- !select47 --
+\N \N
+\N 1
+\N 2
+1 \N
+1 2
+1 3
+2 4
+2 5
+3 3
+3 4
+20 2
+22 3
+24 4
+
+-- !select48 --
+\N \N
+\N 1
+\N 2
+\N 2
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+20 2
+20 2
+22 3
+22 3
+22 3
+24 4
+24 4
+
+-- !select49 --
+\N \N
+\N 1
+\N 2
+\N 2
+1 \N
+1 2
+1 2
+1 3
+1 3
+1 3
+2 4
+2 4
+2 5
+3 3
+3 3
+3 3
+3 4
+3 4
+20 2
+20 2
+22 3
+22 3
+22 3
+24 4
+24 4
+
+-- !select50 --
+\N \N
+\N 1
+\N 2
+\N 2
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+20 2
+20 2
+22 3
+22 3
+22 3
+24 4
+24 4
+
+-- !select51 --
+\N \N
+\N 1
+\N 2
+\N 2
+1 \N
+1 2
+1 2
+1 3
+1 3
+1 3
+2 4
+2 4
+2 5
+3 3
+3 3
+3 3
+3 4
+3 4
+20 2
+20 2
+22 3
+22 3
+22 3
+24 4
+24 4
+
+-- !select52 --
+\N \N
+\N 1
+\N 2
+\N 2
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+20 2
+20 2
+22 3
+22 3
+22 3
+24 4
+24 4
+
+-- !select53 --
+\N \N
+\N 1
+\N 2
+\N 2
+1 \N
+1 2
+1 2
+1 3
+1 3
+1 3
+2 4
+2 4
+2 5
+3 3
+3 3
+3 3
+3 4
+3 4
+20 2
+20 2
+22 3
+22 3
+22 3
+24 4
+24 4
+
+-- !select54 --
+\N \N
+\N 1
+\N 2
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+20 2
+22 3
+24 4
+
+-- !select55 --
+\N \N
+\N 1
+\N 2
+1 \N
+1 2
+1 3
+2 4
+2 5
+3 3
+3 4
+20 2
+22 3
+24 4
+
+-- !select56 --
+\N \N
+\N 1
+\N 2
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+20 2
+22 3
+24 4
+
+-- !select57 --
+\N \N
+\N 1
+\N 2
+1 \N
+1 2
+1 3
+2 4
+2 5
+3 3
+3 4
+20 2
+22 3
+24 4
+
+-- !select58 --
+\N \N
+\N 1
+\N 2
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 2
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+1 3
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 4
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+2 5
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 3
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+3 4
+20 2
+22 3
+24 4
+
+-- !select59 --
+\N \N
+\N 1
+\N 2
+1 \N
+1 2
+1 3
+2 4
+2 5
+3 3
+3 4
+20 2
+22 3
+24 4
+
diff --git a/regression-test/data/nereids_p0/subquery/test_subquery_in_project.out b/regression-test/data/nereids_p0/subquery/test_subquery_in_project.out
index c0d289b51fb..961b0d73c52 100644
--- a/regression-test/data/nereids_p0/subquery/test_subquery_in_project.out
+++ b/regression-test/data/nereids_p0/subquery/test_subquery_in_project.out
@@ -66,3 +66,123 @@ true
5
7
+-- !select_m1 --
+\N
+false
+false
+false
+false
+false
+false
+false
+false
+false
+true
+true
+true
+
+-- !select_m2 --
+false
+false
+false
+false
+false
+false
+true
+true
+true
+true
+true
+true
+true
+
+-- !select_m3 --
+\N
+false
+false
+false
+true
+true
+true
+true
+true
+true
+true
+true
+true
+
+-- !select_m4 --
+false
+false
+false
+false
+false
+false
+false
+true
+true
+true
+true
+true
+true
+
+-- !select_m5 --
+\N
+\N
+\N
+\N
+true
+true
+true
+true
+true
+true
+true
+true
+true
+
+-- !select_m6 --
+true
+true
+true
+true
+true
+true
+true
+true
+true
+true
+true
+true
+true
+
+-- !select_m7 --
+\N
+\N
+\N
+\N
+false
+false
+false
+false
+false
+false
+false
+false
+false
+
+-- !select_m8 --
+false
+false
+false
+false
+false
+false
+false
+false
+false
+false
+false
+false
+false
+
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query10.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query10.out
index 59e87830c0d..83741371afc 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query10.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query10.out
@@ -5,50 +5,53 @@ PhysicalResultSink
----PhysicalTopN[MERGE_SORT]
------PhysicalDistribute[DistributionSpecGather]
--------PhysicalTopN[LOCAL_SORT]
-----------hashAgg[LOCAL]
-------------PhysicalProject
---------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
-----------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
-------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((c.c_customer_sk = web_sales.ws_bill_customer_sk)) otherCondition=()
---------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=() build RFs:RF5 c_current_cdemo_sk->[cd_demo_sk]
-------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF5
-------------------------PhysicalDistribute[DistributionSpecReplicated]
+----------hashAgg[GLOBAL]
+------------PhysicalDistribute[DistributionSpecHash]
+--------------hashAgg[LOCAL]
+----------------PhysicalProject
+------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
+--------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
+----------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((c.c_customer_sk = web_sales.ws_bill_customer_sk)) otherCondition=()
+------------------------PhysicalDistribute[DistributionSpecHash]
--------------------------PhysicalProject
-----------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[ss_customer_sk]
-------------------------------PhysicalDistribute[DistributionSpecHash]
+----------------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=() build RFs:RF5 c_current_cdemo_sk->[cd_demo_sk]
+------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF5
+------------------------------PhysicalDistribute[DistributionSpecReplicated]
--------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[ss_sold_date_sk]
-------------------------------------PhysicalProject
---------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF3 RF4
-------------------------------------PhysicalDistribute[DistributionSpecReplicated]
+----------------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[ss_customer_sk]
+------------------------------------PhysicalDistribute[DistributionSpecHash]
--------------------------------------PhysicalProject
-----------------------------------------filter((date_dim.d_moy <= 6) and (date_dim.d_moy >= 3) and (date_dim.d_year = 2001))
-------------------------------------------PhysicalOlapScan[date_dim]
-------------------------------PhysicalDistribute[DistributionSpecHash]
---------------------------------hashJoin[INNER_JOIN] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF2 ca_address_sk->[c_current_addr_sk]
-----------------------------------PhysicalProject
-------------------------------------PhysicalOlapScan[customer] apply RFs: RF2
-----------------------------------PhysicalDistribute[DistributionSpecReplicated]
-------------------------------------PhysicalProject
---------------------------------------filter(ca_county IN ('Campbell County', 'Cleburne County', 'Escambia County', 'Fairfield County', 'Washtenaw County'))
-----------------------------------------PhysicalOlapScan[customer_address]
---------------------PhysicalDistribute[DistributionSpecReplicated]
-----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ws_sold_date_sk]
+----------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[ss_sold_date_sk]
+------------------------------------------PhysicalProject
+--------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF3 RF4
+------------------------------------------PhysicalDistribute[DistributionSpecReplicated]
+--------------------------------------------PhysicalProject
+----------------------------------------------filter((date_dim.d_moy <= 6) and (date_dim.d_moy >= 3) and (date_dim.d_year = 2001))
+------------------------------------------------PhysicalOlapScan[date_dim]
+------------------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------------------hashJoin[INNER_JOIN] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF2 ca_address_sk->[c_current_addr_sk]
+----------------------------------------PhysicalProject
+------------------------------------------PhysicalOlapScan[customer] apply RFs: RF2
+----------------------------------------PhysicalDistribute[DistributionSpecReplicated]
+------------------------------------------PhysicalProject
+--------------------------------------------filter(ca_county IN ('Campbell County', 'Cleburne County', 'Escambia County', 'Fairfield County', 'Washtenaw County'))
+----------------------------------------------PhysicalOlapScan[customer_address]
+------------------------PhysicalDistribute[DistributionSpecHash]
--------------------------PhysicalProject
-----------------------------PhysicalOlapScan[web_sales] apply RFs: RF1
---------------------------PhysicalDistribute[DistributionSpecReplicated]
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_moy <= 6) and (date_dim.d_moy >= 3) and (date_dim.d_year = 2001))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalDistribute[DistributionSpecReplicated]
---------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
+----------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ws_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1
+------------------------------PhysicalDistribute[DistributionSpecReplicated]
+--------------------------------PhysicalProject
+----------------------------------filter((date_dim.d_moy <= 6) and (date_dim.d_moy >= 3) and (date_dim.d_year = 2001))
+------------------------------------PhysicalOlapScan[date_dim]
+----------------------PhysicalDistribute[DistributionSpecHash]
------------------------PhysicalProject
---------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
-------------------------PhysicalDistribute[DistributionSpecReplicated]
---------------------------PhysicalProject
-----------------------------filter((date_dim.d_moy <= 6) and (date_dim.d_moy >= 3) and (date_dim.d_year = 2001))
-------------------------------PhysicalOlapScan[date_dim]
+--------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
+----------------------------PhysicalDistribute[DistributionSpecReplicated]
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_moy <= 6) and (date_dim.d_moy >= 3) and (date_dim.d_year = 2001))
+----------------------------------PhysicalOlapScan[date_dim]
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query35.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query35.out
index 79412a9b427..a72d3f8673e 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query35.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query35.out
@@ -12,32 +12,33 @@ PhysicalResultSink
------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
--------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
----------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((c.c_customer_sk = web_sales.ws_bill_customer_sk)) otherCondition=()
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=() build RFs:RF5 cd_demo_sk->[c_current_cdemo_sk]
-----------------------------PhysicalDistribute[DistributionSpecHash]
-------------------------------PhysicalProject
---------------------------------hashJoin[INNER_JOIN] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF4 ca_address_sk->[c_current_addr_sk]
-----------------------------------PhysicalDistribute[DistributionSpecHash]
-------------------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[ss_customer_sk]
---------------------------------------PhysicalDistribute[DistributionSpecHash]
-----------------------------------------PhysicalProject
-------------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
---------------------------------------------PhysicalProject
-----------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 RF3
---------------------------------------------PhysicalDistribute[DistributionSpecReplicated]
+------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=() build RFs:RF5 cd_demo_sk->[c_current_cdemo_sk]
+------------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------------PhysicalProject
+----------------------------------hashJoin[INNER_JOIN] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF4 ca_address_sk->[c_current_addr_sk]
+------------------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[ss_customer_sk]
+----------------------------------------PhysicalDistribute[DistributionSpecHash]
+------------------------------------------PhysicalProject
+--------------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
----------------------------------------------PhysicalProject
-------------------------------------------------filter((date_dim.d_qoy < 4) and (date_dim.d_year = 1999))
---------------------------------------------------PhysicalOlapScan[date_dim]
---------------------------------------PhysicalDistribute[DistributionSpecHash]
-----------------------------------------PhysicalProject
-------------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 RF5
-----------------------------------PhysicalDistribute[DistributionSpecHash]
-------------------------------------PhysicalProject
---------------------------------------PhysicalOlapScan[customer_address]
-----------------------------PhysicalDistribute[DistributionSpecHash]
-------------------------------PhysicalProject
---------------------------------PhysicalOlapScan[customer_demographics]
-------------------------PhysicalDistribute[DistributionSpecReplicated]
+------------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 RF3
+----------------------------------------------PhysicalDistribute[DistributionSpecReplicated]
+------------------------------------------------PhysicalProject
+--------------------------------------------------filter((date_dim.d_qoy < 4) and (date_dim.d_year = 1999))
+----------------------------------------------------PhysicalOlapScan[date_dim]
+----------------------------------------PhysicalDistribute[DistributionSpecHash]
+------------------------------------------PhysicalProject
+--------------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 RF5
+------------------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------------------PhysicalProject
+----------------------------------------PhysicalOlapScan[customer_address]
+------------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[customer_demographics]
+------------------------PhysicalDistribute[DistributionSpecHash]
--------------------------PhysicalProject
----------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ws_sold_date_sk]
------------------------------PhysicalProject
@@ -46,7 +47,7 @@ PhysicalResultSink
--------------------------------PhysicalProject
----------------------------------filter((date_dim.d_qoy < 4) and (date_dim.d_year = 1999))
------------------------------------PhysicalOlapScan[date_dim]
-----------------------PhysicalDistribute[DistributionSpecReplicated]
+----------------------PhysicalDistribute[DistributionSpecHash]
------------------------PhysicalProject
--------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
----------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query10.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query10.out
index 49d24a004db..845705681d5 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query10.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query10.out
@@ -12,30 +12,31 @@ PhysicalResultSink
------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
--------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
----------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((c.c_customer_sk = web_sales.ws_bill_customer_sk)) otherCondition=()
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF5 ca_address_sk->[c_current_addr_sk]
-----------------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=()
-------------------------------PhysicalDistribute[DistributionSpecHash]
---------------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[ss_customer_sk]
-----------------------------------PhysicalDistribute[DistributionSpecHash]
-------------------------------------PhysicalProject
---------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
-----------------------------------------PhysicalProject
-------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 RF3
-----------------------------------------PhysicalDistribute[DistributionSpecReplicated]
+------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF5 ca_address_sk->[c_current_addr_sk]
+------------------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=()
+--------------------------------PhysicalDistribute[DistributionSpecHash]
+----------------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[ss_customer_sk]
+------------------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------------------PhysicalProject
+----------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
------------------------------------------PhysicalProject
---------------------------------------------filter((date_dim.d_moy <= 4) and (date_dim.d_moy >= 1) and (date_dim.d_year = 2001))
-----------------------------------------------PhysicalOlapScan[date_dim]
-----------------------------------PhysicalDistribute[DistributionSpecHash]
-------------------------------------PhysicalProject
---------------------------------------PhysicalOlapScan[customer] apply RFs: RF5
-------------------------------PhysicalDistribute[DistributionSpecHash]
---------------------------------PhysicalOlapScan[customer_demographics]
-----------------------------PhysicalDistribute[DistributionSpecReplicated]
-------------------------------PhysicalProject
---------------------------------filter(ca_county IN ('Cochran County', 'Kandiyohi County', 'Marquette County', 'Storey County', 'Warren County'))
-----------------------------------PhysicalOlapScan[customer_address]
-------------------------PhysicalDistribute[DistributionSpecReplicated]
+--------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 RF3
+------------------------------------------PhysicalDistribute[DistributionSpecReplicated]
+--------------------------------------------PhysicalProject
+----------------------------------------------filter((date_dim.d_moy <= 4) and (date_dim.d_moy >= 1) and (date_dim.d_year = 2001))
+------------------------------------------------PhysicalOlapScan[date_dim]
+------------------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------------------PhysicalProject
+----------------------------------------PhysicalOlapScan[customer] apply RFs: RF5
+--------------------------------PhysicalDistribute[DistributionSpecHash]
+----------------------------------PhysicalOlapScan[customer_demographics]
+------------------------------PhysicalDistribute[DistributionSpecReplicated]
+--------------------------------PhysicalProject
+----------------------------------filter(ca_county IN ('Cochran County', 'Kandiyohi County', 'Marquette County', 'Storey County', 'Warren County'))
+------------------------------------PhysicalOlapScan[customer_address]
+------------------------PhysicalDistribute[DistributionSpecHash]
--------------------------PhysicalProject
----------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ws_sold_date_sk]
------------------------------PhysicalProject
@@ -44,7 +45,7 @@ PhysicalResultSink
--------------------------------PhysicalProject
----------------------------------filter((date_dim.d_moy <= 4) and (date_dim.d_moy >= 1) and (date_dim.d_year = 2001))
------------------------------------PhysicalOlapScan[date_dim]
-----------------------PhysicalDistribute[DistributionSpecReplicated]
+----------------------PhysicalDistribute[DistributionSpecHash]
------------------------PhysicalProject
--------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
----------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query35.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query35.out
index a849eb675a5..97a42e09303 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query35.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query35.out
@@ -12,31 +12,32 @@ PhysicalResultSink
------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
--------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
----------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((c.c_customer_sk = web_sales.ws_bill_customer_sk)) otherCondition=()
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=()
-----------------------------PhysicalDistribute[DistributionSpecHash]
-------------------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=()
---------------------------------PhysicalDistribute[DistributionSpecHash]
-----------------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=()
-------------------------------------PhysicalDistribute[DistributionSpecHash]
---------------------------------------PhysicalProject
-----------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
-------------------------------------------PhysicalProject
---------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2
-------------------------------------------PhysicalDistribute[DistributionSpecReplicated]
+------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=()
+------------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=()
+----------------------------------PhysicalDistribute[DistributionSpecHash]
+------------------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=()
+--------------------------------------PhysicalDistribute[DistributionSpecHash]
+----------------------------------------PhysicalProject
+------------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
--------------------------------------------PhysicalProject
-----------------------------------------------filter((date_dim.d_qoy < 4) and (date_dim.d_year = 2001))
-------------------------------------------------PhysicalOlapScan[date_dim]
-------------------------------------PhysicalDistribute[DistributionSpecHash]
---------------------------------------PhysicalProject
-----------------------------------------PhysicalOlapScan[customer]
---------------------------------PhysicalDistribute[DistributionSpecHash]
-----------------------------------PhysicalProject
-------------------------------------PhysicalOlapScan[customer_demographics]
-----------------------------PhysicalDistribute[DistributionSpecHash]
-------------------------------PhysicalProject
---------------------------------PhysicalOlapScan[customer_address]
-------------------------PhysicalDistribute[DistributionSpecReplicated]
+----------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2
+--------------------------------------------PhysicalDistribute[DistributionSpecReplicated]
+----------------------------------------------PhysicalProject
+------------------------------------------------filter((date_dim.d_qoy < 4) and (date_dim.d_year = 2001))
+--------------------------------------------------PhysicalOlapScan[date_dim]
+--------------------------------------PhysicalDistribute[DistributionSpecHash]
+----------------------------------------PhysicalProject
+------------------------------------------PhysicalOlapScan[customer]
+----------------------------------PhysicalDistribute[DistributionSpecHash]
+------------------------------------PhysicalProject
+--------------------------------------PhysicalOlapScan[customer_demographics]
+------------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[customer_address]
+------------------------PhysicalDistribute[DistributionSpecHash]
--------------------------PhysicalProject
----------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ws_sold_date_sk]
------------------------------PhysicalProject
@@ -45,7 +46,7 @@ PhysicalResultSink
--------------------------------PhysicalProject
----------------------------------filter((date_dim.d_qoy < 4) and (date_dim.d_year = 2001))
------------------------------------PhysicalOlapScan[date_dim]
-----------------------PhysicalDistribute[DistributionSpecReplicated]
+----------------------PhysicalDistribute[DistributionSpecHash]
------------------------PhysicalProject
--------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
----------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query10.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query10.out
index 9e185b3a54e..558e86d696e 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query10.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query10.out
@@ -12,30 +12,31 @@ PhysicalResultSink
------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
--------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
----------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((c.c_customer_sk = web_sales.ws_bill_customer_sk)) otherCondition=()
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF5 ca_address_sk->[c_current_addr_sk]
-----------------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=() build RFs:RF4 cd_demo_sk->[c_current_cdemo_sk]
-------------------------------PhysicalDistribute[DistributionSpecHash]
---------------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[ss_customer_sk]
-----------------------------------PhysicalDistribute[DistributionSpecHash]
-------------------------------------PhysicalProject
---------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
-----------------------------------------PhysicalProject
-------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 RF3
-----------------------------------------PhysicalDistribute[DistributionSpecReplicated]
+------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF5 ca_address_sk->[c_current_addr_sk]
+------------------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=() build RFs:RF4 cd_demo_sk->[c_current_cdemo_sk]
+--------------------------------PhysicalDistribute[DistributionSpecHash]
+----------------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[ss_customer_sk]
+------------------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------------------PhysicalProject
+----------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
------------------------------------------PhysicalProject
---------------------------------------------filter((date_dim.d_moy <= 4) and (date_dim.d_moy >= 1) and (date_dim.d_year = 2001))
-----------------------------------------------PhysicalOlapScan[date_dim]
-----------------------------------PhysicalDistribute[DistributionSpecHash]
-------------------------------------PhysicalProject
---------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 RF5
-------------------------------PhysicalDistribute[DistributionSpecHash]
---------------------------------PhysicalOlapScan[customer_demographics]
-----------------------------PhysicalDistribute[DistributionSpecReplicated]
-------------------------------PhysicalProject
---------------------------------filter(ca_county IN ('Cochran County', 'Kandiyohi County', 'Marquette County', 'Storey County', 'Warren County'))
-----------------------------------PhysicalOlapScan[customer_address]
-------------------------PhysicalDistribute[DistributionSpecReplicated]
+--------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 RF3
+------------------------------------------PhysicalDistribute[DistributionSpecReplicated]
+--------------------------------------------PhysicalProject
+----------------------------------------------filter((date_dim.d_moy <= 4) and (date_dim.d_moy >= 1) and (date_dim.d_year = 2001))
+------------------------------------------------PhysicalOlapScan[date_dim]
+------------------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------------------PhysicalProject
+----------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 RF5
+--------------------------------PhysicalDistribute[DistributionSpecHash]
+----------------------------------PhysicalOlapScan[customer_demographics]
+------------------------------PhysicalDistribute[DistributionSpecReplicated]
+--------------------------------PhysicalProject
+----------------------------------filter(ca_county IN ('Cochran County', 'Kandiyohi County', 'Marquette County', 'Storey County', 'Warren County'))
+------------------------------------PhysicalOlapScan[customer_address]
+------------------------PhysicalDistribute[DistributionSpecHash]
--------------------------PhysicalProject
----------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ws_sold_date_sk]
------------------------------PhysicalProject
@@ -44,7 +45,7 @@ PhysicalResultSink
--------------------------------PhysicalProject
----------------------------------filter((date_dim.d_moy <= 4) and (date_dim.d_moy >= 1) and (date_dim.d_year = 2001))
------------------------------------PhysicalOlapScan[date_dim]
-----------------------PhysicalDistribute[DistributionSpecReplicated]
+----------------------PhysicalDistribute[DistributionSpecHash]
------------------------PhysicalProject
--------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
----------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query35.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query35.out
index 03410c1b280..cc92a34b9e9 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query35.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query35.out
@@ -12,31 +12,32 @@ PhysicalResultSink
------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
--------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
----------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((c.c_customer_sk = web_sales.ws_bill_customer_sk)) otherCondition=()
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF5 ca_address_sk->[c_current_addr_sk]
-----------------------------PhysicalDistribute[DistributionSpecHash]
-------------------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=() build RFs:RF4 cd_demo_sk->[c_current_cdemo_sk]
---------------------------------PhysicalDistribute[DistributionSpecHash]
-----------------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[ss_customer_sk]
-------------------------------------PhysicalDistribute[DistributionSpecHash]
---------------------------------------PhysicalProject
-----------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
-------------------------------------------PhysicalProject
---------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 RF3
-------------------------------------------PhysicalDistribute[DistributionSpecReplicated]
+------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF5 ca_address_sk->[c_current_addr_sk]
+------------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=() build RFs:RF4 cd_demo_sk->[c_current_cdemo_sk]
+----------------------------------PhysicalDistribute[DistributionSpecHash]
+------------------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[ss_customer_sk]
+--------------------------------------PhysicalDistribute[DistributionSpecHash]
+----------------------------------------PhysicalProject
+------------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
--------------------------------------------PhysicalProject
-----------------------------------------------filter((date_dim.d_qoy < 4) and (date_dim.d_year = 2001))
-------------------------------------------------PhysicalOlapScan[date_dim]
-------------------------------------PhysicalDistribute[DistributionSpecHash]
---------------------------------------PhysicalProject
-----------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 RF5
---------------------------------PhysicalDistribute[DistributionSpecHash]
-----------------------------------PhysicalProject
-------------------------------------PhysicalOlapScan[customer_demographics]
-----------------------------PhysicalDistribute[DistributionSpecHash]
-------------------------------PhysicalProject
---------------------------------PhysicalOlapScan[customer_address]
-------------------------PhysicalDistribute[DistributionSpecReplicated]
+----------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 RF3
+--------------------------------------------PhysicalDistribute[DistributionSpecReplicated]
+----------------------------------------------PhysicalProject
+------------------------------------------------filter((date_dim.d_qoy < 4) and (date_dim.d_year = 2001))
+--------------------------------------------------PhysicalOlapScan[date_dim]
+--------------------------------------PhysicalDistribute[DistributionSpecHash]
+----------------------------------------PhysicalProject
+------------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 RF5
+----------------------------------PhysicalDistribute[DistributionSpecHash]
+------------------------------------PhysicalProject
+--------------------------------------PhysicalOlapScan[customer_demographics]
+------------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[customer_address]
+------------------------PhysicalDistribute[DistributionSpecHash]
--------------------------PhysicalProject
----------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ws_sold_date_sk]
------------------------------PhysicalProject
@@ -45,7 +46,7 @@ PhysicalResultSink
--------------------------------PhysicalProject
----------------------------------filter((date_dim.d_qoy < 4) and (date_dim.d_year = 2001))
------------------------------------PhysicalOlapScan[date_dim]
-----------------------PhysicalDistribute[DistributionSpecReplicated]
+----------------------PhysicalDistribute[DistributionSpecHash]
------------------------PhysicalProject
--------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
----------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query10.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query10.out
index dc3529bdf25..923159c5f63 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query10.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query10.out
@@ -5,50 +5,51 @@ PhysicalResultSink
----PhysicalTopN[MERGE_SORT]
------PhysicalDistribute[DistributionSpecGather]
--------PhysicalTopN[LOCAL_SORT]
-----------hashAgg[LOCAL]
-------------PhysicalProject
---------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
-----------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
-------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((c.c_customer_sk = web_sales.ws_bill_customer_sk)) otherCondition=()
---------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=() build RFs:RF5 c_current_cdemo_sk->[cd_demo_sk]
-------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF5
-------------------------PhysicalDistribute[DistributionSpecReplicated]
---------------------------PhysicalProject
-----------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[ss_customer_sk]
-------------------------------PhysicalDistribute[DistributionSpecHash]
---------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[ss_sold_date_sk]
-------------------------------------PhysicalProject
---------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF3 RF4
-------------------------------------PhysicalDistribute[DistributionSpecReplicated]
---------------------------------------PhysicalProject
-----------------------------------------filter((date_dim.d_moy <= 4) and (date_dim.d_moy >= 1) and (date_dim.d_year = 2001))
-------------------------------------------PhysicalOlapScan[date_dim]
-------------------------------PhysicalDistribute[DistributionSpecHash]
---------------------------------hashJoin[INNER_JOIN] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF2 ca_address_sk->[c_current_addr_sk]
+----------hashAgg[GLOBAL]
+------------PhysicalDistribute[DistributionSpecHash]
+--------------hashAgg[LOCAL]
+----------------PhysicalProject
+------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
+--------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
+----------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((c.c_customer_sk = web_sales.ws_bill_customer_sk)) otherCondition=()
+------------------------PhysicalProject
+--------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF5 c_customer_sk->[ss_customer_sk]
+----------------------------PhysicalDistribute[DistributionSpecHash]
+------------------------------PhysicalProject
+--------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
----------------------------------PhysicalProject
-------------------------------------PhysicalOlapScan[customer] apply RFs: RF2
+------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF4 RF5
----------------------------------PhysicalDistribute[DistributionSpecReplicated]
------------------------------------PhysicalProject
---------------------------------------filter(ca_county IN ('Cochran County', 'Kandiyohi County', 'Marquette County', 'Storey County', 'Warren County'))
-----------------------------------------PhysicalOlapScan[customer_address]
---------------------PhysicalDistribute[DistributionSpecReplicated]
-----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ws_sold_date_sk]
+--------------------------------------filter((date_dim.d_moy <= 4) and (date_dim.d_moy >= 1) and (date_dim.d_year = 2001))
+----------------------------------------PhysicalOlapScan[date_dim]
+----------------------------PhysicalDistribute[DistributionSpecHash]
+------------------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=() build RFs:RF3 c_current_cdemo_sk->[cd_demo_sk]
+--------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF3
+--------------------------------PhysicalDistribute[DistributionSpecReplicated]
+----------------------------------hashJoin[INNER_JOIN] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF2 ca_address_sk->[c_current_addr_sk]
+------------------------------------PhysicalProject
+--------------------------------------PhysicalOlapScan[customer] apply RFs: RF2
+------------------------------------PhysicalDistribute[DistributionSpecReplicated]
+--------------------------------------PhysicalProject
+----------------------------------------filter(ca_county IN ('Cochran County', 'Kandiyohi County', 'Marquette County', 'Storey County', 'Warren County'))
+------------------------------------------PhysicalOlapScan[customer_address]
+------------------------PhysicalDistribute[DistributionSpecHash]
--------------------------PhysicalProject
-----------------------------PhysicalOlapScan[web_sales] apply RFs: RF1
---------------------------PhysicalDistribute[DistributionSpecReplicated]
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_moy <= 4) and (date_dim.d_moy >= 1) and (date_dim.d_year = 2001))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalDistribute[DistributionSpecReplicated]
---------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
+----------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ws_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1
+------------------------------PhysicalDistribute[DistributionSpecReplicated]
+--------------------------------PhysicalProject
+----------------------------------filter((date_dim.d_moy <= 4) and (date_dim.d_moy >= 1) and (date_dim.d_year = 2001))
+------------------------------------PhysicalOlapScan[date_dim]
+----------------------PhysicalDistribute[DistributionSpecHash]
------------------------PhysicalProject
---------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
-------------------------PhysicalDistribute[DistributionSpecReplicated]
---------------------------PhysicalProject
-----------------------------filter((date_dim.d_moy <= 4) and (date_dim.d_moy >= 1) and (date_dim.d_year = 2001))
-------------------------------PhysicalOlapScan[date_dim]
+--------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
+----------------------------PhysicalDistribute[DistributionSpecReplicated]
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_moy <= 4) and (date_dim.d_moy >= 1) and (date_dim.d_year = 2001))
+----------------------------------PhysicalOlapScan[date_dim]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query35.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query35.out
index 7f0a43176d6..1ea14729a92 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query35.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query35.out
@@ -12,32 +12,33 @@ PhysicalResultSink
------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
--------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
----------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((c.c_customer_sk = web_sales.ws_bill_customer_sk)) otherCondition=()
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=()
-----------------------------PhysicalDistribute[DistributionSpecHash]
-------------------------------PhysicalProject
---------------------------------hashJoin[INNER_JOIN] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=()
-----------------------------------PhysicalDistribute[DistributionSpecHash]
-------------------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=()
---------------------------------------PhysicalDistribute[DistributionSpecHash]
-----------------------------------------PhysicalProject
-------------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
---------------------------------------------PhysicalProject
-----------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2
---------------------------------------------PhysicalDistribute[DistributionSpecReplicated]
+------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=()
+------------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------------PhysicalProject
+----------------------------------hashJoin[INNER_JOIN] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=()
+------------------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=()
+----------------------------------------PhysicalDistribute[DistributionSpecHash]
+------------------------------------------PhysicalProject
+--------------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
----------------------------------------------PhysicalProject
-------------------------------------------------filter((date_dim.d_qoy < 4) and (date_dim.d_year = 2001))
---------------------------------------------------PhysicalOlapScan[date_dim]
---------------------------------------PhysicalDistribute[DistributionSpecHash]
-----------------------------------------PhysicalProject
-------------------------------------------PhysicalOlapScan[customer]
-----------------------------------PhysicalDistribute[DistributionSpecHash]
-------------------------------------PhysicalProject
---------------------------------------PhysicalOlapScan[customer_address]
-----------------------------PhysicalDistribute[DistributionSpecHash]
-------------------------------PhysicalProject
---------------------------------PhysicalOlapScan[customer_demographics]
-------------------------PhysicalDistribute[DistributionSpecReplicated]
+------------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2
+----------------------------------------------PhysicalDistribute[DistributionSpecReplicated]
+------------------------------------------------PhysicalProject
+--------------------------------------------------filter((date_dim.d_qoy < 4) and (date_dim.d_year = 2001))
+----------------------------------------------------PhysicalOlapScan[date_dim]
+----------------------------------------PhysicalDistribute[DistributionSpecHash]
+------------------------------------------PhysicalProject
+--------------------------------------------PhysicalOlapScan[customer]
+------------------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------------------PhysicalProject
+----------------------------------------PhysicalOlapScan[customer_address]
+------------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[customer_demographics]
+------------------------PhysicalDistribute[DistributionSpecHash]
--------------------------PhysicalProject
----------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ws_sold_date_sk]
------------------------------PhysicalProject
@@ -46,7 +47,7 @@ PhysicalResultSink
--------------------------------PhysicalProject
----------------------------------filter((date_dim.d_qoy < 4) and (date_dim.d_year = 2001))
------------------------------------PhysicalOlapScan[date_dim]
-----------------------PhysicalDistribute[DistributionSpecReplicated]
+----------------------PhysicalDistribute[DistributionSpecHash]
------------------------PhysicalProject
--------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
----------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query10.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query10.out
index dc3529bdf25..923159c5f63 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query10.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query10.out
@@ -5,50 +5,51 @@ PhysicalResultSink
----PhysicalTopN[MERGE_SORT]
------PhysicalDistribute[DistributionSpecGather]
--------PhysicalTopN[LOCAL_SORT]
-----------hashAgg[LOCAL]
-------------PhysicalProject
---------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
-----------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
-------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((c.c_customer_sk = web_sales.ws_bill_customer_sk)) otherCondition=()
---------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=() build RFs:RF5 c_current_cdemo_sk->[cd_demo_sk]
-------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF5
-------------------------PhysicalDistribute[DistributionSpecReplicated]
---------------------------PhysicalProject
-----------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[ss_customer_sk]
-------------------------------PhysicalDistribute[DistributionSpecHash]
---------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[ss_sold_date_sk]
-------------------------------------PhysicalProject
---------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF3 RF4
-------------------------------------PhysicalDistribute[DistributionSpecReplicated]
---------------------------------------PhysicalProject
-----------------------------------------filter((date_dim.d_moy <= 4) and (date_dim.d_moy >= 1) and (date_dim.d_year = 2001))
-------------------------------------------PhysicalOlapScan[date_dim]
-------------------------------PhysicalDistribute[DistributionSpecHash]
---------------------------------hashJoin[INNER_JOIN] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF2 ca_address_sk->[c_current_addr_sk]
+----------hashAgg[GLOBAL]
+------------PhysicalDistribute[DistributionSpecHash]
+--------------hashAgg[LOCAL]
+----------------PhysicalProject
+------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
+--------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
+----------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((c.c_customer_sk = web_sales.ws_bill_customer_sk)) otherCondition=()
+------------------------PhysicalProject
+--------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF5 c_customer_sk->[ss_customer_sk]
+----------------------------PhysicalDistribute[DistributionSpecHash]
+------------------------------PhysicalProject
+--------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
----------------------------------PhysicalProject
-------------------------------------PhysicalOlapScan[customer] apply RFs: RF2
+------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF4 RF5
----------------------------------PhysicalDistribute[DistributionSpecReplicated]
------------------------------------PhysicalProject
---------------------------------------filter(ca_county IN ('Cochran County', 'Kandiyohi County', 'Marquette County', 'Storey County', 'Warren County'))
-----------------------------------------PhysicalOlapScan[customer_address]
---------------------PhysicalDistribute[DistributionSpecReplicated]
-----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ws_sold_date_sk]
+--------------------------------------filter((date_dim.d_moy <= 4) and (date_dim.d_moy >= 1) and (date_dim.d_year = 2001))
+----------------------------------------PhysicalOlapScan[date_dim]
+----------------------------PhysicalDistribute[DistributionSpecHash]
+------------------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=() build RFs:RF3 c_current_cdemo_sk->[cd_demo_sk]
+--------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF3
+--------------------------------PhysicalDistribute[DistributionSpecReplicated]
+----------------------------------hashJoin[INNER_JOIN] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF2 ca_address_sk->[c_current_addr_sk]
+------------------------------------PhysicalProject
+--------------------------------------PhysicalOlapScan[customer] apply RFs: RF2
+------------------------------------PhysicalDistribute[DistributionSpecReplicated]
+--------------------------------------PhysicalProject
+----------------------------------------filter(ca_county IN ('Cochran County', 'Kandiyohi County', 'Marquette County', 'Storey County', 'Warren County'))
+------------------------------------------PhysicalOlapScan[customer_address]
+------------------------PhysicalDistribute[DistributionSpecHash]
--------------------------PhysicalProject
-----------------------------PhysicalOlapScan[web_sales] apply RFs: RF1
---------------------------PhysicalDistribute[DistributionSpecReplicated]
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_moy <= 4) and (date_dim.d_moy >= 1) and (date_dim.d_year = 2001))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalDistribute[DistributionSpecReplicated]
---------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
+----------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ws_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1
+------------------------------PhysicalDistribute[DistributionSpecReplicated]
+--------------------------------PhysicalProject
+----------------------------------filter((date_dim.d_moy <= 4) and (date_dim.d_moy >= 1) and (date_dim.d_year = 2001))
+------------------------------------PhysicalOlapScan[date_dim]
+----------------------PhysicalDistribute[DistributionSpecHash]
------------------------PhysicalProject
---------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
-------------------------PhysicalDistribute[DistributionSpecReplicated]
---------------------------PhysicalProject
-----------------------------filter((date_dim.d_moy <= 4) and (date_dim.d_moy >= 1) and (date_dim.d_year = 2001))
-------------------------------PhysicalOlapScan[date_dim]
+--------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
+----------------------------PhysicalDistribute[DistributionSpecReplicated]
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_moy <= 4) and (date_dim.d_moy >= 1) and (date_dim.d_year = 2001))
+----------------------------------PhysicalOlapScan[date_dim]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query35.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query35.out
index 35d0777a6fe..0874f2dd35c 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query35.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query35.out
@@ -12,32 +12,33 @@ PhysicalResultSink
------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
--------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
----------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((c.c_customer_sk = web_sales.ws_bill_customer_sk)) otherCondition=()
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=() build RFs:RF5 cd_demo_sk->[c_current_cdemo_sk]
-----------------------------PhysicalDistribute[DistributionSpecHash]
-------------------------------PhysicalProject
---------------------------------hashJoin[INNER_JOIN] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF4 ca_address_sk->[c_current_addr_sk]
-----------------------------------PhysicalDistribute[DistributionSpecHash]
-------------------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[ss_customer_sk]
---------------------------------------PhysicalDistribute[DistributionSpecHash]
-----------------------------------------PhysicalProject
-------------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
---------------------------------------------PhysicalProject
-----------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 RF3
---------------------------------------------PhysicalDistribute[DistributionSpecReplicated]
+------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=() build RFs:RF5 cd_demo_sk->[c_current_cdemo_sk]
+------------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------------PhysicalProject
+----------------------------------hashJoin[INNER_JOIN] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF4 ca_address_sk->[c_current_addr_sk]
+------------------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[ss_customer_sk]
+----------------------------------------PhysicalDistribute[DistributionSpecHash]
+------------------------------------------PhysicalProject
+--------------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
----------------------------------------------PhysicalProject
-------------------------------------------------filter((date_dim.d_qoy < 4) and (date_dim.d_year = 2001))
---------------------------------------------------PhysicalOlapScan[date_dim]
---------------------------------------PhysicalDistribute[DistributionSpecHash]
-----------------------------------------PhysicalProject
-------------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 RF5
-----------------------------------PhysicalDistribute[DistributionSpecHash]
-------------------------------------PhysicalProject
---------------------------------------PhysicalOlapScan[customer_address]
-----------------------------PhysicalDistribute[DistributionSpecHash]
-------------------------------PhysicalProject
---------------------------------PhysicalOlapScan[customer_demographics]
-------------------------PhysicalDistribute[DistributionSpecReplicated]
+------------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 RF3
+----------------------------------------------PhysicalDistribute[DistributionSpecReplicated]
+------------------------------------------------PhysicalProject
+--------------------------------------------------filter((date_dim.d_qoy < 4) and (date_dim.d_year = 2001))
+----------------------------------------------------PhysicalOlapScan[date_dim]
+----------------------------------------PhysicalDistribute[DistributionSpecHash]
+------------------------------------------PhysicalProject
+--------------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 RF5
+------------------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------------------PhysicalProject
+----------------------------------------PhysicalOlapScan[customer_address]
+------------------------------PhysicalDistribute[DistributionSpecHash]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[customer_demographics]
+------------------------PhysicalDistribute[DistributionSpecHash]
--------------------------PhysicalProject
----------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ws_sold_date_sk]
------------------------------PhysicalProject
@@ -46,7 +47,7 @@ PhysicalResultSink
--------------------------------PhysicalProject
----------------------------------filter((date_dim.d_qoy < 4) and (date_dim.d_year = 2001))
------------------------------------PhysicalOlapScan[date_dim]
-----------------------PhysicalDistribute[DistributionSpecReplicated]
+----------------------PhysicalDistribute[DistributionSpecHash]
------------------------PhysicalProject
--------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
----------------------------PhysicalProject
diff --git a/regression-test/suites/nereids_p0/subquery/subquery_unnesting.groovy b/regression-test/suites/nereids_p0/subquery/subquery_unnesting.groovy
new file mode 100644
index 00000000000..0a830f1a364
--- /dev/null
+++ b/regression-test/suites/nereids_p0/subquery/subquery_unnesting.groovy
@@ -0,0 +1,132 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite ("subquery_unnesting") {
+ sql """ SET enable_nereids_planner=true"""
+ sql "set enable_fallback_to_original_planner=false"
+
+ sql "drop table if exists t1"
+ sql "drop table if exists t2"
+ sql "drop table if exists t3"
+
+ sql """create table t1
+ (k1 bigint, k2 bigint)
+ ENGINE=OLAP
+ DUPLICATE KEY(k1, k2)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(k2) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1"
+ );"""
+ sql """create table t2
+ (k1 int, k2 varchar(128), k3 bigint, v1 bigint, v2 bigint)
+ ENGINE=OLAP
+ DUPLICATE KEY(k1, k2)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(k2) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1"
+ );"""
+ sql """create table t3
+ (k1 bigint, k2 bigint)
+ ENGINE=OLAP
+ DUPLICATE KEY(k1, k2)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(k2) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1"
+ );"""
+ sql """insert into t1 values (1,null),(null,1),(1,2), (null,2),(1,3), (2,4), (2,5), (3,3), (3,4), (20,2), (22,3), (24,4),(null,null);"""
+ sql """insert into t2 values (1,'abc',2,3,4), (1,'abcd',3,3,4), (2,'xyz',2,4,2), (2,'uvw',3,4,2), (2,'uvw',3,4,2), (3,'abc',4,5,3), (3,'abc',4,5,3), (null,null,null,null,null);"""
+ sql """insert into t3 values (1,null),(null,1),(1,4), (1,2), (null,3), (2,4), (3,7), (3,9),(null,null),(5,1);"""
+
+ sql "drop table if exists sub_query_correlated_subquery1"
+ sql "drop table if exists sub_query_correlated_subquery3"
+
+ sql """create table if not exists sub_query_correlated_subquery1
+ (k1 bigint, k2 bigint)
+ duplicate key(k1)
+ distributed by hash(k2) buckets 1
+ properties('replication_num' = '1');"""
+ sql """create table if not exists sub_query_correlated_subquery3
+ (k1 int, k2 varchar(128), k3 bigint, v1 bigint, v2 bigint)
+ distributed by hash(k2) buckets 1
+ properties('replication_num' = '1');"""
+ sql """insert into sub_query_correlated_subquery1 values (1,null),(null,1),(1,2), (null,2),(1,3), (2,4), (2,5), (3,3), (3,4), (20,2), (22,3), (24,4),(null,null);"""
+ sql """insert into sub_query_correlated_subquery3 values (1,"abc",2,3,4), (1,"abcd",3,3,4), (2,"xyz",2,4,2), (2,"uvw",3,4,2), (2,"uvw",3,4,2), (3,"abc",4,5,3), (3,"abc",4,5,3), (null,null,null,null,null);"""
+
+ qt_select1 """select * from sub_query_correlated_subquery1 where sub_query_correlated_subquery1.k1 < (select sum(sub_query_correlated_subquery3.k3) from sub_query_correlated_subquery3 where sub_query_correlated_subquery3.v2 = sub_query_correlated_subquery1.k2) OR k1 < 10 order by k1, k2;"""
+ qt_select2 """select * from sub_query_correlated_subquery1 where sub_query_correlated_subquery1.k1 < (select sum(sub_query_correlated_subquery3.k3) from sub_query_correlated_subquery3 where sub_query_correlated_subquery3.v2 = sub_query_correlated_subquery1.k2) order by k1, k2;"""
+ qt_select3 """SELECT * FROM sub_query_correlated_subquery1 WHERE k1 > (SELECT AVG(k1) FROM sub_query_correlated_subquery3) OR k1 < 10 order by k1, k2;"""
+ qt_select4 """select * from sub_query_correlated_subquery1 where sub_query_correlated_subquery1.k1 < (select sum(sub_query_correlated_subquery3.k3) from sub_query_correlated_subquery3 where sub_query_correlated_subquery3.v2 = 2) order by k1, k2;"""
+ qt_select5 """select * from sub_query_correlated_subquery1 where sub_query_correlated_subquery1.k1 in (select sub_query_correlated_subquery3.k3 from sub_query_correlated_subquery3 where sub_query_correlated_subquery3.v2 = sub_query_correlated_subquery1.k2) OR k1 < 10 order by k1, k2;"""
+ qt_select6 """select * from sub_query_correlated_subquery1 where sub_query_correlated_subquery1.k1 in (select sub_query_correlated_subquery3.k3 from sub_query_correlated_subquery3 where sub_query_correlated_subquery3.v2 = sub_query_correlated_subquery1.k2) order by k1, k2;"""
+ qt_select7 """SELECT * FROM sub_query_correlated_subquery1 WHERE k1 IN (SELECT k1 FROM sub_query_correlated_subquery3 WHERE sub_query_correlated_subquery1.k1 > sub_query_correlated_subquery3.k3) OR k1 < 10 ORDER BY k1, k2;"""
+ qt_select8 """select * from sub_query_correlated_subquery1 where sub_query_correlated_subquery1.k1 in (select sub_query_correlated_subquery3.k3 from sub_query_correlated_subquery3 where sub_query_correlated_subquery3.v2 < sub_query_correlated_subquery1.k2) order by k1, k2;"""
+ qt_select9 """SELECT * FROM sub_query_correlated_subquery1 WHERE k1 IN (SELECT k1 FROM sub_query_correlated_subquery3) OR k1 < 10 order by k1, k2;"""
+ qt_select10 """select * from sub_query_correlated_subquery1 where sub_query_correlated_subquery1.k1 in (select sub_query_correlated_subquery3.k3 from sub_query_correlated_subquery3) order by k1, k2;"""
+ qt_select11 """select * from sub_query_correlated_subquery1 where sub_query_correlated_subquery1.k1 not in (select sub_query_correlated_subquery3.k3 from sub_query_correlated_subquery3 where sub_query_correlated_subquery3.v2 = sub_query_correlated_subquery1.k2) or k1 < 10 order by k1, k2;"""
+ qt_select12 """select * from sub_query_correlated_subquery1 where sub_query_correlated_subquery1.k1 not in (select sub_query_correlated_subquery3.k3 from sub_query_correlated_subquery3 where sub_query_correlated_subquery3.v2 = sub_query_correlated_subquery1.k2) order by k1, k2;"""
+ qt_select13 """select * from sub_query_correlated_subquery1 where sub_query_correlated_subquery1.k1 not in (select sub_query_correlated_subquery3.k3 from sub_query_correlated_subquery3 where sub_query_correlated_subquery3.v2 > sub_query_correlated_subquery1.k2) or k1 < 10 order by k1, k2;"""
+ qt_select14 """select * from sub_query_correlated_subquery1 where sub_query_correlated_subquery1.k1 not in (select sub_query_correlated_subquery3.k3 from sub_query_correlated_subquery3 where sub_query_correlated_subquery3.v2 > sub_query_correlated_subquery1.k2) order by k1, k2;"""
+ qt_select15 """select * from sub_query_correlated_subquery1 where sub_query_correlated_subquery1.k1 not in (select sub_query_correlated_subquery3.k3 from sub_query_correlated_subquery3 ) or k1 < 10 order by k1, k2;"""
+ qt_select16 """select * from sub_query_correlated_subquery1 where sub_query_correlated_subquery1.k1 not in (select sub_query_correlated_subquery3.k3 from sub_query_correlated_subquery3 where sub_query_correlated_subquery3.v2 = 2) order by k1, k2;"""
+ qt_select17 """select * from sub_query_correlated_subquery1 where exists (select sub_query_correlated_subquery3.k3 from sub_query_correlated_subquery3 where sub_query_correlated_subquery1.k2 = sub_query_correlated_subquery3.v2) or k1 < 10 order by k1, k2;"""
+ qt_select18 """select * from sub_query_correlated_subquery1 where exists (select sub_query_correlated_subquery3.k3 from sub_query_correlated_subquery3 where sub_query_correlated_subquery1.k2 = sub_query_correlated_subquery3.v2) order by k1, k2;"""
+ qt_select19 """select * from sub_query_correlated_subquery1 where exists (select sub_query_correlated_subquery3.k3 from sub_query_correlated_subquery3 where sub_query_correlated_subquery1.k2 > sub_query_correlated_subquery3.v2) or k1 < 10 order by k1, k2;"""
+ qt_select20 """select * from sub_query_correlated_subquery1 where exists (select sub_query_correlated_subquery3.k3 from sub_query_correlated_subquery3 where sub_query_correlated_subquery1.k2 > sub_query_correlated_subquery3.v2) order by k1, k2;"""
+ qt_select21 """SELECT * FROM sub_query_correlated_subquery1 WHERE EXISTS (SELECT k1 FROM sub_query_correlated_subquery3 WHERE k1 = 10) OR k1 < 10 order by k1, k2;"""
+ qt_select22 """select * from sub_query_correlated_subquery1 where exists (select sub_query_correlated_subquery3.k3 from sub_query_correlated_subquery3) order by k1, k2;"""
+ qt_select23 """select * from sub_query_correlated_subquery1 where not exists (select sub_query_correlated_subquery3.k3 from sub_query_correlated_subquery3 where sub_query_correlated_subquery1.k2 = sub_query_correlated_subquery3.v2) or k1 < 10 order by k1, k2;"""
+ qt_select24 """select * from sub_query_correlated_subquery1 where not exists (select sub_query_correlated_subquery3.k3 from sub_query_correlated_subquery3 where sub_query_correlated_subquery1.k2 = sub_query_correlated_subquery3.v2) order by k1, k2;"""
+ qt_select25 """select * from sub_query_correlated_subquery1 where not exists (select sub_query_correlated_subquery3.k3 from sub_query_correlated_subquery3 where sub_query_correlated_subquery1.k2 > sub_query_correlated_subquery3.v2) or k1 < 10 order by k1, k2;"""
+ qt_select26 """select * from sub_query_correlated_subquery1 where not exists (select sub_query_correlated_subquery3.k3 from sub_query_correlated_subquery3 where sub_query_correlated_subquery1.k2 != sub_query_correlated_subquery3.v2) order by k1, k2;"""
+ qt_select27 """SELECT * FROM sub_query_correlated_subquery1 WHERE not EXISTS (SELECT k1 FROM sub_query_correlated_subquery3 WHERE k1 = 10) OR k1 < 10 order by k1, k2;"""
+ qt_select28 """select * from sub_query_correlated_subquery1 where not exists (select sub_query_correlated_subquery3.k3 from sub_query_correlated_subquery3) order by k1, k2;"""
+ qt_select29 """select * from sub_query_correlated_subquery1 where exists (select sub_query_correlated_subquery3.k3 from sub_query_correlated_subquery3 where sub_query_correlated_subquery1.k2 = sub_query_correlated_subquery3.v2 limit 1) order by k1, k2;"""
+ qt_select30 """select * from sub_query_correlated_subquery1 where exists (select sub_query_correlated_subquery3.k3 from sub_query_correlated_subquery3 where sub_query_correlated_subquery1.k2 > sub_query_correlated_subquery3.v2 limit 1) order by k1, k2;"""
+ qt_select31 """select * from sub_query_correlated_subquery1 where exists (select sub_query_correlated_subquery3.k3 from sub_query_correlated_subquery3 limit 1) order by k1, k2;"""
+
+ qt_select32 """select t1.* from t1 left join t2 on t1.k2 = t2.k3 and t1.k1 < ( select max(k1) from t3 where t1.k2 = t3.k2 ) OR t1.k1 < 10 order by t1.k1, t1.k2;"""
+ qt_select33 """select t1.* from t1 left join t2 on t1.k2 = t2.k3 and t1.k1 < ( select max(k1) from t3 where t1.k2 = t3.k2 ) order by t1.k1, t1.k2;"""
+ qt_select34 """select t1.* from t1 left join t2 on t1.k2 = t2.k3 and t1.k1 < ( select max(k1) from t3 ) or t1.k1 < 10 order by t1.k1, t1.k2;"""
+ qt_select35 """select t1.* from t1 left join t2 on t1.k2 = t2.k3 and t1.k1 < ( select max(k1) from t3 ) order by t1.k1, t1.k2;"""
+ qt_select36 """select t1.* from t1 left join t2 on t1.k2 = t2.k3 and t1.k1 in ( select t3.k1 from t3 where t1.k2 = t3.k2 ) or t1.k1 < 10 order by t1.k1, t1.k2;"""
+ qt_select37 """select t1.* from t1 left join t2 on t1.k2 = t2.k3 and t1.k1 in ( select t3.k1 from t3 where t1.k2 = t3.k2 ) order by t1.k1, t1.k2;"""
+ qt_select38 """select t1.* from t1 left join t2 on t1.k2 = t2.k3 and t1.k1 in ( select t3.k1 from t3 where t1.k2 < t3.k2 ) or t1.k1 < 10 order by t1.k1, t1.k2;"""
+ qt_select39 """select t1.* from t1 left join t2 on t1.k2 = t2.k3 and t1.k1 in ( select t3.k1 from t3 where t1.k2 < t3.k2 ) order by t1.k1, t1.k2;"""
+ qt_select40 """select t1.* from t1 left join t2 on t1.k2 = t2.k3 and t1.k1 in ( select t3.k1 from t3 ) or t1.k1 < 10 order by t1.k1, t1.k2;"""
+ qt_select41 """select t1.* from t1 left join t2 on t1.k2 = t2.k3 and t1.k1 in ( select t3.k1 from t3 ) order by t1.k1, t1.k2;"""
+ qt_select42 """select t1.* from t1 left join t2 on t1.k2 = t2.k3 and t1.k1 not in ( select t3.k1 from t3 where t1.k2 = t3.k2 ) or t1.k1 < 10 order by t1.k1, t1.k2;"""
+ qt_select43 """select t1.* from t1 left join t2 on t1.k2 = t2.k3 and t1.k1 not in ( select t3.k1 from t3 where t1.k2 = t3.k2 ) order by t1.k1, t1.k2;"""
+ qt_select44 """select t1.* from t1 left join t2 on t1.k2 = t2.k3 and t1.k1 not in ( select t3.k1 from t3 where t1.k2 < t3.k2 ) or t1.k1 < 10 order by t1.k1, t1.k2;"""
+ qt_select45 """select t1.* from t1 left join t2 on t1.k2 = t2.k3 and t1.k1 not in ( select t3.k1 from t3 where t1.k2 < t3.k2 ) order by t1.k1, t1.k2;"""
+ qt_select46 """select t1.* from t1 left join t2 on t1.k2 = t2.k3 and t1.k1 not in ( select t3.k1 from t3 ) or t1.k1 < 10 order by t1.k1, t1.k2;"""
+ qt_select47 """select t1.* from t1 left join t2 on t1.k2 = t2.k3 and t1.k1 not in ( select t3.k1 from t3 ) order by t1.k1, t1.k2;"""
+ qt_select48 """select t1.* from t1 left join t2 on t1.k2 = t2.k3 and exists ( select t3.k1 from t3 where t1.k2 = t3.k2 ) or t1.k1 < 10 order by t1.k1, t1.k2;"""
+ qt_select49 """select t1.* from t1 left join t2 on t1.k2 = t2.k3 and exists ( select t3.k1 from t3 where t1.k2 = t3.k2 ) order by t1.k1, t1.k2;"""
+ qt_select50 """select t1.* from t1 left join t2 on t1.k2 = t2.k3 and exists ( select t3.k1 from t3 where t1.k2 < t3.k2 ) or t1.k1 < 10 order by t1.k1, t1.k2;"""
+ qt_select51 """select t1.* from t1 left join t2 on t1.k2 = t2.k3 and exists ( select t3.k1 from t3 where t1.k2 < t3.k2 ) order by t1.k1, t1.k2;"""
+ qt_select52 """select t1.* from t1 left join t2 on t1.k2 = t2.k3 and exists ( select t3.k1 from t3 ) or t1.k1 < 10 order by t1.k1, t1.k2;"""
+ qt_select53 """select t1.* from t1 left join t2 on t1.k2 = t2.k3 and exists ( select t3.k1 from t3 ) order by t1.k1, t1.k2;"""
+ qt_select54 """select t1.* from t1 left join t2 on t1.k2 = t2.k3 and not exists ( select t3.k1 from t3 where t1.k2 = t3.k2 ) or t1.k1 < 10 order by t1.k1, t1.k2;"""
+ qt_select55 """select t1.* from t1 left join t2 on t1.k2 = t2.k3 and not exists ( select t3.k1 from t3 where t1.k2 = t3.k2 ) order by t1.k1, t1.k2;"""
+ qt_select56 """select t1.* from t1 left join t2 on t1.k2 = t2.k3 and not exists ( select t3.k1 from t3 where t1.k2 < t3.k2 ) or t1.k1 < 10 order by t1.k1, t1.k2;"""
+ qt_select57 """select t1.* from t1 left join t2 on t1.k2 = t2.k3 and not exists ( select t3.k1 from t3 where t1.k2 < t3.k2 ) order by t1.k1, t1.k2;"""
+ qt_select58 """select t1.* from t1 left join t2 on t1.k2 = t2.k3 and not exists ( select t3.k1 from t3 ) or t1.k1 < 10 order by t1.k1, t1.k2;"""
+ qt_select59 """select t1.* from t1 left join t2 on t1.k2 = t2.k3 and not exists ( select t3.k1 from t3 ) order by t1.k1, t1.k2;"""
+}
diff --git a/regression-test/suites/nereids_p0/subquery/test_subquery_in_project.groovy b/regression-test/suites/nereids_p0/subquery/test_subquery_in_project.groovy
index 25920848c6d..32802dbb7a1 100644
--- a/regression-test/suites/nereids_p0/subquery/test_subquery_in_project.groovy
+++ b/regression-test/suites/nereids_p0/subquery/test_subquery_in_project.groovy
@@ -137,4 +137,59 @@ suite("test_subquery_in_project") {
"""
sql """drop table if exists test_sql;"""
+
+ sql """drop table if exists markjoin_t1;"""
+ sql """drop table if exists markjoin_t2;"""
+ sql """drop table if exists markjoin_t3;"""
+
+ sql """create table markjoin_t1
+ (a bigint, b bigint)
+ ENGINE=OLAP
+ DUPLICATE KEY(a, b)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(a) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1"
+ );"""
+ sql """create table markjoin_t2
+ (a int, b varchar(128), c bigint, v1 bigint, v2 bigint)
+ ENGINE=OLAP
+ DUPLICATE KEY(a, b)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(a) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1"
+ );"""
+
+ sql """insert into markjoin_t1 values (1,null),(null,1),(1,2), (null,2),(1,3), (2,4), (2,5), (3,3), (3,4), (20,2), (22,3), (24,4),(null,null);"""
+ sql """insert into markjoin_t2 values (1,'abc',2,3,4), (1,'abcd',3,3,4), (2,'xyz',2,4,2), (2,'uvw',3,4,2), (2,'uvw',3,4,2), (3,'abc',4,5,3), (3,'abc',4,5,3), (null,null,null,null,null);"""
+
+ qt_select_m1 """SELECT markjoin_t1.b IN
+ (SELECT markjoin_t2.c FROM markjoin_t2 WHERE markjoin_t1.a = markjoin_t2.a)
+ FROM markjoin_t1 order by 1;"""
+ qt_select_m2 """SELECT EXISTS
+ (SELECT markjoin_t2.c FROM markjoin_t2 WHERE markjoin_t1.a = markjoin_t2.a)
+ FROM markjoin_t1 order by 1;"""
+ qt_select_m3 """SELECT markjoin_t1.b NOT IN
+ (SELECT markjoin_t2.c FROM markjoin_t2 WHERE markjoin_t1.a = markjoin_t2.a)
+ FROM markjoin_t1 order by 1;"""
+ qt_select_m4 """SELECT NOT EXISTS
+ (SELECT markjoin_t2.c FROM markjoin_t2 WHERE markjoin_t1.a = markjoin_t2.a)
+ FROM markjoin_t1 order by 1;"""
+ qt_select_m5 """SELECT markjoin_t1.b IN
+ (SELECT markjoin_t2.c FROM markjoin_t2)
+ FROM markjoin_t1 order by 1;"""
+ qt_select_m6 """SELECT EXISTS
+ (SELECT markjoin_t2.c FROM markjoin_t2)
+ FROM markjoin_t1 order by 1;"""
+ qt_select_m7 """SELECT markjoin_t1.b NOT IN
+ (SELECT markjoin_t2.c FROM markjoin_t2)
+ FROM markjoin_t1 order by 1;"""
+ qt_select_m8 """SELECT NOT EXISTS
+ (SELECT markjoin_t2.c FROM markjoin_t2)
+ FROM markjoin_t1 order by 1;"""
+ sql """drop table if exists markjoin_t1;"""
+ sql """drop table if exists markjoin_t2;"""
+ sql """drop table if exists markjoin_t3;"""
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org